GazelleBEA [LIVE] [1032712]

need latest JavaXHyperlinker.

// GazelleBEA - Base class of all Gazelle web servers

// About BEA objects with code: There is some confusion about the actions on them.
// There are:
// - compileAndLoadObject
// - activateDynamicObject
// - reactivateDynamicObject
// - deactivateObject
//
// It's not completely clear what each of these actions exactly does
// with respect to the at least 3 internal states of an object:
// - no custom code, or none compiled
// - custom code compiled (or imported from another object)
// - custom code loaded   (actually migrated object to custom class)
// - object activated     (activate listeners etc defined by the
//                         object in _activateImpl())
//
// Right now things seem to work, but we could really use some test cases related to this.

// Ways to put code into a BEA object:
//
// Way 1. Put the JavaX code directly in meta_code (string field).
//        You can use an exports {} block to make things on the global level (e.g. static classes to export)
//        You can use a special "extends" declaration (extends BBla from 123;) in meta_code to subclass a class defined in another object.
//
// Way 2. Point meta_prototype to another object with a custom class.
//        This will use the same class for this object.
//
// Way 3. Set meta_useClass to something like "123.BBla" to use a class
//        called BBla exported by object 123 for this object.

// Note about GazelleBEA:
// If you ever move this class into loadableUtils, make sure to copy the definition of method db() to the subclass
// and override uploadsBaseDir()

mainPackage gazelle
mainClassName main

set flag needToKnowMainLib.

!include once #1030833 // BEACalculations

//set flag defaultDefaultClassFinder_debug.

set flag OurSyncCollections.
set flag CleanImports.
set flag DynModule.
set flag NoNanoHTTPD.
set flag AllPublic. // for dynamic BEA objects
rewrite BEA with BEAObject.

abstract static class GazelleBEA > DynGazelleRocks {
  switchable bool mirrorBEAObjects; // don't do it for now
  switchable bool enableAutoRuns = true;
  switchable bool enableNewBEAObjectNotis = true;
  switchable bool printAllConceptChanges;
  switchable bool useShadowLogger = false;
  switchable bool autoActivateDynamicObjects = true;
  switchable bool reloadWhenMainLibUpdates;
  switchable S baseSystemVersion = "1"; // increase this when all dynamic objects need a recompile
  switchable int backupFrequency = 5; // make new backup file every x minutes
  switchable S gazelleServerGlobalID = aGlobalID();
  
  // proofOfStart
  transient bool allObjectsActivated;
  transient Throwable startError;
  
  switchable S activateOnlyTheseIDs;
  switchable S dontActivateTheseIDs;

  transient ReliableSingleThread_Multi<BEA> rstAutoRuns = dm_rstMulti(this, 1000, lambda1 performAutoRuns);
  transient Q notificationQ;
  transient ReliableSingleThread_Multi<BEA> rstDistributeNewObject = dm_rstMulti(this, 1000, lambda1 distributeNewObject_impl);
  transient Set<BEA> newObjectsDistributed = weakSet();

  transient ReliableSingleThread_Multi<BEA> rstUpdateBEAMirrors = dm_rstMulti(this, 100, c -> c.updateMirrorPost());
  
  transient ReliableSingleThread_Multi<BEA> rstAutoActivateDynamicObjects = dm_rstMulti(this, 100, lambda1 autoActivateDynamicObject);

  transient BEACalculations calculations = new(this);
  transient BEACalculations calc = calculations;
  transient int newFieldsToShow = 3;
  transient bool allowAnonymousInputUpload = true; // TODO
  switchable int maxInputLength = 50000;
  
  switchable S mainDomainWithProtocol = "https://bea.gazelle.rocks";
  
  switchable S grabUsersFromDomain = "https://gazelle.rocks";
  
  switchable bool verboseQStartsAndStops;

  transient ConceptClassesDependentValue<Int> inputsWithoutRewrites;
  transient ConceptClassesDependentValue<Int> inputsWithoutMatches;
  transient ConceptClassesDependentValue<Int> syntacticPatternsWithoutRewrites;
  
  transient Set<File> deadClassPaths = syncLinkedHashSet();
  
  transient new x30_pkg.x30_util.BetterThreadLocal<BEA> beaThreadOwner;
  
  switchable long beaHomePageID;
  
  bool verboseClassLoading;
  
  transient bool saveAllQsAsInput = true;
  transient bool moveNavItemsToMisc = false;
  transient bool additionalCommandsForObjects = true;
  
  transient bool exportUsersInVM, exportUsersGlobally;
  
  transient bool sourceIsPublic = true;
  
  // create an anonymous user for each cookie unless user is logged in
  transient bool makeAnonymousUsers = true;
  
  transient bool showInputAndPatternNavItems = true;
  
  transient IConceptCounter beaModifiedIndex;
  
  transient bool showInputCounts;
  
  transient ConceptFieldIndexCI_certainValues<BEA> serviceIndex;
  
  transient bool runRefChecker = true;

  // add more fields for GazelleBEA here
  
  !include #1030883 // DB quickImport mix-in
  
  void init :: after {
    //set quickDBReloadEnabled;
    botName = heading = adminName = "Gazelle";
    set enableVars;
    unset showTalkToBotLink;
    unset phoneNumberSpecialInputField;
    unset showMetaBotOnEveryPage;
    unset showDeliveredDomains;
    set showCRUDToEveryone;
  }

  
  Class lookForClass(ClassLoader cl, S name) {
    pcall {
      // first thing - filter
      if (!name.startsWith(DynClassName_mainPrefixForVM())) null;
    
      if (verboseClassLoading) print("Looking for class " + name);
      DynClassName dcn = DynClassName_parse(name);
      if (dcn == null) {
        vmBus_send lookingForClass_noDCNParse(name);
        null;
      }
      S subPath = vmClassNameToSubPath(dcn.fullClassNameForVM());
      File byteCode = dcn.byteCodeFile();
      vmBus_send lookingForClass_haveByteCodeFile(byteCode);
      
      // jar already added? done.
      if (classLoaderContainsByteCodePath(cl, byteCode)) {
        vmBus_send lookingForClass_haveByteCodePath(byteCode);
        null;
      }
      
      // class file not found? bad.
      bool exists = fileExistsInInDirOrZip(byteCode, subPath);
      printVars(+exists, +subPath, +byteCode);
      if (!exists) {
        vmBus_send lookingForClass_subPathNotInByteCode(byteCode, subPath);
        fail("Class not found: " + subPath + " in " + byteCode);
      }
        
      // register byte code path
      addByteCodePathToClassLoader(cl, byteCode);
      assertTrue("Byte code added", classLoaderContainsByteCodePath(cl, byteCode));
      vmBus_send lookingForClass_byteCodePathAdded(byteCode);
      
      ret (Class) call(cl, "super_findClass", name);
      
      // null; // just let the class loader do its thing
      
      // recursive call
      //ctex { ret cl.loadClass(name); }
    }
    null;
  }
  
  // MUST be called outside of loadableUtils
  void db {
    main db();
  }

  void startDB {
    ClassLoader cl = dm_moduleClassLoader();
    setOpt(cl, verbose := verboseClassLoading);
    setFieldToIF1Proxy(cl, findClass_extension := (IF1<S, Class>) name -> {
      vmBus_send lookingForClass(cl, name);
      Class c = lookForClass(cl, name);
      vmBus_send lookingForClass_result(cl, name, c);
      ret c;
    });
    
    S mainPrefix = dotToDollar(DynClassName_mainPrefix());
    O classFinder = _defaultClassFinder();
    //Map<S, Class> classesCache = syncMap();
    db_mainConcepts().classFinder = func(S name) -> Class enter {
      //ret syncGetOrCreate(classesCache, name, () -> {
        print("Deserializing class " + name);
        try {
          S name2 = dropPrefix(mcName() + "$", name);
          //printVars(+mainPrefix, +name2);
          if (name2.startsWith(mainPrefix)) {
            S vmName = DynClassName_parse(name2).fullClassNameForVM();
            if (verboseClassLoading) print("Loading dynamic class " + vmName);
            Class c = classLoader_loadClass(module(), vmName);
            //print(+c);
            ret c;
          }
          Class c = cast callF(classFinder, name);
          print(+c);
          ret c;
        } on fail e {
          printStackTrace(e);
        }
      //});
    };
    
    db();
    Concepts concepts = db_mainConcepts();
    concepts.useBackRefsForSearches = true;
    concepts.newBackupEveryXMinutes = backupFrequency;
    
    fixConceptIDs();
    
    /*concepts.makeStructureData = () ->
      concepts.finishStructureData(new structure_Data {
        void writeObject(O o, S shortName, MapSO fv) {
          if (o instanceof Concept && conceptID(o/Concept) == 136925)
            print("writeProblemObject " + shortName + " " + fv);
          super.writeObject(o, shortName, fv);
        }
      });*/
      
    inputsWithoutRewrites = ConceptClassesDependentValue(litset(BEA), () -> countPred(beaList("Input"), c -> empty(beaBackRefs(c, "Rewrite"))));
    inputsWithoutMatches = ConceptClassesDependentValue(litset(BEA), () -> countPred(beaList("Input"), c -> empty(beaBackRefs(c, "Match"))));
    syntacticPatternsWithoutRewrites = ConceptClassesDependentValue(litset(BEA), () -> countPred(beaList("Syntactic Pattern"), c -> empty(beaBackRefs(c, "Rewrite"))));
    
    if (useShadowLogger) {
      ConceptsShadowLogger shadowLogger = new(db_mainConcepts());
      shadowLogger.install();
      //shadowLogger.writer = printWriter(deflaterOutputStream_syncFlush_append(programFile("shadow.log.deflated")));
      shadowLogger.writer = filePrintWriter_append(programFile("shadow.log"));
      dm_doEvery(10.0, r { shadowLogger.flush(); });
      ownResource(shadowLogger);
    }
    
    if (printAllConceptChanges) printAllConceptChanges();
  }
  
  MapSO emergencyFlags() {
    ret (Map) parseEqualsProperties(
      joinNemptiesWithEmptyLines(
        loadTextFile(programFile("gazelle-emergency-options.txt")),
        loadTextFile(javaxDataDir("gazelle-emergency-options.txt")),
      ));
  }
  
  void start {
    try {
      start2();
    } catch print e {
      startError = e;
    }
  }
  
  void start2 {
    print("Module ID: " + dm_moduleID());
    set !useBotNameAsModuleName;
    setOptAllDyn_pcall(module(), pnl(+emergencyFlags()));
    change();
    S name = actualMCDollar() + "BEAObject";
    print(+name);
    assertEquals(BEA, callF(defaultDefaultClassFinder(), name));
    assertEquals(ConceptWithGlobalID, callF(defaultDefaultClassFinder(), mcDollar() + "ConceptWithGlobalID"));
    //seedDBFrom(#1030602);
    set botDisabled;
    set storeBaseClassesInStructure;
    super.start();
    assertSameVerbose("miscMap me", db_mainConcepts().miscMapGet(DynNewBot2), this);
    assertSameVerbose("beaMod()", main beaMod(), this);
    print("main concepts: " + db_mainConcepts() + ", count: " + db_mainConcepts().countConcepts());
    print(renderConceptClassesWithCount(db_mainConcepts()));
    print("Inputs: " + n2(beaCount("Input")));
    //set showOnlySelectedObject;
    
    // reload module when lib changes
      dm_onSnippetTranspiled(mainLibID, r {
        if (reloadWhenMainLibUpdates) {
          setField(reloadWhenMainLibUpdates := false);
          dm_reloadModule();
        }
      });

    if (!enabled) ret;
    
    if (exportUsersInVM) {
       dm_vmBus_answerToMessage lookupGazelleUser(func(S name, S pw, S salt) {
        if (!eq(salt, passwordSalt())) null;
        User user = conceptWhereCI User(+name);
        if (user == null) null;
        if (!eq(getVar(user.passwordMD5), hashPW(pw))) null;
        ret user;
      });
    }

    // mirror all objects to be sure
    rstUpdateBEAMirrors.addAll(list(BEA));
    
    newObjectsDistributed.addAll(list(BEA));
    
    /*onIndividualConceptChange_notOnAllChanged(BEA,
      p -> { calculations.makeSyntacticPattern(p); });*/
      
    onIndividualConceptChange_notOnAllChanged(BEA,
      o -> {
        if (enableAutoRuns) rstAutoRuns.add(o);
        if (enableNewBEAObjectNotis && newObjectsDistributed.add(o))
          rstDistributeNewObject.add(o);
      });
      
    notificationQ = dm_startQ();
    
    // fix refs occasionally
    
    if (runRefChecker)
      dm_doEvery(60*60.0, r-enter {
        print(ConceptsRefChecker(db_mainConcepts()).runAndFixAll());
      });
    
    // call _activate on all activatable objects (initial activations)
    
    setField(activateOnlyTheseIDs := nemptyLinesLL(
      loadTextFile(programDir("gazelle-activate-only")),
      loadTextFile(javaxDataDir("gazelle-activate-only"))));
    Set<Long> ids = activateOnlyTheseIDs();
    
    Set<Long> antiIDs = dontActivateTheseIDs();
    print("Object IDs to activate: " + (ids == null ? "ALL" : ids));
    if (nempty(antiIDs)) print("Object IDs not to activate: " + (ids == null ? "ALL" : ids));
    for (BEA bea)
      if ((ids == null || contains(ids, bea.id)) && !contains(antiIDs, bea.id)) {
        //print("Activating: " + bea);
        callActivate(bea);
      }
      
    cleanReifiedWebSockets();
      
    set allObjectsActivated; // proof of start
    print("All objects activated");
  } // end of start2()
  
  void makeIndices :: after {    
    indexConceptClass(BSettings);
    beaModifiedIndex = (IConceptCounter) indexConceptFieldDesc_v2(BEA, "_modified");
    indexConceptFieldCIWithTopTen(BEA, "type");
    indexConceptFieldIC(BEA, "text");
    indexConceptFieldIC(BEA, "purpose");
    
    serviceIndex = ConceptFieldIndexCI_certainValues<BEA>(BEA, "meta_isService") {
      bool isApplicableValue(O o) { ret o != null; }
    };
  }
  
  bool isMaster(Req req) {
    ret req?.masterAuthed;
  }
  
  @Override
  L<Class> crudClasses(DynGazelleRocks.Req req) {
    var l = super.crudClasses(req);
    l.add(BEA);
    if (masterAuthed(req))
      l.add(AuthedDialogID);
    ret l;
  }
  
  Set<Class> hiddenCrudClasses() {
    Set<Class> set = super.hiddenCrudClasses();
    set.remove(UploadedSound);
    addAll(set, Conversation, ExecutedStep, InputHandled);
    ret set;
  }
  
  S authFormHeading() {
    ret h3(adminName);
  }
  
  swappable S authFormMoreContent() {
    S html = p(html_loggedIn(false));
    
    new HTMLFramer1 framer;
    if (anonymousUserWarning(currentReq(), framer))
      html += lines(framer.contents);
    
    ret html + super.authFormMoreContent();
  }
  
  HTMLFramer1 newFramerInstance() {
    ret new BEAHTMLFramer;
  }

  @Override
  void makeFramer(DynGazelleRocks.Req req) {
    super.makeFramer(req);
    
    if (!allObjectsActivated)
      req.framer.add(p("WARNING: Start failure"));
      
    if (startError != null)
      req.framer.add(pre_htmlEncode("WARNING: Start error\n\n" + renderStackTrace(startError)));

    req.framer.addInHead(hjs_selectize());
    req.framer.addInHead(hjs_selectizeClickable());
    req.framer.addInHead(hjs_autoExpandingTextAreas());
  }

  @Override
  <A extends Concept> HCRUD_Concepts<A> crudData(Class<A> c, DynGazelleRocks.Req _req) {
    Req req = cast _req;
    HCRUD_Concepts<A> cc = super.crudData(c, req);
    
    if (c == UploadedSound || c == UploadedFile || c == UploadedImage) {
      cc.onCreate.add(o -> cset(o, createdBy := currentUser());
      cc.emptyConcept = () -> {
        Concept ccc = cc.emptyConcept_base();
        cset(ccc, isPublic := true);
        ret (A) ccc;
      };
    }
    
    if (c == BEA) {
      cc.lsMagic = true;
      cc.humanizeFieldNames = false;
      cc.convertConceptValuesToRefs = true;
      cc.itemName = () -> "BEA Object";
      
      cc.addRenderer("meta_code", new HCRUD_Data.AceEditor(80, 20));
      
      cc.titleForObjectID = id -> beaHTML(beaGet(toLong(id)));
      
      cc.isEditableValue = o ->
        cc.isEditableValue_base(o)
          && !eq(shortClassName(o), "MultiSet");
      
      cc.conceptClassForComboBoxSearch = (info, query) -> {
        if (endsWith(info, "_nativeValue"))
          ret Concept;
        ret cc.conceptClassForComboBoxSearch_base(info, query);
      };
      
      cc.comboBoxSearchBaseItems = (info, query) -> {
        new Matches m;
        printVars comboBoxSearchBaseItems(+info);
        if (jMatchStart("type=<quoted>", info, m)) {
          var l = beaList($1);
          print("Got " + nItems(l));
          ret cc.comboBoxItems(l);
        }
        ret cc.comboBoxSearchBaseItems_base(info, query);
      };
      
      // prevent non-master users from forging/changing createdBy
      cc.massageItemMapForUpdate = (o, map) -> {
        cc.massageItemMapForUpdate_base(o, map);
        
        if (!isMaster(req)) {
          O createdByOld = cget createdBy(o);
          if (createdByOld != null || map.get("createdBy") != user(req))
            map.remove("createdBy");
        }
      };
      
      // probably already done by massageItemMapForUpdate
      cc.onCreate.add(o ->
        cset(o, createdBy := currentUser()));
      
      cc.getObjectForDuplication = id -> {
        BEA o = beaGet(toLong(id));
        MapSO item = cc.getObjectForDuplication_base(id);
        item.put(createdBy := req.auth.user!); // set default creator to current user
        item.put(meta_createdFrom := o);
        
        // call enhancer object
        item = or((MapSO) pcallOpt(cget cleanObjectForDuplication(websiteEnhancersObject()), "cleanObjectForDuplication", o, item), item);
        
        ret item;
      };
      
      cc.emptyObject = () -> {
        MapSO item = cc.emptyObject_base();
        item.put(type := "");
        ret item;
      };
      
      Set<S> deletableRefs = litciset("Match");
      
      cc.objectCanBeDeleted = id -> {
        BEA o = cast cc.conceptForID(id);
        ret userCanEditObject(user(req), o)
          && all(findBackRefs(BEA, o), x -> contains(deletableRefs, x.type()));
      };

      cc.actuallyDeleteConcept = o -> {
        deleteConcepts(filter(findBackRefs(BEA, o),
          o2 -> contains(deletableRefs, o2.type())));
        cdelete(o);
      };
    }

    ret cc;
  }
  
  bool newLinkTargetBlank_base(Class c) {
    ret super.newLinkTargetBlank_base(c) || c == BEA;
  }

  <A extends Concept> HCRUD makeCRUDBase(Class<A> c, DynGazelleRocks.Req req, HTMLFramer1 framer) {
    HCRUD crud = super.makeCRUD(c, req, framer);
    HCRUD_Concepts<A> data = cast crud.data;
    crud.showOnlySelected = true;
    crud.showSearchField = true;
    if (data.customFilter == null)
      crud.descending = true; // show latest objects first by default except when searching
    crud.cleanItemIDs = true;
    
    if (c == BEA)
      customizeBEACrud(crud, req, framer);
      
    ret crud;
  }

  @Override
  <A extends Concept> HCRUD makeCRUD(Class<A> c, DynGazelleRocks.Req req, HTMLFramer1 framer) {
    HCRUD crud = makeCRUDBase(c, req, framer);
    if (c == BEA)
      optimizeBEACrud(crud);
    ret crud;
  }
    
  void customizeBEACrud(HCRUD crud, DynGazelleRocks.Req req, HTMLFramer1 framer) {
    HCRUD_Concepts<BEA> data = cast crud.data;
    
    crud.allowFieldRenaming = true;
    
    var renderValue_old = crud.renderValue;
    crud.renderValue = (field, value) -> {
      S html = crud.renderValue_fallback(renderValue_old, field, value);
      ret addShowMoreButton(html);
    };
    
    crud.haveSelectizeClickable = true;
    
    crud.cellColumnToolTips = true;
    crud.unshownFields = litset("mirrorPost", "globalID");
    crud.uneditableFields = litset("log", "meta_dependsOnCode");
    crud.showTextFieldsAsAutoExpandingTextAreas = true;
    crud.duplicateInNewTab = true;
    
    crud.customizeACEEditor = ace -> {
      ace.onKeyDown = "function(event) { " + jquery_submitFormOnCtrlEnter() + " }";
    };
    
    HCRUD_Concepts cc = cast crud.data;
    
    S typeFilter = req.get("type");
    if (nempty(typeFilter))
      cc.addCIFilter(type := typeFilter);
    
    crud.renderCmds = map -> {
      BEA o = getConcept BEA(crud.itemIDAsLong(map));
      if (o == null) ret "";
      
      new LS cmds;
      
      // ask the object itself for commands
      
      addAll(cmds, allToString(optCast Cl(pcall(o, "cmds"))));
      
      // special commands for BEA types or objects with certain fields
      
      if (fileNotEmpty(javaSourceFileForObject(o)))
        cmds.add(targetBlank(baseLink + "/javaSource?id=" + o.id, "Show Java source"));
      
      for (S field : ll("text", "what", "type")) {
        S text = getStringOpt(o, field);
        if (nempty(text))
          cmds.add(
            targetBlank_noFollow(addParamsToURL(baseLink + "/query",
              q := text, algorithm := "process input"),
              "Use field " + quote(field) + " as query"));
      }

      if (o.typeIs("Input")) {
        cmds.add(ahref(addParamsToURL(crudLink(BEA),
          cmd := "new",
          title := "Add Pattern For Input",
          f_type := "Pattern",
          f_text := getStringOpt text(o),
          f_shouldMatch := o.id, metaInfo_shouldMatch := "concept",
        ), "Add pattern"));

        cmds.add(ahref(appendParamsToURL(baseLink + "/query", q := o.text(), algorithm := "Apply all text functions"),
         "Apply all text functions"));
      }

      if (o.typeIs("Pattern"))
        cmds.add(ahref(appendParamsToURL(baseLink + "/query", q := o.id, algorithm := "Run pattern"),
         "Try-run pattern against all inputs"));
      
      if (o.typeIsOneOf("Pattern", "Syntactic Pattern")) {
        cmds.add(ahref(appendParamsToURL(baseLink + "/reactAllInputsWithPattern", patternID := o.id),
         "React pattern with all inputs"));
         
        cmds.add(ahref(appendParamsToURL(baseLink + "/convertSyntacticToSemanticMatchesForWholePattern", patternID := o.id),
          "Convert all syntactic matches to semantic matches"));
      }

      if (o.typeIs("Match")) {
        cmds.add(ahref(appendParamsToURL(baseLink + "/query", q := o.id, algorithm := "Find rewrites for match"),
         "Find rewrites"));
        if (beaTypeIs(beaGet pattern(o), "Syntactic Pattern"))
          cmds.add(ahref(appendParamsToURL(baseLink + "/convertSyntacticToSemanticMatches", matchID := o.id),
          "Convert to semantic matches"));
      }

      cmds.add(
        targetBlank(addParamsToURL(crudLink(BEA),
          cmd := "new",
          newField1_name := "inReferenceTo",
          newField1_type := "BEAObject",
          newField1_conceptValue := o.id),
          "Reference this"));
      
      cmds.add(
        ahref_noFollow(addParamsToURL(baseLink + "/markUseful",
          redirect := beaShortURL(o),
          objectID := o.id),
          "Mark useful"));
      
      cmds.add(
        ahref_noFollow(addParamsToURL(baseLink + "/markBad",
          redirect := beaShortURL(o),
          objectID := o.id),
          "Mark bad"));
      
      cmds.add(addCommentHTML(o));
        
      if (o.typeIsOneOf("Script", "Step in script") || eqic(beforeVerticalBar(o.type()), "Instruction"))
        cmds.add(
          ahref(addParamsToURL(baseLink + "/runInstruction",
            instruction := o.id),
            "Run"));
            
      if (o.typeIs("Function Result")) {
        if (eqic_gen(cget resultType(o), "string"))
          cmds.add(
            ahref(addParamsToURL(baseLink + "/convertResultToInput",
              result := o.id),
              "Convert to input"));
      }
      
      if (o.typeIs("Auto Run"))
        cmds.add(
          ahref(addParamsToURL(baseLink + "/performAutoRunOnAllObjects", autoRun := o.id), "Run on all objects"));

      if (o.getClass() != BEA)
        cmds.add(
          ahref(addParamsToURL(baseLink + "/migrateToBase", id := o.id), "Migrate to BEAObject"));
          
      cmds.add(
        ahref(addParamsToURL(baseLink + "/performAutoRuns",
          onObject := o.id),
          "Perform auto runs on this object"));
          
      framer.addInHead(hjs_copyToClipboard());
      
      cmds.add(ahref_onClick(formatFunctionCall copyToClipboard(jsQuote(o.globalIDStr())), "Copy global ID [" + o.globalIDStr() + "]"));
      
      LS items = llNempties(
        crud.renderCmds_base(map),
        !o.typeIsOneOf("Input", "Pattern", "Syntactic Pattern")
          ? null : addRewriteHTML(o),
        !canAutoMigrate(o) ? null 
          : ahref(addParamsToURL(baseLink + "/autoMigrate", id := o.id), "Auto-migrate to " + shortClassName(defaultCustomClass(o))));

      if (isObjectWithCode(o) && isMasterAuthed(req)) {
        S link = baseLink + "/activateDynamicObject?id=" + o.id;
        if (!codeHashUpToDate(o))
          if (cget meta_codeHash(o) == null)
            items.add(ahref(link, "Compile new code", title := "Object has not been compiled yet"));
          else
            items.add(ahref(link, "Compile changed code", title := "Code has changed afer last compilation"));
        else if (compileErrorInObject(o))
          items.add("Code error. " + ahref(link, "Compile again", title := "Object has code errors - maybe recompiling fixes them?"));
        else if (o.customCodeLoaded())
          items.add(ahref(link, "Compile again", title := "Custom code is loaded. Click if you want to recompile anyway"));
        else
          items.add(ahref(link, "Compile &amp; load"));
      }
      
      if (additionalCommandsForObjects) {
        if (o.canRenderHTML())
          items.add(targetBlank(beaMod().baseLink + "/beaHTML/" + o.id, "Show HTML"));
          
        if (nempty(getStringOpt meta_code(o)))
          items.add(targetBlank(addParamsToURL(beaMod().baseLink + "/get-field", obj := o.id, field := "meta_code", enhance := "JavaX"),
          "Show code"));
      
        S codeState = getStringOpt meta_codeState(o);
        if (swic(codeState, "error"))
          items.add(targetBlank(addParamsToURL(beaMod().baseLink + "/beaHTML/249697", obj := o.id, field := "meta_codeState"),
          "Show compile error"));
          
        bool master = beaMod().isMasterAuthed();
      
        if (hasMethod(o, "_activate")) {
          bool active = isTrue(pcallOpt(o, "_active"));
          bool shouldActivate = isTrue(getOpt(o, "meta_shouldActivate"));
          if (active != shouldActivate)
            items.add("Warning: Object is semi-activated");
          else if (active)
            items.add("Object is active");
          else
            items.add("Object is inactive");
            
          if (master) {
            if (!active || !shouldActivate)
              items.add(ahref(addParamsToURL(beaMod().baseLink + "/activateObject", id := o.id), "ACTIVATE"));
            if (active || shouldActivate)
              items.add(ahref(addParamsToURL(beaMod().baseLink + "/deactivateObject", id := o.id), "DEACTIVATE"));
          }
        }
        
        if (master) {
          new LS methodCalls;
          for (Method method : methodsSortedByNameIC(allLiveMethodsBelowClass(o, BEA))) {
            SS params = litorderedmap(id := o.id, name := method.getName());
            bool incomplete = false, problem = false;
            var types = method.getParameterTypes();
            for (int i = 1; i <= l(types); i++) {
              var argType = types[i-1];
              set incomplete;
              if (eq(argType, S))
                params.put("arg" + i, "putArgHere");
              else if (isSubclassOf(argType, BEA))
                params.put("conceptArg" + i, "putArgHere");
              else {
                params.put("arg" + i, "missingConverterProblem");
                set problem;
              }
            }
            S url = addParamsToURL("/callAnyMethod", params);
            S html = method.getName();
            if (problem) html += appendBracketed("probably can't call, need converter");
            else if (incomplete) html += appendBracketed("fill in parameters first");
            methodCalls.add(targetBlank(url, html));
          }
        
          if (nempty(methodCalls)) items.add(hPopDownButtonWithText("Call a method", methodCalls));
        }

      }

      // add more commands here
      
      pcall { addAll(items, o.directCmds()); }
      
      pcall {
        addAll(items, (LS) callOpt(getWebsiteEnhancer("commandsForObject"), 'commandsFor, o));
      }
      
      items.add(hPopDownButton(cmds));
      ret joinNemptiesWithVBar(items);
    };
    
    cc.massageItemMapForList = (item, map) -> {
      BEA o = cast item;
      
      if (o.typeIs("Input")) {
        Cl<BEA> matches = objectsWhereNotIC(objectsWhereCI(findBackRefs(o, BEA), type := "match"),
          label := "bad");
        map/Map.put("Best Matches", HTML(hparagraphs(
          lmap matchDescHTML(takeFirst(3, matches)))));
      }
      S idField = crud.idField();
      O id = map/Map.get(idField);
      if (id instanceof S)
        map/Map.put(idField, HTML(ahref(beaShortURL(o), id)));
      if (o.typeIs("Function Result")) {
        O value = o~.result;
        map/Map.put("result", HTML(javaValueToHTML(value)));
      }
      
      if (o.typeIs("Match")) {
        SS mapping = o.mapping();
        if (mapping != null)
        map/Map.put("mapping", HTML(
          joinWithBR(map(mapping, (k, v) ->
            htmlEncode2(k) + "=" + calculations.bestInputHTML(v)))));
      }
    
      if (o.getClass() != BEA)
        map/Map.put("Java Class", dropPrefix(actualMCDollar(), className(o)));
        
      if (eq(req.get("showSimpleObjectScore"), "1"))
        map/Map.put("Simple object score" := calculations.simpleObjectScore(o));
        
      o.massageItemMapForList(map/Map);
    }; // end of massageItemMapForList
    
    crud.massageFormMatrix = (map, matrix) -> {
      for (int i = 1; i <= newFieldsToShow; i++) {
        S nf = "newField" + i;
        
        S key = "\*nf*/_conceptValue";
        BEA refValue = beaGet(req.get(key));
        S refSelector =
          crud.renderInput(key,
            cc.makeConceptsComboBox(key, BEA), refValue)
          + hjs([[$("[name=]] + key + [[]").hide()]]);
            
        S nativeSelector =
          crud.renderInput("\*nf*/_nativeValue",
            cc.makeConceptsComboBox("\*nf*/_nativeValue", Concept), null)
          + hjs([[$("[name=]] + nf + [[_nativeValue]").hide()]]);
            
        LS types = ll("String", "BEAObject", "Bool", "Native");
            
        S toggleVis = [[
          $("[name=]] + nf + [[_value]").toggle(value == "String" || value == "Bool");
              $("#]] + nf + [[_refBox").toggle(value == "BEAObject");
              $("#]] + nf + [[_nativeSel").toggle(value == "Native");
        ]];

        S type = req.get("\*nf*/_type");
        S typeSelector = hselect_list(types, type, name := "\*nf*/_type",
            onchange := "var value = this.value;" + toggleVis);
        if (nempty(type))
          typeSelector += hscript("(function(value) {" + toggleVis + "})(" + jsQuote(type) + ")");
        
        matrix.add(ll("Add field:<br>" + htextfield("\*nf*/_name", req.get("\*nf*/_name"), title := "New field name"),
          htmlTable2_noHtmlEncode(ll(ll(
            // string input
            //htextfield("\*nf*/_value"),
            crud.renderTextField("\*nf*/_value", "", 40)
            /*htextarea("",
              name := nf + "_value", 
              class := "auto-expand",
              style := "width: 300px",
            )*/ +
            span(refSelector, id := "\*nf*/_refBox", style := "display: none"),
            span(nativeSelector, id := "\*nf*/_nativeSel", style := "display: none"),
          "Type", typeSelector
          )),
          noHeader := true,
          tableParams := litobjectarray(style := "width: 100%"))));
        }
      };
    
    crud.preprocessUpdateParams = params -> {
      params = cloneMap(params);

      // drop empty strings
      //removeFromMapWhereValue(params, v -> eq(v, ""));
      params = mapValues(params, v -> eq(v, "") ? null : v);
      
      for (int i = 1; i <= max(newFieldsToShow, 10); i++) {
        S nf = "newField" + i;
        S name = trim(params.get("\*nf*/_name")),
          type = params.get("\*nf*/_type"),
          refValue = params.get("\*nf*/_conceptValue"),
          nativeValue = params.get("\*nf*/_nativeValue"),
          value = params.get("\*nf*/_value");

        if (eqic(type, "BEAObject")) {
          value = refValue;
          params.put("metaInfo_" + name, "concept");
        } else if (eqic(type, "Native")) {
          value = nativeValue;
          params.put("metaInfo_" + name, "concept");
        } else if (eqic(type, "Bool"))
          params.put("metaInfo_" + name, "bool");
        
        if (eq(value, "")) value = null;
          
        if (nempty(name) /*&& neqOneOf(value, null, "")*/)
          params.put(crud.fieldPrefix + name, value);
      }
      
      ret params;
    };

    // regular users can only edit their own objects
    if (!isMasterAuthed(req))
      cc.objectCanBeEdited = id -> userCanEditObject(user(req), (BEA) cc.conceptForID(id));
  } // end of customizeBEACrud
  
  // optimize listing all BEA objects
  void optimizeBEACrud(HCRUD crud) {
    HCRUD_Concepts<BEA> data = cast crud.data;
    
    if (empty(data.filters) && empty(data.ciFilters)) {
      crud.sortByField = "_modified";
      crud.descending = true;
      data.listConcepts_firstStep = () -> (L) cloneList(beaModifiedIndex.allConcepts());
      data.defaultSortField = () -> pair("_modified", true);
      data.defaultSort = lambda1 id;
    }
  } // end of optimizeBEACrud

  bool userCanEditObject(User user, BEA o) {
    ret user != null && (user.isMaster || cget createdBy(o) == user);
  }
  
  @Override
  O servePossiblyUserlessBotFunction(DynGazelleRocks.Req req, S function, Map data, User user) {
    if (eq(function, "beaExport")) {
      Cl<Long> ids = concatLists(
        tok_integersAsLongs(urldecode(req.subURI)),
        tok_integersAsLongs(req.get("ids")));
        
      var objects = map beaGet(ids);
      ret serveJSON_breakAtLevels(2, result := map(objects, obj ->
        litorderedmap(id := obj.id,
          gid := str(obj.globalID()),
          struct := obj.structureString())));
    }
    
    ret super.servePossiblyUserlessBotFunction(req, function, data, user);
  }
  
  @Override
  O serveBotFunction(DynGazelleRocks.Req req, S function, Map data, User user) {
    if (eq(function, "beaList")) {
      long changedAfter = toLong(data.get("changedAfter"));
      double pollFor = min(bot_maxPollSeconds, toLong(data.get("pollFor"))); // how long to poll (seconds)
      long startTime = sysNow();

      // We're super-anal about catching all changes. This will probably never trigger
      if (changedAfter > 0 && changedAfter == now()) sleep(1);
      
      Cl<BEA> objects;
      while true {
        objects = changedAfter == 0 ? list(BEA)
          : conceptsWithFieldGreaterThan_sorted(BEA, _modified := changedAfter);

        // return when there are results, no polling or poll expired
        if (nempty(objects) || pollFor == 0 || elapsedSeconds_sysNow(startTime) >= pollFor)
          ret serveJSON_breakAtLevels(2, result := map(objects, obj ->
            litorderedmap(gid := str(obj.globalID()), struct := obj.structureString())
          ));

        // sleep and try again
        sleep(bot_pollInterval);
      }
    }
    
    ret super.serveBotFunction(req, function, data, user);
  }

  @Override
  O serveOtherPage2(DynGazelleRocks.Req req) { ret serveOtherPage2((Req) req); }
  O serveOtherPage2(Req req) null {
    //printVars_str serveOtherPage2(uri := req.uri);
    try object super.serveOtherPage2(req);
    
    S uri = dropTrailingSlashIfNemptyAfterwards(req.uri);
    new Matches m;
    
    if (swic_notSame(uri, "/beaCRUD/", m))
      ret renderBEAObjectTable(req, urldecode(m.rest()));
    
    if (eq(uri, "/inputs"))
      ret renderBEAObjectTable(req, "input");
      
    if (eq(uri, "/syntacticMatchesTable"))
      ret hrefresh(baseLink + "/matchesTable?syntacticOnly=1");
      
    if (eq(uri, "/matchesTable")) {
      bool syntacticOnly = eq("1", req.get("syntacticOnly"));

      new L<BEA> list;
      for (BEA match : beaList("Match")) {
        BEA input = cgetOpt BEA(match, "input");
        BEA pat = cgetOpt BEA(match, "pattern");
        if (syntacticOnly && !calculations.isSyntacticPattern(pat)) continue;
        continue if calculations.patternAlwaysMatches(pat);

        SS mapping = match.mapping();
        if (input == null || pat == null || mapping == null) continue;
        list.add(match);
      }
      list = sortedByCalculatedFieldDesc(list,
        match -> conceptID(cgetBEA input(match)));
      
      new HTMLPaginator paginator;
      paginator.processParams(req.params());
      paginator.baseLink = addParamsToURL(baseLink,
        filterKeys(req.params(), p -> eq(p, "syntacticOnly")));
      paginator.max = l(list);
      req.framer.add(divUnlessEmpty(paginator.renderNav()));
      list = subList(list, paginator.visibleRange());
      
      new L<Map> data;
      for (BEA match : list) {
        BEA input = cgetOpt BEA(match, "input");
        BEA pat = cgetOpt BEA(match, "pattern");
        SS mapping = match.mapping();
        S patText = calculations.patternTextWithAngleBracketVars(pat);
        LS tok = javaTokWithAngleBrackets(patText);
        new BitSet bs;
        tok = replaceAngleBracketVars(tok, mapping, bs);
        for (IntRange r, bool b : unpair bitSetStreaksAndNonStreaks(bs, l(tok))) {
          S t = joinSubList(tok, r);
          replaceTokens(tok, r, b
            ? formatSubInput(input, t, "obj" + match.id)
            : htmlEncode2(t));
        }
        S boldenedInput = join(tok);
        
        data.add(litorderedmap(
          "Input" := aname("obj" + match.id, joinWithBR(
            ahref(beaURL(input), htmlEncode2_str(input~.text())),
            boldenedInput)),
          "Pattern" := ahref(beaURL(pat), htmlEncode2(pat.text())), // beaHTML(pat),
          //"Sub-Inputs" := htmlEncode2_str(mapping),
          "DB object" := beaHTML_justID(match),
          "Feedback" := joinNemptiesWithSpace(calculations.feedbackHTML(match),
            addCommentHTML(match, defaultComment := "good match", redirectAfterSave := req.uriWithParams(),
            text := "+")),
        ));
      }
        
      req.framer.title = syntacticOnly ? "Syntactic matches table" : "Matches table";
      req.framer.add(p());
      req.framer.add(htmlTable2_noHtmlEncode(data));
      ret completeFrame(req);
    }
            
    if (eq(uri, "/rewritesTable")) {
      new L<Map> data;
      for (BEA input : beaList("Input"))
        for (BEA r : beaBackRefs(input, "Rewrite"))
          data.add(litorderedmap(
            "Input" := input.text(),
            "Rewrite" := tok_dropCurlyBrackets(r.text()),
            "Rewrite Type" := r~.rewriteType));
            
      req.framer.title = "Rewrites table";
      req.framer.add(p());
      req.framer.add(htmlTable2(data));
      ret completeFrame(req);
    }

    if (eq(uri, "/patternRewritesTable")) {
      new L<Map> data;
      for (BEA pat : beaListAny("Pattern", "Syntactic Pattern"))
        for (BEA r : beaBackRefs(pat, "Rewrite"))
          if (cget isRewriteOf(r) == pat) {
            PairS example = calculations.exampleForPatternRewrite(pat, r);
            data.add(litorderedmap(
              "Pattern" := ahref(beaURL(pat), htmlEncode2(pat.text())),
              "Rewrite" := ahref(beaURL(r), htmlEncode2(tok_dropCurlyBrackets(r.text()))),
              "Example" := joinWithBR(htmlEncode2(example.a), htmlEncode2("=> " + example.b)),
              "Rewrite Type" := htmlEncode2_str(r~.rewriteType),
              "Cmds" := hPopDownButton(
                targetBlank(conceptEditLink(r), "Edit"),
              )));
          }
            
      req.framer.title = "Pattern rewrites table";
      req.framer.add(p());
      req.framer.add(htmlTable2_noHtmlEncode(data));
      ret completeFrame(req);
    }

    if (eq(uri, "/syntacticPatternsWithoutRewrites")) {
      HCRUD crud = makeBEATypeCrud(req, "Syntactic Pattern");
      HCRUD_Concepts<BEA> data = cast crud.data;
      
      IF1 prev = data.customFilter;
      data.customFilter = list -> {
        list = filter(list, c -> empty(beaBackRefs(c, "Rewrite")));
        ret postProcess(prev, list);
      };
      crud.customTitle = "Syntactic patterns without rewrites";
      ret serveCRUD(req, BEA, crud);
    }
      
    if (eq(uri, "/inputsWithoutRewrites")) {
      HCRUD crud = makeBEATypeCrud(req, "input");
      HCRUD_Concepts<BEA> data = cast crud.data;
      
      IF1 prev = data.customFilter;
      data.customFilter = list -> {
        list = filter(list, c -> empty(beaBackRefs(c, "Rewrite")));
        ret postProcess(prev, list);
      };
      crud.customTitle = "Inputs without rewrites";
      ret serveCRUD(req, BEA, crud);
    }
      
    if (eq(uri, "/inputsWithRewrites")) {
      HCRUD crud = makeBEATypeCrud(req, "input");
      HCRUD_Concepts<BEA> data = cast crud.data;
      
      IF1 prev = data.customFilter;
      data.customFilter = list -> {
        list = filter(list, c -> nempty(beaBackRefs(c, "Rewrite")));
        ret postProcess(prev, list);
      };
      crud.customTitle = "Inputs with rewrites";
      ret serveCRUD(req, BEA, crud);
    }
      
    if (eq(uri, "/inputsWithoutMatches")) {
      HCRUD crud = makeBEATypeCrud(req, "input");
      HCRUD_Concepts<BEA> data = cast crud.data;
      
      IF1 prev = data.customFilter;
      data.customFilter = list -> {
        list = filter(list, c -> empty(beaBackRefs(c, "Match")));
        ret postProcess(prev, list);
      };
      crud.customTitle = "Inputs without matches";
      ret serveCRUD(req, BEA, crud);
    }
      
    if (eq(uri, "/patterns"))
      ret renderBEAObjectTable(req, "pattern");

    if (eq(uri, "/syntacticPatterns"))
      ret renderBEAObjectTable(req, "Syntactic Pattern");

    if (eq(uri, "/matches"))
      ret renderBEAObjectTable(req, "match");

    if (eq(uri, "/rewrites"))
      ret renderBEAObjectTable(req, "rewrite");

    if (eq(uri, "/aiTasks"))
      ret renderBEAObjectTable(req, "AI Task");

    if (eq(uri, "/query"))
      ret calculations.serveQueryPage(req);
      
    if (eq(uri, "/queryHome"))
      ret calculations.serveQueryPage(req, true);
      
    if (eq(uri, "/saveInput")) {
      if (userAgentIsBot(currentUserAgent())) ret "You're a bot";
      
      S text = trim(req.get("text"));
      S info = trim(req.get("info"));
      S uncleanedText = trim(req.get("uncleanedText"));
      if (empty(text)) ret subBot_serve500("Input empty");
      if (l(text) > maxInputLength) ret subBot_serve500("Input too long");
      BEA input = uniqCI BEA(type := "Input", +text, createdBy := user(req));
      saveUserAgent(input);
      if (nempty(info) || nempty(uncleanedText))
        cnew BEA(type := "Input Source", +input, source := info, +uncleanedText);
      ret hrefresh(or2(req.get("redirect"), baseLink + "/"));
    }
      
    if (eq(uri, "/markUseful")) {
      BEA o = beaGet objectID(req);
      if (o == null) ret subBot_serve500("Object not found");
      uniqCI BEA(type := "Useful", object := o, createdBy := user(req));
      ret hrefresh(or2(req.get("redirect"), baseLink + "/"));
    }
      
    if (eq(uri, "/markBad")) {
      BEA o = beaGet objectID(req);
      if (o == null) ret subBot_serve500("Object not found");
      uniqCI BEA(type := "Bad", object := o, createdBy := user(req));
      ret hrefresh(or2(req.get("redirect"), baseLink + "/"));
    }
      
    if (eq(uri, "/saveAnswer")) {
      S text = trim(req.get("text"));
      S rewriteType = or2(trim(req.get("rewriteType")), "Suggested Answer");
      long inputID = toLong(req.get("inputID"));
      BEA input = beaGet(inputID);
      if (input == null) ret subBot_serve500("Input not found");
      if (empty(text)) ret subBot_serve500("Text empty");
      if (l(text) > maxInputLength) ret subBot_serve500("Text too long");
      uniqCI BEA(type := "Rewrite", +text, isRewriteOf := input, +rewriteType, createdBy := user(req));
      ret hrefresh(or2(req.get("redirect"), baseLink + "/"));
    }

    if (eq(uri, "/javaSource")) {
      BEA o = beaGet id(req);
      ret serveText(loadTextFile(javaSourceFileForObject(o)));
    }
    
    if (startsWith(uri, "/beaHTML/", m)) {
      S rest = m.rest();
      int i = smartIndexOf(rest, "/");
      BEA o = beaGet(takeFirst(i, rest));
      S subURI = substring(rest, i+1);
      ret serveBEAHTML(req, o, subURI);
    }
    
    if (eq(uri, "/is-a-gazelle")) ret "yes";
    
    if (eq(uri, "/gazelle-server-id")) ret gazelleServerGlobalID;
    
    if (eq(uri, "/get-field")) {
      BEA obj = beaGet("obj", req);
      S field = req.get("field");
      S value = str(cget(field, obj));
      
      if (eqic(req.get("enhance"), "JavaX")) {
        new JavaXHyperlinker hl;
        hl.targetBlank = true;
        hl.addExplainer(ph -> {
          if (eq(ph.token, "from") && isInteger(ph.next))
            ph.link(ph.cIdx+2, beaURL(parseLong(ph.next)));
        });
        
        S html = hl.codeToHTML(value);
        ret
          htitle_h1(beaHTML(obj) + "." + htmlEncode2(field))
          + div(sourceCodeToHTML_noEncode(html), style := hstyle_sourceCodeLikeInRepo());
      }
      
      ret serveText(value);
    }
    
    if (sourceIsPublic || masterAuthed(req))
      if (eqicOneOf(uri, "/source", "/sources", "/src", "sourceCode", "/code"))
        ret "You will be redirected to my source code..." + hrefresh(3.0, snippetURL(programID()));
        
    BEAForward forward = findBEAShortURL(uri);
    if (forward != null)
      ret serveBEAHTML(req, forward.object, forward.subURI);

    if (exportUsersGlobally && eq(uri, "/lookup-user")) {
      S name from req;
      S passwordMD5 from req;
      S salt from req;
      
      print("lookup-user called from " + req.webRequest.clientIP() + ", name: " + name);
      
      if (!eq(salt, passwordSalt()))
        ret serveJSON(litorderedmap(error := "Bad salt"));
        
      if (empty(name))
        ret serveJSON(litorderedmap(error := "No name"));
        
      User user = conceptWhereCI User(+name);
      if (user == null)
        ret serveJSON(litorderedmap(error := "User not found"));
        
      if (!eq(getVar(user.passwordMD5), passwordMD5))
        ret serveJSON(litorderedmap(error := "Bad pw"));
      
      ret serveJSON(litorderedmap(
        userID := user.id,
        contact := getString contact(user),
        isMaster := getBoolOpt isMaster(user)
      ));
    }
    
    // add more public URLs here
      
    if (!inMasterMode(req)) null;
    
    if (eq(uri, "/uploadInputs"))
      ret serveUploadTexts(req, "Input");
    
    if (eq(uri, "/uploadPatterns"))
      ret serveUploadTexts(req, "Pattern");
      
    if (eq(uri, "/analyzeInput"))
      ret calculations.serveAnalyzeInput(req);
      
    if (eq(uri, "/allBEATypes")) {
      ret h2_title("All object types")
        + ul(renderObjectTypes());
    }
    
    if (eq(uri, "/storeMatch")) {
      BEA pattern = beaGet(req.get("pattern"));
      BEA input = beaGet(req.get("input"));
      S label = req.get("label");
      BEA match = calculations.reactInputWithPattern(input, pattern);
      if (match == null) ret "Couldn't match";
      cset(match, +label);
      ret hrefresh(or2(req.get("redirect"), beaObjectURL(match)));
    }
    
    if (eq(uri, "/saveSyntacticPattern") || eq(uri, "/savePattern")) {
      S type = cic(uri, "syntactic") ? "Syntactic Pattern" : "Pattern";
      S text = req.get("text");
      BEA fromInput = beaGet(req.get("fromInput"));
      BEA pat = uniqCI BEA(+type, +text, createdBy := user(req));
      csetIfUnset(pat, +fromInput);
      if (fromInput != null)
        calculations.reactInputWithPattern(fromInput, pat);
      ret hrefresh(or2(req.get("redirect"), baseLink + "/"));
    }
    
    if (eq(uri, "/runInstruction")) {
      BEA o = beaGet(req.get("instruction"));
      runInstruction(o);
      ret hrefresh(beaObjectURL(o));
    }
    
    if (eq(uri, "/convertResultToInput")) {
      BEA result = beaGet(req.get("result"));
      if (result == null) ret subBot_serve500("Object not found");
      S text = unquote(getString result(result));
      if (l(text) > maxInputLength) ret subBot_serve500("Input too long");
      BEA input = uniqCI BEA(type := "Input", +text);
      uniqCI BEA(type := "Input Source", +input, source := result);
      ret hrefresh(beaShortURL(input));
    }
    
    /*if (eq(uri, "/pipeStringListIntoPattern")) {
      BEA r = beaGet(req.get("resultID"));
      BEa pat = beaGet(req.get("patternID"));
      if (r == null || pat == null) ret "Object not found";
      LS result = assertStringList(cget result(r));
      for (S s : result)
        uniq BEA(type := "Statement", );
    }*/
    
    if (eq(uri, "/setWordType")) {
      BEA r = beaGet(req.get("resultID"));
      S wordType = assertNempty(req.get("wordType"));
      LS result = assertStringList(cget result(r));
      for (S s : result)
        uniqCI BEA(type := "Word type", word := s, +wordType);
      ret "OK";
    }
    
    if (eq(uri, "/reload")) {
      dm_reloadModuleIn(3);
      ret "Reloading module in 3";
    }
    
    if (eq(uri, "/performAutoRuns")) {
      BEA o = beaGet(req.get("onObject"));
      if (o == null) ret "Object not found";
      rstAutoRuns.add(o);
      ret hrefresh(beaURL(o));
    }
    
    if (eq(uri, "/reactAllInputsWithAllSyntacticPatterns")) {
      ret str(returnTimed(r { calculations.reactAllInputsWithAllSyntacticPatterns(); }));
    }
    
    if (eq(uri, "/reactAllInputsWithPattern")) {
      BEA pat = beaGet(req.get("patternID"));
      if (pat == null) ret "Pattern not found";
      ret str(returnTimed(r { calculations.reactAllInputsWithPattern(pat); }));
    }
    
    if (eq(uri, "/storeSubInput")) {
      BEA input = beaGet(req.get("input"));
      if (input == null) ret "Input not found";
      S text = req.get("text");
      S type = eqic(req.get("label"), "good") ? "Sub-Input" : "Bad Sub-Input";
      BEA o = uniqCI BEA(
        +type,
        +input,
        +text,
        createdBy := currentUser());
      ret hrefresh(or2(req.get("redirect"), beaURL(o)));
    }
    
    if (eq(uri, "/performAutoRunOnAllObjects")) {
      BEA autoRun = beaGet autoRun(req);
      performAutoRunOnAllObjects(autoRun);
      ret "OK";
    }
    
    if (eq(uri, "/autoMigrate")) {
      BEA o = beaGet id(req);
      if (o == null) ret "Object not found";
      S url = beaURL(o);
      ret hsansserif() + "Migrated " + beaHTML(o) + " to " + className(autoMigrateToCustomClass(o))
        + hrefresh(1.0, url);
    }
      
    if (eq(uri, "/migrateToBase")) {
      BEA o = beaGet id(req);
      if (o == null) ret "Object not found";
      S url = beaURL(o);
      ret hsansserif() + "Migrated " + beaHTML(o) + " to " + className(autoMigrateToBase(o))
        + hrefresh(1.0, url);
    }
      
    if (eq(uri, "/convertSyntacticToSemanticMatches")) {
      BEA match = beaGet matchID(req);
      if (match == null) ret "Match not found";
      ret ul_htmlEncode2(calculations.convertSyntacticToSemanticMatches(match));
    }
    
    if (eq(uri, "/convertSyntacticToSemanticMatchesForWholePattern")) {
      BEA pat = beaGet patternID(req);
      if (pat == null) ret "Pattern not found";
      ret ul_htmlEncode2(calculations.convertSyntacticToSemanticMatchesForWholePattern(pat));
    }
    
    if (eq(uri, "/createPatternListFromUsefulSyntacticPatterns"))
      ret hrefresh(beaURL(calc.createPatternListFromUsefulSyntacticPatterns()));
      
    if (eq(uri, "/createConceptShadows"))
      ret createConceptShadows();

    if (eq(uri, "/compareConceptShadows")) {
      if (conceptShadows == null) ret createConceptShadows();
      L<ConceptShadow> newShadows = allConceptShadows();
      L<CreatedDeletedChanged<ConceptShadow>> diff
        = diffConceptShadows(conceptShadows, newShadows);
      ret subBot_serveText(nDiffs(diff) + ":\n\n"
        + pnlToString(diff));
    }

    if (eq(uri, "/includeScript")) {
      S snippetID = req.get("id");
      O wired = hotwire(snippetID);
      ret "Wired: " + wired;
    }
    
    if (eq(uri, "/dependents")) {
      BEA o = beaGet id(req);
      ret ul(lmap htmlEncode2_str(objectWithDependents(o)));
    }
    
    if (eq(uri, "/prototypeUsers")) {
      BEA o = beaGet id(req);
      ret ul(lmap htmlEncode2_str(prototypeUsers(o)));
    }
    
    // Reminder: We're still in master mode
    
    if (eq(uri, "/activateObject")) {
      BEA o = beaGet id(req);
      if (o == null) ret "Object not found";
      cset(o, meta_shouldActivate := true);
      callActivate(o);
      ret "Object activated." + hrefresh(1.0, beaURL(o));
    }
    
    if (eq(uri, "/deactivateObject")) {
      BEA o = beaGet id(req);
      if (o == null) ret "Object not found";
      cset(o, meta_shouldActivate := false);
      callDeactivate(o);
      ret "Object deactivated." + hrefresh(1.0, beaURL(o));
    }
    
    if (eq(uri, "/reactivateDynamicObject")) {
      BEA o = beaGet id(req);
      if (o == null) ret "Object not found";
      long id = o.id;
      reactivateAndMigratePrototypeUsers(o);
      ret "OK" + hrefresh(1.0, beaURL(id));
    }
    
    if (eq(uri, "/activateDynamicObject")) {
      BEA o = beaGet id(req);
      if (o == null) ret "Object not found";
      compileAndLoadObject(o);
      o = beaGet id(req);
      S url = or2(req.get("redirect"), beaURL(o));
      ret hsansserif() + "Migrated " + beaHTML(o) + " to " + className(o)
        + hrefresh(1.0, url);
    }
    
    if (eq(uri, "/callAnyMethod")) {
      BEA o = beaGet id(req);
      if (o == null) ret "Object not found";
      S name = req.get("name");
      bool showPrints = eq("1", req.get("showPrints"));

      new L args;
      for (S key, val : req.params()) {
        LS groups = regexpExtractGroups("^(arg|conceptArg)(\\d+)$", key);
        if (groups != null) {
          int idx = parseInt(second(groups));
          S type = first(groups);
          O arg = eq(type, "conceptArg") ? getConcept(parseLong(val))
            : val;
          listSet(args, idx-1, arg);
        }
      }
      
      S arg = req.get("arg");
      if (arg != null) listSet(args, 0, arg);
      
      S result = hijackPrintPlusResult_text(() -> {
        try {
          ret call(o, name, toObjectArray(args));
        } catch e {
          printStackTrace(e);
          null;
        }
      });
      ret serveText("Result of " + o + " . " + name + "(" + joinWithComma(args) + "):\n" + result);
    }
    
    if (eq(uri, "/download"))
      ret subBot_serveFileWithName("gazelle-database." + ymd_minus_hms() + ".gz", conceptsFile());
      
    if (eq(uri, "/errorSource"))
      ret subBot_serveText(loadTextFile(transpilerErrorSourceFile()));
      
    if (eq(uri, "/changeUserPassword")) {
      S name = req.get("name"), newPW = req.get("newPW");
      if (empty(name)) ret "Need 'name' parameter (user name)";
      if (empty(newPW)) ret "Need 'newPW' parameter";
      User user = conceptWhereCI User(+name);
      cset(user, passwordMD5 := SecretValue(hashPW(newPW)));
      ret "Password for " + htmlEncode2(user) + " updated!";
    }
    
    // add more master-mode URLs here
    
  } // end of serveOtherPage2
  
  record noeq BEAForward(BEA object, S subURI) {}
  
  Cl<BEA> beaShortURLObjects() {
    ret filter createdByMasterUser(beaList("BEA Short URL"));
  }
  
  Cl<BEA> masterObjects(Cl<BEA> l) {
    ret filter createdByMasterUser(l);
  }

  bool isBEAShortURLObject(BEA o) {
    ret o != null && o.typeIs("BEA Short URL") && createdByMasterUser(o);
  }
  
  BEAForward findBEAShortURL(S uri) {
    new Matches m;
    S uriWithSlash = addSlashSuffix(uri);
    for (BEA o : beaShortURLObjects()) pcall {
      S uriToTest = getStringOpt uri(o);
      if (startsWith_addingSlash(uriWithSlash, uriToTest, m)) {
        BEA o2 = cast cget object(o);
        if (o2 != null) {
          S subURI = m.rest();
          if (l(uriWithSlash) > l(uri))
            subURI = dropSlashSuffix(subURI);
          ret new BEAForward(o2, subURI);
        }
      }
    }
    null;
  }
  
  transient L<ConceptShadow> conceptShadows;

  O createConceptShadows() {  
    time "Make shadows" {
      S profile = profileThisThreadToString(r {
        conceptShadows = allConceptShadows();
      });
    }
    ret subBot_serveText(n2(conceptShadows, "shadow") + " made in " + lastTiming_formatted() + "\n\n" + profile);
  }
  
  HCRUD makeBEATypeCrud(Req req, S type, HTMLFramer1 framer default req.framer) {
    HCRUD crud = makeCRUDBase(BEA, req, framer);
    crud.baseLink = req.uri();
    HCRUD_Concepts data = cast crud.data;
    data.itemName = () -> firstToUpper(type);
    data.addCIFilter(type := eqic(type, "(no type)") ? null : type);
    
    if (eqicOneOf(type, "Input", "Pattern", "AI Task")) {
      IF0<MapSO> prev = data.emptyObject;
      data.emptyObject = () -> {
        MapSO item = data.emptyObject_fallback(prev);
        item.put(text := ""); // show text field when creating new objects
        ret item;
      };
    }
    
    optimizeBEACrud(crud);
    ret crud;
  }
  
  O renderBEAObjectTable(Req req, S type) {
    HCRUD crud = makeBEATypeCrud(req, type);
    ret serveCRUD(req, BEA, crud);
  }
  
  O serveUploadTexts(Req req, S type) {
    S inputs = req.get("text");
    
    new LS output;
    
    if (nempty(inputs)) {
      for (S text : tlft(inputs)) {
        Pair<BEA, Bool> p = uniqCI2_sync BEA(+type, +text);
        if (cget uploadedBy(p.a) == null)
          cset(p.a, uploadedBy := req.auth.user);
        output.add(type + " " + (p.b ? "added" : "exists")
          + " (ID " + p.a.id + "): " + text);
      }
    }
    
    ret h2("Upload " + plural(type))
      + hpostform(
          p(plural(type) + " (one per line):")
        + p(htextarea(inputs, name := "text"))
        + pIfNempty(htmlEncode_nlToBR(lines(output)))
        + hsubmit("Upload " + plural(type)));
  }
  
  Cl<BEA> beaObjectsOfType(S type) {
    ret conceptsWhereCI BEA(+type);
  }
  
  void reactAllInputsWithSomePatterns {
    calculations.reactAllInputsWithSomePatterns();
  }
  
  swappable S navDiv() {
    ret div_vbar(newNavLinks(),
      style := "margin-bottom: 0.5em");
  }
  
  swappable LS newNavLinks() {
    Req req = currentReq();
    HCRUD crud = makeCRUD(BEA, req);

    S allObjectTypesLink = ahref(baseLink + "/allBEATypes", "All object types");

    BEAHTMLFramer framer = cast req.framer;
    framer.unnamedPopDownButton.addAll(
      ahref(baseLink + "/stats", "Stats"),
      !isMasterAuthed() ? null : ahref(baseLink + "/download", "DB download"),
      targetBlank(getProgramURL(), "Source code"),
      !isMasterAuthed() ? null : ahref(baseLink + "/refchecker", "Reference checker"),
      ahref(baseLink + "/inputsWithRewrites", "Inputs with rewrites"),
      ahref(baseLink + "/rewritesTable", "Rewrites table"),
      ahref(baseLink + "/patternRewritesTable", "Pattern rewrites table"),
      ahref(baseLink + "/matchesTable", "Matches table"),
      ahref(baseLink + "/matchesTable?syntacticOnly=1", "Syntactic matches table"),
      //ahref("https://gazelle.rocks/htmlBot/238410", "Classic Gazelle"),
    );

    new LS items;
    if (showInputAndPatternNavItems)
      addAll(items,
        ahref(baseLink + "/query", "Query"),
        ahref(baseLink + "/newSearch", span_title("New BEA-based searched in development", "New Search (dev.)")),
        beaNavLink("Input", crud),
        beaNavLink("Pattern", crud),
        beaNavLink("Syntactic Pattern", crud),
        ahref(baseLink + "/syntacticPatternsWithoutRewrites", (showInputCounts ? n2(syntacticPatternsWithoutRewrites!) + " " : "") + "Syntactic patterns w/o rewrites"),
        beaNavLink("Match", crud),
        beaNavLink("Rewrite", crud),
        beaNavLink("AI Task", crud),
        ahref(baseLink + "/inputsWithoutMatches", (showInputCounts ? n2(inputsWithoutMatches!) + " " : "") + "Inputs w/o matches"),
        ahref(baseLink + "/inputsWithoutRewrites", (showInputCounts ? n2(inputsWithoutRewrites!) + " " : "") + "Inputs w/o rewrites"),
      );
      /*hPopDownButtonWithText("Bot Forum", navLinks(flat := true, withStats := false)),*/
      
    addAll(items,
      inlineSwappable navDiv_allObjectTypes(this, () ->
        allObjectTypesLink + " " + hPopDownButton(
            listPlus(renderObjectTypes(10),
              allObjectTypesLink))),
      framer.unnamedPopDownButton.width(350).height(500).html());
    ret items;
  }
  
  // crud is just the cached BEAObject crud to check for creation rights
  S beaNavLink(S type, HCRUD crud, int count default beaCount(type)) {
    S plural = firstToLower(plural(type));
    S link = baseLink + "/" + camelCase(plural);
    
    ret ahref(link, n2(count) + " " + firstToUpper(plural)) + (!crud.actuallyAllowCreate() ? "" : " " + ahref(addParamToURL(link, cmd := "new"), "+"));
  }
  
  S beaObjectURL(BEA o) {
    ret conceptLink(o, currentReq());
    /*ret o == null ?:
      addParamsToURL(baseLink + "/crud/BEAObject",
        selectObj := o.id) + "#" + o.id;*/
  }
  
  S matchDescHTML(BEA m) {
    pcall {
      BEA pat = cast cget pattern(m);
      SS mapping = cast cget mapping(m);
      ret ahref_undecorated(crudLink(m), htmlEncode2(quote(getString text(pat)))
        + "<br>&nbsp; with " + renderEqualsCommaProperties(mapping));
    }
    ret htmlEncode2(str(m));
  }
  
  Cl<BEA> beaList() {
    ret list BEA();
  }
  
  Cl<BEA> beaList(S type, O... params) {
    ret conceptsWhereCI BEA(paramsPlus_inFront(params, +type));
  }
  
  Cl<BEA> beaListAny(S... types) {
    ret concatLists(lmap beaList(litciset(types)));
  }
  
  Cl<BEA> beaListAny(Cl<S> types) {
    ret concatLists(lmap beaList(asCISet(types)));
  }
  
  BEA beaGet(long id) {
    ret getConceptOpt BEA(id);
  }
  
  BEA beaGet(S id) {
    ret beaGet(parseFirstLong(id));
  }
  
  bool beaTypeIs(BEA o, S type) {
    ret o != null && o.typeIs(type);
  }
  
  BEA mapMethodLike beaGet(S key, Req req) {
    ret beaGet(req.get(key));
  }

  BEA mapMethodLike cgetBEA aka beaGet(S field, BEA o) {
    ret (BEA) cget(field, o);
  }
  
  BEA cgetBEA aka beaGet(BEA o, S field) {
    ret cgetBEA(field, o);
  }
  
  S beaLinkHTML aka beaToHTML aka beaHTML(BEA o, bool targetBlank default false) {  
    ret o == null ?: ahref_targetBlankIf(targetBlank, conceptLink(o), htmlEncode2_nlToBr(str(o)));
  }
  
  S beaHTML_targetBlank(BEA o) {  
    ret o == null ?: targetBlank(conceptLink(o), htmlEncode2_nlToBr(str(o)));
  }
  
  S beaHTML_justID(BEA o) { ret o == null ?: ahref(beaShortURL(o), o.id); }
  
  S beaFullURL(BEA o) {
    ret o == null ?: mainDomainWithProtocol + beaURL(o);
  }
  
  S beaURL(long id) {
    ret baseLink + "/" + id;
  }
  
  S beaShortURL aka beaURL(BEA o) {
    ret o == null ?: baseLink + "/" + o.id;
  }

  Cl<BEA> beaBackRefs(BEA o, S type) {
    ret objectsWhereCI(findBackRefs BEA(o), +type);
  }
  
  Cl<BEA> beaBackRefs(BEA o) {
    ret findBackRefs BEA(o);
  }
  
  @Override swappable O serveDefaultPage(DynGazelleRocks.Req req) {
    HTMLFramer1 framer = req.framer;
    framer.add(centerGazelleLogo());
    ret completeFrame(req);
  }
  
  S centerGazelleLogo() {
    ret hcenter3(hsnippetimg_scaleToWidth(200, #1102967, 200, 110, title := "Gazelle"), style := "margin-top: 100px");
  }
  
  S html_loggedIn(bool offerLogInLink default true) {
    User user = user(currentReq());
    ret user == null
      ? (offerLogInLink ? ahref(baseLink + "/login", "Log in") : "Not logged in")
      : "Logged in as " + htmlEncode2(user.name);
  }
  
  swappable void distributeNewObject_impl(BEA o) {
    if (newObjectIsWorthNotification(o))
      distributeNewObject_impl_noCheck(o);
  }
  
  swappable bool newObjectIsWorthNotification(BEA o) {
    ret o.id != 0 && !o.typeIsOneOf("Match", "Live WebSocket");
  }
  
  swappable void distributeNewObject_impl_noCheck(BEA o) {
    distributeNotification("New object: " + o);
  }
  
  void performAutoRuns(BEA o) enter {
    //print("performAutoRuns", o);
    
    for (BEA autoRun : beaList("Auto Run")) {
      if (!isTrue(getOpt enabled(autoRun)))
        continue with print("Not enabled: " + autoRun);
      performAutoRunOnObject(autoRun, o);
    }
  }
  
  void performAutoRunOnAllObjects(BEA autoRun) {
    for (BEA o : list(BEA))
      performAutoRunOnObject(autoRun, o);
  }
  
  void performAutoRunOnObject(BEA autoRun, BEA o) {
    ping();
    S type = getString onChangedObjectOfType(autoRun);
    if (!o.typeIs(type)) ret; // with print("Wrong type: " + type);
    
    print("Running " + autoRun);
    BEA procedure = cast cget procedure(autoRun);
    
    S internalCode = getString internalCode(procedure);
    printVars_str(+internalCode, +o);
    
    if (eqic(internalCode, "convertInputToPattern")) {
      S text = o.text();
      if (!containsAngleBracketVars(text)) ret with print("No angle bracket vars");
      BEA p = uniqCI_returnIfNew BEA(type := "Pattern", +text);
      cset(p, fromInput := o, byProcedure := procedure, byAutoRun := autoRun);
      print(+p);
    } else if (eqic(internalCode, "convertInputToSyntacticPattern")) {
      S text = o.text();
      if (!containsStars(text)) ret with print("No stars");
      BEA p = uniqCI_returnIfNew BEA(type := "Syntactic Pattern", +text);
      cset(p, fromInput := o, byProcedure := procedure, byAutoRun := autoRun);
      print(+p);
    } else if (eqic(internalCode, "dropPunctuationFromPattern")) {
      S text = o.text();
      S text2 = dropPunctuation(text);
      if (eq(text, text2)) ret;
      
      BEA p = uniqCI BEA(
        type := o.type(),
        text := text2);
      cset(o, withoutPunctuation := p);
    } else if (eqic(internalCode, "makeSyntacticPattern")) {
      print(sp := calculations.makeSyntacticPattern(o));
    } else if (eqic(internalCode, "runFunctionOnInput")) {
      print("runFunctionOnInput");
      BEA function = beaGet(o, "function");
      BEA input = beaGet(o, "input");
      if (function == null || input == null) ret with print("Missing parameters");
      if (cget result(o) != null) ret with print("Has result");

      BEA result = calculations.reactFunctionWithInput(calculations.new BackEndAlgorithm, function, input);
      print(+result);
      if (result != null) {
        cset(result, request := o);
        cset(o, +result);
      }
    } else
      print("Unknown internal code");
  }

  BEA findInput(S text) {
    ret conceptWhereIC(BEA, type := "Input", +text);
  }
  
  S addRewriteHTML(BEA o) {
    ret ahref(addParamsToURL(crudLink(BEA),
      cmd := "new",
      title := "Add Rewrite",
      f_type := "Rewrite",
      f_text := getStringOpt text(o),
      f_isRewriteOf := o.id, metaInfo_isRewriteOf := "concept",
    ), "Add Rewrite");
  }
  
  @Override
  O serveIntegerLink(DynGazelleRocks.Req req, long id) {
    BEA o = getConceptOpt BEA(id);
    if (o != null)
      ret htitle(str(o)) + hrefresh(conceptLink_long(o));
    ret super.serveIntegerLink(req, id);
  }
  
  void distributeTestNotification() {
    distributeNotification("It is " + localTimeWithSeconds());
  }
  
  void distributeNotification(S text) {
    notificationQ.add(r {
      /*for (User user)
        if (nemptyAfterTrim(user.notificationSetting))
          sendNotification(text);*/
          
        S jsCode =
          "window.createNotification({ theme: 'success', showDuration: 3000 })("
          + jsonEncodeMap(message := text) + ");";
          
      sendJavaScriptToAllWebSockets(jsCode);
    });
  }
  
  void sendJavaScriptToAllWebSockets(S jsCode) {
    for (Pair<virtual WebSocket, WebSocketInfo> p : syncMapToPairs(webSockets)) {
      pcall(p.a, "send", jsonEncodeMap(eval := jsCode));
    }
  }
  
  void runInstruction(BEA o) {
    if (o == null) ret;
    try {
      BEA instruction = o;
      if (o.typeIs("Step in script"))
        instruction = (BEA) cget(o, "instruction");
      
      if (instruction.typeIs("Instruction | List objects by type")) {
        saveInstructionResult(o, beaList(getString typeToList(instruction)));
        ret;
      }
      
      if (instruction.typeIs("Instruction | List object types"))
        ret with saveInstructionResult(o, distinctCIFieldValuesOfConcepts(BEA, "type"));

      if (instruction.typeIs("Instruction | Filter list by text starting with")) {
        // find list made before
        BEA scriptRun = cgetBEA scriptRun(o);
        if (scriptRun == null) fail("Need to be run as part of script");
        L<BEA> steps = scriptRunSteps(scriptRun);
        int idx = indexOf(steps, o);
        if (idx < 0) fail("Step not found in script run");
        
        L<BEA> list = firstNotNull(map(reversed(takeFirst(steps, idx)),
          step -> optCast L(cget data(cgetBEA result(step)))));
         
        S prefix = getString prefix(instruction);
        L<BEA> filtered = filter(list, obj -> swic(obj.text(), prefix));
        saveInstructionResult(o, filtered);
        ret;
      }
      
      if (instruction.typeIs("Script")) {
        BEA script = instruction;
        BEA scriptRun = cnew BEA(type := "Script Run", +script);
        
        // Make an instance of all the instructions
        
        int i = 0;
        new L<BEA> steps;
        while not null (instruction = (BEA) cget(instruction, "step" + (++i))) {
          BEA step = cnew(BEA, type := "Step in script", step := i, +scriptRun, +instruction);
          steps.add(step);
        }
        
        cset(scriptRun, +steps);
        
        // TODO: run steps?

        ret;
      }
      
      cnew BEA(type := "Instruction Error", instruction := o, error := "Unknown instruction type");
    } catch e {
      cnew BEA(type := "Instruction Error", instruction := o, error := getStackTrace(e));
    }
  }

  BEA saveInstructionResult(BEA instruction, O data) {
    BEA result = cnew BEA(type := "Instruction Result",
      +instruction, +data);
    cset(instruction, +result);
    ret result;
  }
  
  L<BEA> scriptRunSteps(BEA scriptRun) {
    ret (L) cget steps(scriptRun);
  }
  
  S addCommentHTML(BEA o, O... _) {
    optPar S redirectAfterSave;
    optPar S defaultComment;
    optPar S text = "Add comment";
    
    ret ahref(addParamsToURL(crudLink(BEA),
      cmd := "new",
      title := "Add Comment",
      f_type := "Comment",
      f_on := o.id,
      f_text := unnull(defaultComment),
      +redirectAfterSave,
      autofocus := "f_text",
      metaInfo_on := "concept"), text);
  }
  
  S formatSubInput(BEA input, S text, S anchor) {
    S redirect = addAnchorToURL(currentReq->uriWithParams(), anchor);
    int goodCount = countConceptsCI BEA(type := "Sub-Input", +input, +text);
    int badCount = countConceptsCI BEA(type := "Bad Sub-Input", +input, +text);
    ret calculations.bestInputHTML(text)
      + " " + small(joinNemptiesWithSpace(
        ahref_noFollow(addParamsToURL(
          baseLink + "/storeSubInput", 
          label := "good",
          +text,
          input := input.id,
          +redirect),
        unicode_thumbsUp()),
        goodCount == 0 ? "" : n2(goodCount),
        ahref_noFollow(addParamsToURL(
          baseLink + "/storeSubInput", 
          label := "bad",
          +text,
          input := input.id,
          +redirect),
        unicode_thumbsDown()),
        badCount == 0 ? "" : n2(badCount)
      ));
  }

  LS renderObjectTypes(int max default Int.MAX_VALUE) {
    MultiSet<S> ms = distinctCIFieldValuesOfConcepts_multiSet(BEA, "type");
    ret mapPairs(takeFirst(max, multiSetToPairsByPopularity(ms)),
      (type, count) -> {
        S _type = or2(type, "(no type)");
        ret n2(count) + " " + ahref(baseLink + "/beaCRUD/" + urlencode(_type), htmlEncode2(_type));
      });
  }
  
  BEA autoMigrateToBase(BEA o) {
    ret migrateToClass(o, BEA);
  }
  
  BEA autoMigrateToCustomClass(BEA o) {
    ret migrateToClass(o, defaultCustomClass(o));
  }
  
  BEA migrateToClass(BEA o, Class<? extends BEA> targetClass) {
    // Is the object actually with us?
    if (o == null || o._concepts != db_mainConcepts()) null;
    
    // Do we switch to another class?
    if (targetClass != null && targetClass != _getClass(o)) {
      // make an unlisted copy
      
      BEA newObject = unlistedCopyToClass_withConverter_pcall(targetClass, o);
      
      // give new object chance to grab stuff from old one
      // while that one is still activated.
      // Note that new object is not YET activated at this point
      // and it isn't connected to the concepts either
      // DON'T use beaCall or cCall because enter() doesn't work
      // on "dead" objects.
      callOpt(newObject, "_fixAfterMigration", o);
      
      // actually replace in DB
      ret replaceConceptAndUpdateRefs(o, newObject);
    }
    ret o;
  }

  void setCodeState(BEA o, long timestamp, S codeHash, O state) {
    if (empty(state)) state = null;/* else
      state = "[" + formatGMTWithMilliseconds_24(timestamp) + "] "
        + (empty(codeHash) ? "" : "Code hash: " + codeHash + ". ")
        + state;*/
    cset(o, meta_codeState := state);
  }

  S codeForObject(BEA o) {
    ret getStringOpt meta_code(o);
  }
  
  S actualCodeHash(BEA o) {
    ret md5OrNull(codeForObject(o));
  }
  
  bool codeHashUpToDate(BEA o) {
    ret eq(actualCodeHash(o), getStringOpt meta_codeHash(o));
  }
  
  bool compileErrorInObject(BEA o) {
    ret swic(getStringOpt meta_codeState(o), "error.");
  }
  
  bool isDependentOn(BEA a, BEA b) {
    ret contains(optCast(Concept.RefL.class,
      getOpt meta_dependsOnCode(a)), b);
  }
  
  Cl<BEA> objectWithDependents(BEA o) {
    ret stepAllAndGet(TransitiveHull<>(x -> filter(y -> isDependentOn(y, x), list(BEA)), o));
  }
  
  BEA compileAndLoadObject(BEA o) {
    dm_mediumRefreshTranspiler();
    Cl<BEA> withDependents = objectWithDependents(o);
    
    // So you can prevent broken dependent objects from
    // breaking the compile of the main object (by setting meta_shouldActivate=false on the bad object)
    withDependents = filter(withDependents,
      x -> x == o || !isFalse(getBoolOpt meta_shouldActivate(x)));
    print("Dependents for " + o + ": " + withDependents);
    for (BEA x : withDependents)
      compileAndLoadObject_withoutDependents(x);
    ret (BEA) getMigration(o);
  }
  
  srecord UseClassDecl(long objID, S innerClassName) {}
  
  UseClassDecl parseUseClassDecl(S useClass) {
    LS tok = javaTokC(useClass);
    if (l(tok) == 3 && eqGet(tok, 1, ".") && isIdentifier(tok.get(2)) && isIntegerOrIdentifier(tok.get(0))) {
      S innerClassName = tok.get(2);
      long objID = trailingLong_regexp(first(tok));
      ret new UseClassDecl(objID, innerClassName);
    }
    null;
  }

  Class protoClass(BEA o) {  
    BEA proto = cast cget meta_prototype(o);
    S useClass = cast cget meta_useClass(o);
    
    if (nempty(useClass)) {
      UseClassDecl uc = parseUseClassDecl(useClass);
      if (uc != null) {
        S innerClassName = uc.innerClassName;
        long objID = uc.objID;
        proto = beaGet(objID);
        if (proto == null)
          fail("useClass invalid reference: " + objID + " not found");
        if (!usesLiveCustomCode(proto))
          fail("Prototype is not custom or not loaded: " + proto);
        Class protoClass = proto.getClass();
        Class outerClass = getOuterClass(protoClass, _defaultClassFinder());
        printVars(id := o.id, +useClass, +innerClassName, +protoClass, +outerClass);
        Class innerClass = getInnerClass(outerClass, innerClassName, _defaultClassFinder());
        printVars(+innerClass);
        if (innerClass == null)
          fail("Inner class not found: " + outerClass + " . " + innerClassName);
          
        cset(o, meta_dependsOnCode := ll(proto));
        ret innerClass;
      } else
        fail("meta_useClass has invalid syntax: " + useClass);
    }
      
    Class protoClass = null;
    if (proto != null) {
      if (!usesLiveCustomCode(proto))
        fail("Prototype is not custom or not loaded: " + proto);
      protoClass = proto.getClass();
    }
    
    ret protoClass;
  }
  
  S setCodeHash(BEA o) {
    S code = codeForObject(o);
    S codeHash = empty(code) ? null : md5(code);
    cset(o, meta_codeHash := codeHash);
    ret codeHash;
  }
  
  void compileAndLoadObject_withoutDependents(BEA o) ctex {
    print("compileAndLoadObject_withoutDependents", o);
    assertSame(db_mainConcepts(), o._concepts);
    ClassLoader cl = dm_moduleClassLoader();

    long timestamp = now();
    setCodeState(o, timestamp, null, "Compiling");

    S codeHash = null;
    transpileAndCompileForHotwiring_src.set(null);

    try {
      // Check meta_code and meta_prototype (latter overwrites the former)
      Class protoClass = protoClass(o);
      
      codeHash = setCodeHash(o);
      S code = codeForObject(o);

      if (empty(code)) {
        if (protoClass != null) {
          // no extra code, just instantiate prototype class
          o = migrateToClass(o, protoClass);
          setCodeState(o, timestamp, null, "Migrated to proto class " + shortClassName(protoClass));
          callActivate(o);
        } else {
          // Nothing to activate - migrate back to base
          o = autoMigrateToBase(o);
          setCodeState(o, timestamp, null, null);
        }
        ret;
      }
      
      actualCompile(o);
      
      reactivateAndMigratePrototypeUsers(o);
      
      print("Done compiling");
    } on fail e {
      cset(o, meta_javaClass := null);
      setCodeState(o, timestamp, codeHash, "Error. " + exceptionToStringShort(e));
    }
  }
  
  void setCodeHashAndActualCompile(BEA o) {
    setCodeHash(o);
    actualCompile(o);
  }
  
  void actualCompile(BEA o) {
    Class protoClass = protoClass(o);
    
    // safety check for reference (not currently used, we just check for master user status)
    cset(o, meta_codeSafety := null/*codeSafetyCheckResult(code)*/);

    DynClassName dcn = new(o.id, aGlobalID(), o.id);
    cset(o, meta_javaClass := dcn.fullClassName());
    
    S code = codeForObject(o);
    LS tok = javaTok(code);
    S baseClass = "BEAObject";
    new LS implementedInterfaces;
    
    // "implements ...";
    // e.g. "implements IF1<S>;"
    // e.g. "implements B123.IBla;"
    // e.g. "implements BBla from 12345;"
    
    for (int i : jfindAll(tok, "implements", (_tok, nIdx) -> {
      S left = get(_tok, nIdx-1);
      ret !(isIdentifier(left) || eq(left, ">"));
    })) {
      int iSemi = tok_findEndOfStatement(tok, i)-1;
      LS tokWhat = subList(tok, i+1, iSemi);
      S className;
      if (nCodeTokens(tokWhat) == 3 && eq(tokWhat.get(3), "from")) {
        className = tokWhat.get(1);
        className = "B" + tokWhat.get(5) + "." + className;
      } else
        className = join(dropFirstAndLast(tokWhat));
      className = expandShortClassRef(className);
      implementedInterfaces.add(className);
      clearTokens(tok, i, iSemi+2);
    }

    // e.g. "extends BWithResources;"
    int tokIdx;
    if ((tokIdx = jfind_any(tok,
        "extends <id>;",
        "extends <id>.<id>;",
        "extends <int>.<id>;",
        "extends <id>.<id>.<id>;")) > 0) {
      int j = indexOf(tok, ";", tokIdx);
      baseClass = joinSubList(tok, tokIdx+2, j-1);
      baseClass = expandShortClassRef(baseClass);
      clearTokens(tok, tokIdx, j+2);
    }
    
    jreplace(tok, "extends <id><.<id>>", "extends $2<" + dcn.middleClassName() + ".$5>");

    // e.g. "extends BBla from 12345;"
    if ((tokIdx = jfind_any(tok,
      "extends <id> from *;",
      "extends <id><<id>> from *;",
      "extends <id><<id>.<id>> from *;",
    )) > 0) {
      int iFrom = indexOf(tok, tokIdx, "from");
      S className = joinSubList(tok, tokIdx+2, iFrom-1);
      baseClass = expandShortClassRef("B" + parseFirstLong_regexp(tok.get(iFrom+2)) + "." + className);
      int j = indexOf(tok, ";", iFrom);
      clearTokens(tok, tokIdx, j+2);
    }
    
    new LS imports;
    
    // e.g. "import BBla from 12345;"
    while ((tokIdx = jfind_any(tok,
      "import <id> from <int>;",
      "import <id> from <id>;",
      "import static <id> from <int>;",
      "import static <id> from <id>;")) > 0) {
      
      int i = tokIdx+2;
      if (eqGet(tok, i, "static")) i += 2;
      int iStart = i;
      S className = tok.get(i);
      S bClass = addPrefix("B", tok.get(i += 4));
      S name = expandShortClassRef(bClass + "." + className);
      imports.add(joinSubList(tok, tokIdx, iStart-1) + " " + name + ";");
      int j = indexOf(tok, ";", tokIdx);
      clearTokens(tok, tokIdx, j+2);
    }
    
    S global = getGlobalCode(tok);
    code = join(tok);
    
    S src =
      lines(imports)
      + "mainPackage " + dcn.makePackageName() + "\n"
      + "mainClassName " + dcn.makeMainClassName() + "\n"
      + "concept " + dcn.makeBEAClassName() + " extends " +
        (protoClass == null ? baseClass : protoClass.getName().replace("$", "."))
      + (empty(implementedInterfaces) ? "" : " implements "
        + joinWithComma(implementedInterfaces))
      + " {\n" + code + "\n}\n"
      + global + "\n"
      + customCodePostlude();
    
    //print("SRC> ", src);
    ClassLoader cl = dm_moduleClassLoader();
    var files = concatLists(
      filesFromClassLoader(cl),
      filesFromClassLoader(getVirtualParent(cl)));
    
    print("Compiling with " + n2(files, "byte code path"));
    //pnlIndent(files);
    
    javaCompileToJar_localLibraries.set(files);
    File bytecode;
    new LS libs;
    try {
      bytecode = transpileAndCompileForHotwiring(src, libs);
      libs.remove(str(parseSnippetID(mainLibID)));
      print(+libs);
    } finally {
      S javaSrc = transpileAndCompileForHotwiring_src!;
      saveTextFile(javaSourceFileForObject(o), javaSrc);
      
      // add all mentioned objects as as dependency
      new LinkedHashSet<Concept> dependencies;
      for (S id : tok_identifiers(javaTok(javaSrc))) {
        long objID = regexpToLongIC("^b_?(\\d+)", id);
        if (objID != 0 && objID != o.id)
          dependencies.add(getConceptOrMarkMissingObject BEA(objID));
      }
      cset(o, meta_libraries := nullIfEmpty(joinWithSpace(libs)));
      cset(o, meta_dependsOnCode := nullIfEmpty(asList(dependencies)));
    }
    renameFileVerbose(bytecode, dcn.byteCodeFile());
    
    cset(o, meta_javaClass := dcn.fullClassName());
  }
  
  // TODO: aren't prototype users a part of objectWithDependents?
  void reactivateAndMigratePrototypeUsers(BEA o) {
    var p = reactivateDynamicObject(o);
    if (p == null) fail("Activation of " + o.id + " failed. " + getOpt meta_codeState(o));
    File deadPath = byteCodePathForClass(o); // TODO: isn't that the new path?
    print("New dead class path: " + deadPath + " (total: " + n2(deadClassPaths) + ")");
    deadClassPaths.add(deadPath);
    o = p.a;
    assertNotNull("Activated object", o);
    S codeHash = getStringOpt meta_codeHash(o);
    long timestamp = now();
    setCodeState(o, timestamp, codeHash, "Custom code loaded");
    print("Custom code loaded");
    
    migratePrototypeUsers(o);
  }

  // translate B123.Bla
  S expandShortClassRef(S name) {  
    LS groups = regexpGroups("^B?(\\d+)(\\..+)$", name);
    if (groups != null) {
      long baseID = parseLong(first(groups));
      BEA baseObj = beaGet(baseID);
      name = mainClassNameForObject(baseObj) + second(groups);
      if (empty(name)) fail("Referenced object doesn't have a Java class: " + baseID);
    }
    ret name;
  }

  void migratePrototypeUsers(BEA o) {  
    // migrate prototypeUsers to new class
    // (only those who don't add any code are included right now)
    for (BEA derivative : prototypeUsers(o)) pcall {
      print("Migrating prototype user " + derivative + " of " + o + " to new base class " + o.getClass());
      migrateToClass(derivative, o.getClass());
    }
  }
  
  Cl<BEA> prototypeUsers(BEA o) {
    TransitiveHull<BEA> th = new(x -> filter(
      conceptsWhere BEA(meta_prototype := x),
      d -> empty(codeForObject(d))), o);
    ret setMinus_inPlace(stepAllAndGet(th), o);
  }
  
  Pair<BEA, Bool> reactivateDynamicObject(BEA o) {
    try {
      ret reactivateDynamicObject_impl(o);
    } catch print e {
      cset(o, meta_javaClass := null);
      cset(o, meta_codeState := exceptionToStringShort(e));
      null;
    }
  }
  
  bool usesLiveCustomCode(BEA o) {
    ret startsWith(className(o), packagePrefix());
  }
  
  // pair(possibly updated object reference, migrated or not) 
  Pair<BEA, Bool> reactivateDynamicObject_impl(BEA o) ctex {
    assertNotNull(o);
    assertSame(db_mainConcepts(), o._concepts);
    DynClassName dcn = dcnForObject(o);
    if (dcn == null) {
      cset(o, meta_codeState := null);
      ret pair(o, false);
    }
    File byteCode = dcn.byteCodeFile();
    
    loadObjectsLibraries(o);
    dm_addByteCodePathToModuleClassLoader(byteCode);
    
    S fullName = dcn.fullClassNameForVM();
    print(+fullName);
    ClassLoader cl = dm_moduleClassLoader();
    Class targetClass = cl.loadClass(fullName);
    print(+targetClass);
    assertNotNull("Class not found: " + fullName, targetClass);
    o = migrateToClass(o, targetClass);
    assertNotNull("autoMigrated", o);

    callActivate(o);
    cset(o, meta_codeState := "[" + formatGMTWithMilliseconds_24() + "] " + "Custom code reloaded");
    ret pair(o, true);
  }

  File javaSourceFileForObject(BEA o) {
    ret programFile("BEA Java Sources/" + o.id + ".java");
  }
  
  File byteCodeFile(DynClassName name) {
    ret name == null ?: programFile("Object ByteCode/" + name.objectID + "-" + name.globalID + ".jar");
  }
  
  DynClassName dcnForObject(BEA o) {
    S javaClassName = getString meta_javaClass(o);
    ret DynClassName_parse(javaClassName);
  }
  
  File byteCodeFileForObject(BEA o) {
    DynClassName dcn = dcnForObject(o);
    ret dcn?.byteCodeFile();
  }
  
  BEA autoMigrateUnlistedOrKeep(BEA o) {
    ret canAutoMigrate(o) ? unlistedCopyToClass_withConverter(defaultCustomClass(o), o) : o;
  }
  
  bool canAutoMigrate(BEA o) {
    ret o != null && !eqOneOf(defaultCustomClass(o), _getClass(o), null);
  }
  
  Class<? extends BEA> defaultCustomClass(BEA o) {
    if (o == null) null;
    BEA entry = conceptWhereCI BEA(type := "Auto Custom Class", forType := o.type());
    //printVars_str defaultCustomClass(+o, +entry);
    if (entry == null) null;
    S className = getString toClassName(entry);
    //printVars_str defaultCustomClass(+className);
    Class c = findClassThroughDefaultClassFinder(className);
    //printVars_str defaultCustomClass(+c);
    ret c;
  }
  
  S queryLink(S algorithm, S q) {
    ret addParamsToURL(baseLink + "/query", +algorithm, +q);
  }

  S currentUserAgent() { ret userAgent(currentReq()); }
  
  S userAgent(Req req) {  
    if (req == null) null;
    ret mapGet(req.webRequest.headers(), "user-agent");
  }

  void saveUserAgent(BEA input) {  
    S userAgent = currentUserAgent();
    if (nempty(userAgent) && !userAgentIsBot(userAgent))
      uniqCI2 BEA(type := "Input Source",  +input, +userAgent);
  }

  bool isObjectWithCode(BEA o) {
    ret o != null && o.hasCustomCode();
  }

  bool createdByMasterUser(BEA o) {
    User user = optCast User(cget createdBy(o));
    ret user != null && user.isMaster;
  }

  void autoActivateDynamicObject(BEA o) {
    if (!autoActivateDynamicObjects) ret;
    S code = codeForObject(o);
    if (empty(code)) ret;
    if (!createdByMasterUser(o)) ret;
    S codeState = getString meta_codeState(o);
    S time = leadingAngleBracketStuff(codeState);
    long timestamp = empty(time) ? 0 : parseDateWithMillisecondsGMT(time);
    if (o._modified <= timestamp) ret;
    //S hash = regexpExtractIC("code hash: (\\w+)?", codeState);
    S hash = getString meta_codeHash(o);
    if (eq(hash, md5(code)))
      ret; // Code unchanged
    print("Auto-activating dynamic object: " + o);
    compileAndLoadObject(o);
    sleepSeconds(1); // safety sleep
  }

  S customCodePostlude() {  
    ret "!include early #1031282\n\n";
  }
  
  S makeJavaClassName(BEA o) {
    ret /*"b" + o.id + "_" +*/ aGlobalID();
  }
  
  S packagePrefix() {
    ret "dyn.b_";
  }
  
  S makePackageName(BEA o) {
    ret packagePrefix() + o.id;
  }
  
  BEA websiteEnhancersObject() {
    Cl<BEA> l = beaList("Live Website Enhancers");
    if (l(l) > 1) warn("Multiple Live Website Enhancers: " + l);
    ret first(l);
  }
  
  BEA getWebsiteEnhancer(S key) {
    ret beaGet(websiteEnhancersObject(), key);
  }
  
  S newLinkForCRUD(HCRUD crud, Class c) {
    if (c == User) ret baseLink + "/register";
    else ret super.newLinkForCRUD(crud, c);
  }
  
  S callHTMLFixer(O caller, S html) {
    BEA fixer = beaGet htmlFixer(websiteEnhancersObject());
    ret or((S) callOpt(fixer, "fixHTML", html, caller), html);
  }
  
  @Override
  O html3(DynGazelleRocks.Req req) { ret html3((Req) req); }
  O html3(Req req) {
    assertSame(realMC(), mc());
    assertSame(main beaMod(), me());
    vmBus_send logMethodCall(this, "html3", req);
    vmBus_send html3(this, req);
    
    BHTTPRequest reified = null;
    if (reifyHTTPRequests())
      reified = cnew BHTTPRequest(
        type := "Live HTTP Request",
        +req,
        uri := req.uri(),
        ip := req.webRequest.clientIP(),
        user := user(req));

    long time = sysNow();
    try {
      ret html4(req);
    } finally {
      time = sysNow()-time;
      if (reified != null)
        cset(reified, type := "Past HTTP Request", processingTime := time);
    }
  }

  O html4(Req req) {
    // generally save all "q" params as input
    // except if it's a live search
    // and only if someone is logged in
    if (saveAllQsAsInput && !cic(req.uri(), "/live")
      && user(req) != null) {
      S text = trim(req.get("q"));
      if (nempty(text)) {
        BEA input = uniqCI BEA(type := "Input", +text);
        uniqCI BEA(type := "Input Source", +input, source := "Web Request (q parameter)");
      }
    }

    O response = html5(req);
    if (response cast S)
      response = routeThroughAll(pagePostProcessors, response);
    ret response;
  }
  
  transient L<IF1<S>> pagePostProcessors = syncL();
  
  O html5(Req req) {
    S uri = req.uri();
    for (BEA rewrite : filter createdByMasterUser(beaList("Live URL Rewrite"))) pcall {
      S input = getString input(rewrite);
      bool ci = isTrueOpt caseInsensitive(rewrite);
      vmBus_send testingURLRewrite(rewrite, uri);
      if (eqOrEqic(ci, uri, input)) {
        vmBus_send applyingURLRewrite(rewrite, uri);
        S output = getString output(rewrite);
        /*
        SS newParams = paramsFromURL(output);
        req.uri = urlWithoutQuery(output);
        ModifiedWebRequest request2 = webRequest_modifyURI(req.webRequest, req.uri);
        request2.params = req.params = joinMaps(newParams, req.params());
        vmBus_send urlRewrite_newURI(req.uri);
        vmBus_send urlRewrite_newParams(req.params);
        break;
        */
        ret hrefresh(appendParamsToURL(output, req.params()));
      }
    }
    
    ret super.html3(req);
  }
  
  @Override
  O serveOtherPage(DynGazelleRocks.Req req) { ret serveOtherPage((Req) req); }
  O serveOtherPage(Req req) {
    if (eq(req.uri, "/login"))
      ret serveAuthForm(null);
      
    if (startsWith_addingSlash(req.uri, "/myLoginData"))
      ret serveMyLoginData(req);
      
    //try object serveSpecialCRUD(req);
      
    ret super.serveOtherPage(req);
  }

  // CRUD for non-master users (filtered)
  /*O serveSpecialCRUD(Req req) {
    S uri = req.uri();
    if (!uri.startsWith(crudBase())) null;
    
    //printVars_str serveSpecialCRUD(+uri);
      if (eq(uri, dropUriPrefix(baseLink, crudLink(UploadedImage)))) {
        HCRUD crud = makeCRUD(c, req);
        crud.filte
        ret serveCRUD(req, c, crud);
      }
  }*/
  
  
  void cleanMeUp_deactivateBEAObjects {
    Cl<BEA> objects = filter(beaList(), o -> hasMethod(o, "_deactivate"));
    print("Deactivating " + nObjects(objects) + " for shutdown");
    for (BEA bea : objects)
      callDeactivate(bea);
    if (nempty(objects)) print("Done");
  }
  
  // parses class names of the form
  //     dyn.b_123_bla.B456
  // or: dyn.b_123_bla.B456$Inner
  DynClassName DynClassName_parse(S className) {
    className = replace(className, "$", ".");
    LS groups = regexpFirstGroups("^dyn\\.b_(\\d+)_([a-z]+)\\.B(\\d+)\\$?(.*)", className);
    if (verboseClassLoading) printVars("DynClassName_parse", +className, +groups);
    if (groups != null)
      ret new DynClassName(
        parseLong(first(groups)),
        second(groups),
        parseLong(third(groups)),
        fourth(groups));
        
    ret DynClassName_parseOther(className);
  }
  
  // parses class names of the form
  //     dyn.b_123_bla.Anything
  DynClassName DynClassName_parseOther(S className) {
    className = replace(className, "$", ".");
    LS groups = regexpFirstGroups("^dyn\\.b_(\\d+)_([a-z]+)\\.(.*)", className);
    if (verboseClassLoading) printVars("DynClassName_parseOther", +className, +groups);
    
    if (groups != null)
      ret new DynClassName(
        parseLong(first(groups)),
        second(groups),
        0,
        third(groups));
        
    null;
  }
  
  S DynClassName_mainPrefix() { ret "dyn."; }
  S DynClassName_mainPrefixForVM() { ret "dyn."; }
  
  class DynClassName {
    long compileBaseObjectID;
    S globalID;
    long objectID; // can be 0 if innerClass is the class we look for
    S innerClass; // optional
    
    *(long *compileBaseObjectID, S *globalID, long *objectID) {}
    *(long *compileBaseObjectID, S *globalID, long *objectID, S *innerClass) {}
    
    S middleClassName() { ret objectID == 0 ? "" : "B" + objectID; }
    
    S makePackageName() { ret dropDotSuffix(DynClassName_mainPrefix()); }
    S makeMainClassName() { ret "b_" + compileBaseObjectID + "_" + globalID; }
    S makeBEAClassName() { ret joinNempties("$", middleClassName(), innerClass); }
    
    S fullClassName() {
      ret joinWithDot(makePackageName(), makeMainClassName(), makeBEAClassName());
    }
    
    S fullClassNameForVM() {
      ret makePackageName()
        + "." + makeMainClassName() + "$" + makeBEAClassName();
    }
    
    File byteCodeFile() {
      ret programFile("Object ByteCode/" + baseSystemVersion
        + "/" + compileBaseObjectID + "/" + globalID + ".jar");
    }
  } // end of DynClassName

  S mainClassNameForObject(BEA o) {
    if (o == null) null;
    S fullBaseClass = getString meta_javaClass(o);
      
    // Note: We are doing some DynClassName_parse stuff here basically
    ret takeFirst(fullBaseClass, lastIndexOf(fullBaseClass, "."));
  }
  
  void loadObjectsLibraries(BEA o) {
    LS libs = splitAtSpace_trim(getString meta_libraries(o));
    // TODO: use loaded version of library instead of latest
    if (addLibrariesToClassLoader(dm_moduleClassLoader(), libs))
      print("Loaded libs for " + o + ": " + libs);
  }

  void callActivate(BEA bea) {
    if (getBoolOpt meta_shouldActivate(bea, true)) {
      loadObjectsLibraries(bea);
      pcallOpt(bea, "_activate");
    }
  }
  
  void callDeactivate(BEA bea) {
    pcallOpt(bea, "_deactivate");
  }
  
  void onNewWebSocket(WebSocketInfo ws) {
    super.onNewWebSocket(ws);
    
    Req req = cast ws.req;
    S uri = ws.uri;
    BEA reifiedWebSocket = null;
    
    if (webSocketsToBEA()) { time "Reify WebSocket" {
      var webRequest = req?.webRequest;
      var cookie = cookieFromWebRequest(webRequest);
      //var headers = webRequest?.headers();
      User user = user(req);
      reifiedWebSocket = cnew(webSocketClass,
        webSocketInfo := ws,
        type := "Live WebSocket",
        +uri,
        //hasRequest := req != null,
        //hasWebRequest := webRequest != null,
        //cookie := dropLast(4, cookie),
        //headers := keysList(headers),
        //auth := req?.auth,
        +user);
      ws.dbRepresentation = reifiedWebSocket;
      //if (isTrue(ws.misc.get("jqueryLoaded")))
        ws.eval(replaceDollarVars(
          [[if (typeof $ !== "undefined") $(".webSocketPlaceholder").html($html);]],
          html := jsQuote(beaHTML(reifiedWebSocket))));
    }}
    
    onNewWebSocket2(ws);
    
    BEAForward forward = null;

    if (eq(uri, "/")) {    
      BEA o = beaGet(beaHomePageID);
      if (o != null)
        forward = new BEAForward(o, dropLeadingSlash(uri));
    }
    
    if (forward == null) {
      new Matches m;
      if (startsWith(uri, "/beaHTML/", m)) {
        BEA handler = beaGet(m.rest());
        if (handler != null)
          forward = new BEAForward(handler, afterSlash(m.rest()));
      } else
        forward = findBEAShortURL(uri);
    }
      
    printVars onNewWebSocket(+uri, +forward);
      
    if (forward != null) {
      var handler = forward.object;
      req.subURI = forward.subURI;
      
      print("Have WebSocket " + reifiedWebSocket + "/" + ws + " at " + uri + " for " + handler);
      if (!handler._isAlive())
        ret with print("HAndler not alive!?");

      cset(reifiedWebSocket, +handler);
      beaPcall(handler, "useWebSocket", ws);
    }
  }
  
  swappable void onNewWebSocket2(WebSocketInfo ws) {}
  
  transient Class<? extends BWebSocket> webSocketClass = BWebSocket;
  
  void onWebSocketClose(WebSocketInfo ws) enter {
    super.onWebSocketClose(ws);
    retireReifiedWebSocket((BWebSocket) ws.dbRepresentation);
  }
  
  void retireReifiedWebSocket(BWebSocket ws) {
    if (!keepReifiedWebSockets())
      cdelete(ws);
    else
      cset(ws, type := "Closed WebSocket");
  }
  
  transient timedCached[10.0] bool webSocketsToBEA() {
    ret getBoolOpt webSocketsToBEA(liveGazelleSettings());
  }
  
  transient timedCached[10.0] bool keepReifiedWebSockets() {
    ret getBoolOpt keepReifiedWebSockets(liveGazelleSettings());
  }
  
  transient timedCached[10.0] bool reifyHTTPRequests() {
    ret getBoolOpt reifyHTTPRequests(liveGazelleSettings());
  }
  
  BSettings gazelleSettings() {
    ret conceptWhere(BSettings);
  }
  
  BEA liveGazelleSettings() {
    ret first(filter createdByMasterUser(beaList("Live Gazelle Settings")));
  }
  
  O serveWithNavigation aka withFrame(S html) {
    var framer = framer();
    framer.add(html);
    ret completeFrame();
  }
  
  @Override
  O completeFrame_base(DynGazelleRocks.Req req) {
    cast req to Req;
    req.framer.addNavItems(pageBy_navItems(req.makers));
    ret super.completeFrame_base(req);
  }

  LS pageBy_navItems(Cl/O... makers) {
    makers = nonNulls(makers);
    if (empty(makers)) ret ll();
    
    L<BEA> beas = instancesOf BEA(makers);
    L nonBEAs = nonInstancesOf BEA(makers);
    
    ret concatLists(
      map(beas, o ->
        targetBlank(beaURL(o),
          repS(2, html_gazelle_madeByIcon()),
          title := "Page made by: " + o)),
          
      empty(nonBEAs) ? ll() : ll(new HPopDownButton(empty(beas) ? "Page by" : "Page also by",
        map(makers, o -> {
          if (o cast BEA) ret beaHTML_targetBlank(o);
          ret htmlEncode2(shortenStr(o, 40));
        })).html()));
  }

  // it's swappable
  LLS renderStats_base() {
    ret listPlusItems_inPlace(super.renderStats_base(),
      ll("Live / dead class paths", n2(classLoaderPathsCount(dm_moduleClassLoader())) + " / " + n2(deadClassPaths)),
    );
  }
  
  @Override
  swappable void makeNavItems(DynGazelleRocks.Req req, HTMLFramer1 framer) {
    super.makeNavItems(req, framer);
    S webSocketPlaceholder = span("", class := "webSocketPlaceholder");
    if (moveNavItemsToMisc) {
      var navItems = cloneList(framer.navItems);
      framer.clearNavItems();
      framer.addNavItem(hPopDownButtonWithText("Misc",
        map(navItems, ni -> framer.renderNavItem(ni)))
        + " " + webSocketPlaceholder);
    } else
      framer.addNavItem(webSocketPlaceholder);
    if (findConcept(CMissingObject) != null)
      framer.addNavItem(makeClassNavItem(CMissingObject, req));
  }

  // when HTML is longer than this bytes, restrict to fixed height and show the "more" button
  // we could always do this, but it's probably quite some overhead
  // (we currently do it once per table cell)
  transient int showMoreThreshold = 500;
  transient int showMoreHeight = 250;
  
  swappable S addShowMoreButton(S html) {  
    if (l(html) < showMoreThreshold) ret html;
    ret HDivWithVerticalExpandButton(showMoreHeight, html).html();
  }
  
  bool deleteQsWhenEmpty() {
    BSettings settings = gazelleSettings();
    ret settings != null && settings.deleteQsWhenEmpty;
  }
  
  Set<Long> activateOnlyTheseIDs() {
    if (empty(activateOnlyTheseIDs)) null;
    ret mapToLinkedHashSet parseLong(tok_integers(activateOnlyTheseIDs));
  }
  
  Set<Long> dontActivateTheseIDs() {
    if (empty(dontActivateTheseIDs)) null;
    ret mapToLinkedHashSet parseLong(tok_integers(dontActivateTheseIDs));
  }
  
  GazelleBEA beaMod() { this; }
  
  !include #1031610 // Methods for BEAObject as well as GazelleBEA

  S getGlobalCode(LS tok) {  
    S global = "";
    
    // I guess exports {} is the new preferred syntax
    int idx;
    while ((idx = jfind_any(tok, "global {", "exports {")) >= 0) {
      int j = findEndOfBracketPart(tok, idx+2);
      global = joinSubList(tok, idx+3, j-1);
      clearTokens(tok, idx, j+1);
    }
    
    ret global;
  }
  
  S getGlobalCode(BEA o) {
    ret getGlobalCode(javaTok(codeForObject(o)));
  }
  
  AutoCloseable beaEnter(BEA o) {
    if (o == null) null;
    ret combineAutoCloseables(
      tempSetTL(beaThreadOwner, o),
      enter());
  }
  
  // GUI stuff
  
  afterVisualize { addControls(); }
  
  void addControls {
  }
  
  swappable S beaObjectToString_long(BEA o) { 
    ret beaObjectToString_long_static(o);
  }
  
  sS beaObjectToString_long_static(BEA o) {
    if (o == null) ret "null";
    
    S type = strOrNull(o.typeForToString());
    S s = or2(type, "(no type)");
    
    s +=  " " + o.id;
    
    s += appendBracketed(strOrNull(o~.label));

    if (eqic(type, "Match"))
      s += " " + o.~mapping
        /*+ appendBracketed(o.~input + " + " + o~.pattern)*/;

    bool enabled = eq(true, getOpt enabled(o));
    if (enabled) s += " [enabled]";
    
    S purpose = getStringOpt purpose(o);
    if (nempty(purpose))
      s += " " + quote(purpose);
    
    S text = or2(o.text(),
      getStringOpt name(o),
      getStringOpt internalCode(o));
    if (text != null)
      s += " " + quote(text);
      
    O fc = o~.functionCalled;
    if (fc != null) s += " " + fc;
      
    O result = o~.result;
    if (result != null) s += " = " + shorten_str(result);
      
    ret s;
  }
  
  // for instance replacement
  bool useErrorHandling() { false; }
  
  User findOrCreateUserForLogin(S name, S pw) {
    try object User u = super.findOrCreateUserForLogin(name, pw);
    
    // At this point we know there is no user with that name
    
    /*virtual User u = vmBus_query lookupGazelleUser(name, pw, passwordSalt());
    if (u == null) null;
    
    S contact = getString contact(u);
    bool isMaster = getBoolOpt isMaster(u);*/
    
    if (empty(grabUsersFromDomain)) null;
    
    pcall {
      var passwordMD5 = SecretValue(hashPW(pw));
      S url = addSlashSuffix(grabUsersFromDomain) + "lookup-user";
      Map map = jsonDecodeMap(postPage(url, +name, +passwordMD5, salt := passwordSalt()));
  
      print("Got map? " + (map != null));
      S error = cast mapGet(map, "error");
      if (nempty(error))
        ret null with print(+error);
        
      if (map != null)
        ret cnew User(+name,
          contact := (S) map.get("contact"),
          +passwordMD5,
          isMaster := isTrue(map.get("isMaster")),
          copiedFromMainDB := true);
    }
    
    null;
  }
  
  S conceptLink(Concept c) {
    if (c cast BEA) ret beaShortURL(c);
    ret super.conceptLink(c);
  }
  
  S conceptLink_long(Concept c) {
    ret super.conceptLink(c);
  }
  
  S conceptToHTML_targetBlank(Concept c) {
    ret c == null ? "" : targetBlank(conceptLink(c), htmlEncode2(c));
  }
  
  Req newReq() {
    ret new Req;
  }
  
  class Req extends DynGazelleRocks.Req {
    // objects involved in handling the request
    Set makers = syncCompactSet();
    new Timestamp started;
    
    void add(O html) {
      framer.add(html);
    }
    
    bool debug() {
      ret eq(get("debug"), "1");
    }
    
    BEAHTMLFramer framer() { ret (BEAHTMLFramer) framer; }
  }
  
  @Override swappable O serveHomePage() {
    BEA o = beaGet(beaHomePageID);
    if (o != null) {
      Req req = currentReq();
      ret serveBEAHTML(req, o, dropLeadingSlash(req.uri()));
    }
    ret super.serveHomePage();
  }
  
  Req currentReq() {
    ret (Req) super.currentReq();
  }

  void cleanReifiedWebSockets enter {
    for (BWebSocket o : instancesOf BWebSocket(cloneList(beaList("Live WebSocket"))))
      if (o.webSocket() == null)
        retireReifiedWebSocket(o);
  }
  
  Class<? extends Concept> defaultCRUDClass() { ret BEA; }
  
  O serveFavIcon() {
    O response = super.serveFavIcon();
    print("serveFavIcon: " + response);
    ret response;
  }
  
  bool alwaysShowLogInLink() { true; }
  
  S logInLink() { ret baseLink + "/login"; }
  
  S defaultRedirectAfterLogin() { ret baseLink + "/crud/BEAObject"; }
  
  @Override void fillReqAuthFromCookie(DynGazelleRocks.Req req,
    S cookie, AuthedDialogID auth) {
    super.fillReqAuthFromCookie(req, cookie, auth);
    
    if (makeAnonymousUsers && auth != null && auth.user! == null) {
      S pw = aGlobalID();
      User userObj = cnew User(
        passwordMD5 := SecretValue(hashPW(pw)),
        password := SecretValue(pw)); // store pw clear so we can export it to user
      cset(userObj, name := "guest" + userObj.id);
      cset(auth, user := userObj);
      vmBus_send userCreated(userObj);
    }
  }
  
  bool userHasData(User user) {
    pcall {
      Cl<Concept> refs = allBackRefs(user);
      refs = withoutInstancesOf AuthedDialogID(refs);
      refs = withoutInstancesOf BWebSocket(refs);
      pnl("USERDATA", refs);
      ret nempty(refs);
    }
    true;
  }
  
  bool anonymousUserWarning(Req req, HTMLFramer1 framer) {
    User user = user(req);
    if (print(confirm := empty(req.get("confirmLogout")))
      && print(isAnonymousUser := isAnonymousUser(user))
      && print(userHasData := userHasData(user))) {
      framer.add(h2("Anonymous User Warning"));
      framer.add(p("You are currently logged in as anonymous user " + b(htmlEncode2(user)) + " and seem to have some data stored on the server. Do you want to download your login data before logging out?"));
      framer.add(p(joinWithSpace(
        hlinkButton(baseLink + "/myLoginData", "Download Login Data"),
        hlinkButton(baseLink + req.uri + "?logout=1&confirmLogout=1", "Just log me out")
      )));
      true;
    }
    false;
  }
  
  @Override O handleLogout(DynGazelleRocks.Req req) {
    cast req to Req;
    print("handleLogout " + user(req));
    if (anonymousUserWarning(req, framer(req)))
      ret completeFrame(req);

    print("Actual logout");
    ret super.handleLogout(req);
  }
  
  bool isAnonymousUser(User user) {
    // anonymous users have their password stored as plain text
    ret cget password(user) != null;
  }
  
  O serveMyLoginData(Req req) {
    User user = user(req);
    SecretValue<S> pw = cast cget password(user);
    bool isAnon = pw != null;
      
    if (!isAnon) {
      req.framer.add(p("You are not an anonymous user. Only anonymous users can download their login data."));
      ret beaMod().completeFrame(req);
    }
  
    if (eq(req.uri(), "/myLoginData/download")) {
      S host = mapGet(req.headers(), "host");
      S text = jsonEncode_breakAtLevel1(litorderedmap(
        site := host,
        user := user.name,
        password := pw!,
        date := dateWithSecondsUTC(),
        ip := getClientIPFromHeaders(req.headers())));
      S filename = "my-login-at-" + host + ".txt";
      ret subBot_noCacheHeaders(addFilenameHeader(filename, subBot_serveWithContentType(text, binaryMimeType())));
    }
    
    req.framer.add(hcenter(linesLL(
      h1(req.framer.htmlTitle("Download Login Data")),
      p("You are currently logged in as an anonymous user and are able to upload data to the site. In order to access, modify or delete this data later, you should definitely download your login data now. Otherwise you risk losing this data and/or being unable to delete it."),
      p(hbuttonLink(baseLink + "/myLoginData/download", "Download my login data"))
    )));
    ret completeFrame(req);
  }
  
  HTMLFramer1 framer(Req req) {
    if (req.framer == null) makeFramer(req);
    ret req.framer;
  }
  
  bool userCanSeeObject(User user, O o) {
    if (user == null) false;
    if (user.isMaster) true;
    o = derefRef(o);
    
    if (user == o) true;
    
    if (o cast Concept) {
      if (cget createdBy(o) == user) true;
      if (isTrueOpt isPublic(o)) true;
    }
    false;
  }
  
  @Override File uploadsBaseDir() {
    ret programFile("uploads");
  }
  
  class BEAHTMLFramer extends HTMLFramer1 {
    new HTMLPopDownButton unnamedPopDownButton;
  }
  
  S makeAbsoluteURL(Req req, S url) {
    ret main makeAbsoluteURL(
      (req.isHttps() ? "https://" : "http://") + req.domain(),
      url);
  }

  O serveBEAHTML(Req req, BEA o, S subURI) {  
    req.noSpam();
    req.subURI = subURI;
    req.makers.add(o);
      
    framer(req).title = str(o);

    IF0 result = call_optional(o, "html", req);
    if (result != null) ret result!;
    result = call_optional(o, "html", req.webRequest);
    if (result != null) ret result!;
    ret call(o, "html");
  }
  
  S optBeaHTML(O o, bool targetBlank default false) {
    if (o cast BEA) ret beaHTML(o, targetBlank);
    ret htmlEncode2(str(o));
  }
  
  S beaHTML_preferRendered(BEA o, bool targetBlank default false) {
   if (o != null && o.canRenderHTML())
      ret targetBlankIf(targetBlank, o.myUri(), htmlEncode2_nlToBr(str(o)));
    ret beaHTML(o, targetBlank);
  }
  
  // API
  
  // e.g. for webssh.gaz.ai - check if request is master authed
  bool checkCookie(S cookie, S domain) {
    print("checkCookie " + takeFirst(4, cookie));
    AuthedDialogID auth = authObject(cookie);
    if (auth == null) false;
    new Req req; // dummy request object
    fillReqAuthFromCookie(req, cookie, auth);
    ret masterAuthed(req);
  }
  
  bool masterAuthed(WebSocketInfo ws) {
    ret masterAuthed(ws?.req);
  }
  
  S redirectToLoginURL(Req req) {
    ret addParamsToURL(baseLink + "/login", redirect := req.uri());
  }
  
  O redirectToLogin(Req req) {
    ret subBot_serveRedirect(redirectToLoginURL(req));
  }
  
  S redirectUnlessMasterAuthed(WebSocketInfo ws) {
    if (masterAuthed(ws)) null;
    ret hrefresh(redirectToLoginURL((Req) ws.req));
  }
  
  S redirectToLogin_hrefresh(Req req) {
    ret hrefresh(addParamsToURL(baseLink + "/login", redirect := req.uri()));
  }
  
  @Override
  O html(IWebRequest request) enter {
    bool profile = eq(request.get("_profile"), "1");
    if (profile) {
      // Should reset _profile but whatever
      ret subBot_serveText(profileThisThreadToString(() -> {
        super.html(request);
      }));
    }
      
    ret super.html(request);
  }
  
  <A> A findService(Class<A> type) {
    if (type == null) null;
    S name = shortClassName(type);
    ret (A) findService(name);
  }
  
  BEA findService(S name) {
    if (empty(name)) null;
    var services = cloneList(serviceIndex.getAll(name));
    if (l(services) > 1)
      warn("Duplicate service " + name + ": " + joinWithSlash(services));
    ret first(services);
  }
  
  S beaAttribution(BEA o) {
    ret o == null ?: html_gazelle_madeByIcon(beaURL(o));
  }
  
  // data directory specifically for an object
  File beaDataDir(BEA o) {
    if (o == null) null;
    long id = o.id;
    ret id == 0 ? null : programFile("BEA Data " + id);
  }
  
  S beaTypeCRUDURL(S type) {
    ret baseLink + "/beaCRUD/" + urlencode(type);
  }
 
  @Override
  O serveCRUD(DynNewBot2.Req req) {
    if (eq(req.uri(), "/crud/BEA"))
      ret hrefresh(baseLink + "/crud/BEAObject");
    ret super.serveCRUD(req);
  }
  
  Cl<User> allUsers() { ret list(User); }
} // end of module / end of GazelleBEA

// not used. also, we can't extend User anymore as it's in loadableUtils
/*extend User {
  S notificationSetting;
}*/

concept BEAObject > ConceptWithGlobalID {
  // optional new Ref<UserPost> mirrorPost;
  UserPost mirrorPost() { ret (UserPost) cget mirrorPost(this); }
  
  delegate WebSocketInfo to GazelleBEA.

  void change :: after {
    var mod = beaMod();
    if (mod != null) {
      mod.rstUpdateBEAMirrors.add(this);
      if (mod.autoActivateDynamicObjects)
        mod.rstAutoActivateDynamicObjects.add(this);
    }
  }

  void delete :: before {
    cdelete(mirrorPost());
    pcallOpt(this, "_deactivate");
  }
  
  void updateMirrorPost {
    GazelleBEA mod = beaMod();
    if (isDeleted() || !mod.mirrorBEAObjects) ret;

    if (mirrorPost() == null)
      cset(this, mirrorPost := cnew UserPost(
        type := "BEA Object",
        creator := mod.internalUser(),
        botInfo := "BEA Mirror Bot"));

    S text = structureString();

    cset(mirrorPost(), 
      title := str(this),
      +text);
  }

  S structureString() {
    S text = "Error";
    pcall {
      structure_Data data = new {
        structure_ClassInfo newClass(Class c) {
          structure_ClassInfo info = super.newClass(c);
          if (c == Concept.Ref.class) {
            info.special = true;
            info.serializeObject = o -> {
              Concept cc = cast deref((Concept.Ref) o);
              //append("cu CRef " + (cc != null ? str(cc.id) : "null"), 3);
              if (cc != null)
                append("CRef(id=" + cc.id + ", c=" + quote(dynShortClassName(cc)) + ")", 6);
              else
                append("CRef", 1);
            };
          }
          ret info;
        }

        void setFields(structure_ClassInfo info, L<Field> fields) {
          if (isSubclassOf(info.c, BEA)) {
            // Don't serialize "refs" and "backRefs" fields
            removeAll(fields,
              getField(BEA, "refs"),
              getField(BEA, "backRefs"));
          }
          super.setFields(info, fields);
        }
      };
      
      S struct = structure(this, data);
      struct = structure_convertTokenMarkersToExplicit(struct);
      struct = dropLoadableUtilsPackageFromStruct(struct);
      text = indentStructureString_firstLevels(1, struct);
    }

    ret text;
  }
      
  toString {
    ret shorten(toString_long());
  }
  
  S toString_long() {
    var mod = beaMod();
    if (mod != null) ret mod.beaObjectToString_long(this);
    ret "[NO MODULE] " + mod.beaObjectToString_long_static(this);
  }
  
  bool isAlive aka _isAlive() {
    ret !_conceptsDefunctOrUnregistered();
  }
  
  bool hasCustomCode() {
    ret cget meta_code(this) != null
      || cget meta_prototype(this) != null
      || cget meta_useClass(this) != null;
  }
  
  bool customCodeLoaded() {
    S className = getStringOpt meta_javaClass(this);
    if (className != null) ret eq(replace(className(this), "$", "."), className);
    
    // quick & dirty test only checking for inner class name
    S useClass = getStringOpt meta_useClass(this);
    GazelleBEA.UseClassDecl uc = beaMod().parseUseClassDecl(useClass);
    if (uc != null) ret eq(shortClassName(this), uc.innerClassName);
    
    false;
  }

  // Note: beaGet is not mapMethodLike in here
  
  // access typical fields BEA objects have
  
  S type() { ret getStringOpt type(this); }
  bool typeIs(S type) { ret eqic(type(), type); }
  bool typeIsOneOf aka typeIsAny(S... types) { ret eqicOneOf(type(), types); }
  
  S text() { ret getStringOpt text(this); }
  
  BEA input() { ret (BEA) cget input(this); }
  BEA isRewriteOf() { ret (BEA) cget isRewriteOf(this); }
  
  S inputText() {
    BEA input = or(input(), isRewriteOf());
    ret input?.text();
  }
  
  SS mapping() {
    ret keysAndValuesToString(cgetOpt Map(this, "mapping"));
  }
  
  LS directCmds() { ret ll(); }
  
  Cl<BEA> allObjects() { ret list(_concepts, BEA); }
  
  S baseLink() { ret beaMod().baseLink; }
  
  BEA beaGet(S field, BEA o) { ret beaMod().beaGet(field, o); }
  BEA beaGet(long id) { ret beaMod().beaGet(id); }
  
  // TODO: this is confusing, it differs from beaMod().beaGet(S id)
  BEA beaGet(S field) { ret beaMod().beaGet(field, this); }
  
  !include #1031610 // Methods for BEAObject as well as GazelleBEA

  gazelle.main.GazelleBEA beaMod() {
    ret _concepts == null
      ? /*null*/ // XXX - considering this object dead in this case?
        main beaMod() 
      : (GazelleBEA) _concepts.miscMapGet(DynNewBot2);
  }
  
  Cl<BEA> beaList aka beaAll() { ret beaMod().beaList(); }
  Cl<BEA> beaListAny(S... types) { ret beaMod().beaListAny(types); }
  Cl<BEA> beaList(S type, O... params) { ret beaMod().beaList(type, params); }
  
  S beaHTML(BEA o, bool targetBlank default false) { ret beaMod().beaHTML(o, targetBlank); }
  
  S beaShortURL aka beaURL(BEA o) { ret beaMod().beaURL(o); }
  
  S fixHTML aka fixHtml aka htmlFixer aka htmlFix(S html) { ret beaMod().callHTMLFixer(this, html); }
  
  bool createdByMasterUser(BEA o) { ret beaMod().createdByMasterUser(o); }
  
  S myURI aka myUri() {
    S shortURI = first(myShortURLs());
    if (shortURI != null)
      ret beaMod().baseLink + shortURI;
    ret beaMod().baseLink + "/beaHTML/" + id;
  }
  
  void massageItemMapForList(MapSO map) {}
  
  BEA mapMethodLike beaGet(S key, GazelleBEA.Req req) {
    ret beaMod().beaGet(key, req);
  }
  
  selfType me() { this; }
  
  O completeFrame(GazelleBEA.Req req) { ret beaMod().completeFrame(req); }
  
  S typeForToString() { ret strOrEmpty(cget type(this)); }
  
  bool masterAuthed(GazelleBEA.Req req) {
    ret beaMod().masterAuthed(req);
  }
  
  GazelleBEA.BEAHTMLFramer framer(GazelleBEA.Req req) {
    ret (GazelleBEA.BEAHTMLFramer) beaMod().framer(req);
  }
  
  User user(GazelleBEA.Req req) {
    ret beaMod().user(req);
  }
  
  GazelleBEA.Req currentReq() {
    ret beaMod().currentReq();
  }
  
  bool isHomePage() { ret beaMod().beaHomePageID == id; }
  
  S optBeaHTML(O o, bool targetBlank default false) {
    ret beaMod().optBeaHTML(o, targetBlank);
  }
  
  bool haveShortURL(S uri) {
    var forward = beaMod().findBEAShortURL(uri);
    ret forward != null && forward.object == this;
  }
  
  Cl<S> myShortURLs() {
    ret collectStrings uri(filter(findBackRefs BEA(this), o -> beaMod().isBEAShortURLObject(o)));
  }
  
  bool canRenderHTML() {
    ret hasMethodNamed(this, "html");
  }
  
  S beaAttribution(BEA o) {
    ret beaMod().beaAttribution(o);
  }
  
  S madeByMeStamp() { ret beaAttribution(this); }
  
  <A> A findService(Class<A> type) {
    ret beaMod().findService(type);
  }
  
  BEA findService(S name) {
    ret beaMod().findService(name);
  }
  
  Cl<BEA> masterObjects(Cl<BEA> l) {
    ret beaMod().masterObjects(l);
  }
  
  S redirectUnlessMasterAuthed(WebSocketInfo ws) {
    ret beaMod().redirectUnlessMasterAuthed(ws);
  }
  
  bool masterAuthed(WebSocketInfo ws) {
    ret beaMod().masterAuthed(ws);
  }
  
  File myDataDir() {
    ret assertNotNull(beaMod().beaDataDir(this));
  }
} // end of BEAObject / end of class BEAObject

beaConcept BEARegExp {
  S text;
  bool caseInsensitive = true;
  
  bool valid() { ret nempty(text); }
  
  java.util.regex.Pattern compile() {
    ret compileRegexpPossiblyIC_unicodeCase(text, caseInsensitive);
  }
}

beaConcept BEARegExpReplacement > BEARegExp {
  S replacement;

  S apply(S text) {
    try {
      ret regexpReplace_directWithRefs(compile().matcher(text), unnull(replacement));
    } catch e {
      fail(format_quoted("Was searching * in * ", this.text, text));
    }
  }
  
  LS directCmds() {
    ret listPlus(super.directCmds(),
      !valid() ? "Note: not valid"
      : targetBlank(beaMod().queryLink("Apply regular expression replacement to all inputs", str(id)), "Apply to all inputs"));
  }
}

beaConcept BEAPatternList {
  new RefL<BEA> patterns;
  
  toString {
    ret super.toString() + ": " + nPatterns(patterns);
  }
}

static GazelleBEA beaMod() {
  //ret (GazelleBEA) botMod();
  ret (GazelleBEA) dm_current_mandatory();
}

set flag hotwire_here.

// share ISpec interface with sub-modules
static JavaXClassLoader hotwire_makeClassLoader(L<File> files) {
  ClassLoader cl = myClassLoader();
   ret new JavaXClassLoaderWithParent2(null, files, cl, ll(/*TODO*/));
}

// class BWebSocket
beaConcept BWebSocket {
  transient GazelleBEA.WebSocketInfo webSocketInfo;
  transient WithTimestamp<byte[]> screenShot;
  transient Set<Runnable> onScreenShotChanged = syncLinkedHashSet();
  
  virtual WebSocket webSocket() {
    ret getWeakRef(webSocketInfo?.webSocket);
  }
  
  void sendJavaScript(S js) {
    if (empty(js)) ret;
    virtual WebSocket ws = webSocket();
    call(ws, "send", jsonEncode(litorderedmap(eval := js)));
  }
  
  void setScreenShot(WithTimestamp<byte[]> screenShot) {
    this.screenShot = screenShot;
    vmBus_send screenShotChanged(this, screenShot);
    print("Listeners: " + cloneList(onScreenShotChanged));
    pcallFAll(onScreenShotChanged);
  }
}

concept BWithResources > BEA {
  transient new CloseablesHolder resources;
  transient Q q = startQ(); // object's queue
  L<WithTimestamp<S>> log; // object's log
  transient bool activated;
  bool meta_shouldActivate = true;
  
  AutoCloseable enter() {
    ret beaMod().beaEnter(this);
  }
  
  void mapMethodLike beaPrintVars(O... _) {
    beaPrint(renderVars(_));
  }
  
  void _handleException(Throwable e) {
    beaPrint(getStackTrace(e));
  }
  
  bool _active() { ret activated; }
  
  // TODO: sync
  final void _activate {
    addToQ(r {
      if (activated) ret;
      set activated;
      _activateImpl();
    });
  }

  void _activateImpl() {}

  void _deactivate {
    addToQ(r {
      _deactivateImpl();
      resources.close();
      unset activated;
    });
  }
  
  // must be idempotent
  void _deactivateImpl() {}
  
  void delete :: before {
    _deactivate();
    waitForQToEmpty(q());
  }
  
  void addResource aka ownResource(AutoCloseable r) {
    resources.add(r);
  }
  
  void clearBEAPrintLog() {
    inQ(r {
      log = null;
      change();
    });
  }
  
  // per printed line
  int maxBeaPrintLength() { ret 10000; }
  
  <A> A beaPrintAndReturn(S s default "", A a) {
    beaPrint(s, a);
    ret a;
  }
  
  // if you change this, gotta recompile many objects
  // <A> A beaPrint(S s default "", A o) enter {
  void beaPrint(S s default "", O o) enter {
    inQ(r {
      S text = combinePrintParameters(s, o);
      text = shorten(maxBeaPrintLength(), text);
      if (log == null) log = new L;
      synchronized(log) {
        truncateListFromStart(log, maxLogSize()-1);
        printIndent(id + "|", text);
        log.add(WithTimestamp(text));
      }
      change();
    });
    //ret o;
  }
  
  // can change through dynamic field
  int maxLogSize() { ret getIntOpt maxLogSize(this, 10); }
  
  selfType me() { this; }
  
  // run directly if in queue
  void inQ(Runnable r) {
    if (r == null) ret;
    if (isInQ(q))
      r.run();
    else
      addToQ(r);
  }
  
  // always add to end of queue even when called in queue
  /*synchronized*/ void addToQ(Runnable r) {
    if (r == null) ret;
    getQ().add(rEnter {
      try {
        r.run();
      } catch e {
        beaPrint(getStackTrace(e));
      }
    });
  }
  
  /*synchronized*/ Q getQ aka q() {
    /*if (q == null) {
      if (beaMod().verboseQStartsAndStops) print("Starting Q for " + me());
      q = new Q(str(me())) {
        void onIdle() { if (beaMod().deleteQsWhenEmpty()) deleteQIfEmpty(); }
      };
    }*/
    ret q;
  }
  
  /*synchronized void deleteQIfEmpty() {
    if (q == null) ret;
    synchronized(q.mutex()) {
      if (q.isEmpty()) {
        if (beaMod().verboseQStartsAndStops) print("Stopping Q for " + me());
        dispose q;
      }
    }
  }*/
  
  void useWebSocket(GazelleBEA.WebSocketInfo ws) {}
}

concept BSettings {
  bool deleteQsWhenEmpty; // experimental
}

beaConcept BHTTPRequest {
  transient GazelleBEA.Req req;
}

sclass WebSocketSet {
  delegate WebSocketInfo to GazelleBEA.
  
  Set<WebSocketInfo> set = syncWeakSet();
  event countChanged;
  event wsAdded(WebSocketInfo ws);
  event wsRemoved(WebSocketInfo ws);
  
  void add(WebSocketInfo ws) {
    if (ws == null) ret;
    ws.onClose(() -> remove(ws));
    if (set.add(ws)) {
      wsAdded(ws);
      countChanged();
    }
  }
  
  void remove(WebSocketInfo ws) {
    if (set.remove(ws)) {
      wsRemoved(ws);
      countChanged();
    }
  }
  
  int size() { ret l(set); }
  bool empty() { ret size() == 0; }
  bool nempty() { ret size() != 0; }
  
  L<WebSocketInfo> getList() { ret cloneList(set); }
  
  void eval(S js, O... _) {
    broadcast(jsonEvalMsg(js, _));
  }
  
  void broadcast(S data, WebSocketInfo excludeSender default null) {
    var sockets = cloneList(set);
    //beaPrint("Broadcasting to " + n2(sockets, "web socket") + ": " + shorten(data, 20));
    for (WebSocketInfo ws : sockets) pcall {
      if (ws == excludeSender) continue;
      try {
        ws.send(data);
      } catch e {
        //print("Removing faulty WebSocket");
        remove(ws);
        ws.close();
      }
    }
  }
}

// a BEA object managing a set of websockets

concept BWithWebSockets extends BWithResources {
  delegate WebSocketInfo to GazelleBEA.
  
  transient new WebSocketSet webSockets;
  transient new FlexibleRateTimer timer;
  double updateFrequency = 0.5; // Hz
  
  {
    webSockets.onCountChanged(r { inQ(r webSocketCountChanged) });
    webSockets.onWsAdded(lambda1 wsAdded);
    webSockets.onWsRemoved(lambda1 wsRemoved);
  }
  
  void webSocketCountChanged {}
  void wsAdded(WebSocketInfo ws) {}
  void wsRemoved(WebSocketInfo ws) {}
    
  @Override void useWebSocket(WebSocketInfo ws) {
    if (acceptWebSocket(ws))
      webSockets.add(ws);
    else
      ws.send(jsonEncodeMap(webSocketAccepted := false));
  }
  
  // overridable
  bool acceptWebSocket(WebSocketInfo ws) { true; }
}

// a BEA object that delivers ALL its content by websocket

sclass BByWebSocket extends BWithResources {
  delegate WebSocketInfo to GazelleBEA.
  delegate Req to GazelleBEA.
  
  noeq record Rendered(S html, Runnable afterSent) {
    *(S *html) {}
  }
  
  O html(Req req) {
    new HInitWebSocket iws;
    iws.autoOpen = false;
    if (req.isPost())
      iws.params.put("_hasPostData", "1");
    iws.reconnectParams = "reconnect=1";
    
    ret hhtml(linesLL(
      hhead(nemptyLinesLL(
        hmobilefix(),
        iws!,
        !eq(req.get("wsVerbose"), "1") ? null : hjs([[wsVerbose = true;]]),
        // hold POST params for websocket
        !req.isPost() ? null : hjs_escapedDollarVars([[
          var _postParams = $params;
          wsOnOpen(function() { wsSend(JSON.stringify({"postData": _postParams})); });
        ]], params := req.params()),
        hsansserif(),
        hjs(js_nodeScriptReplace2()), // We need it later anyway, so let's use it for inserting the loadingContent too (only if JS enabled)
      )),
      hbody(
        // Satisfy the noscripters
          hnoscript(noScriptContent())
        + loadingContent()
        + hjs([[
          var lc = document.getElementById("loading-content");
          if (lc) lc.style.display = "table";
          console.log("Opening WebSocket"); ws.open();
        ]])
      ) // end of body
    );
  }
  
  S loadingContent() {
    ret hcss("body { background-color: #ecf0f1; #loading-content { display: none; }")
      + hfullcenter(hsnippetimg(#1102948, title := "Initializing WebSocket")
      + hnoscript(noScriptContent()),
      id := "loading-content");
  }
  
  // what to show to Non-JavaScript users.
  // Can be a hrefresh.
  S noScriptContent() {
    ret "Please enable JavaScript to see this page.";
  }
  
  // override me
  Rendered htmlX(WebSocketInfo ws) {
    ret new Rendered(html(ws));
  }
  
  // or me
  S html(WebSocketInfo ws) {
    ret hhead(linesLL(
      htitle("Hello world"),
      hsansserif(),
    )
      + hbody(hfullcenter(span("Actual contents!", style := "font-size: 30px")));
  }
  
  Rendered htmlX_safe(WebSocketInfo ws) {
    try {
      ret htmlX(ws);
    } catch print e {
      ret new Rendered("Error: " + e);
    }
  }
  
  /*void handleWSMessage(S s) {
    pcall-short {
      Map json = jsonDecodeMap(s);
      handleWSMessage(json);
    }
  }
  
  // override me
  void handleWSMessage(Map map) {
  }*/
  
  bool isReconnected(WebSocketInfo ws) {
    ret ws != null && eq(ws.get("reconnect"), "1");
  }
  
  // override me
  void handleReconnect(WebSocketInfo ws) {}
  
  void useWebSocket(WebSocketInfo ws) {
    try {
      bool re = isReconnected(ws);
      beaPrint("Have " + stringIf(re, "reconnected ") + " websocket: " + ws);
      //ws.onStringMessage(s -> handleWSMessage(s));
      if (re) {
        handleReconnect(ws);
      } else {
        double delay = min(10.0, parseDouble(ws.get("_wsDelay")));
        sleepSeconds(delay);
        Rendered html = htmlX_safe(ws);
        if (html != null && html.html != null) {
          sendHTML(ws, html.html);
          callF(html.afterSent);
        }
      }
    } catch e {
      beaPrint(e);
    }
  }
  
  void sendHTML(WebSocketInfo ws, S html) {
    ws?.eval(js_replaceHTML(html));
  }
  
  Rendered renderedIfNempty(S html) {
    ret empty(html) ? null : new Rendered(html);
  }
} // end of BByWebSocket

sS mainLibID;
Travelled to 4 computer(s): bhatertpkbcr, mowyntqkapby, mqqgnosmbjvj, onxytkatvevr
Snippet ID:	#1032712
Snippet name:	GazelleBEA [LIVE]
Eternal ID of this version:	#1032712/95
Text MD5:	891bd39614cf530fbd088be272d3bb97
Author:	stefan
Category:	javax
Type:	JavaX fragment (include)
Public (visible to everyone):	Yes
Archived (hidden from active list):	No
Created/modified:	2022-11-16 16:30:51
Source code size:	133025 bytes / 4031 lines
Pitched / IR pitched:	No / No
Views / Downloads:	388 / 735
Version history:	94 change(s)
Referenced in:	[show references]
< > BotCompany Repo | #1032712 // GazelleBEA [LIVE]

JavaX fragment (include) [tags: use-pretranspiled]

Author comment
