GazelleBEA [backup before new dynamic class names] [1031465]

// TODO: prevent non-master users from setting createdBy field manually

mainPackage gazelle
mainClassName main

set flag needToKnowMainLib.
!include early #1031277 // BEA general includes

!include once #1030833 // BEACalculations

set flag defaultDefaultClassFinder_debug.

set flag CleanImports.
set flag DynModule.
set flag NoNanoHTTPD.
set flag AllPublic. // for dynamic BEA objects
rewrite BEA with BEAObject.

abstract static class GazelleBEA > DynGazelleRocks {
  switchable bool mirrorBEAObjects; // don't do it for now
  switchable bool enableAutoRuns = true;
  switchable bool enableNewBEAObjectNotis = true;
  switchable bool printAllConceptChanges;
  switchable bool useShadowLogger = true;
  switchable bool autoActivateDynamicObjects = true;
  switchable bool reloadWhenMainLibUpdates;

  transient ReliableSingleThread_Multi<BEAObject> rstAutoRuns = new(1000, lambda1 performAutoRuns);
  transient Q notificationQ;
  transient ReliableSingleThread_Multi<BEAObject> rstDistributeNewObject = new(1000, lambda1 distributeNewObject_impl);
  transient Set<BEAObject> newObjectsDistributed = weakSet();

  transient ReliableSingleThread_Multi<BEAObject> rstUpdateBEAMirrors = new(100, c -> enter { c.updateMirrorPost(); });
  
  transient ReliableSingleThread_Multi<BEAObject> rstAutoActivateDynamicObjects = new(100, c -> enter { autoActivateDynamicObject(c); });

  transient BEACalculations calculations = new(this);
  transient BEACalculations calc = calculations;
  transient int newFieldsToShow = 3;
  transient bool allowAnonymousInputUpload = true; // TODO
  switchable int maxInputLength = 50000;
  
  transient ConceptClassesDependentValue<Int> inputsWithoutRewrites;
  transient ConceptClassesDependentValue<Int> inputsWithoutMatches;
  transient ConceptClassesDependentValue<Int> syntacticPatternsWithoutRewrites;

  // add more fields here
  
  !include #1030883 // DB quickImport mix-in
  
  void init {
    super.init();
    //set quickDBReloadEnabled;
    botName = heading = adminName = "Gazelle BEA";
    set enableVars;
    unset showTalkToBotLink;
    unset phoneNumberSpecialInputField;
    unset showMetaBotOnEveryPage;
  }

  void startDB {
    // TODO: also load classes referenced between objects
    // e.g. because of subclassing
    O classFinder = _defaultClassFinder();
    db_mainConcepts().classFinder = func(S name) -> Class enter {
      print("Looking for class " + name);
      if (name.startsWith("dyn.")) {
        long conceptID = firstLong_regexp(name);
        BEA o = beaGet(conceptID);
        
        if (o != null) {
          File bytecode = byteCodeFileForObject(o, shortenClassName(name));
          print("Found byte code: " + bytecode);
          dm_addByteCodePathToModuleClassLoader(bytecode);
        }
              
        ret print("Found dynamic: ", _getClass(module(), name));
      }
      ret (Class) callF(classFinder, name);
    };
    
    db();
    db_mainConcepts().useBackRefsForSearches = true;
    inputsWithoutRewrites = ConceptClassesDependentValue(litset(BEA), () -> countPred(beaList("Input"), c -> empty(beaBackRefs(c, "Rewrite"))));
    inputsWithoutMatches = ConceptClassesDependentValue(litset(BEA), () -> countPred(beaList("Input"), c -> empty(beaBackRefs(c, "Match"))));
    syntacticPatternsWithoutRewrites = ConceptClassesDependentValue(litset(BEA), () -> countPred(beaList("Syntactic Pattern"), c -> empty(beaBackRefs(c, "Rewrite"))));
    
    if (useShadowLogger) {
      ConceptsShadowLogger shadowLogger = new(db_mainConcepts());
      shadowLogger.install();
      //shadowLogger.writer = printWriter(deflaterOutputStream_syncFlush_append(programFile("shadow.log.deflated")));
      shadowLogger.writer = filePrintWriter_append(programFile("shadow.log"));
      dm_doEvery(10.0, r { shadowLogger.flush(); });
      ownResource(shadowLogger);
    }
    
    if (printAllConceptChanges) printAllConceptChanges();
  }

  void start {
    print("Module ID: " + dm_moduleID());
    set !useBotNameAsModuleName;
    S name = mcDollar() + "BEAObject";
    print(+name);
    assertEquals(BEAObject, callF(defaultDefaultClassFinder(), name));
    assertEquals(ConceptWithGlobalID, callF(defaultDefaultClassFinder(), mcDollar() + "ConceptWithGlobalID"));
    //seedDBFrom(#1030602);
    set botDisabled;
    set storeBaseClassesInStructure;
    super.start();
    print("main concepts: " + db_mainConcepts() + ", count: " + db_mainConcepts().countConcepts());
    print(renderConceptClassesWithCount(db_mainConcepts()));
    print("Inputs: " + n2(beaCount("Input")));
    set showCRUDToEveryone;
    //set showOnlySelectedObject;
    
    // reload module when lib changes
      dm_onSnippetTranspiled(mainLibID, r {
        if (reloadWhenMainLibUpdates)
          dm_reloadModule();
      });

    if (!enabled) ret;

    // mirror all objects to be sure
    rstUpdateBEAMirrors.addAll(list(BEAObject));
    
    newObjectsDistributed.addAll(list(BEAObject));
    
    /*onIndividualConceptChange_notOnAllChanged(BEAObject,
      p -> { calculations.makeSyntacticPattern(p); });*/
      
    onIndividualConceptChange_notOnAllChanged(BEAObject,
      o -> {
        if (enableAutoRuns) rstAutoRuns.add(o);
        if (enableNewBEAObjectNotis && newObjectsDistributed.add(o))
          rstDistributeNewObject.add(o);
      });
      
    notificationQ = dm_startQ();
    
    // fix refs occasionally
    
    dm_doEvery(5*60.0, r-enter {
      print(ConceptsRefChecker(db_mainConcepts()).runAndFixAll());
    });
    
    // reactivate the dynamic objects
    
    if (autoActivateDynamicObjects)
      dm_startThread("Reactivator", r {
        var list = filter(beaList(), o -> nempty(getString meta_javaClass(o)));
        
        new L<BEA> toRecompile;
        
        bool progress;
        do {
          new L<BEA> postponed; // because prototype not activated yet
          progress = false;
          
          for ping (int idx, BEA o : unpair iterateWithIndex1(list)) {
            printWithMilliseconds("Reactivating object " + idx + "/" + l(list) + ": " + o);
            if (basedOnUnactivatedPrototype(o)) {
              print("Postponing " + o + ", waiting for " + cget meta_protype(o));
              postponed.add(o);
            } else pcall {
              set progress;
              if (!reactivateDynamicObject(o))
                toRecompile.add(o);
            }
          }
          list = postponed;
        } while (progress);
        
        if (nempty(list))
          pnl("Couldn't reactivate: ", list);
        
        print("Done reactivating, will recompile " + n2(toRecompile));
        for (BEA o : toRecompile) pcall {
          activateDynamicObject(o);
        }
      });
      
    // call _activate on all objects
    for (BEA bea)
      pcallOpt(bea, "_activate");
      
    print("All objects activated");
  }

  void makeIndices :: after {    
    indexConceptFieldDesc(BEAObject, "_modified");
    indexConceptFieldCIWithTopTen(BEAObject, "type");
    indexConceptFieldIC(BEAObject, "text");
    indexConceptFieldIC(BEAObject, "purpose");
  }
  
  L<Class> crudClasses(Req req) {
    ret listPlus(super.crudClasses(req), BEAObject, /*BEAPatternList*/);
  }
  
  Set<Class> hiddenCrudClasses() {
    Set<Class> set = super.hiddenCrudClasses();
    addAll(set, Conversation, ExecutedStep, InputHandled);
    ret set;
  }
  
  S authFormHeading() {
    ret h3("Gazelle BEA");
  }
  
  void makeFramer(Req req) {
    super.makeFramer(req);
    
    req.framer.addInHead(hjs_autoExpandingTextAreas());
  }

  <A extends Concept> HCRUD_Concepts<A> crudData(Class<A> c, Req req) {
    HCRUD_Concepts<A> cc = super.crudData(c, req);
    
    if (c == BEAObject) {
      cc.humanizeFieldNames = false;
      cc.convertConceptValuesToRefs = true;
      cc.itemName = () -> "BEA Object";
      
      cc.conceptClassForComboBoxSearch = (info, query) -> {
        if (endsWith(info, "_nativeValue"))
          ret Concept;
        ret cc.conceptClassForComboBoxSearch_base(info, query);
      };
      
      cc.onCreate.add(o ->
        cset(o, createdBy := currentUser()));
      
      cc.getObjectForDuplication = id -> {
        BEA o = beaGet(toLong(id));
        MapSO item = cc.getObjectForDuplication_base(id);
        item.put(createdBy := req.auth.user!); // set default creator to current user
        item.put(createdFrom := o);
        
        // call enhancer object
        item = (MapSO) pcallOpt(cget cleanObjectForDuplication(websiteEnhancersObject()), "cleanObjectForDuplication", o, item);
        
        ret item;
      };
      
      cc.emptyObject = () -> {
        MapSO item = cc.emptyObject_base();
        item.put(type := "");
        ret item;
      };
      
      Set<S> deletableRefs = litciset("Match");
      
      cc.objectCanBeDeleted = id -> {
        BEA o = cast cc.conceptForID(id);
        ret userCanEditObject(user(req), o)
          && all(findBackRefs(BEAObject, o), x -> contains(deletableRefs, x.type()));
      };

      cc.actuallyDeleteConcept = o -> {
        deleteConcepts(filter(findBackRefs(BEAObject, o),
          o2 -> contains(deletableRefs, o2.type())));
        cdelete(o);
      };
    }

    ret cc;
  }

  <A extends Concept> HCRUD makeCRUD(Class<A> c, Req req, HTMLFramer1 framer) {
    HCRUD crud = super.makeCRUD(c, req, framer);
    HCRUD_Concepts data = cast crud.data;
    crud.showOnlySelected = true;
    crud.showSearchField = true;
    if (data.customFilter == null)
      crud.descending = true; // show latest objects first by default except when searching
    crud.cleanItemIDs = true;

    if (c == BEAObject) {
      crud.cellColumnToolTips = true;
      crud.unshownFields = litset("mirrorPost", "globalID");
      crud.showTextFieldsAsAutoExpandingTextAreas = true;
      HCRUD_Concepts cc = cast crud.data;
      
      S typeFilter = req.get("type");
      if (nempty(typeFilter))
        cc.addCIFilter(type := typeFilter);
      
      crud.renderCmds = map -> {
        BEA o = getConcept BEA(crud.itemIDAsLong(map));
        
        new LS cmds;
        
        // ask the object itself for commands
        
        addAll(cmds, allToString(optCast Cl(pcall(o, "cmds"))));
        
        // special commands for BEA types or objects with certain fields
        
        if (fileNotEmpty(javaSourceFileForObject(o)))
          cmds.add(targetBlank(baseLink + "/javaSource?id=" + o.id, "Show Java source"));
        
        for (S field : ll("text", "what", "type")) {
          S text = getStringOpt(o, field);
          if (nempty(text))
            cmds.add(
              targetBlank(addParamsToURL(baseLink + "/query",
                q := text, algorithm := "process input"),
                "Use field " + quote(field) + " as query"));
        }

        if (o.typeIs("Input")) {
          cmds.add(ahref(addParamsToURL(crudLink(BEAObject),
            cmd := "new",
            title := "Add Pattern For Input",
            f_type := "Pattern",
            f_text := getStringOpt text(o),
            f_shouldMatch := o.id, metaInfo_shouldMatch := "concept",
          ), "Add pattern"));

          cmds.add(ahref(appendParamsToURL(baseLink + "/query", q := o.text(), algorithm := "Apply all text functions"),
           "Apply all text functions"));
        }

        if (o.typeIs("Pattern"))
          cmds.add(ahref(appendParamsToURL(baseLink + "/query", q := o.id, algorithm := "Run pattern"),
           "Try-run pattern against all inputs"));
        
        if (o.typeIsOneOf("Pattern", "Syntactic Pattern")) {
          cmds.add(ahref(appendParamsToURL(baseLink + "/reactAllInputsWithPattern", patternID := o.id),
           "React pattern with all inputs"));
           
          cmds.add(ahref(appendParamsToURL(baseLink + "/convertSyntacticToSemanticMatchesForWholePattern", patternID := o.id),
            "Convert all syntactic matches to semantic matches"));
        }

        if (o.typeIs("Match")) {
          cmds.add(ahref(appendParamsToURL(baseLink + "/query", q := o.id, algorithm := "Find rewrites for match"),
           "Find rewrites"));
          if (beaTypeIs(beaGet pattern(o), "Syntactic Pattern"))
            cmds.add(ahref(appendParamsToURL(baseLink + "/convertSyntacticToSemanticMatches", matchID := o.id),
            "Convert to semantic matches"));
        }

        cmds.add(
          ahref(addParamsToURL(baseLink + "/markUseful",
            redirect := beaShortURL(o),
            objectID := o.id),
            "Mark useful"));
        
        cmds.add(
          ahref(addParamsToURL(baseLink + "/markBad",
            redirect := beaShortURL(o),
            objectID := o.id),
            "Mark bad"));
        
        cmds.add(addCommentHTML(o));
          
        if (o.typeIsOneOf("Script", "Step in script") || eqic(beforeVerticalBar(o.type()), "Instruction"))
          cmds.add(
            ahref(addParamsToURL(baseLink + "/runInstruction",
              instruction := o.id),
              "Run"));
              
        if (o.typeIs("Function Result")) {
          if (eqic(cget resultType(o), "string"))
            cmds.add(
              ahref(addParamsToURL(baseLink + "/convertResultToInput",
                result := o.id),
                "Convert to input"));
        }
        
        if (o.typeIs("Auto Run"))
          cmds.add(
            ahref(addParamsToURL(baseLink + "/performAutoRunOnAllObjects", autoRun := o.id), "Run on all objects"));

        if (o.getClass() != BEA)
          cmds.add(
            ahref(addParamsToURL(baseLink + "/migrateToBase", id := o.id), "Migrate to BEAObject"));
            
        cmds.add(
          ahref(addParamsToURL(baseLink + "/performAutoRuns",
            onObject := o.id),
            "Perform auto runs on this object"));
            
        framer.addInHead(hjs_copyToClipboard());
        
        cmds.add(ahref_onClick(formatFunctionCall copyToClipboard(jsQuote(o.globalIDStr())), "Copy global ID [" + o.globalIDStr() + "]"));
        
        LS items = llNempties(
          crud.renderCmds_base(map),
          !o.typeIsOneOf("Input", "Pattern", "Syntactic Pattern")
            ? null : addRewriteHTML(o),
          !canAutoMigrate(o) ? null 
            : ahref(addParamsToURL(baseLink + "/autoMigrate", id := o.id), "Auto-migrate to " + shortClassName(defaultCustomClass(o))));

        if (isObjectWithCode(o) && isMasterAuthed(req))
          items.add(ahref(baseLink + "/activateDynamicObject?id=" + o.id, "Activate custom code"));

        // add more commands here
        
        pcall { addAll(items, o.directCmds()); }
        
        pcall {
          addAll(items, (LS) callOpt(getWebsiteEnhancer("commandsForObject"), 'commandsFor, o));
        }
        
        items.add(hPopDownButton(cmds));
        ret joinNemptiesWithVBar(items);
      };
      
      cc.massageItemMapForList = (item, map) -> {
        BEAObject o = cast item;
        
        if (o.typeIs("Input")) {
          Cl<BEAObject> matches = objectsWhereNotIC(objectsWhereCI(findBackRefs(o, BEAObject), type := "match"),
            label := "bad");
          map/Map.put("Best Matches", HTML(hparagraphs(
            lmap matchDescHTML(takeFirst(3, matches)))));
        }
        S idField = crud.idField();
        O id = map/Map.get(idField);
        if (id instanceof S)
          map/Map.put(idField, HTML(ahref(beaShortURL(o), id)));
        if (o.typeIs("Function Result")) {
          O value = o~.result;
          map/Map.put("result", HTML(javaValueToHTML(value)));
        }
        
        if (o.typeIs("Match")) {
          SS mapping = o.mapping();
          if (mapping != null)
          map/Map.put("mapping", HTML(
            joinWithBR(map(mapping, (k, v) ->
              htmlEncode2(k) + "=" + calculations.bestInputHTML(v)))));
        }
      
        if (o.getClass() != BEA)
          map/Map.put("Java Class", className(o));
          
        if (eq(req.get("showSimpleObjectScore"), "1"))
          map/Map.put("Simple object score" := calculations.simpleObjectScore(o));
      }; // end of massageItemMapForList
      
      crud.massageFormMatrix = (map, matrix) -> {
        for (int i = 1; i <= newFieldsToShow; i++) {
          S nf = "newField" + i;
          
          S refSelector =
            crud.renderInput("\*nf*/_conceptValue",
              cc.makeConceptsComboBox("\*nf*/_conceptValue", BEAObject), null)
            + hjs([[$("[name=]] + nf + [[_conceptValue]").hide()]]);
              
          S nativeSelector =
            crud.renderInput("\*nf*/_nativeValue",
              cc.makeConceptsComboBox("\*nf*/_nativeValue", Concept), null)
            + hjs([[$("[name=]] + nf + [[_nativeValue]").hide()]]);
              
          LS types = ll("String", "BEAObject", "Bool", "Native");
              
          S typeSelector = hselect_list(types, name := "\*nf*/_type",
              onchange := [[
                var value = this.value;
                $("[name=]] + nf + [[_value]").toggle(value == "String" || value == "Bool");
                $("#]] + nf + [[_refBox").toggle(value == "BEAObject");
                $("#]] + nf + [[_nativeSel").toggle(value == "Native");
              ]]);
          
          matrix.add(ll("Add field:<br>" + htextfield("\*nf*/_name", title := "New field name"),
            htmlTable2_noHtmlEncode(ll(ll(
              // string input
              //htextfield("\*nf*/_value"),
              htextarea("",
                name := "\*nf*/_value", 
                class := "auto-expand",
                style := "width: 300px",
              ) +
              span(refSelector, id := "\*nf*/_refBox", style := "display: none"),
              span(nativeSelector, id := "\*nf*/_nativeSel", style := "display: none"),
            "Type", typeSelector
            )),
            noHeader := true,
            tableParams := litobjectarray(style := "width: 100%"))));
          }
        };
      
      crud.preprocessUpdateParams = params -> {
        params = cloneMap(params);

        // drop empty strings
        //removeFromMapWhereValue(params, v -> eq(v, ""));
        params = mapValues(params, v -> eq(v, "") ? null : v);
        
        for (int i = 1; i <= max(newFieldsToShow, 10); i++) {
          S nf = "newField" + i;
          S name = params.get("\*nf*/_name"),
            type = params.get("\*nf*/_type"),
            refValue = params.get("\*nf*/_conceptValue"),
            nativeValue = params.get("\*nf*/_nativeValue"),
            value = params.get("\*nf*/_value");
  
          if (eqic(type, "BEAObject")) {
            value = refValue;
            params.put("metaInfo_" + name, "concept");
          } else if (eqic(type, "Native")) {
            value = nativeValue;
            params.put("metaInfo_" + name, "concept");
          } else if (eqic(type, "Bool"))
            params.put("metaInfo_" + name, "bool");
          
          if (eq(value, "")) value = null;
            
          if (nempty(name) /*&& neqOneOf(value, null, "")*/)
            params.put(crud.fieldPrefix + name, value);
        }
        
        ret params;
      };

      // regular users can only edit their own objects
      if (!isMasterAuthed(req))
        cc.objectCanBeEdited = id -> userCanEditObject(user(req), (BEA) cc.conceptForID(id));
    } // end of CRUD customization for BEAObject
    
    ret crud;
  }

  bool userCanEditObject(User user, BEA o) {
    ret cget createdBy(o) == user;
  }
  
  O serveBotFunction(Req req, S function, Map data, User user) {
    if (eq(function, "beaList")) {
      long changedAfter = toLong(data.get("changedAfter"));
      double pollFor = min(bot_maxPollSeconds, toLong(data.get("pollFor"))); // how long to poll (seconds)
      long startTime = sysNow();

      // We're super-anal about catching all changes. This will probably never trigger
      if (changedAfter > 0 && changedAfter == now()) sleep(1);
      
      Cl<BEAObject> objects;
      while true {
        objects = changedAfter == 0 ? list(BEAObject)
          : conceptsWithFieldGreaterThan_sorted(BEAObject, _modified := changedAfter);

        // return when there are results, no polling or poll expired
        if (nempty(objects) || pollFor == 0 || elapsedSeconds_sysNow(startTime) >= pollFor)
          ret serveJSON_breakAtLevels(2, result := map(objects, obj ->
            litorderedmap(gid := str(obj.globalID()), struct := obj.structureString())
          ));

        // sleep and try again
        sleep(bot_pollInterval);
      }
    }
    
    ret super.serveBotFunction(req, function, data, user);
  }

  O serveOtherPage2(Req req) null {
    printVars_str serveOtherPage2(uri := req.uri);
    try object super.serveOtherPage2(req);
    
    S uri = dropTrailingSlashIfNemptyAfterwards(req.uri);
    new Matches m;
    
    if (swic_notSame(uri, "/beaCRUD/", m))
      ret renderBEAObjectTable(req, urldecode(m.rest()));
    
    if (eq(uri, "/inputs"))
      ret renderBEAObjectTable(req, "input");
      
    if (eq(uri, "/syntacticMatchesTable"))
      ret hrefresh(baseLink + "/matchesTable?syntacticOnly=1");
      
    if (eq(uri, "/matchesTable")) {
      bool syntacticOnly = eq("1", req.get("syntacticOnly"));

      new L<BEA> list;
      for (BEA match : beaList("Match")) {
        BEA input = cgetOpt BEA(match, "input");
        BEA pat = cgetOpt BEA(match, "pattern");
        if (syntacticOnly && !calculations.isSyntacticPattern(pat)) continue;
        continue if calculations.patternAlwaysMatches(pat);

        SS mapping = match.mapping();
        if (input == null || pat == null || mapping == null) continue;
        list.add(match);
      }
      list = sortedByCalculatedFieldDesc(list,
        match -> conceptID(cgetBEA input(match)));
      
      new HTMLPaginator paginator;
      paginator.processParams(req.params());
      paginator.baseLink = addParamsToURL(baseLink,
        filterKeys(req.params(), p -> eq(p, "syntacticOnly")));
      paginator.max = l(list);
      req.framer.add(divUnlessEmpty(paginator.renderNav()));
      list = subList(list, paginator.visibleRange());
      
      new L<Map> data;
      for (BEA match : list) {
        BEA input = cgetOpt BEA(match, "input");
        BEA pat = cgetOpt BEA(match, "pattern");
        SS mapping = match.mapping();
        S patText = calculations.patternTextWithAngleBracketVars(pat);
        LS tok = javaTokWithAngleBrackets(patText);
        new BitSet bs;
        tok = replaceAngleBracketVars(tok, mapping, bs);
        for (IntRange r, bool b : unpair bitSetStreaksAndNonStreaks(bs, l(tok))) {
          S t = joinSubList(tok, r);
          replaceTokens(tok, r, b
            ? formatSubInput(input, t, "obj" + match.id)
            : htmlEncode2(t));
        }
        S boldenedInput = join(tok);
        
        data.add(litorderedmap(
          "Input" := aname("obj" + match.id, joinWithBR(
            ahref(beaURL(input), htmlEncode2_str(input~.text())),
            boldenedInput)),
          "Pattern" := ahref(beaURL(pat), htmlEncode2(pat.text())), // beaHTML(pat),
          //"Sub-Inputs" := htmlEncode2_str(mapping),
          "DB object" := beaHTML_justID(match),
          "Feedback" := joinNemptiesWithSpace(calculations.feedbackHTML(match),
            addCommentHTML(match, defaultComment := "good match", redirectAfterSave := req.uriWithParams(),
            text := "+")),
        ));
      }
        
      req.framer.title = syntacticOnly ? "Syntactic matches table" : "Matches table";
      req.framer.add(p());
      req.framer.add(htmlTable2_noHtmlEncode(data));
      ret completeFrame(req);
    }
            
    if (eq(uri, "/rewritesTable")) {
      new L<Map> data;
      for (BEA input : beaList("Input"))
        for (BEA r : beaBackRefs(input, "Rewrite"))
          data.add(litorderedmap(
            "Input" := input.text(),
            "Rewrite" := tok_dropCurlyBrackets(r.text()),
            "Rewrite Type" := r~.rewriteType));
            
      req.framer.title = "Rewrites table";
      req.framer.add(p());
      req.framer.add(htmlTable2(data));
      ret completeFrame(req);
    }

    if (eq(uri, "/patternRewritesTable")) {
      new L<Map> data;
      for (BEA pat : beaListAny("Pattern", "Syntactic Pattern"))
        for (BEA r : beaBackRefs(pat, "Rewrite"))
          if (cget isRewriteOf(r) == pat) {
            PairS example = calculations.exampleForPatternRewrite(pat, r);
            data.add(litorderedmap(
              "Pattern" := ahref(beaURL(pat), htmlEncode2(pat.text())),
              "Rewrite" := ahref(beaURL(r), htmlEncode2(tok_dropCurlyBrackets(r.text()))),
              "Example" := joinWithBR(htmlEncode2(example.a), htmlEncode2("=> " + example.b)),
              "Rewrite Type" := htmlEncode2_str(r~.rewriteType),
              "Cmds" := hPopDownButton(
                targetBlank(conceptEditLink(r), "Edit"),
              )));
          }
            
      req.framer.title = "Pattern rewrites table";
      req.framer.add(p());
      req.framer.add(htmlTable2_noHtmlEncode(data));
      ret completeFrame(req);
    }

    if (eq(uri, "/syntacticPatternsWithoutRewrites")) {
      HCRUD crud = makeBEAObjectCRUD(req, "Syntactic Pattern");
      HCRUD_Concepts<BEAObject> data = cast crud.data;
      
      IF1 prev = data.customFilter;
      data.customFilter = list -> {
        list = filter(list, c -> empty(beaBackRefs(c, "Rewrite")));
        ret postProcess(prev, list);
      };
      crud.customTitle = "Syntactic patterns without rewrites";
      ret serveCRUD(req, BEAObject, crud);
    }
      
    if (eq(uri, "/inputsWithoutRewrites")) {
      HCRUD crud = makeBEAObjectCRUD(req, "input");
      HCRUD_Concepts<BEAObject> data = cast crud.data;
      
      IF1 prev = data.customFilter;
      data.customFilter = list -> {
        list = filter(list, c -> empty(beaBackRefs(c, "Rewrite")));
        ret postProcess(prev, list);
      };
      crud.customTitle = "Inputs without rewrites";
      ret serveCRUD(req, BEAObject, crud);
    }
      
    if (eq(uri, "/inputsWithRewrites")) {
      HCRUD crud = makeBEAObjectCRUD(req, "input");
      HCRUD_Concepts<BEAObject> data = cast crud.data;
      
      IF1 prev = data.customFilter;
      data.customFilter = list -> {
        list = filter(list, c -> nempty(beaBackRefs(c, "Rewrite")));
        ret postProcess(prev, list);
      };
      crud.customTitle = "Inputs with rewrites";
      ret serveCRUD(req, BEAObject, crud);
    }
      
    if (eq(uri, "/inputsWithoutMatches")) {
      HCRUD crud = makeBEAObjectCRUD(req, "input");
      HCRUD_Concepts<BEAObject> data = cast crud.data;
      
      IF1 prev = data.customFilter;
      data.customFilter = list -> {
        list = filter(list, c -> empty(beaBackRefs(c, "Match")));
        ret postProcess(prev, list);
      };
      crud.customTitle = "Inputs without matches";
      ret serveCRUD(req, BEAObject, crud);
    }
      
    if (eq(uri, "/patterns"))
      ret renderBEAObjectTable(req, "pattern");

    if (eq(uri, "/syntacticPatterns"))
      ret renderBEAObjectTable(req, "Syntactic Pattern");

    if (eq(uri, "/matches"))
      ret renderBEAObjectTable(req, "match");

    if (eq(uri, "/rewrites"))
      ret renderBEAObjectTable(req, "rewrite");

    if (eq(uri, "/aiTasks"))
      ret renderBEAObjectTable(req, "AI Task");

    if (eq(uri, "/query"))
      ret calculations.serveQueryPage(req);
      
    if (eq(uri, "/saveInput")) {
      S text = trim(req.get("text"));
      S info = trim(req.get("info"));
      if (empty(text)) ret subBot_serve500("Input empty");
      if (l(text) > maxInputLength) ret subBot_serve500("Input too long");
      BEAObject input = uniqCI BEAObject(type := "Input", +text, createdBy := user(req));
      saveUserAgent(input);
      if (nempty(info))
        uniqCI BEAObject(type := "Input Source", +input, source := info);
      ret hrefresh(or2(req.get("redirect"), baseLink + "/"));
    }
      
    if (eq(uri, "/markUseful")) {
      BEA o = beaGet objectID(req);
      if (o == null) ret subBot_serve500("Object not found");
      uniqCI BEA(type := "Useful", object := o, createdBy := user(req));
      ret hrefresh(or2(req.get("redirect"), baseLink + "/"));
    }
      
    if (eq(uri, "/markBad")) {
      BEA o = beaGet objectID(req);
      if (o == null) ret subBot_serve500("Object not found");
      uniqCI BEA(type := "Bad", object := o, createdBy := user(req));
      ret hrefresh(or2(req.get("redirect"), baseLink + "/"));
    }
      
    if (eq(uri, "/saveAnswer")) {
      S text = trim(req.get("text"));
      S rewriteType = or2(trim(req.get("rewriteType")), "Suggested Answer");
      long inputID = toLong(req.get("inputID"));
      BEAObject input = beaGet(inputID);
      if (input == null) ret subBot_serve500("Input not found");
      if (empty(text)) ret subBot_serve500("Text empty");
      if (l(text) > maxInputLength) ret subBot_serve500("Text too long");
      uniqCI BEAObject(type := "Rewrite", +text, isRewriteOf := input, +rewriteType, createdBy := user(req));
      ret hrefresh(or2(req.get("redirect"), baseLink + "/"));
    }

    if (eq(uri, "/javaSource")) {
      BEA o = beaGet id(req);
      ret serveText(loadTextFile(javaSourceFileForObject(o)));
    }
    
    if (startsWith(uri, "/beaHTML/", m)) {
      req.noSpam();
      BEA o = beaGet(m.rest());
      IF0 result = call_optional(o, "html", req);
      if (result != null) ret result!;
      result = call_optional(o, "html", req.webRequest);
      if (result != null) ret result!;
      ret call(o, "html");
    }
    
    // add more public URLs here
      
    if (!inMasterMode(req)) null;
    
    if (eq(uri, "/uploadInputs"))
      ret serveUploadTexts(req, "Input");
    
    if (eq(uri, "/uploadPatterns"))
      ret serveUploadTexts(req, "Pattern");
      
    if (eq(uri, "/analyzeInput"))
      ret calculations.serveAnalyzeInput(req);
      
    if (eq(uri, "/allBEATypes")) {
      ret h2_title("All object types")
        + ul(renderObjectTypes());
    }
    
    if (eq(uri, "/storeMatch")) {
      BEAObject pattern = beaGet(req.get("pattern"));
      BEAObject input = beaGet(req.get("input"));
      S label = req.get("label");
      BEAObject match = calculations.reactInputWithPattern(input, pattern);
      if (match == null) ret "Couldn't match";
      cset(match, +label);
      ret hrefresh(or2(req.get("redirect"), beaObjectURL(match)));
    }
    
    if (eq(uri, "/saveSyntacticPattern") || eq(uri, "/savePattern")) {
      S type = cic(uri, "syntactic") ? "Syntactic Pattern" : "Pattern";
      S text = req.get("text");
      BEAObject fromInput = beaGet(req.get("fromInput"));
      BEAObject pat = uniqCI BEAObject(+type, +text, createdBy := user(req));
      csetIfUnset(pat, +fromInput);
      if (fromInput != null)
        calculations.reactInputWithPattern(fromInput, pat);
      ret hrefresh(or2(req.get("redirect"), baseLink + "/"));
    }
    
    if (eq(uri, "/runInstruction")) {
      BEAObject o = beaGet(req.get("instruction"));
      runInstruction(o);
      ret hrefresh(beaObjectURL(o));
    }
    
    if (eq(uri, "/convertResultToInput")) {
      BEAObject result = beaGet(req.get("result"));
      if (result == null) ret subBot_serve500("Object not found");
      S text = unquote(getString result(result));
      if (l(text) > maxInputLength) ret subBot_serve500("Input too long");
      BEAObject input = uniqCI BEAObject(type := "Input", +text);
      uniqCI BEAObject(type := "Input Source", +input, source := result);
      ret hrefresh(beaShortURL(input));
    }
    
    /*if (eq(uri, "/pipeStringListIntoPattern")) {
      BEA r = beaGet(req.get("resultID"));
      BEa pat = beaGet(req.get("patternID"));
      if (r == null || pat == null) ret "Object not found";
      LS result = assertStringList(cget result(r));
      for (S s : result)
        uniq BEA(type := "Statement", );
    }*/
    
    if (eq(uri, "/setWordType")) {
      BEA r = beaGet(req.get("resultID"));
      S wordType = assertNempty(req.get("wordType"));
      LS result = assertStringList(cget result(r));
      for (S s : result)
        uniqCI BEA(type := "Word type", word := s, +wordType);
      ret "OK";
    }
    
    if (eq(uri, "/reload")) {
      dm_reloadModuleIn(3);
      ret "Reloading module in 3";
    }
    
    if (eq(uri, "/performAutoRuns")) {
      BEA o = beaGet(req.get("onObject"));
      if (o == null) ret "Object not found";
      rstAutoRuns.add(o);
      ret hrefresh(beaURL(o));
    }
    
    if (eq(uri, "/reactAllInputsWithAllSyntacticPatterns")) {
      ret str(returnTimed(r { calculations.reactAllInputsWithAllSyntacticPatterns(); }));
    }
    
    if (eq(uri, "/reactAllInputsWithPattern")) {
      BEA pat = beaGet(req.get("patternID"));
      if (pat == null) ret "Pattern not found";
      ret str(returnTimed(r { calculations.reactAllInputsWithPattern(pat); }));
    }
    
    if (eq(uri, "/storeSubInput")) {
      BEA input = beaGet(req.get("input"));
      if (input == null) ret "Input not found";
      S text = req.get("text");
      S type = eqic(req.get("label"), "good") ? "Sub-Input" : "Bad Sub-Input";
      BEA o = uniqCI BEA(
        +type,
        +input,
        +text,
        createdBy := currentUser());
      ret hrefresh(or2(req.get("redirect"), beaURL(o)));
    }
    
    if (eq(uri, "/performAutoRunOnAllObjects")) {
      BEA autoRun = beaGet autoRun(req);
      performAutoRunOnAllObjects(autoRun);
      ret "OK";
    }
    
    if (eq(uri, "/autoMigrate")) {
      BEA o = beaGet id(req);
      if (o == null) ret "Object not found";
      S url = beaURL(o);
      ret hsansserif() + "Migrated " + beaHTML(o) + " to " + className(autoMigrateToCustomClass(o))
        + hrefresh(1.0, url);
    }
      
    if (eq(uri, "/migrateToBase")) {
      BEA o = beaGet id(req);
      if (o == null) ret "Object not found";
      S url = beaURL(o);
      ret hsansserif() + "Migrated " + beaHTML(o) + " to " + className(autoMigrateToBase(o))
        + hrefresh(1.0, url);
    }
      
    if (eq(uri, "/convertSyntacticToSemanticMatches")) {
      BEA match = beaGet matchID(req);
      if (match == null) ret "Match not found";
      ret ul_htmlEncode2(calculations.convertSyntacticToSemanticMatches(match));
    }
    
    if (eq(uri, "/convertSyntacticToSemanticMatchesForWholePattern")) {
      BEA pat = beaGet patternID(req);
      if (pat == null) ret "Pattern not found";
      ret ul_htmlEncode2(calculations.convertSyntacticToSemanticMatchesForWholePattern(pat));
    }
    
    if (eq(uri, "/createPatternListFromUsefulSyntacticPatterns"))
      ret hrefresh(beaURL(calc.createPatternListFromUsefulSyntacticPatterns()));
      
    if (eq(uri, "/createConceptShadows"))
      ret createConceptShadows();

    if (eq(uri, "/compareConceptShadows")) {
      if (conceptShadows == null) ret createConceptShadows();
      L<ConceptShadow> newShadows = allConceptShadows();
      L<CreatedDeletedChanged<ConceptShadow>> diff
        = diffConceptShadows(conceptShadows, newShadows);
      ret subBot_serveText(nDiffs(diff) + ":\n\n"
        + pnlToString(diff));
    }

    if (eq(uri, "/includeScript")) {
      S snippetID = req.get("id");
      O wired = hotwire(snippetID);
      ret "Wired: " + wired;
    }
    
    if (eq(uri, "/activateDynamicObject")) {
      BEA o = beaGet id(req);
      if (o == null) ret "Object not found";
      activateDynamicObject(o);
      o = beaGet id(req);
      S url = beaURL(o);
      ret hsansserif() + "Migrated " + beaHTML(o) + " to " + className(o)
        + hrefresh(1.0, url);
    }
    
    if (eq(uri, "/callAnyMethod")) {
      BEA o = beaGet id(req);
      if (o == null) ret "Object not found";
      S name = req.get("name");
      bool showPrints = eq("1", req.get("showPrints"));

      new L args;
      for (S key, val : req.params()) {
        int idx = parseIntOpt(regexpExtractGroup("^conceptArg(\\d+)$", key));
        if (idx > 0)
          listSet(args, idx-1, getConcept(parseLong(val)));
      }
      
      S arg = req.get("arg");
      if (arg != null) listSet(args, 0, arg);
      
      S result = hijackPrintPlusResult_text(() -> call(o, name, toObjectArray(args)));
      ret serveText("Result of " + o + " . " + name + "(" + joinWithComma(args) + "):\n" + result);
    }
    
    if (eq(uri, "/download"))
      ret subBot_serveFileWithName("gazelle-database." + ymd_minus_hms() + ".gz", conceptsFile());
    
    // add more master-mode URLs here
  } // end of serveOtherPage2
  
  transient L<ConceptShadow> conceptShadows;

  O createConceptShadows() {  
    time "Make shadows" {
      S profile = profileThisThreadToString(r {
        conceptShadows = allConceptShadows();
      });
    }
    ret subBot_serveText(n2(conceptShadows, "shadow") + " made in " + lastTiming_formatted() + "\n\n" + profile);
  }
  
  HCRUD makeBEAObjectCRUD(Req req, S type) {
    HCRUD crud = makeCRUD(BEAObject, req);
    crud.baseLink = req.uri();
    HCRUD_Concepts data = cast crud.data;
    data.itemName = () -> firstToUpper(type);
    data.addCIFilter(type := eqic(type, "(no type)") ? null : type);
    
    if (eqicOneOf(type, "Input", "Pattern", "AI Task")) {
      IF0<MapSO> prev = data.emptyObject;
      data.emptyObject = () -> {
        MapSO item = data.emptyObject_fallback(prev);
        item.put(text := ""); // show text field when creating new objects
        ret item;
      };
    }
    ret crud;
  }
  
  O renderBEAObjectTable(Req req, S type) {
    HCRUD crud = makeBEAObjectCRUD(req, type);
    ret serveCRUD(req, BEAObject, crud);
  }
  
  O serveUploadTexts(Req req, S type) {
    S inputs = req.get("text");
    
    new LS output;
    
    if (nempty(inputs)) {
      for (S text : tlft(inputs)) {
        Pair<BEAObject, Bool> p = uniqCI2_sync BEAObject(+type, +text);
        if (cget uploadedBy(p.a) == null)
          cset(p.a, uploadedBy := req.auth.user);
        output.add(type + " " + (p.b ? "added" : "exists")
          + " (ID " + p.a.id + "): " + text);
      }
    }
    
    ret h2("Upload " + plural(type))
      + hpostform(
          p(plural(type) + " (one per line):")
        + p(htextarea(inputs, name := "text"))
        + pIfNempty(htmlEncode_nlToBR(lines(output)))
        + hsubmit("Upload " + plural(type)));
  }
  
  Cl<BEAObject> beaObjectsOfType(S type) {
    ret conceptsWhereCI BEAObject(+type);
  }
  
  void reactAllInputsWithSomePatterns {
    calculations.reactAllInputsWithSomePatterns();
  }
  
  S navDiv() {
    HCRUD crud = makeCRUD(BEAObject, currentReq!);

    long time = sysNow();
    int inputsWithoutRewrites = this.inputsWithoutRewrites!;
    int inputsWithoutMatches = this.inputsWithoutMatches!;
    int syntacticPatternsWithoutRewrites = this.syntacticPatternsWithoutRewrites!;
    /*int inputsWithoutRewrites = 0;
    int inputsWithoutMatches = 0;
    int syntacticPatternsWithoutRewrites = 0;*/
    done2_always("Count Things", time);
    
    ret joinNemptiesWithVBar(
      ahref(baseLink + "/query", "Query"),
      beaNavLink("Input", crud),
      beaNavLink("Pattern", crud),
      beaNavLink("Syntactic Pattern", crud),
      ahref(baseLink + "/syntacticPatternsWithoutRewrites", n2(syntacticPatternsWithoutRewrites) + " Syntactic patterns w/o rewrites"),
      beaNavLink("Match", crud),
      beaNavLink("Rewrite", crud),
      beaNavLink("AI Task", crud),
      ahref(baseLink + "/inputsWithoutMatches", n2(inputsWithoutMatches) + " Inputs w/o matches"),
      ahref(baseLink + "/inputsWithoutRewrites", n2(inputsWithoutRewrites) + " Inputs w/o rewrites"),
      ahref(baseLink + "/allBEATypes", "All object types")
        + " " + hPopDownButton(renderObjectTypes(10)),
      hPopDownButtonWithText("Bot Forum", navLinks(flat := true, withStats := false)),
      HTMLPopDownButton(
        ahref(baseLink + "/stats", "Stats"),
        !isMasterAuthed() ? null : ahref(baseLink + "/download", "DB download"),
        !isMasterAuthed() ? null : ahref(baseLink + "/refchecker", "Reference checker"),
        ahref(baseLink + "/inputsWithRewrites", "Inputs with rewrites"),
        ahref(baseLink + "/rewritesTable", "Rewrites table"),
        ahref(baseLink + "/patternRewritesTable", "Pattern rewrites table"),
        ahref(baseLink + "/matchesTable", "Matches table"),
        ahref(baseLink + "/matchesTable?syntacticOnly=1", "Syntactic matches table"),
        ahref("https://gazelle.rocks/htmlBot/238410", "Classic Gazelle"),
      ).width(350).height(500).html());
  }
  
  // crud is just the cached BEAObject crud to check for creation rights
  S beaNavLink(S type, HCRUD crud, int count default beaCount(type)) {
    S plural = firstToLower(plural(type));
    S link = baseLink + "/" + camelCase(plural);
    
    ret ahref(link, n2(count) + " " + firstToUpper(plural)) + (!crud.actuallyAllowCreate() ? "" : " " + ahref(addParamToURL(link, cmd := "new"), "+"));
  }
  
  S beaObjectURL(BEAObject o) {
    ret conceptLink(o, currentReq!);
    /*ret o == null ?:
      addParamsToURL(baseLink + "/crud/BEAObject",
        selectObj := o.id) + "#" + o.id;*/
  }
  
  S matchDescHTML(BEAObject m) {
    pcall {
      BEAObject pat = cget pattern(m);
      SS mapping = cast cget mapping(m);
      ret ahref_undecorated(crudLink(m), htmlEncode2(quote(getString text(pat)))
        + "<br>&nbsp; with " + renderEqualsCommaProperties(mapping));
    }
    ret htmlEncode2(str(m));
  }
  
  int beaCount(S type) {
    ret countConceptsWhereCI BEAObject(+type);
  }
  
  Cl<BEA> beaList() {
    ret list BEA();
  }
  
  Cl<BEAObject> beaList(S type, O... params) {
    ret conceptsWhereCI BEAObject(paramsPlus_inFront(params, +type));
  }
  
  Cl<BEAObject> beaListAny(S... types) {
    ret concatLists(lmap beaList(litciset(types)));
  }
  
  BEA beaGet(long id) {
    ret getConceptOpt BEAObject(id);
  }
  
  BEA beaGet(S id) {
    ret beaGet(parseFirstLong(id));
  }
  
  // first param can be type
  BEA beaNew(O... _) {
    _ = prependParamIfOddCount("type", _);
    ret cnew BEA(_);
  }
  
  bool beaTypeIs(BEA o, S type) {
    ret o != null && o.typeIs(type);
  }
  
  BEA mapMethodLike beaGet(S key, Req req) {
    ret beaGet(req.get(key));
  }

  BEA mapMethodLike cgetBEA aka beaGet(S field, BEA o) {
    ret (BEA) cget(field, o);
  }
  
  BEAObject cgetBEA aka beaGet(BEAObject o, S field) {
    ret cgetBEA(field, o);
  }

  S beaLinkHTML aka beaToHTML aka beaHTML(BEAObject o) {  
    ret o == null ?: ahref(conceptLink(o), htmlEncode2_nlToBr(str(o)));
  }
  
  S beaHTML_justID(BEA o) { ret o == null ?: ahref(beaShortURL(o), o.id); }
  
  S beaShortURL aka beaURL(BEAObject o) {
    ret o == null ?: baseLink + "/" + o.id;
  }

  Cl<BEAObject> beaBackRefs(BEAObject o, S type) {
    ret objectsWhereCI(findBackRefs BEAObject(o), +type);
  }
  
  Cl<BEAObject> beaBackRefs(BEAObject o) {
    ret findBackRefs BEAObject(o);
  }
  
  O serveDefaultPage(Req req) {
    HTMLFramer1 framer = req.framer;
    framer.add(hcenter3(hsnippetimg_scaleToWidth(200, #1102967, 200, 110, title := "Gazelle"), style := "margin-top: 100px"));
    ret completeFrame(req);
  }
  
  S html_loggedIn() {
    User user = user(currentReq!);
    ret user == null
      ? /*"Not logged in"*/ ahref(baseLink + "/", "Log in")
      : "Logged in as " + htmlEncode2(user.name);
  }
  
  void distributeNewObject_impl(BEAObject o) {
    if (o.typeIs("Match")) ret;
    distributeNotification("New object: " + o);
  }
  
  void performAutoRuns(BEAObject o) enter {
    //print("performAutoRuns", o);
    
    for (BEAObject autoRun : beaList("Auto Run")) {
      if (!isTrue(getOpt enabled(autoRun)))
        continue with print("Not enabled: " + autoRun);
      performAutoRunOnObject(autoRun, o);
    }
  }
  
  void performAutoRunOnAllObjects(BEA autoRun) {
    for (BEA o : list(BEA))
      performAutoRunOnObject(autoRun, o);
  }
  
  void performAutoRunOnObject(BEA autoRun, BEA o) {
    ping();
    S type = getString onChangedObjectOfType(autoRun);
    if (!o.typeIs(type)) ret; // with print("Wrong type: " + type);
    
    print("Running " + autoRun);
    BEAObject procedure = cast cget procedure(autoRun);
    
    S internalCode = getString internalCode(procedure);
    printVars_str(+internalCode, +o);
    
    if (eqic(internalCode, "convertInputToPattern")) {
      S text = o.text();
      if (!containsAngleBracketVars(text)) ret with print("No angle bracket vars");
      BEAObject p = uniqCI_returnIfNew BEAObject(type := "Pattern", +text);
      cset(p, fromInput := o, byProcedure := procedure, byAutoRun := autoRun);
      print(+p);
    } else if (eqic(internalCode, "convertInputToSyntacticPattern")) {
      S text = o.text();
      if (!containsStars(text)) ret with print("No stars");
      BEAObject p = uniqCI_returnIfNew BEAObject(type := "Syntactic Pattern", +text);
      cset(p, fromInput := o, byProcedure := procedure, byAutoRun := autoRun);
      print(+p);
    } else if (eqic(internalCode, "dropPunctuationFromPattern")) {
      S text = o.text();
      S text2 = dropPunctuation(text);
      if (eq(text, text2)) ret;
      
      BEAObject p = uniqCI BEA(
        type := o.type(),
        text := text2);
      cset(o, withoutPunctuation := p);
    } else if (eqic(internalCode, "makeSyntacticPattern")) {
      print(sp := calculations.makeSyntacticPattern(o));
    } else if (eqic(internalCode, "runFunctionOnInput")) {
      print("runFunctionOnInput");
      BEA function = beaGet(o, "function");
      BEA input = beaGet(o, "input");
      if (function == null || input == null) ret with print("Missing parameters");
      if (cget result(o) != null) ret with print("Has result");

      BEA result = calculations.reactFunctionWithInput(calculations.new BackEndAlgorithm, function, input);
      print(+result);
      if (result != null) {
        cset(result, request := o);
        cset(o, +result);
      }
    } else
      print("Unknown internal code");
  }

  BEAObject findInput(S text) {
    ret conceptWhereIC(BEAObject, type := "Input", +text);
  }
  
  S addRewriteHTML(BEAObject o) {
    ret ahref(addParamsToURL(crudLink(BEAObject),
      cmd := "new",
      title := "Add Rewrite",
      f_type := "Rewrite",
      f_text := getStringOpt text(o),
      f_isRewriteOf := o.id, metaInfo_isRewriteOf := "concept",
    ), "Add Rewrite");
  }
  
  O serveIntegerLink(Req req, long id) {
    BEAObject o = getConceptOpt BEAObject(id);
    if (o != null)
      ret htitle(str(o)) + hrefresh(conceptLink(o));
    ret super.serveIntegerLink(req, id);
  }
  
  void distributeTestNotification() {
    distributeNotification("It is " + localTimeWithSeconds());
  }
  
  void distributeNotification(S text) {
    notificationQ.add(r {
      /*for (User user)
        if (nemptyAfterTrim(user.notificationSetting))
          sendNotification(text);*/
          
      for (Pair<virtual WebSocket, WebSocketInfo> p : syncMapToPairs(webSockets)) {
        // TODO: check user
        S jsCode =
          "window.createNotification({ theme: 'success', showDuration: 3000 })("
          + jsonEncodeMap(message := text) + ");";
        call(p.a, "send", jsonEncodeMap(eval := jsCode));
      }
    });
  }
  
  void runInstruction(BEAObject o) {
    if (o == null) ret;
    try {
      BEAObject instruction = o;
      if (o.typeIs("Step in script"))
        instruction = (BEAObject) cget(o, "instruction");
      
      if (instruction.typeIs("Instruction | List objects by type")) {
        saveInstructionResult(o, beaList(getString typeToList(instruction)));
        ret;
      }
      
      if (instruction.typeIs("Instruction | List object types"))
        ret with saveInstructionResult(o, distinctCIFieldValuesOfConcepts(BEAObject, "type"));

      if (instruction.typeIs("Instruction | Filter list by text starting with")) {
        // find list made before
        BEAObject scriptRun = cgetBEA scriptRun(o);
        if (scriptRun == null) fail("Need to be run as part of script");
        L<BEAObject> steps = scriptRunSteps(scriptRun);
        int idx = indexOf(steps, o);
        if (idx < 0) fail("Step not found in script run");
        
        L<BEAObject> list = firstNotNull(map(reversed(takeFirst(steps, idx)),
          step -> optCast L(cget data(cgetBEA result(step)))));
         
        S prefix = getString prefix(instruction);
        L<BEAObject> filtered = filter(list, obj -> swic(obj.text(), prefix));
        saveInstructionResult(o, filtered);
        ret;
      }
      
      if (instruction.typeIs("Script")) {
        BEAObject script = instruction;
        BEAObject scriptRun = cnew BEAObject(type := "Script Run", +script);
        
        // Make an instance of all the instructions
        
        int i = 0;
        new L<BEAObject> steps;
        while not null (instruction = (BEAObject) cget(instruction, "step" + (++i))) {
          BEAObject step = cnew(BEAObject, type := "Step in script", step := i, +scriptRun, +instruction);
          steps.add(step);
        }
        
        cset(scriptRun, +steps);
        
        // TODO: run steps?

        ret;
      }
      
      cnew BEAObject(type := "Instruction Error", instruction := o, error := "Unknown instruction type");
    } catch e {
      cnew BEAObject(type := "Instruction Error", instruction := o, error := getStackTrace(e));
    }
  }

  BEAObject saveInstructionResult(BEAObject instruction, O data) {
    BEAObject result = cnew BEAObject(type := "Instruction Result",
      +instruction, +data);
    cset(instruction, +result);
    ret result;
  }
  
  L<BEAObject> scriptRunSteps(BEAObject scriptRun) {
    ret (L) cget steps(scriptRun);
  }
  
  S addCommentHTML(BEA o, O... _) {
    optPar S redirectAfterSave;
    optPar S defaultComment;
    optPar S text = "Add comment";
    
    ret ahref(addParamsToURL(crudLink(BEAObject),
      cmd := "new",
      title := "Add Comment",
      f_type := "Comment",
      f_on := o.id,
      f_text := unnull(defaultComment),
      +redirectAfterSave,
      autofocus := "f_text",
      metaInfo_on := "concept"), text);
  }
  
  S formatSubInput(BEA input, S text, S anchor) {
    S redirect = addAnchorToURL(currentReq->uriWithParams(), anchor);
    int goodCount = countConceptsCI BEA(type := "Sub-Input", +input, +text);
    int badCount = countConceptsCI BEA(type := "Bad Sub-Input", +input, +text);
    ret calculations.bestInputHTML(text)
      + " " + small(joinNemptiesWithSpace(
        ahref(addParamsToURL(
          baseLink + "/storeSubInput", 
          label := "good",
          +text,
          input := input.id,
          +redirect),
        unicode_thumbsUp()),
        goodCount == 0 ? "" : n2(goodCount),
        ahref(addParamsToURL(
          baseLink + "/storeSubInput", 
          label := "bad",
          +text,
          input := input.id,
          +redirect),
        unicode_thumbsDown()),
        badCount == 0 ? "" : n2(badCount)
      ));
  }

  LS renderObjectTypes(int max default Int.MAX_VALUE) {
    MultiSet<S> ms = distinctCIFieldValuesOfConcepts_multiSet(BEAObject, "type");
    ret mapPairs(takeFirst(max, multiSetToPairsByPopularity(ms)),
      (type, count) -> {
        S _type = or2(type, "(no type)");
        ret n2(count) + " " + ahref(baseLink + "/beaCRUD/" + urlencode(_type), htmlEncode2(_type));
      });
  }
  
  BEA autoMigrateToBase(BEA o) {
    ret autoMigrateToClass(o, BEA);
  }
  
  BEA autoMigrateToCustomClass(BEA o) {
    ret autoMigrateToClass(o, defaultCustomClass(o));
  }
  
  // accepting Concept here to allow for fixes after broken unstructuring
  // (a problem we will probably never have again)
  BEA autoMigrateToClass(Concept o, Class<? extends BEAObject> targetClass) {
    pcall {
      if (o == null || o._concepts != db_mainConcepts()) null;
      if (targetClass != null && targetClass != _getClass(o))
        ret replaceConceptAndUpdateRefs(o, unlistedCopyToClass_withConverter(targetClass, o));
    }
    ret (BEA) o;
  }

  void setCodeState(BEA o, long timestamp, S codeHash, O state) {
    if (empty(state)) state = null; else
      state = "[" + formatGMTWithMilliseconds_24(timestamp) + "] "
        + (empty(codeHash) ? "" : "Code hash: " + codeHash + ". ")
        + state;
    cset(o, meta_codeState := state);
  }

  S codeForObject(BEA o) {
    ret getString meta_code(o);
  }
  
  void activateDynamicObject(BEA o) ctex {
    ClassLoader cl = dm_moduleClassLoader();
    print(+cl);

    long timestamp = now();
    setCodeState(o, timestamp, null, "Compiling");

    S codeHash = null;
    transpileAndCompileForHotwiring_src.set(null);

    // TODO: there is still a slight possibility that erroneous code will be compiled again and again

    try {
      // Check meta_code and meta_prototype (latter overwrites the former)
      BEA proto = cget meta_prototype(o);
      S code = codeForObject(o);
      
      Class protoClass = null;
      if (proto != null) {
        if (!isActivatedDynamicObject(proto))
          fail("Prototype not activated: " + proto);
        protoClass = proto.getClass();
      }
      
      codeHash = empty(code) ? null : md5(code);
      cset(o, meta_codeHash := codeHash);
  
      if (empty(code)) {
        if (protoClass != null) {
          // no extra code, just instantiate prototype class
          o = autoMigrateToClass(o, protoClass);
          setCodeState(o, timestamp, null, "Migrated to proto class " + shortClassName(protoClass));
        } else {
          // Nothing to activate - migrate back to base
          o = autoMigrateToBase(o);
          setCodeState(o, timestamp, null, null);
        }
        ret;
      }
      
      // safety check for reference (not currently used, we just check for master user status)
      cset(o, meta_codeSafety := null/*codeSafetyCheckResult(code)*/);

      S id = makeJavaClassName(o);
      S mcName = makePackageName(o);
      cset(o, meta_javaClass := mcName + "." + id);
      
      S src = 
        "mainClassName " + mcName + "\n"
        + "concept " + id + " extends " +
          (protoClass == null ? "BEAObject" : protoClass.getName().replace("$", "."))
        + " {\n" + code + "\n}\n"
        + customCodePostlude();
      
      print("SRC> ", src);
      var files = filesFromClassLoader(cl);
      
      print("Compiling with path:");
      pnlIndent(files);
      
      javaCompileToJar_localLibraries.set(files);
      dm_mediumRefreshTranspiler();
      File bytecode = transpileAndCompileForHotwiring(src);
      saveTextFile(javaSourceFileForObject(o), transpileAndCompileForHotwiring_src!);
      copyFile(bytecode, byteCodeFileForObject(o, id));
      
      cset(o, meta_javaClass := mcName + "." + id);
      var p = reactivateDynamicObject_impl(o);
      o = p.a;
      setCodeState(o, timestamp, codeHash, "Custom code loaded");
      
      // migrate derivatives to new class
      // (only those who don't add any code are included right now)
      for (BEA derivative : directDerivatives(o)) pcall {
        print("Migrating derivative " + derivative + " of " + o + " to new base class " + o.getClass());
        autoMigrateToClass(derivative, o.getClass());
      }

    } catch print e {
      cset(o, meta_javaClass := null);
      setCodeState(o, timestamp, codeHash, exceptionToStringShort(e));
    }
  }
  
  bool basedOnUnactivatedPrototype(BEA o) {
    BEA proto = optCast BEA(cget meta_prototype(o));
    ret proto != null && !isActivatedDynamicObject(proto);
  }
  
  Cl<BEA> directDerivatives(BEA o) {
    ret transitiveHull(x -> filter(
      conceptsWhere BEA(meta_prototype := x),
      d -> empty(codeForObject(d))), o);
  }
  
  bool reactivateDynamicObject(BEA o) {
    try {
      ret reactivateDynamicObject_impl(o).b;
    } catch print e {
      cset(o, meta_javaClass := null);
      cset(o, meta_codeState := exceptionToStringShort(e));
      false;
    }
  }
  
  bool isActivatedDynamicObject(BEA o) {
    ret startsWith(className(o), packagePrefix());
  }
  
  Pair<BEA, Bool> reactivateDynamicObject_impl(BEA o) ctex {
    S id = getString meta_javaClass(o);
    if (empty(id)) {
      cset(o, meta_codeState := null);
      ret pair(o, false);
    }
    File bytecode = byteCodeFileForObject(o, shortenClassName(id));
    dm_addByteCodePathToModuleClassLoader(bytecode);
    S mcName = makePackageName(o);
    int idx = lastIndexOf(id, '.');
    S fullName = takeFirst(id, idx)/*.replace(".", "/")*/
      + substring(id, idx).replace(".", "$");

    //Class targetClass = _getClass(fullName);
    ClassLoader cl = dm_moduleClassLoader();
    Class targetClass = cl.loadClass(fullName);
    print(+targetClass);
    assertNotNull("Class not found: " + fullName, targetClass);
    o = autoMigrateToClass(o, targetClass);

    cset(o, meta_codeState := "[" + formatGMTWithMilliseconds_24() + "] " + "Custom code reloaded");
    ret pair(o, true);
  }

  File javaSourceFileForObject(BEA o) {
    ret programFile("BEA Java Sources/" + o.id + ".java");
  }
  
  File byteCodeFile(DynClassName name) {
    ret name == null ?: programFile("Object ByteCode/" + name.objectID + "-" + name.globalID + ".jar");
  }
  
  File byteCodeFileForObject(BEA o, S javaClassName
    default getString meta_javaClass(o)) {
    if (empty(javaClassName)) null;
    ret programFile("Object ByteCode/" + o.id + "-" + javaClassName + ".jar");
  }
  
  BEA autoMigrateUnlistedOrKeep(BEA o) {
    ret canAutoMigrate(o) ? unlistedCopyToClass_withConverter(defaultCustomClass(o), o) : o;
  }
  
  bool canAutoMigrate(BEA o) {
    ret o != null && !eqOneOf(defaultCustomClass(o), _getClass(o), null);
  }
  
  Class<? extends BEA> defaultCustomClass(BEA o) {
    if (o == null) null;
    BEA entry = conceptWhereCI BEA(type := "Auto Custom Class", forType := o.type());
    //printVars_str defaultCustomClass(+o, +entry);
    if (entry == null) null;
    S className = getString toClassName(entry);
    //printVars_str defaultCustomClass(+className);
    Class c = findClassThroughDefaultClassFinder(className);
    //printVars_str defaultCustomClass(+c);
    ret c;
  }
  
  S queryLink(S algorithm, S q) {
    ret addParamsToURL(baseLink + "/query", +algorithm, +q);
  }

  S currentUserAgent() {  
    Req req = currentReq!;
    if (req == null) null;
    ret mapGet(req.webRequest.headers(), "user-agent");
  }

  void saveUserAgent(BEA input) {  
    S userAgent = currentUserAgent();
    if (nempty(userAgent))
      uniqCI2 BEA(type := "Input Source",  +input, +userAgent);
  }

  bool isObjectWithCode(BEA o) {
    ret o != null && (cget meta_code(o) != null || cget meta_prototype(o) != null);
  }

  bool createdByMasterUser(BEA o) {
    User user = optCast User(cget createdBy(o));
    ret user != null && user.isMaster;
  }

  void autoActivateDynamicObject(BEA o) {
    if (!autoActivateDynamicObjects) ret;
    S code = codeForObject(o);
    if (empty(code)) ret;
    if (!createdByMasterUser(o)) ret;
    S codeState = getString meta_codeState(o);
    S time = leadingAngleBracketStuff(codeState);
    long timestamp = empty(time) ? 0 : parseDateWithMillisecondsGMT(time);
    if (o._modified <= timestamp) ret;
    //S hash = regexpExtractIC("code hash: (\\w+)?", codeState);
    S hash = getString meta_codeHash(o);
    if (eq(hash, md5(code)))
      ret; // Code unchanged
    print("Auto-activating dynamic object: " + o);
    activateDynamicObject(o);
    sleepSeconds(1); // safety sleep
  }

  S customCodePostlude() {  
    ret "!include early #1031282\n\n";
  }
  
  S makeJavaClassName(BEA o) {
    ret /*"b" + o.id + "_" +*/ aGlobalID();
  }
  
  S packagePrefix() {
    ret "dyn.b_";
  }
  
  S makePackageName(BEA o) {
    ret packagePrefix() + o.id;
  }
  
  BEA websiteEnhancersObject() {
    Cl<BEA> l = beaList("Live Website Enhancers");
    if (l(l) > 1) warn("Multiple Live Website Enhancers: " + l);
    ret first(l);
  }
  
  BEA getWebsiteEnhancer(S key) {
    ret beaGet(websiteEnhancersObject(), key);
  }
  
  S newLinkForCRUD(HCRUD crud, Class c) {
    if (c == User) ret baseLink + "/register";
    else ret super.newLinkForCRUD(crud, c);
  }
  
  S callHTMLFixer(O caller, S html) {
    BEA fixer = beaGet htmlFixer(websiteEnhancersObject());
    ret or((S) callOpt(fixer, "fixHTML", html, caller), html);
  }
  
  O html3(Req req) {
    S uri = req.uri();
    for (BEA rewrite : filter createdByMasterUser(beaList("Live URL Rewrite"))) pcall {
      S input = getString input(rewrite);
      vmBus_send testingURLRewrite(rewrite, uri);
      if (eq(uri, input)) {
        vmBus_send applyingURLRewrite(rewrite, uri);
        S output = getString output(rewrite);
        /*
        SS newParams = paramsFromURL(output);
        req.uri = urlWithoutQuery(output);
        ModifiedWebRequest request2 = webRequest_modifyURI(req.webRequest, req.uri);
        request2.params = req.params = joinMaps(newParams, req.params());
        vmBus_send urlRewrite_newURI(req.uri);
        vmBus_send urlRewrite_newParams(req.params);
        break;
        */
        ret hrefresh(output);
      }
    }
    ret super.html3(req);
  }
  
  void cleanMeUp_deactivateBEAObjects {
    Cl<BEA> objects = filter(beaList(), o -> hasMethod(o, "_deactivate"));
    print("Deactivating " + nObjects(objects) + " for shutdown");
    for (BEA bea : objects)
      pcallOpt(bea, "_deactivate");
    if (nempty(objects)) print("Done");
  }

} // end of module

extend User {
  S notificationSetting;
}

concept BEAObject > ConceptWithGlobalID {
  // optional new Ref<UserPost> mirrorPost;
  UserPost mirrorPost() { ret (UserPost) cget mirrorPost(this); }

  void change :: after {
    var mod = beaMod();
    if (mod != null) {
      mod.rstUpdateBEAMirrors.add(this);
      if (mod.autoActivateDynamicObjects)
        mod.rstAutoActivateDynamicObjects.add(this);
    }
  }

  void delete :: before {
    cdelete(mirrorPost());
    pcallOpt(this, "_deactivate");
  }
  
  void updateMirrorPost {
    GazelleBEA mod = beaMod();
    if (isDeleted() || !mod.mirrorBEAObjects) ret;

    if (mirrorPost() == null)
      cset(this, mirrorPost := cnew UserPost(
        type := "BEA Object",
        creator := mod.internalUser(),
        botInfo := "BEA Mirror Bot"));

    S text = structureString();

    cset(mirrorPost(), 
      title := str(this),
      +text);
  }

  S structureString() {
    S text = "Error";
    pcall {
      structure_Data data = new {
        structure_ClassInfo newClass(Class c) {
          structure_ClassInfo info = super.newClass(c);
          if (c == Concept.Ref.class) {
            info.special = true;
            info.serializeObject = o -> {
              Concept cc = cast deref((Concept.Ref) o);
              //append("cu CRef " + (cc != null ? str(cc.id) : "null"), 3);
              if (cc cast BEAObject)
                append("CRef(gid=" + quote(cc.globalID()) + ")", 6);
              else if (cc != null)
                append("CRef(id=" + cc.id + ")", 6);
              else
                append("CRef", 1);
            };
          }
          ret info;
        }

        void setFields(structure_ClassInfo info, L<Field> fields) {
          if (isSubclassOf(info.c, BEAObject)) {
            // Don't serialize "refs" and "backRefs" fields
            removeAll(fields,
              getField(BEAObject, "refs"),
              getField(BEAObject, "backRefs"));
          }
          super.setFields(info, fields);
        }
      };
      
      S struct = structure(this, data);
      struct = dropLoadableUtilsPackageFromStruct(struct);
      text = indentStructureString_firstLevels(1, struct);
    }

    ret text;
  }
      
  toString {
    ret shorten(toString_long());
  }
  
  S toString_long() {
    S type = strOrNull(cget type(this));
    S s = super.toString();
    
    if (nempty(type)) {
      s = type + " " + id;
      
      s += appendBracketed(strOrNull(this~.label));

      if (eqic(type, "Match"))
        s += " " + this.~mapping
          /*+ appendBracketed(this.~input + " + " + this~.pattern)*/;
    }
    
    bool enabled = eq(true, getOpt enabled(this));
    if (enabled) s += " [enabled]";
    
    S purpose = getStringOpt purpose(this);
    if (nempty(purpose))
      s += " " + quote(purpose);
    
    S text = or2(text(),
      getStringOpt name(this),
      getStringOpt internalCode(this));
    if (text != null)
      s += " " + quote(text);
      
    O fc = this~.functionCalled;
    if (fc != null) s += " " + fc;
      
    O result = this~.result;
    if (result != null) s += " = " + shorten_str(result);
      
    ret s;
  }

  !include #1031300 // BEAObject methods

} // end of BEAObject

beaConcept BEARegExp {
  S text;
  bool caseInsensitive = true;
  
  bool valid() { ret nempty(text); }
  
  java.util.regex.Pattern compile() {
    ret compileRegexpPossiblyIC_unicodeCase(text, caseInsensitive);
  }
}

beaConcept BEARegExpReplacement > BEARegExp {
  S replacement;

  S apply(S text) {
    try {
      ret regexpReplace_directWithRefs(compile().matcher(text), unnull(replacement));
    } catch e {
      fail(format_quoted("Was searching * in * ", this.text, text));
    }
  }
  
  LS directCmds() {
    ret listPlus(super.directCmds(),
      !valid() ? "Note: not valid"
      : targetBlank(beaMod().queryLink("Apply regular expression replacement to all inputs", str(id)), "Apply to all inputs"));
  }
}

beaConcept BEAPatternList {
  new RefL<BEA> patterns;
  
  toString {
    ret super.toString() + ": " + nPatterns(patterns);
  }
}

static GazelleBEA beaMod() {
  ret (GazelleBEA) botMod();
}

set flag hotwire_here.

// share ISpec interface with sub-modules
static JavaXClassLoader hotwire_makeClassLoader(L<File> files) {
  ClassLoader cl = myClassLoader();
   ret new JavaXClassLoaderWithParent2(null, files, cl, ll(/*TODO*/));
}

sclass DynClassName {
  long objectID;
  S globalID;
  
  *(long *objectID, S *globalID) {}
  
  static DynClassName parse(S className) {
    LS groups = regexpFirstGroups("^dyn\\.b_(\\d+)_([a-z]+)\\.B(\\d+)", className);
    if (groups == null) null;
    ret DynClassName(parseLong(first(groups)), second(groups));
  }

  S makePackageName() { ret "dyn.b_" + objectID + "_" + globalID; }
  S makeClassName() { ret "B" + objectID; }
  S fullClassName() { ret makePackageName() + "." + makeClassName(); }
}
Travelled to 4 computer(s): bhatertpkbcr, mqqgnosmbjvj, pyentgdyhuwx, vouqrxazstgt
Snippet ID:	#1031465
Snippet name:	GazelleBEA [backup before new dynamic class names]
Eternal ID of this version:	#1031465/1
Text MD5:	83a743ba83f1448a9eef4e91fd6144b0
Author:	stefan
Category:	javax
Type:	JavaX fragment (include)
Public (visible to everyone):	Yes
Archived (hidden from active list):	No
Created/modified:	2021-06-14 03:59:25
Source code size:	68153 bytes / 1932 lines
Pitched / IR pitched:	No / No
Views / Downloads:	144 / 788
Referenced in:	[show references]
< > BotCompany Repo | #1031465 // GazelleBEA [backup before new dynamic class names]

JavaX fragment (include)

Author comment
