!7

!include early #1030953 // specialized Compact Module Include

import loadableUtils.utils.User;

import loadableUtils.utils.UserPost;

import loadableUtils.utils.Conversation;

import loadableUtils.utils.ExecutedStep;

import loadableUtils.utils.InputHandled;

import static loadableUtils.utils.botMod;

!include once #1030833 // BEACalculations

set flag NoNanoHTTPD.

rewrite BEA with BEAObject.

module GazelleBEA > DynGazelleRocks {

  switchable bool mirrorBEAObjects; // don't do it for now

  switchable bool enableAutoRuns = true;

  switchable bool enableNewBEAObjectNotis = true;

  switchable bool printAllConceptChanges;

  transient ReliableSingleThread_Multi<BEAObject> rstAutoRuns = new(1000, lambda1 performAutoRuns);

  transient Q notificationQ;

  transient ReliableSingleThread_Multi<BEAObject> rstDistributeNewObject = new(1000, lambda1 distributeNewObject_impl);

  transient Set<BEAObject> newObjectsDistributed = weakSet();

  transient BEACalculations calculations = new(this);

  transient BEACalculations calc = calculations;

  transient int newFieldsToShow = 3;

  transient bool allowAnonymousInputUpload = true; // TODO

  switchable int maxInputLength = 50000;

  transient ConceptClassesDependentValue<Int> inputsWithoutRewrites;

  transient ConceptClassesDependentValue<Int> inputsWithoutMatches;

  transient ConceptClassesDependentValue<Int> syntacticPatternsWithoutRewrites;

  // add more fields here

  !include #1030883 // DB quickImport mix-in

  void init {

    super.init();

    set quickDBReloadEnabled;

    botName = heading = adminName = "Gazelle BEA";

    set enableVars;

    unset showTalkToBotLink;

    unset phoneNumberSpecialInputField;

    unset showMetaBotOnEveryPage;

  }

  void startDB {

    bootstrapDBFrom(#1030602);

    db();

    db_mainConcepts().useBackRefsForSearches = true;

    inputsWithoutRewrites = ConceptClassesDependentValue(litset(BEA), () -> countPred(beaList("Input"), c -> empty(beaBackRefs(c, "Rewrite"))));

    inputsWithoutMatches = ConceptClassesDependentValue(litset(BEA), () -> countPred(beaList("Input"), c -> empty(beaBackRefs(c, "Match"))));

    syntacticPatternsWithoutRewrites = ConceptClassesDependentValue(litset(BEA), () -> countPred(beaList("Syntactic Pattern"), c -> empty(beaBackRefs(c, "Rewrite"))));

    if (printAllConceptChanges) printAllConceptChanges();

  }

  void start {

    assertEquals(BEAObject, callF(defaultDefaultClassFinder(), "main$BEAObject"));

    assertEquals(ConceptWithGlobalID, callF(defaultDefaultClassFinder(), "main$ConceptWithGlobalID"));

    seedDBFrom(#1030602);

    set botDisabled;

    set storeBaseClassesInStructure;

    super.start();

    print("main concepts: " + db_mainConcepts() + ", count: " + db_mainConcepts().countConcepts());

    print(renderConceptClassesWithCount(db_mainConcepts()));

    print("Inputs: " + n2(beaCount("Input")));

    set showCRUDToEveryone;

    //set showOnlySelectedObject;

    // reload module when lib changes

    dm_onSnippetTranspiled(voidfunc(S snippetID) {

      if (sameSnippetID(snippetID, #1030952))

        dm_reloadModule();

    });

    if (!enabled) ret;

    // mirror all objects to be sure

    rstUpdateBEAMirrors.addAll(list(BEAObject));

    newObjectsDistributed.addAll(list(BEAObject));

    /*onIndividualConceptChange_notOnAllChanged(BEAObject,

      p -> { calculations.makeSyntacticPattern(p); });*/

    onIndividualConceptChange_notOnAllChanged(BEAObject,

      o -> {

        if (enableAutoRuns) rstAutoRuns.add(o);

        if (enableNewBEAObjectNotis && newObjectsDistributed.add(o))

          rstDistributeNewObject.add(o);

      });

    notificationQ = dm_startQ();

    // fix refs occasionally

    dm_doEvery(5*60.0, r-enter {

      print(ConceptsRefChecker(db_mainConcepts()).runAndFixAll());

    });

  }

  void makeIndices :: after {    

    indexConceptFieldDesc(BEAObject, "_modified");

    indexConceptFieldIC(BEAObject, "type");

    // TODO: don't index nulls, also maybe just use backRefs

    indexConceptField(BEAObject, "input"); // e.g. for counting how many matches we have for a given input

  }

  L<Class> crudClasses(Req req) {

    ret listPlus(super.crudClasses(req), BEAObject, /*BEAPatternList*/);

  }

  Set<Class> hiddenCrudClasses() {

    Set<Class> set = super.hiddenCrudClasses();

    addAll(set, Conversation, ExecutedStep, InputHandled);

    ret set;

  }

  S authFormHeading() {

    ret h3("Gazelle BEA");

  }

  void makeFramer(Req req) {

    super.makeFramer(req);

    req.framer.addInHead(hjs_autoExpandingTextAreas());

  }

  <A extends Concept> HCRUD_Concepts<A> crudData(Class<A> c, Req req) {

    HCRUD_Concepts<A> cc = super.crudData(c, req);

    if (c == BEAObject) {

      cc.humanizeFieldNames = false;

      cc.convertConceptValuesToRefs = true;

      cc.itemName = () -> "BEA Object";

      cc.onCreate.add(o ->

        cset(o, createdBy := currentUser()));

      cc.getObjectForDuplication = id -> {

        MapSO item = cc.getObjectForDuplication_base(id);

        item.put(creator := req.auth.user!); // set default creator to current user

        item.put(createdFrom := getConcept(toLong(id)));

        ret item;

      };

      cc.emptyObject = () -> {

        MapSO item = cc.emptyObject_base();

        item.put(type := "");

        ret item;

      };

      Set<S> deletableRefs = litciset("Match");

      cc.objectCanBeDeleted = id ->

        all(findBackRefs(BEAObject, cc.conceptForID(id)),

          o -> contains(deletableRefs, o.type()));

      cc.actuallyDeleteConcept = o -> {

        deleteConcepts(filter(findBackRefs(BEAObject, o),

          o2 -> contains(deletableRefs, o2.type())));

        cdelete(o);

      };

    }

    ret cc;

  }

  <A extends Concept> HCRUD makeCRUD(Class<A> c, Req req, HTMLFramer1 framer) {

    HCRUD crud = super.makeCRUD(c, req, framer);

    HCRUD_Concepts data = cast crud.data;

    crud.showOnlySelected = true;

    crud.showSearchField = true;

    if (data.customFilter == null)

      crud.descending = true; // show latest objects first by default except when searching

    crud.cleanItemIDs = true;

    if (c == BEAObject) {

      crud.cellColumnToolTips = true;

      crud.unshownFields = litset("mirrorPost", "globalID");

      crud.showTextFieldsAsAutoExpandingTextAreas = true;

      HCRUD_Concepts cc = cast crud.data;

      S typeFilter = req.get("type");

      if (nempty(typeFilter))

        cc.addCIFilter(type := typeFilter);

      crud.renderCmds = map -> {

        BEAObject o = getConcept BEAObject(crud.itemIDAsLong(map));

        new LS cmds;

        // special commands for BEA types or objects with certain fields

        if (nempty(o.text()))

          cmds.add(

            targetBlank(addParamsToURL(baseLink + "/query",

              q := o.text(), algorithm := "process input"),

              "Use text as query"));

        if (o.typeIs("Input")) {

          cmds.add(ahref(addParamsToURL(crudLink(BEAObject),

            cmd := "new",

            title := "Add Pattern For Input",

            f_type := "Pattern",

            f_text := getStringOpt text(o),

            f_shouldMatch := o.id, metaInfo_shouldMatch := "concept",

          ), "Add pattern"));

          cmds.add(ahref(appendParamsToURL(baseLink + "/query", q := o.text(), algorithm := "Apply all text functions"),

           "Apply all text functions"));

        }

        if (o.typeIs("Pattern"))

          cmds.add(ahref(appendParamsToURL(baseLink + "/query", q := o.id, algorithm := "Run pattern"),

           "Try-run pattern against all inputs"));

        if (o.typeIsOneOf("Pattern", "Syntactic Pattern")) {

          cmds.add(ahref(appendParamsToURL(baseLink + "/reactAllInputsWithPattern", patternID := o.id),

           "React pattern with all inputs"));

          cmds.add(ahref(appendParamsToURL(baseLink + "/convertSyntacticToSemanticMatchesForWholePattern", patternID := o.id),

            "Convert all syntactic matches to semantic matches");

        }

        if (o.typeIs("Match")) {

          cmds.add(ahref(appendParamsToURL(baseLink + "/query", q := o.id, algorithm := "Find rewrites for match"),

           "Find rewrites"));

          if (beaTypeIs(beaGet pattern(o), "Syntactic Pattern"))

            cmds.add(ahref(appendParamsToURL(baseLink + "/convertSyntacticToSemanticMatches", matchID := o.id),

            "Convert to semantic matches");

        }

        cmds.add(

          ahref(addParamsToURL(baseLink + "/markUseful",

            redirect := beaShortURL(o),

            objectID := o.id),

            "Mark useful"));

        cmds.add(

          ahref(addParamsToURL(baseLink + "/markBad",

            redirect := beaShortURL(o),

            objectID := o.id),

            "Mark bad"));

        cmds.add(addCommentHTML(o));

        if (o.typeIsOneOf("Script", "Step in script") || eqic(beforeVerticalBar(o.type()), "Instruction"))

          cmds.add(

            ahref(addParamsToURL(baseLink + "/runInstruction",

              instruction := o.id),

              "Run"));

        if (o.typeIs("Function Result")) {

          if (eqic(cget resultType(o), "string"))

            cmds.add(

              ahref(addParamsToURL(baseLink + "/convertResultToInput",

                result := o.id),

                "Convert to input"));

        }

        if (o.typeIs("Auto Run"))

          cmds.add(

            ahref(addParamsToURL(baseLink + "/performAutoRunOnAllObjects", autoRun := o.id), "Run on all objects");

        if (o.getClass() != BEA)

          cmds.add(

            ahref(addParamsToURL(baseLink + "/migrateToBase", id := o.id), "Migrate to BEAObject");

        cmds.add(

          ahref(addParamsToURL(baseLink + "/performAutoRuns",

            onObject := o.id),

            "Perform auto runs on this object"));

        framer.addInHead(hjs_copyToClipboard());

        cmds.add(ahref_onClick(formatFunctionCall copyToClipboard(jsQuote(o.globalIDStr())), "Copy global ID [" + o.globalIDStr() + "]"));

        LS items = llNempties(

          crud.renderCmds_base(map),

          !o.typeIsOneOf("Input", "Pattern", "Syntactic Pattern")

            ? null : addRewriteHTML(o),

          !canAutoMigrate(o) ? null 

            : ahref(addParamsToURL(baseLink + "/autoMigrate", id := o.id), "Auto-migrate to " + shortClassName(defaultCustomClass(o))));

        addAll(items, o.directCmds());

        items.add(hPopDownButton(cmds));

        ret joinNemptiesWithVBar(items);

      };

      cc.massageItemMapForList = (item, map) -> {

        BEAObject o = cast item;

        if (o.typeIs("Input")) {

          Cl<BEAObject> matches = objectsWhereNotIC(objectsWhereCI(findBackRefs(o, BEAObject), type := "match"),

            label := "bad");

          map/Map.put("Best Matches", HTML(hparagraphs(

            lmap matchDescHTML(takeFirst(3, matches)))));

        }

        S idField = crud.idField();

        O id = map/Map.get(idField);

        if (id instanceof S)

          map/Map.put(idField, HTML(ahref(beaShortURL(o), id)));

        if (o.typeIs("Function Result")) {

          O value = o~.result;

          map/Map.put("result", HTML(javaValueToHTML(value)));

        }

        if (o.typeIs("Match")) {

          SS mapping = o.mapping();

          if (mapping != null)

          map/Map.put("mapping", HTML(

            joinWithBR(map(mapping, (k, v) ->

              htmlEncode2(k) + "=" + calculations.bestInputHTML(v)))));

        }

        if (o.getClass() != BEA)

          map/Map.put("Java Class", shortClassName(o));

        if (eq(req.get("showSimpleObjectScore"), "1"))

          map/Map.put("Simple object score" := calculations.simpleObjectScore(o));

      }; // end of massageItemMapForList

      crud.massageFormMatrix = (map, matrix) -> {

        for (int i = 1; i <= newFieldsToShow; i++) {

          S nf = "newField" + i;

          S refSelector =

            crud.renderInput("\*nf*/_conceptValue",

              cc.makeConceptsComboBox("\*nf*/_conceptValue", BEAObject), null)

            + hjs([[$("[name=]] + nf + [[_conceptValue]").hide()]]);

          LS types = ll("String", "BEAObject", "Bool");

          S typeSelector = hselect_list(types, name := "\*nf*/_type",

              onchange := [[

                var value = this.value;

                $("[name=]] + nf + [[_value]").toggle(value != "BEAObject");

                $("#]] + nf + [[_refBox").toggle(value == "BEAObject");

              ]]);

          matrix.add(ll("Add field:<br>" + htextfield("\*nf*/_name", title := "New field name"),

            htmlTable2_noHtmlEncode(ll(ll(

              // string input

              htextfield("\*nf*/_value"),

              span(refSelector, id := "\*nf*/_refBox", style := "display: none"),

            "Type", typeSelector

            )),

            noHeader := true,

            tableParams := litobjectarray(style := "width: 100%"))));

          }

        };

      crud.preprocessUpdateParams = params -> {

        params = cloneMap(params);

        // drop empty strings

        //removeFromMapWhereValue(params, v -> eq(v, ""));

        params = mapValues(params, v -> eq(v, "") ? null : v);

        for (int i = 1; i <= max(newFieldsToShow, 10); i++) {

          S nf = "newField" + i;

          S name = params.get("\*nf*/_name"),

            type = params.get("\*nf*/_type"),

            refValue = params.get("\*nf*/_conceptValue"),

            value = params.get("\*nf*/_value");

          if (eqic(type, "BEAObject")) {

            value = refValue;

            params.put("metaInfo_" + name, "concept");

          } else if (eqic(type, "Bool"))

            params.put("metaInfo_" + name, "bool");

          if (eq(value, "")) value = null;

          if (nempty(name) /*&& neqOneOf(value, null, "")*/)

            params.put(crud.fieldPrefix + name, value);

        }

        ret params;

      };

    }

    ret crud;

  }

  O serveBotFunction(Req req, S function, Map data, User user) {

    if (eq(function, "beaList")) {

      long changedAfter = toLong(data.get("changedAfter"));

      double pollFor = min(bot_maxPollSeconds, toLong(data.get("pollFor"))); // how long to poll (seconds)

      long startTime = sysNow();

      // We're super-anal about catching all changes. This will probably never trigger

      if (changedAfter > 0 && changedAfter == now()) sleep(1);

      Cl<BEAObject> objects;

      while true {

        objects = changedAfter == 0 ? list(BEAObject)

          : conceptsWithFieldGreaterThan_sorted(BEAObject, _modified := changedAfter);

        // return when there are results, no polling or poll expired

        if (nempty(objects) || pollFor == 0 || elapsedSeconds_sysNow(startTime) >= pollFor)

          ret serveJSON_breakAtLevels(2, result := map(objects, obj ->

            litorderedmap(gid := str(obj.globalID()), struct := obj.structureString())

          ));

        // sleep and try again

        sleep(bot_pollInterval);

      }

    }

    ret super.serveBotFunction(req, function, data, user);

  }

  transient ReliableSingleThread_Multi<BEAObject> rstUpdateBEAMirrors = new(100, c -> enter { c.updateMirrorPost(); });

  O serveOtherPage2(Req req) null {

    printVars_str serveOtherPage2(uri := req.uri);

    try object super.serveOtherPage2(req);

    S uri = dropTrailingSlashIfNemptyAfterwards(req.uri);

    new Matches m;

    if (swic_notSame(uri, "/beaCRUD/", m))

      ret renderBEAObjectTable(req, urldecode(m.rest()));

    if (eq(uri, "/inputs"))

      ret renderBEAObjectTable(req, "input");

    if (eq(uri, "/syntacticMatchesTable"))

      ret hrefresh(baseLink + "/matchesTable?syntacticOnly=1");

    if (eq(uri, "/matchesTable")) {

      bool syntacticOnly = eq("1", req.get("syntacticOnly"));

      new L<BEA> list;

      for (BEA match : beaList("Match")) {

        BEA input = cgetOpt BEA(match, "input");

        BEA pat = cgetOpt BEA(match, "pattern");

        if (syntacticOnly && !calculations.isSyntacticPattern(pat)) continue;

        continue if calculations.patternAlwaysMatches(pat);

        SS mapping = match.mapping();

        if (input == null || pat == null || mapping == null) continue;

        list.add(match);

      }

      list = sortedByCalculatedFieldDesc(list,

        match -> conceptID(cgetBEA input(match)));

      new HTMLPaginator paginator;

      paginator.processParams(req.params());

      paginator.baseLink = addParamsToURL(baseLink,

        filterKeys(req.params(), p -> eq(p, "syntacticOnly")));

      paginator.max = l(list);

      req.framer.add(divUnlessEmpty(paginator.renderNav()));

      list = subList(list, paginator.visibleRange());

      new L<Map> data;

      for (BEA match : list) {

        BEA input = cgetOpt BEA(match, "input");

        BEA pat = cgetOpt BEA(match, "pattern");

        SS mapping = match.mapping();

        S patText = calculations.patternTextWithAngleBracketVars(pat);

        LS tok = javaTokWithAngleBrackets(patText);

        new BitSet bs;

        tok = replaceAngleBracketVars(tok, mapping, bs);

        for (IntRange r, bool b : unpair bitSetStreaksAndNonStreaks(bs, l(tok))) {

          S t = joinSubList(tok, r);

          replaceTokens(tok, r, b

            ? formatSubInput(input, t, "obj" + match.id)

            : htmlEncode2(t));

        }

        S boldenedInput = join(tok);

        data.add(litorderedmap(

          "Input" := aname("obj" + match.id, joinWithBR(

            ahref(beaURL(input), htmlEncode2_str(input~.text())),

            boldenedInput)),

          "Pattern" := ahref(beaURL(pat), htmlEncode2(pat.text())), // beaHTML(pat),

          //"Sub-Inputs" := htmlEncode2_str(mapping),

          "DB object" := beaHTML_justID(match),

          "Feedback" := joinNemptiesWithSpace(calculations.feedbackHTML(match),

            addCommentHTML(match, defaultComment := "good match", redirectAfterSave := req.uriWithParams(),

            text := "+")),

        ));

      }

      req.framer.title = syntacticOnly ? "Syntactic matches table" : "Matches table";

      req.framer.add(p());

      req.framer.add(htmlTable2_noHtmlEncode(data));

      ret completeFrame(req);

    }

    if (eq(uri, "/rewritesTable")) {

      new L<Map> data;

      for (BEA input : beaList("Input"))

        for (BEA r : beaBackRefs(input, "Rewrite"))

          data.add(litorderedmap(

            "Input" := input.text(),

            "Rewrite" := tok_dropCurlyBrackets(r.text()),

            "Rewrite Type" := r~.rewriteType));

      req.framer.title = "Rewrites table";

      req.framer.add(p());

      req.framer.add(htmlTable2(data));

      ret completeFrame(req);

    }

    if (eq(uri, "/patternRewritesTable")) {

      new L<Map> data;

      for (BEA pat : beaListAny("Pattern", "Syntactic Pattern"))

        for (BEA r : beaBackRefs(pat, "Rewrite"))

          if (cget isRewriteOf(r) == pat) {

            PairS example = calculations.exampleForPatternRewrite(pat, r);

            data.add(litorderedmap(

              "Pattern" := ahref(beaURL(pat), htmlEncode2(pat.text())),

              "Rewrite" := ahref(beaURL(r), htmlEncode2(tok_dropCurlyBrackets(r.text()))),

              "Example" := joinWithBR(htmlEncode2(example.a), htmlEncode2("=> " + example.b)),

              "Rewrite Type" := htmlEncode2_str(r~.rewriteType),

              "Cmds" := hPopDownButton(

                targetBlank(conceptEditLink(r), "Edit"),

              )));

          }

      req.framer.title = "Pattern rewrites table";

      req.framer.add(p());

      req.framer.add(htmlTable2_noHtmlEncode(data));

      ret completeFrame(req);

    }

    if (eq(uri, "/syntacticPatternsWithoutRewrites")) {

      HCRUD crud = makeBEAObjectCRUD(req, "Syntactic Pattern");

      HCRUD_Concepts<BEAObject> data = cast crud.data;

      IF1 prev = data.customFilter;

      data.customFilter = list -> {

        list = filter(list, c -> empty(beaBackRefs(c, "Rewrite")));

        ret postProcess(prev, list);

      };

      crud.customTitle = "Syntactic patterns without rewrites";

      ret serveCRUD(req, BEAObject, crud);

    }

    if (eq(uri, "/inputsWithoutRewrites")) {

      HCRUD crud = makeBEAObjectCRUD(req, "input");

      HCRUD_Concepts<BEAObject> data = cast crud.data;

      IF1 prev = data.customFilter;

      data.customFilter = list -> {

        list = filter(list, c -> empty(beaBackRefs(c, "Rewrite")));

        ret postProcess(prev, list);

      };

      crud.customTitle = "Inputs without rewrites";

      ret serveCRUD(req, BEAObject, crud);

    }

    if (eq(uri, "/inputsWithRewrites")) {

      HCRUD crud = makeBEAObjectCRUD(req, "input");

      HCRUD_Concepts<BEAObject> data = cast crud.data;

      IF1 prev = data.customFilter;

      data.customFilter = list -> {

        list = filter(list, c -> nempty(beaBackRefs(c, "Rewrite")));

        ret postProcess(prev, list);

      };

      crud.customTitle = "Inputs with rewrites";

      ret serveCRUD(req, BEAObject, crud);

    }

    if (eq(uri, "/inputsWithoutMatches")) {

      HCRUD crud = makeBEAObjectCRUD(req, "input");

      HCRUD_Concepts<BEAObject> data = cast crud.data;

      IF1 prev = data.customFilter;

      data.customFilter = list -> {

        list = filter(list, c -> empty(beaBackRefs(c, "Match")));

        ret postProcess(prev, list);

      };

      crud.customTitle = "Inputs without matches";

      ret serveCRUD(req, BEAObject, crud);

    }

    if (eq(uri, "/patterns"))

      ret renderBEAObjectTable(req, "pattern");

    if (eq(uri, "/syntacticPatterns"))

      ret renderBEAObjectTable(req, "Syntactic Pattern");

    if (eq(uri, "/matches"))

      ret renderBEAObjectTable(req, "match");

    if (eq(uri, "/rewrites"))

      ret renderBEAObjectTable(req, "rewrite");

    if (eq(uri, "/aiTasks"))

      ret renderBEAObjectTable(req, "AI Task");

    if (eq(uri, "/query"))

      ret calculations.serveQueryPage(req);

    if (eq(uri, "/saveInput")) {

      S text = trim(req.get("text"));

      S info = trim(req.get("info"));

      if (empty(text)) ret subBot_serve500("Input empty");

      if (l(text) > maxInputLength) ret subBot_serve500("Input too long");

      BEAObject input = uniqCI BEAObject(type := "Input", +text, createdBy := user(req));

      if (nempty(info))

        uniqCI BEAObject(type := "Input Source", +input, source := info);

      ret hrefresh(or2(req.get("redirect"), baseLink + "/"));

    }

    if (eq(uri, "/markUseful")) {

      BEA o = beaGet objectID(req);

      if (o == null) ret subBot_serve500("Object not found");

      uniqCI BEA(type := "Useful", object := o, createdBy := user(req));

      ret hrefresh(or2(req.get("redirect"), baseLink + "/"));

    }

    if (eq(uri, "/markBad")) {

      BEA o = beaGet objectID(req);

      if (o == null) ret subBot_serve500("Object not found");

      uniqCI BEA(type := "Bad", object := o, createdBy := user(req));

      ret hrefresh(or2(req.get("redirect"), baseLink + "/"));

    }

    if (eq(uri, "/saveAnswer")) {

      S text = trim(req.get("text"));

      S rewriteType = or2(trim(req.get("rewriteType")), "Suggested Answer";

      long inputID = toLong(req.get("inputID"));

      BEAObject input = beaGet(inputID);

      if (input == null) ret subBot_serve500("Input not found");

      if (empty(text)) ret subBot_serve500("Text empty");

      if (l(text) > maxInputLength) ret subBot_serve500("Text too long");

      uniqCI BEAObject(type := "Rewrite", +text, isRewriteOf := input, +rewriteType, createdBy := user(req));

      ret hrefresh(or2(req.get("redirect"), baseLink + "/"));

    }

    // add more public URLs here

    if (!inMasterMode(req)) null;

    if (eq(uri, "/uploadInputs"))

      ret serveUploadTexts(req, "Input");

    if (eq(uri, "/uploadPatterns"))

      ret serveUploadTexts(req, "Pattern");

    if (eq(uri, "/analyzeInput"))

      ret calculations.serveAnalyzeInput(req);

    if (eq(uri, "/allBEATypes")) {

      ret h2_title("All object types")

        + ul(renderObjectTypes());

    }

    if (eq(uri, "/storeMatch")) {

      BEAObject pattern = beaGet(req.get("pattern"));

      BEAObject input = beaGet(req.get("input"));

      S label = req.get("label");

      BEAObject match = calculations.reactInputWithPattern(input, pattern);

      if (match == null) ret "Couldn't match";

      cset(match, +label);

      ret hrefresh(or2(req.get("redirect"), beaObjectURL(match)));

    }

    if (eq(uri, "/saveSyntacticPattern") || eq(uri, "/savePattern")) {

      S type = cic(uri, "syntactic") ? "Syntactic Pattern" : "Pattern";

      S text = req.get("text");

      BEAObject fromInput = beaGet(req.get("fromInput"));

      BEAObject pat = uniqCI BEAObject(+type, +text, createdBy := user(req));

      csetIfUnset(pat, +fromInput);

      if (fromInput != null)

        calculations.reactInputWithPattern(fromInput, pat);

      ret hrefresh(or2(req.get("redirect"), baseLink + "/"));

    }

    if (eq(uri, "/runInstruction")) {

      BEAObject o = beaGet(req.get("instruction"));

      runInstruction(o);

      ret hrefresh(beaObjectURL(o));

    }

    if (eq(uri, "/convertResultToInput")) {

      BEAObject result = beaGet(req.get("result"));

      if (result == null) ret subBot_serve500("Object not found");

      S text = unquote(getString result(result));

      if (l(text) > maxInputLength) ret subBot_serve500("Input too long");

      BEAObject input = uniqCI BEAObject(type := "Input", +text);

      uniqCI BEAObject(type := "Input Source", +input, source := result);

      ret hrefresh(beaShortURL(input));

    }

    /*if (eq(uri, "/pipeStringListIntoPattern")) {

      BEA r = beaGet(req.get("resultID"));

      BEa pat = beaGet(req.get("patternID"));

      if (r == null || pat == null) ret "Object not found";

      LS result = assertStringList(cget result(r));

      for (S s : result)

        uniq BEA(type := "Statement", );

    }*/

    if (eq(uri, "/setWordType")) {

      BEA r = beaGet(req.get("resultID"));

      S wordType = assertNempty(req.get("wordType"));

      LS result = assertStringList(cget result(r));

      for (S s : result)

        uniqCI BEA(type := "Word type", word := s, +wordType);

      ret "OK";

    }

    if (eq(uri, "/reload")) {

      dm_reloadModuleIn(3);

      ret "Reloading module in 3";

    }

    if (eq(uri, "/performAutoRuns")) {

      BEA o = beaGet(req.get("onObject"));

      if (o == null) ret "Object not found";

      rstAutoRuns.add(o);

      ret hrefresh(beaURL(o));

    }

    if (eq(uri, "/reactAllInputsWithAllSyntacticPatterns")) {

      ret str(returnTimed(r { calculations.reactAllInputsWithAllSyntacticPatterns(); }));

    }

    if (eq(uri, "/reactAllInputsWithPattern")) {

      BEA pat = beaGet(req.get("patternID"));

      if (pat == null) ret "Pattern not found";

      ret str(returnTimed(r { calculations.reactAllInputsWithPattern(pat); }));

    }

    if (eq(uri, "/storeSubInput")) {

      BEA input = beaGet(req.get("input"));

      if (input == null) ret "Input not found";

      S text = req.get("text");

      S type = eqic(req.get("label"), "good") ? "Sub-Input" : "Bad Sub-Input";

      BEA o = uniqCI BEA(

        +type,

        +input,

        +text,

        createdBy := currentUser());

      ret hrefresh(or2(req.get("redirect"), beaURL(o)));

    }

    if (eq(uri, "/performAutoRunOnAllObjects")) {

      BEA autoRun = beaGet autoRun(req);

      performAutoRunOnAllObjects(autoRun);

      ret "OK";

    }

    if (eq(uri, "/autoMigrate")) {

      BEA o = beaGet id(req);

      if (o == null) ret "Object not found";

      S url = beaURL(o);

      ret hsansserif() + "Migrated " + beaHTML(o) + " to " + className(autoMigrateToCustomClass(o))

        + hrefresh(1.0, url);

    }

    if (eq(uri, "/migrateToBase")) {

      BEA o = beaGet id(req);

      if (o == null) ret "Object not found";

      S url = beaURL(o);

      ret hsansserif() + "Migrated " + beaHTML(o) + " to " + className(autoMigrateToBase(o))

        + hrefresh(1.0, url);

    }

    if (eq(uri, "/convertSyntacticToSemanticMatches")) {

      BEA match = beaGet matchID(req);

      if (match == null) ret "Match not found";

      ret ul_htmlEncode2(calculations.convertSyntacticToSemanticMatches(match));

    }

    if (eq(uri, "/convertSyntacticToSemanticMatchesForWholePattern")) {

      BEA pat = beaGet patternID(req);

      if (pat == null) ret "Pattern not found";

      ret ul_htmlEncode2(calculations.convertSyntacticToSemanticMatchesForWholePattern(pat));

    }

    if (eq(uri, "/createPatternListFromUsefulSyntacticPatterns"))

      ret hrefresh(beaURL(calc.createPatternListFromUsefulSyntacticPatterns()));

    // add more master-mode URLs here

  }

  HCRUD makeBEAObjectCRUD(Req req, S type) {

    HCRUD crud = makeCRUD(BEAObject, req);

    crud.baseLink = req.uri();

    HCRUD_Concepts data = cast crud.data;

    data.itemName = () -> firstToUpper(type);

    data.addCIFilter(type := eqic(type, "(no type)") ? null : type);

    if (eqicOneOf(type, "Input", "Pattern", "AI Task")) {

      IF0<MapSO> prev = data.emptyObject;

      data.emptyObject = () -> {

        MapSO item = data.emptyObject_fallback(prev);

        item.put(text := ""); // show text field when creating new objects

        ret item;

      };

    }

    ret crud;

  }

  O renderBEAObjectTable(Req req, S type) {

    HCRUD crud = makeBEAObjectCRUD(req, type);

    ret serveCRUD(req, BEAObject, crud);

  }

  O serveUploadTexts(Req req, S type) {

    S inputs = req.get("text");

    new LS output;

    if (nempty(inputs)) {

      for (S text : tlft(inputs)) {

        Pair<BEAObject, Bool> p = uniqCI2_sync BEAObject(+type, +text);

        if (cget uploadedBy(p.a) == null)

          cset(p.a, uploadedBy := req.auth.user);

        output.add(type + " " + (p.b ? "added" : "exists")

          + " (ID " + p.a.id + "): " + text);

      }

    }

    ret h2("Upload " + plural(type))

      + hpostform(

          p(plural(type) + " (one per line):")

        + p(htextarea(inputs, name := "text"))

        + pIfNempty(htmlEncode_nlToBR(lines(output)))

        + hsubmit("Upload " + plural(type)));

  }

  Cl<BEAObject> beaObjectsOfType(S type) {

    ret conceptsWhereCI BEAObject(+type);

  }

  void reactAllInputsWithSomePatterns {

    calculations.reactAllInputsWithSomePatterns();

  }

  S navDiv() {

    HCRUD crud = makeCRUD(BEAObject, currentReq!);

    long time = sysNow();

    int inputsWithoutRewrites = this.inputsWithoutRewrites!;

    int inputsWithoutMatches = this.inputsWithoutMatches!;

    int syntacticPatternsWithoutRewrites = this.syntacticPatternsWithoutRewrites!;

    /*int inputsWithoutRewrites = 0;

    int inputsWithoutMatches = 0;

    int syntacticPatternsWithoutRewrites = 0;*/

    done2_always("Count Things", time);

    ret joinNemptiesWithVBar(

      ahref(baseLink + "/query", "Query"),

      beaNavLink("Input", crud),

      beaNavLink("Pattern", crud),

      beaNavLink("Syntactic Pattern", crud),

      ahref(baseLink + "/syntacticPatternsWithoutRewrites", n2(syntacticPatternsWithoutRewrites) + " Syntactic patterns w/o rewrites"),

      beaNavLink("Match", crud),

      beaNavLink("Rewrite", crud),

      beaNavLink("AI Task", crud),

      ahref(baseLink + "/inputsWithoutMatches", n2(inputsWithoutMatches) + " Inputs w/o matches"),

      ahref(baseLink + "/inputsWithoutRewrites", n2(inputsWithoutRewrites) + " Inputs w/o rewrites"),

      hPopDownButtonWithText("Bot Forum", navLinks(flat := true, withStats := false)),

      HTMLPopDownButton(

        ahref(baseLink + "/allBEATypes", "All object types")

          + " " + hPopDownButton(renderObjectTypes()),

        ahref(baseLink + "/stats", "Stats"),

        !inMasterMode(currentReq!) ? null : ahref(baseLink + "/refchecker", "Reference checker"),

        ahref(baseLink + "/inputsWithRewrites", "Inputs with rewrites"),

        ahref(baseLink + "/rewritesTable", "Rewrites table"),

        ahref(baseLink + "/patternRewritesTable", "Pattern rewrites table"),

        ahref(baseLink + "/matchesTable", "Matches table"),

        ahref(baseLink + "/matchesTable?syntacticOnly=1", "Syntactic matches table"),

      ).width(350).height(500).html());

  }

  // crud is just the cached BEAObject crud to check for creation rights

  S beaNavLink(S type, HCRUD crud, int count default beaCount(type)) {

    S plural = firstToLower(plural(type));

    S link = baseLink + "/" + camelCase(plural);

    ret ahref(link, n2(count) + " " + firstToUpper(plural)) + (!crud.actuallyAllowCreate() ? "" : " " + ahref(addParamToURL(link, cmd := "new"), "+"));

  }

  S beaObjectURL(BEAObject o) {

    ret conceptLink(o, currentReq!);

    /*ret o == null ?:

      addParamsToURL(baseLink + "/crud/BEAObject",

        selectObj := o.id) + "#" + o.id;*/

  }

  S matchDescHTML(BEAObject m) {

    pcall {

      BEAObject pat = cget pattern(m);

      SS mapping = cast cget mapping(m);

      ret ahref_undecorated(crudLink(m), htmlEncode2(quote(getString text(pat)))

        + "<br>&nbsp; with " + renderEqualsCommaProperties(mapping));

    }

    ret htmlEncode2(str(m));

  }

  int beaCount(S type) {

    ret countConceptsWhereCI BEAObject(+type);

  }

  Cl<BEAObject> beaList(S type, O... params) {

    ret conceptsWhereCI BEAObject(paramsPlus_inFront(params, +type));

  }

  Cl<BEAObject> beaListAny(S... types) {

    ret concatLists(lmap beaList(litciset(types)));

  }

  BEA beaGet(long id) {

    ret getConceptOpt BEAObject(id);

  }

  BEA beaGet(S id) {

    ret beaGet(parseFirstLong(id));

  }

  // first param can be type

  BEA beaNew(O... _) {

    _ = prependParamIfOddCount("type", _);

    ret cnew BEA(_);

  }

  bool beaTypeIs(BEA o, S type) {

    ret o != null && o.typeIs(type);

  }

  BEA mapMethodLike beaGet(S key, Req req) {

    ret beaGet(req.get(key));

  }

  BEA mapMethodLike cgetBEA aka beaGet(S field, BEA o) {

    ret (BEA) cget(field, o);

  }

  BEAObject cgetBEA aka beaGet(BEAObject o, S field) {

    ret cgetBEA(field, o);

  }

  S beaLinkHTML aka beaToHTML aka beaHTML(BEAObject o) {  

    ret o == null ?: ahref(conceptLink(o), htmlEncode2_nlToBr(str(o));

  }

  S beaHTML_justID(BEA o) { ret o == null ?: ahref(beaShortURL(o), o.id); }

  S beaShortURL aka beaURL(BEAObject o) {

    ret o == null ?: baseLink + "/" + o.id;

  }

  Cl<BEAObject> beaBackRefs(BEAObject o, S type) {

    ret objectsWhereCI(findBackRefs BEAObject(o), +type);

  }

  Cl<BEAObject> beaBackRefs(BEAObject o) {

    ret findBackRefs BEAObject(o);

  }

  O serveDefaultPage(Req req) {

    HTMLFramer1 framer = req.framer;

    framer.add(hcenter3(hsnippetimg_scaleToWidth(200, #1102967, 200, 110, title := "Gazelle"), style := "margin-top: 100px"));

    ret completeFrame(req);

  }

  S html_loggedIn() {

    User user = user(currentReq!);

    ret user == null

      ? /*"Not logged in"*/ ahref(baseLink + "/", "Log in")

      : "Logged in as " + htmlEncode2(user.name);

  }

  void distributeNewObject_impl(BEAObject o) {

    if (o.typeIs("Match")) ret;

    distributeNotification("New object: " + o);

  }

  void performAutoRuns(BEAObject o) enter {

    print("performAutoRuns", o);

    for (BEAObject autoRun : beaList("Auto Run")) {

      if (!isTrue(getOpt enabled(autoRun)))

        continue with print("Not enabled: " + autoRun);

      performAutoRunOnObject(autoRun, o);

    }

  }

  void performAutoRunOnAllObjects(BEA autoRun) {

    for (BEA o : list(BEA))

      performAutoRunOnObject(autoRun, o);

  }

  void performAutoRunOnObject(BEA autoRun, BEA o) {

    ping();

    S type = getString onChangedObjectOfType(autoRun);

    if (!o.typeIs(type)) ret with print("Wrong type: " + type);

    print("Running " + autoRun);

    BEAObject procedure = cast cget procedure(autoRun);

    S internalCode = getString internalCode(procedure);

    printVars_str(+internalCode, +o);

    if (eqic(internalCode, "convertInputToPattern")) {

      S text = o.text();

      if (!containsAngleBracketVars(text)) ret with print("No angle bracket vars");

      BEAObject p = uniqCI_returnIfNew BEAObject(type := "Pattern", +text);

      cset(p, fromInput := o, byProcedure := procedure, byAutoRun := autoRun);

      print(+p);

    } else if (eqic(internalCode, "convertInputToSyntacticPattern")) {

      S text = o.text();

      if (!containsStars(text)) ret with print("No stars");

      BEAObject p = uniqCI_returnIfNew BEAObject(type := "Syntactic Pattern", +text);

      cset(p, fromInput := o, byProcedure := procedure, byAutoRun := autoRun);

      print(+p);

    } else if (eqic(internalCode, "dropPunctuationFromPattern")) {

      S text = o.text();

      S text2 = dropPunctuation(text);

      if (eq(text, text2)) ret;

      BEAObject p = uniqCI BEA(

        type := o.type(),

        text := text2);

      cset(o, withoutPunctuation := p);

    } else if (eqic(internalCode, "makeSyntacticPattern")) {

      print(sp := calculations.makeSyntacticPattern(o));

    } else if (eqic(internalCode, "runFunctionOnInput")) {

      print("runFunctionOnInput");

      BEA function = beaGet(o, "function");

      BEA input = beaGet(o, "input");

      if (function == null || input == null) ret with print("Missing parameters");

      if (cget result(o) != null) ret with print("Has result");

      BEA result = calculations.reactFunctionWithInput(calculations.new BackEndAlgorithm, function, input);

      print(+result);

      if (result != null) {

        cset(result, request := o);

        cset(o, +result);

      }

    } else

      print("Unknown internal code");

  }

  BEAObject findInput(S text) {

    ret conceptWhereIC(BEAObject, type := "Input", +text);

  }

  S addRewriteHTML(BEAObject o) {

    ret ahref(addParamsToURL(crudLink(BEAObject),

      cmd := "new",

      title := "Add Rewrite",

      f_type := "Rewrite",

      f_text := getStringOpt text(o),

      f_isRewriteOf := o.id, metaInfo_isRewriteOf := "concept",

    ), "Add Rewrite");

  }

  O serveIntegerLink(Req req, long id) {

    BEAObject o = getConceptOpt BEAObject(id);

    if (o != null)

      ret htitle(str(o)) + hrefresh(conceptLink(o));

    ret super.serveIntegerLink(req, id);

  }

  void distributeTestNotification() {

    distributeNotification("It is " + localTimeWithSeconds());

  }

  void distributeNotification(S text) {

    notificationQ.add(r {

      /*for (User user)

        if (nemptyAfterTrim(user.notificationSetting))

          sendNotification(text);*/

      for (Pair<virtual WebSocket, WebSocketInfo> p : syncMapToPairs(webSockets)) {

        // TODO: check user

        S jsCode =

          "window.createNotification({ theme: 'success', showDuration: 3000 })("

          + jsonEncodeMap(message := text) + ");";

        call(p.a, "send", jsonEncodeMap(eval := jsCode));

      }

    });

  }

  void runInstruction(BEAObject o) {

    if (o == null) ret;

    try {

      BEAObject instruction = o;

      if (o.typeIs("Step in script"))

        instruction = (BEAObject) cget(o, "instruction");

      if (instruction.typeIs("Instruction | List objects by type")) {

        saveInstructionResult(o, beaList(getString typeToList(instruction)));

        ret;

      }

      if (instruction.typeIs("Instruction | List object types"))

        ret with saveInstructionResult(o, distinctCIFieldValuesOfConcepts(BEAObject, "type"));

      if (instruction.typeIs("Instruction | Filter list by text starting with")) {

        // find list made before

        BEAObject scriptRun = cgetBEA scriptRun(o);

        if (scriptRun == null) fail("Need to be run as part of script");

        L<BEAObject> steps = scriptRunSteps(scriptRun);

        int idx = indexOf(steps, o);

        if (idx < 0) fail("Step not found in script run");

        L<BEAObject> list = firstNotNull(map(reversed(takeFirst(steps, idx)),

          step -> optCast L(cget data(cgetBEA result(step)))));

        S prefix = getString prefix(instruction);

        L<BEAObject> filtered = filter(list, obj -> swic(obj.text(), prefix);

        saveInstructionResult(o, filtered);

        ret;

      }

      if (instruction.typeIs("Script")) {

        BEAObject script = instruction;

        BEAObject scriptRun = cnew BEAObject(type := "Script Run", +script);

        // Make an instance of all the instructions

        int i = 0;

        new L<BEAObject> steps;

        while not null (instruction = (BEAObject) cget(instruction, "step" + (++i))) {

          BEAObject step = cnew(BEAObject, type := "Step in script", step := i, +scriptRun, +instruction);

          steps.add(step);

        }

        cset(scriptRun, +steps);

        // TODO: run steps?

        ret;

      }

      cnew BEAObject(type := "Instruction Error", instruction := o, error := "Unknown instruction type");

    } catch e {

      cnew BEAObject(type := "Instruction Error", instruction := o, error := getStackTrace(e));

    }

  }

  BEAObject saveInstructionResult(BEAObject instruction, O data) {

    BEAObject result = cnew BEAObject(type := "Instruction Result",

      +instruction, +data);

    cset(instruction, +result);

    ret result;

  }

  L<BEAObject> scriptRunSteps(BEAObject scriptRun) {

    ret (L) cget steps(scriptRun);

  }

  S addCommentHTML(BEA o, O... _) {

    optPar S redirectAfterSave;

    optPar S defaultComment;

    optPar S text = "Add comment";

    ret ahref(addParamsToURL(crudLink(BEAObject),

      cmd := "new",

      title := "Add Comment",

      f_type := "Comment",

      f_on := o.id,

      f_text := unnull(defaultComment),

      +redirectAfterSave,

      autofocus := "f_text",

      metaInfo_on := "concept"), text);

  }

  S formatSubInput(BEA input, S text, S anchor) {

    S redirect = addAnchorToURL(currentReq->uriWithParams(), anchor);

    int goodCount = countConceptsCI BEA(type := "Sub-Input", +input, +text);

    int badCount = countConceptsCI BEA(type := "Bad Sub-Input", +input, +text);

    ret calculations.bestInputHTML(text)

      + " " + small(joinNemptiesWithSpace(

        ahref(addParamsToURL(

          baseLink + "/storeSubInput", 

          label := "good",

          +text,

          input := input.id,

          +redirect),

        unicode_thumbsUp()),

        goodCount == 0 ? "" : n2(goodCount),

        ahref(addParamsToURL(

          baseLink + "/storeSubInput", 

          label := "bad",

          +text,

          input := input.id,

          +redirect),

        unicode_thumbsDown()),

        badCount == 0 ? "" : n2(badCount)

      ));

  }

  LS renderObjectTypes() {

    MultiSet<S> ms = distinctCIFieldValuesOfConcepts_multiSet(BEAObject, "type");

    ret mapPairs(multiSetToPairsByPopularity(ms),

      (type, count) -> {

        S _type = or2(type, "(no type)");

        ret count + " " + ahref(baseLink + "/beaCRUD/" + urlencode(_type), htmlEncode2(_type));

      });

  }

  BEA autoMigrateToBase(BEA o) {

    ret autoMigrateToClass(o, BEA);

  }

  BEA autoMigrateToCustomClass(BEA o) {

    ret autoMigrateToClass(o, defaultCustomClass(o));

  }

  BEA autoMigrateToClass(BEA o, Class<? extends BEAObject> targetClass) {

    pcall {

      if (o == null) null;

      if (targetClass != null && targetClass != _getClass(o))

        ret replaceConceptAndUpdateRefs(o, unlistedCopyToClass(targetClass, o));

    }

    ret o;

  }

  BEA autoMigrateUnlistedOrKeep(BEA o) {

    ret canAutoMigrate(o) ? unlistedCopyToClass(defaultCustomClass(o), o) : o;

  }

  bool canAutoMigrate(BEA o) {

    ret o != null && !eqOneOf(defaultCustomClass(o), _getClass(o), null);

  }

  Class<? extends BEA> defaultCustomClass(BEA o) {

    if (o == null) null;

    BEA entry = conceptWhereCI BEA(type := "Auto Custom Class", forType := o.type());

    //printVars_str defaultCustomClass(+o, +entry);

    if (entry == null) null;

    S className = getString toClassName(entry);

    //printVars_str defaultCustomClass(+className);

    Class c = findClassThroughDefaultClassFinder(className);

    //printVars_str defaultCustomClass(+c);

    ret c;

  }

  S queryLink(S algorithm, S q) {

    ret addParamsToURL(baseLink + "/query", +algorithm, +q);

  }

} // end of module

extend User {

  S notificationSetting;

}

concept BEAObject > ConceptWithGlobalID {

  new Ref<UserPost> mirrorPost;

  void change :: after {

    beaMod().rstUpdateBEAMirrors.add(this);

  }

  void delete :: before {

    cdelete(mirrorPost!);

  }

  void updateMirrorPost {

    GazelleBEA mod = beaMod();

    if (isDeleted() || !mod.mirrorBEAObjects) ret;

    if (!mirrorPost.has())

      cset(this, mirrorPost := cnew UserPost(

        type := "BEA Object",

        creator := mod.internalUser(),

        botInfo := "BEA Mirror Bot"));

    S text = structureString();

    cset(mirrorPost!, 

      title := str(this),

      +text);

  }

  S structureString() {

    S text = "Error";

    pcall {

      structure_Data data = new {

        structure_ClassInfo newClass(Class c) {

          structure_ClassInfo info = super.newClass(c);

          if (c == Concept.Ref.class) {

            info.special = true;

            info.serializeObject = o -> {

              Concept cc = cast deref((Concept.Ref) o);

              //append("cu CRef " + (cc != null ? str(cc.id) : "null"), 3);

              if (cc cast BEAObject)

                append("CRef(gid=" + quote(cc.globalID()) + ")", 6);

              else if (cc != null)

                append("CRef(id=" + cc.id + ")", 6);

              else

                append("CRef", 1);

            };

          }

          ret info;

        }

        void setFields(structure_ClassInfo info, L<Field> fields) {

          if (isSubclassOf(info.c, BEAObject)) {

            // Don't serialize "refs" and "backRefs" fields

            removeAll(fields,

              getField(BEAObject, "refs"),

              getField(BEAObject, "backRefs"));

          }

          super.setFields(info, fields);

        }

      };

      S struct = structure(this, data);

      struct = dropLoadableUtilsPackageFromStruct(struct);

      text = indentStructureString_firstLevels(1, struct);

    }

    ret text;

  }

  toString {