sclass BEACalculations {

  GazelleBEA mod;  

  *(GazelleBEA *mod) {}

  transient int liveOutputDelay = 1000;

  transient double algoTimeout = 30.0;

  void reactAllInputsWithSomePatterns() {

    for (BEAObject input : mod.beaObjectsOfType("input")) {

      new ReactInputWithSomePatterns r;

      r.input = input;

      r.ccOut = db_mainConcepts();

      r.max = 10;

      r.start();

      stepAllWithTimeout(10.0, r);

    }

  }

  void reactAllInputsWithAllSyntacticPatterns() {

    for ping (BEA input : mod.beaList("input")) {

      for ping (BEA pattern : mod.beaList("Syntactic Pattern")) {

        continue if patternAlwaysMatches(pattern);

        reactInputWithPattern(input, pattern);

      }

    }

  }

  void reactAllInputsWithPattern(BEA pattern) {

    for ping (BEA input : mod.beaList("input"))

      reactInputWithPattern(input, pattern);

  }

  bool patternAlwaysMatches(BEA pat) {

    ret isTrue_getOpt alwaysMatches(pat);

  }

  class ReactInputWithSomePatterns implements Steppable {

    BEAObject input;

    Concepts ccOut;

    int max = 3, n;

    Iterator<BEAObject> patterns;

    void start {

      n = countConceptsWhereIC(ccOut, BEAObject, +input, type := "match");

      patterns = iterator(sortByConceptID(conceptsWhereIC BEAObject(type := "Pattern")));

    }

    public bool step() {

      if (n >= max) false;

      if (!patterns.hasNext()) false;

      if (reactInputWithPattern(input, patterns.next(), ccOut) != null)

        ++n;

      true;

    }

  }

  // (patern, mapping)

  Pair<BEA, SS> matchInputWithPatternList(S text, BEA patternList) {

    L<BEA> patterns = cast cget patterns(patternList);

    fOr (BEA pattern : patterns) {

      SS mapping = matchInputWithPattern(text, pattern);

      if (mapping != null) ret pair(pattern, mapping);

    }

    null;

  }

  // we don't do this anymore

  new ThreadLocal<Bool> matchInput_droppedPunctuation;

  SS matchInputWithPattern(BEAObject input, BEAObject pattern) {

    S text = getString text(input);

    ret matchInputWithPattern(text, pattern);

  }

  SS matchInputWithPattern(S text, BEAObject pattern) {

    S pat = getString text(pattern);

    bool punctuationOnly = isTrue_getOpt punctuationOnly(pattern);

    if (pattern.typeIs("Syntactic Pattern"))

      ret matchInputWithSyntacticPattern(text, pat, punctuationOnly);

    else if (eqic_gen(pattern~.subType, "PhraseCache"))

      ret phraseCache(pat, text) ? litmap() : null;

    else

      ret matchInputWithPattern(text, pat, punctuationOnly);

  }

  SS matchInputWithSyntacticPattern(S text, S pat, bool punctuationOnly default false) {

    pat = starsToAngleBracketedNumbers(pat);

    ret matchInputWithPattern(text, pat, punctuationOnly);

  }

  SS matchInputWithPattern(S text, S pat, bool punctuationOnly default false) {

    SS mapping = flexMatchAngleBracketVarsIC_honorPunctuation_noBrackets_first(pat, text);

    if (mapping != null || punctuationOnly) ret mapping;

    /*matchInput_droppedPunctuation.set(true);

    ret flexMatchAngleBracketVarsIC_first(pat, text);*/

    null; // not doing the implicit punctuation drop anymore (it's now an explicit pattern)

  }

  BEAObject reactInputWithPattern(BEAObject input, BEAObject pattern, Concepts ccOut default db_mainConcepts()) {

    if (input == null || pattern == null) null;

    matchInput_droppedPunctuation.set(false);

    SS mapping = matchInputWithPattern(input, pattern);

    //printVars_str("reactInputWithPattern", +input, +pattern, +mapping);

    if (mapping == null) null;

    BEA match = uniqCI(ccOut, BEA,

      type := "Match",

      +input,

      +pattern);

    cset(match,

      +mapping,

      droppedPunctuation := trueOrNull(matchInput_droppedPunctuation!),

    );

    ret match;

  }

  O serveAnalyzeInput(GazelleBEA.Req req) {

    new Concepts ccOut;

    BEAObject input = getConcept BEAObject(parseLong(req.get("id")));

    if (input == null)

      input = cnew(ccOut, BEAObject, type := "Input", text := req.get("q"));

    S text = getString text(input);

    new ReactInputWithSomePatterns r;

    r.input = input;

    r.ccOut = ccOut;

    r.max = 1000;

    r.start();

    stepAllWithTimeout(10.0, r);

    Cl matches = conceptsWhereIC(ccOut, BEAObject, type := "Match");

    ret h1_title("Analyze Input")

      + p("Input: " + b(htmlEncode_nlToBr_withIndents(text)))

      + p(empty(matches) ? "No matches" : "Matches")

      + ul_htmlEncode(matches);

  }

  O serveQueryPage(GazelleBEA.Req req, bool withFrame default false) {

    S q = req.get("q"), algorithm = req.get("algorithm");

    bool dontRun = eq(req.get("dontRun"), "1");

    bool liveOutput = eq(req.get("liveOutput"), "1");

    if (liveOutput) req.noSpam();

    if (eqic(algorithm, "process input"))

      algorithm = "Process new input v3";

    //if (liveOutput) ret jsonEncode(litmap(answer := "Live output demo: " + htmlEncode2(q)));

    Map<S, IF0<Algorithm>> algorithms = algorithms();

    S output = empty(algorithm) || dontRun ? null : doAlgorithm(algorithms, algorithm, q, liveOutput);

    if (liveOutput)

      ret jsonEncode(litorderedmap(answer := output));

    S right = withFrame ? "" : mod.html_loggedIn();

    Algorithm alg = callF(lookupPossiblyCI(algorithms, algorithm));

    if (alg != null) {

      alg.q = q;

      alg.req = mod.currentReq();

    }

    S centered = 

        (withFrame ? "" : htitle_h2(ahref(mod.baseLink + "/", "Gazelle") + " Query"

        + htmlEncode2((empty(q) ? "" : ": " + q)

        + appendSquareBracketed(algorithm))))

      + hscript(replaceVars([[

          var sugLoading = false, sugTriggerAgain = false;

          function sugTrigger() {

            console.log("sugTrigger");

            if (sugLoading) { sugTriggerAgain = true; return; }

            // get form data as JSON

            var data = {

              "liveOutput" : "1",

              "q": $("textarea[name=q]").val(),

              "algorithm": $("select[name=algorithm]").val(),

              "algorithmID": $("select[name=algorithmID]").val(), // XXX: do this for all additional alg parameters

            };

            const url = "/query";

            console.log("Loading " + url + " with " + data);

            sugLoading = true;

            $.post(url, data,

              function(result) {

                console.log("Suggestor result: " + result);

                const answer = !result ? "" : JSON.parse(result).answer;

                if (answer) {

                  $("#liveResult .sectionContents").html(answer);

                  $("#liveResult").show();

                } else

                  $("#liveResult").hide();

                //$("#suggestorResult").html(answer ? "Suggestor says: " + answer : "");

              }

            ).always(function() {

              console.log("sug loading done");

              setTimeout(function() {

                sugLoading = false;

                if (sugTriggerAgain) { sugTriggerAgain = false; sugTrigger(); }

              }, liveOutputDelay);

            });

          }

          $(document).ready(function() {

            $("textarea[name=q], input[name=q], select[name=algorithm]").on('input propertychange change', sugTrigger);

            sugTrigger();

          });

        ]], +liveOutputDelay))

      + hform(p("Query (Ctrl+Enter): "

          + htextarea(q,

              id := "mainInput", 

              name := "q", 

              class := "auto-expand",

              style := "width: 300px",

              onkeydown := jquery_submitFormOnCtrlEnter(),

              autofocus := dontRun || empty(q) ? true : null // autofocus for live output mode

            ) + " "

          /*+ hjs([[$(document).ready(function() {

              autoExpandTextArea(document.getElementById("mainInput"));

          });]])*/

        //+ htextfield(+q, style := "width: 300px; text-align: center", autofocus := true)

        + (alg == null ? "" : alg.additionalInputs())

        + "   Algorithm: " 

        + hselect_list(keys(algorithms), algorithm, name := "algorithm") + "   " + hbutton("Go")));

    S content = (empty(right) ? "" : div_alignRight(right))

      + div_center(centered)

      + htitledSectionWithDiv("Live Result", "",

         id := "liveResult", style := "display: none",

         innerDivStyle := "max-height: 150px; overflow: auto")

      + (output == null ? "" : h3(algorithm) + output);

    if (withFrame) {

      req.framer.addInHead(hjs_handleScrollToAnchor());

      req.framer.add(content);

      ret mod.completeFrame(req);

    } else

      ret hhtml(hhead(

          hmobilefix() + hsansserif() + 

          loadJQuery2() + 

          hjs_handleScrollToAnchor() +

          hjs_autoExpandingTextAreas() +

          mod.webSocketHeadStuff(req) +

          hNotificationPopups())

        + hbody(content));

  }

  S doAlgorithm(Map<S, IF0<Algorithm>> algorithms, S algorithm, S q, bool liveMode) {

    GazelleBEA.Req req = mod.currentReq();

    StringBufferWithMaxSize out = new(1000000); // TODO: make sure HTML tags are closed properly

    IF0<Algorithm> algo = lookupPossiblyCI(algorithms, algorithm);

    if (algo == null)

      out.append("Algorithm not found: " + algorithm);

    else {

      if (isB(evalWithTimeout(algoTimeout, r {

        temp mod.enter();

        temp tempSetTL(mod.currentReq, req);

        Algorithm alg = algo!;

        alg.liveMode = liveMode;

        alg.out = out;

        alg.q = q;

        alg.req = mod.currentReq();

        alg.runIt();

      })))

        out.append("\n[algorithm timeout]");

    }

    ret str(out);

  }

  class Algorithm implements Runnable {

    S q;

    GazelleBEA.Req req;

    Appendable out;

    //Appendable outPrio = new StringBufferWithMaxSize(10000);

    bool liveMode;

    Concepts localConcepts = db_mainConcepts(); //= childConcepts(db_mainConcepts());

    L<Runnable> nonMatches; // null so not filled by default

    // override for normal mode

    run {}

    // override for live mode

    void runLiveMode {}

    /*void cleanForStorage {

      out = outPrio = null;

    }*/

    // overridable

    S additionalInputs() { ret ""; }

    void appendHTML(Appendable out default this.out, S html) ctex {

      if (empty(html)) ret;

      html = wrapInDiv(html, style := "margin-top: 0.5em");

      out.append(html).append("\n");

    }

    void appendHTML_raw(S html) ctex { out.append(html).append("\n"); }

    void appendText(Appendable out default this.out, S text) ctex {

      if (empty(text)) ret;

      out.append(hpre_htmlencode(rtrim(text)));

    }

    void appendTextBold(Appendable out default this.out, S text) ctex {

      if (empty(text)) ret;

      out.append(b(hpre_htmlencode(rtrim(text))));

    }

    void allowSuggestingAnswer(BEAObject input) {

      if (input == null) ret;

      S text = input.text();

      S url = mod.baseLink + "/saveAnswer";

      appendHTML(

        hpostform("Suggest an answer to " + htmlEncode_quote(text) + ": "

          //+ hinputfield(+text, style := "width: 300px")

          + htextarea(text, id := "suggestAnswer", name := "text", class := "auto-expand",

            style := "width: 300px")

          + hhidden(inputID := input.id)

          + hhidden(redirect := "")

          + " " + hsubmit("Save answer")

          + " [Optional rewrite type: " + hinputfield("rewriteType", style := "width: 100px") + " ]",

          action := url,

          onsubmit := js_setRedirect()));

    }

    void allowSuggestingPattern(BEAObject input) {

      if (input == null) ret;

      S text = input.text();

      S url = mod.baseLink + "/savePattern";

      appendHTML(

        hpostform("Suggest a pattern (with angle bracket vars): "

          + htextarea(text, name := "text", class := "auto-expand",

            style := "width: 300px")

          + hhidden(fromInput := conceptIDOrNull(input))

          + hhidden(userTyped := true)

          + hhidden(redirect := "")

          + " " + hsubmit("Save pattern"),

          action := url,

          onsubmit := js_setRedirect()));

    }

    void allowSuggestingSyntacticPattern(BEAObject input) {

      if (input == null) ret;

      S text = input.text();

      S url = mod.baseLink + "/saveSyntacticPattern";

      appendHTML(

        hpostform("Suggest a syntactic pattern (with * vars): "

          + htextarea(text, name := "text", class := "auto-expand",

            style := "width: 300px")

          + hhidden(fromInput := conceptIDOrNull(input))

          + hhidden(userTyped := true)

          + hhidden(redirect := "")

          + " " + hsubmit("Save syntactic pattern"),

          action := url,

          onsubmit := js_setRedirect()));

    }

    BEA saveInput(S q) {

      if (userAgentIsBot(beaMod().currentUserAgent())) null;

      Pair<BEA, Bool> p = uniqCI2 BEA(type := "Input", text := q);

      BEA input = p.a;

      mod.saveUserAgent(input);

      if (p.b) {

        csetIfUnset(input, createdBy := mod.user(mod.currentReq()));

        appendHTML("Input saved");

      }

      ret input;

    }  

    void runNonMatches {

      if (nempty(nonMatches)) { appendHTML(hr()); callFAll(nonMatches); }

    }

    void addNonMatch(Runnable r) { add(nonMatches, r); }

    void runIt {

      if (liveMode) runLiveMode(); else run();

      runNonMatches();

    }

  } // end of Algorithm

  class ReverseInput extends Algorithm {

    run { appendText(reversed(q)); }

    void runLiveMode { run(); }

  }

  class TimeoutTest extends Algorithm {

    run { sleepSeconds(120); }

  }

  class RunPattern extends Algorithm {

    run {

      BEAObject pattern = mod.beaGet(parseFirstLong(q));

      if (pattern == null) ret with appendText("Pattern not found");

      appendText("Pattern: " + pattern);

      for (BEAObject input : mod.beaList("Input")) {

        S text = input.text();

        SS mapping = matchInputWithPattern(input, pattern);

        if (mapping == null)

          addNonMatch(r {

            appendText("Checking Input: " + input + " - No match");

          });

        else {

          S result = "\n" + renderMappingAsText(mapping);

          appendText("Checking Input: " + input + result);

          allowStoringMatch(this, input, pattern, mapping);

        }

      }

    }

  }

  S renderMappingAsText(SS mapping) {

    ret indent(formatDoubleArrowMap_horizontallyAligned(mapping));

  }

  class FindRewrites extends Algorithm {

    run {

      // argument is ID of a match

      BEAObject match = mod.beaGet(parseFirstLong(q));

      if (match == null || !match.typeIs("Match")) ret with appendText("Match not found");

      BEAObject pattern = cast cget pattern(match);

      BEAObject input = cast cget input(match);

      SS mapping = cast get mapping(match);

      appendText("Match: " + match);

      appendText("Input: " + input);

      appendText("Pattern: " + pattern);

      appendText("Mapping:\n"

        + renderMappingAsText(mapping));

      appendRewrites(this, input, input.text(), pattern, match, mapping);

    }

  }

  // match and input can be null, pattern has to be set

  LS appendRewrites(Algorithm algo, BEAObject input, S inputText, BEAObject pattern, BEAObject match, SS mapping) {

    Cl<BEAObject> rewrites = mod.beaBackRefs(pattern, "Rewrite");

    new LS texts;

    for (BEAObject r : rewrites) {

      S text = r.text();

      S textWithVars = text;

      if (pattern.typeIs("Syntactic pattern"))

        textWithVars = starsToAngleBracketedNumbers(text);

      S text2 = replaceAngleBracketVars_curly(textWithVars, mapping);

      algo.appendText(" Using pattern rewrite: " + text);

      algo.appendTextBold(

          "    " + inputText + "\n"

        + " => " + text2);

      texts.add(text2);

      // look for existing rewrite

      BEAObject existing = match == null || input == null ?: conceptWhereIC BEAObject(

        type := "Rewrite", +input, text := text2,

          +match, patternRewrite := r, label := "good"

      );

      if (existing != null)

        algo.appendHTML("Rewrite exists: " + mod.beaLinkHTML(existing));

      else if (match == null)

        algo.appendText("[Can't save, no match]");

      else if (input == null)

        algo.appendText("[Can't save, no input object]");

      else

        algo.appendHTML(

          hinlinepostform(

            hhiddenMulti(

              action := "create",

              f_type := "Rewrite",

              f_text := text2,

              f_input := input.id, metaInfo_input := "concept",

              f_match := match.id, metaInfo_match := "concept",

              f_patternRewrite := r.id, metaInfo_patternRewrite := "concept",

              f_label := "good")

            + hbutton("Good rewrite"),

            target := "_blank",

            action := mod.crudLink(BEAObject)));

    }

    ret texts;

  }

  class ProcessNewInput extends Algorithm {

    run {

      BEAObject input = handleInput(this, q = trim(q), false);

      for (BEAObject pattern : mod.beaListAny("Pattern", "Syntactic Pattern")) {

        checkPattern(this, input, q, pattern);

      }

    }

  }

  void checkPattern(Algorithm algo, BEA input, S q, BEA pattern) {

    SS mapping = matchInputWithPattern(q, pattern);

    if (mapping == null)

      algo.addNonMatch(r {

        algo.appendText("Checking Pattern: " + pattern + " - No match");

      });

    else {

      S result = "\n" + renderMappingAsText(mapping);

      algo.appendText("Checking Pattern: " + pattern + result);

      if (input == null) {

        //appendHTML("Click \"Save input\" first to store this match");

      } else

        allowStoringMatch(algo, input, pattern, mapping);

      appendRewrites(algo, input, q, pattern, null, mapping);

    }

  }

  class ProcessNewInput_v2 extends Algorithm {

    run {

      q = trim(q);

      saveInput(q);

      BEAObject input = handleInput(this, q, false);

      if (input != null) {

        Cl<BEAObject> rewrites = mod.beaBackRefs(input, "Rewrite");

        BEAObject rewrite = random(rewrites);

        if (rewrite != null)

          appendHTML("My answer: " + b(htmlEncode2(rewrite.text())) + " " + ahref(mod.beaShortURL(rewrite), "[source]"));

      }

      allowSuggestingAnswer(input);

      allowSuggestingPattern(input);

      allowSuggestingSyntacticPattern(input);

      for (BEAObject pattern : syntacticPatternsSortedBySpecificityDesc()) {

        continue if patternAlwaysMatches(pattern);

        SS mapping = matchInputWithPattern(q, pattern);

        if (mapping == null)

          addNonMatch(r {

            appendHTML("Checking Syntactic Pattern: " + mod.beaLinkHTML(pattern) + " - No match");

          });

        else {

          S result = "Grouped as: " + replaceAngleBracketVars_curly(starsToAngleBracketedNumbers(pattern.text()), mapping) + "\n"

            + renderMappingAsText(mapping);

          appendHTML(hr());

          appendHTML("Checking Syntactic Pattern: " + mod.beaLinkHTML(pattern));

          appendText(result);

          BEAObject match = null;

          if (input == null) {

            //appendHTML("Click \"Save input\" first to store this match");

          } else

            match = allowStoringMatch(this, input, pattern, mapping);

          MultiMap<S, BEAObject> subInputRewrites = ciMultiMap();

          for (S subInput : uniquifyCI(values(mapping)))

            if (!eqic(subInput, q)) {

              BEAObject subInputObj = handleInput(this, subInput, "Sub-Input", true);

              // find rewrites for sub-input

              if (subInputObj != null) {

                L<BEAObject> subRewrites = concatLists(

                  conceptsWhereCI(BEAObject, type := "Rewrite", input := subInputObj),

                  conceptsWhereCI(BEAObject, type := "Rewrite", isRewriteOf := subInputObj));

                for (BEAObject r : subRewrites)

                  appendHTML("  Sub-Rewrite found: " + r);

                subInputRewrites.putAll(subInput, subRewrites);

              }

            }

          // show all rewrites for the pattern  

          appendRewrites(this, input, q, pattern, match, mapping);

          // apply some sub-rewrites randomly

          SS rewrittenMapping = mapValues(mapping,

            s -> {

              BEAObject rewrite = random(subInputRewrites.get(s));

              ret rewrite == null ? s : uncurly(rewrite.text());

            });

          S randomRewrite = replaceAngleBracketVars_curly(starsToAngleBracketedNumbers(pattern.text()), rewrittenMapping);

          if (neq(randomRewrite, q))

            appendText("Random rewrite from sub-inputs: " + randomRewrite);

        }

      } // for pattern

    }

  }

  class ProcessNewInput_v3 extends Algorithm {

    run {

      q = trim(q);

      saveInput(q);

      BEAObject input = handleInput(this, q, false);

      new HTMLTabs tabs;

      {

        new StringBuffer toTab;

        Algorithm alg = new ProcessNewInput_v2;

        alg.q = q;

        alg.out = toTab;

        alg.runIt();

        tabs.add("Match input with patterns", toTab);

      }

      {

        new StringBuffer toTab;

        Algorithm alg = new SyntacticPatternFromInput;

        alg.q = q;

        alg.out = toTab;

        alg.runIt();

        tabs.add("Make syntactic pattern", toTab);

      }

      appendHTML(tabs.html());

    }

  }

  BEAObject allowStoringMatch(Algorithm algo, BEAObject input, BEAObject pattern, SS mapping) {

    BEAObject match = conceptWhereCI(BEAObject, type := "Match",

      +pattern, +input, +mapping);

    if (match != null)

      ret match with algo.appendHTML("Match exists: " + mod.beaLinkHTML(match));

    /*S url = appendParamsToURL(mod.baseLink + "/storeMatch", pattern := pattern.id, input := input.id);

    algo.appendHTML(

      hbuttonLink(appendParamsToURL(url, label := "good"), "Good match") + " "

      + hbuttonLink(appendParamsToURL(url, label := "bad"), "Bad match"));*/

    algo.appendHTML(joinWithSpace(

      storeMatchForm(input, pattern, "good"),

      storeMatchForm(input, pattern, "bad")));

    null;

  }

  S storeMatchForm(BEAObject input, BEAObject pattern, S label) {

    ret hinlinepostform(

      hhiddenMulti(

        pattern := pattern.id,

        input := input.id,

        +label,

        redirect := "")

        + hbutton(firstToUpper(label) + " match"),

      action := mod.baseLink + "/storeMatch",

      onsubmit := js_redirectAutoScroll2());

  }

  runnable class MakeSyntacticPatterns extends Algorithm {

    for (BEAObject p : mod.beaList("Pattern")) {

      appendText(p + " => " + makeSyntacticPattern(p));

    }

  }

  S syntacticPatternText(BEAObject p) {  

    ret simpleSpacesTrim_javaTok(angleBracketVarsToStars(p.text()));

  }

  // convert angle bracket var pattern to syntactic pattern

  BEAObject makeSyntacticPattern(BEAObject p) {

    if (p == null || !p.typeIs("Pattern") || empty(p.text())) null;

    //print("makeSyntacticPattern", p);

    S text = syntacticPatternText(p);

    BEAObject sp = uniqCI BEAObject(type := "Syntactic Pattern", +text);

    cset(p, syntacticPattern := sp);

    ret sp;

  }

  S processInputURL(S q) {

    ret addParamsToURL(mod.baseLink + "/query", +q, algorithm := "process input");

  }

  BEAObject handleInput(Algorithm algo, S q, S desc default "Input", bool showProcessLink) {

    algo.appendHTML(hSingleRowTable_withSpacing(

      "\*desc*/:",

      tt(htmlEncode_nlToBr_withIndents(q)),

      !showProcessLink ? "" : targetBlank_noFollow(processInputURL(q), "Process")));

    BEAObject input = mod.findInput(q);

    algo.appendHTML(inputHTML(input, desc, q));

    ret input;

  }

  // Link to or allow to save input object

  S inputHTML(BEAObject input, S desc default "Input", S q) {

    ret input != null

      ? "\*desc*/ exists: " + mod.beaLinkHTML(input)

      : !mod.requestAuthed() && !mod.allowAnonymousInputUpload ? null : hinlinepostform(

          hhiddenMulti(

            text := q,

            redirect := "")

          + hbutton("Save \*desc*/"),

          action := mod.baseLink + "/saveInput",

          onsubmit := js_setRedirect());

  }

  // Link to or allow to save syntactic pattern

  S syntacticPatternLinkHTML(BEAObject obj, S text, BEAObject fromInput default null) {

    S desc = "Syntactic pattern";

    ret obj != null

      ? mod.beaLinkHTML(obj)

      : !mod.requestAuthed() ? null : hinlinepostform(

          hhiddenMulti(

            +text,

            fromInput := conceptIDOrNull(fromInput),

            redirect := "")

          + hbutton("Save \*firstToLower(desc)*/"),

          action := mod.baseLink + "/saveSyntacticPattern",

          onsubmit := js_setRedirect());

  }

  runnable class SyntacticPatternFromInput extends Algorithm {

    //appendHTML(inputHTML(mod.findInput(q), q));

    BEAObject input = handleInput(this, q, false);

    if (countTokens(q) > 10) ret with appendText("Input too long to extract syntactic patterns");

    S q2 = dropPunctuation(q);

    new L<Map> data;

    for (S pat : sortedIC(gazelle_allSyntacticPatternsFromInput(q))) {

      SS mapping = matchInputWithSyntacticPattern(q2, pat);

      BEAObject obj = conceptWhereIC(BEAObject, type := "Syntactic Pattern", text := pat);

      /*appendHTML(htmlEncode2(pat) + " " + syntacticPatternLinkHTML(obj, pat)

        + (empty(mapping) ? "" : 

        hpre_htmlencode(indent(formatDoubleArrowMap_horizontallyAligned(mapping)))));*/

      Map row = litorderedmap(

        "Syntactic Pattern" := htmlEncode2(pat),

        "In database" := syntacticPatternLinkHTML(obj, pat, input));

      for (S key, S val : mapping)

        row.put("Argument " + key, htmlEncode2(val));

      data.add(row);

    }

    appendHTML(htmlTable2_noHtmlEncode(data));

  }

  class ApplyTextFunctions extends Algorithm {

    run {

      q = trim(q);

      saveInput(q);

      BEAObject input = handleInput(this, q, false);

      new LinkedHashSet<BEAObject> objectsMade;

      for (BEAObject f : mod.beaList("JavaX Standard Function")) pcall {

        addIfNotNull(objectsMade, reactFunctionWithInput(this, f, input));

      }

      appendHTML(h3("Results of function calls"));

      //appendHTML(ul_htmlencode(allToString(list(localConcepts, BEAObject))));

      /*HCRUD_Concepts<BEAObject> crudData = new(localConcepts, BEAObject);

      HCRUD crud = new(null, crudData);

      crud.unshownFields = litset("globalID", "mirrorPost");

      crudData.massageItemMapForList = (item, map) -> {

        BEAObject o = item/BEAObject;

        O value = o~.result;

        map.put("result", HTML(javaValueToHTML(value, identityHashSet())));

        //map.put("Save", HTML(hbutton()));

      };*/

      HCRUD crud = mod.makeCRUD(BEAObject, mod.currentReq());

      HCRUD_Concepts crudData = cast crud.data;

      crudData.listConcepts_firstStep = () -> objectsMade;

      appendHTML(crud.renderTable(true));

    }

  }

  class FindPath extends Algorithm {

    new BreadthFirstPathFinder_withLinkType<BEAObject, FieldIndicator> pathFinder;

    BEAObject start, dest;

    run {

      start = mod.beaGet(parseFirstLong(q));

      dest = mod.beaGet(parseSecondLong(q));

      if (start == null || dest == null)

        ret with appendText("Please supply object IDs of start and end");

      appendHTML("Looking for path from " + mod.beaToHTML(start) + " to " + mod.beaToHTML(dest));

      pathFinder.getChildren = o -> cForwardAndBackRefsWithFieldIndicator BEAObject(o);

      pathFinder.add(start);

      while (!pathFinder.nodeReached(dest) && pathFinder.step()) {}

      //appendHTML(hcomment(struct(pathFinder)));

      LPair<BEAObject, FieldIndicator> path = pathFinder.examplePathWithTypes(start, dest);

      if (path == null)

        appendText("No path found!");

      else {

        appendHTML(b("Path found!"));

        appendHTML(ol(map(path, p -> {

          S html = mod.beaHTML(p.a);

          if (p.b == null) ret html;

          S arrow = ifThenElse(p.b.forward, ">>", "<<");

          ret html + " " + htmlEncode2(

            joinNemptiesWithSpace(arrow, p.b.field, arrow));

        })));

      }

    }

  }

  class ExecuteTextQuery extends Algorithm {

    run {

      saveInput(q = trim(q));

      appendHTML(h3("Result"));

      appendHTML(htmlEncode2(strOrEmpty(executeTextQuery(q))));

    }

  }

  class InputSummary extends Algorithm {

    run {

      BEA mainInput = saveInput(q = trim(q));

      Cl<BEA> inputs = conceptsWhere BEAObject(type := "Input", text := q);

      if (l(inputs) > 1)

        appendHTML("Note: Input appears " + l(inputs) + " times in the database: "

          + joinWithSpace(map(o -> mod.beaToHTML(o), inputs)));

      for (BEA input : inputs) {

        if (input != mainInput) {

          appendHTML(hr());

          appendHTML("Input: " + input);

        }

        // rewrites

        Cl<BEA> rewrites = mod.beaBackRefs(input, "Rewrite");

        if (empty(rewrites)) appendHTML(h3("No rewrites"));

        else

          for (int i, BEA r : unpair iterateWithIndex1(rewrites))

            appendHTML("Rewrite " + i + ": " + ahref(mod.beaURL(r), tok_dropCurlyBrackets(r.text())));

        // matches

        //beaBackRefs("Match

      }

    }

  }

  class ApplyPatternList extends Algorithm {

    S additionalInputs() {

      ret hselect("patternListID", mapToKey conceptID(mod.beaList("Pattern List")),

        req.get("patternListID"));

    }

    run {

      BEAObject input = handleInput(this, q = trim(q), false);

      BEA patternList = mod.beaGet patternListID(req);

      if (patternList == null) ret;

      L<BEA> patterns = cast cget patterns(patternList);

      appendHTML("Pattern list: " + mod.beaHTML(patternList));

      fOr (BEA pattern : patterns) {

        appendText("Trying pattern " + pattern);

        checkPattern(this, input, q, pattern);

      }

    }

  }

  class ApplyAlgorithmObject extends Algorithm {

    S additionalInputs() {

      ret hselect("algorithmID", mapToKey conceptID(

          concatLists(

            wideningListCast Concept(list BEARegExpReplacement()),

            wideningListCast Concept(conceptsWhereIC UserPost(type := "JavaX Code (HTML Bot)"))

          )),

        req.get("algorithmID"));

    }

    void runLiveMode() {

      //BEAObject input = handleInput(this, q = trim(q), false);

      Concept algo = getConcept(parseLongOpt(req.get("algorithmID")));

      //appendText("Test test. Algorithm: " + algo);

      appendHTML("Algorithm object: " + htmlEncode2_nlToBr(str(algo)));

      if (algo cast UserPost) {

        //O result = algObject.apply(q);

        O result = mod.callHtmlBot_dropMadeByComment(algo.id, req.params());

        appendText("Result: " + result);

      } else if (algo cast BEARegExpReplacement) {

        //appendHTML("Algorithm object: " + mod.beaHTML(algObject));

        O result = algo.apply(q);

        appendText("Result: " + result);

      } else if (algo == null)

        appendText("Algorithm object not found");

      else

        appendText("Unknown algorithm object type: " + _className(algo));

    }

  }

  class ApplyRegExpReplacement > Algorithm {

    run {

      BEARegExpReplacement regExpReplacement = cast mod.beaGet(q);

      if (regExpReplacement == null) ret with appendText("Please enter reg exp replacement ID");

      appendHTML("Applying " + beaHTML(regExpReplacement));

      for (BEA input : sortedByConceptIDDesc(mod.beaList("Input"))) {

        S text = input.text();

        S replaced = regExpReplacement.apply(text);

        if (neq(text, replaced)) {

          appendHTML(hr());

          appendText("   " + text + "\n=> " + replaced);

          appendPossibleRewrite(this, input, replaced, litmap(

            f_regularExpression := regExpReplacement.id,

            metaInfo_regularExpression := "concept"));

        }

      }

    }

  }

  // prints instead of rendering to page

  class BackEndAlgorithm > Algorithm {

    S prefix() { ret shortClassName(this) + ": "; }

    @Override void appendText(S s) { printWithIndent(prefix(), s); }

    @Override void appendHTML(S s) { appendText(htmlDecode_dropTags(s)); }

    @Override void addNonMatch(Runnable r) { r.run(); }

  }

  class MatchInputWithAllPatternLists > Algorithm {

    run {

      BEAObject input = saveInput(q);

      for (BEA patternList : sortedByConceptIDDesc(beaList("Pattern List")))

        reactInputWithPatternList(this, input, q, patternList, 1, 2);

    }

  }

  class RunTestCasesOnCodePost extends Algorithm {

    S additionalInputs() {

      MultiSet<S> ms = distinctCIFieldValuesOfConcepts_multiSet(BEAObject, "type");

      LS types = filter(pairsA(multiSetToPairsByPopularity(ms)), 

        type -> cic(type, "test case"));

      ret "Test case type: " + hselect_list("testCaseType", types, req.get("testCaseType"))

        + " " + hselect("algorithmID", mapToKey conceptID(

          conceptsWhereIC UserPost(type := "JavaX Code (HTML Bot)")),

        req.get("algorithmID"));

    }

    run {

      //BEAObject input = handleInput(this, q = trim(q), false);

      S testCaseType = req.get("testCaseType");

      Concept algo = getConcept(parseLongOpt(req.get("algorithmID")));

      //appendText("Test test. Algorithm: " + algo);

      Cl<BEA> testCases = beaList(testCaseType);

      appendText(n2(testCases, "test case") + " found for type " + quote(testCaseType));

      appendHTML("Algorithm object: " + htmlEncode2_nlToBr(str(algo)));

      if (!algo instanceof UserPost) ret;

      new Scorer scorer;

      for ping (BEA testCase : testCases) {

        MapSO fields = toCIMap(objectToMap(testCase));

        S input = str(getAny(fields, "input"));

        S expected = str(getAny(fields, "output", "result"));

        S result = str(mod.callHtmlBot_dropMadeByComment(algo.id, litmap(q := input)));

        bool ok = eq(result, expected);

        appendText((ok ? "OK" : "BAD") + ": Input: " + quote(input) + ", expected output: " + quote(expected)

          + ", result: " + quote(result));

        scorer.add(ok);

      }

      appendText(str(scorer));

    }

  }

  // add algorithms here

  void reactInputWithPatternList(Algorithm algo, BEA input, S text, BEA patternList, int level, int recursionLevels) {

    Pair<BEA, SS> match = matchInputWithPatternList(text, patternList);

    algo.appendHTML(hr());

    algo.appendHTML(joinNemptiesWithColon(

      level == 1 ? beaHTML(patternList) : null,

      stringIf(match == null, "No match")));

    if (match != null) {

      BEA pattern = match.a;

      SS mapping = match.b;

      algo.appendHTML("Matched as " + beaHTML(pattern));

      algo.appendText(renderMappingAsText(mapping));

      if (input != null) {

        BEA matchObj = uniqCI(algo.localConcepts, BEA,

          type := "Match",

          +input,

          +patternList,

          winningPattern := pattern,

          +mapping);

        algo.appendHTML("Match saved as: " + beaHTML(matchObj));

      }

      if (level < recursionLevels) {

        algo.appendHTML_raw("<ul>");

        for (S key, subInput : mapping) {

          if (!eqic(key, text)) {

            algo.appendHTML_raw("<li>");

            algo.appendText("Processing sub-Input: " + subInput);

            reactInputWithPatternList(algo, null, subInput, patternList, level+1, recursionLevels);

            algo.appendHTML_raw("</li>");

          }

        }

        algo.appendHTML_raw("</ul>");

      }

    }

  }

  int patternSpecificity(BEAObject pattern) {

    ret countWordsWithoutAngleBracketedAndStars(pattern.text());

  }

  L<BEAObject> syntacticPatternsSortedBySpecificityDesc() {

    ret sortedByCalculatedFieldDesc patternSpecificity(mod.beaList("Syntactic Pattern"));

  }

  O executeTextQuery(S s) {

    new Matches m;

    if (match3_withIntsAndIDs("all references in field <id> of object <int>", s, m)) {

      O value = cget(mod.beaGet($2), $1);

      ret value instanceof Cl ? instancesOf BEAObject((Cl) value)

        : value instanceof BEAObject ? ll((BEAObject) value)

        : ll();

    }

    ret new QueryNotUnderstood(s);

  }

  srecord QueryNotUnderstood(S s) {}

  S beaToHTML(BEA o) { ret mod.beaToHTML(o); }

  S beaHTML(BEA o) { ret mod.beaHTML(o); }

  S beaURL(BEA o) { ret mod.beaURL(o); }

  bool isSyntacticPattern(BEA pat) {

    ret pat != null && pat.typeIs("Syntactic pattern");

  }

  S patternTextWithAngleBracketVars(BEA pat) {

    S text = pat.text();

    ret isSyntacticPattern(pat) ? starsToAngleBracketedNumbers(text) : text;

  }

  bool isCommentOrFeedback(BEA o) {

    ret o != null && o.typeIsOneOf("Comment", "Useful", "Feedback");

  }

  S feedbackHTML(BEA o) {

    Cl<BEA> l = filter isCommentOrFeedback(mod.beaBackRefs(o));

    ret joinWithBR(map(l, c -> c.typeIs("Comment")

      ? ahref(beaURL(c), htmlEncode2(c.text()))

      : beaHTML(c)));

  }

  S bestInputURL(S text) {

    text = trim(text);

    BEA input = mod.findInput(text);

    ret input != null ? beaURL(input) : processInputURL(text);

  }

  // also shaded

  S bestInputHTML(S text) {

    if (empty(text)) ret "";

    ret ahref(bestInputURL(text), span(htmlEncode2(text),

      style := "background-color: #ccc"));

  }

  S performRewrite(BEA rewrite, SS mapping) {

    S rewriteText = starsToAngleBracketedNumbers(rewrite.text());

    ret replaceAngleBracketVars(rewriteText, mapping);

  }

  PairS exampleForPatternRewrite(BEA pattern, BEA rewrite) {

    S patText = patternTextWithAngleBracketVars(pattern);

    Set<S> vars = collectAngleBracketVars(patText);

    SS mapping = mapToValues_ciMap(vars, var -> "$" + var);

    ret pair(replaceAngleBracketVars(patText, mapping),

      performRewrite(rewrite, mapping));

  }

  /* if there is a regular expression

   * and an input

   * and the regular expression matches the input

   * generate a sub-input with the matched part

   */

  record RegExpToPossibleSubInputs(BEA input, BEA regExp) extends Algorithm {

    run {

      S text = input.text();

      S pat = regExp.text();

      for (IntRange r : regexpFindRangesIC(pat, text))

        appendHTML(beaHTML(csetAndReturn(uniq(localConcepts, BEA,

          type := "Possible Sub-Input",

          +input,

          characterRange := r,

          text := substring(text, r)),

          +regExp,

          byAlgorithm := shortName(this))));

    }

  }

  void appendPossibleRewrite(Algorithm algo, BEA input, S text, Map moreInfo default null) {

    // look for existing rewrite

    BEAObject existing = input == null ?: conceptWhereIC BEAObject(

      type := "Rewrite", +input, +text,

      label := "good"

    );

    if (existing != null)

      algo.appendHTML("Rewrite exists: " + mod.beaLinkHTML(existing));

    else if (input != null)

      algo.appendHTML(

        joinWithSpace(map(ll("good", "bad"), label ->

          hinlinepostform(

            hhiddenMulti(

              action := "create",

              f_type := "Rewrite",

              f_text := text,

              f_input := input.id, metaInfo_input := "concept",

              f_label := label,

              redirectAfterSave := ""/*algo.req.uriWithParams()*/)

            + hhiddenMulti(moreInfo)

            + hbutton(firstToUpper(label) + " rewrite",

              onclick := js_redirectAutoScroll2()),

            action := mod.crudLink(BEA)))));

  }

  BEA reactFunctionWithInput(Algorithm algo, BEA f, BEA input) {

    S fName = getString name(f);

    if (!isIdentifier(fName)) null;

    if (!isSafeStandardFunction(fName))

      ret null with algo?.addNonMatch(r { algo.appendText("Function " + fName + " not safe"); });

    BEAObject o = cnew(algo.localConcepts, BEAObject,

      type := "Function Result",

      function := f,

      functionCalled := fName,

      +input,

      calculating := true);

    print("Function result: " + o);

    try {

      O holder = getStandardFunctionHolder(fName);

      O result = evalWithTimeoutOrFail(10.0, () -> call(holder, fName, input.text()));

      // convert string collections to list

      if (!result instanceof L && isStringCollection(result)) result = asList((Cl) result);

      if (result == null)

        cset(o, resultType := "null");

      else if (result instanceof S)

        cset(o, resultType := "string", +result);

      else if (isStringList(result))

        cset(o, resultType := "list of strings", +result);

      else

        cset(o, result := "other" + appendRoundBracketed(className(result)),

          result := shorten(1000, str(result)));

    } catch e {

      cset(o, resultType := "error", error := getStackTrace(e));

    }

    cset(o, calculating := null);

    ret o;

  }

  BEA convertSyntacticToSemanticMatch(BEA match, BEA semanticPattern) {

    BEA syntacticPattern = beaGet pattern(match);

    BEA input = beaGet input(match);

    S pat2 = syntacticPattern.text();

    S pat1 = semanticPattern.text();

    new LS vars;

    S pat2b = angleBracketVarsToStars(pat1, vars);

    S warning = eq(pat2, pat2b) ? null

      : "Patterns don't match: " + quote(pat2) + " / " + quote(pat2b);

    SS mapping = cast cget mapping(match);

    SS mapping2 = mapKeys(mapping, key -> or(_get(vars, parseInt(key)-1), key));

    mapping2 = putKeysFirst(vars, mapping2); // reorder variables in order of appearance in input

    BEA match2 = uniqCI BEA(

      type := "Match",

      +input,

      pattern := semanticPattern,

      fromSyntacticMatch := match);

    cset(match2,

      mapping := mapping2);

    ret match2;  

  }

  Cl<BEA> convertSyntacticToSemanticMatches(BEA match) {

    new L<BEA> out;

    BEA syntacticPattern = beaGet pattern(match);

    for (BEA semanticPattern : mod.beaList("Pattern", +syntacticPattern)) pcall {

      out.add(convertSyntacticToSemanticMatch(match, semanticPattern));

    }

    ret out;

  }

  Cl<BEA> convertSyntacticToSemanticMatchesForWholePattern(BEA pattern) {

    if (pattern.typeIs("Syntactic Pattern"))

      // This is huge - uses all semantic patterns

      ret concatMap_lists convertSyntacticToSemanticMatches(mod.beaList("Match", +pattern));

    else {

      new L<BEA> out;

      BEA syntacticPattern = beaGet syntacticPattern(pattern);

      if (syntacticPattern != null)

        for (BEA match : mod.beaList("Match", pattern := syntacticPattern))

          out.add(convertSyntacticToSemanticMatch(match, pattern));

      ret out;

    }

  }

  BEA mapMethodLike beaGet(S field, BEA o) { ret mod.beaGet(field, o); }

  Cl<BEA> beaList(S type, O... params) { ret mod.beaList(type, params); }

  int simpleObjectScore(BEA o) {

    int score = 0;

    if (eqic_gen(o~.label, "good")) ++score;

    else if (eqic_gen(o~.label, "bad")) --score;

    Cl<BEA> l = findBackRefsWithField BEA(object := o);

    score += countWhereCI(l, type := "Useful");

    score -= countWhereCI(l, type := "Bad");

    ret score;

  }

  L<BEA> sortedBySimpleObjectScore(int minScore, Cl<BEA> l) {

    ret sortedByCalculatedFieldDesc simpleObjectScore(filter(l, o -> simpleObjectScore(o) >= minScore));

  }

  BEA createPatternListFromUsefulSyntacticPatterns() {

    L<BEA> patterns = sortedBySimpleObjectScore(0, beaList("Syntactic Pattern"));

    ret mod.autoMigrateToCustomClass(uniqCI BEA(type := "Pattern List",

      byFunction := "createPatternListFromUsefulSyntacticPatterns",

      +patterns));

  }

  Map<S, IF0<Algorithm>> algorithms() {  

    ret litorderedmap(

      "Process new input v3" := (IF0) () -> new ProcessNewInput_v3,

      "Process new input v2" := (IF0) () -> new ProcessNewInput_v2,

      "Process new input" := (IF0) () -> new ProcessNewInput,

      "Run pattern" := (IF0) () -> new RunPattern,

      "Find rewrites for match" := (IF0) () -> new FindRewrites,

      "Reverse input (test algorithm)" := (IF0) () -> new ReverseInput,

      "Timeout test" := (IF0) () -> new TimeoutTest,

      "Syntactic pattern from input" := (IF0) () -> new SyntacticPatternFromInput,

      "Apply all text functions" := (IF0) () -> new ApplyTextFunctions,

      "Find Path" := (IF0) () -> new FindPath,

      "Execute text query" := (IF0) () -> new ExecuteTextQuery,

      "Make syntactic patterns for all patterns in DB" := (IF0) () -> new MakeSyntacticPatterns,

      "Input summary" := (IF0) () -> new InputSummary,

      "Apply pattern list" := (IF0) () -> new ApplyPatternList,

      "Apply regular expression replacement to all inputs" := (IF0) () -> new ApplyRegExpReplacement,

      "Match input with all pattern lists" := (IF0) () -> new MatchInputWithAllPatternLists,

      "Apply algorithm object" := (IF0) () -> new ApplyAlgorithmObject,

      "Run test cases on code post" := (IF0) () -> new RunTestCasesOnCodePost,

      // add algorithms here

    );

  }

  void putAlgorithmsInDatabase {

    for (S name : keys(algorithms()))

      uniq BEA(type := "Classic Query Algorithm", +name);

  }

} // end of Calculations