sclass BEACalculations {
  GazelleBEA mod;  
  *(GazelleBEA *mod) {}

  transient int liveOutputDelay = 1000;
  transient double algoTimeout = 30.0;
  
  void reactAllInputsWithSomePatterns() {
    for (BEAObject input : mod.beaObjectsOfType("input")) {
      new ReactInputWithSomePatterns r;
      r.input = input;
      r.ccOut = db_mainConcepts();
      r.max = 10;
      r.start();
      stepAllWithTimeout(10.0, r);
    }
  }

  void reactAllInputsWithAllSyntacticPatterns() {
    for ping (BEA input : mod.beaList("input")) {
      for ping (BEA pattern : mod.beaList("Syntactic Pattern")) {
        continue if patternAlwaysMatches(pattern);
        reactInputWithPattern(input, pattern);
      }
    }
  }

  void reactAllInputsWithPattern(BEA pattern) {
    for ping (BEA input : mod.beaList("input"))
      reactInputWithPattern(input, pattern);
  }

  bool patternAlwaysMatches(BEA pat) {
    ret isTrue_getOpt alwaysMatches(pat);
  }
  
  class ReactInputWithSomePatterns implements Steppable {
    BEAObject input;
    Concepts ccOut;
    int max = 3, n;
    Iterator<BEAObject> patterns;

    void start {
      n = countConceptsWhereIC(ccOut, BEAObject, +input, type := "match");
      patterns = iterator(sortByConceptID(conceptsWhereIC BEAObject(type := "Pattern")));
    }
    
    public bool step() {
      if (n >= max) false;
      if (!patterns.hasNext()) false;
      if (reactInputWithPattern(input, patterns.next(), ccOut) != null)
        ++n;
      true;
    }
  }

  // (patern, mapping)
  Pair<BEA, SS> matchInputWithPatternList(S text, BEA patternList) {
    L<BEA> patterns = cast cget patterns(patternList);
    fOr (BEA pattern : patterns) {
      SS mapping = matchInputWithPattern(text, pattern);
      if (mapping != null) ret pair(pattern, mapping);
    }
    null;
  }

  // we don't do this anymore
  new ThreadLocal<Bool> matchInput_droppedPunctuation;

  SS matchInputWithPattern(BEAObject input, BEAObject pattern) {
    S text = getString text(input);
    ret matchInputWithPattern(text, pattern);
  }
    
  SS matchInputWithPattern(S text, BEAObject pattern) {
    S pat = getString text(pattern);
    bool punctuationOnly = isTrue_getOpt punctuationOnly(pattern);

    if (pattern.typeIs("Syntactic Pattern"))
      ret matchInputWithSyntacticPattern(text, pat, punctuationOnly);
    else if (eqic_gen(pattern~.subType, "PhraseCache"))
      ret phraseCache(pat, text) ? litmap() : null;
    else
      ret matchInputWithPattern(text, pat, punctuationOnly);
  }

  SS matchInputWithSyntacticPattern(S text, S pat, bool punctuationOnly default false) {
    pat = starsToAngleBracketedNumbers(pat);
    ret matchInputWithPattern(text, pat, punctuationOnly);
  }
      
  SS matchInputWithPattern(S text, S pat, bool punctuationOnly default false) {
    SS mapping = flexMatchAngleBracketVarsIC_honorPunctuation_noBrackets_first(pat, text);
    if (mapping != null || punctuationOnly) ret mapping;
    /*matchInput_droppedPunctuation.set(true);
    ret flexMatchAngleBracketVarsIC_first(pat, text);*/
    null; // not doing the implicit punctuation drop anymore (it's now an explicit pattern)
  }
  
  BEAObject reactInputWithPattern(BEAObject input, BEAObject pattern, Concepts ccOut default db_mainConcepts()) {
    if (input == null || pattern == null) null;
    matchInput_droppedPunctuation.set(false);
    SS mapping = matchInputWithPattern(input, pattern);
    //printVars_str("reactInputWithPattern", +input, +pattern, +mapping);
    if (mapping == null) null;
    BEA match = uniqCI(ccOut, BEA,
      type := "Match",
      +input,
      +pattern);
    cset(match,
      +mapping,
      droppedPunctuation := trueOrNull(matchInput_droppedPunctuation!),
    );
    ret match;
  }
  
  O serveAnalyzeInput(GazelleBEA.Req req) {
    new Concepts ccOut;
    BEAObject input = getConcept BEAObject(parseLong(req.get("id")));
    if (input == null)
      input = cnew(ccOut, BEAObject, type := "Input", text := req.get("q"));
    S text = getString text(input);
    new ReactInputWithSomePatterns r;
    r.input = input;
    r.ccOut = ccOut;
    r.max = 1000;
    r.start();
    stepAllWithTimeout(10.0, r);
    Cl matches = conceptsWhereIC(ccOut, BEAObject, type := "Match");
    ret h1_title("Analyze Input")
      + p("Input: " + b(htmlEncode_nlToBr_withIndents(text)))
      + p(empty(matches) ? "No matches" : "Matches")
      + ul_htmlEncode(matches);
  }
  
  O serveQueryPage(GazelleBEA.Req req, bool withFrame default false) {
    S q = req.get("q"), algorithm = req.get("algorithm");
    bool dontRun = eq(req.get("dontRun"), "1");
    bool liveOutput = eq(req.get("liveOutput"), "1");
    if (liveOutput) req.noSpam();

    if (eqic(algorithm, "process input"))
      algorithm = "Process new input v3";

    //if (liveOutput) ret jsonEncode(litmap(answer := "Live output demo: " + htmlEncode2(q)));

    Map<S, IF0<Algorithm>> algorithms = algorithms();

    S output = empty(algorithm) || dontRun ? null : doAlgorithm(algorithms, algorithm, q, liveOutput);

    if (liveOutput)
      ret jsonEncode(litorderedmap(answer := output));
    
    S right = withFrame ? "" : mod.html_loggedIn();
    
    Algorithm alg = callF(lookupPossiblyCI(algorithms, algorithm));
    if (alg != null) {
      alg.q = q;
      alg.req = mod.currentReq();
    }
    
    S centered = 
        (withFrame ? "" : htitle_h2(ahref(mod.baseLink + "/", "Gazelle") + " Query"
        + htmlEncode2((empty(q) ? "" : ": " + q)
        + appendSquareBracketed(algorithm))))
      + hscript(replaceVars([[
          var sugLoading = false, sugTriggerAgain = false;

          function sugTrigger() {
            console.log("sugTrigger");
            if (sugLoading) { sugTriggerAgain = true; return; }

            // get form data as JSON
            
            var data = {
              "liveOutput" : "1",
              "q": $("textarea[name=q]").val(),
              "algorithm": $("select[name=algorithm]").val(),
              "algorithmID": $("select[name=algorithmID]").val(), // XXX: do this for all additional alg parameters
            };
            
            const url = "/query";
            console.log("Loading " + url + " with " + data);
            sugLoading = true;
            $.post(url, data,
              function(result) {
                console.log("Suggestor result: " + result);
                const answer = !result ? "" : JSON.parse(result).answer;
                if (answer) {
                  $("#liveResult .sectionContents").html(answer);
                  $("#liveResult").show();
                } else
                  $("#liveResult").hide();
                //$("#suggestorResult").html(answer ? "Suggestor says: " + answer : "");
              }
            ).always(function() {
              console.log("sug loading done");
              setTimeout(function() {
                sugLoading = false;
                if (sugTriggerAgain) { sugTriggerAgain = false; sugTrigger(); }
              }, liveOutputDelay);
            });
          }

          $(document).ready(function() {
            $("textarea[name=q], input[name=q], select[name=algorithm]").on('input propertychange change', sugTrigger);
            sugTrigger();
          });
        ]], +liveOutputDelay))
      + hform(p("Query (Ctrl+Enter): "
          + htextarea(q,
              id := "mainInput", 
              name := "q", 
              class := "auto-expand",
              style := "width: 300px",
              onkeydown := jquery_submitFormOnCtrlEnter(),
              autofocus := dontRun || empty(q) ? true : null // autofocus for live output mode
            ) + " "
          /*+ hjs([[$(document).ready(function() {
              autoExpandTextArea(document.getElementById("mainInput"));
          });]])*/
        //+ htextfield(+q, style := "width: 300px; text-align: center", autofocus := true)
        + (alg == null ? "" : alg.additionalInputs())
        + " &nbsp; Algorithm: " 
        + hselect_list(keys(algorithms), algorithm, name := "algorithm") + " &nbsp; " + hbutton("Go")));
    
    S content = (empty(right) ? "" : div_alignRight(right))
      + div_center(centered)
      + htitledSectionWithDiv("Live Result", "",
         id := "liveResult", style := "display: none",
         innerDivStyle := "max-height: 150px; overflow: auto")
      + (output == null ? "" : h3(algorithm) + output);
      
    if (withFrame) {
      req.framer.addInHead(hjs_handleScrollToAnchor());
      req.framer.add(content);
      ret mod.completeFrame(req);
    } else
      ret hhtml(hhead(
          hmobilefix() + hsansserif() + 
          loadJQuery2() + 
          hjs_handleScrollToAnchor() +
          hjs_autoExpandingTextAreas() +
          mod.webSocketHeadStuff(req) +
          hNotificationPopups())
        + hbody(content));
  }
  
  S doAlgorithm(Map<S, IF0<Algorithm>> algorithms, S algorithm, S q, bool liveMode) {
    GazelleBEA.Req req = mod.currentReq();
    StringBufferWithMaxSize out = new(1000000); // TODO: make sure HTML tags are closed properly
    
    IF0<Algorithm> algo = lookupPossiblyCI(algorithms, algorithm);
    if (algo == null)
      out.append("Algorithm not found: " + algorithm);
    else {
      if (isB(evalWithTimeout(algoTimeout, r {
        temp mod.enter();
        temp tempSetTL(mod.currentReq, req);
        Algorithm alg = algo!;
        alg.liveMode = liveMode;
        alg.out = out;
        alg.q = q;
        alg.req = mod.currentReq();
        alg.runIt();
      })))
        out.append("\n[algorithm timeout]");
    }
    
    ret str(out);
  }
  
  class Algorithm implements Runnable {
    S q;
    GazelleBEA.Req req;
    Appendable out;
    //Appendable outPrio = new StringBufferWithMaxSize(10000);
    bool liveMode;
    Concepts localConcepts = db_mainConcepts(); //= childConcepts(db_mainConcepts());
    L<Runnable> nonMatches; // null so not filled by default

    // override for normal mode
    run {}

    // override for live mode
    void runLiveMode {}

    /*void cleanForStorage {
      out = outPrio = null;
    }*/

    // overridable
    S additionalInputs() { ret ""; }
    
    void appendHTML(Appendable out default this.out, S html) ctex {
      if (empty(html)) ret;
      html = wrapInDiv(html, style := "margin-top: 0.5em");
      out.append(html).append("\n");
    }

    void appendHTML_raw(S html) ctex { out.append(html).append("\n"); }
    
    void appendText(Appendable out default this.out, S text) ctex {
      if (empty(text)) ret;
      out.append(hpre_htmlencode(rtrim(text)));
    }
    
    void appendTextBold(Appendable out default this.out, S text) ctex {
      if (empty(text)) ret;
      out.append(b(hpre_htmlencode(rtrim(text))));
    }
    
    void allowSuggestingAnswer(BEAObject input) {
      if (input == null) ret;
      S text = input.text();
        
      S url = mod.baseLink + "/saveAnswer";
      appendHTML(
        hpostform("Suggest an answer to " + htmlEncode_quote(text) + ": "
          //+ hinputfield(+text, style := "width: 300px")
          + htextarea(text, id := "suggestAnswer", name := "text", class := "auto-expand",
            style := "width: 300px")
          + hhidden(inputID := input.id)
          + hhidden(redirect := "")
          + " " + hsubmit("Save answer")
          + " [Optional rewrite type: " + hinputfield("rewriteType", style := "width: 100px") + " ]",
          action := url,
          onsubmit := js_setRedirect()));
    }

    void allowSuggestingPattern(BEAObject input) {
      if (input == null) ret;
      S text = input.text();

      S url = mod.baseLink + "/savePattern";
      appendHTML(
        hpostform("Suggest a pattern (with angle bracket vars): "
          + htextarea(text, name := "text", class := "auto-expand",
            style := "width: 300px")
          + hhidden(fromInput := conceptIDOrNull(input))
          + hhidden(userTyped := true)
          + hhidden(redirect := "")
          + " " + hsubmit("Save pattern"),
          action := url,
          onsubmit := js_setRedirect()));
    }

    void allowSuggestingSyntacticPattern(BEAObject input) {
      if (input == null) ret;
      S text = input.text();

      S url = mod.baseLink + "/saveSyntacticPattern";
      appendHTML(
        hpostform("Suggest a syntactic pattern (with * vars): "
          + htextarea(text, name := "text", class := "auto-expand",
            style := "width: 300px")
          + hhidden(fromInput := conceptIDOrNull(input))
          + hhidden(userTyped := true)
          + hhidden(redirect := "")
          + " " + hsubmit("Save syntactic pattern"),
          action := url,
          onsubmit := js_setRedirect()));
    }

    BEA saveInput(S q) {
      if (userAgentIsBot(beaMod().currentUserAgent())) null;
      Pair<BEA, Bool> p = uniqCI2 BEA(type := "Input", text := q);
      BEA input = p.a;
      mod.saveUserAgent(input);
      
      if (p.b) {
        csetIfUnset(input, createdBy := mod.user(mod.currentReq()));
        appendHTML("Input saved");
      }
      ret input;
    }  

    void runNonMatches {
      if (nempty(nonMatches)) { appendHTML(hr()); callFAll(nonMatches); }
    }

    void addNonMatch(Runnable r) { add(nonMatches, r); }

    void runIt {
      if (liveMode) runLiveMode(); else run();
      runNonMatches();
    }
  } // end of Algorithm
    
  class ReverseInput extends Algorithm {
    run { appendText(reversed(q)); }
    void runLiveMode { run(); }
  }
  
  class TimeoutTest extends Algorithm {
    run { sleepSeconds(120); }
  }
  
  class RunPattern extends Algorithm {
    run {
      BEAObject pattern = mod.beaGet(parseFirstLong(q));
      if (pattern == null) ret with appendText("Pattern not found");
      appendText("Pattern: " + pattern);
      for (BEAObject input : mod.beaList("Input")) {
        S text = input.text();
        SS mapping = matchInputWithPattern(input, pattern);
        if (mapping == null)
          addNonMatch(r {
            appendText("Checking Input: " + input + " - No match");
          });
        else {
          S result = "\n" + renderMappingAsText(mapping);
          appendText("Checking Input: " + input + result);
          allowStoringMatch(this, input, pattern, mapping);
        }
      }
    }
  }

  S renderMappingAsText(SS mapping) {
    ret indent(formatDoubleArrowMap_horizontallyAligned(mapping));
  }

  class FindRewrites extends Algorithm {
    run {
      // argument is ID of a match
      BEAObject match = mod.beaGet(parseFirstLong(q));
      if (match == null || !match.typeIs("Match")) ret with appendText("Match not found");
      BEAObject pattern = cast cget pattern(match);
      BEAObject input = cast cget input(match);
      SS mapping = cast get mapping(match);
      appendText("Match: " + match);
      appendText("Input: " + input);
      appendText("Pattern: " + pattern);
      appendText("Mapping:\n"
        + renderMappingAsText(mapping));
        
      appendRewrites(this, input, input.text(), pattern, match, mapping);
    }
  }
  
  // match and input can be null, pattern has to be set
  LS appendRewrites(Algorithm algo, BEAObject input, S inputText, BEAObject pattern, BEAObject match, SS mapping) {
    Cl<BEAObject> rewrites = mod.beaBackRefs(pattern, "Rewrite");
    new LS texts;
    for (BEAObject r : rewrites) {
      S text = r.text();
      S textWithVars = text;
      if (pattern.typeIs("Syntactic pattern"))
        textWithVars = starsToAngleBracketedNumbers(text);
        
      S text2 = replaceAngleBracketVars_curly(textWithVars, mapping);
      algo.appendText(" Using pattern rewrite: " + text);
      algo.appendTextBold(
          "    " + inputText + "\n"
        + " => " + text2);
      texts.add(text2);

      // look for existing rewrite
      BEAObject existing = match == null || input == null ?: conceptWhereIC BEAObject(
        type := "Rewrite", +input, text := text2,
          +match, patternRewrite := r, label := "good"
      );
      
      if (existing != null)
        algo.appendHTML("Rewrite exists: " + mod.beaLinkHTML(existing));
      else if (match == null)
        algo.appendText("[Can't save, no match]");
      else if (input == null)
        algo.appendText("[Can't save, no input object]");
      else
        algo.appendHTML(
          hinlinepostform(
            hhiddenMulti(
              action := "create",
              f_type := "Rewrite",
              f_text := text2,
              f_input := input.id, metaInfo_input := "concept",
              f_match := match.id, metaInfo_match := "concept",
              f_patternRewrite := r.id, metaInfo_patternRewrite := "concept",
              f_label := "good")
            + hbutton("Good rewrite"),
            target := "_blank",
            action := mod.crudLink(BEAObject)));
    }
    ret texts;
  }
  
  class ProcessNewInput extends Algorithm {
    run {
      BEAObject input = handleInput(this, q = trim(q), false);

      for (BEAObject pattern : mod.beaListAny("Pattern", "Syntactic Pattern")) {
        checkPattern(this, input, q, pattern);
      }
    }
  }

  void checkPattern(Algorithm algo, BEA input, S q, BEA pattern) {
    SS mapping = matchInputWithPattern(q, pattern);
    if (mapping == null)
      algo.addNonMatch(r {
        algo.appendText("Checking Pattern: " + pattern + " - No match");
      });
    else {
      S result = "\n" + renderMappingAsText(mapping);
      algo.appendText("Checking Pattern: " + pattern + result);
      if (input == null) {
        //appendHTML("Click \"Save input\" first to store this match");
      } else
        allowStoringMatch(algo, input, pattern, mapping);
        
      appendRewrites(algo, input, q, pattern, null, mapping);
    }
  }
  
  class ProcessNewInput_v2 extends Algorithm {
    run {
      q = trim(q);
      saveInput(q);
      BEAObject input = handleInput(this, q, false);

      if (input != null) {
        Cl<BEAObject> rewrites = mod.beaBackRefs(input, "Rewrite");
        BEAObject rewrite = random(rewrites);
        if (rewrite != null)
          appendHTML("My answer: " + b(htmlEncode2(rewrite.text())) + " " + ahref(mod.beaShortURL(rewrite), "[source]"));
      }

      allowSuggestingAnswer(input);
      allowSuggestingPattern(input);
      allowSuggestingSyntacticPattern(input);

      for (BEAObject pattern : syntacticPatternsSortedBySpecificityDesc()) {
        continue if patternAlwaysMatches(pattern);
        SS mapping = matchInputWithPattern(q, pattern);
        if (mapping == null)
          addNonMatch(r {
            appendHTML("Checking Syntactic Pattern: " + mod.beaLinkHTML(pattern) + " - No match");
          });
        else {
          S result = "Grouped as: " + replaceAngleBracketVars_curly(starsToAngleBracketedNumbers(pattern.text()), mapping) + "\n"
            + renderMappingAsText(mapping);
          appendHTML(hr());
          appendHTML("Checking Syntactic Pattern: " + mod.beaLinkHTML(pattern));
          appendText(result);
          BEAObject match = null;
          if (input == null) {
            //appendHTML("Click \"Save input\" first to store this match");
          } else
            match = allowStoringMatch(this, input, pattern, mapping);
            
          MultiMap<S, BEAObject> subInputRewrites = ciMultiMap();
          
          for (S subInput : uniquifyCI(values(mapping)))
            if (!eqic(subInput, q)) {
              BEAObject subInputObj = handleInput(this, subInput, "Sub-Input", true);
              
              // find rewrites for sub-input
              if (subInputObj != null) {
                L<BEAObject> subRewrites = concatLists(
                  conceptsWhereCI(BEAObject, type := "Rewrite", input := subInputObj),
                  conceptsWhereCI(BEAObject, type := "Rewrite", isRewriteOf := subInputObj));
                for (BEAObject r : subRewrites)
                  appendHTML("&nbsp; Sub-Rewrite found: " + r);
                subInputRewrites.putAll(subInput, subRewrites);
              }
            }
          
          // show all rewrites for the pattern  
          appendRewrites(this, input, q, pattern, match, mapping);
          
          // apply some sub-rewrites randomly
          SS rewrittenMapping = mapValues(mapping,
            s -> {
              BEAObject rewrite = random(subInputRewrites.get(s));
              ret rewrite == null ? s : uncurly(rewrite.text());
            });
            
          S randomRewrite = replaceAngleBracketVars_curly(starsToAngleBracketedNumbers(pattern.text()), rewrittenMapping);
          if (neq(randomRewrite, q))
            appendText("Random rewrite from sub-inputs: " + randomRewrite);
        }
      } // for pattern
    }
  }
  
  class ProcessNewInput_v3 extends Algorithm {
    run {
      q = trim(q);
      saveInput(q);
      BEAObject input = handleInput(this, q, false);

      new HTMLTabs tabs;

      {
        new StringBuffer toTab;
        Algorithm alg = new ProcessNewInput_v2;
        alg.q = q;
        alg.out = toTab;
        alg.runIt();
        tabs.add("Match input with patterns", toTab);
      }
      
      {
        new StringBuffer toTab;
        Algorithm alg = new SyntacticPatternFromInput;
        alg.q = q;
        alg.out = toTab;
        alg.runIt();
        tabs.add("Make syntactic pattern", toTab);
      }
      
      appendHTML(tabs.html());
    }
  }

  BEAObject allowStoringMatch(Algorithm algo, BEAObject input, BEAObject pattern, SS mapping) {
    BEAObject match = conceptWhereCI(BEAObject, type := "Match",
      +pattern, +input, +mapping);
    if (match != null)
      ret match with algo.appendHTML("Match exists: " + mod.beaLinkHTML(match));
      
    /*S url = appendParamsToURL(mod.baseLink + "/storeMatch", pattern := pattern.id, input := input.id);
    algo.appendHTML(
      hbuttonLink(appendParamsToURL(url, label := "good"), "Good match") + " "
      + hbuttonLink(appendParamsToURL(url, label := "bad"), "Bad match"));*/

    algo.appendHTML(joinWithSpace(
      storeMatchForm(input, pattern, "good"),
      storeMatchForm(input, pattern, "bad")));
      
    null;
  }

  S storeMatchForm(BEAObject input, BEAObject pattern, S label) {
    ret hinlinepostform(
      hhiddenMulti(
        pattern := pattern.id,
        input := input.id,
        +label,
        redirect := "")
        + hbutton(firstToUpper(label) + " match"),
      action := mod.baseLink + "/storeMatch",
      onsubmit := js_redirectAutoScroll2());
  }
  
  runnable class MakeSyntacticPatterns extends Algorithm {
    for (BEAObject p : mod.beaList("Pattern")) {
      appendText(p + " => " + makeSyntacticPattern(p));
    }
  }

  S syntacticPatternText(BEAObject p) {  
    ret simpleSpacesTrim_javaTok(angleBracketVarsToStars(p.text()));
  }
  
  // convert angle bracket var pattern to syntactic pattern
  BEAObject makeSyntacticPattern(BEAObject p) {
    if (p == null || !p.typeIs("Pattern") || empty(p.text())) null;
    //print("makeSyntacticPattern", p);
    S text = syntacticPatternText(p);
    BEAObject sp = uniqCI BEAObject(type := "Syntactic Pattern", +text);
    cset(p, syntacticPattern := sp);
    ret sp;
  }

  S processInputURL(S q) {
    ret addParamsToURL(mod.baseLink + "/query", +q, algorithm := "process input");
  }
  
  BEAObject handleInput(Algorithm algo, S q, S desc default "Input", bool showProcessLink) {
    algo.appendHTML(hSingleRowTable_withSpacing(
      "\*desc*/:",
      tt(htmlEncode_nlToBr_withIndents(q)),
      !showProcessLink ? "" : targetBlank_noFollow(processInputURL(q), "Process")));
      
    BEAObject input = mod.findInput(q);
    algo.appendHTML(inputHTML(input, desc, q));
    ret input;
  }

  // Link to or allow to save input object
  S inputHTML(BEAObject input, S desc default "Input", S q) {
    ret input != null
      ? "\*desc*/ exists: " + mod.beaLinkHTML(input)
      : !mod.requestAuthed() && !mod.allowAnonymousInputUpload ? null : hinlinepostform(
          hhiddenMulti(
            text := q,
            redirect := "")
          + hbutton("Save \*desc*/"),
          action := mod.baseLink + "/saveInput",
          onsubmit := js_setRedirect());
  }

  // Link to or allow to save syntactic pattern
  S syntacticPatternLinkHTML(BEAObject obj, S text, BEAObject fromInput default null) {
    S desc = "Syntactic pattern";
    ret obj != null
      ? mod.beaLinkHTML(obj)
      : !mod.requestAuthed() ? null : hinlinepostform(
          hhiddenMulti(
            +text,
            fromInput := conceptIDOrNull(fromInput),
            redirect := "")
          + hbutton("Save \*firstToLower(desc)*/"),
          action := mod.baseLink + "/saveSyntacticPattern",
          onsubmit := js_setRedirect());
  }

  runnable class SyntacticPatternFromInput extends Algorithm {
    //appendHTML(inputHTML(mod.findInput(q), q));
    BEAObject input = handleInput(this, q, false);

    if (countTokens(q) > 10) ret with appendText("Input too long to extract syntactic patterns");
    S q2 = dropPunctuation(q);
    new L<Map> data;
    for (S pat : sortedIC(gazelle_allSyntacticPatternsFromInput(q))) {
      SS mapping = matchInputWithSyntacticPattern(q2, pat);
      BEAObject obj = conceptWhereIC(BEAObject, type := "Syntactic Pattern", text := pat);

      /*appendHTML(htmlEncode2(pat) + " " + syntacticPatternLinkHTML(obj, pat)
        + (empty(mapping) ? "" : 
        hpre_htmlencode(indent(formatDoubleArrowMap_horizontallyAligned(mapping)))));*/

      Map row = litorderedmap(
        "Syntactic Pattern" := htmlEncode2(pat),
        "In database" := syntacticPatternLinkHTML(obj, pat, input));
      for (S key, S val : mapping)
        row.put("Argument " + key, htmlEncode2(val));
      data.add(row);
    }
    appendHTML(htmlTable2_noHtmlEncode(data));
  }

  class ApplyTextFunctions extends Algorithm {
    run {
      q = trim(q);
      saveInput(q);
      BEAObject input = handleInput(this, q, false);

      new LinkedHashSet<BEAObject> objectsMade;
      
      for (BEAObject f : mod.beaList("JavaX Standard Function")) pcall {
        addIfNotNull(objectsMade, reactFunctionWithInput(this, f, input));
      }
      
      appendHTML(h3("Results of function calls"));
        
      //appendHTML(ul_htmlencode(allToString(list(localConcepts, BEAObject))));

      /*HCRUD_Concepts<BEAObject> crudData = new(localConcepts, BEAObject);
      HCRUD crud = new(null, crudData);
      crud.unshownFields = litset("globalID", "mirrorPost");

      crudData.massageItemMapForList = (item, map) -> {
        BEAObject o = item/BEAObject;
        O value = o~.result;
        map.put("result", HTML(javaValueToHTML(value, identityHashSet())));
        //map.put("Save", HTML(hbutton()));
      };*/

      HCRUD crud = mod.makeCRUD(BEAObject, mod.currentReq());
      HCRUD_Concepts crudData = cast crud.data;
      crudData.listConcepts_firstStep = () -> objectsMade;
    
      appendHTML(crud.renderTable(true));
    }
  }

  class FindPath extends Algorithm {
    new BreadthFirstPathFinder_withLinkType<BEAObject, FieldIndicator> pathFinder;
    BEAObject start, dest;
      
    run {
      start = mod.beaGet(parseFirstLong(q));
      dest = mod.beaGet(parseSecondLong(q));
      if (start == null || dest == null)
        ret with appendText("Please supply object IDs of start and end");

      appendHTML("Looking for path from " + mod.beaToHTML(start) + " to " + mod.beaToHTML(dest));

      pathFinder.getChildren = o -> cForwardAndBackRefsWithFieldIndicator BEAObject(o);
      pathFinder.add(start);

      while (!pathFinder.nodeReached(dest) && pathFinder.step()) {}

      //appendHTML(hcomment(struct(pathFinder)));
      LPair<BEAObject, FieldIndicator> path = pathFinder.examplePathWithTypes(start, dest);
      
      if (path == null)
        appendText("No path found!");
      else {
        appendHTML(b("Path found!"));
        appendHTML(ol(map(path, p -> {
          S html = mod.beaHTML(p.a);
          if (p.b == null) ret html;
          S arrow = ifThenElse(p.b.forward, ">>", "<<");
          ret html + " " + htmlEncode2(
            joinNemptiesWithSpace(arrow, p.b.field, arrow));
        })));
      }
    }
  }

  class ExecuteTextQuery extends Algorithm {
    run {
      saveInput(q = trim(q));
      appendHTML(h3("Result"));
      appendHTML(htmlEncode2(strOrEmpty(executeTextQuery(q))));
    }
  }

  class InputSummary extends Algorithm {
    run {
      BEA mainInput = saveInput(q = trim(q));

      Cl<BEA> inputs = conceptsWhere BEAObject(type := "Input", text := q);
      if (l(inputs) > 1)
        appendHTML("Note: Input appears " + l(inputs) + " times in the database: "
          + joinWithSpace(map(o -> mod.beaToHTML(o), inputs)));

      for (BEA input : inputs) {
        if (input != mainInput) {
          appendHTML(hr());
          appendHTML("Input: " + input);
        }

        // rewrites

        Cl<BEA> rewrites = mod.beaBackRefs(input, "Rewrite");

        if (empty(rewrites)) appendHTML(h3("No rewrites"));
        else
          for (int i, BEA r : unpair iterateWithIndex1(rewrites))
            appendHTML("Rewrite " + i + ": " + ahref(mod.beaURL(r), tok_dropCurlyBrackets(r.text())));

        // matches

        //beaBackRefs("Match
      }
    }
  }

  class ApplyPatternList extends Algorithm {
    S additionalInputs() {
      ret hselect("patternListID", mapToKey conceptID(mod.beaList("Pattern List")),
        req.get("patternListID"));
    }

    run {
      BEAObject input = handleInput(this, q = trim(q), false);
      
      BEA patternList = mod.beaGet patternListID(req);
      if (patternList == null) ret;
      L<BEA> patterns = cast cget patterns(patternList);
      appendHTML("Pattern list: " + mod.beaHTML(patternList));

      fOr (BEA pattern : patterns) {
        appendText("Trying pattern " + pattern);
        checkPattern(this, input, q, pattern);
      }
    }
  }

  class ApplyAlgorithmObject extends Algorithm {
    S additionalInputs() {
      ret hselect("algorithmID", mapToKey conceptID(
          concatLists(
            wideningListCast Concept(list BEARegExpReplacement()),
            wideningListCast Concept(conceptsWhereIC UserPost(type := "JavaX Code (HTML Bot)"))
          )),
        req.get("algorithmID"));
    }

    void runLiveMode() {
      //BEAObject input = handleInput(this, q = trim(q), false);

      Concept algo = getConcept(parseLongOpt(req.get("algorithmID")));
      //appendText("Test test. Algorithm: " + algo);

      appendHTML("Algorithm object: " + htmlEncode2_nlToBr(str(algo)));
      
      if (algo cast UserPost) {
        //O result = algObject.apply(q);
        O result = mod.callHtmlBot_dropMadeByComment(algo.id, req.params());
        appendText("Result: " + result);
      } else if (algo cast BEARegExpReplacement) {
        //appendHTML("Algorithm object: " + mod.beaHTML(algObject));
        O result = algo.apply(q);
        appendText("Result: " + result);
      } else if (algo == null)
        appendText("Algorithm object not found");
      else
        appendText("Unknown algorithm object type: " + _className(algo));
    }
  }

  class ApplyRegExpReplacement > Algorithm {
    run {
      BEARegExpReplacement regExpReplacement = cast mod.beaGet(q);
      if (regExpReplacement == null) ret with appendText("Please enter reg exp replacement ID");

      appendHTML("Applying " + beaHTML(regExpReplacement));
      
      for (BEA input : sortedByConceptIDDesc(mod.beaList("Input"))) {
        S text = input.text();
        S replaced = regExpReplacement.apply(text);
        if (neq(text, replaced)) {
          appendHTML(hr());
          appendText("   " + text + "\n=> " + replaced);
          appendPossibleRewrite(this, input, replaced, litmap(
            f_regularExpression := regExpReplacement.id,
            metaInfo_regularExpression := "concept"));
        }
      }
    }
  }

  // prints instead of rendering to page
  class BackEndAlgorithm > Algorithm {
    S prefix() { ret shortClassName(this) + ": "; }
    @Override void appendText(S s) { printWithIndent(prefix(), s); }
    @Override void appendHTML(S s) { appendText(htmlDecode_dropTags(s)); }
    @Override void addNonMatch(Runnable r) { r.run(); }
  }

  class MatchInputWithAllPatternLists > Algorithm {
    run {
      BEAObject input = saveInput(q);
      
      for (BEA patternList : sortedByConceptIDDesc(beaList("Pattern List")))
        reactInputWithPatternList(this, input, q, patternList, 1, 2);
    }
  }
  
  class RunTestCasesOnCodePost extends Algorithm {
    S additionalInputs() {
      MultiSet<S> ms = distinctCIFieldValuesOfConcepts_multiSet(BEAObject, "type");
      LS types = filter(pairsA(multiSetToPairsByPopularity(ms)), 
        type -> cic(type, "test case"));

      ret "Test case type: " + hselect_list("testCaseType", types, req.get("testCaseType"))
        + " " + hselect("algorithmID", mapToKey conceptID(
          conceptsWhereIC UserPost(type := "JavaX Code (HTML Bot)")),
        req.get("algorithmID"));
    }

    run {
      //BEAObject input = handleInput(this, q = trim(q), false);

      S testCaseType = req.get("testCaseType");
      Concept algo = getConcept(parseLongOpt(req.get("algorithmID")));
      //appendText("Test test. Algorithm: " + algo);

      Cl<BEA> testCases = beaList(testCaseType);

      appendText(n2(testCases, "test case") + " found for type " + quote(testCaseType));
      appendHTML("Algorithm object: " + htmlEncode2_nlToBr(str(algo)));

      if (!algo instanceof UserPost) ret;

      new Scorer scorer;

      for ping (BEA testCase : testCases) {
        MapSO fields = toCIMap(objectToMap(testCase));
        S input = str(getAny(fields, "input"));
        S expected = str(getAny(fields, "output", "result"));
        S result = str(mod.callHtmlBot_dropMadeByComment(algo.id, litmap(q := input)));
        bool ok = eq(result, expected);
        appendText((ok ? "OK" : "BAD") + ": Input: " + quote(input) + ", expected output: " + quote(expected)
          + ", result: " + quote(result));
        scorer.add(ok);
      }

      appendText(str(scorer));
    }
  }

  // add algorithms here

  void reactInputWithPatternList(Algorithm algo, BEA input, S text, BEA patternList, int level, int recursionLevels) {
    Pair<BEA, SS> match = matchInputWithPatternList(text, patternList);
    algo.appendHTML(hr());
    algo.appendHTML(joinNemptiesWithColon(
      level == 1 ? beaHTML(patternList) : null,
      stringIf(match == null, "No match")));
      
    if (match != null) {
      BEA pattern = match.a;
      SS mapping = match.b;
      algo.appendHTML("Matched as " + beaHTML(pattern));
      algo.appendText(renderMappingAsText(mapping));

      if (input != null) {
        BEA matchObj = uniqCI(algo.localConcepts, BEA,
          type := "Match",
          +input,
          +patternList,
          winningPattern := pattern,
          +mapping);
        algo.appendHTML("Match saved as: " + beaHTML(matchObj));
      }

      if (level < recursionLevels) {
        algo.appendHTML_raw("<ul>");
        for (S key, subInput : mapping) {
          if (!eqic(key, text)) {
            algo.appendHTML_raw("<li>");
            algo.appendText("Processing sub-Input: " + subInput);
            reactInputWithPatternList(algo, null, subInput, patternList, level+1, recursionLevels);
            algo.appendHTML_raw("</li>");
          }
        }
        algo.appendHTML_raw("</ul>");
      }
    }
  }

  int patternSpecificity(BEAObject pattern) {
    ret countWordsWithoutAngleBracketedAndStars(pattern.text());
  }

  L<BEAObject> syntacticPatternsSortedBySpecificityDesc() {
    ret sortedByCalculatedFieldDesc patternSpecificity(mod.beaList("Syntactic Pattern"));
  }

  O executeTextQuery(S s) {
    new Matches m;
    if (match3_withIntsAndIDs("all references in field <id> of object <int>", s, m)) {
      O value = cget(mod.beaGet($2), $1);
      ret value instanceof Cl ? instancesOf BEAObject((Cl) value)
        : value instanceof BEAObject ? ll((BEAObject) value)
        : ll();
    }

    ret new QueryNotUnderstood(s);
  }

  srecord QueryNotUnderstood(S s) {}

  S beaToHTML(BEA o) { ret mod.beaToHTML(o); }
  S beaHTML(BEA o) { ret mod.beaHTML(o); }
  S beaURL(BEA o) { ret mod.beaURL(o); }

  bool isSyntacticPattern(BEA pat) {
    ret pat != null && pat.typeIs("Syntactic pattern");
  }

  S patternTextWithAngleBracketVars(BEA pat) {
    S text = pat.text();
    ret isSyntacticPattern(pat) ? starsToAngleBracketedNumbers(text) : text;
  }

  bool isCommentOrFeedback(BEA o) {
    ret o != null && o.typeIsOneOf("Comment", "Useful", "Feedback");
  }

  S feedbackHTML(BEA o) {
    Cl<BEA> l = filter isCommentOrFeedback(mod.beaBackRefs(o));
    ret joinWithBR(map(l, c -> c.typeIs("Comment")
      ? ahref(beaURL(c), htmlEncode2(c.text()))
      : beaHTML(c)));
  }

  S bestInputURL(S text) {
    text = trim(text);
    BEA input = mod.findInput(text);
    ret input != null ? beaURL(input) : processInputURL(text);
  }

  // also shaded
  S bestInputHTML(S text) {
    if (empty(text)) ret "";
    ret ahref(bestInputURL(text), span(htmlEncode2(text),
      style := "background-color: #ccc"));
  }

  S performRewrite(BEA rewrite, SS mapping) {
    S rewriteText = starsToAngleBracketedNumbers(rewrite.text());
    ret replaceAngleBracketVars(rewriteText, mapping);
  }

  PairS exampleForPatternRewrite(BEA pattern, BEA rewrite) {
    S patText = patternTextWithAngleBracketVars(pattern);
    Set<S> vars = collectAngleBracketVars(patText);
    SS mapping = mapToValues_ciMap(vars, var -> "$" + var);

    ret pair(replaceAngleBracketVars(patText, mapping),
      performRewrite(rewrite, mapping));
  }

  /* if there is a regular expression
   * and an input
   * and the regular expression matches the input
   * generate a sub-input with the matched part
   */
  record RegExpToPossibleSubInputs(BEA input, BEA regExp) extends Algorithm {
    run {
      S text = input.text();
      S pat = regExp.text();
      for (IntRange r : regexpFindRangesIC(pat, text))
        appendHTML(beaHTML(csetAndReturn(uniq(localConcepts, BEA,
          type := "Possible Sub-Input",
          +input,
          characterRange := r,
          text := substring(text, r)),
          +regExp,
          byAlgorithm := shortName(this))));
    }
  }

  void appendPossibleRewrite(Algorithm algo, BEA input, S text, Map moreInfo default null) {
    // look for existing rewrite
    BEAObject existing = input == null ?: conceptWhereIC BEAObject(
      type := "Rewrite", +input, +text,
      label := "good"
    );
      
    if (existing != null)
      algo.appendHTML("Rewrite exists: " + mod.beaLinkHTML(existing));
    else if (input != null)
      algo.appendHTML(
        joinWithSpace(map(ll("good", "bad"), label ->
          hinlinepostform(
            hhiddenMulti(
              action := "create",
              f_type := "Rewrite",
              f_text := text,
              f_input := input.id, metaInfo_input := "concept",
              f_label := label,
              redirectAfterSave := ""/*algo.req.uriWithParams()*/)
            + hhiddenMulti(moreInfo)
            + hbutton(firstToUpper(label) + " rewrite",
              onclick := js_redirectAutoScroll2()),
            action := mod.crudLink(BEA)))));
  }

  BEA reactFunctionWithInput(Algorithm algo, BEA f, BEA input) {
    S fName = getString name(f);
    if (!isIdentifier(fName)) null;
    if (!isSafeStandardFunction(fName))
      ret null with algo?.addNonMatch(r { algo.appendText("Function " + fName + " not safe"); });

    BEAObject o = cnew(algo.localConcepts, BEAObject,
      type := "Function Result",
      function := f,
      functionCalled := fName,
      +input,
      calculating := true);
    print("Function result: " + o);

    try {
      O holder = getStandardFunctionHolder(fName);
      O result = evalWithTimeoutOrFail(10.0, () -> call(holder, fName, input.text()));

      // convert string collections to list
      if (!result instanceof L && isStringCollection(result)) result = asList((Cl) result);

      if (result == null)
        cset(o, resultType := "null");
      else if (result instanceof S)
        cset(o, resultType := "string", +result);
      else if (isStringList(result))
        cset(o, resultType := "list of strings", +result);
      else
        cset(o, result := "other" + appendRoundBracketed(className(result)),
          result := shorten(1000, str(result)));
    } catch e {
      cset(o, resultType := "error", error := getStackTrace(e));
    }
    cset(o, calculating := null);
    ret o;
  }

  BEA convertSyntacticToSemanticMatch(BEA match, BEA semanticPattern) {
    BEA syntacticPattern = beaGet pattern(match);
    BEA input = beaGet input(match);
    S pat2 = syntacticPattern.text();
    S pat1 = semanticPattern.text();
    new LS vars;
    S pat2b = angleBracketVarsToStars(pat1, vars);
    S warning = eq(pat2, pat2b) ? null
      : "Patterns don't match: " + quote(pat2) + " / " + quote(pat2b);
    SS mapping = cast cget mapping(match);
    SS mapping2 = mapKeys(mapping, key -> or(_get(vars, parseInt(key)-1), key));
    mapping2 = putKeysFirst(vars, mapping2); // reorder variables in order of appearance in input
    BEA match2 = uniqCI BEA(
      type := "Match",
      +input,
      pattern := semanticPattern,
      fromSyntacticMatch := match);
    cset(match2,
      mapping := mapping2);
    ret match2;  
  }

  Cl<BEA> convertSyntacticToSemanticMatches(BEA match) {
    new L<BEA> out;
    BEA syntacticPattern = beaGet pattern(match);
    for (BEA semanticPattern : mod.beaList("Pattern", +syntacticPattern)) pcall {
      out.add(convertSyntacticToSemanticMatch(match, semanticPattern));
    }
    ret out;
  }

  Cl<BEA> convertSyntacticToSemanticMatchesForWholePattern(BEA pattern) {
    if (pattern.typeIs("Syntactic Pattern"))
      // This is huge - uses all semantic patterns
      ret concatMap_lists convertSyntacticToSemanticMatches(mod.beaList("Match", +pattern));
    else {
      new L<BEA> out;
      BEA syntacticPattern = beaGet syntacticPattern(pattern);
      if (syntacticPattern != null)
        for (BEA match : mod.beaList("Match", pattern := syntacticPattern))
          out.add(convertSyntacticToSemanticMatch(match, pattern));
      ret out;
    }
      
  }

  BEA mapMethodLike beaGet(S field, BEA o) { ret mod.beaGet(field, o); }
  Cl<BEA> beaList(S type, O... params) { ret mod.beaList(type, params); }

  int simpleObjectScore(BEA o) {
    int score = 0;
    if (eqic_gen(o~.label, "good")) ++score;
    else if (eqic_gen(o~.label, "bad")) --score;
    Cl<BEA> l = findBackRefsWithField BEA(object := o);
    score += countWhereCI(l, type := "Useful");
    score -= countWhereCI(l, type := "Bad");
    ret score;
  }

  L<BEA> sortedBySimpleObjectScore(int minScore, Cl<BEA> l) {
    ret sortedByCalculatedFieldDesc simpleObjectScore(filter(l, o -> simpleObjectScore(o) >= minScore));
  }

  BEA createPatternListFromUsefulSyntacticPatterns() {
    L<BEA> patterns = sortedBySimpleObjectScore(0, beaList("Syntactic Pattern"));
    ret mod.autoMigrateToCustomClass(uniqCI BEA(type := "Pattern List",
      byFunction := "createPatternListFromUsefulSyntacticPatterns",
      +patterns));
  }

  Map<S, IF0<Algorithm>> algorithms() {  
    ret litorderedmap(
      "Process new input v3" := (IF0) () -> new ProcessNewInput_v3,
      "Process new input v2" := (IF0) () -> new ProcessNewInput_v2,
      "Process new input" := (IF0) () -> new ProcessNewInput,
      "Run pattern" := (IF0) () -> new RunPattern,
      "Find rewrites for match" := (IF0) () -> new FindRewrites,
      "Reverse input (test algorithm)" := (IF0) () -> new ReverseInput,
      "Timeout test" := (IF0) () -> new TimeoutTest,
      "Syntactic pattern from input" := (IF0) () -> new SyntacticPatternFromInput,
      "Apply all text functions" := (IF0) () -> new ApplyTextFunctions,
      "Find Path" := (IF0) () -> new FindPath,
      "Execute text query" := (IF0) () -> new ExecuteTextQuery,
      "Make syntactic patterns for all patterns in DB" := (IF0) () -> new MakeSyntacticPatterns,
      "Input summary" := (IF0) () -> new InputSummary,
      "Apply pattern list" := (IF0) () -> new ApplyPatternList,
      "Apply regular expression replacement to all inputs" := (IF0) () -> new ApplyRegExpReplacement,
      "Match input with all pattern lists" := (IF0) () -> new MatchInputWithAllPatternLists,
      "Apply algorithm object" := (IF0) () -> new ApplyAlgorithmObject,
      "Run test cases on code post" := (IF0) () -> new RunTestCasesOnCodePost,
      
      // add algorithms here
    );
  }
  
  void putAlgorithmsInDatabase {
    for (S name : keys(algorithms()))
      uniq BEA(type := "Classic Query Algorithm", +name);
  }

} // end of Calculations