do not include class And.
set flag OurSyncCollections.

sS mainBotID;
static int maxTypos = 1;

sS adminTitle = "Chat Bot Admin";
sS shortLink; // catchy link to admin if available
sbool enableWorkerChat;

sS embedderLink; // where chat bot will go
sS goDaddyEmbedCode;

static Cache<Scorer<ConsistencyError>> consistencyCheckResult = new(lambda0 consistencyCheck);

concept Category {
  int index;
  S name;
}

concept Language {
  int index;
  S code;
}

concept QA {
  int index;
  S language;
  S questions; // line by line
  S patterns; // to be determined
  S answers; // one answer, or "random { answers separated by empty lines }"
  S category; // "business", "smalltalk"

  transient MMOPattern parsedPattern;

  sS _fieldOrder = "language category index questions patterns answers";
  
  void change {
    parsedPattern = null;
    super.change();
  }
  
  MMOPattern parsedPattern() {
    if (parsedPattern == null)
      parsedPattern = mmo2_parsePattern(patterns);
    ret parsedPattern;
  }
}

concept Defaults {
  S lastLanguage;
}

concept Settings {
  bool botOn, botAutoOpen;
}

svoid pQADB {
  print("fieldOrder", getFieldOrder(QA));
  dbIndexing(Language, 'index, Category, 'index);
  indexSingletonConcept(Defaults);
  indexSingletonConcept(Settings);
  
  // languages, default language
  cset(conceptsWhere QA(language := null), language := 'en);
  uniq(Language, code := 'en);
  removeConceptsWhere(Language, code := 'de);
  
  // categories, default category
  int i = 0;
  for (S category : ll("business", "smalltalk"))
    cset(uniq(Category, name := category), index := i++);
  cset(conceptsWhere QA(category := null), category := "business");
  
  onConceptsChange(r { consistencyCheckResult.clear(); });
  reindexQAs();
}

html { try {
  if (swic(addSlash(uri), "/answer/")) {
    temp tempSetTL(opt_noDefault, valueIs1 noDefault(params));
    ret serveText(unnull(answer(params.get("q"), "en")));
  }
    
  // force auth
  try answer (S) callHtmlMethod2(mainBot(), "/auth-only", mapPlus(
params, uri := rawLink(uri)));
    
  if (eq(uri, "/demo"))
    ret hrefresh(appendQueryToURL(rawBotLink(mainBotID), _botDemo := 1, bot := 1));
    
  if (eq(uri, "/download"))
    ret subBot_serveText(conceptsStructure());
    
  if (eq(uri, "/embedCode"))
    ret hsansserif() + htitle_h2("Chat bot embed code")
      + (empty(goDaddyEmbedCode) ? "" :
        h3("GoDaddy Site Builder")
      
        + p("Add an HTML box with this code:")
      
        + pre(htmlEncode2(goDaddyEmbedCode))
        
        + h3("Other"))
        
      + p("Add this code in front of your " + tt(htmlEncode2("</body>")) + " tag:")
      
      + pre(htmlEncode2(hjavascript_src_withType(rawBotLink(mainBotID), defer := html_valueLessParam())));
  
  if (eq(uri, "/dbUploads/import")) {
    long id = parseLong(params.get("id"));
    DBUpload dbUpload = getConcept DBUpload(id);
    if (dbUpload == null) ret "Not found";
    
    Map<Long, O> qaMap = cast safeUnstructure(dbUpload.text);
    Map<S, QA> existingByPattern = indexByFieldCI patterns(list(QA));
    new LS msgs;
    msgs.add("Importing...");

    for (S _qaID : startingWith_dropPrefix("qa_", keys(params))) {
      long qaID = parseLong(_qaID);
      virtual QA qa = qaMap.get(qaID);
      if (qa == null) continue;

      S patterns = getString patterns(qa), answers = getString answers(qa);
      QA existing = existingByPattern.get(patterns);
      if (existing != null) {
        msgs.add("QA item exists for patterns: " + patterns);
        msgs.add("Answers are " + (eq(answers, existing.answers) ? "identical" : "different"));
      } else {
        QA imported = cnew QA();
        copyFields(qa, imported, "index", "language", "questions", "patterns", "answers", "category");
        msgs.add("Imported item " + qaID + " as " + imported.id + " (patterns: " + patterns + ")");
      }
    }

    ret htmlEncode2_nlToBr(lines(msgs));
  }
    
  if (eq(uri, "/dbUploads/view")) {
    long id = parseLong(params.get("id"));
    DBUpload dbUpload = getConcept DBUpload(id);

    S contents;
    if (dbUpload == null)
      contents = "Not found";
    else {
      new L<Map> data;
      Map<Long, O> qaMap = cast safeUnstructure(dbUpload.text);

      // filter and sort
      qaMap = filterByValuePredicate(qaMap, c -> dynShortNameIs QA(c));
      qaMap = mapSortedByFunctionOnValue(qaMap, c -> neqic(getString category(c), "smalltalk"));

      for (long qaID, O c : qaMap) {
        S category = getString category(c);
        data.add(litorderedmap(
          "ID" := qaID,
          "Select" := hcheckbox("qa_" + qaID, eqic(category, "smalltalk")),
          "Category" := htmlEncode2(category),
          "Questions" := htmlEncode2(getString questions(c)),
          "Patterns" := htmlEncode2(getString patterns(c)),
          "Answers" := htmlEncode2(getString answers(c)),
          ));
      }
      contents = hpostform(
          hhidden(+id)
        + p(hsubmit("Import selected questions"))
        + htmlTable2_noHtmlEncode(data),
        action := rawLink("dbUploads/import")
      );
    }

    S title = "Import questions";
    ret hhtml(hhead_title_decode(title)
      + hbody(h2(title)
      + p(makeNav())
      + contents));
    
  }
  
  if (eq(uri, "/dbUploads")) {
    HCRUD_Concepts<DBUpload> data = new HCRUD_Concepts<DBUpload>(DBUpload) {
      Renderer getRenderer(S field) {
        if (eq(field, "text"))
          ret new TextArea(80, 20);
        ret super.getRenderer(field);
      }
    };

    HCRUD crud = new(rawLink("dbUploads"), data) {
      S frame(S title, S contents) {
        title = ahref(or2(shortLink, rawLink("")), adminTitle) + " | " + title;
        ret hhtml(hhead_title_decode(title)
          + hbody(h2(title)
          + p(makeNav())
          + contents));
      }
      
      S renderCmds(MapSO item) {
        O id = item.get(data.idField());
        ret joinNemptiesWithVBar(super.renderCmds(item), ahref(baseLink + "/view?id=" + urlencode(str(id)), "View questions"));
      }
    };
    
    crud.tableClass = "responstable";
    ret hsansserif() + hcss_responstable() + crud.renderPage(params);
  }  
  
  // make QA CRUD_
  
  HCRUD_Concepts<QA> data = new HCRUD_Concepts<QA>(QA) {
    S itemName() { ret "question"; }
    
    S fieldHelp(S field) {
      if (eq(field, "category"))
        ret "smalltalk has a lower precedence than business";
      if (eq(field, "index"))
        ret "lower index = higher matching precedence";
      if (eq(field, "patterns"))
        ret [[Phrases to match. Use "+" as "and" operator, commas as "or" operator. Round brackets for grouping. Quotes for special characters. Put exclamation mark after phrase to disable typo correction.]];
      if (eq(field, "answers"))
        ret "Text of answer. Use random { ... } for multiple answers (separated from each other by an empty line)";
      null;
    }
    
    Renderer getRenderer(S field) {
      if (eqOneOf(field, 'questions, 'answers))
        ret new TextArea(80, 10);
      if (eq(field, 'patterns))
        ret new TextField(80);
      if (eq(field, 'language))
        ret new ComboBox(collect code(conceptsSortedByField(Language, 'index)));
      if (eq(field, 'category))
        ret new ComboBox(collect name(conceptsSortedByField(Category, 'index)));
      ret super.getRenderer(field);
    }
    
    // sort by language first, then by priority (category + index)
    L<QA> defaultSort(L<QA> l) {
      ret sortByFieldDesc language(sortQAs(l));
    }
    
    Map<S, O> emptyObject() {
      ret mapPlus(super.emptyObject(), language := uniq(Defaults).lastLanguage);
    }
    
    O createObject(SS map, S fieldPrefix) {
      S language = cast map.get('language);
      if (language != null) cset(uniq(Defaults), lastLanguage := language);
      ret super.createObject(map, fieldPrefix);
    }
  };
  data.onCreateOrUpdate.add(qa -> reindexQAs());
  
  HCRUD crud = new(rawLink(), data) {
    S frame(S title, S contents) {
      S stats = (S) pcallOpt(mainBot(), 'dbStats);
      title = ahref(or2(shortLink, rawLink("")), adminTitle) + " | " + title;
      ret hhtml(hhead_title_decode(title)
        + hbody(h2(title)
        + pIfNempty(stats)
        + p(makeNav())
        + contents));
    }
    
    S renderTable(bool withCmds) {
      Scorer<ConsistencyError> scorer = consistencyCheckResult!;
      ret (empty(scorer.errors)
        ? p("Pattern analysis: No problems found. " + nEntries(countConcepts(QA)) + ".")
        : joinMap(scorer.errors, lambda1 renderErrorAsParagraph))
        + super.renderTable(withCmds);
    }
    
    S renderErrorAsParagraph(ConsistencyError error) {
      S fix = error.renderFix();
      ret p("Error: " + htmlEncode2(error.msg) + " " +
        joinMap(error.items, qa -> ahref(editLink(qa.id), "[item]"))
        + (empty(fix) ? "" : " " + fix));
    }
  };
  
  // actions

  if (eqGet(params, action := 'reindex)) {
    long id = parseLong(params.get('id));
    int newIndex = parseInt(params.get('newIndex));
    QA qa = getConcept(QA, id);
    if (qa == null) ret crud.refreshWithMsgs("Item not found");
    cset(qa, index := newIndex);
    reindexQAs();
    ret crud.refreshWithMsgs("Index of item " + qa.id + " changed");
  }

  if (eqGet(params, botOn := "1")) {
    cset(uniq(Settings), botOn := true);
    ret crud.refreshWithMsgs("Bot turned on!");
  }
  
  if (eqGet(params, botOff := "1")) {
    cset(uniq(Settings), botOn := false);
    ret crud.refreshWithMsgs("Bot turned off!");
  }
  
  if (nemptyGet botAutoOpen(params)) {
    cset(uniq(Settings), botAutoOpen := eq("1", params.get("botAutoOpen")));
    ret crud.refreshWithMsgs("Bot auto-open turned " + onOff(botAutoOpen()) + "!");
  }
  
  crud.tableClass = "responstable";
  ret hsansserif() + hcss_responstable() + crud.renderPage(params);
 } catch print e {
  ret "ERROR.";
 }
}

static new ThreadLocal<S> language_out;
static new ThreadLocal<Bool> opt_noDefault; // true = return null instead of #default text

// main API function for other bots
sS answer(S s, S language) {
  language_out.set(null);
  ret trim(answer_main(s, language));
}

sS answer_main(S s, S language) {
  S raw = findRawAnswer(s, language, true);
  S contents = extractKeywordPlusBracketed_keepComments("random", raw);
  if (contents != null)
    ret random(splitAtEmptyLines(contents));
  ret raw;
}

sS findRawAnswer(S s, S language, bool allowTypos) {
  ret selectQA(findMatchingQA(s, language, allowTypos));
}

static QA findQAWithTypos(S s, S language) {
  Lowest<QA> qa = findClosestQA(s, language);
  if (!qa.has()) null;
  if (qa.score() > maxTypos) {
    print("Rejecting " + nTypos((int) qa.score()) + ": " + s + " / " + qa->patterns);
    null;
  }
  print("Accepting " + nTypos((int) qa.score()) + ": " + s + " / " + qa->patterns);
  ret qa!;
}

static QA findMatchingQA(S s, S language, bool allowTypos) {
  try object QA qa = findMatchingQA(s, conceptsWhere(QA, +language));
  try object QA qa = findMatchingQA(s, filter(list(QA), q -> neq(q.language, language)));
  if (allowTypos)
    try object QA qa = findQAWithTypos(s, language);
  if (!eq(s, "#default") && !isTrue(opt_noDefault!)) ret findMatchingQA("#default", language, false);
  null;
}

static Lowest<QA> findClosestQA(S s, S language) {
  time "findClosestQA" {
    new Lowest<QA> qa;
    findClosestQA(s, qa, conceptsWhere(QA, +language));
    findClosestQA(s, qa, filter(list(QA), q -> neq(q.language, language)));
  }
  ret qa;
}

static L<QA> sortQAs(Cl<QA> qas) {
  Map<S, Int> categoryToIndex = fieldToFieldIndex('name, 'index, list(Category));
  ret sortedByCalculatedField(qas, q -> pair(categoryToIndex.get(q.category), q.index));
}

svoid reindexQAs() {
  for (Language lang) {
    int index = 1;
    for (QA qa : sortQAs(conceptsWhere(QA, language := lang.code)))
      cset(qa, index := index++);
  }
}

static QA findMatchingQA(S s, Cl<QA> qas) {
  for (QA qa : sortQAs(qas))
    if (mmo2_match(qa.parsedPattern(), s))
      ret qa;
  null;
}

svoid findClosestQA(S s, Lowest<QA> best, Cl<QA> qas) {
  for (QA qa : sortQAs(qas)) {
    Int score = mmo2_levenWithSwapsScore(qa.parsedPattern(), s);
    if (score != null)
      best.put(qa, score);
  }
}

// returns answers
sS selectQA(QA qa) {
  if (qa == null) null;
  language_out.set(qa.language);
  ret qa.answers;
}

sclass ConsistencyError {
  S msg;
  L<QA> items;
  
  *(S *msg, QA... items) { this.items = asList(items); }
  
  S renderFix() { null; }
}

svoid checkLocalConsistency(QA qa, Scorer<ConsistencyError> scorer) {
  LS questions = tlft(qa.questions);
  for (S q : questions)
    if (mmo2_match(qa.parsedPattern(), q))
      scorer.ok();
    else
      scorer.error(ConsistencyError("Question " + quote(q) + " not matched by patterns " + quote(qa.patterns), qa));
}

svoid checkGlobalConsistency(QA qa, Scorer<ConsistencyError> scorer) {
  for (S q : tlft(qa.questions)) {
    QA found = findMatchingQA(q, qa.language, false);
    if (found != null && found != qa)
      scorer.error(new ConsistencyError("Question " + quote(q) + " (item " + qa.id + ") shadowed by patterns " + quote(found.patterns) + " (item " + found.id + ")", qa, found) {
        S renderFix() {
          ret ahref(rawLink("?action=reindex&id=" + qa.id + "&newIndex=" + (found.index-1)), "[fix by changing index]");
        }
      });
    else
      scorer.ok();
  }
}

static Scorer<ConsistencyError> consistencyCheck() {
  new Scorer<ConsistencyError> scorer;
  scorer.collectErrors();
  for (QA qa) {
    checkLocalConsistency(qa, scorer);
    checkGlobalConsistency(qa, scorer);
  }
  ret scorer;
}

// API

sbool botOn() {
  ret uniq(Settings).botOn;
}

sbool botAutoOpen() {
  ret uniq(Settings).botAutoOpen;
}

svoid importQA(virtual QA qa_external) {
  QA qa = shallowCloneToUnlistedConcept QA(qa_external);
  uniq QA(allConceptFieldsAsParams(qa));
}

static swappable S makeNav() {
  S s = joinWithVBar(
    ahref("?logout=1", "log out"),
    targetBlank(relativeRawBotLink(mainBotID, "/logs"), "chat logs"),
    targetBlank(rawLink("demo?bot=1"), "demo"),
    targetBlank(rawLink("download"), "export brain"),
    ahref(rawLink("embedCode"), "show embed code"),
    botOn()
      ? "Bot is ON (appears on home page) " + ahrefWithConfirm("Switch bot off?", "?botOff=1", "[switch bot off]")
      : "Bot is OFF (appears only with " + targetBlank(appendQueryToURL(embedderLink, bot := 1), "?bot=1") + ") " + ahrefWithConfirm("Switch bot on?", "?botOn=1", "[switch bot on]"),
    ahref(rawLink("dbUploads"), "db uploads"),
  );
  
  if (enableWorkerChat)
    s +=
      " | " + ahref(rawBotLink(mainBotID, "workers-admin"), "workers admin")
    + " | " + targetBlank(rawBotLink(mainBotID, "worker"), "worker chat");
    
  ret s;
}

sO mainBot() {
  ret getBot(mainBotID);
}

concept DBUpload {
  S name;
  S text;
}

Began life as a copy of #1026409