!include once #1029875 // concept Worker

abstract sclass DynNewBot2 > DynPrintLogAndEnabled {
  set flag NoNanoHTTPD.
  !include #1029545 // API for Eleu

  int maxRefsToShow = 5;
  volatile long requestsServed;
  
  transient S templateID = #1029809;
  sS cssID = #1029808;

  transient S botName = "DynNewBot2";
  transient S heading = "DynNewBot2";
  transient S adminName = "DynNewBot2 Admin";
  transient S botImageID = #1102935;
  transient S userImageID = #1102803;
  transient S chatHeaderImageID = #1102802;
  transient S timeZone = ukTimeZone_string(); 
  
  transient S baseLink = "";
  
  transient bool newDesign = true; // use new chat bot design
  transient bool ariaLiveTrick = false;
  transient bool ariaLiveTrick2 = true;
  
  !include once #1029876 // WorkerChat 
  transient new WorkerChat workerChat;
  
  transient ReliableSingleThread rstBotActions = dm_rst(this, r botActions);
  
  transient S defaultHeaderColorLeft = "#2a27da";
  transient S defaultHeaderColorRight = "#00ccff";
  transient bool enableUsers;
  transient bool useWebSockets;
  transient bool showRegisterLink;
  transient bool showTalkToBotLink;
  transient bool alwaysRedirectToHttps;
  transient bool redirectOnLogout; // remove that ugly logout=1 from url
  transient bool showFullErrors;
  transient bool lockWhileDoingBotActions;
  transient bool standardButtonsAsSymbols;
  transient bool enableVars; // $userName etc. in bot messages
  transient bool enableAvatars;
  transient bool recordExecutionsAndInputs = true;
  transient bool phoneNumberSpecialInputField = true;

  transient new ThreadLocal<Req> currentReq;

  transient volatile Scorer consistencyCheckResults;

  transient Lock statsLock = lock();

  transient SS specialPurposes = litcimap(
    "bad email address", "Please enter a valid e-mail address",
    "bad phone number", "Please enter a valid phone number");

  void start {
    super.start();
    dm_setModuleName(botName);
    dm_assertFirstSibling();
    concepts_setUnlistedByDefault(true);
    standardTimeZone();
    standardTimeZone_name = timeZone;
    print("DB program ID: " + dbProgramID());
    realPW(); // make password
    pWebChatBot();
    doEvery(60.0, r cleanConversations);
    rstBotActions.trigger();
  }

  afterVisualize {
    addToControlArea(jPopDownButton_noText(popDownButtonEntries()));
  }

  O[] popDownButtonEntries() {
    ret litobjectarray(
      "Show/edit master password...", rThreadEnter editMasterPassword,
      );
  }

  sclass Req {
    IWebRequest webRequest;
    S uri;
    SS params;
    AuthedDialogID auth;
    Domain authDomainObj;
    S authDomain;
    HTMLFramer1 framer;
    bool masterAuthed;
    Conversation conv;

    S uri() { ret uri; }
    bool requestAuthed() { ret auth != null; }

    void markNoSpam { webRequest.noSpam(); }
  }

  bool calcMasterAuthed(Req req) {
    ret req.auth != null && req.auth.master;
  }

  O html(IWebRequest request) enter {
    try {
      ret html2(request);
    } catch print e {
      ret serve500(showFullErrors ? getStackTrace(e) : "Error.");
    }
  }

  void requestServed {}

  O html2(IWebRequest request) {
    htmlencode_forParams_useV2();

    {
      lock statsLock;
      requestsServed++;
      change();
      requestServed();
    }
    
    S uri = request.uri();
    SS params = request.params();

    new Req req;
    req.webRequest = request;
    req.uri = uri;
    req.params = params;

    if (alwaysRedirectToHttps)
      try object redirectToHttps(req);

    if (eq(params.get("_newConvCookie"), "1"))
      ret hrefresh(appendQueryToURL(req.uri, mapPlus(mapMinus(req.params, "_newConvCookie"),
        cookie := "test_" + aGlobalID())));

    // cookie comes either from a URI parameter or from the request header
    // Note: this can be either a JS-generated (in dynamic chat bot part)
    // or server-generated cookie (when loading initial chat bot script or admin)
    // To distinguish: JS-generated cookies are shorter and can contain numbers
    S cookie = request.cookie();
    //print("Request cookie: " + cookie);
   
    bool workerMode = nempty(params.get("workerMode")) || startsWith(uri, "/worker");
    
    // find out which domain we're on ("delivered domain")

    S domain = request.domain(), _domain = domain;
    saveDeliveredDomain(domain);
    Domain domainObj = findDomainObj(domain);

    // get conversation
    S convCookie = params.get("cookie");
    //if (eq(params.get("_newConvCookie"), "1")) convCookie = "conv-" + aGlobalID();
    Conversation conv = isRequestFromBot(req) ? null
      : nempty(convCookie) ? getConv(convCookie)
      : nempty(cookie) ? getConv(cookie) : null;
    req.conv = conv;

    AutoCloseable tempThing = conv == null ? null : temp_printPrefix("Conv " + conv.cookie + ": ");
    temp tempThing;
    temp tempSetTL(currentReq, req);

    S botConfig = params.get("_botConfig");
    SS botConfigParams = decodeURIParams(botConfig);
    S simulatedDomain = botConfigParams.get("domain");
    Domain domainObj2 = nempty(simulatedDomain) ? findDomainObj(simulatedDomain) : domainObj;
    if (nempty(botConfigParams)) cset(conv, botConfig := botConfigParams);

    // save ip & domain in conversation
    if (conv != null && !workerMode)
      if (cset_trueIfChanged(conv, ip := request.clientIP(), +domain, domainObj := domainObj2)) {
        calcCountry(conv);
        initAvatar(conv);
      }
      
    //print("URI: " + uri + ", cookie: " + cookie + ", msgs: " + l(conv.msgs));

    try object handleAuth(req, cookie);

    // TODO: instead of maxCache, check file date & request header (If-Unmodified-Since?)
    
    new Matches m;
    if (startsWith(uri, "/worker-image/", m)) {
      long id = parseLong(m.rest());
      ret subBot_serveFile_maxCache(workerImageFile(id), "image/jpeg");
    }
    
    if (startsWith(uri, "/uploaded-image/", m)) {
      long id = parseLong(m.rest());
      UploadedImage img = getConcept UploadedImage(id);
      ret img == null ? serve404() : subBot_serveFile_maxCache(img.imageFile(), "image/jpeg");
    }
    
    if (startsWith(uri, "/uploaded-sound/", m)) {
      long id = parseLong(m.rest());
      UploadedSound sound = getConcept UploadedSound(id);
      ret sound == null ? serve404() : subBot_serveFile_maxCache(sound.soundFile(), mp3mimeType());
    }
    
    if (startsWith(uri, "/uploaded-file/", m)) {
      long id = parseLong(dropAfterSlash(m.rest()));
      UploadedFile f = getConcept UploadedFile(id);
      ret f == null ? serve404() : subBot_serveFile_maxCache(f.theFile(),
        or2(params.get("ct"), or2(trim(f.mimeType), binaryMimeType())));
    }
    
    AuthedDialogID auth = authObject(cookie);
    if (eq(params.get('logout), "1")) {
      cdelete(auth);
      if (redirectOnLogout) ret hrefresh(req.uri);
      auth = null;
    }
    
    req.auth = auth;
    bool requestAuthed = auth != null; // any authentication
    bool masterAuthed = req.masterAuthed = calcMasterAuthed(req);
    Domain authDomainObj = !requestAuthed ? null : auth.restrictedToDomain!;
    req.authDomainObj = authDomainObj;
    S authDomain = !requestAuthed ? null : auth.domain();
    req.authDomain = authDomain;

    try object serve2(req);

    if (!requestAuthed && settings().talkToBotOnlyWithAuth)
      ret serveAuthForm(params.get('uri));

    if (eq(uri, "/emoji-picker/index.js"))
      //ret serveWithContentType(loadTextFile(loadLibrary(#1400436)), "text/javascript"); // TODO: optimize
      ret withHeader(subBot_maxCacheHeaders(serveInputStream(bufferedFileInputStream(loadLibrary(#1400436)), "text/javascript")));

    if (eq(uri, "/emoji-picker-test"))
      ret loadSnippet(#1029870);

    if (eq(uri, "/logs")) {
      if (!masterAuthed) ret serveAuthForm(rawLink(uri));
      ret webChatBotLogsHTML2(rawLink(uri), params);
    }
    
    if (eq(uri, "/refchecker")) {
      if (!masterAuthed) ret serveAuthForm(rawLink(uri));
      ConceptsRefChecker refChecker = new (db_mainConcepts());
      L errors = refChecker.run();
      if (eq(params.get("fix"), "1"))
        ret serveText(refChecker.fixAll());
      else
        ret serveText(jsonEncode_breakAtLevels(2, litorderedmap(errors := allToString(errors))));
    }
    
    if (eq(uri, "/backupDBNow")) {
      if (!masterAuthed) ret serveAuthForm(rawLink(uri));
      ret serveText("Backed up DB as " + fileInfo(backupConceptsNow()));
    }
    
    if (eq(uri, "/auth-only")) {
      if (!requestAuthed) ret serveAuthForm(params.get('uri));
      ret "";
    }

    if (eq(uri, "/leads-api"))
      ret serveLeadsAPI(request);
    
    if (workerChat != null) // TODO: don't permit when worker chat is disabled
      try object workerChat.html(req);
  
    S message = trim(params.get("btn"));
    if (empty(message)) message = trim(params.get("message"));
    
    if (match("new dialog", message)) {
      lock dbLock();
      conv.newDialog();
      message = null;
    }

    if (eqic(message, "!toggle notifications")) {
      cset(conv, notificationsOn := !conv.notificationsOn);
      message = null;
    }
    
    this.conv.set(conv);
    
    if (nempty(message) && !lastUserMessageWas(conv, message)) {
      lock dbLock();
      print("Adding message: " + message);

      possiblyTriggerNewDialog(conv);
      
      if (workerMode) {
        Msg msg = new(false, message);
        msg.fromWorker = auth.loggedIn;
        conv.add(msg);
      } else {
        // add user message
        conv.add(new Msg(true, message));
        //conv.botTyping = now();
        addScheduledAction(new OnUserMessage(conv));
      }
    }
    
    S testMode = params.get("testMode");
    if (nempty(testMode)) {
      print("Setting testMode", testMode);
      cset(conv, testMode := eq("1", testMode));
    }
    
    if (eq(uri, "/msg")) ret withHeader("OK");
    
    if (eq(uri, "/typing")) {
      if (workerMode) {
        conv.botTyping = now();
        print(conv.botTyping + " Bot typing in: " + conv.cookie);
      } else {
        conv.userTyping = now();
        print(conv.userTyping + " User typing in: " + conv.cookie);
      }
      ret withHeader("OK");
    }

    // render all of a conversation's messages for inspection
    if (eq(uri, "/renderMessages")) {
      long msgTime = parseLong(params.get("msgTime"));
      L<Msg> msgs = conv.allMsgs();
      if (msgTime != 0)
        msgs = filter(msgs, msg -> msg.time == msgTime);
      new StringBuilder buf;
      renderMessages(conv, buf, msgs,
        msg -> false); // don't show buttons
      ret hhtml_title_body(nMessages(msgs),
        hstylesheetsrc(cssURL())
        + p(nMessages(msgs) + " found")
        + buf);
    }

    if (eq(uri, "/incremental")) {
      vmBus_send chatBot_userPolling(mc(), conv);
      cset(conv, lastPing := now());
      possiblyTriggerNewDialog(conv);
      int a = parseInt(params.get("a"));
  
      int reloadCounter = conv.reloadCounter;
      long start = sysNow(), start2 = now();
      L<Msg> msgs;
      bool first = true;
      int timeout = toInt(req.params.get("timeout"));
      long endTime = start+(timeout <= 0 || timeout > longPollMaxWait/1000 ? longPollMaxWait : timeout*1000L);
      while (licensed() && sysNow() < endTime) {
        int as = conv.archiveSize();
        msgs = cloneSubList(conv.msgs, a-as); // just the new messages
        bool reloadTriggered = conv.reloadCounter > reloadCounter;
        bool actuallyNewDialog = a < as;
        bool newDialog = actuallyNewDialog || reloadTriggered;
        if (newDialog)
          msgs = cloneList(conv.msgs);
        long typing = workerMode ? conv.userTyping : conv.botTyping;
        bool otherPartyTyping = typing > start2;

        bool anyEvent = nempty(msgs) || newDialog || otherPartyTyping;
        
        if (!anyEvent) {
          if (first) {
            //print("Long poll starting on " + cookie + ", " + a + "/" + a);
            first = false;
          }
          sleep(longPollTick);
        } else {
          if (!first) print("Long poll ended.");

          if (eq(req.params.get("json"), "1"))
            ret serveJSON_breakAtLevels(2, litorderedmap(
              n := conv.allCount(),
              newDialog := trueOrNull(newDialog),
              otherPartyTyping := trueOrNull(otherPartyTyping),
              msgs := map(msgs, msg ->
                litorderedmap(
                  time := msg.time,
                  fromUser := msg.fromUser,
                  fromWorker := msg.fromWorker == null ?: msg.fromWorker.displayName,
                  text := msg.text,
                  // TODO: out
                  labels := msg.labels))));
          
          new StringBuilder buf;

          if (newDialog) {
            // send header colors & notification status
            S l = or2_trim(domainObj2.headerColorLeft, defaultDomain().headerColorLeft, defaultHeaderColorLeft),
              r = or2_trim(domainObj2.headerColorRight, defaultDomain().headerColorRight, defaultHeaderColorRight);
            buf.append(hcss(".chat_header { background: linear-gradient(135deg, "
              + hexColorToCSSRGB(l) + " 0%, " + hexColorToCSSRGB(r) + " 100%); }"));
            buf.append(hscript("$('#chatBot_notiToggleText').text(" + jsQuote("Turn " + (conv.notificationsOn ? "off" : "on")
              + " notifications") + ");"));
          }
          
          if (otherPartyTyping) {
            print("Noticed " + (workerMode ? "user" : "bot") + " typing in " + conv.cookie);
            buf.append(hscript("showTyping();"));
          }
          renderMessages(conv, buf, msgs);
          if (ariaLiveTrick2 && !workerMode) {
            Msg msg = lastBotMsg(msgs);
            if (msg != null) {
              S author = msg.fromWorker != null ? htmlEncode2(msg.fromWorker.displayName) : botName;
              buf.append(hscript([[$("#screenreadertrick").html(]] + jsQuote(author + " says: " + msg.text) + ");"));
              }
          }
          if (a != 0 && anyInterestingMessages(msgs, workerMode))
            buf.append(hscript(
              stringIf(conv.notificationsOn, "window.playChatNotification();\n") +
              "window.setTitleStatus(" + jsQuote((workerMode ? "User" : botName) + " says…") + ");"
            ));
          S html = str(buf);

          // TODO: hack for notransition
          //if (newDialog && !actuallyNewDialog) html = html.replace([[class="chat_msg_item]], [[class="notransition chat_msg_item]]);
          
          ret withHeader("<!-- " + conv.allCount() + " " + (newDialog ? actuallyNewDialog ? "NEW DIALOG " : "NO NOTIFY NEW DIALOG " : "") + "-->\n"  + html);
        }
      }
      ret withHeader("");
    }

    if (eqOneOf(uri, "/script", "/demo")) {
      lock dbLock();
      S html = loadSnippet_cached(templateID);

      html = modifyTemplateBeforeDelivery(html, req);
      
      S heading = or2(headingForReq(req), or2(trim(domainObj2.botName), this.heading));
      S botImg = botImageForDomain(domainObj);

      UploadedSound sound = settings().notificationSound!;
      S notificationSound = sound != null ? sound.soundURL() : defaultNotificationSound();

      S workerModeParam = workerMode ? "workerMode=1&" : "";
      S langlinks = "<!-- langlinks here -->";
      if (html.contains(langlinks))
        html = html.replace(langlinks,
          ahref(rawLink("eng"), "English") + " | " + ahref(rawLink("deu"), "German"));

      //html = html.replace("#COUNTRYCODE_OPTIONS#", countryCodeOptions(conv));
      html = html.replace("#COUNTRY#", lower(conv.country));
      html = html.replace("#BOTIMG#", botImg);
      html = html.replace("#N#", "0");
      html = html.replace("#INCREMENTALURL#", baseLink + "/incremental?" + workerModeParam + "a=");
      html = html.replace("#MSGURL#", baseLink + "/msg?" + workerModeParam + "message=");
      html = html.replace("#TYPINGURL#", baseLink + "/typing?" + workerModeParam);
      html = html.replace("#CSS_ID#", psI_str(cssID));
      if (ariaLiveTrick || ariaLiveTrick2)
        html = html.replace([[aria-live="polite">]], ">");
      html = html.replace("#OTHERSIDE#", workerMode ? "User" : "Representative");
      if (nempty(params.get("debug")))
        html = html.replace("var showActions = false;", "var showActions = true;");
    
      html = html.replace("#AUTOOPEN#", jsBool(workerMode || eq(params.get("_autoOpenBot"), "1") || botAutoOpen(domainObj2)));
      html = html.replace("#BOT_ON#", jsBool(botOn() || eq(uri, "/demo")));
      html = html.replace("$HEADING", heading);
      html = html.replace("#WORKERMODE", jsBool(workerMode));
      html = html.replace("#NOTIFICATIONSOUND#", notificationSound);
      html = html.replace("<!-- MSGS HERE -->", "");
      html = hreplaceTitle(html, heading);

      if (eq(uri, "/demo"))
        ret hhtml(hhead(
          htitle(heading)
          + loadJQuery2()
        ) + hbody(hjavascript(html)));
      else
        ret withHeader(subBot_serveJavaScript(html));
    }

    try object serveOtherPage(req);
    
    if (eq(uri, "/")) try object serveHomePage();

    // serve uploaded files

    if (!startsWith(uri, "/crud/")) {
      // TODO: more caching
      UploadedFile fileToServe = conceptWhere UploadedFile(liveURI := urldecode(dropSlashPrefix(uri)));
      if (fileToServe != null)
        ret subBot_serveFile(fileToServe.theFile(), nempty(fileToServe.mimeType)
          ? fileToServe.mimeType
          : guessMimeTypeFromFileName(afterLastSlash(fileToServe.liveURI)));
    }

    // add more uris here

    // serve admin
    
    if (!requestAuthed) ret serveAuthForm(params.get('uri));
    // authed from here (not necessarily master-authed though)

    if (masterAuthed && eq(uri, "/dialogTree")) {
      BotStep step = getConcept BotStep(toLong(params.get("stepID")));
      if (step == null) ret serve404("Step not found");
      ret hhtml_head_title_body("Dialog Tree for " + step,
        hmobilefix() + hsansserif() + h2("Dialog Tree")
          + hcss_linkColorInherit()
          + hcss([[
            .dialogTree li { margin-top: 0.8em; }
          ]])
          + renderDialogTree(step));
    }

    if (eq(uri, "/thoughts"))
      ret serveThoughts(req);

    if (masterAuthed && eq(uri, "/search"))
      ret serveSearch(req);

    if (eq(uri, "/leads-csv")) {
      S text = leadsToCSV(conceptsWhere(Lead, mapToParams(filtersForClass(Lead, req))));
      S name = "bot-leads"
        + (authDomainObj == null ? "" : "-" + replace(str(authDomainObj), "/", "-"))
        + "-" + ymd_minus_hm() + ".csv";
      ret serveCSVWithFileName(name, text);
    }

    if (eq(uri, "/cleanConversations") && masterAuthed) {
      cleanConversations();
      ret hrefresh(baseLink + "/crud/Conversation");
    }
    
    if (eq(uri, "/deleteDeliveredDomains") && masterAuthed) {
      deleteDeliveredDomains();
      ret hrefresh(baseLink + "/crud/DeliveredDomain");
    }

    makeFramer(req);
    HTMLFramer1 framer = req.framer;

    L<Class> classes = crudClasses(req);
    L<Class> cmdClasses = req.masterAuthed ? botCmdClasses() : null;
    for (Class c : (Set<Class>) asSet(flattenList2(classes, DeliveredDomain.class, cmdClasses)))
      if (eq(uri, dropUriPrefix(baseLink, crudLink(c)))) {
        S help = mapGet(crudHelp(), c);
        if (nempty(help)) framer.add(p(help));

        HCRUD crud = makeCRUD(c, req);

        if (c == UserKeyword) {
          Scorer scorer = consistencyCheckResults();
          framer.add(p("Consistency check results: " + nTests(scorer.successes) + " OK, " + nErrors(scorer.errors)));
          if (nempty(scorer.errors))
            framer.add(ul(map(scorer.errors, e -> "Error: " + e)));
        }

        // render CRUD

        crud.processSortParameter(params);
        framer.add(crud.renderPage(params));

        // javascript magic to highlight table row according to anchor in URL
        framer.add(hjs_markRowMagic());
      }
      
    if (eq(uri, "/emojis")) {
      framer.title = "Emoji List";
      framer.contents.add(htmlTable2(map(emojiShortNameMap(),
        (code, emoji) -> litorderedmap("Shortcode" := code, "Emoji" := emoji))));
    }

    if (eq(uri, "/embedCode")) {
      S goDaddyEmbedCode = "";
      framer.title = "Embed Code";
      framer.contents.add(h2("Chat bot embed code")
        + (empty(goDaddyEmbedCode) ? "" :
          h3("GoDaddy Site Builder")
          + p("Add an HTML box with this code:")
          + pre(htmlEncode2(goDaddyEmbedCode))
          + h3("Other"))
          
        + p("Add this code in front of your " + tt(htmlEncode2("</body>")) + " tag:")
        + pre(htmlEncode2(hjavascript_src_withType("https://" + request.domain() + baseLink + "/script"))));
    }
  
    if (eq(uri, "/stats")) {
      long us = db_mainConcepts().uncompressedSize;
      framer.add(htableRaw2(llNonNulls(
        ll("DB size on disk", htmlEncode2(toK_str(fileSize(conceptsFile())))),
        us == 0 ? null : ll("DB size uncompressed", htmlEncode2(toK_str(us))),
        ll("DB load time", htmlEncode2(renderDBLoadTime())),
        ll("DB save time", htmlEncode2(renderDBSaveTime())),
        ll("Last saved", htmlEncode2(renderHowLongAgo(db_mainConcepts().lastSaveWas))),
        ), ll(class := "responstable"), null, null));
    }

    if (eq(uri, "/unpackToWeb") && req.masterAuthed) {
      UploadedFile file = getConcept UploadedFile(parseLong(req.params.get("fileID")));
      if (file == null) ret serve404("File not found");
      S path = unnull(req.params.get("path"));
      File zipFile = file.theFile();
      int changes = 0;
      for (S filePath : listZIP(zipFile)) {
        S liveURI = appendSlash(path) + filePath;
        UploadedFile entry = uniq UploadedFile(+liveURI);
        byte[] data = loadBinaryFromZip(zipFile, filePath);
        if (!fileContentsIs(entry.theFile(), data)) {
          ++changes;
          saveBinaryFile(entry.theFile(), data);
          touchConcept(entry);
        }
      }

      ret "Unpacked, " + nChanges(changes);
    }

    // put more AUTHED uris here

    //if (masterAuthed) framer.addNavItem(baseLink + "/search", "Search");

    if (settings().enableWorkerChat)
      framer.addNavItem(baseLink + "/worker", "Worker chat");
      
    framer.addNavItem(baseLink + "?logout=1", "Log out");
    ret framer.render();
  } // end of html2

  // put early stuff here
  O serve2(Req req) { null; }

  O serveOtherPage(Req req) { null; }

  <A> A getDomainValue(Domain domainObj, IF1<Domain, A> getter, A defaultValue) {
    if (domainObj != null)
      try object A val = getter.get(domainObj);
    try object A val = getter.get(defaultDomain());
    ret defaultValue;
  }

  void makeFramer(Req req) {
    new HTMLFramer1 framer;
    req.framer = framer;
    framer.title = adminName + " " + squareBracket(
      loggedInUserDesc(req));
    framer.addInHead(hsansserif() + hmobilefix() + hresponstable()
      + hcss_responstableForForms());
    framer.addInHead(loadJQuery2());
    framer.addInHead(hjs_selectize());
    framer.addInHead(hjs_copyToClipboard());
    framer.addInHead(hNotificationPopups());
    framer.addInHead(hcss_linkColorInherit());
    if (useWebSockets) {
      framer.addInHead(hreconnectingWebSockets());

      framer.addInHead(hjs([[
        var webSocketQuery = "";
        var ws;
        $(document).ready(function() {
          ws = new ReconnectingWebSocket(((window.location.protocol === "https:") ? "wss://" : "ws://") + window.location.host + ]]
            + jsQuote(baseLink + req.uri) + [[ + webSocketQuery);
          var wsReady = false;
          ws.onopen = function(event) {
            wsReady = true;
            console.log("WebSocket ready!");
            ws.onmessage = ]] + js_evalOnWebSocketMessage() + [[;
          };
        });
      ]]));
    }
    if (showTalkToBotLink)
      framer.addNavItem(simulateDomainLink(req.authDomain), "Talk to bot", targetBlank := true);
    L<Class> classes = crudClasses(req);
    L<Class> cmdClasses = req.masterAuthed ? botCmdClasses() : null;
    for (Class c : classes)
      framer.addNavItem(makeClassNavItem(c, req));
    if (nempty(cmdClasses))
      framer.contents.add(p("Bot actions: " + joinWithVBar(map(cmdClasses, c -> makeClassNavItem(c, req)))));
  }

  // make and adapt CRUD for class    
  <A extends Concept> HCRUD makeCRUD(Class<A> c, Req req, HTMLFramer1 framer default req.framer) {
    HCRUD_Concepts data = crudData(c, req);
    data.referencesBlockDeletion = true;
    HCRUD crud = new(crudLink(c), data);
    crud.params = req.params;
    crud.buttonsOnTop = true;
    crud.haveJQuery = crud.haveSelectizeJS = true;
    crud.sortable = true;
    crud.paginate = true;
    crud.paginator.step = 25;
    crud.cmdsLeft = true;
    crud.showCheckBoxes = true;
    crud.tableClass = "responstable";
    //framer.addInHead(hcss(".crudForm { width: 800px; }"));
    crud.formTableClass = "responstableForForms";

    crud.renderValue_inner = value -> {
      S html = crud.renderValue_inner_base(value);
      if (value cast Concept)
        ret ahref(conceptLink(value), html);
      ret html;
    };
    
    if (c == Conversation) {
      // Note: fields for list are created from scratch in massageItemMapForList
      crud.nav = () -> joinNemptiesWithVBar(crud.nav_base(),
        ahref(baseLink + "/cleanConversations", "Clean list"));
      crud.unshownFields = litset("oldDialogs", "worker", "botOn", "lastPing", "cookie", "form", "testMode", "userMessageProcessed", "newDialogTriggered");
    }
    if (c == DeliveredDomain || c == Conversation || c == Lead || c == ConversationFeedback) 
      crud.allowCreate = crud.allowEdit = false;
      
    if (c == Settings) {
      crud.singleton = countConcepts(Settings) == 1;
      if (!settings().multiLanguageMode) crud.unshownFields = litset("defaultLanguage");
      framer?.add(p(joinNemptiesWithVBar(
        ahref(baseLink + "/emojis", "Emojis"),
        ahref(baseLink + "/stats", "Stats"),
        ahref(baseLink + "/embedCode", "Embed Code"))));
    }

    if (c == Lead)
      framer?.add(p(ahref(baseLink + "/leads-csv", "Export as CSV")));
      
    if (c == BotOutgoingQuestion) {
      crud.unlistedFields = litset("multipleChoiceSeparator", "placeholder");

      crud.massageFormMatrix = (map, matrix) -> {
        int idx = indexOfPred(matrix, row -> cic(first(row), "Actions"));
        BotOutgoingQuestion item = getConcept BotOutgoingQuestion(toLong(map.get(crud.idField())));
        printVars_str("massageFormMatrix", +item, +idx);
        if (item == null) ret;
        if (idx < 0) ret;
        LS row = matrix.get(idx);
        row.set(1, hcrud_mergeTables(row.get(1), tag table(map(s -> tr(td() + td(htmlEncode2(s))), item.buttons)), "for:"));
      };
    }

    data.addFilters(filtersForClass(c, req));

    if (c == Domain) {
      crud.renderCmds = map -> {
        Domain dom = getConcept Domain(toLong(crud.itemID(map)));
        ret joinNemptiesWithVBar(crud.renderCmds_base(map),
          targetBlank(simulateDomainLink(dom.domainAndPath), "Talk to bot"));
      };
      framer?.add(p("See also the " + ahref(crudLink(DeliveredDomain), "list of domains the bot was delivered on")));
    }
    
    if (c == DeliveredDomain) {
      crud.nav = () -> joinNemptiesWithVBar(crud.nav_base(),
        ahref("/deleteDeliveredDomains", "Delete all"));
      crud.renderCmds = map -> {
        DeliveredDomain dom = getConcept DeliveredDomain(toLong(crud.itemID(map)));
        ret joinNemptiesWithVBar(crud.renderCmds_base(map),
          targetBlank(simulateDomainLink(dom.domain), "Talk to bot"));
      };
    }

    if (c == UploadedImage) {
      crud.massageFormMatrix = (map, matrix) -> {
        UploadedImage item = getConcept UploadedImage(toLong(crud.itemID(map)));
        addInFront(matrix, ll("Upload image",
          hjs_imgUploadBase64Encoder()
          + himageupload(id := "imgUploader")
          + hhiddenWithIDAndName("f_img_base64")));
      };
      crud.formParameters = () -> paramsPlus(crud.formParameters_base(), onsubmit := "return submitWithImageConversion(this)");
    }

    if (c == UploadedSound) {
      crud.massageFormMatrix = (map, matrix) -> {
        UploadedSound item = getConcept UploadedSound(toLong(crud.itemID(map)));
        matrix.add(ll("Upload MP3",
          hjs_fileUploadBase64Encoder()
          + hmp3upload(id := "fileUploader")
          + hhiddenWithIDAndName("f_file_base64")));
      };
      crud.formParameters = () -> litparams(onsubmit := "return submitWithFileConversion(this)");
    }

    if (c == UploadedFile) {
      crud.massageFormMatrix = (map, matrix) -> {
        UploadedFile item = getConcept UploadedFile(toLong(crud.itemID(map)));
        matrix.add(ll("Upload File",
          hjs_fileUploadBase64Encoder()
          + hfileupload(id := "fileUploader")
          + hhiddenWithIDAndName("f_file_base64")
          + hjs([[
            $("input[name=thefile]").change(function(e) {
              var name = e.target.files[0].name;
              if (name)
                $("input[name=f_name]").val(name);              
            });
           ]]
          )));
      };
      crud.formParameters = () -> litparams(onsubmit := "return submitWithFileConversion(this)");
    }

    if (isSubclassOf(c, BotStep)) {
      crud.renderCmds = map -> {
        BotStep step = getConcept BotStep(toLong(crud.itemID(map)));
        ret joinNemptiesWithVBar(crud.renderCmds_base(map),
          targetBlank(simulateScriptLink(step), "Test in bot"),
          hPopDownButton(
            targetBlank(baseLink + "/dialogTree?stepID=" + step.id, "Show Dialog Tree")));
      };
      framer?.add(p("See also the " + ahref(crudLink(DeliveredDomain), "list of domains the bot was delivered on")));
    }

    if (c == Worker) {
      crud.unshownFields = litset("lastOnline");
      crud.uneditableFields = litset("available", "lastOnline");
    }

    // show references
    
    crud.postProcessTableRow = (item, rendered) -> {
      long id = parseLong(item.get("id"));
      Concept concept = getConcept(id);
      if (concept == null) ret rendered;
      Cl<Concept> refs = allBackRefs(concept);
      if (empty(refs)) ret rendered;
      refs = sortedByConceptID(refs);
      int more = l(refs)-maxRefsToShow;
      ret mapPlus(rendered, span_title("Where is this object used", "References"),
        joinMap(takeFirst(maxRefsToShow, refs), ref -> p(ahref(conceptLink(ref),
          htmlEncode_nlToBr_withIndents(str(ref)))))
        + (more > 0 ? "<br>+" + more + " more" : ""));
    };

    ret crud;
  }

  bool isLinkableConcept(Concept c, Req req) {
    ret req != null && c != null && contains(allLinkableClasses(req), _getClass(c));
  }

  S conceptLink(Concept c, Req req default null) {
    if (req != null && !isLinkableConcept(c, req)) null;
    ret c == null ? null : appendQueryToURL(crudLink(c.getClass()), selectObj := c.id) + "#obj" + c.id;
  }

  S conceptEditLink(Concept c, O... _) {
    ret c == null ? null : appendQueryToURL(crudLink(c.getClass()), paramsPlus(_, edit := c.id));
  }

  S conceptDuplicateLink(Concept c) {
    ret c == null ? null : appendQueryToURL(crudLink(c.getClass()), duplicate := c.id);
  }

  <A extends Concept> S makeClassNavItem(Class<A> c, Req req) {
    HCRUD_Concepts<A> data = crudData(c, req);
    HCRUD crud = makeCRUD(c, req, null);
    Map filters = filtersForClass(c, req);
    int count = countConcepts(c, mapToParams(filters));
    //print("Count for " + c + " with " + filters + ": " + count);
    ret (crud.singleton ? ""
      : n2(count) + " ")
      + ahref(crudLink(c), count == 1 ? data.itemName() : data.itemNamePlural())
      + (!crud.actuallyAllowCreate() ? "" : " " + ahref(crud.newLink(), "+", title := "Create New " + data.itemName()));
  }

  Cl<Class> allLinkableClasses(Req req) {
    ret joinSets(crudClasses(req), botCmdClasses());
  }

  L<Class> botCmdClasses() {
    ret dynNewBot2_botCmdClasses();
  }

  L<Class> dynNewBot2_botCmdClasses() {
    L<Class> l = ll(BotMessage, UploadedImage, BotImage, BotOutgoingQuestion, BotPause, Sequence);
    if (settings().multiLanguageMode) l.add(BotSwitchLanguage);
    ret l;
  }

  L<Class> crudClasses(Req req) {
    ret dynNewBot2_crudClasses(req);
  }
  
  L<Class> dynNewBot2_crudClasses(Req req) {
    if (req.masterAuthed) {
      L<Class> l = ll(Conversation, Lead, ConversationFeedback, Domain, UserKeyword, UploadedSound, UploadedFile);
      if (enableAvatars) l.add(Avatar);
      l.add(Settings);
      if (settings().multiLanguageMode) l.add(Language);
      if (settings().enableWorkerChat) l.add(Worker);
      if (recordExecutionsAndInputs) addAll(l, ExecutedStep, InputHandled);
      ret l;
    } else
      ret ll(Conversation, Lead, ConversationFeedback);
  }

  MapSO filtersForClass(Class c, Req req) {
    if (c == Conversation.class && req.authDomainObj != null)
      ret litmap(domainObj := req.authDomainObj);
    if (eqOneOf(c, Lead.class, ConversationFeedback.class) && req.authDomainObj != null)
      ret litmap(domain := req.authDomainObj);
    null;
  }

  S crudLink(Class c) {
    ret baseLink + "/crud/" + shortName(c);
  }

  <A extends Concept> HCRUD_Concepts<A> crudData(Class<A> c, Req req) {
    HCRUD_Concepts<A> cc = new(c);
    cc.trimAllSingleLineValues = true;
    cc.fieldHelp("comment", "Put any comment about this object here");
    cc.itemName = () -> replaceIfEquals(
      dropPrefix("Bot ", cc.itemName_base()), "Jump Button", "Button");
      
    cc.valueConverter = new DefaultValueConverterForField {
      public OrError<O> convertValue(O object, Field field, O value) {
        // e.g. for "buttons" field
        print("convertValue " + field + " " + className(value));
        if (value instanceof S && eq(field.getGenericType(), type_LS())) {
          print("tlft");
          ret OrError(tlft((S) value));
        }
        ret super.convertValue(object, field, value);
      }
    };

    if (c == InputHandled) {
      cc.itemNamePlural = () -> "Inputs Handled";
    }
    
    if (c == Domain) {
      cc.fieldHelp(
        "domainAndPath", "without http:// or https://",
        "botName", "Bot name for this domain (optional)",
        "headerColorLeft", "Hex color for left end of header gradient (optional)",
        "headerColorRight", "Hex color for right end of header gradient (optional)",
        autoOpenBot := "Open the bot when page is loaded");
      cc.massageItemMapForList = (item, map) -> {
        map.put("domainAndPath", HTML(b(htmlEncode2(item/Domain.domainAndPath))));
        map.put("password", SecretValue(map.get("password")));
      };
    }

    if (c == BotOutgoingQuestion) {
      cc.dropEmptyListValues = false; // for button actions
      cc.addRenderer("displayText", new HCRUD_Data.TextArea(80, 10));
      cc.addRenderer("buttons", new HCRUD_Data.TextArea(80, 10, o -> lines_rtrim((L) o)));
      cc.fieldHelp(
        displayText := displayTextHelp(),
        key := [[Internal key for question (any format, can be empty, put "-" to disable storing answer)]],
        defaultValue := "What the input field is prefilled with",
        placeholder := "A text the empty input field shows as a hint",
        buttons := "Buttons to offer as standard answers (one per line, use | to add a shortened version submitted as user input)",
        allowFreeText := "Can user enter free text in addition to clicking a button?",
        multipleChoice := "Can user select multiple buttons?",
        optional := "Can user skip the question by entering nothing?",
        multipleChoiceSeparator := "Internal field, just leave as it is",
        answerCheck := "Select this to validate user's answer against a pattern",
        buttonActions := "Optional actions (one for each button in the list above)",
      );
      cc.addRenderer("answerCheck", new HCRUD_Data.ComboBox("", "email address", "phone number"));
      cc.massageItemMapForList = (item, map) -> {
        map.put("buttons", HTML(ol_htmlEncode(item/BotOutgoingQuestion.buttons)));
        map.put("steps", HTML(ul(
          map(item/BotOutgoingQuestion.buttonActions, action ->
            ahref(conceptLink(action), htmlEncode2_nlToBr(str(action)))))));
       };
    }

    if (c == UserKeyword) {
      cc.fieldHelp(
        language := "Language that this keyword matches in (optional)",
        pattern := targetBlank("http://code.botcompany.de:8081/getraw.php?id=1030319", "'PhraseCache'") + " matching pattern (most important field)",
        examples := "Example inputs that should be matched (one per line, optional)",
        counterexamples := "Example inputs that should be NOT matched (one per line, optional)",
        action := "What bot does when input is matched",
        enabled := "Uncheck to disable this keyword",
        priority := "If set, this keyword overrides other input handlers (e.g. from outgoing questions)",
        precedence := "Precedence over other keywords (higher value = match first)",
      );
        
      for (S field : ll("examples", "counterexamples"))
        cc.addRenderer(field, new HCRUD_Data.TextArea(80, 5, o -> lines_rtrim((L) o)));
      cc.massageItemMapForList = (item, map) -> {
        map.put("examples", lines_rtrim(item/UserKeyword.examples));
        map.put("counterexamples", lines_rtrim(item/UserKeyword.counterexamples));
      };
    }

    if (c == Settings) {
      cc.fieldHelp(
        mainDomainName := "Domain where bot is hosted (to correctly send image and sound links)",
        botTypingDelay := "Delay in seconds before bot sends message",
        preferredCountryCodes := [[Country codes to display first in list (e.g. "+1, +91")]],
        notificationSound := "Bot message notification sound (can leave empty, then we use the "
          + ahref(defaultNotificationSound(), "default sound"),
        talkToBotOnlyWithAuth := "Show bot only to authorized persons");        
    }

    if (c == Conversation)
      cc.massageItemMapForList = (item, map) -> {
        replaceMap(map, litorderedmap(
          id := map.get("id"),
          "IP" := map.get("ip"),
          "started" := formatLocalDateWithMinutes(item/Conversation.created),
          country := map.get("country"),
          msgs := lines_rtrim(item/Conversation.msgs),
          avatar := map.get("avatar"),
          answers := htmlEncode2(renderColonProperties(item/Conversation.answers)),
          stack := HTML(ol(
            cloneMap(item/Conversation.stack, activeSeq -> htmlEncode2(str(activeSeq))))),
        ));
      };

    if (c == BotMessage) {
      cc.addRenderer("text", new HCRUD_Data.TextArea(80, 10));
      cc.addRenderer("specialPurpose", new HCRUD_Data.ComboBox(itemPlus("", keys(specialPurposes))));
      cc.fieldHelp(
        text := displayTextHelp(),
        specialPurpose := "Special occasion when to display this message");
    }

    if (c == Form)
      cc.massageItemMapForList = (item, map) -> {
        map.put("steps", pnlToStringWithEmptyLines_rtrim(item/Form.steps));
      };

    if (c == Sequence)
      cc.massageItemMapForList = (item, map) -> {
        map.put("steps", HTML(ol(
          map(item/Sequence.steps, step ->
            ahref(conceptLink(step), htmlEncode2_nlToBr(str(step)))))));
      };
      
    if (c == BotImage)
      cc.massageItemMapForList = (item, map) -> {
        S url = item/BotImage.imageURL;
        if (isURL(url))
          map.put("Image Preview", HTML(himg(url, style := hcrud_imagePreviewStyle())));
        map.put(cc.fieldNameToHTML("imageURL"), HTML(
            htmlEncode2(url) + " "
          + himgsnippet(#1101381, onclick := 
              "copyToClipboard(" + jsQuote(url) + "); "
            + "window.createNotification({ theme: 'success', showDuration: 3000 })({ message: "
            + jsQuote("Image URL copied to clipboard")
            + "});")));
      };

    if (c == UploadedImage) {
      cc.massageItemMapForList = (item, map) -> {
        S url = item/UploadedImage.imageURL();
        File f = item/UploadedImage.imageFile();
        map.put("Image Size", toK_str(fileSize(f)));
        if (fileSize(f) > 0)
          map.put("Image Preview", HTML(himg(url, style := hcrud_imagePreviewStyle())));
      };
      IVF2 massageItemMapForUpdate_old = cc.massageItemMapForUpdate;
      cc.massageItemMapForUpdate = (item, map) -> {
        cc.massageItemMapForUpdate_fallback(massageItemMapForUpdate_old, item, map);
        S base64 = (S) map.get("img_base64");
        print("Got base64 data: " + l(base64));
        if (nempty(base64)) {
          File f = item/UploadedImage.imageFile();
          saveFileVerbose(f, base64decode(base64));
        }
        map.remove("img_base64");
      };
    }
    
    if (c == UploadedSound) {
      cc.massageItemMapForList = (item, map) -> {
        S url = item/UploadedSound.soundURL();
        File f = item/UploadedSound.soundFile();
        map.put("Sound Size", toK_str(fileSize(f)));
        if (fileSize(f) > 0)
          map.put("Test Sound", HTML(ahref(url, "Test Sound")));
      };
      IVF2 massageItemMapForUpdate_old = cc.massageItemMapForUpdate;
      cc.massageItemMapForUpdate = (item, map) -> {
        cc.massageItemMapForUpdate_fallback(massageItemMapForUpdate_old, item, map);
        S base64 = (S) map.get("file_base64");
        print("Got base64 data: " + l(base64));
        if (nempty(base64)) {
          File f = item/UploadedSound.soundFile();
          saveFileVerbose(f, base64decode(base64));
        }
        map.remove("file_base64");
      };
    }

    if (c == UploadedFile) {
      cc.fieldHelp(
        name := "Also used as file name when downloading",
        mimeType := "Content type (MIME type) for this file (optional)");
      cc.massageItemMapForList = (item, map) -> {
        S url = item/UploadedFile.downloadURL();
        File f = item/UploadedFile.theFile();
        map.put("File Size", toK_str(fileSize(f)));
        if (fileSize(f) > 0)
          map.put("Download", HTML(ahref(url, "Download")));
      };
      IVF2 massageItemMapForUpdate_old = cc.massageItemMapForUpdate;
      cc.massageItemMapForUpdate = (item, map) -> {
        cc.massageItemMapForUpdate_fallback(massageItemMapForUpdate_old, item, map);
        S base64 = (S) map.get("file_base64");
        print("Got base64 data: " + l(base64));
        if (nempty(base64)) {
          File f = item/UploadedFile.theFile();
          saveFileVerbose(f, base64decode(base64));
        }
        map.remove("file_base64");
      };
    }

    if (c == Worker) {
      cc.massageItemMapForList = (item, map) -> {
        AbstractBotImage image = item/Worker.image!;
        if (image != null)
          map.put("Image Preview", HTML(himg(image.imageURL(), style := hcrud_imagePreviewStyle())));
      };
    }

    if (c == Avatar) {
      cc.fieldHelp(shift := [[hours when avatar is active, e.g. 8-18 (leave empty if always on shift, put 0-0 if never on shift)]]);
      cc.massageItemMapForList = (item, map) -> {
        map.put("On Shift Now" := item/Avatar.onShift());
      };
    }

    ret cc;
  }

  Map<Class, S> crudHelp() {
    ret litmap(DeliveredDomain, "This list is filled by the bot with the domains it was delivered on. "
      + "Some may be bogus, we write down whatever the browser sends.");
  }

  void saveDeliveredDomain(S domain) {
    if (empty(domain)) ret;
    uniqCI(DeliveredDomain, domain := beforeColonOrAll(domain));
  }

  void cleanConversations {
    withDBLock(r {
      cdelete(filter(list(Conversation), c -> l(c.msgs) <= 1
        && elapsedSeconds_timestamp(c.created) >= 60 & c.lastPing == 0));
    });
  }
  
  void deleteDeliveredDomains {
    cdelete DeliveredDomain();
  }

  O serveThoughts(Req req) {
    new HTMLFramer1 framer;
    framer.addInHead(hsansserif() + hmobilefix());
    framer.add(div_floatRight(hbutton("Reload", onclick := "location.reload()")));
    framer.add(h2("Bot Thoughts"));
    if (req.conv == null) framer.add("No conversation");
    else {
      framer.add(p("Conversation cookie: " + req.conv.cookie));
      Conversation conv = req.conv;
      framer.add(h3("Stack"));
      framer.add(empty(conv.stack) ? p("empty")
        : ol(lmap htmlEncode2_gen(reversed(conv.stack))));
    }
    ret framer.render();
  }

  O serveSearch(Req req) {
    makeFramer(req);
    HTMLFramer1 framer = req.framer;
    framer.add("Search here");
    ret framer.render();
  }
  
  transient bool debug;

  // always returns an object
  Domain findDomainObj(S domain) {
    Domain domainObj = conceptWhereCI Domain(domainAndPath := domain);
    if (domainObj == null) domainObj = defaultDomain();
    ret domainObj;
  }
  
  Domain defaultDomain() {
    ret uniqCI Domain(domainAndPath := "<default>");
  }
  
  // Web Chat Bot Include
  
  transient int longPollTick = 200;
  transient int longPollMaxWait = 1000*30; // lowered to 30 seconds
  transient int activeConversationSafetyMargin = 15000; // allow client 15 seconds to reload
  
  transient Set<S> specialButtons = litciset("Cancel", "Back", "Abbrechen", "Zurück");
  
  S dbStats() {
    Cl<Conversation> all = list(Conversation);
    int nRealConvos = countPred(all, c -> l(c.msgs) > 1);
    //int nTestConvos = countPred(all, c -> startsWith(c.cookie, "test_"));
    //ret n2(l(all)-nTestConvos, "real conversation") + ", " + n2(nTestConvos, "test conversation");
    ret nConversations(nRealConvos);
  }
  
  S realPW() {
    ret loadTextFileOrCreateWithRandomID(realPWFile());
  }

  File realPWFile() {
    ret secretProgramFile("password.txt");
  }
  
  S serveAuthForm(S redirect) {
    redirect = dropParamFromURL(redirect, "logout"); // don't log us out right again
    if (empty(redirect)) redirect = baseLink + "/";
    ret hhtml(hhead(htitle(botName + ": Log-In")
      + hsansserif() + hmobilefix()) + hbody(hfullcenter(
      authFormHeading()
      + hpostform(
          hhidden(+redirect)
          + tag table(
            (enableUsers ? tr(td("User:") + td(hinputfield("user")) + td()) : "")
          + tr(td("Password:") + td(hpassword("pw")) + td(hsubmit("Log in")))),
        action := baseLink)
      + (!showTalkToBotLink? "" : p(ahref(baseLink + "/demo", "Talk to bot")))
      + (!showRegisterLink ? "" : p(hbuttonLink(baseLink + "/register", "Register as new user")))
      + authFormMoreContent()
      )));
  }

  S authFormHeading() { ret h3_htmlEncode(adminName); }  
  S authFormMoreContent() { ret ""; }
  
  S leadsToCSV(Cl<Lead> leads) {
    LS fields = ciContentsIndexedList();
    new LL rows;
    fOr (Lead lead : leads) {
      new L row;
      fOr (S key, O val : mapPlus_inFront((MapSO) (Map) lead.answers,
        "Date" := formatLocalDateWithMinutes(lead.date.date),
        "Domain" := lead.domain!,
        "Conversation ID" := conceptID(lead.conversation!))) {
        int idx = fields.indexOf(key);
        if (idx < 0) idx = addAndReturnIndex(fields, key);
        listSet(row, idx, val, "");
      }
      rows.add(row);
    }
    ret formatCSVFileForExcel2(itemPlusList(fields, rows));
  }
  
  S countryCodeOptions(Conversation conv) {
    Cl<S> countryCodes = putSetElementsFirst(keys(countryDialCodesMultiMap()), splitAtComma_trim(settings().preferredCountryCodes));
    S selected = dialCodeStringForCountryCode(conv.country);
    ret mapToLines(c -> { // c == dial code
      L<CountryDialCode> cdc = countryDialCodesMultiMap().get(c);
      S text = nempty(cdc) ? c + " [" + joinWithComma(collectSorted countryCode(cdc)) + "]" : c;
      ret tag option(text, value := c, selected := eq(c, selected) ? html_valueLessParam() : null);
    }, itemPlus("", countryCodes));
  }
  
  L<BotStep> addTypingDelays(L<BotStep> steps) {
    if (settings().botTypingDelay <= 0) ret steps;
    //printStackTrace();
    ret concatMap(steps, step -> {
      double delay = step.preTypingDelay();
      if (delay <= 0) ret ll(step);
      ret ll(new BotSendTyping, new BotPause(delay), step);
    });
  }
  
  BotMessage messageForPurpose(S specialPurpose) {
    ret conceptWhere BotMessage(+specialPurpose);
  }
  
  O serveLeadsAPI(virtual WebRequest req) {
    SS headers = cast rcall headers(req);
    SS params = cast get params(req);
    S tenantID = params.get("tenantID");
    if (empty(tenantID)) ret serveJSONError("Empty tenantID");
    S pw = params.get("pw");
    if (empty(pw)) pw = dropPrefix_trim("Bearer ", headers.get("Authorization"));
    if (empty(pw)) ret serveJSONError("Empty password");
    Domain domain = conceptWhere Domain(+tenantID);
    if (domain == null) ret serveJSONError("Tenant ID not found");
    if (neq(domain.password, pw)) ret serveJSONError("Bad passsword");
  
    Cl<Lead> leads = conceptsWhere Lead(+domain);
    ret serveJSON(map(leads, lead -> {
      Map map = litorderedmap(+tenantID,
        leadID := lead.id,
        domain := str(lead.domain),
        date := formatLocalDateWithSeconds(lead.created),
        data := lead.answers
      );
      ret map;
    }));
  }
  
  O serveJSONError(S error) {
    ret serveJSON(litorderedmap(+error));
  }
  
  S botImageForDomain(Domain domainObj) {
    S botImg = imageSnippetURLOrEmptyGIF(chatHeaderImageID);
    if (domainObj != null && domainObj.botImage.has())
      botImg = domainObj.botImage->imageURL();
    else if (defaultDomain().botImage.has())
      botImg = defaultDomain().botImage->imageURL();
    ret botImg;
  }
  
  S simulateDomainLink(S domain) {
    ret appendQueryToURL(baseLink + "/demo", _botConfig := makePostData(+domain), _newConvCookie := 1, _autoOpenBot := 1);
  }
  
  S simulateScriptLink(BotStep script) {
    ret appendQueryToURL(baseLink + "/demo", _botConfig := makePostData(mainScript := script.id), _newConvCookie := 1, _autoOpenBot := 1);
  }

  Scorer consistencyCheckResults() {
    if (consistencyCheckResults == null)
      consistencyCheckResults = doConsistencyChecks();
    ret consistencyCheckResults;
  }

  Scorer doConsistencyChecks() {
    Scorer scorer = scorerWithSuccessesAndErrors();
    for (UserKeyword uk) try {
      localConsistencyCheck(uk, scorer);
    } catch e {
      scorer.addError(htmlEncode2(str(e)));
    }
    try {
      globalConsistencyCheck(scorer);
    } catch e {
      scorer.addError(htmlEncode2(str(e)));
    }
    ret scorer;
  }

  void localConsistencyCheck(UserKeyword uk, Scorer scorer) {
    S patHTML = span_title(str(uk.parsedPattern()), htmlEncode2(uk.pattern));
    S item = "[Entry " + ahref(conceptLink(uk), uk.id) + "] ";
    
    fOr (S input : uk.examples) {
      bool ok = uk.matchWithTypos(input);
      scorer.add(ok, ok ? item + "Example OK: " + patHTML + " => " + htmlEncode2(quote(input))
        : item + "Example doesn't match pattern: " + patHTML + " => " + htmlEncode2(quote(input)));
    }
    
    fOr (S input : uk.counterexamples) {
      bool ok = !uk.matchWithTypos(input);
      scorer.add(ok, ok ? item + "Counterexample OK: " + patHTML + " => " + htmlEncode2(quote(input))
        : item + "Counterexample matches pattern: " + patHTML + " => " + htmlEncode2(quote(input)));
    }
  }

  void globalConsistencyCheck(Scorer scorer) {
    L<UserKeyword> list = concatLists(enabledUserKeywordsForPriority(true), enabledUserKeywordsForPriority(false));
    for (UserKeyword uk : list)
      fOr (S example : uk.examples) {
        S item1 = ahref(conceptLink(uk), uk.id);
        UserKeyword found = firstThat(list, uk2 -> uk2.matchWithTypos(example));
        if (found == null) continue; // Error should be found in local check
        if (found == uk)
          scorer.addOK("[Entry " + item1 + "] Global check OK for input " + htmlEncode2(quote(example)));
        else {
          S item2 = ahref(conceptLink(found), found.id);
          S items = "[Entries " + item1 + " and " + item2 + "] ";
          scorer.addError(items + "Input " + htmlEncode2(quote(example)) + " shadowed by pattern: " + htmlEncode2(quote(found.pattern))
            + ". Try reducing precedence value of entry " + item2 + " or increasing precedence value of entry " + item1);
        }
      }
  }

  // sorted by precedence
  static Cl<UserKeyword> enabledUserKeywordsForPriority(bool priority) {
    ret sortByFieldDesc precedence(conceptsWhere UserKeyword(onlyNonNullParams(+priority)));
  }
  
  bool handleGeneralUserInput(Conversation conv, Msg msg, Bool priority) {
    for (UserKeyword qa : enabledUserKeywordsForPriority(priority)) {
      if (qa.enabled && qa.matchWithTypos(msg.text)) {
        print("Matched pattern: " + qa.pattern + " / " + msg.text);
        conv.noteEvent(EvtMatchedUserKeyword(qa, msg));
        conv.callSubroutine(qa.action!);
        conv.scheduleNextStep();
        true;
      }
    }
  
    false;
  }
  
  void didntUnderstandUserInput(Conversation conv, Msg msg) {
    // TODO
  }
  
  transient simplyCached Settings settings() {
    ret conceptWhere(Settings);
  }
  
  S displayTextHelp() {
    ret "Text to show to user (can include HTML and " + targetBlank(baseLink + "/emojis", "emojis") + ")";
  }  
  
  void botActions {
    lock lockWhileDoingBotActions ? dbLock() : null;
    while licensed {
      ScheduledAction action = lowestConceptByField ScheduledAction("time");
      if (action == null) break;
      if (action.time > now()) {
        //print("Postponing next bot action - " + (action.time-now()));
        doAfter(100, rstBotActions);
        break;
      }
      cdelete(action);
      print("Executing action " + action);
      pcall { action.run(); }
    }
  }
  
  // action must be unlisted
  void addScheduledAction(ScheduledAction action, long delay default 0) {
    action.time = now()+delay;
    registerConcept(action);
    rstBotActions.trigger();
  }
  
  transient new ThreadLocal<Out> out;
  transient new ThreadLocal<Conversation> conv;

  transient long lastConversationChange = now();
  
  void pWebChatBot {
    dbIndexing(Conversation, 'cookie, Conversation, 'worker,
      Conversation, 'lastPing,
      Worker, 'loginName, AuthedDialogID, 'cookie);
    indexConceptFieldDesc(ScheduledAction, 'time);
    indexConceptFieldCI(Domain, 'domainAndPath);
    indexConceptFieldCI(DeliveredDomain, 'domain);
    indexConceptFieldCI(CannedAnswer, 'hashTag);
    indexConceptFieldCI(Language, 'languageName);
    indexConceptField(Lead, 'domain);
    indexConceptField(ConversationFeedback, 'domain);
    indexConceptField(UploadedFile, 'liveURI);
    
    db_mainConcepts().miscMapPut(DynNewBot2, this);
  
    // legacy clean-up

    // make Settings object, index it
    indexSingletonConcept(Settings);
    uniq(Settings);
  
    // Make other singleton concepts
    uniq(BotSaveLead);
    uniq(BotSaveFeedback);
    uniq(BotClearStack);
    uniq(BotEndConversation);
    uniq(BotDoNothing);
    uniq(BotIdleMode);
    if (enableAvatars) uniq(RandomAvatar);
  }
  
  void addReplyToConvo(Conversation conv, IF0<S> think) {
    out.set(new Out);
    S reply = "";
    pcall {
      reply = think!;
    }
    Msg msg = new Msg(false, reply);
    msg.out = out!;
    conv.add(msg);
  }
  
  Msg msgFromThinkFunction(IF0<S> think) {
    out.set(new Out);
    S reply = "";
    pcall {
      reply = think!;
    }
    Msg msg = new Msg(false, reply);
    msg.out = out!;
    ret msg;
  }
  
  
  O withHeader(S html) {
    ret withHeader(subBot_noCacheHeaders(subBot_serveHTML(html)));
  }
  
  O withHeader(O response) {
    call(response, 'addHeader, "Access-Control-Allow-Origin", "*");
    ret response;
  }
  
  S renderMessageText(S text, bool htmlEncode) {
    text = trim(text);
    if (eqic(text, "!rate conversation")) text = "Rate this conversation";
    if (htmlEncode) text = htmlEncode2(text);
    text = nlToBr(text);
    ret html_emojisToUnicode(text);
  
  }
  
  void renderMessages(Conversation conv, StringBuilder buf, L<Msg> msgs,
   IPred<Msg> showButtons default msg -> msg == last(msgs)) {
    if (empty(msgs)) ret;
    for (Msg m : msgs) {
      if (m.fromUser || neqOneOf(m.text, "-", "")) {
        S html = renderMessageText(m.text, shouldHtmlEncodeMsg(m));
        appendMsg(conv, buf, m.fromUser ? defaultUserName() : botName, formatTime(m.time), html, !m.fromUser, m.fromWorker, m.out, m.labels);
      }
  
      if (showButtons.get(m)) {
        // yeah we're doing it too much
        buf.append(hscript([[$("#chat_telephone, .iti").hide(); $("#chat_message").show().prop('disabled', false).focus();]]));
        
        if (m.out != null && nempty(m.out.javaScript))
          buf.append(hscript(m.out.javaScript));
      }
    }
        
    appendButtons(buf, last(msgs).out, null);
  }
  
  void appendMsg(Conversation conv, StringBuilder buf, S name, S time, S text, bool bot, Worker fromWorker,
    Out out, LS labels) {
    bool useTrick = ariaLiveTrick;
    S tag = useTrick ? "div" : "span";
    if (bot) { // msg from bot (show avatar)
      S id = randomID();
      S author = fromWorker != null ? htmlEncode2(fromWorker.displayName ): botName;
      if (fromWorker != null) buf.append([[<div class="chat_botname"><p>]] + author + [[</p>]]);
      buf.append("<" + tag + [[ class="chat_msg_item chat_msg_item_admin"]] + (useTrick ? [[ id="]] + id + [[" aria-live="polite" tabindex="-1"]] : "") + ">");
      S imgURL;
      if (fromWorker != null && fileExists(workerImageFile(fromWorker.id)))
        imgURL = fullRawLink("worker-image/" + fromWorker.id);
      else if (conv.avatar.has())
        imgURL = conv.avatar->image->imageURL();
      else
        imgURL = botImageForDomain(conv.domainObj!);
        
      if (nempty(imgURL))
        buf.append([[
          <div class="chat_avatar">
            <img src="$IMG"/>
          </div>]]
          .replace("$IMG", imgURL));
        
      buf.append([[<span class="sr-only">]] + (fromWorker != null ? "" : botName + " ") + [[says</span>]]);
      buf.append(text);
      buf.append([[</]] + tag + [[>]]);
      if (fromWorker != null) buf.append("</div>");
      if (useTrick) buf.append(hscript("$('#" + id + "').focus();"));
    } else // msg from user (no avatar)
      buf.append(([[
        <span class="sr-only">You say</span>
        <]] + tag + [[ class="chat_msg_item chat_msg_item_user"]] + (useTrick ? [[ aria-live="polite"]] : "") + [[>$TEXT</]] + tag + [[>
      ]]).replace("$TEXT", text));

    if (nempty(labels))
      buf.append(span(
        join(" &nbsp; ", map(labels, lbl -> span("&nbsp; " + lbl + " &nbsp;"))),
        class := "labels chat_msg_item chat_msg_item_user")).append("\n");
  }
  
  S replaceButtonText(S s) {
    if (standardButtonsAsSymbols) {
      if (eqicOneOf(s, "back", "zurück")) ret unicode_undoArrow();
      if (eqicOneOf(s, "cancel", "Abbrechen")) ret unicode_crossProduct();
    }
    ret s;
  }
  
  S renderMultipleChoice(LS buttons, Cl<S> selections, S multipleChoiceSeparator) {
    print(+selections);
    Set<S> selectionSet = asCISet(selections);
    S rand = randomID();
    S className = "chat_multiplechoice_" + rand;
    S allCheckboxes = [[$(".]] + className + [[")]];
    ret joinWithBR(map(buttons, name ->
      hcheckbox("", contains(selectionSet, name),
        value := name,
        class := className) + " " + name))
      + "<br>" + hbuttonOnClick_returnFalse("OK", "submitMsg()", class := "btn btn-ok")
      + hscript(allCheckboxes
        + ".change(function() {"
        //+ "  console.log('multiple choice change');"
        + " var theList = $('." + className + ":checkbox:checked').map(function() { return this.value; }).get();"
        + "  console.log('theList: ' + theList);"
        + "  $('#chat_message').val(theList.join(" + jsQuote(multipleChoiceSeparator) + "));"
        + "});");
  }
  
  // render single-choice buttons
  S renderSingleChoice(LS buttons) {
    new LS out;
    for i over buttons: {
      S code = buttons.get(i);
      S text = replaceButtonText(code);
      S userInput = htmldecode_dropAllTags(text);
      LS split = splitAtVerticalBar(text);
      if (l(split) == 2) {
        userInput = second(split);
        text = code = first(split);
      }
      out.add(hbuttonOnClick_returnFalse(text, "submitAMsg(" + jsQuote(userInput) + ")", class := "chatbot-choice-button", title := eq(code, text) ? null : code));
      if (!specialButtons.contains(code)
        && i+1 < l(buttons) && specialButtons.contains(buttons.get(i+1)))
        out.add("&nbsp;&nbsp;");
    }
    ret lines(out);
  }
  
  void appendButtons(StringBuilder buf, Out out, Set<S> buttonsToSkip) {
    S placeholder = out == null ? "" : unnull(out.placeholder);
    S defaultInput = out == null ? "" : unnull(out.defaultInput);
    buf.append(hscript("chatBot_setInput(" + jsQuote(defaultInput) + ", " + jsQuote(placeholder) + ");"));
    if (out == null) ret;
    LS buttons = listMinusSet(out.buttons, buttonsToSkip);
    if (empty(buttons)) ret;
    printVars_str(+buttons, +buttonsToSkip);
    S buttonsHtml;
    if (out.multipleChoice)
      buttonsHtml = renderMultipleChoice(buttons,
        text_multipleChoiceSplit(out.defaultInput, out.multipleChoiceSeparator), out.multipleChoiceSeparator);
    else
      buttonsHtml = renderSingleChoice(buttons);
    
    buf.append(span(buttonsHtml,
      class := "chat_msg_item chat_msg_item_admin chat_buttons"
        + stringIf(!out.multipleChoice, " single-choice")));
  }
  
  void appendDate(StringBuilder buf, S date) {
    buf.append([[
    <div class="chat-box-single-line">
      <abbr class="timestamp">DATE</abbr>
    </div>]].replace("DATE", date));
  }
  
  bool lastUserMessageWas(Conversation conv, S message) {
    Msg m = last(conv.msgs);
    ret m != null && m.fromUser && eq(m.text, message);
  }
  
  S formatTime(long time) {
    ret timeInTimeZoneWithOptionalDate_24(timeZone, time);
  }
  
  S formatDialog(S id, L<Msg> msgs) {
    new L<S> lc;
    for (Msg m : msgs)
      lc.add(htmlencode((m.fromUser ? "> " : "< ") + m.text));
    ret id + ul(lc);
  }
  
  Conversation getConv(fS cookie) {
    ret uniq(Conversation, +cookie);
  }
  
  S errorMsg(S msg) {
    ret hhtml(hhead_title("Error") + hbody(hfullcenter(msg + "<br><br>" + ahref(jsBackLink(), "Back"))));
  }
  
  S defaultUserName() {
    ret "You";
  }
  
  bool botOn() {
    true;
  }
  
  bool botAutoOpen(Domain domain) {
    ret getDomainValue(domain, d -> d.autoOpenBot, false);
  }
  
  File workerImageFile(long id) {
    ret id == 0 ? null : javaxDataDir("adaptive-bot/images/" + id + ".jpg");
  }
  
  long activeConversationTimeout() {
    ret longPollMaxWait+activeConversationSafetyMargin;
  }
  
  AuthedDialogID authObject(S cookie) {
    AuthedDialogID auth = empty(cookie) ? null : conceptWhere AuthedDialogID(+cookie);
    printVars_str(+cookie, +auth);
    ret auth;
  }
  
  bool anyInterestingMessages(L<Msg> msgs, bool workerMode) {
    ret any(msgs, m -> m.fromUser == workerMode);
  }
  
  bool shouldHtmlEncodeMsg(Msg msg) {
    ret msg.fromUser;
  }
  
  void calcCountry(Conversation c) {
    cset(c, country := "");
    getCountry(c);
  }

  S getCountry(Conversation c) {
    if (empty(c.country) && nempty(c.ip))
      cset(c, country := ipToCountry2020_safe(c.ip));
    ret or2(c.country, "?");
  }
  
  void initAvatar(Conversation c) {
    printVars_str("initAvatar", +c, +enableAvatars, domain := c.domainObj, avatar := c.avatar);
    if (enableAvatars && !c.avatar.has() && c.domainObj.has()) {
      AbstractAvatar a = c.domainObj->avatar!;
      Avatar avatar = a?.getActualAvatar();
      print("initAvatar " + c + " => " + avatar);
      cset(c, +avatar);
    }
  }
  
  void noteConversationChange {
    lastConversationChange = now();
  }
  
  S template(S hashtag, O... params) {
    ret replaceSquareBracketVars(getCannedAnswer(hashtag), params);
  }
  
  // TODO: redefine this
  S getCannedAnswer(S hashTag, Conversation conv default null) {
    ret hashTag;
    //if (!startsWith(hashTag, "#")) ret hashTag;
    //ret or2(trim(text(conceptWhereCI CannedAnswer(+hashTag))), hashTag);
  }
  
  LS text_multipleChoiceSplit(S input, S multipleChoiceSeparator) {
    ret trimAll(splitAt(input, dropSpaces(multipleChoiceSeparator)));
  }
  
  Msg lastBotMsg(L<Msg> l) {
    ret lastThat(l, msg -> !msg.fromUser);
  }
  
  // TODO
  File uploadedImagesDir() {
    ret programDir("uploadedImages");
  }
  
  File uploadedSoundsDir() {
    ret programDir("uploadedSounds");
  }

  File uploadedFilesDir() {
    ret programDir("uploadedFiles");
  }

  S defaultNotificationSound() {
    ret "https://botcompany.de/files/1400403/notification.mp3";
  }

  S convertToAbsoluteURL(S url) {
    S domain = settings().mainDomainName;
    ret empty(domain) ? url : "https://" + domain + addSlashPrefix(url);
  }

  // only an absolute URL if settings().mainDomainName is set
  S absoluteAdminURL() {
    ret convertToAbsoluteURL(baseLink);
  }

  void editMasterPassword {
    if (!confirmOKCancel("Are you sure? This will reveal the password.")) ret;
    JTextField tf = jtextfield(realPW());
    showFormTitled2(botName + " master password",
      "Password", tf,
      r {
        S pw = gtt(tf);
        if (empty(pw)) infoBox("Empty password, won't save");
        else {
          saveTextFile(realPWFile(), pw);
          infoBox("Saved new password for " + adminName);
        }
      });
  }

  // handle user log-in (create/update AuthedDialogID concept)
  O handleAuth(Req req, S cookie) null {
    S pw = trim(req.params.get('pw));
    if (nempty(pw) && nempty(cookie)) {
      Domain authDomain;
      if (eq(pw, realPW())) 
        cset(uniq AuthedDialogID(+cookie), restrictedToDomain := null, master := true);
      else if ((authDomain = conceptWhere Domain(password := pw)) != null)
        cset(uniq AuthedDialogID(+cookie), restrictedToDomain := authDomain, master := false);
      else
        ret errorMsg("Bad password, please try again");

      S redirect = req.params.get('redirect);
      if (nempty(redirect))
        ret hrefresh(redirect);
    }
  }

  S loggedInUserDesc(Req req) {
    ret req.masterAuthed ? "super user" : htmlEncode2(req.authDomain);
  }

  O redirectToHttps(Req req) null {
    if (!req.webRequest.isHttps())
      ret subBot_serveRedirect("https://" + req.webRequest.domain() + req.uri + htmlQuery(req.params));
  }

  void startMainScript(Conversation conv) {
    initAvatar(conv);

    Domain domain = conv.domainObj!;
    if (domain == null)
      ret with botMod().addReplyToConvo(conv, () -> "Error: No domain set");
    BotStep seq = getConcept BotStep(parseLong(mapGet(conv.botConfig, "mainScript")));
    if (seq == null)
      seq = domain.mainScript!;
    if (seq == null)
      ret with botMod().addReplyToConvo(conv, () -> "Error: No main script set for " + htmlEncode2(str(domain)));
    if (executeStep(seq, conv))
      nextStep(conv);
  }

  bool isRequestFromBot(Req req) { false; }

  S modifyTemplateBeforeDelivery(S html, Req req) { ret html; }

  S headingForReq(Req req) { null; }

  O serveHomePage() { null; }

  void possiblyTriggerNewDialog(Conversation conv) {
    if (empty(conv.msgs) && conv.newDialogTriggered < conv.archiveSize()) {
      cset(conv, newDialogTriggered := conv.archiveSize());
      addScheduledAction(OnNewDialog(conv));
    }
  }

  S rewriteBotMessage(Conversation conv, S text) {
    if (!enableVars || conv == null) ret text;

    // replace <if> tags
    text = html_evaluateIfTags(text, var -> eqic("true",
      print("if " + var + ": ", calcVar(conv, dropDollarPrefix(var)))));
    
    // replace dollar vars
    print(+text);
    text = replaceDollarVars_dyn(text, var -> print("calcVar " + var, calcVar(conv, var)));

    ret text;
  }

  S calcVar(Conversation conv, S var) {
    if (conv == null) null;
    try object S val = syncGet(conv.answers, var);
    if (eqic(var, "botName") && conv.avatar.has())
      ret conv.avatar->name;
    null;
  }

  // must match what template does with CSS_ID
  S cssURL() {
    ret serveSnippetURL(cssID);
  }

  class RenderDialogTree {
    new Set<Concept> seen;

    S render(BotStep step) {
      if (!seen.add(step))
        ret "[see above] " + htmlEncode2(str(step));
        
      new LS children;
        
      if (step cast Sequence)
        for (int i, BotStep step2 : unpair iterateWithIndex1(step.steps))
          children.add("Step " + i + ": " + renderDialogTree(step2));

      if (step cast BotOutgoingQuestion)
        for (S input, BotStep step2 : unpair step.buttonsWithActions())
          children.add("If user says " + b(htmlEncode2(quote(last(splitAtVerticalBar(input)))))
            + "<br><br> => " + (step2 == null ? "no action defined"
              : renderDialogTree(step2)));
  
      ret stepToHTMLFull(step) + ulIfNempty(children, class := "dialogTree");
    }
  }

  S stepToHTMLFull(BotStep step) {
    if (step == null) ret "-";
    S link = conceptLink(step, currentReq!);
    ret ahref_unstyled(link, prependSquareBracketed(step.id))
      + ahref(link, htmlEncode2_nlToBr(step.fullToString()));
  }

  S renderDialogTree(BotStep step) {
    ret new RenderDialogTree().render(step);
  }
} // end of module

// start of concepts

concept ExecutedStep {
  new Ref<BotStep> step;
  new Ref<Conversation> conversation;
  Msg msg; // msg generated by step (if any)
}

concept InputHandled {
  // what was the input and where/when did it occur?
  
  S input; // store again here in case msg is edited later or something
  new Ref<Conversation> conversation;
  Msg msg;

  // what was the input handler?
  
  new Ref inputHandler;
  bool handled; // did inputHandler return true?
}

abstract concept BotStep {
  S comment;

  // returns true iff execution should continue immediately
  bool run(Conversation conv) { false; }

  ScheduledAction nextStepAction(Conversation conv) {
    ret Action_NextStep(conv, conv.executedSteps);
  }

  S fullToString() { ret toString(); }

  double preTypingDelay() { ret 0; }
}

concept Sequence > BotStep {
  new RefL<BotStep> steps;

  toString { ret "Sequence " + quoteOr(comment, "(unnamed)")
    + spaceRoundBracketed(nSteps(steps))
    /*+ ": " + joinWithComma(steps)*/;
  }

  bool run(Conversation conv) {
    // add me to stack
    syncAdd(conv.stack, new ActiveSequence(this));
    conv.change();
    true;
  }
}

concept BotMessage > BotStep {
  S text;
  S specialPurpose;
  //S hashtag; // optional

  sS _fieldOrder = "text specialPurpose";

  public S toString(int shorten default 40) { ret "Message" + spacePlusRoundBracketedIfNempty(comment) + ": " + newLinesToSpaces2(shorten(text, shorten)); }

  S fullToString() {
    ret toString(Int.MAX_VALUE);
  }

  bool run(Conversation conv) {
    botMod().addReplyToConvo(conv, () -> botMod().rewriteBotMessage(conv, text));
    true;
  }
  
  double preTypingDelay() { ret botMod().settings().botTypingDelay; }
}

abstract concept AbstractBotImage > BotStep {
  S altText;

  bool run(Conversation conv) {
    botMod().addReplyToConvo(conv, () -> himgsrc(imageURL(), title := altText, alt := altText, class := "chat_contentImage"));
    true;
  }

  abstract S imageURL();
  
  double preTypingDelay() { ret botMod().settings().botTypingDelay; }
}

// external image
concept BotImage > AbstractBotImage {
  S imageURL, altText;

  sS _fieldOrder = "imageURL";

  S imageURL() { ret imageURL; }

  toString { ret "Image" + spacePlusRoundBracketedIfNempty(comment) + ": " + imageURL; }
}

concept UploadedImage > AbstractBotImage {
  S imageURL() { ret botMod().convertToAbsoluteURL(botMod().baseLink + "/uploaded-image/" + id); }

  File imageFile() { ret newFile(botMod().uploadedImagesDir(), id + ".png"); }
  
  toString { ret "Image" + spacePlusRoundBracketedIfNempty(altText)
    + spacePlusRoundBracketedIfNempty(comment); }

  void delete :: before {
    if (concepts() == db_mainConcepts())
      deleteFile(imageFile());
  }
}

concept UploadedSound {
  S comment;
  
  S soundURL() { ret botMod().convertToAbsoluteURL(botMod().baseLink + "/uploaded-sound/" + id); }

  File soundFile() { ret newFile(botMod().uploadedSoundsDir(), id + ".mp3"); }
  
  toString { ret "Sound" + spacePlusRoundBracketedIfNempty(comment); }

  void delete :: before {
    if (concepts() == db_mainConcepts())
      deleteFile(soundFile());
  }
}

concept UploadedFile {
  S name, comment, mimeType, liveURI;
  
  S downloadURL(S contentType default null) {
    ret botMod().convertToAbsoluteURL(appendQueryToURL(botMod().baseLink
      + "/uploaded-file/" + id + (empty(name) ? "" : "/" + urlencode(name)),
      ct := contentType));
  }

  File theFile() { ret newFile(botMod().uploadedFilesDir(), id + ".png"); }
  
  toString { ret "File " + spacePlusRoundBracketedIfNempty(name)
    + spacePlusRoundBracketedIfNempty(comment); }

  void delete :: before {
    if (concepts() == db_mainConcepts())
      deleteFile(theFile());
  }
}

concept BotDoNothing > BotStep {
  bool run(Conversation conv) { false; }

  toString { ret "Do nothing"; }
}

concept BotSendTyping > BotStep {
  bool run(Conversation conv) {
    conv.botTyping = now();
    true;
  }
}

concept BotPause > BotStep {
  double seconds;

  *() {}
  *(double *seconds) {}

  bool run(Conversation conv) {
    botMod().addScheduledAction(nextStepAction(conv), toMS(seconds));
    false;
  }

  toString { ret "Pause for " + (seconds == 1 ? "1 second" : formatDouble(seconds, 1) + " seconds"); }
}

concept BotOutgoingQuestion > BotStep implements IInputHandler {
  S displayText;
  S key;
  S defaultValue;
  S placeholder; // null for same as displayText, "" for none
  LS buttons;
  bool allowFreeText = true; // only matters when there are buttons
  bool multipleChoice; // buttons are checkboxes
  S multipleChoiceSeparator = ", "; // how to join choices into value
  bool optional; // can be empty
  S answerCheck; // e.g. "email address"
  new RefL<BotStep> buttonActions; // what to do on each button

  sS _fieldOrder = "displayText key defaultValue placeholder buttons buttonActions allowFreeText multipleChoice optional answerCheck";

  toString { ret "Question: " + orEmptyQuotes(displayText); }

  LPair<S, BotStep> buttonsWithActions() {
    ret zipTwoListsToPairs_longer(buttons, buttonActions);
  }

  bool run(Conversation conv) {
    S text = botMod().rewriteBotMessage(conv, displayText);
    Msg msg = new Msg(false, text);
    msg.out = makeMsgOut();
    conv.add(msg);
    cset(conv, inputHandler := this);
    false;
  }

  Out makeMsgOut() {
    new Out out;
    out.placeholder = or(placeholder, displayText);
    out.defaultInput = defaultValue;
    out.buttons = cloneList(buttons);
    out.multipleChoice = multipleChoice;
    out.multipleChoiceSeparator = multipleChoiceSeparator;
    if (eqic(answerCheck, "phone number") && botMod().phoneNumberSpecialInputField)
      //out.javaScript = [[$("#chat_countrycode").addClass("visible");]];
      out.javaScript = [[$("#chat_message").hide(); $("#chat_telephone, .iti").show(); $("#chat_telephone").val("").focus();]];
    else if (!allowFreeText) {
      out.javaScript = jsDisableInputField();
      if (empty(out.placeholder)) out.placeholder = " ";
    }

    ret out;
  }

  public bool handleInput(S s, Conversation conv) {
    print("BotOutgoingQuestion handleInput " + s);

    // Store answer
    s = trim(s);
    S theKey = or2(key, displayText);
    if (nempty(theKey) && !eq(theKey, "-")) syncPut(conv.answers, theKey, s);
    conv.change();

    // Validate
    if (eqic(answerCheck, "email address") && !isValidEmailAddress_simple(s))
      ret true with handleValidationFail(conv, "bad email address");
    if (eqic(answerCheck, "phone number") && !isValidInternationalPhoneNumber(s))
      ret true with handleValidationFail(conv, "bad phone number");

    conv.removeInputHandler(this);
    
    int idx = indexOfIC(trimAll(lmap dropStuffBeforeVerticalBar(buttons)), s);
    print("Button index of " + quote(s) + " in " + sfu(buttons) + " => " + idx);
    BotStep target = get(buttonActions, idx);
    print("Button action: " + target);
    if (target != null)
      //conv.jumpTo(target);
      conv.callSubroutine(target); // We now interpret the target as a subroutine call

    // Go to next step
    print("Scheduling next step in " + conv);
    conv.scheduleNextStep();
    true; // acknowledge that we handled the input
  }
  
  double preTypingDelay() { ret empty(displayText) ? 0 : botMod().settings().botTypingDelay; }

  // show error msg and reschedule input handling
  void handleValidationFail(Conversation conv, S purpose) {
    BotMessage msg = botMod().messageForPurpose(purpose);
    S text = or2(or2(msg?.text, getOrKeep(botMod().specialPurposes, purpose)), "Invalid input, please try again");
    printVars_str("handleValidationFail", +purpose, +msg, +text);
    new Msg m;
    m.text = text;
    m.out = makeMsgOut();
    conv.add(m);
    cset(conv, inputHandler := this);
    ret;
  }
}

concept BotIdleMode > BotStep {
  toString { ret "Idle Mode (wait indefinitely)"; }
  
  bool run(Conversation conv) {
    false;
  }
}

concept BotEndConversation > BotStep {
  toString { ret "End Conversation (disable user input)"; }
  
  bool run(Conversation conv) {
    Msg msg = new Msg(false, "");
    msg.out = new Out;
    msg.out.javaScript = jsDisableInputField();
    msg.out.placeholder = " ";
    conv.add(msg);
    true;
  }
}

sS jsDisableInputField() {
  ret [[$("#chat_message").prop('disabled', true);]];
}

concept BotSaveLead > BotStep {
  toString { ret "Save Lead"; }
  
  bool run(Conversation conv) {
    cnew Lead(conversation := conv,
      domain := conv.domainObj,
      date := Timestamp(now()),
      answers := cloneMap(conv.answers));
    true;
  }
}

concept BotSaveFeedback > BotStep {
  toString { ret "Save Conversation Feedback"; }
  
  bool run(Conversation conv) {
    cnew ConversationFeedback(conversation := conv,
      domain := conv.domainObj,
      date := Timestamp(now()),
      answers := filterKeys(conv.answers, swic$("Conversation Feedback:")));
    true;
  }
}

concept BotClearStack > BotStep {
  bool run(Conversation conv) {
    syncRemoveAllExceptLast(conv.stack);
    conv.change();
    true;
  }

  toString { ret "Clear Stack [forget all running procedures in conversation]"; }
}

concept BotSwitchLanguage > BotStep {
  new Ref<Language> language;
  
  bool run(Conversation conv) {
    cset(conv, +language);
    true;
  }

  toString { ret "Switch to " + language; }
}

concept UserKeyword {
  new Ref<Language> language;
  S pattern;
  LS examples, counterexamples;
  new Ref<BotStep> action;
  bool enabled = true;
  bool priority;
  int precedence;
  transient MMOPattern parsedPattern;

  void change { parsedPattern = null; botMod().consistencyCheckResults = null; super.change(); }
  
  MMOPattern parsedPattern() {
    if (parsedPattern == null) ret parsedPattern = mmo2_parsePattern(pattern);
    ret parsedPattern;
  }

  toString { ret "User Keyword: " + pattern; }

  bool matchWithTypos(S s) {
    ret mmo2_matchWithTypos(parsedPattern(), s);
  }
}

concept Language {
  S languageName;
  S comment;

  toString { ret languageName + spaceRoundBracketed(comment); }
}

// event concepts are stored as part of the Conversation object, so they don't have IDs
// (might change this)

concept Evt {}

concept EvtMatchedUserKeyword > Evt {
  new Ref<UserKeyword> userKeyword;
  Msg msg;

  *() {}
  *(UserKeyword userKeyword, Msg *msg) { this.userKeyword.set(userKeyword); }
 }
 
concept EvtJumpTo > Evt {
  new Ref<BotStep> target;

  *() {}
  *(BotStep target) { this.target.set(target); }
}

concept EvtCallSubroutine > Evt {
  new Ref<BotStep> target;

  *() {}
  *(BotStep target) { this.target.set(target); }
}

abstract concept AbstractAvatar {
  abstract Avatar getActualAvatar();
}

concept RandomAvatar > AbstractAvatar {
  Avatar getActualAvatar() {
    ret random(filter(list(Avatar), a -> a.onShift()));
  }

  toString { ret "Choose a random avatar that is on shift"; }
}

concept Avatar > AbstractAvatar {
  S name, comment;
  S shift; // hours (e.g. "8-18", optional)
  new Ref<AbstractBotImage> image;

  Avatar getActualAvatar() { this; }

  bool onShift() {
    L<IntRange> shifts = parseBusinessHours_pcall(shift);
    if (empty(shifts)) true;
    shifts = splitBusinessHoursAtMidnight(shifts);
    int minute = minuteInDay(timeZone(botMod().timeZone));
    ret anyIntRangeContains(shifts, minute);
  }

  toString { ret empty(name) ? super.toString() : "Avatar " + name + appendBracketed(comment); }
}

/*concept ActorInConversation {
  new Ref<Conversation> conv;
  new Ref<Avatar> avatar;
  new L<ActiveSequence> stack;
}*/

// end of concepts

// special options for a message
sclass Out extends DynamicObject {
  LS buttons;
  bool multipleChoice;
  S multipleChoiceSeparator;
  S placeholder;
  S defaultInput;
  S javaScript;
}

sclass Msg extends DynamicObject {
  long time;
  bool fromUser;
  //Avatar avatar;
  Worker fromWorker;
  S text;
  Out out;
  LS labels;
  
  *() {}
  *(bool *fromUser, S *text) { time = now(); }
  *(S *text, bool *fromUser) { time = now(); }

  toString { ret (fromUser ? "User" : "Bot") + ": " + text; }

}

concept AuthedDialogID {
  S cookie;
  bool master;
  new Ref<Domain> restrictedToDomain;
  Worker loggedIn; // who is logged in with this cookie
  
  S domain() {
    ret !restrictedToDomain.has() ? null : restrictedToDomain->domainAndPath;
  }
}

//concept Session {} // LEGACY

sclass ActiveSequence {
  Sequence originalSequence;
  L<BotStep> steps;
  int stepIndex;

  *() {}
  *(BotStep step) {
    if (step cast Sequence) {
      steps = cloneList(step.steps);
      originalSequence = step;
    } else
      steps = ll(step);
    steps = botMod().addTypingDelays(steps);
  }
    
  bool done() { ret stepIndex >= l(steps); }
  BotStep currentStep() { ret get(steps, stepIndex); }
  void nextStep() { ++stepIndex; }

  toString {
    ret (stepIndex >= l(steps) ? "DONE " : "Step " + (stepIndex+1) + "/" + l(steps) + " of ")
      + (originalSequence != null ? str(originalSequence) : squareBracket(joinWithComma(steps)));
  }
}

// our base concept - a conversation between a user and a bot or sales representative
concept Conversation {
  S cookie, ip, country, domain;
  new Ref<Domain> domainObj;
  new LL<Msg> oldDialogs;
  new L<Msg> msgs;
  long lastPing;
  bool botOn = true;
  bool notificationsOn = true;
  Worker worker; // who are we talking to?
  transient long userTyping, botTyping; // sysNow timestamps
  bool testMode;
  long nextActionTime = Long.MAX_VALUE;
  long userMessageProcessed; // timestamp
  int newDialogTriggered = -1;
  transient bool dryRun;
  new Ref<Avatar> avatar;
  
  new L<ActiveSequence> stack;
  int executedSteps;
  IInputHandler inputHandler;
  new Ref<Language> language;
  //Long lastProposedDate;
  SS answers = litcimap();
  SS botConfig;
  new RefL<Concept> events; // just logging stuff
  int reloadCounter; // increment this to have whole conversation reloaded in browser

  void add(Msg m) {
    m.text = trim(m.text);
    //if (!m.fromUser && empty(m.text) && (m.out == null || empty(m.out.buttons))) ret; // don't store empty msgs from bot
    syncAdd(msgs, m);
    botMod().noteConversationChange();
    change();
    vmBus_send chatBot_messageAdded(mc(), this, m);
  }
  
  int allCount() { ret archiveSize() + syncL(msgs); }
  int archiveSize() { ret syncLengthLevel2(oldDialogs) + syncL(oldDialogs); }
  L<Msg> allMsgs() { ret concatLists_syncIndividual(syncListPlus(oldDialogs, msgs)); }
  
  long lastMsgTime() { Msg m = last(msgs); ret m == null ? 0 : m.time; }

  void incReloadCounter() {
    cset(this, reloadCounter := reloadCounter+1);
  }
  
  void turnBotOff {
    cset(this, botOn := false);
    botMod().noteConversationChange();
  }

  void turnBotOn {  
    cset(this, botOn := true, worker := null);
    S backMsg = botMod().getCannedAnswer("#botBack", this);
    if (empty(msgs) || lastMessageIsFromUser() || !eq(last(msgs).text, backMsg))
      add(new Msg(backMsg, false));
    botMod().noteConversationChange();
  }
  
  bool lastMessageIsFromUser() {
    ret nempty(msgs) && last(msgs).fromUser;
  }
  
  void newDialog {
    syncAdd(oldDialogs, msgs);
    cset(this, msgs := new L);
    syncClear(answers);
    syncClear(stack);
    // TODO: clear scheduled actions?
    change();
    print("newDialog " + archiveSize() + "/" + allCount());

    vmBus_send chatBot_clearedSession(mc(), this);
    //botMod().addScheduledAction(new OnNewDialog(this));
  }

  void jumpTo(BotStep step) {
    syncPopLast(stack); // terminate inner script
    print("Jumping to " + step);
    noteEvent(EvtJumpTo(step));
    if (step == null) ret;
    syncAdd(stack, new ActiveSequence(step));
    change();
  }

  void callSubroutine(BotStep step) {
    if (step == null) ret;
    noteEvent(EvtCallSubroutine(step));
    print("Calling subroutine " + step);
    syncAdd(stack, new ActiveSequence(step));
    change();
  }

  void removeInputHandler(IInputHandler h) {
    if (inputHandler == h)
      cset(this, inputHandler := null);
  }

  void scheduleNextStep {
    botMod().addScheduledAction(new Action_NextStep(this));
  }

  void noteEvent(Concept event) {
    events.add(event);
  }
} // end of Conversation

abstract concept ScheduledAction implements Runnable {
  long time;
}

concept Action_NextStep > ScheduledAction {
  new Ref<Conversation> conv;
  int executedSteps = -1; // if >= 0, we verify against conv.executedSteps

  *() {}
  *(Conversation conv) { this.conv.set(conv); }
  *(Conversation conv, int *executedSteps) { this.conv.set(conv); }

  run {
    nextStep(conv!, executedSteps);
  }
}

concept OnNewDialog > ScheduledAction {
  new Ref<Conversation> conv;
   
  *() {}
  *(Conversation conv) { this.conv.set(conv); }
  
  run {
    botMod().startMainScript(conv!);
  }
}

svoid nextStep(Conversation conv, int expectedExecutedSteps default -1) {
  if (expectedExecutedSteps >= 0 && conv.executedSteps != expectedExecutedSteps) ret;
  
  while licensed {
    if (empty(conv.stack)) ret;
    ActiveSequence seq = syncLast(conv.stack);
    bool done = seq.done();
    printVars_str("Active sequence", +conv, +seq, +done, stackSize := syncL(conv.stack));
    if (done)
      continue with syncPopLast(conv.stack);
    BotStep step = seq.currentStep();
    seq.nextStep();
    ++conv.executedSteps;
    conv.change();
    if (!executeStep(step, conv)) {
      print("Step returned false: " + step);
      break;
    }
  }
}

sbool executeStep(BotStep step, Conversation conv) {
  if (step == null || conv == null) false;
  print("Executing step " + step + " in " + conv);

  ExecutedStep executed = null;
  if (botMod().recordExecutionsAndInputs)
    executed = cnew ExecutedStep(+step, conversation := conv);
    
  Msg lastMsg = syncLast(conv.msgs);
  bool result = step.run(conv);
  Msg newMsg = syncLast(conv.msgs);
  if (newMsg != null && !newMsg.fromUser && newMsg != lastMsg)
    cset(executed, msg := newMsg);
  ret result;
}

concept OnUserMessage > ScheduledAction {
  new Ref<Conversation> conv;
  
  *() {}
  *(Conversation conv) { this.conv.set(conv); }

  run {
    // TODO: handle multiple messages in case handling was delayed
    Msg msg = syncLast(conv->msgs);
    if (msg == null || !msg.fromUser) ret;

    print("OnUserMessage: " + msg);

    if (botMod().handleGeneralUserInput(conv!, msg, true))
      ret;

    IInputHandler inputHandler = conv->inputHandler;
    if (inputHandler != null) {
      InputHandled rec = null;
      S input = msg.text;
      if (botMod().recordExecutionsAndInputs)
        rec = cnew InputHandled(+inputHandler, conversation := conv!, +msg, +input);
      
      bool result = conv->inputHandler.handleInput(input, conv!);
      cset(rec, handled := result);
      if (result)
        ret;
    }

    botMod().handleGeneralUserInput(conv!, msg, false);
  }
}

// application-specific concepts

// a server we're running - either a domain name or domain.bla/path
concept Domain {
  S domainAndPath;
  S tenantID; // for external software

  // bot config - av
  new Ref<AbstractBotImage> botImage;
  S botName; // if empty, keep default
  new Ref<AbstractAvatar> avatar;
  
  S headerColorLeft, headerColorRight; // ditto
  Bool autoOpenBot;
  
  new Ref<BotStep> mainScript;

  S password = aGlobalID();

  sS _fieldOrder = "domainAndPath botImage mainScript";
  
  toString { ret domainAndPath; }
}

concept Form {
  S name;
  new RefL<BotStep> steps;

  toString { ret name; }
}

concept DeliveredDomain {
  S domain;
}

concept CannedAnswer {
  S hashTag, text;
}

sS text(CannedAnswer a) { ret a?.text; }

concept Lead {
  new Ref<Domain> domain;
  new Ref<Conversation> conversation;
  Timestamp date;
  SS answers;
}

concept ConversationFeedback {
  new Ref<Domain> domain;
  new Ref<Conversation> conversation;
  Timestamp date;
  SS answers;
}

concept Settings {
  S mainDomainName;
  double botTypingDelay;
  S preferredCountryCodes; // dial codes, comma-separated
  bool multiLanguageMode;
  new Ref<UploadedSound> notificationSound;
  bool talkToBotOnlyWithAuth;
  bool enableWorkerChat;
  new Ref<Language> defaultLanguage;
}

sinterface IInputHandler {
  // true if handled
  bool handleInput(S s, Conversation conv);
}

static DynNewBot2 botMod() {
  ret (DynNewBot2) db_mainConcepts().miscMapGet(DynNewBot2);
}

Began life as a copy of #1029877