!include once #1029875 // concept Worker

abstract sclass DynNewBot2 > DynPrintLogAndEnabled {
  set flag NoNanoHTTPD.
  !include #1029545 // API for Eleu

  int maxRefsToShow = 5;
  
  transient S templateID = #1029809;
  sS cssID = #1029808;

  transient S botName = "DynNewBot2";
  transient S heading = "DynNewBot2";
  transient S adminName = "DynNewBot2 Admin";
  transient S botImageID = #1102935;
  transient S userImageID = #1102803;
  transient S chatHeaderImageID = #1102802;
  transient S timeZone = ukTimeZone_string(); 
  
  transient S baseLink = "";
  
  transient bool newDesign = true; // use new chat bot design
  transient bool ariaLiveTrick = false;
  transient bool ariaLiveTrick2 = true;
  
  !include once #1029876 // WorkerChat 
  transient new WorkerChat workerChat;
  
  transient ReliableSingleThread rstBotActions = new(r botActions);
  
  transient S defaultHeaderColorLeft = "#2a27da";
  transient S defaultHeaderColorRight = "#00ccff";
  
  void start {
    super.start();
    dm_setModuleName(botName);
    dm_assertFirstSibling();
    concepts_setUnlistedByDefault(true);
    standardTimeZone();
    standardTimeZone_name = timeZone;
    baseLink = "";
    realPW(); // make password
    pWebChatBot();
    doEvery(60.0, r cleanConversations);
    rstBotActions.trigger();
  }

  sclass Req {
    S uri;
    SS params;
    AuthedDialogID auth;
    Domain authDomainObj;
    S authDomain;
    HTMLFramer1 framer;
    bool masterAuthed;
    Conversation conv;

    bool requestAuthed() { ret auth != null; }
  }

  O html(IWebRequest request) {
    temp enter();
    //temp tempRegisterThread();
    
    S uri = request.uri();
    SS params = request.params();

    new Req req;
    req.uri = uri;
    req.params = params;

    // cookie comes either from a URI parameter or from the request header
    // Note: this can be either a JS-generated (in dynamic chat bot part)
    // or server-generated cookie (when loading initial chat bot script or admin)
    // To distinguish: JS-generated cookies are shorter and can contain numbers
    S cookie = params.get('cookie);
    if (empty(cookie)) cookie = request.cookie();
   
    bool workerMode = nempty(params.get("workerMode")) || startsWith(uri, "/worker");
    
    // find out which domain we're on ("delivered domain")

    S domain = request.domain(), _domain = domain;
    saveDeliveredDomain(domain);
    Domain domainObj = findDomainObj(domain);

    // get conversation
    Conversation conv = nempty(cookie) ? getConv(cookie) : null;
    req.conv = conv;

    temp temp_printPrefix("Conv " + conv.cookie + ": ");

    S botConfig = params.get("_botConfig");
    SS botConfigParams = decodeURIParams(botConfig);
    S simulatedDomain = botConfigParams.get("domain");
    Domain domainObj2 = nempty(simulatedDomain) ? findDomainObj(simulatedDomain) : domainObj;
    if (nempty(botConfigParams)) cset(conv, botConfig := botConfigParams);

    // save ip & domain in conversation
    if (conv != null && !workerMode)
      if (cset_trueIfChanged(conv, ip := request.clientIP(), +domain, domainObj := domainObj2))
        calcCountry(conv);
      
    print("URI: " + uri + ", cookie: " + cookie + ", msgs: " + l(conv.msgs));
    
    S pw = trim(params.get('pw));
    if (nempty(pw) && nempty(cookie)) {
      Domain authDomain;
      if (eq(pw, realPW())) 
        cset(uniq AuthedDialogID(+cookie), restrictedToDomain := null, master := true);
      else if ((authDomain = conceptWhere Domain(password := pw)) != null)
        cset(uniq AuthedDialogID(+cookie), restrictedToDomain := authDomain, master := false);
      else
        ret errorMsg("Bad password, please try again");
        
      if (nempty(params.get('redirect)))
        ret hrefresh(params.get('redirect));
    }
    
    new Matches m;
    if (startsWith(uri, "/worker-image/", m)) {
      long id = parseLong(m.rest());
      ret subBot_serveFile(workerImageFile(id), "image/jpeg");
    }
    
    if (startsWith(uri, "/uploaded-image/", m)) {
      long id = parseLong(m.rest());
      UploadedImage img = getConcept UploadedImage(id);
      ret img == null ? serve404() : subBot_serveFile(img.imageFile(), "image/jpeg");
    }
    
    AuthedDialogID auth = authObject(cookie);
    if (eq(params.get('logout), "1")) {
      cdelete(auth);
      auth = null;
    }
    req.auth = auth;
    bool requestAuthed = auth != null; // any authentication
    bool masterAuthed = req.masterAuthed = requestAuthed && auth.master;
    Domain authDomainObj = !requestAuthed ? null : auth.restrictedToDomain!;
    req.authDomainObj = authDomainObj;
    S authDomain = !requestAuthed ? null : auth.domain();
    req.authDomain = authDomain;

    if (eq(uri, "/emoji-picker/index.js"))
      //ret serveWithContentType(loadTextFile(loadLibrary(#1400436)), "text/javascript"); // TODO: optimize
      ret subBot_maxCacheHeaders(serveInputStream(bufferedFileInputStream(loadLibrary(#1400436)), "text/javascript"));

    if (eq(uri, "/emoji-picker-test"))
      ret loadSnippet(#1029870);

    if (eq(uri, "/stats")) {
      if (!requestAuthed) ret serveAuthForm(rawLink(uri));
      ret "Threads: " + ul_htmlEncode(getThreadNames(registeredThreads()));
    }
        
    if (eq(uri, "/logs")) {
      if (!masterAuthed) ret serveAuthForm(rawLink(uri));
      ret webChatBotLogsHTML2(rawLink(uri), params);
    }
    
    if (eq(uri, "/refchecker")) {
      if (!masterAuthed) ret serveAuthForm(rawLink(uri));
      L errors = ConceptsRefChecker(db_mainConcepts()).run();
      ret serveText(jsonEncode_breakAtLevels(2, litorderedmap(errors := allToString(errors))));
    }
    
    if (eq(uri, "/auth-only")) {
      if (!requestAuthed) ret serveAuthForm(params.get('uri));
      ret "";
    }

    if (eq(uri, "/leads-api"))
      ret serveLeadsAPI(request);
    
    if (workerChat != null)
      try object workerChat.html(uri, params, conv, auth);
  
    {
      lock dbLock();
      
      S message = trim(params.get("btn"));
      if (empty(message)) message = trim(params.get("message"));
      
      if (match("new dialog", message)) {
        conv.newDialog();
        message = null;
      }

      if (eqic(message, "!toggle notifications")) {
        cset(conv, notificationsOn := !conv.notificationsOn);
        message = null;
      }
      
      this.conv.set(conv);
      
      if (nempty(message) && !lastUserMessageWas(conv, message)) {
        print("Adding message: " + message);
        if (workerMode) {
          Msg msg = new(false, message);
          msg.fromWorker = auth.loggedIn;
          conv.add(msg);
        } else {
          // add user message
          conv.add(new Msg(true, message));
          //conv.botTyping = now();
          addScheduledAction(new OnUserMessage(conv));
        }
      }
      
      S testMode = params.get("testMode");
      if (nempty(testMode)) {
        print("Setting testMode", testMode);
        cset(conv, testMode := eq("1", testMode));
      }
    } // locked
    
    if (eq(uri, "/msg")) ret withHeader("OK");
    
    if (eq(uri, "/typing")) {
      if (workerMode) {
        conv.botTyping = now();
        print(conv.botTyping + " Bot typing in: " + conv.cookie);
      } else {
        conv.userTyping = now();
        print(conv.userTyping + " User typing in: " + conv.cookie);
      }
      ret withHeader("OK");
    }
    
    if (eq(uri, "/incremental")) {
      vmBus_send chatBot_userPolling(mc(), conv);
      cset(conv, lastPing := now());
      if (empty(conv.msgs) && conv.newDialogTriggered < conv.archiveSize()) {
        cset(conv, newDialogTriggered := conv.archiveSize());
        addScheduledAction(OnNewDialog(conv));
      }
      int a = parseInt(params.get("a"));
  
      long start = sysNow(), start2 = now();
      L msgs;
      bool first = true;
      while (licensed() && sysNow() < start+longPollMaxWait) {
        int as = conv.archiveSize();
        msgs = cloneSubList(conv.msgs, a-as); // just the new messages
        bool newDialog = a < as;
        long typing = workerMode ? conv.userTyping : conv.botTyping;
        bool otherPartyTyping = typing > start2;

        bool anyEvent = nempty(msgs) || newDialog || otherPartyTyping;
        
        if (!anyEvent) {
          if (first) {
            //print("Long poll starting on " + cookie + ", " + a + "/" + a);
            first = false;
          }
          sleep(longPollTick);
        } else {
          if (!first) print("Long poll ended.");
          new StringBuilder buf;

          if (newDialog) {
            // send header colors & notification status
            S l = or2_trim(domainObj2.headerColorLeft, defaultDomain().headerColorLeft, defaultHeaderColorLeft),
              r = or2_trim(domainObj2.headerColorRight, defaultDomain().headerColorRight, defaultHeaderColorRight);
            buf.append(hcss(".chat_header { background: linear-gradient(135deg, "
              + hexColorToCSSRGB(l) + " 0%, " + hexColorToCSSRGB(r) + " 100%); }"));
            buf.append(hscript("$('#chatBot_notiToggleText').text(" + jsQuote("Turn " + (conv.notificationsOn ? "off" : "on")
              + " notifications") + ");"));
          }
          
          if (otherPartyTyping) {
            print("Noticed " + (workerMode ? "user" : "bot") + " typing in " + conv.cookie);
            buf.append(hscript("showTyping();"));
          }
          renderMessages(conv, buf, msgs);
          if (ariaLiveTrick2 && !workerMode) {
            Msg msg = lastBotMsg(msgs);
            if (msg != null) {
              S author = msg.fromWorker != null ? htmlEncode2(msg.fromWorker.displayName) : botName;
              buf.append(hscript([[$("#screenreadertrick").html(]] + jsQuote(author + " says: " + msg.text) + ");"));
            }
          }
          if (a != 0 && anyInterestingMessages(msgs, workerMode))
            buf.append(hscript(
              stringIf(conv.notificationsOn, "window.playChatNotification();\n") +
              "window.setTitleStatus(" + jsQuote((workerMode ? "User" : botName) + " says…") + ");"
            ));
          ret withHeader("<!-- " + conv.allCount() + " " + (newDialog ? "NEW DIALOG " : "") + "-->\n"  + buf);
        }
      }
      ret withHeader("");
    }

    if (eqOneOf(uri, "/script", "/demo")) {
      lock dbLock();
      S html = loadSnippet_cached(templateID);

      S heading = or2(trim(domainObj2.botName), this.heading);
      S botImg = botImageForDomain(domainObj);

      S workerModeParam = workerMode ? "workerMode=1&" : "";
      S langlinks = "<!-- langlinks here -->";
      if (html.contains(langlinks))
        html = html.replace(langlinks,
          ahref(rawLink("eng"), "English") + " | " + ahref(rawLink("deu"), "German"));

      //html = html.replace("#COUNTRYCODE_OPTIONS#", countryCodeOptions(conv));
      html = html.replace("#COUNTRY#", lower(conv.country));
      html = html.replace("#BOTIMG#", botImg);
      html = html.replace("#N#", "0");
      html = html.replace("#INCREMENTALURL#", baseLink + "/incremental?" + workerModeParam + "a=");
      html = html.replace("#MSGURL#", baseLink + "/msg?" + workerModeParam + "message=");
      html = html.replace("#TYPINGURL#", baseLink + "/typing?" + workerModeParam);
      html = html.replace("#CSS_ID#", psI_str(cssID));
      if (ariaLiveTrick || ariaLiveTrick2)
        html = html.replace([[aria-live="polite">]], ">");
      html = html.replace("#OTHERSIDE#", workerMode ? "User" : "Representative");
      if (nempty(params.get("debug")))
        html = html.replace("var showActions = false;", "var showActions = true;");
    
      html = html.replace("#AUTOOPEN#", jsBool(workerMode || eq(params.get("_autoOpenBot"), "1") || botAutoOpen()));
      html = html.replace("#BOT_ON#", jsBool(botOn() || eq(uri, "/demo")));
      html = html.replace("$HEADING", heading);
      html = html.replace("#WORKERMODE", jsBool(workerMode));
      html = html.replace("<!-- MSGS HERE -->", "");
      html = hreplaceTitle(html, heading);
      
      if (eq(uri, "/demo"))
        ret hhtml(hhead(
          htitle(heading)
          + loadJQuery2()
        ) + hbody(hjavascript(html)));
      else
        ret withHeader(subBot_serveJavaScript(html));
    }
    
    // serve admin
    
    if (!requestAuthed) ret serveAuthForm(params.get('uri));
    // authed from here (not necessarily master-authed though)

    if (eq(uri, "/thoughts"))
      ret serveThoughts(req);

    if (masterAuthed && eq(uri, "/search"))
      ret serveSearch(req);

    if (eq(uri, "/leads-csv")) {
      S text = leadsToCSV(conceptsWhere(Lead, mapToParams(filtersForClass(Lead, authDomainObj))));
      S name = "bot-leads"
        + (authDomainObj == null ? "" : "-" + replace(str(authDomainObj), "/", "-"))
        + "-" + ymd_minus_hm() + ".csv";
      ret serveCSVWithFileName(name, text);
    }

    if (eq(uri, "/cleanConversations") && masterAuthed) {
      cleanConversations();
      ret hrefresh(baseLink + "/crud/Conversation");
    }
    
    if (eq(uri, "/deleteDeliveredDomains") && masterAuthed) {
      deleteDeliveredDomains();
      ret hrefresh(baseLink + "/crud/DeliveredDomain");
    }

    makeFramer(req);
    HTMLFramer1 framer = req.framer;

    L<Class> classes = crudClasses(req.masterAuthed);
    L<Class> cmdClasses = req.masterAuthed ? botCmdClasses() : null;
    for (Class c : (Set<Class>) asSet(flattenList2(classes, DeliveredDomain.class, cmdClasses)))
      if (eq(uri, dropUriPrefix(baseLink, crudLink(c)))) {
        S help = mapGet(crudHelp(), c);
        if (nempty(help)) framer.contents.add(p(help));

        HCRUD crud = makeCRUD(c, req);

        // render CRUD

        crud.processSortParameter(params);
        framer.contents.add(crud.renderPage(params));

        // javascript magic to highlight table row according to anchor in URL
        framer.contents.add(hjs_markRowMagic());
      }
      
    if (eq(uri, "/emojis"))
      framer.contents.add(htmlTable2(map(emojiShortNameMap(),
        (code, emoji) -> litorderedmap("Shortcode" := code, "Emoji" := emoji))));

    //if (masterAuthed) framer.addNavItem(baseLink + "/search", "Search");
    
    framer.addNavItem(baseLink + "?logout=1", "Log out");
    ret framer.render();
  }

  void makeFramer(Req req) {
    new HTMLFramer1 framer;
    req.framer = framer;
    framer.title = adminName + " " + squareBracket(
      req.masterAuthed ? "super user" : htmlEncode2(req.authDomain));
    framer.addInHead(hsansserif() + hmobilefix() + hresponstable()
      + hcss_responstableForForms());
    framer.addInHead(loadJQuery2());
    framer.addInHead(hjs_selectize());
    framer.addInHead(hjs_copyToClipboard());
    framer.addInHead(hNotificationPopups());
    framer.addInHead(hcss_linkColorInherit());
    framer.addNavItem(simulateDomainLink(req.authDomain), "Talk to bot", targetBlank := true);
    L<Class> classes = crudClasses(req.masterAuthed);
    L<Class> cmdClasses = req.masterAuthed ? botCmdClasses() : null;
    for (Class c : classes)
      framer.addNavItem(makeClassNavItem(c, req.authDomainObj));
    if (nempty(cmdClasses))
      framer.contents.add(p("Bot actions: " + joinWithVBar(map(cmdClasses, c -> makeClassNavItem(c, req.authDomainObj)))));
  }

  // make and adapt CRUD for class    
  <A extends Concept> HCRUD makeCRUD(Class<A> c, Req req) {
    HTMLFramer1 framer = req.framer;
    HCRUD_Concepts data = crudData(c);
    data.referencesBlockDeletion = true;
    HCRUD crud = new(req.uri, data);
    crud.haveJQuery = crud.haveSelectizeJS = true;
    crud.sortable = true;
    crud.paginate = true;
    crud.paginator.step = 25;
    crud.baseLink = crudLink(c);
    crud.cmdsLeft = true;
    crud.showCheckBoxes = true;
    crud.tableClass = "responstable";
    //framer.addInHead(hcss(".crudForm { width: 800px; }"));
    crud.formTableClass = "responstableForForms";

    crud.renderValue_inner = value -> {
      S html = crud.renderValue_inner_base(value);
      if (value cast Concept)
        ret ahref(conceptLink(value), html);
      ret html;
    };
    
    if (c == Conversation) {
      crud.nav = () -> joinNemptiesWithVBar(crud.nav_base(),
        ahref(baseLink + "/cleanConversations", "Clean list"));
      crud.unshownFields = litset("oldDialogs", "worker", "botOn", "lastPing", "cookie", "form", "testMode", "userMessageProcessed", "newDialogTriggered");
    }
    if (c == DeliveredDomain || c == Conversation || c == Lead || c == ConversationFeedback) 
      crud.allowCreate = crud.allowEdit = false;
      
    if (c == Settings) {
      crud.singleton = true;
      framer.contents.add(p(ahref(baseLink + "/emojis", "Emojis")));
    }

    if (c == Lead)
      framer.contents.add(p(ahref(baseLink + "/leads-csv", "Export as CSV")));
      
    if (c == BotOutgoingQuestion)
      crud.unlistedFields = litset("multipleChoiceSeparator", "placeholder");

    data.addFilters(filtersForClass(c, req.authDomainObj));

    if (c == Domain) {
      crud.renderCmds = map -> {
        Domain dom = getConcept Domain(toLong(crud.itemID(map)));
        ret joinNemptiesWithVBar(crud.renderCmds_base(map),
          targetBlank(simulateDomainLink(dom.domainAndPath), "Talk to bot"));
      };
      framer.contents.add(p("See also the " + ahref(crudLink(DeliveredDomain), "list of domains the bot was delivered on")));
    }
    
    if (c == DeliveredDomain) {
      crud.nav = () -> joinNemptiesWithVBar(crud.nav_base(),
        ahref("/deleteDeliveredDomains", "Delete all"));
      crud.renderCmds = map -> {
        DeliveredDomain dom = getConcept DeliveredDomain(toLong(crud.itemID(map)));
        ret joinNemptiesWithVBar(crud.renderCmds_base(map),
          targetBlank(simulateDomainLink(dom.domain), "Talk to bot"));
      };
    }

    if (c == UploadedImage) {
      crud.massageFormMatrix = (map, matrix) -> {
        UploadedImage item = getConcept UploadedImage(toLong(crud.itemID(map)));
        matrix.add(ll("Upload image",
          hjs_imgUploadBase64Encoder()
          + himageupload(id := "imgUploader")
          + hhiddenWithIDAndName("f_img_base64")));
      };
      crud.formParameters = () -> litparams(onsubmit := "return submitWithImageConversion(this)");
    }

    if (isSubclassOf(c, BotStep)) {
      crud.renderCmds = map -> {
        BotStep step = getConcept BotStep(toLong(crud.itemID(map)));
        ret joinNemptiesWithVBar(crud.renderCmds_base(map),
          targetBlank(simulateScriptLink(step), "Test in bot"));
      };
      framer.contents.add(p("See also the " + ahref(crudLink(DeliveredDomain), "list of domains the bot was delivered on")));
    }

    // show references
    
    crud.postProcessTableRow = (item, rendered) -> {
      long id = parseLong(item.get("id"));
      Concept concept = getConcept(id);
      if (concept == null) ret rendered;
      Cl<Concept> refs = allBackRefs(concept);
      if (empty(refs)) ret rendered;
      refs = sortedByConceptID(refs);
      int more = l(refs)-maxRefsToShow;
      ret mapPlus(rendered, span_title("Where is this object used", "References"),
        joinMap(takeFirst(maxRefsToShow, refs), ref -> p(ahref(conceptLink(ref),
          htmlEncode_nlToBr_withIndents(str(ref)))))
        + (more > 0 ? "<br>+" + more + " more" : ""));
    };

    ret crud;
  }

  S conceptLink(Concept c) {
    ret c == null ? null : crudLink(c.getClass()) + "#obj" + c.id;
  }

  <A extends Concept> S makeClassNavItem(Class<A> c, Domain authDomainObj) {
    HCRUD_Concepts<A> data = crudData(c);
    Map filters = filtersForClass(c, authDomainObj);
    int count = countConcepts(c, mapToParams(filters));
    //print("Count for " + c + " with " + filters + ": " + count);
    ret (c == Settings.class ? ""
      : count + " ")
      + ahref(crudLink(c), data.itemNamePlural());
  }

  L<Class> botCmdClasses() {
    ret ll(BotMessage, UploadedImage, BotImage, BotOutgoingQuestion, BotPause, Sequence);
  }

  L<Class> crudClasses(bool masterAuthed) {
    if (masterAuthed) {
      L<Class> l = ll(Conversation, Lead, ConversationFeedback, Domain, UserKeyword, Settings);
      if (settings().multiLanguageMode) l.add(Language);
      ret l;
    } else
      ret ll(Conversation, Lead, ConversationFeedback);
  }

  MapSO filtersForClass(Class c, Domain authDomainObj) {
    if (c == Conversation.class && authDomainObj != null)
      ret litmap(domainObj := authDomainObj);
    if (eqOneOf(c, Lead.class, ConversationFeedback.class) && authDomainObj != null)
      ret litmap(domain := authDomainObj);
    null;
  }

  S crudLink(Class c) {
    ret baseLink + "/crud/" + shortName(c);
  }

  <A extends Concept> HCRUD_Concepts<A> crudData(Class<A> c) {
    HCRUD_Concepts<A> cc = new(c);
    cc.trimAllSingleLineValues = true;
    cc.fieldHelp("comment", "Put any comment about this object here");
    cc.itemName = () -> replaceIfEquals(
      dropPrefix("Bot ", cc.itemName_base()), "Jump Button", "Button");
    cc.valueConverter = new DefaultValueConverterForField {
      public OrError<O> convertValue(O object, Field field, O value) {
        // e.g. for "buttons" field
        if (value instanceof S && eq(field.getGenericType(), type_LS()))
          ret OrError(tlft((S) value));
        ret super.convertValue(object, field, value);
      }
    };
    if (c == Domain) {
      cc.fieldHelp(
        "domainAndPath", "without http:// or https://",
        "botName", "Bot name for this domain (optional)",
        "headerColorLeft", "Hex color for left end of header gradient (optional)",
        "headerColorRight", "Hex color for right end of header gradient (optional)");
      cc.massageItemMapForList = (item, map) -> {
        map.put("domainAndPath", HTML(b(htmlEncode2(item/Domain.domainAndPath))));
        map.put("password", SecretValue(map.get("password")));
      };
    }

    if (c == BotOutgoingQuestion) {
      cc.addRenderer("displayText", new HCRUD_Data.TextArea(80, 10));
      cc.addRenderer("buttons", new HCRUD_Data.TextArea(80, 10, o -> lines_rtrim((L) o)));
      cc.fieldHelp(
        displayText := displayTextHelp(),
        key := "Internal key for question (any format, can be empty)",
        defaultValue := "What the input field is prefilled with",
        placeholder := "A text the empty input field shows as a hint",
        buttons := "Buttons to offer as standard answers (one per line)",
        allowFreeText := "Can user enter free text in addition to clicking a button?",
        multipleChoice := "Can user select multiple buttons?",
        optional := "Can user skip the question by entering nothing?",
        multipleChoiceSeparator := "Internal field, just leave as it is",
        answerCheck := "Select this to validate user's answer against a pattern",
        buttonActions := "Optional actions (one for each button in the list above)",
      );
      cc.addRenderer("answerCheck", new HCRUD_Data.ComboBox("", "email address", "phone number"));
      cc.massageItemMapForList = (item, map) -> {
        map.put("buttons", HTML(ol_htmlEncode(item/BotOutgoingQuestion.buttons)));
      };
    }

    if (c == UserKeyword) {
      for (S field : ll("examples", "counterexamples"))
        cc.addRenderer(field, new HCRUD_Data.TextArea(80, 5, o -> lines_rtrim((L) o)));
      cc.massageItemMapForList = (item, map) -> {
        map.put("examples", lines_rtrim(item/UserKeyword.examples));
        map.put("counterexamples", lines_rtrim(item/UserKeyword.counterexamples));
      };
    }

    if (c == Settings)
      cc.fieldHelp(
        "botTypingDelay", "Delay in seconds before bot sends message",
        "preferredCountryCodes", [[Country codes to display first in list (e.g. "+1, +91")]]);

    if (c == Conversation)
      cc.massageItemMapForList = (item, map) -> {
        replaceMap(map, litorderedmap(
          id := map.get("id"),
          "IP" := map.get("ip"),
          "started" := formatLocalDateWithMinutes(item/Conversation.created),
          "country" := map.get("country"),
          msgs := lines_rtrim(item/Conversation.msgs)));
      };

    if (c == BotMessage) {
      cc.addRenderer("text", new HCRUD_Data.TextArea(80, 10));
      cc.addRenderer("specialPurpose", new HCRUD_Data.ComboBox("", "bad email address", "bad phone number"));
      cc.fieldHelp(
        text := displayTextHelp(),
        specialPurpose := "Special occasion when to display this message");
    }

    if (c == Form)
      cc.massageItemMapForList = (item, map) -> {
        map.put("steps", pnlToStringWithEmptyLines_rtrim(item/Form.steps));
      };

    if (c == Sequence)
      cc.massageItemMapForList = (item, map) -> {
        map.put("steps", HTML(ol(
          map(item/Sequence.steps, step ->
            ahref(conceptLink(step), htmlEncode2_nlToBr(str(step)))))));
      };
      
    if (c == BotImage)
      cc.massageItemMapForList = (item, map) -> {
        S url = item/BotImage.imageURL;
        if (isURL(url))
          map.put("Image Preview", HTML(himg(url, style := "max-width: 200px; max-height: 100px; width: auto; height: auto")));
        map.put(cc.fieldNameToHTML("imageURL"), HTML(
            htmlEncode2(url) + " "
          + himgsnippet(#1101381, onclick := 
              "copyToClipboard(" + jsQuote(url) + "); "
            + "window.createNotification({ theme: 'success', showDuration: 3000 })({ message: "
            + jsQuote("Image URL copied to clipboard")
            + "});")));
      };

    if (c == UploadedImage) {
      cc.massageItemMapForList = (item, map) -> {
        S url = item/UploadedImage.imageURL();
        File f = item/UploadedImage.imageFile();
        map.put("Image Size", toK_str(fileSize(f)));
        if (fileSize(f) > 0)
          map.put("Image Preview", HTML(himg(item/UploadedImage.imageURL(), style := "max-width: 200px; max-height: 100px; width: auto; height: auto")));
      };
      IVF2 massageItemMapForUpdate_old = cc.massageItemMapForUpdate;
      cc.massageItemMapForUpdate = (item, map) -> {
        cc.massageItemMapForUpdate_fallback(massageItemMapForUpdate_old, item, map);
        S base64 = (S) map.get("img_base64");
        print("Got base64 data: " + l(base64));
        if (nempty(base64)) {
          File f = item/UploadedImage.imageFile();
          saveFileVerbose(f, base64decode(base64));
        }
        map.remove("img_base64");
      };
    }

    ret cc;
  }

  Map<Class, S> crudHelp() {
    ret litmap(DeliveredDomain, "This list is filled by the bot with the domains it was delivered on. "
      + "Some may be bogus, we write down whatever the browser sends.");
  }

  void saveDeliveredDomain(S domain) {
    if (empty(domain)) ret;
    uniqCI(DeliveredDomain, domain := beforeColonOrAll(domain));
  }

  void cleanConversations {
    withDBLock(r {
      cdelete(filter(list(Conversation), c -> l(c.msgs) <= 1
        && elapsedSeconds_timestamp(c.created) >= 60 & c.lastPing == 0));
    });
  }
  
  void deleteDeliveredDomains {
    cdelete DeliveredDomain();
  }

  O serveThoughts(Req req) {
    new HTMLFramer1 framer;
    framer.addInHead(hsansserif() + hmobilefix());
    framer.add(div_floatRight(hbutton("Reload", onclick := "location.reload()")));
    framer.add(h2("Bot Thoughts"));
    if (req.conv == null) framer.add("No conversation");
    else {
      framer.add(p("Conversation cookie: " + req.conv.cookie));
      Conversation conv = req.conv;
      framer.add(h3("Stack"));
      framer.add(empty(conv.stack) ? p("empty")
        : ol(lmap htmlEncode2_gen(reversed(conv.stack))));
    }
    ret framer.render();
  }

  O serveSearch(Req req) {
    makeFramer(req);
    HTMLFramer1 framer = req.framer;
    framer.add("Search here");
    ret framer.render();
  }
  
  transient bool debug;

  // always returns an object
  Domain findDomainObj(S domain) {
    Domain domainObj = conceptWhereCI Domain(domainAndPath := domain);
    if (domainObj == null) domainObj = defaultDomain();
    ret domainObj;
  }
  
  Domain defaultDomain() {
    ret uniqCI Domain(domainAndPath := "<default>");
  }
  
  // Web Chat Bot Include
  
  transient int longPollTick = 200;
  transient int longPollMaxWait = 1000*30; // lowered to 30 seconds
  transient int activeConversationSafetyMargin = 15000; // allow client 15 seconds to reload
  
  transient Set<S> specialButtons = litciset("Cancel", "Back", "Abbrechen", "Zurück");
  
  S dbStats() {
    Cl<Conversation> all = list(Conversation);
    int nRealConvos = countPred(all, c -> l(c.msgs) > 1);
    //int nTestConvos = countPred(all, c -> startsWith(c.cookie, "test_"));
    //ret n2(l(all)-nTestConvos, "real conversation") + ", " + n2(nTestConvos, "test conversation");
    ret nConversations(nRealConvos);
  }
  
  S realPW() {
    print("realMC: " + realMC() + " (" + getProgramID(realMC()) + ")");
    print("Secret program dir: " + getSecretProgramDir());
    ret loadSecretTextFileOrCreateWithRandomID("password.txt");
  }
  
  // can't un-static this as includes (WorkerChat) require it
  S serveAuthForm(S redirect) {
    ret hhtml(hhead(htitle("Authorization required")
      + hsansserif() + hmobilefix()) + hbody(hfullcenter(
      h3_htmlEncode(adminName)
      + hpostform(
        hhidden(+redirect)
      + "Password: " + hpassword("pw") + " " + hsubmit("Log in"),
        action := baseLink)
      + p(ahref(baseLink + "/demo", "Talk to bot"))
      )));
  }
  
  S leadsToCSV(Cl<Lead> leads) {
    LS fields = ciContentsIndexedList();
    new LL rows;
    fOr (Lead lead : leads) {
      new L row;
      fOr (S key, O val : mapPlus_inFront((MapSO) (Map) lead.answers,
        "Date" := formatLocalDateWithMinutes(lead.date.date),
        "Domain" := lead.domain!,
        "Conversation ID" := conceptID(lead.conversation!))) {
        int idx = fields.indexOf(key);
        if (idx < 0) idx = addAndReturnIndex(fields, key);
        listSet(row, idx, val, "");
      }
      rows.add(row);
    }
    ret formatCSVFileForExcel2(itemPlusList(fields, rows));
  }
  
  S countryCodeOptions(Conversation conv) {
    Cl<S> countryCodes = putSetElementsFirst(keys(countryDialCodesMultiMap()), splitAtComma_trim(settings().preferredCountryCodes));
    S selected = dialCodeStringForCountryCode(conv.country);
    ret mapToLines(c -> { // c == dial code
      L<CountryDialCode> cdc = countryDialCodesMultiMap().get(c);
      S text = nempty(cdc) ? c + " [" + joinWithComma(collectSorted countryCode(cdc)) + "]" : c;
      ret tag option(text, value := c, selected := eq(c, selected) ? html_valueLessParam() : null);
    }, itemPlus("", countryCodes));
  }
  
  L<BotStep> addTypingDelays(L<BotStep> steps) {
    if (settings().botTypingDelay <= 0) ret steps;
    //printStackTrace();
    ret concatMap(steps, step -> {
      double delay = step.preTypingDelay();
      if (delay <= 0) ret ll(step);
      ret ll(new BotSendTyping, new BotPause(delay), step);
    });
  }
  
  BotMessage messageForPurpose(S specialPurpose) {
    ret conceptWhere BotMessage(+specialPurpose);
  }
  
  O serveLeadsAPI(virtual WebRequest req) {
    SS headers = cast rcall headers(req);
    SS params = cast get params(req);
    S tenantID = params.get("tenantID");
    if (empty(tenantID)) ret serveJSONError("Empty tenantID");
    S pw = params.get("pw");
    if (empty(pw)) pw = dropPrefix_trim("Bearer ", headers.get("Authorization"));
    if (empty(pw)) ret serveJSONError("Empty password");
    Domain domain = conceptWhere Domain(+tenantID);
    if (domain == null) ret serveJSONError("Tenant ID not found");
    if (neq(domain.password, pw)) ret serveJSONError("Bad passsword");
  
    Cl<Lead> leads = conceptsWhere Lead(+domain);
    ret serveJSON(map(leads, lead -> {
      Map map = litorderedmap(+tenantID,
        leadID := lead.id,
        domain := str(lead.domain),
        date := formatLocalDateWithSeconds(lead.created),
        data := lead.answers
      );
      ret map;
    }));
  }
  
  O serveJSONError(S error) {
    ret serveJSON(litorderedmap(+error));
  }
  
  S botImageForDomain(Domain domainObj) {
    S botImg = imageSnippetURLOrEmptyGIF(chatHeaderImageID);
    if (domainObj != null && domainObj.botImage.has())
      botImg = domainObj.botImage->imageURL();
    else if (defaultDomain().botImage.has())
      botImg = defaultDomain().botImage->imageURL();
    ret botImg;
  }
  
  S simulateDomainLink(S domain) {
    ret appendQueryToURL(baseLink + "/demo", _botConfig := makePostData(+domain), cookie := "test_" + aRandomID());
  }
  
  S simulateScriptLink(BotStep script) {
    ret appendQueryToURL(baseLink + "/demo", _botConfig := makePostData(mainScript := script.id), cookie := "test_" + aRandomID());
  }
  
  bool handleGeneralUserInput(Conversation conv, Msg msg, Bool priority) {
    for (UserKeyword qa : conceptsWhere UserKeyword(onlyNonNullParams(+priority))) {
      if (qa.enabled && mmo2_match(qa.parsedPattern(), msg.text)) {
        print("Matched pattern: " + qa.pattern + " / " + msg.text);
        conv.noteEvent(EvtMatchedUserKeyword(qa, msg));
        conv.callSubroutine(qa.action!);
        conv.scheduleNextStep();
        true;
      }
    }
  
    false;
  }
  
  void didntUnderstandUserInput(Conversation conv, Msg msg) {
    // TODO
  }
  
  Settings settings() {
    ret conceptWhere(Settings);
  }
  
  S displayTextHelp() {
    ret "Text to show to user (can include HTML and " + targetBlank(baseLink + "/emojis", "emojis") + ")";
  }  
  
  void botActions {
    withDBLock(r {
      while licensed {
        ScheduledAction action = lowestConceptByField ScheduledAction("time");
        if (action == null) break;
        if (action.time > now()) {
          //print("Postponing next bot action - " + (action.time-now()));
          doAfter(100, rstBotActions);
          break;
        }
        cdelete(action);
        print("Executing action " + action);
        pcall { action.run(); }
      }
    });
  }
  
  // action must be unlisted
  void addScheduledAction(ScheduledAction action, long delay default 0) {
    action.time = now()+delay;
    registerConcept(action);
    rstBotActions.trigger();
  }
  
  transient new ThreadLocal<Out> out;
  transient new ThreadLocal<Conversation> conv;

  transient long lastConversationChange = now();
  
  void pWebChatBot {
    dbIndexing(Conversation, 'cookie, Conversation, 'worker,
      Conversation, 'lastPing,
      Worker, 'loginName, AuthedDialogID, 'cookie);
    indexConceptFieldDesc(ScheduledAction, 'time);
    indexConceptFieldCI(Domain, 'domainAndPath);
    indexConceptFieldCI(DeliveredDomain, 'domain);
    indexConceptFieldCI(CannedAnswer, 'hashTag);
    indexConceptFieldCI(Language, 'languageName);
    indexConceptField(Lead, 'domain);
    indexConceptField(ConversationFeedback, 'domain);
    
    db_mainConcepts().miscMap = litmap(DynNewBot2, this);
  
    // legacy clean-up

    // make Settings object, index it
    indexSingletonConcept(Settings);
    uniq(Settings);
  
    // Make singleton commands
    uniq(BotSaveLead);
    uniq(BotSaveFeedback);
    uniq(BotClearStack);
  }
  
  void addReplyToConvo(Conversation conv, IF0<S> think) {
    out.set(new Out);
    S reply = "";
    pcall {
      reply = think!;
    }
    Msg msg = new Msg(false, reply);
    msg.out = out!;
    conv.add(msg);
  }
  
  Msg msgFromThinkFunction(IF0<S> think) {
    out.set(new Out);
    S reply = "";
    pcall {
      reply = think!;
    }
    Msg msg = new Msg(false, reply);
    msg.out = out!;
    ret msg;
  }
  
  
  O withHeader(S html) {
    ret withHeader(subBot_noCacheHeaders(subBot_serveHTML(html)));
  }
  
  O withHeader(O response) {
    call(response, 'addHeader, "Access-Control-Allow-Origin", "*");
    ret response;
  }
  
  S renderMessageText(S text, bool htmlEncode) {
    text = trim(text);
    if (eqic(text, "!rate conversation")) text = "Rate this conversation";
    if (htmlEncode) text = htmlEncode2(text);
    text = nlToBr(text);
    ret html_emojisToUnicode(text);
  
  }
  
  void renderMessages(Conversation conv, StringBuilder buf, L<Msg> msgs) {
    if (empty(msgs)) ret;
    for (Msg m : msgs) {
      if (!m.fromUser && eq(m.text, "-")) continue;
      S html = renderMessageText(m.text, shouldHtmlEncodeMsg(m));
  
      appendMsg(conv, buf, m.fromUser ? defaultUserName() : botName, formatTime(m.time), html, !m.fromUser, m.fromWorker, m.out);
  
      if (m == last(msgs)) {
        // yeah we're doing it too much
        buf.append(hscript([[$("#chat_telephone, .iti").hide(); $("#chat_message").show();]]));
        
        if (m.out != null && nempty(m.out.javaScript))
          buf.append(hscript(m.out.javaScript));
      }
    }
        
    appendButtons(buf, last(msgs).out, null);
  }
  
  void appendMsg(Conversation conv, StringBuilder buf, S name, S time, S text, bool bot, Worker fromWorker,
    Out out) {
    bool useTrick = ariaLiveTrick;
    S tag = useTrick ? "div" : "span";
    if (bot) {
      S id = randomID();
      S author = fromWorker != null ? htmlEncode2(fromWorker.displayName ): botName;
      if (fromWorker != null) buf.append([[<div class="chat_botname"><p>]] + author + [[</p>]]);
      buf.append("<" + tag + [[ class="chat_msg_item chat_msg_item_admin"]] + (useTrick ? [[ id="]] + id + [[" aria-live="polite" tabindex="-1"]] : "") + ">");
      S imgURL;
      if (fromWorker != null && fileExists(workerImageFile(fromWorker.id)))
        imgURL = fullRawLink("worker-image/" + fromWorker.id);
      else 
        imgURL = botImageForDomain(conv.domainObj!);
        
      if (nempty(imgURL))
        buf.append([[
          <div class="chat_avatar">
            <img src="$IMG"/>
          </div>]]
          .replace("$IMG", imgURL));
        
      buf.append([[<span class="sr-only">]] + (fromWorker != null ? "" : botName + " ") + [[says</span>]]);
      buf.append(text);
      buf.append([[</]] + tag + [[>]]);
      if (fromWorker != null) buf.append("</div>");
      if (useTrick) buf.append(hscript("$('#" + id + "').focus();"));
    } else
      buf.append(([[
        <span class="sr-only">You say</span>
        <]] + tag + [[ class="chat_msg_item chat_msg_item_user"]] + (useTrick ? [[ aria-live="polite"]] : "") + [[>$TEXT</]] + tag + [[>
      ]]).replace("$TEXT", text));
  }
  
  S replaceButtonText(S s) {
    if (eqicOneOf(s, "back", "zurück")) ret unicode_undoArrow();
    if (eqicOneOf(s, "cancel", "Abbrechen")) ret unicode_crossProduct();
    ret s;
  }
  
  S renderMultipleChoice(LS buttons, Cl<S> selections, S multipleChoiceSeparator) {
    print(+selections);
    Set<S> selectionSet = asCISet(selections);
    S rand = randomID();
    S className = "chat_multiplechoice_" + rand;
    S allCheckboxes = [[$(".]] + className + [[")]];
    ret joinWithBR(map(buttons, name ->
      hcheckbox("", contains(selectionSet, name),
        value := name,
        class := className) + " " + name))
      + "<br>" + hbuttonOnClick_returnFalse("OK", "submitMsg()", class := "btn btn-ok")
      + hscript(allCheckboxes
        + ".change(function() {"
        //+ "  console.log('multiple choice change');"
        + " var theList = $('." + className + ":checkbox:checked').map(function() { return this.value; }).get();"
        + "  console.log('theList: ' + theList);"
        + "  $('#chat_message').val(theList.join(" + jsQuote(multipleChoiceSeparator) + "));"
        + "});");
  }
  
  // render single-choice buttons
  S renderSingleChoice(LS buttons) {
    new LS out;
    for i over buttons: {
      S code = buttons.get(i);
      S text = replaceButtonText(code);
      out.add(hbuttonOnClick_returnFalse(text, "submitAMsg(" + jsQuote(text) + ")", class := "chatbot-choice-button", title := eq(code, text) ? null : code));
      if (!specialButtons.contains(code)
        && i+1 < l(buttons) && specialButtons.contains(buttons.get(i+1)))
        out.add("&nbsp;&nbsp;");
    }
    ret lines(out);
  }
  
  void appendButtons(StringBuilder buf, Out out, Set<S> buttonsToSkip) {
    S placeholder = out == null ? "" : unnull(out.placeholder);
    S defaultInput = out == null ? "" : unnull(out.defaultInput);
    buf.append(hscript("chatBot_setInput(" + jsQuote(defaultInput) + ", " + jsQuote(placeholder) + ");"));
    if (out == null) ret;
    LS buttons = listMinusSet(out.buttons, buttonsToSkip);
    if (empty(buttons)) ret;
    printVars_str(+buttons, +buttonsToSkip);
    S buttonsHtml;
    if (out.multipleChoice)
      buttonsHtml = renderMultipleChoice(buttons,
        text_multipleChoiceSplit(out.defaultInput, out.multipleChoiceSeparator), out.multipleChoiceSeparator);
    else
      buttonsHtml = renderSingleChoice(buttons);
    
    buf.append(span(buttonsHtml,
      class := "chat_msg_item chat_msg_item_admin chat_buttons"
        + stringIf(!out.multipleChoice, " single-choice")));
  }
  
  void appendDate(StringBuilder buf, S date) {
    buf.append([[
    <div class="chat-box-single-line">
      <abbr class="timestamp">DATE</abbr>
    </div>]].replace("DATE", date));
  }
  
  bool lastUserMessageWas(Conversation conv, S message) {
    Msg m = last(conv.msgs);
    ret m != null && m.fromUser && eq(m.text, message);
  }
  
  S formatTime(long time) {
    ret timeInTimeZoneWithOptionalDate_24(timeZone, time);
  }
  
  S formatDialog(S id, L<Msg> msgs) {
    new L<S> lc;
    for (Msg m : msgs)
      lc.add(htmlencode((m.fromUser ? "> " : "< ") + m.text));
    ret id + ul(lc);
  }
  
  Conversation getConv(fS cookie) {
    ret withDBLock(func -> Conversation {
      uniq(Conversation, +cookie)
    });
  }
  
  S errorMsg(S msg) {
    ret hhtml(hhead_title("Error") + hbody(hfullcenter(msg + "<br><br>" + ahref(jsBackLink(), "Back"))));
  }
  
  S defaultUserName() {
    ret "You";
  }
  
  bool botOn() {
    true;
  }
  
  bool botAutoOpen() {
    false;
  }
  
  File workerImageFile(long id) {
    ret id == 0 ? null : javaxDataDir("adaptive-bot/images/" + id + ".jpg");
  }
  
  long activeConversationTimeout() {
    ret longPollMaxWait+activeConversationSafetyMargin;
  }
  
  AuthedDialogID authObject(S cookie) {
    AuthedDialogID auth = empty(cookie) ? null : conceptWhere AuthedDialogID(+cookie);
    printVars_str(+cookie, +auth);
    ret auth;
  }
  
  bool anyInterestingMessages(L<Msg> msgs, bool workerMode) {
    ret any(msgs, m -> m.fromUser == workerMode);
  }
  
  bool shouldHtmlEncodeMsg(Msg msg) {
    ret msg.fromUser;
  }
  
  void calcCountry(Conversation c) {
    cset(c, country := "");
    getCountry(c);
  }
  
  S getCountry(Conversation c) {
    if (empty(c.country) && nempty(c.ip))
      cset(c, country := ipToCountry2020_safe(c.ip));
    ret or2(c.country, "?");
  }
  
  void noteConversationChange {
    lastConversationChange = now();
  }
  
  S template(S hashtag, O... params) {
    ret replaceSquareBracketVars(getCannedAnswer(hashtag), params);
  }
  
  // TODO: redefine this
  S getCannedAnswer(S hashTag, Conversation conv default null) {
    ret hashTag;
    //if (!startsWith(hashTag, "#")) ret hashTag;
    //ret or2(trim(text(conceptWhereCI CannedAnswer(+hashTag))), hashTag);
  }
  
  LS text_multipleChoiceSplit(S input, S multipleChoiceSeparator) {
    ret trimAll(splitAt(input, dropSpaces(multipleChoiceSeparator)));
  }
  
  Msg lastBotMsg(L<Msg> l) {
    ret lastThat(l, msg -> !msg.fromUser);
  }
  
  // TODO
  File uploadedImagesDir() {
    ret programDir("uploadedImages");
  }
} // end of module

// start of concepts

abstract concept BotStep {
  S comment;

  // returns true iff execution should continue immediately
  bool run(Conversation conv) { false; }

  ScheduledAction nextStepAction(Conversation conv) {
    ret Action_NextStep(conv, conv.executedSteps);
  }

  double preTypingDelay() { ret 0; }
}

concept Sequence > BotStep {
  new RefL<BotStep> steps;

  toString { ret "Sequence " + quoteOr(comment, "(unnamed)")
    + spaceRoundBracketed(nSteps(steps))
    /*+ ": " + joinWithComma(steps)*/;
  }

  bool run(Conversation conv) {
    // add me to stack
    syncAdd(conv.stack, new ActiveSequence(this));
    conv.change();
    true;
  }
}

concept BotMessage > BotStep {
  S text;
  S specialPurpose;
  //S hashtag; // optional

  sS _fieldOrder = "text specialPurpose";

  toString { ret "Message" + spacePlusRoundBracketedIfNempty(comment) + ": " + newLinesToSpaces2(shorten(text, 40)); }

  bool run(Conversation conv) {
    botMod().addReplyToConvo(conv, () -> text);
    true;
  }
  
  double preTypingDelay() { ret botMod().settings().botTypingDelay; }
}

abstract concept AbstractBotImage > BotStep {
  S altText;

  bool run(Conversation conv) {
    botMod().addReplyToConvo(conv, () -> himgsrc(imageURL(), title := altText, alt := altText, class := "chat_contentImage"));
    true;
  }

  abstract S imageURL();
  
  double preTypingDelay() { ret botMod().settings().botTypingDelay; }
}

// external image
concept BotImage > AbstractBotImage {
  S imageURL, altText;

  sS _fieldOrder = "imageURL";

  S imageURL() { ret imageURL; }

  toString { ret "Image" + spacePlusRoundBracketedIfNempty(comment) + ": " + imageURL; }
}

concept UploadedImage > AbstractBotImage {
  S imageURL() { ret botMod().baseLink + "/uploaded-image/" + id; }

  File imageFile() { ret newFile(botMod().uploadedImagesDir(), id + ".png"); }
  
  toString { ret "Image" + spacePlusRoundBracketedIfNempty(altText)
    + spacePlusRoundBracketedIfNempty(comment); }

  void delete {
    if (concepts() == db_mainConcepts())
      deleteFile(imageFile());
    super.delete();
  }
}

concept BotSendTyping > BotStep {
  bool run(Conversation conv) {
    conv.botTyping = now();
    true;
  }
}

concept BotPause > BotStep {
  double seconds;

  *() {}
  *(double *seconds) {}

  bool run(Conversation conv) {
    botMod().addScheduledAction(nextStepAction(conv), toMS(seconds));
    false;
  }

  toString { ret "Pause for " + (seconds == 1 ? "1 second" : formatDouble(seconds, 1) + " seconds"); }
}

concept BotOutgoingQuestion > BotStep implements IInputHandler {
  S displayText;
  S key;
  S defaultValue;
  S placeholder; // null for same as displayText, "" for none
  LS buttons;
  bool allowFreeText; // only matters when there are buttons
  bool multipleChoice; // buttons are checkboxes
  S multipleChoiceSeparator = ", "; // how to join choices into value
  bool optional; // can be empty
  S answerCheck; // e.g. "email address"
  new RefL<BotStep> buttonActions; // what to do on each button

  sS _fieldOrder = "displayText key defaultValue placeholder buttons allowFreeText multipleChoice optional answerCheck";

  toString { ret "Question: " + orEmptyQuotes(displayText); }

  bool run(Conversation conv) {
    Msg msg = new Msg(false, displayText);
    msg.out = makeMsgOut();
    conv.add(msg);
    cset(conv, inputHandler := this);
    false;
  }

  Out makeMsgOut() {
    new Out out;
    out.placeholder = or(placeholder, displayText);
    out.defaultInput = defaultValue;
    out.buttons = cloneList(buttons);
    out.multipleChoice = multipleChoice;
    out.multipleChoiceSeparator = multipleChoiceSeparator;
    if (eqic(answerCheck, "phone number"))
      //out.javaScript = [[$("#chat_countrycode").addClass("visible");]];
      out.javaScript = [[$("#chat_telephone, .iti").show(); $("#chat_message").hide();]];
    ret out;
  }

  public bool handleInput(S s, Conversation conv) {
    print("BotOutgoingQuestion handleInput " + s);

    // Store answer
    s = trim(s);
    syncPut(conv.answers, or2(key, displayText), s);
    conv.change();

    // Validate
    if (eqic(answerCheck, "email address") && !isValidEmailAddress_simple(s))
      ret true with handleValidationFail(conv, "bad email address");
    if (eqic(answerCheck, "phone number") && !isValidInternationalPhoneNumber(s))
      ret true with handleValidationFail(conv, "bad phone number");

    conv.removeInputHandler(this);
    
    int idx = indexOfIC(buttons, s);
    print("Button index of " + quote(s) + " in " + sfu(buttons) + " => " + idx);
    BotStep target = get(buttonActions, idx);
    print("Button action: " + target);
    if (target != null)
      conv.jumpTo(target);

    // Go to next step
    print("Scheduling next step in " + conv);
    conv.scheduleNextStep();
    true; // acknowledge that we handled the input
  }
  
  double preTypingDelay() { ret empty(displayText) ? 0 : botMod().settings().botTypingDelay; }

  // show error msg (if found) and reschedule input handling
  void handleValidationFail(Conversation conv, S purpose) {
    BotMessage msg = botMod().messageForPurpose(purpose);
    if (msg != null) {
      new Msg m;
      m.text = msg.text;
      m.out = makeMsgOut();
      conv.add(m);
    }
    cset(conv, inputHandler := this);
    ret;
  }
}

concept BotSaveLead > BotStep {
  toString { ret "Save Lead"; }
  
  bool run(Conversation conv) {
    cnew Lead(conversation := conv,
      domain := conv.domainObj,
      date := Timestamp(now()),
      answers := cloneMap(conv.answers));
    true;
  }
}

concept BotSaveFeedback > BotStep {
  toString { ret "Save Conversation Feedback"; }
  
  bool run(Conversation conv) {
    cnew ConversationFeedback(conversation := conv,
      domain := conv.domainObj,
      date := Timestamp(now()),
      answers := filterKeys(conv.answers, swic$("Conversation Feedback:")));
    true;
  }
}

concept BotClearStack > BotStep {
  bool run(Conversation conv) {
    syncRemoveAllExceptLast(conv.stack);
    conv.change();
    true;
  }

  toString { ret "Clear Stack [forget all running procedures in conversation]"; }
}

concept UserKeyword {
  new Ref<Language> language;
  S pattern;
  LS examples, counterexamples;
  new Ref<BotStep> action;
  bool enabled = true;
  bool priority;
  transient MMOPattern parsedPattern;

  void change { parsedPattern = null; super.change(); }
  
  MMOPattern parsedPattern() {
    if (parsedPattern == null) parsedPattern = mmo2_parsePattern(pattern);
    ret parsedPattern;
  }

  toString { ret "User Keyword: " + pattern; }
}

concept Language {
  S languageName;
  S comment;

  toString { ret languageName + spaceRoundBracketed(comment); }
}

concept Evt {}

concept EvtMatchedUserKeyword > Evt {
  new Ref<UserKeyword> userKeyword;
  Msg msg;

  *() {}
  *(UserKeyword userKeyword, Msg *msg) { this.userKeyword.set(userKeyword); }
 }
 
concept EvtJumpTo > Evt {
  new Ref<BotStep> target;

  *() {}
  *(BotStep target) { this.target.set(target); }
}

concept Avatar {
  S name, comment;
  new Ref<AbstractBotImage> image;
}

/*concept ActorInConversation {
  new Ref<Conversation> conv;
  new Ref<Avatar> avatar;
  new L<ActiveSequence> stack;
}*/

// end of concepts

// special options for a message
sclass Out extends DynamicObject {
  LS buttons;
  bool multipleChoice;
  S multipleChoiceSeparator;
  S placeholder;
  S defaultInput;
  S javaScript;
}

sclass Msg extends DynamicObject {
  long time;
  bool fromUser;
  //Avatar avatar;
  Worker fromWorker;
  S text;
  Out out;
  
  *() {}
  *(bool *fromUser, S *text) { time = now(); }
  *(S *text, bool *fromUser) { time = now(); }

  toString { ret (fromUser ? "User" : "Bot") + ": " + text; }
}

concept AuthedDialogID {
  S cookie;
  bool master;
  new Ref<Domain> restrictedToDomain;
  Worker loggedIn; // who is logged in with this cookie
  
  S domain() {
    ret !restrictedToDomain.has() ? null : restrictedToDomain->domainAndPath;
  }
}

//concept Session {} // LEGACY

sclass ActiveSequence {
  Sequence originalSequence;
  L<BotStep> steps;
  int stepIndex;

  *() {}
  *(BotStep step) {
    if (step cast Sequence) {
      steps = cloneList(step.steps);
      originalSequence = step;
    } else
      steps = ll(step);
    steps = botMod().addTypingDelays(steps);
  }
    
  bool done() { ret stepIndex > l(steps); }
  BotStep currentStep() { ret get(steps, stepIndex); }
  void nextStep() { ++stepIndex; }

  toString {
    ret (stepIndex >= l(steps) ? "DONE " : "Step " + (stepIndex+1) + "/" + l(steps) + " of ")
      + (originalSequence != null ? str(originalSequence) : squareBracket(joinWithComma(steps)));
  }
}

// our base concept - a conversation between a user and a bot or sales representative
concept Conversation {
  S cookie, ip, country, domain;
  new Ref<Domain> domainObj;
  new LL<Msg> oldDialogs;
  new L<Msg> msgs;
  long lastPing;
  bool botOn = true;
  bool notificationsOn = true;
  Worker worker; // who are we talking to?
  transient long userTyping, botTyping; // sysNow timestamps
  bool testMode;
  long nextActionTime = Long.MAX_VALUE;
  long userMessageProcessed; // timestamp
  int newDialogTriggered = -1;
  transient bool dryRun;
  
  new L<ActiveSequence> stack;
  int executedSteps;
  IInputHandler inputHandler;
  new Ref<Language> language;
  //Long lastProposedDate;
  SS answers = litcimap();
  SS botConfig;
  new RefL<Concept> events; // just logging stuff

  void add(Msg m) {
    m.text = trim(m.text);
    //if (!m.fromUser && empty(m.text) && (m.out == null || empty(m.out.buttons))) ret; // don't store empty msgs from bot
    syncAdd(msgs, m);
    botMod().noteConversationChange();
    change();
    vmBus_send chatBot_messageAdded(mc(), this, m);
  }
  
  int allCount() { ret archiveSize() + syncL(msgs); }
  int archiveSize() { ret syncLengthLevel2(oldDialogs) + syncL(oldDialogs); }
  
  long lastMsgTime() { Msg m = last(msgs); ret m == null ? 0 : m.time; }
  
  S language() { ret "en"; }
  
  void turnBotOff {
    cset(this, botOn := false);
    botMod().noteConversationChange();
  }

  void turnBotOn {  
    cset(this, botOn := true, worker := null);
    S backMsg = botMod().getCannedAnswer("#botBack", this);
    if (empty(msgs) || lastMessageIsFromUser() || !eq(last(msgs).text, backMsg))
      add(new Msg(backMsg, false));
    botMod().noteConversationChange();
  }
  
  bool lastMessageIsFromUser() {
    ret nempty(msgs) && last(msgs).fromUser;
  }
  
  void newDialog {
    syncAdd(oldDialogs, msgs);
    cset(this, msgs := new L);
    syncClear(answers);
    syncClear(stack);
    // TODO: clear scheduled actions?
    change();
    print("newDialog " + archiveSize() + "/" + allCount());

    vmBus_send chatBot_clearedSession(mc(), this);
    //botMod().addScheduledAction(new OnNewDialog(this));
  }

  void jumpTo(BotStep step) {
    syncPopLast(stack); // terminate inner script
    print("Jumping to " + step);
    noteEvent(EvtJumpTo(step));
    if (step == null) ret;
    syncAdd(stack, new ActiveSequence(step));
    change();
  }

  void callSubroutine(BotStep step) {
    if (step == null) ret;
    print("Calling subroutine " + step);
    syncAdd(stack, new ActiveSequence(step));
    change();
  }

  void removeInputHandler(IInputHandler h) {
    if (inputHandler == h)
      cset(this, inputHandler := null);
  }

  void scheduleNextStep {
    botMod().addScheduledAction(new Action_NextStep(this));
  }

  void noteEvent(Concept event) {
    events.add(event);
  }
} // end of Conversation

abstract concept ScheduledAction implements Runnable {
  long time;
}

concept Action_NextStep > ScheduledAction {
  new Ref<Conversation> conv;
  int executedSteps = -1; // if >= 0, we verify against conv.executedSteps

  *() {}
  *(Conversation conv) { this.conv.set(conv); }
  *(Conversation conv, int *executedSteps) { this.conv.set(conv); }

  run {
    nextStep(conv!, executedSteps);
  }
}

concept OnNewDialog > ScheduledAction {
  new Ref<Conversation> conv;
   
  *() {}
  *(Conversation conv) { this.conv.set(conv); }
  
  run {
    Domain domain = conv->domainObj!;
    if (domain == null)
      ret with botMod().addReplyToConvo(conv!, () -> "Error: No domain set");
    BotStep seq = getConcept BotStep(parseLong(mapGet(conv->botConfig, "mainScript")));
    if (seq == null)
      seq = domain.mainScript!;
    if (seq == null)
      ret with botMod().addReplyToConvo(conv!, () -> "Error: No main script set for " + htmlEncode2(str(domain)));
    if (executeStep(seq, conv!))
      nextStep(conv!);
  }
}

svoid nextStep(Conversation conv, int expectedExecutedSteps default -1) {
  if (expectedExecutedSteps >= 0 && conv.executedSteps != expectedExecutedSteps) ret;
  
  while licensed {
    if (empty(conv.stack)) ret;
    ActiveSequence seq = last(conv.stack);
    if (seq.done())
      continue with syncPopLast(conv.stack);
    BotStep step = seq.currentStep();
    seq.nextStep();
    ++conv.executedSteps;
    conv.change();
    if (!executeStep(step, conv))
      break;
  }
}

sbool executeStep(BotStep step, Conversation conv) {
  if (step == null || conv == null) false;
  print("Executing step " + step + " in " + conv);
  ret step.run(conv);
}

concept OnUserMessage > ScheduledAction {
  new Ref<Conversation> conv;
  
  *() {}
  *(Conversation conv) { this.conv.set(conv); }

  run {
    // TODO: handle multiple messages in case handling was delayed
    Msg msg = syncLast(conv->msgs);
    if (msg == null || !msg.fromUser) ret;

    print("OnUserMessage: " + msg);

    if (botMod().handleGeneralUserInput(conv!, msg, true))
      ret;
    
    if (conv->inputHandler != null) {
      if (conv->inputHandler.handleInput(msg.text, conv!))
        ret;
    }

    botMod().handleGeneralUserInput(conv!, msg, false);
  }
}

// application-specific concepts

// a server we're running - either a domain name or domain.bla/path
concept Domain {
  S domainAndPath;
  S tenantID; // for external software

  // bot config
  new Ref<AbstractBotImage> botImage;
  S botName; // if empty, keep default
  S headerColorLeft, headerColorRight; // ditto
  
  new Ref<BotStep> mainScript;

  S password = aGlobalID();

  sS _fieldOrder = "domainAndPath botImage mainScript";
  
  toString { ret domainAndPath; }
}

concept Form {
  S name;
  new RefL<BotStep> steps;

  toString { ret name; }
}

concept DeliveredDomain {
  S domain;
}

concept CannedAnswer {
  S hashTag, text;
}

sS text(CannedAnswer a) { ret a?.text; }

concept Lead {
  new Ref<Domain> domain;
  new Ref<Conversation> conversation;
  Timestamp date;
  SS answers;
}

concept ConversationFeedback {
  new Ref<Domain> domain;
  new Ref<Conversation> conversation;
  Timestamp date;
  SS answers;
}

concept Settings {
  double botTypingDelay;
  S preferredCountryCodes; // dial codes, comma-separated
  bool multiLanguageMode;
}

sinterface IInputHandler {
  // true if handled
  bool handleInput(S s, Conversation conv);
}

static DynNewBot2 botMod() {
  ret (DynNewBot2) db_mainConcepts().miscMap.get(DynNewBot2);
}