!7

set flag OurSyncCollections.

!include #1015743 // concept AIList

static ConceptFieldIndexCI<AIList> nameIndex;
static volatile long totalChars = -1;

concept Session {
  S cookie;
  S mech_filter;
}

p {
  dbSaveEvery(60);
  dbIndexing(AIList, 'name);
  nameIndex = indexConceptFieldCI(AIList, 'name);
  thread { mech_guessLanguage(""); } // preload
  calcOnConceptChanges(10000, r calcTotalSize, true);
  addConceptIndex(new IConceptIndex {
    public void update(Concept c) {
      pcall { if (c cast AIList) sendToUpdatesBot('mechListChange, c.name); }
    }
    public void remove(Concept c) {
      pcall { if (c cast AIList) sendToUpdatesBot('removeMechList, c.name); }
    }
  });
  updateListNamesList();
}

set flag NoNanoHTTPD. html {
  temp countDispatch('html);
  O response;
  new Matches m;
  
  S cookie = cookieFromUser();
  S filter = params.get('mech_filter);
  print("cookie=" + cookie + ", filter=" + filter + ", dialogID=" + getDialogID());
  Session session = empty(cookie) ? null : uniq(Session, +cookie);
  if (session != null && filter != null) {
    cset(session, mech_filter := filter);
    ret hrefresh(rawSelfLink());
  }

  bool authed = webAuthed(params);
  
  if (eq(uri, "/list-count")) ret serveText(countConcepts(AIList));
  if ((response = serveListText(uri, params, authed)) != null) ret response;
  
  // Append API
  if (swic(uri, "/bot-list-append/", m)) {
    params.put(name := urldecode(m.rest());
    uri = "/bot-list-append";
  }

  if (eq(uri, "/bot-list-append")) {
    S mode = params.get('mode);
    lock dbLock();
    S name = params.get("name"), text = params.get("text");
    if (!authed)
      name = "Unauthorized Appends | " + name;
    AIList l = getList(name);
    //if (!authed) cset(l, status := "PUBLIC READ");
    
    if (eq(mode, "uniqCI")) {
      text = lines(listMinusSet(tlft(text), asCISet(tlft(l.text))));
      if (empty(text)) ret "No change";
    }
        
    if (eq(mode, "uniq")) {
      text = lines(listMinusSet(tlft(text), tlft(l.text)));
      if (empty(text)) ret "No change";
    }
    
    listAppendWithLog(l, text);
    ret "Changed";
  }

  if (!authed && eq(uri, "/list-names"))
    ret serveText(jsonEncode(sortedIC(collect(publicReadLists(), 'name))));
    
  if (!authed && eq(uri, "/list-md5s"))
    ret serveText(jsonEncode(map(publicReadLists(), func(AIList l) -> LS { ll(l.name, md5OfList(l)) })));
    
  if (!authed && eq(uri, "/list-md5s-and-statuses"))
    ret serveText(jsonEncode(map(publicReadLists(), func(AIList l) -> LS { ll(l.name, md5OfList(l), l.status, str(lineCountOfList(l))) })));
    
  if (eq(uri, "/authed")) ret yesno(authed);
  
  // Show list (unauthed)
  if (!authed && startsWith(uri, "/list/", m)) {
    AIList l = getList_noCreate(urldecode(m.get(0)));
    if (l == null || !l.isPublicRead()) ret serve404("List not found");
    ret htitle_h2(htmlencode(l.name)) + hpre_htmlencode(l.text);
  }
  
  bool alphabetical = eq(uri, "/alphabetical");
  if (!authed && (eq(uri, "/") || alphabetical)) {
    L<AIList> list = alphabetical
    ? sortedByFieldIC('name, publicReadLists())
    : sortedByFieldDesc('modified, publicReadLists());

    ret h3_title("Bot Data")
    + hBoolSelector(alphabetical, "/", "By date", "/alphabetical", "Alphabetical")
    + listLists(list);
  }
  
  // Search results
  S q = params.get('q);
  if (nempty(q)) {
    L<AIList> found = scoredSearch_AIList(q, listsForAuth(authed));
    
    if (eq(uri, "/json-search"))
      ret serveText(jsonEncode(collect name(found)));
      
    ret hmobilefix()
      + htitle_h3("mech.botcompany.de: Search results for " + htmlencode(singleQuote(q)))
      + listLists(found);
  }

  if (!authed) ret redirectToWebAuth();
  
  // AUTHED FROM HERE ON
  
  try object html_serveDispatches(uri);

  if (swic(uri, "/list-id/", m)) {
    AIList l = getList_noCreate(urldecode(m.rest()));
    ret l == null ? "List not found" : str(l.id);
  }

  if (eq(uri, "/fix-unauth")) {
    for (AIList l) if (swic(l.name, "Unauthorized Appends |"))
      cset(l, status := "");
    ret "OK";
  }
  
  // API
  if (eq(uri, "/download"))
    ret subBot_serveFileWithName("mechlists-" + ymd_minus_hms() + ".gz", conceptsFile()); // TODO: concurrent writes?
  
  if (eq(uri, "/list-names"))
    ret serveText(jsonEncode(sortedIC(collect(list(AIList), 'name))));

  if (eq(uri, "/list-md5s"))
    ret serveText(jsonEncode(map(list(AIList), func(AIList l) -> LS { ll(l.name, md5OfList(l)) })));
    
  if (eq(uri, "/list-md5s-and-statuses"))
    ret serveText(jsonEncode(map(list(AIList), func(AIList l) -> LS { ll(l.name, md5OfList(l), l.status, str(lineCountOfList(l))) })));
    
  if (eq(uri, "/bot-edited-lists"))
    ret serveText(jsonEncode(sortedIC(collect(conceptsWhere(AIList, botFlag := true), 'name))));

  // Create API
  if (eq(uri, "/bot-list-create")) {
    lock dbLock();
    S name = params.get("name");
    getList(name);
    ret "OK";
  }

  // Edit API
  if (eq(uri, "/bot-list-edit")) {
    lock dbLock();
    S name = params.get("name"), text = params.get("text");
    AIList l = getList(name);
    if (eq(l.text, text)) ret "No change";
    listReplaceWithLog(l, text);
    ret "Changed";
  }

  // Show list (authed)
  if (startsWith(uri, "/list/", m)) {
    S name = urldecode(m.get(0));
    AIList l = eq("1", params.get("create")) ? getList(name) : getList_noCreate(name);
    if (l == null)
      ret "List not found"
        + (!authed ? "" : hpostform(hhidden(create := 1) + hsubmit("Create")));
    
    bool delete = eq(params.get("delete"), "1");
    
    // Rename list
    S renameTo = params.get('renameTo);
    if (nempty(renameTo) && !delete) {
      if (neqic(l.name, renameTo) && getList_noCreate(renameTo) != null)
        ret "Can't rename - list " + quote(renameTo) + " already exists";
      S oldName = l.name;
      cset(l, name := renameTo);
      updateListNamesList();
      ret nav() + "List " + quote(oldName) + " renamed to " + quote(renameTo);
    }
    
    if (delete) {
      logStructure("versions.log", litmap(id := l.id, status := "delete me", date := now()));
      deleteConcept(l);
      updateListNamesList();
      ret nav() + "List deleted";
    }

    // Update text
    S text = params.get("human");
    S status = params.get("status");
    if (text != null) {
      lock dbLock();
      //printStruct(ll(+text, +status));
      if (neqAny(text, l.text, status, unnull(l.status))) {
        logStructure("versions.log", litmap(id := l.id, +text, +status, date := now()));
        if (status != null) cset(l, +status);
        cset(l, +text, textMD5 := md5(text), lines := -1, botFlag := false, modified := now());
        listTextChanged(l);
        //printStruct("l.text=" + sfu(text));
      }
      ret hrefresh(0, rawLink(uri) + "?random=" + randomID());
    }
    
    S textArea = htextarea(l.text, name := 'human, cols := 80, rows := 30);
    
    L<S> tok = javaTok(l.status);
    jreplace(tok, "bb", "iframe 1018300");
    int idx = jfind(tok, "iframe <int>");
    if (idx > 0) {
      S analyzer = tok.get(idx+2);
      S subUri = joinSubList(tok, idx+4, smartIndexOf(tok, idx, ",")-1);
      textArea = htableRaw2_singleRow(ll(textArea, iframe(relativeRawBotLink(analyzer, subUri + "?list=" + urlencode(l.name)), width := "500", height := "500")),
        null, null, ll(valign := 'top));
    }
    
    // Serve list text (editable)
    
    ret nav() + htitle(l.name)
      + h2(ahref(neatMechListURL(l.name), htmlencode2(l.name)) + " [" + nLines(countLines(l.text)) + "]")
      + hpostform(
        "Status: " + htextinput('status, value := l.status)
      + h3("Contents (" + (l.botFlag ? "bot" : "human") + "-edited)")
      + p(hsubmit("Save"))
      + textArea
      + p(hsubmit("Save"))
      )
      + hpostform(
        "New name: " + htextinput('renameTo, value := l.name)
        + " "
        + hsubmit("Rename list") + " or " + hbutton("Delete list", name := "delete", type := "submit", value := 1, onClick := "return confirm('Really delete?')")
      );
  }
  
  // New list
  S newList = trim(params.get('newList));
  if (nempty(newList)) {
    AIList l = getList(newList);
    ret hrefresh(rawLink("/list/" + urlencode(l.name)));
  }
  
  // Overview
  
  L<AIList> list = alphabetical
    ? sortedByFieldIC('name, list(AIList))
    : sortedByFieldDesc('modified, list(AIList));
    
  fS prefix = session == null ? null : session.mech_filter;
  if (nempty(prefix))
    list = [AIList l : list | swic(l.name, prefix)];

  ret hmobilefix()
    + hcomment("cookie=" + cookie)
    + htitle_h3("mech.botcompany.de [" + n2(l(list), "list")
      + (totalChars >= 0 ? ", " + toK(totalChars) + "K chars" : "")
    + "]")
    + htableRaw_singleRow(ll(
      hform("Search: " + htextinput('q, style := "width: 200px") + " " + hsubmit("Search")),
      hpostform("| New list: " + htextinput('newList, style := "background-color: yellow") + " " + hsubmit("Create list"))))
    + hBoolSelector(alphabetical, "/", "By date", "/alphabetical", "Alphabetical")
    + listLists(list)
    + hpostform("Only show lists starting with: "
      + htextinput('mech_filter, value := prefix)
      + " " + hsubmit("OK"));
}

sS nav() {
  ret hmobilefix() + p(ahref(rawLink(), "&lt;&lt; back"));
}

static AIList getList_noCreate(S name) {
  ret getList(name, false);
}

static AIList getList(S name) {
  ret getList(name, true);
}

static AIList getList(S name, bool create) {
  lock dbLock();
  AIList l = findConcept(AIList, +name);
  if (l == null) l = nameIndex.get(name);
  if (l == null) if (!create) null; else {
    l = cnew(AIList, +name);
    logStructure("versions.log", litmap(id := l.id, +name, date := now(), mode := "List created"));
    updateListNamesList();
  }
  if (l.modified == 0) cset(l, modified := now());
  ret l;
}

sS listLists(L<AIList> list) {
  ret ul(map(list, func(AIList l) -> S {
    int n = lineCountOfList(l);
    new L<S> status;
    bool german = eq('german, mech_guessLanguage_quick(l.name));
    //if (l.mighty) status.add(german ? "GENUTZT" : "USED");
    addIfNempty(status, htmlencode(l.status));
    if (n != 0) status.add(n2(n, german ? "Zeile" : "line", german ? "Zeilen": "lines"));
    ret ahref(rawLink("/list/" + urlencode(l.name)),
        htmlencode(or2(l.name, "[no name]")) + appendBracketed(joinWithComma(status));
  }));
}

static L<AIList> scoredSearch_AIList(S query, Iterable<AIList> data) {
  new Map<AIList, Int> scores;
  L<S> prepared = scoredSearch_prepare(query);
  for (AIList l : data)
    putUnlessZero(scores, l,
        3*scoredSearch_score(l.name, prepared)
      + 2*scoredSearch_score(l.status, prepared)
      +   scoredSearch_score(l.text, prepared));
  ret keysSortedByValuesDesc(scores);
}

svoid calcTotalSize {
  long total = 0;
  for (AIList l)
    total += l(l.text);
  totalChars = total;
}

sS export_getListText(S listName) {
  AIList l = getList_noCreate(listName);
  ret l == null ? "" : l.text;
}

sS export_setListText(S listName, S text) {
  AIList l = getList(listName);
  if (eq(l.text, text)) ret "No change";
  logStructure("versions.log", litmap(id := l.id, +text, date := now(), mode := "bot edit"));
  cset(l, +text, textMD5 := md5(text), lines := -1, botFlag := true, modified := now());
  listTextChanged(l);
  ret "Changed";
}

sO serveListText(S uri, SS params, bool authed) {
  new Matches m;
  if (startsWith(uri, "/list-text/", m)) {
    bool opt = eq("1", params.get("opt"));
    bool withStatus = eq("1", params.get("withStatus"));
    bool create = authed && eq("1", params.get("create"));
    S md5 = params.get("md5");
    int md5Len = parseIntOpt(params.get('l));
    S name = urldecode(m.get(0));
    lock dbLock();
    AIList l = create ? getList(name) : getList_noCreate(name);
    if (!authed && l != null && !l.isPublicRead())
      l = null;
      //ret subBot_serve404("Not logged in");
    cset(l, mighty := true);
    if (l == null)
      if (opt)
        ret serveText(jsonEncode(litmap("Text" := "")));
      else
        ret serveText(jsonEncode(litmap("Error", ll("List not found", name))));
    else {
      Map map = litmap("Name" := l.name, "Status" := withStatus ? l.status : null);
      if (md5 != null) {
        S actualMD5 = md5OfList(l);
        print("md5: " + md5 + " / " + actualMD5);
        if (eq(actualMD5, md5))
          ret serveText(jsonEncode(mapPlus(map, "Same" := true)));
        else if (md5Len != 0 && eq(md5(takeFirst(l.text, md5Len)), md5))
          ret serveText(jsonEncode(mapPlus(map, "Appended" := true, "Text" := substring(l.text, md5Len))));
      }
      ret serveText(jsonEncode(mapPlus(map, "Text" := l.text)));
    }
  }
  null;
}

sS md5OfList(AIList l) {
  lock dbLock();
  if (l.textMD5 == null) l.textMD5 = md5(l.text);
  ret l.textMD5;
}

static int lineCountOfList(AIList l) {
  lock dbLock();
  if (l.lines < 0) l.lines = countLines(l.text);
  ret l.lines;
}

static L<AIList> publicReadLists() {
  ret [AIList l : list(AIList) | l.isPublicRead()];
}

svoid updateListNamesList {
  lock dbLock();
  AIList l = getList("All public-read mech lists");
  TreeSet<S> actual = new TreeSet(collect(publicReadLists(), 'name));
  Set<S> listed = asHashSet(lines(l.text));
  if (nempty(setMinusSet(listed, actual)))
    listReplaceWithLog(l, lines(actual));
  else
    listAppendWithLog(l, lines(setMinusSet(actual, listed)));
}

svoid listAppendWithLog(AIList l, S text) {
  if (emptyAfterTrim(text)) ret;
  logStructure("versions.log", litmap(id := l.id, +text, date := now(), mode := "bot append"));
  cset(l, text := appendNewLineIfNempty(rtrim(l.text)) + text, botFlag := true, modified := now(), textMD5 := null, lines := -1);
  listTextChanged(l);
}

svoid listReplaceWithLog(AIList l, S text) {
  if (eq(text, l.text)) ret;
  logStructure("versions.log", litmap(id := l.id, +text, date := now(), mode := "bot edit"));
  cset(l, +text, textMD5 := md5(text), lines := -1, botFlag := true, modified := now());
  listTextChanged(l);
}

static new ThreadLocal<Bool> listTextChanged_noRecurse;

svoid listTextChanged(AIList l) {
  if (!isTrue(listTextChanged_noRecurse!) && cic(l.status, "auto global ids")) pcall {
    L<S> entries = splitAtEmptyLines(l.text);
    S text = joinWithEmptyLines(map nlLogic_addGlobalID(entries));
    temp tempSetThreadLocal(listTextChanged_noRecurse, true);
    listReplaceWithLog(l, text);
  }
  
  if (!isTrue(listTextChanged_noRecurse!) && cic(l.status, "gazelle ids")) pcall {
    L<S> entries = splitAtEmptyLines(l.text);
    S text = joinWithEmptyLines(map gazelle_addGlobalID(entries));
    temp tempSetThreadLocal(listTextChanged_noRecurse, true);
    listReplaceWithLog(l, text);
  }
}

sS mechList_opt_raw_fresh(S name) {
  ret export_getListText(name);
}

static L<AIList> listsForAuth(bool authed) {
  ret authed ? list(AIList) : publicReadLists();
}