!7

set flag OurSyncCollections.

!include #1015743 // concept AIList

static ConceptFieldIndexCI<AIList> nameIndex;

static volatile long totalChars = -1;

concept Session {

  S cookie;

  S mech_filter;

}

p {

  dbSaveEvery(60);

  dbIndexing(AIList, 'name);

  nameIndex = indexConceptFieldCI(AIList, 'name);

  thread { mech_guessLanguage(""); } // preload

  calcOnConceptChanges(10000, r calcTotalSize, true);

  addConceptIndex(new IConceptIndex {

    public void update(Concept c) {

      pcall { if (c cast AIList) sendToUpdatesBot('mechListChange, c.name); }

    }

    public void remove(Concept c) {

      pcall { if (c cast AIList) sendToUpdatesBot('removeMechList, c.name); }

    }

  });

  updateListNamesList();

}

set flag NoNanoHTTPD. html {

  temp countDispatch('html);

  O response;

  new Matches m;

  S cookie = cookieFromUser();

  S filter = params.get('mech_filter);

  print("cookie=" + cookie + ", filter=" + filter + ", dialogID=" + getDialogID());

  Session session = empty(cookie) ? null : uniq(Session, +cookie);

  if (session != null && filter != null) {

    cset(session, mech_filter := filter);

    ret hrefresh(rawSelfLink());

  }

  bool authed = webAuthed(params);

  if (eq(uri, "/list-count")) ret serveText(countConcepts(AIList));

  if ((response = serveListText(uri, params, authed)) != null) ret response;

  // Append API

  if (swic(uri, "/bot-list-append/", m)) {

    params.put(name := urldecode(m.rest());

    uri = "/bot-list-append";

  }

  if (eq(uri, "/bot-list-append")) {

    S mode = params.get('mode);

    lock dbLock();

    S name = params.get("name"), text = params.get("text");

    if (!authed)

      name = "Unauthorized Appends | " + name;

    AIList l = getList(name);

    //if (!authed) cset(l, status := "PUBLIC READ");

    if (eq(mode, "uniqCI")) {

      text = lines(listMinusSet(tlft(text), asCISet(tlft(l.text))));

      if (empty(text)) ret "No change";

    }

    if (eq(mode, "uniq")) {

      text = lines(listMinusSet(tlft(text), tlft(l.text)));

      if (empty(text)) ret "No change";

    }

    listAppendWithLog(l, text);

    ret "Changed";

  }

  if (!authed && eq(uri, "/list-names"))

    ret serveText(jsonEncode(sortedIC(collect(publicReadLists(), 'name))));

  if (!authed && eq(uri, "/list-md5s"))

    ret serveText(jsonEncode(map(publicReadLists(), func(AIList l) -> LS { ll(l.name, md5OfList(l)) })));

  if (!authed && eq(uri, "/list-md5s-and-statuses"))

    ret serveText(jsonEncode(map(publicReadLists(), func(AIList l) -> LS { ll(l.name, md5OfList(l), l.status, str(lineCountOfList(l))) })));

  if (eq(uri, "/authed")) ret yesno(authed);

  // Show list (unauthed)

  if (!authed && startsWith(uri, "/list/", m)) {

    AIList l = getList_noCreate(urldecode(m.get(0)));

    if (l == null || !l.isPublicRead()) ret serve404("List not found");

    ret htitle_h2(htmlencode(l.name)) + hpre_htmlencode(l.text);

  }

  bool alphabetical = eq(uri, "/alphabetical");

  if (!authed && (eq(uri, "/") || alphabetical)) {

    L<AIList> list = alphabetical

    ? sortedByFieldIC('name, publicReadLists())

    : sortedByFieldDesc('modified, publicReadLists());

    ret h3_title("Bot Data")

    + hBoolSelector(alphabetical, "/", "By date", "/alphabetical", "Alphabetical")

    + listLists(list);

  }

  // Search results

  S q = params.get('q);

  if (nempty(q)) {

    L<AIList> found = scoredSearch_AIList(q, listsForAuth(authed));

    if (eq(uri, "/json-search"))

      ret serveText(jsonEncode(collect name(found)));

    ret hmobilefix()

      + htitle_h3("mech.botcompany.de: Search results for " + htmlencode(singleQuote(q)))

      + listLists(found);

  }

  if (!authed) ret redirectToWebAuth();

  // AUTHED FROM HERE ON

  try object html_serveDispatches(uri);

  if (swic(uri, "/list-id/", m)) {

    AIList l = getList_noCreate(urldecode(m.rest()));

    ret l == null ? "List not found" : str(l.id);

  }

  if (eq(uri, "/fix-unauth")) {

    for (AIList l) if (swic(l.name, "Unauthorized Appends |"))

      cset(l, status := "");

    ret "OK";

  }

  // API

  if (eq(uri, "/download"))

    ret subBot_serveFileWithName("mechlists-" + ymd_minus_hms() + ".gz", conceptsFile()); // TODO: concurrent writes?

  if (eq(uri, "/list-names"))

    ret serveText(jsonEncode(sortedIC(collect(list(AIList), 'name))));

  if (eq(uri, "/list-md5s"))

    ret serveText(jsonEncode(map(list(AIList), func(AIList l) -> LS { ll(l.name, md5OfList(l)) })));

  if (eq(uri, "/list-md5s-and-statuses"))

    ret serveText(jsonEncode(map(list(AIList), func(AIList l) -> LS { ll(l.name, md5OfList(l), l.status, str(lineCountOfList(l))) })));

  if (eq(uri, "/bot-edited-lists"))

    ret serveText(jsonEncode(sortedIC(collect(conceptsWhere(AIList, botFlag := true), 'name))));

  // Create API

  if (eq(uri, "/bot-list-create")) {

    lock dbLock();

    S name = params.get("name");

    getList(name);

    ret "OK";

  }

  // Edit API

  if (eq(uri, "/bot-list-edit")) {

    lock dbLock();

    S name = params.get("name"), text = params.get("text");

    AIList l = getList(name);

    if (eq(l.text, text)) ret "No change";

    listReplaceWithLog(l, text);

    ret "Changed";

  }

  // Show list (authed)

  if (startsWith(uri, "/list/", m)) {

    S name = urldecode(m.get(0));

    AIList l = eq("1", params.get("create")) ? getList(name) : getList_noCreate(name);

    if (l == null)

      ret "List not found"

        + (!authed ? "" : hpostform(hhidden(create := 1) + hsubmit("Create")));

    bool delete = eq(params.get("delete"), "1");

    // Rename list

    S renameTo = params.get('renameTo);

    if (nempty(renameTo) && !delete) {

      if (neqic(l.name, renameTo) && getList_noCreate(renameTo) != null)

        ret "Can't rename - list " + quote(renameTo) + " already exists";

      S oldName = l.name;

      cset(l, name := renameTo);

      updateListNamesList();

      ret nav() + "List " + quote(oldName) + " renamed to " + quote(renameTo);

    }

    if (delete) {

      logStructure("versions.log", litmap(id := l.id, status := "delete me", date := now()));

      deleteConcept(l);

      updateListNamesList();

      ret nav() + "List deleted";

    }

    // Update text

    S text = params.get("human");

    S status = params.get("status");

    if (text != null) {

      lock dbLock();

      //printStruct(ll(+text, +status));

      if (neqAny(text, l.text, status, unnull(l.status))) {

        logStructure("versions.log", litmap(id := l.id, +text, +status, date := now()));

        if (status != null) cset(l, +status);

        cset(l, +text, textMD5 := md5(text), lines := -1, botFlag := false, modified := now());

        listTextChanged(l);

        //printStruct("l.text=" + sfu(text));

      }

      ret hrefresh(0, rawLink(uri) + "?random=" + randomID());

    }

    S textArea = htextarea(l.text, name := 'human, cols := 80, rows := 30);

    L<S> tok = javaTok(l.status);

    jreplace(tok, "bb", "iframe 1018300");

    int idx = jfind(tok, "iframe <int>");

    if (idx > 0) {

      S analyzer = tok.get(idx+2);

      S subUri = joinSubList(tok, idx+4, smartIndexOf(tok, idx, ",")-1);

      textArea = htableRaw2_singleRow(ll(textArea, iframe(relativeRawBotLink(analyzer, subUri + "?list=" + urlencode(l.name)), width := "500", height := "500")),

        null, null, ll(valign := 'top));

    }

    // Serve list text (editable)

    ret nav() + htitle(l.name)

      + h2(ahref(neatMechListURL(l.name), htmlencode2(l.name)) + " [" + nLines(countLines(l.text)) + "]")

      + hpostform(

        "Status: " + htextinput('status, value := l.status)

      + h3("Contents (" + (l.botFlag ? "bot" : "human") + "-edited)")

      + p(hsubmit("Save"))

      + textArea

      + p(hsubmit("Save"))

      )

      + hpostform(

        "New name: " + htextinput('renameTo, value := l.name)

        + " "

        + hsubmit("Rename list") + " or " + hbutton("Delete list", name := "delete", type := "submit", value := 1, onClick := "return confirm('Really delete?')")

      );

  }

  // New list

  S newList = trim(params.get('newList));

  if (nempty(newList)) {

    AIList l = getList(newList);

    ret hrefresh(rawLink("/list/" + urlencode(l.name)));

  }

  // Overview

  L<AIList> list = alphabetical

    ? sortedByFieldIC('name, list(AIList))

    : sortedByFieldDesc('modified, list(AIList));

  fS prefix = session == null ? null : session.mech_filter;

  if (nempty(prefix))

    list = [AIList l : list | swic(l.name, prefix)];

  ret hmobilefix()

    + hcomment("cookie=" + cookie)

    + htitle_h3("mech.botcompany.de [" + n2(l(list), "list")

      + (totalChars >= 0 ? ", " + toK(totalChars) + "K chars" : "")

    + "]")

    + htableRaw_singleRow(ll(

      hform("Search: " + htextinput('q, style := "width: 200px") + " " + hsubmit("Search")),

      hpostform("| New list: " + htextinput('newList, style := "background-color: yellow") + " " + hsubmit("Create list"))))

    + hBoolSelector(alphabetical, "/", "By date", "/alphabetical", "Alphabetical")

    + listLists(list)

    + hpostform("Only show lists starting with: "

      + htextinput('mech_filter, value := prefix)

      + " " + hsubmit("OK"));

}

sS nav() {

  ret hmobilefix() + p(ahref(rawLink(), "<< back"));

}

static AIList getList_noCreate(S name) {

  ret getList(name, false);

}

static AIList getList(S name) {

  ret getList(name, true);

}

static AIList getList(S name, bool create) {

  lock dbLock();

  AIList l = findConcept(AIList, +name);

  if (l == null) l = nameIndex.get(name);

  if (l == null) if (!create) null; else {

    l = cnew(AIList, +name);

    logStructure("versions.log", litmap(id := l.id, +name, date := now(), mode := "List created"));

    updateListNamesList();

  }

  if (l.modified == 0) cset(l, modified := now());

  ret l;

}

sS listLists(L<AIList> list) {

  ret ul(map(list, func(AIList l) -> S {

    int n = lineCountOfList(l);

    new L<S> status;

    bool german = eq('german, mech_guessLanguage_quick(l.name));

    //if (l.mighty) status.add(german ? "GENUTZT" : "USED");

    addIfNempty(status, htmlencode(l.status));

    if (n != 0) status.add(n2(n, german ? "Zeile" : "line", german ? "Zeilen": "lines"));

    ret ahref(rawLink("/list/" + urlencode(l.name)),

        htmlencode(or2(l.name, "[no name]")) + appendBracketed(joinWithComma(status));

  }));

}

static L<AIList> scoredSearch_AIList(S query, Iterable<AIList> data) {

  new Map<AIList, Int> scores;

  L<S> prepared = scoredSearch_prepare(query);

  for (AIList l : data)

    putUnlessZero(scores, l,

        3*scoredSearch_score(l.name, prepared)

      + 2*scoredSearch_score(l.status, prepared)

      +   scoredSearch_score(l.text, prepared));

  ret keysSortedByValuesDesc(scores);

}

svoid calcTotalSize {

  long total = 0;

  for (AIList l)

    total += l(l.text);

  totalChars = total;

}

sS export_getListText(S listName) {

  AIList l = getList_noCreate(listName);

  ret l == null ? "" : l.text;

}

sS export_setListText(S listName, S text) {

  AIList l = getList(listName);

  if (eq(l.text, text)) ret "No change";

  logStructure("versions.log", litmap(id := l.id, +text, date := now(), mode := "bot edit"));

  cset(l, +text, textMD5 := md5(text), lines := -1, botFlag := true, modified := now());

  listTextChanged(l);

  ret "Changed";

}

sO serveListText(S uri, SS params, bool authed) {

  new Matches m;

  if (startsWith(uri, "/list-text/", m)) {

    bool opt = eq("1", params.get("opt"));

    bool withStatus = eq("1", params.get("withStatus"));

    bool create = authed && eq("1", params.get("create"));

    S md5 = params.get("md5");

    int md5Len = parseIntOpt(params.get('l));

    S name = urldecode(m.get(0));

    lock dbLock();

    AIList l = create ? getList(name) : getList_noCreate(name);

    if (!authed && l != null && !l.isPublicRead())

      l = null;

      //ret subBot_serve404("Not logged in");

    cset(l, mighty := true);

    if (l == null)

      if (opt)

        ret serveText(jsonEncode(litmap("Text" := "")));

      else

        ret serveText(jsonEncode(litmap("Error", ll("List not found", name))));

    else {

      Map map = litmap("Name" := l.name, "Status" := withStatus ? l.status : null);

      if (md5 != null) {

        S actualMD5 = md5OfList(l);

        print("md5: " + md5 + " / " + actualMD5);

        if (eq(actualMD5, md5))

          ret serveText(jsonEncode(mapPlus(map, "Same" := true)));

        else if (md5Len != 0 && eq(md5(takeFirst(l.text, md5Len)), md5))

          ret serveText(jsonEncode(mapPlus(map, "Appended" := true, "Text" := substring(l.text, md5Len))));

      }

      ret serveText(jsonEncode(mapPlus(map, "Text" := l.text)));

    }

  }

  null;

}

sS md5OfList(AIList l) {

  lock dbLock();

  if (l.textMD5 == null) l.textMD5 = md5(l.text);

  ret l.textMD5;

}

static int lineCountOfList(AIList l) {

  lock dbLock();

  if (l.lines < 0) l.lines = countLines(l.text);

  ret l.lines;

}

static L<AIList> publicReadLists() {

  ret [AIList l : list(AIList) | l.isPublicRead()];

}

svoid updateListNamesList {

  lock dbLock();

  AIList l = getList("All public-read mech lists");

  TreeSet<S> actual = new TreeSet(collect(publicReadLists(), 'name));

  Set<S> listed = asHashSet(lines(l.text));

  if (nempty(setMinusSet(listed, actual)))

    listReplaceWithLog(l, lines(actual));

  else

    listAppendWithLog(l, lines(setMinusSet(actual, listed)));

}

svoid listAppendWithLog(AIList l, S text) {

  if (emptyAfterTrim(text)) ret;

  logStructure("versions.log", litmap(id := l.id, +text, date := now(), mode := "bot append"));

  cset(l, text := appendNewLineIfNempty(rtrim(l.text)) + text, botFlag := true, modified := now(), textMD5 := null, lines := -1);

  listTextChanged(l);

}

svoid listReplaceWithLog(AIList l, S text) {

  if (eq(text, l.text)) ret;

  logStructure("versions.log", litmap(id := l.id, +text, date := now(), mode := "bot edit"));

  cset(l, +text, textMD5 := md5(text), lines := -1, botFlag := true, modified := now());

  listTextChanged(l);

}

static new ThreadLocal<Bool> listTextChanged_noRecurse;

svoid listTextChanged(AIList l) {

  if (!isTrue(listTextChanged_noRecurse!) && cic(l.status, "auto global ids")) pcall {

    L<S> entries = splitAtEmptyLines(l.text);

    S text = joinWithEmptyLines(map nlLogic_addGlobalID(entries));

    temp tempSetThreadLocal(listTextChanged_noRecurse, true);

    listReplaceWithLog(l, text);

  }

  if (!isTrue(listTextChanged_noRecurse!) && cic(l.status, "gazelle ids")) pcall {

    L<S> entries = splitAtEmptyLines(l.text);

    S text = joinWithEmptyLines(map gazelle_addGlobalID(entries));

    temp tempSetThreadLocal(listTextChanged_noRecurse, true);

    listReplaceWithLog(l, text);

  }

}

sS mechList_opt_raw_fresh(S name) {

  ret export_getListText(name);

}

static L<AIList> listsForAuth(bool authed) {

  ret authed ? list(AIList) : publicReadLists();

}