!7

// See #1023660 for the older 99 lines version

concept PhysicalSlice {

  new Ref<Page> slicePage; // page that describes this slice

}

concept Page {

  new Ref<PhysicalSlice> slice; // where are we stored

  S globalID = aGlobalID();

  S url, q;

}

concept Entry {

  S globalID = aGlobalID();

  new Ref<Page> page;

  int count;

  S key, value;

  S ip;

  new Ref<Signer> signer;

}

concept Signer {

  S globalID = aGlobalID();

  S publicKey;

  bool trusted;

  S approvedBy;

}

concept Session {

  S cookie;

  Page slicePage;

}

static SS searchTypeToText = litmap(

  leven := "Leven",

  literal := "Literal",

  scored := "Scored");

static int displayLength = 140;

static int sideDisplayLength = 50; // when in side bar

static int searchResultsToShow = 50;

sS sideSearchType = 'literal;

static int lines;

sbool asyncSearch = false;

sbool allowMultipleCasesInValues = true;

sbool showSliceSelector = false;

static ConceptFieldIndexDesc idx_latestEntries, idx_latestCreatedPages, idx_latestChangedPages;

p {

  dbIndexingCI(Page, 'url, Page, 'q, Entry, 'key, Entry, 'value);

  dbIndexing(Signer, 'publicKey);

  dbIndexing(Session, 'cookie);

  // legacy conversions!

  //for (Page p) cset(p, q := p.url);

  moveSlicelessPagesToTheWildSlice();

  idx_latestCreatedPages = new ConceptFieldIndexDesc(Page, 'created);

  idx_latestChangedPages = new ConceptFieldIndexDesc(Page, '_modified);

  idx_latestEntries = new ConceptFieldIndexDesc(Entry, 'created);

  // Approve this machine's key

  PKIKeyPair machineKey = agiBot_trustedKeyForMachine();

  if (machineKey != null) {

    print("Approving this machine's key: " + machineKey.publicKey);

    cset(uniq_sync(Signer, publicKey := machineKey.publicKey), trusted := true, approvedBy := "local");

  }

  lines = countLines(mySource());

}

// DB functions

sbool hasPage(S q) { ret hasConceptWhereIC(Page, +q); }

sS getValue(Page page, S key) {

  ret page == null || empty(key) ? null : getString value(highestByField count(objectsWhereIC(findBackRefs(page, Entry), +key)));

}

sS getValue(S page, S key) {

  ret getValue(findPageFromQ(page), key);

}

sS pageDisplayName(Page page) {

  /*S name = getValue(page, "read as");

  bool unnaturalName = nempty(name) && !eq(makeAGIDomain(name), page.url);

  ret unnaturalName ? name + " " + squareBracketed(page.url) : or2(name, unpackAGIDomainOpt(page.url));*/

  ret page.q;

}

// Serve page

set flag NoNanoHTTPD. html {

  ret new Request().serve(uri, params);

}

sclass Request {

  S cookie;

  Session session;

  S uri;

  SS params;

  // should also work for standalone

  bool isHomeDomain() {

    S domain = domain();

    ret eqic(domain, "www.agi.blue") || !ewic(domain, ".agi.blue");

  }

  O serve(S uri, SS params) {

    this.uri = uri;

    this.params = params;

    new Matches m;

    print(uri + " ? " + unnull(subBot_query()));

    cookie = cookieFromUser();

    if (cookie != null)

      session = uniq_sync(Session, +cookie);

    else

      session = unlistedWithValues(Session);

    // Check for special URIs

    if (swic(uri, "/bot/")) ret serveBot();

    // eleu appends a slash to the URI if it's a single identifier, so we drop it again

    S uri2 = dropTrailingSlash(uri);

    if (eqic(uri2, "/search")) ret serveScoredSearch();

    if (eqic(uri2, "/literalSearch")) ret serveLiteralSearch();

    if (eqic(uri2, "/levenSearch")) ret serveLevenSearch();

    if (eqic(uri2, "/query")) ret serveQueryPage();

    S selectSlice = params.get('slice);

    if (nempty(selectSlice))

      cset(session, slicePage := pageFromQ(selectSlice));

    S q = params.get('q);

    S domain = or2(params.get('domain), domain());

    S raw = firstKeyWithValue("", params); // agi.blue?something

    if (nempty(raw) && empty(q)) q = raw;

    /*if (nempty(q)) {

      domain = makeAGIDomain(q);

      if (l(domain) > maximumDomainPartLength()) // escape with "domain="

        ret hrefresh("http://agi.blue/" + hquery(+domain, key := "read as", value := q));

      ret hrefresh("http://" + domain + (eq(q, domain) ? "" : "/" +  hquery(key := "read as", value := q)));

      //uri = "/"; replaceMapWithParams(params, key := "read as", value := q);

    }*/

    S url = domain + dropTrailingSlash(uri);

    // domain to query

    //if (empty(q)) q = url;

    if (empty(q)) q = agiBlue_urlToQuery(url);

    Page page, bool newPage = unpair uniqCI2_sync(Page, +q);

    if (newPage) dbLog("New page", +q);

    //printStructs(+params, +raw, +q);

    S top = hcomment("cookie: " + takeFirst(4, session.cookie))

      + hSilentComputatorWithFlag("agi.blue: " + q)

      + p(ahref("http://agi.blue",

        //hsnippetimg(#1101682, width := 565/5, height := 800/5, title := "Robot by Peerro @ DeviantArt")

        hsnippetimg(#1101778, width := 96, height := 96, title := "agi.blue - a database for everything")

      ))

        + p(small(b(ahref("http://agi.blue", "agi.blue"))

        + " " /*+ htooltip("It's very new.", "alpha")*/ + " | " + targetBlank(progLink(), "source code") + " of this web site (" + nLines(lines) + ") | " + targetBlank("https://gitter.im/agi-blue/community", "sponsor https?") + " | by " + targetBlank("https://BotCompany.de", "BC") + " | " + targetBlank("http://fiverr.tinybrain.de/", "Fiverr") + " | " + targetBlank("https://discordapp.com/invite/SEAjPqk", "Discord") + " | " + targetBlank("https://www.youtube.com/watch?v=b6jtRdV3Ev8", "Video")

        + " | " + targetBlank("http://code.botcompany.de/1024233", "Notes")

        + " | " + ahref("/query", "Query")

      ));

    if (empty(params.get('q)) && empty(raw) && isHomeDomain()) {

      L<Page> pages = sortedByFieldDesc _modified(list(Page)); // TODO: use index

      int start = parseInt(params.get("start")), step = 100;

      S nav = pageNav2("/", l(pages), start, step, "start");

      ret hhtml2(hhead_title("agi.blue Overview") // SERVE HOME PAGE

        + hbody(hfullcenterAndTopLeft(top

          + hform(b("GIVE ME INPUT:") + "<br><br>" + htextinput('q, autofocus := true) + " " + hsubmit("Ask"))

          + h1("agi.blue has " + nPages(countConcepts(Page)) + " and " + nEntries(countConcepts(Entry)))

          + p(nav)

          + p_nemptyLines(map(pageToHTMLLink(), subList(pages, start, start+step))),

          !showSliceSelector ? "" : hform("Select reality slice: " + hselect_list(availableSlices(), getString q(session.slicePage), name := 'slice) + " " + hsubmit("Go"))

        )));

    }

    S key = trim(params.get('key)), value = trim(params.get('value));

    L<Entry> entries = GetEntriesAndPost().go(page, params).entries;

    Set<S> get = asCISet(nempties(subBot_paramsAsMultiMap().get('get)));

    //S get = params.get('get);

    if (nempty(get))

      ret serveJSON(collect value(llNotNulls(firstThat(entries, e -> get.contains(e.key)))));

    S key2 = key, value2 = value; if (nempty(key) && nempty(value)) key2 = value2 = ""; // input reset

    bool withHidden = eq(params.get('withHidden), "1");

    new Set<Int> hide;

    if (!withHidden) for (Entry e : entries) if (eqic(e.key, 'hide) && isSquareBracketedInt(e.value)) addAll(hide, e.count, parseInt(unSquareBracket(e.value)));

    new MultiMap<Int, S> mmMeta;

    for (Entry e : entries) if (isSquareBracketedInt(e.key)) mmMeta.put(parseInt(unSquareBracket(e.key)), e.value);

    new MultiMap<S> mm;

    for (Entry e : entries) mm.put(e.key, e.value);

    //S name = or2(/* ouch */ last(mm.get("read as")), /* end ouch */ unpackAGIDomain(page.url), page.url);

    S name = page.q;

    // Find references

    L<Entry> refs = concatLists(conceptsWhereIC Entry(value := name),

      conceptsWhereIC Entry(key := name));

    Set<Page> refPages = asSet(ccollect page(refs));

    refPages.remove(page); // don't list current page as reference

    // Search in page names (depending on default search type)

    L<Page> searchResults;

    if (eq(sideSearchType, 'leven))

      searchResults = levenSearch(page.q, max := 50);

    else if (eq(sideSearchType, 'literal))

      searchResults = literalSearch(page.q, max := 50);

    else

      searchResults = (L<Page>) dm_call('agiBlueSearch, 'search, page.q, maxResult := searchResultsToShow+1);

    searchResults.remove(page);

    S mainContents =

        top + h1(ahref_unstyled("http://" + url + hquery(+q), htmlEncode2(shorten(displayLength, name))) + (newPage ? " [huh????]" : ""))

      + p_nemptyLines(map(entries, func(Entry e) -> S {

        !withHidden && (hide.contains(e.count) || eqic(e.key, "read as") && eqic(e.value, name)) ? ""

        : "[" + e.count + "] " +

          renderThing(e.key, false) + ": " +

          b(renderThing(e.value, cic(mmMeta.get(e.count), "is a URL")))

        }))

      + hpostform(h3("Add an entry")

        + "Key: " + hinputfield(key := key2) + " Value: " + hinputfield(value := value2) + "<br><br>" + hsubmit("Add")

        )

      + p(ahref("http://agi.blue/literalSearch" + hquery(q := page.q), "[literal search]", title := "Search pages with a name containing this page's name literally")

       + " " +

       ahref("http://agi.blue/levenSearch" + hquery(q := page.q), "[leven search 1]", title := "Search pages with a Levenshtein similarity of 1 containing this page's name literally")

       + " " +

       ahref("http://agi.blue/search" + hquery(q := page.q), "[scored search]", title := "Search pages with ScoredSearch"));

    S sideContents =

      hform(b("GIVE ME INPUT:") + " "

        + htextinput('q) + " "

        + hsubmit("Ask", onclick := "document.getElementById('newInputForm').target = '';") + " "

        + hsubmit("+Tab", title := "Ask and show result in a new tab", onclick := "document.getElementById('newInputForm').target = '_blank';"),

        id := 'newInputForm)

      + h3("References (" + l(refPages) + ")")

      + p_nemptyLines_showFirst(10, map(pageToHTMLLink(displayLength := sideDisplayLength), refPages))

      + h3(searchTypeToText.get(sideSearchType) + " search results (" + (l(searchResults) >= searchResultsToShow ? searchResultsToShow + "+" : str(l(searchResults))) + ")")

      + p_nemptyLines_showFirst(searchResultsToShow, map(pageToHTMLLink(displayLength := sideDisplayLength), searchResults))

      + hdiv("", id := 'extraStuff);

    // TODO: sync search delivery with WebSocket creation

    if (asyncSearch)

      doLater(6.0, r { dm_call('agiBlueSearch, 'searchAndPost, page.q) });

    // serve a concept page

    ret hhtml2(hhead_title(pageDisplayName(page)) + hbody(

      tag('table, tr(

        td(mainContents, align := 'center, valign := 'top) +

        td(sideContents, align := 'right, valign := 'top)),

        width := "100%", height := "100%")));

  }

  O servePagesToBot(Iterable<Page> pages) {

    ret serveListToBot(map(pageToMap(wrapMapAsParams(params)), pages));

  }

  O serveListToBot(Collection l) {

    if (nempty(params.get('max)))

      l = takeFirst(parseInt(params.get('max)), l);

    ret serveJSON(l);

  }

  // uri starts with "/bot/"

  O serveBot() {

    S q = params.get('q);

    if (eqic(uri, "/bot/hello")) ret serveJSON("hello");

    if (eqic(uri, "/bot/hasPage")) ret serveJSON(hasPage(q));

    if (eqic(uri, "/bot/randomPageContaining")) {

      assertNempty(q);

      ret servePageToBot(random(filter(list(Page), p -> cic(p.q, q))), params);

    }

    if (eqic(uri, "/bot/allPages"))

      ret servePagesToBot(list(Page));

    if (eqic(uri, "/bot/allPagesStartingWith")) {

      assertNempty(q);

      ret servePagesToBot(filter(list(Page), p -> swic(p.q, q)));

    }

    if (eqic(uri, "/bot/allPagesEndingWith")) {

      assertNempty(q);

      ret servePagesToBot(filter(list(Page), p -> ewic(p.q, q)));

    }

    if (eqic(uri, "/bot/allPagesContaining")) {

      assertNempty(q);

      ret servePagesToBot(filter(list(Page), p -> cic(p.q, q)));

    }

    if (eqic(uri, "/bot/allPagesContainingRegexp")) {

      assertNempty(q);

      Pattern pat = regexpIC(q);

      ret servePagesToBot(filter(list(Page), p -> regexpFindIC(pat, p.q)));

    }

    if (eqicOneOf(uri, "/bot/postSigned", "/bot/makePhysicalSlice", "/bot/approveTrustRequest")) {

      S text = rtrim(params.get('text));

      S key = getSignerKey(text);

      if (empty(key)) ret subBot_serve500("Please include your public key");

      if (!isSignedWithKey(text, key)) ret subBot_serve500("Signature didn't verify");

      text = dropLastTwoLines(text); // drop signer + sig line

      Signer signer = uniq_sync Signer(publicKey := key);

      if (eqic(uri, "/bot/makePhysicalSlice")) {

        if (!signer.trusted) ret subBot_serve500("Untrusted signer");

        Page page = findPageFromParams(jsonDecodeMap(text));

        if (page == null) ret subBot_serve500("Page not found");

        ret serveJSON(uniq2_sync(PhysicalSlice, slicePage := page).b ? "Slice made" : "Slice exists");

      }

      if (eqic(uri, "/bot/postSigned")) {

        new L out;

        for (S line : tlft(text)) {

          SS map = jsonDecodeMap(line);

          new GetEntriesAndPost x;

          x.signer = signer;

          Page page = findOrMakePageFromParams(map);

          if (page == null) continue with out.add("Invalid page reference");

          x.go(page, map);

          out.add(x.newEntry ? "Saved" : x.entry != null ? "Entry exists" : "Need key and value");

        }

        ret serveJSON(out);

      }

      if (eqic(uri, "/bot/approveTrustRequest")) {

        if (!signer.trusted) ret subBot_serve500("Untrusted signer");

        Signer toApprove = conceptWhere Signer(publicKey := trim(text));

        if (toApprove == null) ret subBot_serve500("Signer to approve not found");

        cset(toApprove, trusted := true, approvedBy := signer.globalID);

        ret serveJSON("Approved: " + trim(text));

      }

      ret subBot_serve500("CONFUSION");

    }

    if (eqic(uri, "/bot/post")) {

      new GetEntriesAndPost x;

      x.go(pageFromQ(q), params);

      ret serveJSON(x.newEntry ? "Saved" : x.entry != null ? "Entry exists" : "Need key and value");

    }

    if (eqic(uri, "/bot/entriesOnPage"))

      ret serveJSON(map(entriesOnPage(findPageFromParams(params)), entryToMap(false)));

    if (eqic(uri, "/bot/lookup")) {

      S key = params.get('key);

      if (empty(key)) ret serveJSON("Need key");

      S value = getValue(findPageFromParams(params), key);

      ret serveJSON(empty(value) ? "" : litmap(+value));

    }

    if (eqic(uri, "/bot/latestEntries"))

      ret serveJSON(map(takeFirst(10, idx_latestEntries.objectIterator()), entryToMap(true)));

    if (eqic(uri, "/bot/latestPages"))

      ret serveJSON(map(takeFirst(10, idx_latestCreatedPages.objectIterator()), pageToMap()));

    if (eqic(uri, "/bot/latestChangedPages"))

      ret serveJSON(map(takeFirst(10, idx_latestChangedPages.objectIterator()), pageToMap()));

    if (eqic(uri, "/bot/totalPageCount"))

      ret serveJSON(countConcepts(Page));

    if (eqic(uri, "/bot/pageWithoutPhysicalSliceCount"))

      ret serveJSON(countConceptsWhere(Page, slice := null));

    if (eqic(uri, "/bot/physicalSliceCount"))

      ret serveJSON(countConcepts(PhysicalSlice));

    if (eqic(uri, "/bot/trustedSignersCount"))

      ret serveJSON(countConcepts(Signer, trusted := true));

    if (eqic(uri, "/bot/valueSearch")) {

      S value = params.get('value);

      L<Entry> entries = conceptsWhereIC Entry(+value);

      ret serveJSON(map(takeFirst(100, entries), entryToMap(true)));

    }

    if (eqic(uri, "/bot/keyAndValueSearch")) {

      S key = params.get('key), value = params.get('value);

      Cl<Page> pages = pagesForKeyAndValue(key, value);

      ret servePagesToBot(pages);

    }

    if (eqic(uri, "/bot/keyValuePairsByPopularity")) {

      L<PairS> pairs = map(list(Entry), e -> pair(e.key, e.value));

      LPair<S, Int> pairs2 = multiSetTopPairs(ciMultiSet(map pairToUglyStringForCIComparison(pairs)));

      ret serveJSON(map(pairs2, p -> {

        S key, value = unpair pairFromUglyString(p.a);

        ret litorderedmap(n := p.b, +key, +value);

      }));

    }

    if (eqic(uri, "/bot/allKeys"))

      ret serveListToBot(distinctCIFieldValuesOfConcepts Entry('key));

    if (eqic(uri, "/bot/allKeysByPopularity"))

      ret serveListToBot(mapMultiSetByPopularity(distinctCIFieldValuesOfConcepts_multiSet Entry('key), (key, n) -> litorderedmap(+n, +key)));

    if (eqic(uri, "/bot/dbSize"))

      ret serveJSON(l(conceptsFile()));

    if (eqic(uri, "/bot/query")) {

      S query = params.get('query);

      L<ALQLLine> lines = agiBlue_parseQueryScript(query);

      SS vars = ciMap();

      for (ALQLLine line : lines) {

        if (line cast ALQLReturn)

          ret serveJSON(ll(getOrKeep(vars, line.var)));

        else if (line cast ALQLTriple) {

          T3S t = line.triple;

          t = tripleMap(t, s -> getOrKeep(vars, s));

          Cl<Page> pages;

          S var;

          if (isDollarVar(t.c)) {

            var = t.c;

            if (isDollarVar(t.a)) {

              if (isDollarVar(t.b)) todo(t);

              Entry e = random(conceptsWhereCI Entry(key := t.b));

              if (e == null) ret serveJSON("No results for " + var);

              pages = pagesForKeyAndValue(t.b, t.c);

              vars.put(t.a, e.page->q);

              vars.put(t.c, e.value);

              continue;

            } else if (isDollarVar(t.b)) {

              Page page = findPageFromQ(t.a);

              Entry e = random(findBackRefs(page, Entry));

              if (e == null) ret serveJSON("No results for " + var);

              vars.put(t.b, e.key);

              vars.put(t.c, e.value);

              continue;

            } else {

              S val = getValue(t.a, t.b);

              if (val == null) ret serveJSON("No results for " + var);

              pages = ll(pageFromQ(val));

            }

          } else if (isDollarVar(t.b)) {

            var = t.b;

            if (isDollarVar(t.c)) todo(t);

            if (isDollarVar(t.a)) {

              L<Entry> entries = conceptsWhereCI Entry(value := t.c);

              if (empty(entries)) ret serveJSON("No results for " + var);

              Entry e = random(entries);

              vars.put(t.a, e.page->q);

              vars.put(t.b, e.key);

              continue;

            } else {

              Cl<S> keys = keysForPageAndValue(t.a, t.c);

              if (empty(keys)) ret serveJSON("No results for " + var);

              pages = map pageFromQ(keys);

            }

          } else {

            var = t.a;

            if (!isDollarVar(t.a)) todo(t);

            if (isDollarVar(t.b)) todo(t);

            if (isDollarVar(t.c)) todo(t);

            pages = pagesForKeyAndValue(t.b, t.c);

          }

          if (empty(pages)) ret serveJSON("No results for " + var);

          vars.put(var, random(pages).q);

        } else

          fail("Can't interpret: " + line);

      }

      ret serveJSON("No return statement");

    }

    // end of serveBot()

    ret subBot_serve404();

  }

  O serveLiteralSearch() {

    S q = params.get('q);

    L<Page> searchResults = literalSearch(q);

    ret serveSearchResults("literal search" , q, searchResultsToShow, searchResults);

  }

  L<Page> literalSearch(S q, O... _) {

    int searchResultsToShow = optPar max(_, 100);

    // quick search in random order

    //L<Page> searchResults = takeFirst(searchResultsToShow+1, filterIterator(iterator(list(Page)), p -> cic(p.q, q));

    // full search, order by length

    ret takeFirst(searchResultsToShow+1, pagesSortedByLength(filter(list(Page), p -> cic(p.q, q))));

  }

  O serveScoredSearch() {

    S q = params.get('q);

    L<Page> searchResults = (L<Page>) dm_call('agiBlueSearch, 'search, q);

    ret serveSearchResults("scored search" , q, searchResultsToShow, searchResults);

  }

  O serveLevenSearch() {

    S q = params.get('q);

    L<Page> searchResults = levenSearch(q);

    ret serveSearchResults("leven search with distance 1" , q, searchResultsToShow, searchResults);

  }

  L<Page> levenSearch(S q, O... _) {

    int searchResultsToShow = optPar max(_, 100);

    int maxEditDistance = 1;

    new Map<Page, Int> map;

    for (Page p) {

      int distance = leven_limitedIC(q, p.q, maxEditDistance+1);

      if (distance <= maxEditDistance) map.put(p, distance);

    }

    ret takeFirst(searchResultsToShow+1, keysSortedByValue(map));

  }

  O serveSearchResults(S searchType, S q, int searchResultsToShow, Collection<Page> searchResults) {

    S title = "agi.blue " + searchType + " for " + htmlEncode2(quote(q)) + " (" + (l(searchResults) >= searchResultsToShow ? searchResultsToShow + "+ results" : nResults(l(searchResults))) + ")";

    ret hhtml2(hhead_title(htmldecode_dropAllTags(title))

      + hbody(hfullcenter(//top +

        h3(title)

      + p_nemptyLines_showFirst(searchResultsToShow, map(pageToHTMLLink(), searchResults)))));

  }

  O serveQueryPage() {

    S query = params.get('query);

    if (query == null) query = loadSnippet(#1024258);

    S title = ahref("/", "agi.blue") + " | Execute a query script (" + targetBlank("http://code.botcompany.de/1024274", "ALQL") + ")";

    ret hhtml2(hhead_title(htmldecode_dropAllTags(title))

      + hbody(hfullcenter(

        h3(title)

      + form(

        htextarea(query, name := 'query, cols := 80, rows := 10, autofocus := true)

        + "<br><br>"

        + hsubmit("Execute"), action := "/bot/query")

      )));    

  }

} // end of class Request

svoid dbLog(O... params) {

  logStructure(programFile("db.log"), ll(params));

}

static IF1<Page, Map> pageToMap(O... _) {

  optPar bool withEntries;

  bool nameOnly = eqOneOf(optPar nameOnly(_), "1", true);

  if (nameOnly) ret (IF1<Page, Map>) p -> litmap(q := p.q);

  ret (IF1<Page, Map>) p -> {

    L<Entry> entries = findBackRefs(p, Entry);

    ret litorderedmap(

      q := p.q,

      nEntries := l(entries),

      created := p.created,

      modified := p._modified,

      entries := !withEntries ? null : map(entries, entryToMap(false)));

  };

}

static IF1<Entry, Map> entryToMap(bool withPage) {

  ret (IF1<Entry, Map>) e -> litorderedmap(

    created := e.created,

    i := e.count,

    key := e.key,

    value := e.value,

    q := withPage ? e.page->q : null,

    signer := getString globalID(e.signer!));

}

sO servePageToBot(Page page, SS params) {

  if (page == null) ret serveJSON(null);

  params = asCIMap(params);

  Map map = pageToMap(withEntries := valueIs1 withEntries(params)).get(page);

  ret serveJSON(map);

}

sclass GetEntriesAndPost {

  L<Entry> entries;

  Entry entry;

  bool newEntry;

  Signer signer;

  GetEntriesAndPost go(Page page, SS params) {

    S key = trim(params.get('key)), value = trim(params.get('value));

    print("GetEntriesAndPost: " + quote(page.q) + ", " + quote(key) + " := " + quote(value));

    withDBLock {

      entries = findBackRefs(page, Entry);

      if (nempty(key) && nempty(value)) {

        S ip = subBot_clientIP();

        entry = firstThat(e -> eqic(e.key, key) && eq_icIf(!allowMultipleCasesInValues, e.value, value) && eq(e.ip, ip), entries);

        if (entry == null) {

          print("SAVING");

          Entry e = cnew Entry(+page, +key, +value, +ip, count := l(entries) + 1, +signer);

          page.change(); // bump modification date

          entry = e;

          newEntry = true;

          entries.add(entry);

          dbLog("New entry", page := page.q, globalID := e.globalID, count := e.count, +key, +value);

        }

      }

    }

    sortByFieldInPlace created(entries);

    numberEntriesInConceptField count(entries);

    this;

  }

}

static Page findPageFromParams(Map map) {

  S q = getString q(map);

  ret empty(q) ? null : findPageFromQ(q);

}

static Page findOrMakePageFromParams(Map map) {

  ret pageFromQ(getString q(map));

}

static Page findPageFromQ(S q) {

  ret conceptWhereCI Page(+q);

}

static Page pageFromQ(S q) {

  ret empty(q) ? null : uniqCI_sync Page(+q);

}

static F1<Page, S> pageToHTMLLink(O... _) {

  optPar int displayLength = main.displayLength;

  ret func(Page p) -> S {

    S name = pageDisplayName(p);

    ret ahref("http://agi.blue" + hquery(q := p.q),

      htmlEncode2(shorten(displayLength, name)),

      title := name);

  };

}

static Collection<S> backSearch(S key, S value) {

  // we query the index for the value field because that yields fewer results

  ret map(conceptsWhereCI Entry(value := "a slice of reality", key := "is"), e -> e.page->q);

}

static Collection<S> availableSlices() {

  ret moveElementToBeginningIC("the default slice", backSearch("is", "a slice of reality"));

  //ret ll("the default slice", "the everything slice", "the robot slice");

}

static PhysicalSlice getOrMakePhysicalSlice(Page p) {

  PhysicalSlice slice = first(findBackRefs PhysicalSlice(p));

  if (slice == null)

    slice = uniq_sync(PhysicalSlice, slicePage := p);

  ret slice;

}

static PhysicalSlice theWildSlice() {

  ret getOrMakePhysicalSlice(pageFromQ("the wild slice"));

}

svoid moveSlicelessPagesToTheWildSlice() {

  PhysicalSlice slice = theWildSlice();

  for (Page p : conceptsWhere Page(slice := null))  {

    print("Moving to wild slice: " + p.q);

    cset_sync(p, +slice);

  }

}

sS renderThing(S s, bool forceURLDisplay) {

  ret forceURLDisplay || isURL(s) || isAGIDomain(s)

    ? ahref(fixAGILink(absoluteURL(s)), htmlencode2(shorten(displayLength, s)))

    : ahref(agiBlue_linkForPhrase(s), htmlencode2(shorten(displayLength, s)));

}

static L<Entry> entriesOnPage(Page p) {

  ret p == null ? null : sortedByField count(findBackRefs(p, Entry));

}

static L<Page> pagesSortedByLength(L<Page> l) {

  ret sortedByCalculatedField(l, p -> l(p.q));

}

sS hhtml2(S contents) {

  ret hhtml(hAddToHead_fast(contents, 

    hIncludeGoogleFont("Source Sans Pro")

    + hmobilefix()

    + hstylesheet("body { font-family: Source Sans Pro }")));

}

static Set<Page> pagesForKeyAndValue(S key, S value) {

  L<Entry> entries = conceptsWhereIC Entry(+value, +key);

  ret asSet(ccollect page(entries));

}

static Cl<S> keysForPageAndValue(S q, S value) {

  Page page = findPageFromQ(q);

  if (page == null) null;

  ret collect key(conceptsWhereIC Entry(+page, +value));

}