!7

// See #1023660 for the older 99 lines version

// Customizations:

sbool asyncSearch = false;
sbool allowMultipleCasesInValues = true;
sbool showSliceSelector = true;
sbool showGoogleLogIn = true;
sbool makeAllKeysPages = true;
sbool makeAllValuesPages = true; 
sbool legacy = true; // We still have slices without applied fixes
sbool dumpSlicesOnAnyChange = true; // testing

static int maxSliceNameLength = 1024;
static int displayLength = 140;
static int sideDisplayLength = 50; // when in side bar
static int searchResultsToShow = 50;
sS sideSearchType = 'literal;
static int maxBackgroundSlices = 1000;

// end of customizations

!include once #1024472 // agi.blue core db

// global constants

static SS searchTypeToText = litmap(
  leven := "Leven",
  literal := "Literal",
  scored := "Scored");
  
// global vars

static volatile bool started, broken;
static int sourceCodeLines;
static TimedCache<Long> sizeOnDisk = new(60.0, f guessClusteredSizeOfProgramDirWithoutBackups);

// end of global data

sclass LoadedSlice { // non-persistent class
  S caseID;
  Concepts cc;
  Slice sliceConcept;
  long lastAccess = sysNow();
  ReliableSingleThread rstDumpSlice = new(r {
    dumpSliceToFile(LoadedSlice.this);
    rst_index.trigger(); // rebuild index too - TODO: only this slice
  });
  bool haltSliceDumping;
  
  ConceptFieldIndexDesc idx_latestEntries, idx_latestCreatedPages, idx_latestChangedPages;
  
  *(S *caseID, Concepts *cc) {
    csetAll(list(cc, Page), slice := null, url := null); // clear legacy fields
    releaseEmptyFieldValuesOfAllConcepts(cc);
    indexThings();
    
    // forward changes to Slice concept
    onConceptsChange(cc, rOnceAtATimeOnly(r {
      cset(sliceConcept, _modified := now())
;
      if (!haltSliceDumping && dumpSlicesOnAnyChange && nempty(caseID)) {
        rstDumpSlice.trigger();
      }
    }));
    
    if (makeAllKeysPages && legacy)
      for (Entry e : list(cc, Entry))
        pageFromQ(e.key);
        
    if (makeAllValuesPages && legacy)
      for (Entry e : list(cc, Entry))
        pageFromQ(e.value);
  }
  
  void initialSetup(GlobalID globalID) {
    SliceInfo info = uniq(cc, SliceInfo);
    cset(info, +globalID);
  }
  
  // create if not there
  Page pageFromQ(S q) {
    ret main.pageFromQ(cc, q);
  }

  void indexThings() {
    indexConceptFieldsCI(cc, Page, 'q, Entry, 'key, Entry, 'value);
    indexConceptFields(cc, Signer, 'publicKey);
    indexConceptFields(cc, Session, 'cookie);
    indexSingletonConcept(cc, SliceInfo);
    idx_latestCreatedPages = new ConceptFieldIndexDesc(cc, Page, 'created);
    idx_latestChangedPages = new ConceptFieldIndexDesc(cc, Page, '_modified);
    idx_latestEntries = new ConceptFieldIndexDesc(cc, Entry, 'created);
  }
  
  bool isMainSlice() { ret empty(caseID); }
  GlobalID globalID() { ret sliceConcept.globalID; }
  S name() { ret sliceConcept.name; }
}

p {
  try {
    if (bootstrapDataFrom(#1023558)) deleteMyBackups();
    thinMyBackups();
    fan = slicesLoadedOnDemand(loadedSlices);
    mainConcepts = fan.get("", null);
    
    // index main DB

    indexConceptFields(Slice, 'globalID, Slice, 'caseID);
    indexConceptField(CookieUser, 'cookie);
    idx_slicesByName = new ConceptFieldIndexCI(Slice, 'name);
    idx_slicesByModification = new ConceptFieldIndexDesc(Slice, '_modified);
    idx_usersByName = new ConceptFieldIndexCI(GoogleUser, 'googleLastName);
    idx_usersBySeen = new ConceptFieldIndexDesc(User, 'lastSeen);

    cset(uniq_returnIfNew Slice(caseID := ""), name := "main slice");
    loadedSlices.get("").initialSetup(toGlobalIDObj(mainSliceGlobalID()));
    loadedSlices.get("").sliceConcept = sliceConceptForCaseID("");
    
    backgroundFan = slicesLoadedOnDemand(backgroundSlices);
    backgroundFan.lock = fan.lock;
    
    // legacy conversions!
    //deleteConcepts(Slice, globalID := GlobalID('wftlawbagrwprywn));
  
    // Approve this machine's key
    PKIKeyPair machineKey = agiBot_trustedKeyForMachine();
    if (machineKey != null) {
      print("Approving this machine's key: " + machineKey.publicKey);
      cset(uniq_sync(Signer, publicKey := machineKey.publicKey), trusted := true, approvedBy := "local");
    }
    
    sourceCodeLines = countLines(mySource());
    
    rst_index.trigger();
  } catch print e {
    set broken;
  } finally {
    set started;
  }
  
  // do stuff after we are officially started
  
  // calculate DB size regularly iff there were changes
  sizeOnDisk.keepValueWhileCalculating = true;
  doEveryAndNow(60.0, rWatcher(() -> db_mainConceptsChangeCount(), r { sizeOnDisk! }));
  
} // end of main program

// Serve page

set flag NoNanoHTTPD. html {
  sleepWhile(() -> !started);
  if (broken) ret subBot_serve500("Internal error");
  ret new Request().serve(uri, params);
}

sclass WorksOnSlice {
  Concepts cc;
  LoadedSlice slice;
  
  *() {}
  *(LoadedSlice *slice) { cc = slice.cc; }
  
  void setSlice(LoadedSlice slice) {
    this.slice = slice;
    cc = slice.cc;
  }
  
  Page findPageFromParams(Map map) {
    S q = getString q(map);
    ret empty(q) ? null : findPageFromQ(q);
  }

  Page findOrMakePageFromParams(Map map) {
    ret pageFromQ(getString q(map));
  }
  
  Page findPageFromQ(S q) {
    ret conceptWhereCI(cc, Page, +q);
  }
  
  Page pageFromQ(S q) {
    ret slice.pageFromQ(q);
  }
  
  Set<Page> pagesForKeyAndValue(S key, S value) {
    L<Entry> entries = conceptsWhereIC(cc, Entry, +value, +key);
    ret asSet(ccollect page(entries));
  }
  
  Cl<S> keysForPageAndValue(S q, S value) {
    Page page = findPageFromQ(q);
    if (page == null) null;
    ret collect key(conceptsWhereIC(cc, Entry, +page, +value));
  }

  // DB functions

  bool hasPage(S q) { ret hasConceptWhereIC(Page, +q); }
  S getValue(Page page, S key) {
    ret page == null || empty(key) ? null : getString value(highestByField count(objectsWhereIC(findBackRefs(page, Entry), +key)));
  }
  S getValue(S page, S key) {
    ret getValue(findPageFromQ(page), key);
  }
  S pageDisplayName(Page page) {
    /*S name = getValue(page, "read as");
    bool unnaturalName = nempty(name) && !eq(makeAGIDomain(name), page.url);
    ret unnaturalName ? name + " " + squareBracketed(page.url) : or2(name, unpackAGIDomainOpt(page.url));*/
    ret page.q;
  }
  
  S renderThing(S s, bool forceURLDisplay) {
    ret forceURLDisplay || isURL(s) || isAGIDomain(s)
      ? ahref(fixAGILink(absoluteURL(s)), htmlencode2(shorten(displayLength, s)))
      : ahref(agiBlue_linkForPhrase(s,
          slice := slice.isMainSlice() ? null : slice.sliceConcept.globalID),
        htmlencode2(shorten(displayLength, s)));
  }

  GlobalID sliceID() { ret slice.globalID(); }

} // end of WorksOnSlice

sclass Request extends WorksOnSlice {
  S cookie;
  Session session;
  S uri;
  SS params;
  long started = sysNow();
  S q, domain;
  Set<S> get;
  
  // when serving a concept page, info that may be grabbed
  // by thought bots
  L<Entry> usesAsKey;
  
  // should also work for standalone
  bool isHomeDomain() {
    S domain = domain();
    ret eqic(domain, "www.agi.blue") || !ewic(domain, ".agi.blue");
  }
  
  O serve(S uri, SS params) {
    this.uri = dropMultipleSlashes(uri);
    this.params = params;
    new Matches m;
    
    print(uri + " ? " + unnull(subBot_query()));
    
    // get cookie & session
    
    cookie = cookieFromUser();
    if (cookie != null)
 {
      session = uniq_sync(Session, +cookie);
      if (session.user! == null)
        cset(session, user := uniq_sync CookieUser(+cookie));
    } else
      session = unlistedWithValues(Session);
      
    // check if user wants to change slices
      
    S selectSlice = params.get('slice);
    if (isGlobalID(selectSlice))
      cset(session, selectedSlice := selectSlice);
    if (session.selectedSlice == null)
      cset(session, selectedSlice := str(mainSliceGlobalID()));

    // get slice's case ID
    
    S caseID = "";
    Slice sliceConcept = sliceConceptForGlobalID(session.selectedSlice);
    print("Selected slice: " + session.selectedSlice + ", obj? " + (sliceConcept != null));
    if (sliceConcept != null) caseID = sliceConcept.caseID;
    print("caseID: " + caseID);
    
    // load slice
    
    slice = assertNotNull(loadSlice(caseID, session.selectedSlice));
    cc = slice.cc;
    
    // Check for special URIs

    if (swic(uri, "/bot/")) ret serveBot();

    if (swic(uri, "/user/", m)) {
      User user = conceptWhere User(globalID := toGlobalIDObj(assertGlobalID(m.rest())));
      if (user == null) ret subBot_serve404("User not found");
      S title = "User " + user;
      ret hhtml_miniPage(title, h3(agiBlueNameHTML() + " | " + title)
        + "User type: " + classShortName(user));
    }

    // eleu appends a slash to the URI if it's a single identifier, so we drop it again
    S uri2 = dropTrailingSlash(uri);
    
    if (eqic(uri2, "/google-verify")) {
      print("Google-verify started.");
      Payload payload = printStruct(googleVerifyUserToken2(googleSignInID(), params.get("token")));
      if (payload == null) ret print("google-verify", "No");
      S email = payload.getEmail();
      if (empty(email)) ret print("google-verify", "No");
      
      GoogleUser user = uniqCI_sync GoogleUser(googleEmail := email);
      cset(user,
        googleEmailVerified := payload.getEmailVerified(),
        googleFirstName := strOrNull(payload.get("given_name")),
        googleLastName := strOrNull(payload.get("family_name")));
      
      cset(session,
 +user);
        
      ret print("google-verify", payload.getEmail() + " " + (payload.getEmailVerified() ? "(verified)" : "(not verified)"));
    }

    cset(session.user!, lastSeen := now());      
    
    if (eqic(uri2, "/users")) ret serveUsersList();
    
    if (eqic(uri2, "/slices")) ret serveSlicesList();
    
    if (eqic(uri2, "/search")) ret serveScoredSearch();
    if (eqic(uri2, "/literalSearch")) ret serveLiteralSearch();
    if (eqic(uri2, "/levenSearch")) ret serveLevenSearch();
    
    if (eqic(uri2, "/query")) ret serveQueryPage();
    if (eqic(uri2, "/createSlice")) ret serveCreateSlicePage();
    if (eqic(uri2, "/deletePage")) ret serveDeletePage();

    if (eqic(uri2, "/checkboxes")) ret serveCheckboxes();
    if (eqic(uri2, "/multiAdd")) ret serveMultiAdd();
    
    q = params.get('q);
    get = asCISet(nempties(subBot_paramsAsMultiMap().get('get)));
    
    domain = or2(params.get('domain), domain());
    
    S raw = firstKeyWithValue("", params); // agi.blue?something
    if (nempty(raw) && empty(q)) q = raw;
    /*if (nempty(q)) {
      domain = makeAGIDomain(q);
      if (l(domain) > maximumDomainPartLength()) // escape with "domain="
        ret hrefresh(agiBlueURL() + hquery(+domain, key := "read as", value := q));
      ret hrefresh("http://" + domain + (eq(q, domain) ? "" : "/" +  hquery(key := "read as", value := q)));
      //uri = "/"; replaceMapWithParams(params, key := "read as", value := q);
    }*/
    S url = domain + dropTrailingSlash(uri);
    
    // domain to query
    //if (empty(q)) q = url;
    if (empty(q)) {
      S qq = agiBlue_urlToQuery(url);
      if (neqic(qq, "agi.blue")) q = qq;
    }
    
    Page page, bool newPage = unpair uniqCI2_sync(cc, Page, +q);
    if (newPage) dbLog("New page", +q);
    //printStructs(+params, +raw, +q);
  
    if (empty(params.get('q)) && empty(raw) && isHomeDomain()) {
      //L<Page> pages = sortedByFieldDesc _modified(list(cc, Page)); // TODO: use index
      L<Page> pages = cloneList(slice.idx_latestChangedPages.objectIterator());
      int start = parseInt(params.get("start")), step = 100;
      S nav = pageNav2("/", l(pages), start, step, "start");
      S content = hform(b("GIVE ME INPUT: &nbsp; ") + htextinput('q, autofocus := true) + " " + hsubmit("Ask"))
        + h1(sliceAsHTML() + " has " + nPages(countConcepts(cc, Page)) + " and " + nConnections(countConcepts(cc, Entry)))
        + p(nav)
        + p_nemptyLines(map(pageToHTMLLink(), subList(pages, start, start+step)));

      ret hhtml_agiBlue(hhead_title("Slice " + sliceConcept().name + " | agi.blue") // SERVE SLICE HOME PAGE
        + hbody(hfullcenterAndTopLeft(top() + content
          + footer(),
          sliceSelector()
        )));
    }
        
    S key = trim(params.get('key)), value = trim(params.get('value));
    L<Entry> entries = GetEntriesAndPost(cc).go(page, params).entries;
    
    //S get = params.get('get);
    if (nempty(get))
      ret serveJSON(collect value(llNotNulls(firstThat(entries, e -> get.contains(e.key)))));
    
    S key2 = key, value2 = value; if (nempty(key) && nempty(value)) key2 = value2 = ""; // input reset
  
    bool withHidden = eq(params.get('withHidden), "1");
    new Set<Int> hide;
    if (!withHidden) for (Entry e : entries) if (eqic(e.key, 'hide) && isSquareBracketedInt(e.value)) addAll(hide, e.count, parseInt(unSquareBracket(e.value)));
    new MultiMap<Int, S> mmMeta;
    for (Entry e : entries) if (isSquareBracketedInt(e.key)) mmMeta.put(parseInt(unSquareBracket(e.key)), e.value);
    new MultiMap<S> mm;
    for (Entry e : entries) mm.put(e.key, e.value);
    
    //S name = or2(/* ouch */ last(mm.get("read as")), /* end ouch */ unpackAGIDomain(page.url), page.url);
    S name = page.q;
    
    // Find references
    
    L<Entry> refs = concatLists(conceptsWhereIC(cc, Entry, value := name),
      conceptsWhereIC(cc, Entry, key := name));
    Set<Page> refPages = asSet(ccollect page(refs));
    refPages.remove(page); // don't list current page as reference
    
    // Search in page names (depending on default search type)
    
    L<Page> searchResults;
    int totalResults = 0, searchResultsToShow = 50;
    if (eq(sideSearchType, 'leven))
      searchResults = levenSearch(page.q, max := searchResultsToShow);
    else if (eq(sideSearchType, 'literal))
 {
      searchResults = literalSearch(page.q, max := maxInt());
      print("totalResults", totalResults = l(searchResults));
      searchResults = takeFirst(searchResultsToShow, searchResults);
    } else
      searchResults = (L<Page>) dm_call('agiBlueSearch, 'search, page.q, maxResult := clipIntPlus(searchResultsToShow, 1), agiBlueBotID := programID(), +cc);
    searchResults.remove(page);
    
    // Find uses as key
    usesAsKey = conceptsWhereIC(cc, Entry, key := page.q);

    // Find page in other slices
    CentralIndexEntry cie = centralIndex.get(page.q);
    L<Slice> pageInOtherSlices = cie == null ? null : listMinus(cie.slices, slice.sliceConcept);
    
    S pageName_html = htmlEncode2(shorten(displayLength, name));

    S mainContents =
        top() + h1(ahref_unstyled(agiBlue_pageURLWithSlice(page), pageName_html) + (newPage ? " [huh????]" : ""))
      + p_nemptyLines(map(entries, func(Entry e) -> S {
        !withHidden && (hide.contains(e.count) || eqic(e.key, "read as") && eqic(e.value, name)) ? ""
        : "[" + e.count + "] " +
          renderThing(e.key, false) + ": " +
          b(renderThing(e.value, cic(mmMeta.get(e.count), "is a URL")))
        }))
      + hpostform(h3("Add an entry")
        + "Key: " + hinputfield(key := key2) + " Value: " + hinputfield(value := value2) + "<br><br>" + hsubmit("Add")
        )
        
      + p(ahref(agiBlueURL() + "/literalSearch" + hquery(q := page.q), "[literal search]", title := "Search pages with a name containing this page's name literally")
         + " " +
         ahref(agiBlueURL() + "/levenSearch" + hquery(q := page.q), "[leven search 1]", title := "Search pages with a Levenshtein similarity of 1 containing this page's name literally")
         + " " +
         ahref(agiBlueURL() + "/search" + hquery(q := page.q), "[scored search]", title := "Search pages with ScoredSearch")
         + " " +
         ahref(agiBlueURL() + "/deletePage" + hquery(slice := sliceID(), q := page.q), "[delete page]")
       )
       
       // optional "uses as key" section
       
       + (empty(usesAsKey) ? "" : h3(quote(pageName_html) + " as key")
       + p_nemptyLines_showFirst(50, map(usesAsKey, e ->
         pageToHTMLLink().get(e.page!) + unicode_spacedRightPointingTriangle() +
           pageName_html + unicode_spacedRightPointingTriangle() +
         pageToHTMLLink().get(pageFromQ(e.value))
       )))
       
       // optional "in other slices" section
       
       + (empty(pageInOtherSlices) ? "" : h3(quote(pageName_html) + " in other slices")
       + p_nemptyLines_showFirst(50, map(slice ->
         ahref(agiBlueURL() + hquery(slice := slice.globalID, q := page.q), htmlEncode2(slice.name)), pageInOtherSlices)))
       ;
       
      // end of mainContents
        
    S sideContents =
      hform(b("GIVE ME INPUT:") + " "
        + htextinput('q) + " "
        + hsubmit("Ask", onclick := "document.getElementById('newInputForm').target = '';") + " "
        + hsubmit("+Tab", title := "Ask and show result in a new tab", onclick := "document.getElementById('newInputForm').target = '_blank';"),
        id := 'newInputForm)
      
      + h3("References (" + l(refPages) + ")")
      
      + p_nemptyLines_showFirst(10, map(pageToHTMLLink(displayLength := sideDisplayLength), refPages))
      
      + h3(searchTypeToText.get(sideSearchType) + " search results ("
        + (l(searchResults) >= searchResultsToShow ? searchResultsToShow + "+" : str(l(searchResults)))
        + (totalResults > l(searchResults) ? " of " + n2(totalResults) : "") + ")")
      
      + p_nemptyLines_showFirst(searchResultsToShow, map(pageToHTMLLink(displayLength := sideDisplayLength), searchResults))
      
      + hdiv("", id := 'extraStuff);
      
    // TODO: sync search delivery with WebSocket creation
    if (asyncSearch)
      doLater(6.0, r { dm_call('agiBlueSearch, 'searchAndPost, page.q, agiBlueBotID := programID(), +cc) });
      
    // notify local interested parties
    vmBus_send('agiBlue_servingConceptPage, this, page);
    
    S iframe = params.get('render_iframe);
      
    // serve a concept page
    ret hhtml_agiBlue(hhead_title(pageDisplayName(page)) + hbody(
      tag('table,
        tr(
          td(sliceSelector(), valign := 'top)
        + td(sideContents, align := 'right, valign := 'top, rowspan := 2))
      + tr(td(mainContents +
        (empty(iframe) ? "" : iframe(iframe, width := 600, height := 400)) +
      footer(), align := 'center, valign := 'top)),
      width := "100%", height := "100%")));
  }
  
  O servePagesToBot(Iterable<Page> pages) {
    ret serveListToBot(map(pageToMap(wrapMapAsParams(params)), pages));
  }

  O serveListToBot(Collection l) {
    if (nempty(params.get('max)))
      l = takeFirst(parseInt(params.get('max)), l);
    ret serveJSON(l);
  }

  // uri starts with "/bot/"
  O serveBot() {
    S q = params.get('q);
    
    if (eqic(uri, "/bot/hello")) ret serveJSON("hello");
    if (eqic(uri, "/bot/hasPage")) ret serveJSON(hasPage(q));
    if (eqic(uri, "/bot/randomPageContaining")) {
      assertNempty(q);
      ret servePageToBot(random(filter(list(cc, Page), p -> cic(p.q, q))), params);
    }
    if (eqic(uri, "/bot/allPages"))
      ret servePagesToBot(list(cc, Page));
    if (eqic(uri, "/bot/allPagesStartingWith")) {
      assertNempty(q);
      ret servePagesToBot(filter(list(cc, Page), p -> swic(p.q, q)));
    }
    if (eqic(uri, "/bot/allPagesEndingWith")) {
      assertNempty(q);
      ret servePagesToBot(filter(list(cc, Page), p -> ewic(p.q, q)));
    }
    if (eqic(uri, "/bot/allPagesContaining")) {
      assertNempty(q);
      ret servePagesToBot(filter(list(cc, Page), p -> cic(p.q, q)));
    }
    if (eqic(uri, "/bot/allPagesContainingRegexp")) {
      assertNempty(q);
      Pattern pat = regexpIC(q);
      ret servePagesToBot(filter(list(cc, Page), p -> regexpFindIC(pat, p.q)));
    }
    
    if (eqicOneOf(uri, "/bot/postSigned", /*"/bot/makePhysicalSlice",*/ "/bot/approveTrustRequest")) {
      S text = rtrim(params.get('text));
      S key = getSignerKey(text);
      if (empty(key)) ret subBot_serve500("Please include your public key");
      if (!isSignedWithKey(text, key)) ret subBot_serve500("Signature didn't verify");
      text = dropLastTwoLines(text); // drop signer + sig line
      
      Signer signer = uniq_sync Signer(publicKey := key);
      
      /*if (eqic(uri, "/bot/makePhysicalSlice")) {
        if (!signer.trusted) ret subBot_serve500("Untrusted signer");
        Page page = findPageFromParams(jsonDecodeMap(text));
        if (page == null) ret subBot_serve500("Page not found");
        ret serveJSON(uniq2_sync(PhysicalSlice, slicePage := page).b ? "Slice made" : "Slice exists");
      }*/
    
      if (eqic(uri, "/bot/postSigned")) {
        new L out;
        for (S line : tlft(text)) {
          SS map = jsonDecodeMap(line);
          GetEntriesAndPost x = new(db_mainConcepts());
          x.signer = signer;
          Page page = findOrMakePageFromParams(map);
          if (page == null) continue with out.add("Invalid page reference");
          x.go(page, map);
          out.add(x.newEntry ? "Saved" : x.entry != null ? "Entry exists" : "Need key and value");
        }
        ret serveJSON(out);
      }
      
      if (eqic(uri, "/bot/approveTrustRequest")) {
        if (!signer.trusted) ret subBot_serve500("Untrusted signer");
        Signer toApprove = conceptWhere Signer(publicKey := trim(text));
        if (toApprove == null) ret subBot_serve500("Signer to approve not found");
        cset(toApprove, trusted := true, approvedBy := signer.globalID);
        ret serveJSON("Approved: " + trim(text));
      }
      
      ret subBot_serve500("CONFUSION");
    }
    
    if (eqic(uri, "/bot/post")) {
      GetEntriesAndPost x = new(db_mainConcepts());
      x.go(pageFromQ(q), params);
      ret serveJSON(x.newEntry ? "Saved" : x.entry != null ? "Entry exists" : "Need key and value");
    }
    
    if (eqic(uri, "/bot/entriesOnPage"))
      ret serveJSON(map(entriesOnPage(findPageFromParams(params)), entryToMap(false)));
      
    if (eqic(uri, "/bot/entriesForKey"))
      ret serveJSON(map(conceptsWhereIC(cc, Entry, key := params.get('key)), entryToMap(true)));
     
    // look up value from page & key in current slice
    if (eqic(uri, "/bot/lookup")) {
      S key = params.get('key);
      if (empty(key)) ret serveJSON("Need key");
      S value = getValue(findPageFromParams(params), key);
      ret serveJSON(empty(value) ? "" : litmap(+value));
    }

    if (eqic(uri, "/bot/multiLookup")) {
      S key = params.get('key);
      if (empty(key)) ret serveJSON("Need key");
      ret serveJSON(collect value(objectsWhereIC(findBackRefs(findPageFromParams(params), Entry), +key)));
    }
  
    // look up value from page & key in all slices
    if (eqic(uri, "/bot/multiLookupInAllSlices")) {
      S key = params.get('key);
      if (empty(key)) ret serveJSON("Need key");
      Cl<Slice> slices = centralIndexGetSlices(key);
      new L<Map> results;
      fOr (Slice s : slices) {
        WorksOnSlice wos = new(loadBackgroundSlice(s));
        Page page = wos.findPageFromParams(params);
        if (page != null)
          fOr (Entry e : objectsWhereIC(findBackRefs(page, Entry), +key))
            results.add(litorderedmap("c" := e.value, "slice" := str(s.globalID())));
      }
      ret serveJSON(results);
    }
    
    if (eqic(uri, "/bot/latestEntries"))
      ret serveJSON(map(takeFirst(10, slice.idx_latestEntries.objectIterator()), entryToMap(true)));
    if (eqic(uri, "/bot/latestPages"))
      ret serveJSON(map(takeFirst(10, slice.idx_latestCreatedPages.objectIterator()), pageToMap()));
    if (eqic(uri, "/bot/latestChangedPages"))
      ret serveJSON(map(takeFirst(10, slice.idx_latestChangedPages.objectIterator()), pageToMap()));
      
    if (eqic(uri, "/bot/googleUsersCount"))
      ret serveJSON(countConcepts(GoogleUser));
      
    if (eqic(uri, "/bot/totalPageCount"))
      ret serveJSON(countConcepts(Page));
    /*if (eqic(uri, "/bot/pageWithoutPhysicalSliceCount"))
      ret serveJSON(countConceptsWhere(Page, slice := null));
    if (eqic(uri, "/bot/physicalSliceCount"))
      ret serveJSON(countConcepts(PhysicalSlice));*/
    if (eqic(uri, "/bot/trustedSignersCount"))
      ret serveJSON(countConcepts(Signer, trusted := true));
      
    if (eqic(uri, "/bot/valueSearch")) {
      S value = params.get('value);
      L<Entry> entries = conceptsWhereIC(cc, Entry, +value);
      ret serveJSON(map(takeFirst(100, entries), entryToMap(true)));
    }
      
    if (eqic(uri, "/bot/keyAndValueSearch")) {
      S key = params.get('key), value = params.get('value);
      Cl<Page> pages = pagesForKeyAndValue(key, value);
      ret servePagesToBot(pages);
    }
    
    if (eqic(uri, "/bot/keyValuePairsByPopularity")) {
      L<PairS> pairs = map(list(Entry), e -> pair(e.key, e.value));
      LPair<S, Int> pairs2 = multiSetTopPairs(ciMultiSet(map pairToUglyStringForCIComparison(pairs)));
      ret serveJSON(map(pairs2, p -> {
        S key, value = unpair pairFromUglyString(p.a);
        ret litorderedmap(n := p.b, +key, +value);
      }));
    }
    
    if (eqic(uri, "/bot/allKeys"))
      ret serveListToBot(distinctCIFieldValuesOfConcepts(cc, Entry, 'key));
      
    if (eqic(uri, "/bot/allKeysByPopularity"))
      ret serveListToBot(mapMultiSetByPopularity(distinctCIFieldValuesOfConcepts_multiSet(cc, Entry, 'key), (key, n) -> litorderedmap(+n, +key)));
      
    if (eqic(uri, "/bot/dbSize"))
      ret serveJSON(l(conceptsFile()));
      
    if (eqic(uri, "/bot/query"))
      ret serveBotQuery();

    if (eqic(uri, "/bot/createSlice")) {
      Slice slice = createSlice(assertNempty(params.get('name)));
      ret serveJSON(litorderedmap(id := str(slice.globalID), name := slice.name));
    }
    
    if (eqic(uri, "/bot/dumpAllSlices") && authed()) {
      new L<Map> out;
      for (Slice slice) try {
        out.add(litorderedmap(slice := slice.caseID, ms := returnTimeFor(() -> dumpSliceToFile(loadBackgroundSlice(slice)))));
      } catch print e {
        out.add(litorderedmap(slice.caseID, error := exceptionToStringShort(e)));
      }
      ret serveJSON(out);
    }
    
    if (eqic(uri, "/bot/dumpSlice")) {
      File f = dumpSliceToFile(slice);
      ret "OK, saved as: " + f2s(f);
    }
    
    if (eqic(uri, "/bot/diskStats"))
      ret serveJSON(litcimap(
        sizeOnDisk := sizeOnDisk!
      ));

    if (eqic(uri, "/bot/memStats")) {
      Map fg = cloneMap(loadedSlices);
      Map bg = cloneMap(backgroundSlices);
      ret serveJSON(litcimap(
        error := checkDoubleLoads(),
        centralIndexEntries := l(centralIndex),
        centralIndexMade := renderHowLongAgo(centralIndexMade),
        centralIndexMadeInMS := centralIndexMadeIn,
        numLoadedForegroundSlices := l(fg),
        numLoadedBackgroundSlices := l(bg),
        loadedForegroundSlicesEstimatedSize := longSum(map estimatedSliceDataSize(keys(fg))),
        loadedBackgroundSlicesEstimatedSize := longSum(map estimatedSliceDataSize(keys(bg))),
        loadedForegroundSlices := sortedIC(keysList(fg)),
        loadedBackgroundSlices := sortedIC(keysList(bg)),
        sessionsWithGoogleUsers := countPred(list(Session), s -> s.user! instanceof GoogleUser),
        userObjects := countConcepts(User),
      ));
    }
    
    if (eqic(uri, "/bot/centralIndexGet")) {
      CentralIndexEntry e = centralIndex.get(q);
      ret serveJSON(e == null ? ll() : map(e.slices, sliceToMap()));
    }
    
    // return primary triple from each slice
    if (eqic(uri, "/bot/centralIndexGrab")) {
      CentralIndexEntry ie = centralIndex.get(q);
      new L<Map> out;
      if (ie != null) for (Slice slice : ie.slices) {
        LoadedSlice ls = loadBackgroundSlice(slice);
        WorksOnSlice wos = new(ls);
        Page page = wos.findPageFromQ(q);
        Iterable<Entry> entries = entriesOnPage(page);
        print("Page for " + q + " in slice " + ls.caseID + ": " + yesNo(page != null) + " - " + l(entries));
        Entry e = first(entries);
        if (e != null)
          out.add(litorderedmap(a := page.q, b := e.key, c := e.value, slice := slice.caseID));
      }
      ret serveJSON(out);
    }
    
    if (eqic(uri, "/bot/updateCentralIndex") && authed()) {
      rst_index.trigger();
      ret "OK";
    }

    if (eqic(uri, "/bot/allGoogleEmails") && authed())
      ret serveJSON(collect googleEmail(list(GoogleUser)));

    if (eqic(uri, "/bot/words2_spaces_all"))
      ret serveJSON(mapToValues_ciMap words2_spaces_cached(collect q(conceptsSortedByFieldCI(slice.cc, Page, 'q))));
    
    if (eqic(uri, "/bot/words2_spaces_collapse_all"))
      ret serveJSON(mapToValues_ciMap words2_spaces_collapse_cached(collect q(conceptsSortedByFieldCI(slice.cc, Page, 'q))));

    if (eqic(uri, "/bot/makeManyPages")) {
      //LS qs = nempties(subBot_paramsAsMultiMap().get('q));
      LS qs = tlft(params.get("pages"));
      slice.haltSliceDumping = true;
      try {
        fOr ping (S _q : qs)
          pageFromQ(_q);
      } finally {
        slice.haltSliceDumping = true;
        slice.rstDumpSlice.trigger();
      }
      ret "OK (" + l(qs) + ")";
    }

    // end of bot methods
  
    ret subBot_serve404();
  }
  
  O serveBotQuery() {
    S query = params.get('query);
    ret new Query(slice).process(query);
  }
  
  O serveLiteralSearch() {
    S q = params.get('q);
    L<Page> searchResults = literalSearch(q);
    ret serveSearchResults("literal search" , q, searchResultsToShow, searchResults);
  }
  
  L<Page> literalSearch(S q, O... _) {
    int searchResultsToShow = optPar max(_, 100);
    
    // quick search in random order
    //L<Page> searchResults = takeFirst(clipIntPlus(searchResultsToShow, 1), filterIterator(iterator(list(cc, Page)), p -> cic(p.q, q));
    
    // full search, order by length
    ret takeFirst(clipIntPlus(searchResultsToShow, 1), pagesSortedByLength(filter(list(cc, Page), p -> cic(p.q, q))));
  }
  
  O serveScoredSearch() {
    S q = params.get('q);
    L<Page> searchResults = (L<Page>) dm_call('agiBlueSearch, 'search, q, +cc);
    ret serveSearchResults("scored search" , q, searchResultsToShow, searchResults);
  }
  
  O serveLevenSearch() {
    S q = params.get('q);
    L<Page> searchResults = levenSearch(q);
    ret serveSearchResults("leven search with distance 1" , q, searchResultsToShow, searchResults);
  }
  
  L<Page> levenSearch(S q, O... _) {
    int searchResultsToShow = optPar max(_, 100);
    int maxEditDistance = 1;
    
    new Map<Page, Int> map;
    for (Page p : list(cc, Page)) {
      int distance = leven_limitedIC(q, p.q, maxEditDistance+1);
      if (distance <= maxEditDistance) map.put(p, distance);
    }
    ret takeFirst(clipIntPlus(searchResultsToShow, 1), keysSortedByValue(map));
  }
  
  O serveSearchResults(S searchType, S q, int searchResultsToShow, Collection<Page> searchResults) {
    S title = "agi.blue " + searchType + " for " + htmlEncode2(quote(q)) + " (" + (l(searchResults) >= searchResultsToShow ? searchResultsToShow + "+ results" : nResults(l(searchResults))) + ")";
    
    ret hhtml_agiBlue(hhead_title(htmldecode_dropAllTags(title))
      + hbody(hfullcenter(//top() +
        h3(title)
      + p_nemptyLines_showFirst(searchResultsToShow, map(pageToHTMLLink(), searchResults)))));
  }
  
  O serveQueryPage() {
    S query = params.get('query);
    if (query == null) query = loadSnippet(#1024258);
    S title = agiBlueNameHTML() + " | Execute a query script (" + targetBlank("http://code.botcompany.de/1024274", "ALQL") + ")";
    ret hhtml_miniPage(title, h3(title)
 + form(
      htextarea(query, name := 'query, cols := 80, rows := 10, autofocus := true)
      + "<br><br>"
      + hsubmit("Execute"), action := "/bot/query"
    ));
  }

  O hhtml_miniPage(S htmlTitle, O contents) {
    ret hhtml_agiBlue(hhead_title(htmldecode_dropAllTags(htmlTitle))
      + hbody(hfullcenter(contents)));
  }
  
  S sliceHomeURL() {
    ret slice == null ? agiBlueURL() : sliceHomeURL(slice.sliceConcept.globalID);
  }
  
  S sliceHomeURL(Slice slice) { ret sliceHomeURL(slice.globalID); }

  S sliceHomeURL(GlobalID slice) {
    ret agiBlueURL() + hquery(+slice);
  }

  S sliceLinkHTML(Slice slice) {
    ret slice == null ? "-": ahref(sliceHomeURL(slice), htmlEncode2(slice.name));
  }
  
  O serveCreateSlicePage() {
    S sliceName = trim(params.get('sliceName));
    if (eq(params.get('doIt), "1") && nempty(sliceName)) {
      // TODO: check for existing name
      Slice slice = createSlice(sliceName);
      ret hrefresh(sliceHomeURL(slice.globalID));
    }
    
    S title = agiBlueNameHTML() + " | Create slice";
    ret hhtml_agiBlue(hhead_title(htmldecode_dropAllTags(title))
      + hbody(hfullcenter(
        h3(title)
      + form(
          hhidden(doIt := 1)
        + "Slice name: " + htextinput(+sliceName, autofocus := true)
        + "<br><br>"
        + hsubmit("Create slice"))
      )));    
  }

  O serveDeletePage() {
    q = params.get('q);
    Page page = findPageFromQ(q);
    if (page == null) ret "Page " + quote(q) + " not found in slice " + htmlEncode(slice.name());
    bool ownage = slice.sliceConcept.owner! == session.user!;
    if (!ownage) ret "Sorry, you don't own slice " + htmlEncode(slice.name());
    deleteConcepts(findBackRefs(page, AbstractEntry));
    cdelete(page);
    ret hrefresh(1.0, sliceHomeURL()) + "Page deleted";
  }
  
  F1<Page, S> pageToHTMLLink(O... _) {
    optPar int displayLength = main.displayLength;
    ret func(Page p) -> S {
      S name = pageDisplayName(p);
      ret ahref(agiBlue_pageURLWithSlice(p),
        htmlEncode2(shorten(displayLength, name)),
        title := name);
    };
  }
  
  // and google log-in
  S sliceSelector() {
    ret htag table(tr(
      (!showGoogleLogIn ? "" : td(
        googleSignIn_signInButton(agiBlueURL() + "/google-verify", "console.log(data);", ""), style := "padding-right: 10px;")
        + td(nobr(ahref("javascript:signOut()", "Sign out")), style := "padding-right: 10px; padding-bottom: 0.15em"))
    
    + (!showSliceSelector ? "" : hform(td(
        "Select reality slice: "
      + hselect(availableSlices(), session.selectedSlice, name := 'slice, onchange := "this.form.submit()")
      /*+ " " + hsubmit("Go")*/
      + " &nbsp; | " +
      ahref(agiBlueURL("/slices"), nSlices(countConcepts(Slice))) + " | " +
      ahref(agiBlueURL() + "/createSlice", "Create slice...")
    )))));
  }
  
  S sliceAsHTML() {
    if (slice.isMainSlice()) ret htmlEncode2(agiBlueName() + "'s main slice");
    if (slice.sliceConcept == null) ret "Slice ???";
    ret htmlEncode2("Slice " + quote(slice.sliceConcept.name));
  }
  
  Slice sliceConcept() { ret slice.sliceConcept; }
  
  S footer() {
    ret p(small(elapsedMS_sysNow(started) + " ms"));
  }
  
  S verifiedEmail() {
    if (session == null || !(session.user! instanceof GoogleUser)) null;
    ret ((GoogleUser) session.user!).googleEmail;
  }
  
  bool authed() {
    bool ver = eqic(verifiedEmail(), "stefan.reich.maker.of.eye@googlemail.com");
    print("Auth check " + verifiedEmail() + " -> " + ver);
    ret ver;
  }
  
  S googleSignInID() {
    ret eqic(domain(), "botcompany.de") ? botCompanyGoogleSignInID() : agiBlueGoogleSignInID();
  }

  S hhtml_agiBlue(S contents) {
    ret hhtml(hAddToHead_fast(contents, 
      hIncludeGoogleFont("Source Sans Pro")
      //+ [[<meta name="google-site-verification" content="oRvyeTqgSuv-FE_c-2UKM1Vp0oDqx8h9WruHYWqA-NQ" />]]
      + loadJQuery()
      + hmobilefix()
      + googleSignIn_header("", googleSignInID())
      + hstylesheet("body { font-family: Source Sans Pro }")));
  }
  
  O serveSlicesList() {
    S sort = getAny(params, 'sort, 'by);
    L<Slice> slices;
    S shown;
    if (eqic(sort, 'name)) {
      slices = asList(idx_slicesByName.objectIterator());
      shown = "sorted by name";
    } else {
      sort = 'date;
      slices = asList(idx_slicesByModification.objectIterator());
      shown = "last modified first";
    }
      
    S title = agiBlueNameHTML() + " has " + nSlices(slices) /*+ " (" + shown + ")"*/;
    new PreIncLongCounter counter;
    ret hhtml_agiBlue(hhead_title_decode(title)
      + hbody(hfullcenter(
        h1(title)
      + p(joinWithSpacedVBar(
        ahrefIf(neqic(sort, 'name), agiBlueURL() + "/slices" + hquery(sort := 'name), "List by name"),
        ahrefIf(neqic(sort, 'date), agiBlueURL() + "/slices" + hquery(sort := 'date), "List by date"),
        ahref(agiBlueURL() + "/createSlice", "Create slice...")))

      + htmlTable2(map(slices, slice ->
        litorderedmap(
          "Name" := ahref(sliceHomeURL(slice), htmlEncode2(slice.name)),
          "ID" := ahref(sliceHomeURL(slice), str(slice.globalID)),
          "Owner" := str(slice.owner!)
        )),
        htmlEncode := false)
      )));
  }

  O serveUsersList() {
    S sort = getAny(params, 'sort, 'by);
    L<User> users;
    S shown;
    if (eqic(sort, 'name))
      users = asList(idx_usersByName.objectIterator());
    else {
      sort = 'seen;
      users= asList(idx_usersBySeen.objectIterator());
    }
      
    S title = agiBlueNameHTML() + " has " + nUsers(users);
    new PreIncLongCounter counter;
    ret hhtml_agiBlue(hhead_title_decode(title)
      + hbody(hfullcenter(
        h1(title)
      + p(joinWithSpacedVBar(
        ahrefIf(neqic(sort, 'name), agiBlueURL() + "/users" + hquery(sort := 'name), "List by name"),
        ahrefIf(neqic(sort, 'seen), agiBlueURL() + "/users" + hquery(sort := 'seen), "List by last seen"),
        ahref(agiBlueURL() + "/createSlice", "Create slice...")))

      + htmlTable2(map(users, user ->
        litorderedmap(
          "Name" := ahref(userHomeURL(user), htmlEncode2(str(user))),
          "Last seen" := renderHowLongAgo(user.lastSeen)
        )),
        htmlEncode := false)
      )));
  }

  S sliceInfoHTML() {
    User owner = slice.sliceConcept.owner!;
    ret owner == null ? "" : p("This slice is owned by " + userHTML(owner));
  }

  S userHomeURL(User user) {
    ret user == null ? null : agiBlueURL() + "/user/" + user.globalID;
  }
  
  S userHTML(User user) {
    ret user == null ? "nobody" : ahref(userHomeURL(user), htmlEncode2(str(user)));
  }
  
  S top() {
    ret nempty(get) ? "" : hcomment("cookie: " + takeFirst(4, session.cookie))
      + hSilentComputatorWithFlag("agi.blue: " + q)
      + p(ahref(sliceHomeURL(),
        //hsnippetimg(#1101682, width := 565/5, height := 800/5, title := "Robot by Peerro @ DeviantArt")
        //hsnippetimg(#1101778, width := 96, height := 96, title := "agi.blue - a database for everything")
        hsnippetimg(#1101822, width := 314, height := 125, title := "agi.blue - Wikipedia for robots")
      ))
      + sliceInfoHTML()
      + p(small(
          agiBlueNameHTML_boldWithSize()
        + (agiBlue_isOriginal() ? "" : " " + targetBlank("http://agi.blue", "[original]"))
        + " | " + ahref(agiBlueURL("/slices"), nSlices(countConcepts(Slice)))
        + " | " + targetBlank(progLink(), "source code") + " of this web site (" + nLines(sourceCodeLines) + ") | " + targetBlank("https://gitter.im/agi-blue/community", "sponsor https?") + " | by " + targetBlank("https://BotCompany.de", "BC") + " | " + targetBlank("http://fiverr.tinybrain.de/", "Fiverr") + " | " + targetBlank("https://discordapp.com/invite/SEAjPqk", "Discord") + " | " + targetBlank("https://www.youtube.com/watch?v=b6jtRdV3Ev8", "Video")
        + " | " + targetBlank("http://code.botcompany.de/1024233", "Notes")
        + " | " + ahref(agiBlueURL() + "/query", "Query")
      ));
  }

  Slice createSlice(S name) {
    name = shorten(name, maxSliceNameLength);
    Slice slice = cnew Slice(+name, owner := session.user);
    cset(slice, caseID := slice.defaultCaseID());
    loadSlice(slice).initialSetup(slice.globalID);
    ret slice;
  }

  O serveCheckboxes() {
    set conceptsSortedByFieldCI_verbose;
    int showMax = 1000;
    Cl<Page> pages = takeFirst(showMax, conceptsSortedByFieldCI(slice.cc, Page, 'q));
    S title = "Select pages";
    ret hhtml_miniPage(title, h3(title) + hpostform(
      htmlTable2(map(pages, page -> litorderedmap(
        "Page" := hcheckbox("page_" + page.globalID()) + " " + pageToHTMLLink().get(page)
      )), 
      htmlEncode := false)
      + h3("Add an entry to all selected pages")
        + p("Key: " + hinputfield("key") +
          " Value: " + hinputfield("value"))
      + p(hsubmit("Add entries")), action := agiBlueURL() + "/multiAdd"));
  }

  O serveMultiAdd() {
    S key = trim(params.get('key)), value = trim(params.get('value));
    if (empty(key) || empty(value)) ret "Need key and value";
    new Matches m;
    new L<Page> pages;
    for (S a, b : params)
      if (startsWith(a, "page_", m))
        addIfNotNull(pages, conceptWhere(cc, Page, globalID := m.rest()));
    if (empty(pages)) ret "No pages selected";
    for (Page page : pages)
      new GetEntriesAndPost(cc).go(page, params);
    ret "Entry added to " + nPages(pages);
  }

} // end of Request

svoid dbLog(O... params) {
  logStructure(programFile("db.log"), ll(params));
}

static IF1<Page, Map> pageToMap(O... _) {
  optPar bool withEntries;
  
  bool nameOnly = eqOneOf(optPar nameOnly(_), "1", true);
  if (nameOnly) ret (IF1<Page, Map>) p -> litmap(q := p.q);
  
  ret (IF1<Page, Map>) p -> {
    L<Entry> entries = findBackRefs(p, Entry);
    ret litorderedmap(
      q := p.q,
      nEntries := l(entries),
      created := p.created,
      modified := p._modified,
      entries := !withEntries ? null : map(entries, entryToMap(false)));
  };
}

static IF1<Entry, Map> entryToMap(bool withPage) {
  ret (IF1<Entry, Map>) e -> litorderedmap(
    created := e.created,
    i := e.count,
    key := e.key,
    value := e.value,
    q := withPage ? e.page->q : null,
    signer := getString globalID(e.signer!));
}

static IF1<Slice, Map> sliceToMap() {
  ret (IF1<Slice, Map>) s -> litorderedmap(
    created := s.created,
    globalID := str(s.globalID),
    name := s.name,
    caseID := s.caseID);
}

sO servePageToBot(Page page, SS params) {
  if (page == null) ret serveJSON(null);
  params = asCIMap(params);
  Map map = pageToMap(withEntries := valueIs1 withEntries(params)).get(page);
  ret serveJSON(map);
}

sclass GetEntriesAndPost {
  Concepts cc;
  L<Entry> entries;
  Entry entry;
  bool newEntry;
  Signer signer;
  
  *(Concepts *cc) {}
  
  GetEntriesAndPost go(Page page, SS params) {
    S key = trim(params.get('key)), value = trim(params.get('value));
    print("GetEntriesAndPost: " + quote(page.q) + ", " + quote(key) + " := " + quote(value));
    withDBLock {
      entries = findBackRefs(page, Entry);
      if (nempty(key) && nempty(value)) {
        S ip = subBot_clientIP();
        entry = firstThat(e -> eqic(e.key, key) && eq_icIf(!allowMultipleCasesInValues, e.value, value) && eq(e.ip, ip), entries);
        if (entry == null) {
          print("SAVING");
          Entry e = cnew(cc, Entry, +page, +key, +value, +ip, count := l(entries) + 1, +signer);
          if (makeAllKeysPages) pageFromQ(cc, key);
          if (makeAllValuesPages) pageFromQ(cc, value);
          page.change(); // bump modification date
          entry = e;
          newEntry = true;
          entries.add(entry);
          dbLog("New entry", page := page.q, globalID := e.globalID, count := e.count, +key, +value);
        }
      }
    }

    sortByFieldInPlace created(entries);
    numberEntriesInConceptField count(entries);
    this;
  }
}

static SS availableSlices() {
  L<Slice> slices = asList(idx_slicesByName.objectIterator());
  ret mapToOrderedMap(s -> pair(str(s.globalID), s.name + " [ID: " + s.globalID + "]"), slices);
}

static L<Entry> entriesOnPage(Page p) {
  ret p == null ? null : sortedByField count(findBackRefs(p, Entry));
}

static L<Page> pagesSortedByLength(L<Page> l) {
  ret sortedByCalculatedField(l, p -> l(p.q));
}

sbool agiBlue_isOriginal() {
  ret amProgram(#1023558);
}

sS agiBlueURL(S subUri) { ret agiBlueURL() + prependSlash(subUri); }

sS agiBlueURL() {
  ret agiBlue_isOriginal() ? "https://agi.blue" :  "/" + psI(programID()) + "/raw";
}

sS agiBlueName() {
  ret agiBlue_isOriginal() ? "agi.blue" : "agi.blue clone " + programID();
}

svoid cleanMeUp {
  cleanUp(fan);
}

static LoadedSlice loadSlice(Slice slice) {
  ret loadSlice(slice.caseID, str(slice.globalID));
}

static LoadedSlice loadSlice(S caseID, S globalID) {
  caseID = unnull(caseID);
  lock fan.lock;
  
  // move slice from background to foreground
  
  Concepts cc = backgroundFan.getIfLoaded(caseID);
  if (cc != null) {
    LoadedSlice slice = assertNotNull(backgroundSlices.get(caseID));
    backgroundFan.loaded.remove(caseID);
    fan.loaded.put(caseID, cc);
    backgroundSlices.remove(caseID);
    loadedSlices.put(caseID, slice);
    ret slice;
  }
  
  fan.get(caseID, globalID);
  ret loadedSlices.get(caseID);
}

// load background slice
static LoadedSlice loadBackgroundSlice(Slice slice) {
  ret loadBackgroundSlice(slice.caseID, str(slice.globalID));
}

static LoadedSlice loadBackgroundSlice(S caseID, S globalID) {
  caseID = unnull(caseID);
  lock fan.lock;
  
  // check if in foreground
  
  if (fan.getIfLoaded(caseID) != null)
    ret loadedSlices.get(caseID);

  backgroundFan.get(caseID, globalID);
  LoadedSlice slice = assertNotNull(backgroundSlices.get(caseID));
  slice.lastAccess = sysNow();
  while (l(backgroundSlices) > maxBackgroundSlices)
    unloadLeastRecentlyAccessedBackgroundSlice();
  assertNotNull("slice.sliceConcept", slice.sliceConcept);
  ret slice;
}

// TODO: sync unloading slice with accessors
svoid unloadLeastRecentlyAccessedBackgroundSlice() {
  lock fan.lock;
  S caseID = lowestByField lastAccess(keys(backgroundSlices));
  if (caseID != null)
    backgroundFan.unloadCase(caseID);
}

static Slice sliceConceptForGlobalID(S globalID) {
  ret sliceConceptForGlobalID(toGlobalIDObj(globalID));
}

static Slice sliceConceptForGlobalID(GlobalID globalID) {
  ret conceptWhere Slice(+globalID);
}

static Slice sliceConceptForCaseID(S caseID) {
  ret conceptWhere Slice(+caseID);
}

sS agiBlueNameHTML() {
  ret ahref(agiBlueURL(), htmlEncode2(agiBlueName()));
}

sS agiBlueNameHTML_boldWithSize() {
  Long size = sizeOnDisk.peek();
  ret b(agiBlueNameHTML()) + (size == null ? "" : " " + spanTitle("Size of agi.blue's database on disk", "(" + toM(size) + " MB)"));
}

static Slice sliceForName(S name) {
  ret conceptWhereCI Slice(+name);
}

sclass Query extends WorksOnSlice {
  SS vars = ciMap();
    
  *(LoadedSlice slice) { super(slice); }
  
  O process(S query) {
    try object processLines(agiBlue_parseQueryScript(query));
    ret serveJSON("No return statement");
  }
    
  O processLines(L<ALQLLine> lines) {
    for (ALQLLine line : lines) {
      if (line cast ALQLSlice) {
        Slice slice = sliceForName(line.slice);
        if (slice == null) ret serveJSON("Slice not found: " + line.slice);
        LoadedSlice oldSlice = Query.this.slice;
        try {
          setSlice(loadSlice(slice));
          try object processLines(line.contents);
        } finally {
          setSlice(oldSlice);
        }
      } else if (line cast ALQLReturn)
        ret serveJSON(ll(getOrKeep(vars, line.var)));
      else if (line cast ALQLTriple) {
        T3S t = line.triple;
        t = tripleMap(t, s -> getOrKeep(vars, s));
        Cl<Page> pages;
        S var;
        if (isDollarVar(t.c)) {
          var = t.c;
          if (isDollarVar(t.a)) {
            if (isDollarVar(t.b)) todo(t);
            Entry e = random(conceptsWhereCI(cc, Entry, key := t.b));
            if (e == null) ret serveJSON("No results for " + var);
            pages = pagesForKeyAndValue(t.b, t.c);
            vars.put(t.a, e.page->q);
            vars.put(t.c, e.value);
            continue;
          } else if (isDollarVar(t.b)) {
            Page page = findPageFromQ(t.a);
            Entry e = random(findBackRefs(page, Entry));
            if (e == null) ret serveJSON("No results for " + var);
            vars.put(t.b, e.key);
            vars.put(t.c, e.value);
            continue;
          } else {
            S val = getValue(t.a, t.b);
            if (val == null) ret serveJSON("No results for " + var);
            pages = ll(pageFromQ(val));
          }
        } else if (isDollarVar(t.b)) {
          var = t.b;
          if (isDollarVar(t.c)) todo(t);
          if (isDollarVar(t.a)) {
            L<Entry> entries = conceptsWhereCI(cc, Entry, value := t.c);
            if (empty(entries)) ret serveJSON("No results for " + var);
            Entry e = random(entries);
            vars.put(t.a, e.page->q);
            vars.put(t.b, e.key);
            continue;
          } else {
            Cl<S> keys = keysForPageAndValue(t.a, t.c);
            if (empty(keys)) ret serveJSON("No results for " + var);
            pages = map(f<S, Page> pageFromQ, keys);
          }
        } else {
          var = t.a;
          if (!isDollarVar(t.a)) todo(t);
          if (isDollarVar(t.b)) todo(t);
          if (isDollarVar(t.c)) todo(t);
          pages = pagesForKeyAndValue(t.b, t.c);
        }
        if (empty(pages)) ret serveJSON("No results for " + var);
        vars.put(var, random(pages).q);
      } else if (line cast ALQLPage) {
        if (eq(line.matchMethod, 'eqic))
          findPageFromQ(line.page);
        else if (eq(line.matchMethod, 'flexMatchDollarVarsIC_first)) {
          new L<SS> l;
          for (Page p : list(cc, Page))
            addIfNotNull(l, flexMatchDollarVarsIC_first(line.page, p.q));
          if (empty(l)) ret serveJSON("No results for " + curly(line.page));
          vars.putAll(random(l));
        } else
          fail("Unknown match method: " + line.matchMethod);
      } else
        fail("Can't interpret: " + line);
    }
    
    null;
  }
}

static Page pageFromQ(Concepts cc, S q) {
  ret empty(q) ? null : uniqCI_sync(cc, Page, +q);
}

/*sS globalIDFromCaseID(S caseID) {
  ret assertGlobalID(takeLast(globalIDLength(), caseID));
}*/

sS agiBlue_pageURLWithSlice(Page p) {
  ret p == null ? null : agiBlueURL() + hquery(slice := _get(sliceForPage(p), 'globalID), q := p.q);
}

static Slice sliceForPage(Page p) {
  if (p == null) null;
  SliceInfo info = conceptWhere(p._concepts, SliceInfo);
  ret info == null ? null : sliceConceptForGlobalID(info.globalID);
}

static GlobalID mainSliceGlobalID() {
  ret conceptWhere Slice(caseID := "").globalID;
}

static File sliceDumpFile(Slice slice) {
  ret programFile("slice-dumps/" + or2(slice.caseID, "main") + ".slice");
}

static File dumpSliceToFile(LoadedSlice slice) {
  ret withDBLock(slice.cc, func -> File {
    cset(slice.sliceConcept, sliceDumped := now());
    File file = sliceDumpFile(slice.sliceConcept);
    new LS lines;
    for (Page p : list(slice.cc, Page))
      lines.add(quote(p.q));
    for (Entry e : list(slice.cc, Entry))
      lines.add(sfu(tripleToList(entryToTriple(e))));
    saveLinesAsTextFile(file, lines);
    ret file;
  });
}

static T3S entryToTriple(Entry e) {
  ret e == null ? null : t3(e.page->q, e.key, e.value);
}

static ConceptsLoadedOnDemand slicesLoadedOnDemand(Map<S, LoadedSlice> loadedSlices) {
  new ConceptsLoadedOnDemand fan;
  fan.onCaseLoaded(voidfunc(S caseID, O globalID, Concepts concepts) {
    LoadedSlice ls = new(caseID, concepts);
    ls.sliceConcept = sliceConceptForCaseID(caseID);

    if (nempty(globalID))
      ls.initialSetup(toGlobalIDObj((S) globalID));
    loadedSlices.put(caseID, ls);
  });
  fan.onUnloadingCase(voidfunc(S caseID, Concepts concepts) {
    loadedSlices.remove(caseID);
  });
  ret fan;
}

// check if slices are loaded both in foreground and background (BAD)
sS checkDoubleLoads() {
  LS l = sharedKeys(loadedSlices, backgroundSlices);
  ret empty(l) ? null : "ERROR: Slice loaded in background and foreground: " + first(l);
}

// just use size of concepts.structure.gz
static long estimatedSliceDataSize(Slice slice) {
  ret estimatedSliceDataSize(slice.caseID);
}

static long estimatedSliceDataSize(S caseID) {
  ret l(conceptsFile(fan.dbID(caseID)));
}

static CentralIndexEntry centralIndexGet(S s) { ret getOrCreate CentralIndexEntry(centralIndex, s); }

static CentralIndexEntry newCentralIndexGet(S s) { ret getOrCreate CentralIndexEntry(newCentralIndex, s); }

static Cl<Slice> centralIndexGetSlices(S s) {
  CentralIndexEntry e = centralIndexGet(s);
  ret e == null ? null : e.slices;
}

// executed by rst_index
svoid makeCentralIndex() {
  newCentralIndex.clear();
  long time = now();
  pcall {
    for (Slice slice : conceptsWhere(Slice, indexed := true))
      addToNewCentralIndex(slice);
  }
  centralIndex.putAll(newCentralIndex);
  removeAllKeysNotIn(centralIndex, newCentralIndex);
  newCentralIndex.clear();
  centralIndexMade = now();
  centralIndexMadeIn = now()-time;
}

svoid addToNewCentralIndex(Slice slice) {
  for (S s : sliceDump_pageNamesIncludingKeys(sliceDumpFile(slice)))
    newCentralIndexGet(s).slices.add(slice);
}

Began life as a copy of #1023558