!7

// See #1023660 for the older 99 lines version

// Customizations:

sbool asyncSearch = false;

sbool allowMultipleCasesInValues = true;

sbool showSliceSelector = true;

sbool showGoogleLogIn = true;

sbool makeAllKeysPages = true;

sbool makeAllValuesPages = true; 

sbool legacy = true; // We still have slices without applied fixes

static int maxSliceNameLength = 1024;

static int displayLength = 140;

static int sideDisplayLength = 50; // when in side bar

static int searchResultsToShow = 50;

sS sideSearchType = 'literal;

static int maxBackgroundSlices = 1000;

// end of customizations

// in main DB

concept Slice {

  GlobalID globalID = aGlobalIDObject();

  S caseID;

  S name;

  bool indexed = true;

  long sliceDumped; // wall time, set before dump starts

  new Ref<User> owner;

  S defaultCaseID() {

    ret uniqueFileNameUsingMD5_80_v2(name + " " + globalID);

  }

}

// in any DB

concept Page {

  S globalID = aGlobalID();

  S url, q;

  // for in-VM helper bots

  void quickPost(S key, S value) {

    new GetEntriesAndPost(_concepts).go(this, litmap(+key, +value));

  }

}

abstract concept AbstractEntry {

  new Ref<Page> page;

  int count;

  S key;

  S ip;

  new Ref<Signer> signer;

  S q() { Page p = page!; ret p == null ? null : page->q; }

}

concept Entry > AbstractEntry {

  S globalID = aGlobalID(); // TODO: migrate to object

  S value;

}

concept MultiLineEntry > AbstractEntry {

  GlobalID globalID = aGlobalIDObject();

  S value;

}

concept BlobEntry > AbstractEntry {

  GlobalID globalID = aGlobalIDObject();

  long length;

  S md5;

}

// in main DB?

concept Signer {

  S globalID = aGlobalID();

  S publicKey;

  bool trusted;

  S approvedBy;

}

concept User {

  GlobalID globalID = aGlobalIDObject();

  long lastSeen;

}

concept CookieUser > User {

  S cookie;

  toString { ret "[cookieUser]"; }

}

concept GoogleUser > User {

  long googleLogInDate;

  S googleEmail, googleFirstName, googleLastName;

  bool googleEmailVerified;

  toString { ret googleFirstName + " " + googleLastName; }

}

concept MinimalUser > User {

  S userName, encryptedPassword; // TODO

  toString { ret "[minimalUser] " + userName; }

}

// in main DB

concept Session {

  S cookie;

  S selectedSlice; // global ID

  Page slicePage; // phasing out

  new Ref<User> user;

}

// singleton concept in every DB

concept SliceInfo {

  GlobalID globalID;

  LS mandatoryBotVMBusIDs;

}

sclass CentralIndexEntry {

  Set<Slice> slices = synchroLinkedHashSet();

}

// global constants

static SS searchTypeToText = litmap(

  leven := "Leven",

  literal := "Literal",

  scored := "Scored");

// global vars

static volatile bool started, broken;

static int sourceCodeLines;

static ConceptsLoadedOnDemand fan;

static ConceptsLoadedOnDemand backgroundFan;

static Map<S, LoadedSlice> loadedSlices = syncMap();

static Map<S, LoadedSlice> backgroundSlices = syncMap();

static Map<S, CentralIndexEntry> centralIndex = ciMap();

static Map<S, CentralIndexEntry> newCentralIndex = ciMap();

static long centralIndexMade; // wall time

static ReliableSingleThread rst_index = new(r makeCentralIndex);

static TimedCache<Long> sizeOnDisk = new(60.0, f guessClusteredSizeOfProgramDirWithoutBackups);

static ConceptFieldIndexDesc idx_slicesByModification, idx_usersBySeen;

static ConceptFieldIndexCI idx_slicesByName, idx_usersByName;

// end of global data

sclass LoadedSlice { // non-persistent class

  S caseID;

  Concepts cc;

  Slice sliceConcept;

  long lastAccess = sysNow();

  ConceptFieldIndexDesc idx_latestEntries, idx_latestCreatedPages, idx_latestChangedPages;

  *(S *caseID, Concepts *cc) {

    indexThings();

    // forward changes to Slice concept

    onConceptsChange(cc, r {

      cset(sliceConcept, _modified := now())

    });

    if (makeAllKeysPages && legacy)

      for (Entry e : list(cc, Entry))

        pageFromQ(e.key);

    if (makeAllValuesPages && legacy)

      for (Entry e : list(cc, Entry))

        pageFromQ(e.value);

  }

  void initialSetup(GlobalID globalID) {

    SliceInfo info = uniq(cc, SliceInfo);

    cset(info, +globalID);

  }

  // create if not there

  Page pageFromQ(S q) {

    ret main.pageFromQ(cc, q);

  }

  void indexThings() {

    indexConceptFieldsCI(cc, Page, 'url, Page, 'q, Entry, 'key, Entry, 'value);

    indexConceptFields(cc, Signer, 'publicKey);

    indexConceptFields(cc, Session, 'cookie);

    indexSingletonConcept(cc, SliceInfo);

    idx_latestCreatedPages = new ConceptFieldIndexDesc(cc, Page, 'created);

    idx_latestChangedPages = new ConceptFieldIndexDesc(cc, Page, '_modified);

    idx_latestEntries = new ConceptFieldIndexDesc(cc, Entry, 'created);

  }

  bool isMainSlice() { ret empty(caseID); }

  GlobalID globalID() { ret sliceConcept.globalID; }

  S name() { ret sliceConcept.name; }

}

p {

  try {

    if (bootstrapDataFrom(#1023558)) deleteMyBackups();

    thinMyBackups();

    fan = slicesLoadedOnDemand(loadedSlices);

    mainConcepts = fan.get("", null);

    // index main DB

    indexConceptFields(Slice, 'globalID, Slice, 'caseID);

    indexConceptField(CookieUser, 'cookie);

    idx_slicesByName = new ConceptFieldIndexCI(Slice, 'name);

    idx_slicesByModification = new ConceptFieldIndexDesc(Slice, '_modified);

    idx_usersByName = new ConceptFieldIndexCI(GoogleUser, 'googleLastName);

    idx_usersBySeen = new ConceptFieldIndexDesc(User, 'lastSeen);

    cset(uniq_returnIfNew Slice(caseID := ""), name := "main slice");

    loadedSlices.get("").initialSetup(toGlobalIDObj(mainSliceGlobalID()));

    loadedSlices.get("").sliceConcept = sliceConceptForCaseID("");

    backgroundFan = slicesLoadedOnDemand(backgroundSlices);

    backgroundFan.lock = fan.lock;

    // legacy conversions!

    //deleteConcepts(Slice, globalID := GlobalID('wftlawbagrwprywn));

    // Approve this machine's key

    PKIKeyPair machineKey = agiBot_trustedKeyForMachine();

    if (machineKey != null) {

      print("Approving this machine's key: " + machineKey.publicKey);

      cset(uniq_sync(Signer, publicKey := machineKey.publicKey), trusted := true, approvedBy := "local");

    }

    sourceCodeLines = countLines(mySource());

    rst_index.trigger();

  } catch print e {

    set broken;

  } finally {

    set started;

  }

  // do stuff after we are officially started

  // calculate DB size regularly iff there were changes

  sizeOnDisk.keepValueWhileCalculating = true;

  doEveryAndNow(60.0, rWatcher(() -> db_mainConceptsChangeCount(), r { sizeOnDisk! }));

} // end of main program

// Serve page

set flag NoNanoHTTPD. html {

  sleepWhile(() -> !started);

  if (broken) ret serve500("Internal error");

  ret new Request().serve(uri, params);

}

sclass WorksOnSlice {

  Concepts cc;

  LoadedSlice slice;

  *() {}

  *(LoadedSlice *slice) { cc = slice.cc; }

  void setSlice(LoadedSlice slice) {

    this.slice = slice;

    cc = slice.cc;

  }

  Page findPageFromParams(Map map) {

    S q = getString q(map);

    ret empty(q) ? null : findPageFromQ(q);

  }

  Page findOrMakePageFromParams(Map map) {

    ret pageFromQ(getString q(map));

  }

  Page findPageFromQ(S q) {

    ret conceptWhereCI(cc, Page, +q);

  }

  Page pageFromQ(S q) {

    ret slice.pageFromQ(q);

  }

  Set<Page> pagesForKeyAndValue(S key, S value) {

    L<Entry> entries = conceptsWhereIC(cc, Entry, +value, +key);

    ret asSet(ccollect page(entries));

  }

  Cl<S> keysForPageAndValue(S q, S value) {

    Page page = findPageFromQ(q);

    if (page == null) null;

    ret collect key(conceptsWhereIC(cc, Entry, +page, +value));

  }

  // DB functions

  bool hasPage(S q) { ret hasConceptWhereIC(Page, +q); }

  S getValue(Page page, S key) {

    ret page == null || empty(key) ? null : getString value(highestByField count(objectsWhereIC(findBackRefs(page, Entry), +key)));

  }

  S getValue(S page, S key) {

    ret getValue(findPageFromQ(page), key);

  }

  S pageDisplayName(Page page) {

    /*S name = getValue(page, "read as");

    bool unnaturalName = nempty(name) && !eq(makeAGIDomain(name), page.url);

    ret unnaturalName ? name + " " + squareBracketed(page.url) : or2(name, unpackAGIDomainOpt(page.url));*/

    ret page.q;

  }

  S renderThing(S s, bool forceURLDisplay) {

    ret forceURLDisplay || isURL(s) || isAGIDomain(s)

      ? ahref(fixAGILink(absoluteURL(s)), htmlencode2(shorten(displayLength, s)))

      : ahref(agiBlue_linkForPhrase(s,

          slice := slice.isMainSlice() ? null : slice.sliceConcept.globalID),

        htmlencode2(shorten(displayLength, s)));

  }

  GlobalID sliceID() { ret slice.globalID(); }

} // end of WorksOnSlice

sclass Request extends WorksOnSlice {

  S cookie;

  Session session;

  S uri;

  SS params;

  long started = sysNow();

  S q, domain;

  Set<S> get;

  // when serving a concept page, info that may be grabbed

  // by thought bots

  L<Entry> usesAsKey;

  // should also work for standalone

  bool isHomeDomain() {

    S domain = domain();

    ret eqic(domain, "www.agi.blue") || !ewic(domain, ".agi.blue");

  }

  O serve(S uri, SS params) {

    this.uri = dropMultipleSlashes(uri);

    this.params = params;

    new Matches m;

    print(uri + " ? " + unnull(subBot_query()));

    // get cookie & session

    cookie = cookieFromUser();

    if (cookie != null)

 {

      session = uniq_sync(Session, +cookie);

      if (session.user! == null)

        cset(session, user := uniq_sync CookieUser(+cookie));

    } else

      session = unlistedWithValues(Session);

    // check if user wants to change slices

    S selectSlice = params.get('slice);

    if (isGlobalID(selectSlice))

      cset(session, selectedSlice := selectSlice);

    if (session.selectedSlice == null)

      cset(session, selectedSlice := str(mainSliceGlobalID()));

    // get slice's case ID

    S caseID = "";

    Slice sliceConcept = sliceConceptForGlobalID(session.selectedSlice);

    print("Selected slice: " + session.selectedSlice + ", obj? " + (sliceConcept != null));

    if (sliceConcept != null) caseID = sliceConcept.caseID;

    print("caseID: " + caseID);

    // load slice

    slice = assertNotNull(loadSlice(caseID, session.selectedSlice));

    cc = slice.cc;

    // Check for special URIs

    if (swic(uri, "/bot/")) ret serveBot();

    if (swic(uri, "/user/", m)) {

      User user = conceptWhere User(globalID := toGlobalIDObj(assertGlobalID(m.rest())));

      if (user == null) ret subBot_serve404("User not found");

      S title = "User " + user;

      ret hhtml_miniPage(title, h3(agiBlueNameHTML() + " | " + title)

        + "User type: " + classShortName(user));

    }

    // eleu appends a slash to the URI if it's a single identifier, so we drop it again

    S uri2 = dropTrailingSlash(uri);

    if (eqic(uri2, "/google-verify")) {

      print("Google-verify started.");

      Payload payload = printStruct(googleVerifyUserToken2(googleSignInID(), params.get("token")));

      if (payload == null) ret print("google-verify", "No");

      S email = payload.getEmail();

      if (empty(email)) ret print("google-verify", "No");

      GoogleUser user = uniqCI_sync GoogleUser(googleEmail := email);

      cset(user,

        googleEmailVerified := payload.getEmailVerified(),

        googleFirstName := strOrNull(payload.get("given_name")),

        googleLastName := strOrNull(payload.get("family_name")));

      cset(session,

 +user);

      ret print("google-verify", payload.getEmail() + " " + (payload.getEmailVerified() ? "(verified)" : "(not verified)"));

    }

    cset(session.user!, lastSeen := now());      

    if (eqic(uri2, "/users")) ret serveUsersList();

    if (eqic(uri2, "/slices")) ret serveSlicesList();

    if (eqic(uri2, "/search")) ret serveScoredSearch();

    if (eqic(uri2, "/literalSearch")) ret serveLiteralSearch();

    if (eqic(uri2, "/levenSearch")) ret serveLevenSearch();

    if (eqic(uri2, "/query")) ret serveQueryPage();

    if (eqic(uri2, "/createSlice")) ret serveCreateSlicePage();

    if (eqic(uri2, "/deletePage")) ret serveDeletePage();

    if (eqic(uri2, "/checkboxes")) ret serveCheckboxes();

    if (eqic(uri2, "/multiAdd")) ret serveMultiAdd();

    q = params.get('q);

    get = asCISet(nempties(subBot_paramsAsMultiMap().get('get)));

    domain = or2(params.get('domain), domain());

    S raw = firstKeyWithValue("", params); // agi.blue?something

    if (nempty(raw) && empty(q)) q = raw;

    /*if (nempty(q)) {

      domain = makeAGIDomain(q);

      if (l(domain) > maximumDomainPartLength()) // escape with "domain="

        ret hrefresh(agiBlueURL() + hquery(+domain, key := "read as", value := q));

      ret hrefresh("http://" + domain + (eq(q, domain) ? "" : "/" +  hquery(key := "read as", value := q)));

      //uri = "/"; replaceMapWithParams(params, key := "read as", value := q);

    }*/

    S url = domain + dropTrailingSlash(uri);

    // domain to query

    //if (empty(q)) q = url;

    if (empty(q)) {

      S qq = agiBlue_urlToQuery(url);

      if (neqic(qq, "agi.blue")) q = qq;

    }

    Page page, bool newPage = unpair uniqCI2_sync(cc, Page, +q);

    if (newPage) dbLog("New page", +q);

    //printStructs(+params, +raw, +q);

    if (empty(params.get('q)) && empty(raw) && isHomeDomain()) {

      //L<Page> pages = sortedByFieldDesc _modified(list(cc, Page)); // TODO: use index

      L<Page> pages = cloneList(slice.idx_latestChangedPages.objectIterator());

      int start = parseInt(params.get("start")), step = 100;

      S nav = pageNav2("/", l(pages), start, step, "start");

      S content = hform(b("GIVE ME INPUT:   ") + htextinput('q, autofocus := true) + " " + hsubmit("Ask"))

        + h1(sliceAsHTML() + " has " + nPages(countConcepts(cc, Page)) + " and " + nConnections(countConcepts(cc, Entry)))

        + p(nav)

        + p_nemptyLines(map(pageToHTMLLink(), subList(pages, start, start+step)));

      ret hhtml_agiBlue(hhead_title("Slice " + sliceConcept().name + " | agi.blue") // SERVE SLICE HOME PAGE

        + hbody(hfullcenterAndTopLeft(top() + content

          + footer(),

          sliceSelector()

        )));

    }

    S key = trim(params.get('key)), value = trim(params.get('value));

    L<Entry> entries = GetEntriesAndPost(cc).go(page, params).entries;

    //S get = params.get('get);

    if (nempty(get))

      ret serveJSON(collect value(llNotNulls(firstThat(entries, e -> get.contains(e.key)))));

    S key2 = key, value2 = value; if (nempty(key) && nempty(value)) key2 = value2 = ""; // input reset

    bool withHidden = eq(params.get('withHidden), "1");

    new Set<Int> hide;

    if (!withHidden) for (Entry e : entries) if (eqic(e.key, 'hide) && isSquareBracketedInt(e.value)) addAll(hide, e.count, parseInt(unSquareBracket(e.value)));

    new MultiMap<Int, S> mmMeta;

    for (Entry e : entries) if (isSquareBracketedInt(e.key)) mmMeta.put(parseInt(unSquareBracket(e.key)), e.value);

    new MultiMap<S> mm;

    for (Entry e : entries) mm.put(e.key, e.value);

    //S name = or2(/* ouch */ last(mm.get("read as")), /* end ouch */ unpackAGIDomain(page.url), page.url);

    S name = page.q;

    // Find references

    L<Entry> refs = concatLists(conceptsWhereIC(cc, Entry, value := name),

      conceptsWhereIC(cc, Entry, key := name));

    Set<Page> refPages = asSet(ccollect page(refs));

    refPages.remove(page); // don't list current page as reference

    // Search in page names (depending on default search type)

    L<Page> searchResults;

    if (eq(sideSearchType, 'leven))

      searchResults = levenSearch(page.q, max := 50);

    else if (eq(sideSearchType, 'literal))

      searchResults = literalSearch(page.q, max := 50);

    else

      searchResults = (L<Page>) dm_call('agiBlueSearch, 'search, page.q, maxResult := searchResultsToShow+1, agiBlueBotID := programID(), +cc);

    searchResults.remove(page);

    // Find uses as key

    usesAsKey = conceptsWhereIC(cc, Entry, key := page.q);

    // Find page in other slices

    CentralIndexEntry cie = centralIndex.get(page.q);

    L<Slice> pageInOtherSlices = cie == null ? null : listMinus(cie.slices, slice.sliceConcept);

    S pageName_html = htmlEncode2(shorten(displayLength, name));

    S mainContents =

        top() + h1(ahref_unstyled(agiBlue_pageURLWithSlice(page), pageName_html) + (newPage ? " [huh????]" : ""))

      + p_nemptyLines(map(entries, func(Entry e) -> S {

        !withHidden && (hide.contains(e.count) || eqic(e.key, "read as") && eqic(e.value, name)) ? ""

        : "[" + e.count + "] " +

          renderThing(e.key, false) + ": " +

          b(renderThing(e.value, cic(mmMeta.get(e.count), "is a URL")))

        }))

      + hpostform(h3("Add an entry")

        + "Key: " + hinputfield(key := key2) + " Value: " + hinputfield(value := value2) + "<br><br>" + hsubmit("Add")

        )

      + p(ahref(agiBlueURL() + "/literalSearch" + hquery(q := page.q), "[literal search]", title := "Search pages with a name containing this page's name literally")

         + " " +

         ahref(agiBlueURL() + "/levenSearch" + hquery(q := page.q), "[leven search 1]", title := "Search pages with a Levenshtein similarity of 1 containing this page's name literally")

         + " " +

         ahref(agiBlueURL() + "/search" + hquery(q := page.q), "[scored search]", title := "Search pages with ScoredSearch")

         + " " +

         ahref(agiBlueURL() + "/deletePage" + hquery(slice := sliceID(), q := page.q), "[delete page]")

       )

       // optional "uses as key" section

       + (empty(usesAsKey) ? "" : h3(quote(pageName_html) + " as key")

       + p_nemptyLines_showFirst(50, map(usesAsKey, e ->

         pageToHTMLLink().get(e.page!) + unicode_spacedRightPointingTriangle() +

           pageName_html + unicode_spacedRightPointingTriangle() +

         pageToHTMLLink().get(pageFromQ(e.value))

       )))

       // optional "in other slices" section

       + (empty(pageInOtherSlices) ? "" : h3(quote(pageName_html) + " in other slices")

       + p_nemptyLines_showFirst(50, map(slice ->

         ahref(agiBlueURL() + hquery(slice := slice.globalID, q := page.q), htmlEncode2(slice.name)), pageInOtherSlices)))

       ;

      // end of mainContents

    S sideContents =

      hform(b("GIVE ME INPUT:") + " "

        + htextinput('q) + " "

        + hsubmit("Ask", onclick := "document.getElementById('newInputForm').target = '';") + " "

        + hsubmit("+Tab", title := "Ask and show result in a new tab", onclick := "document.getElementById('newInputForm').target = '_blank';"),

        id := 'newInputForm)

      + h3("References (" + l(refPages) + ")")

      + p_nemptyLines_showFirst(10, map(pageToHTMLLink(displayLength := sideDisplayLength), refPages))

      + h3(searchTypeToText.get(sideSearchType) + " search results (" + (l(searchResults) >= searchResultsToShow ? searchResultsToShow + "+" : str(l(searchResults))) + ")")

      + p_nemptyLines_showFirst(searchResultsToShow, map(pageToHTMLLink(displayLength := sideDisplayLength), searchResults))

      + hdiv("", id := 'extraStuff);

    // TODO: sync search delivery with WebSocket creation

    if (asyncSearch)

      doLater(6.0, r { dm_call('agiBlueSearch, 'searchAndPost, page.q, agiBlueBotID := programID(), +cc) });

    // notify local interested parties

    vmBus_send('agiBlue_servingConceptPage, this, page);

    S iframe = params.get('render_iframe);

    // serve a concept page

    ret hhtml_agiBlue(hhead_title(pageDisplayName(page)) + hbody(

      tag('table,

        tr(

          td(sliceSelector(), valign := 'top)

        + td(sideContents, align := 'right, valign := 'top, rowspan := 2))

      + tr(td(mainContents +

        (empty(iframe) ? "" : iframe(iframe, width := 600, height := 400)) +

      footer(), align := 'center, valign := 'top)),

      width := "100%", height := "100%")));

  }

  O servePagesToBot(Iterable<Page> pages) {

    ret serveListToBot(map(pageToMap(wrapMapAsParams(params)), pages));

  }

  O serveListToBot(Collection l) {

    if (nempty(params.get('max)))

      l = takeFirst(parseInt(params.get('max)), l);

    ret serveJSON(l);

  }

  // uri starts with "/bot/"

  O serveBot() {

    S q = params.get('q);

    if (eqic(uri, "/bot/hello")) ret serveJSON("hello");

    if (eqic(uri, "/bot/hasPage")) ret serveJSON(hasPage(q));

    if (eqic(uri, "/bot/randomPageContaining")) {

      assertNempty(q);

      ret servePageToBot(random(filter(list(cc, Page), p -> cic(p.q, q))), params);

    }

    if (eqic(uri, "/bot/allPages"))

      ret servePagesToBot(list(cc, Page));

    if (eqic(uri, "/bot/allPagesStartingWith")) {

      assertNempty(q);

      ret servePagesToBot(filter(list(cc, Page), p -> swic(p.q, q)));

    }

    if (eqic(uri, "/bot/allPagesEndingWith")) {

      assertNempty(q);

      ret servePagesToBot(filter(list(cc, Page), p -> ewic(p.q, q)));

    }

    if (eqic(uri, "/bot/allPagesContaining")) {

      assertNempty(q);

      ret servePagesToBot(filter(list(cc, Page), p -> cic(p.q, q)));

    }

    if (eqic(uri, "/bot/allPagesContainingRegexp")) {

      assertNempty(q);

      Pattern pat = regexpIC(q);

      ret servePagesToBot(filter(list(cc, Page), p -> regexpFindIC(pat, p.q)));

    }

    if (eqicOneOf(uri, "/bot/postSigned", /*"/bot/makePhysicalSlice",*/ "/bot/approveTrustRequest")) {

      S text = rtrim(params.get('text));

      S key = getSignerKey(text);

      if (empty(key)) ret subBot_serve500("Please include your public key");

      if (!isSignedWithKey(text, key)) ret subBot_serve500("Signature didn't verify");

      text = dropLastTwoLines(text); // drop signer + sig line

      Signer signer = uniq_sync Signer(publicKey := key);

      /*if (eqic(uri, "/bot/makePhysicalSlice")) {

        if (!signer.trusted) ret subBot_serve500("Untrusted signer");

        Page page = findPageFromParams(jsonDecodeMap(text));

        if (page == null) ret subBot_serve500("Page not found");

        ret serveJSON(uniq2_sync(PhysicalSlice, slicePage := page).b ? "Slice made" : "Slice exists");

      }*/

      if (eqic(uri, "/bot/postSigned")) {

        new L out;

        for (S line : tlft(text)) {

          SS map = jsonDecodeMap(line);

          GetEntriesAndPost x = new(db_mainConcepts());

          x.signer = signer;

          Page page = findOrMakePageFromParams(map);

          if (page == null) continue with out.add("Invalid page reference");

          x.go(page, map);

          out.add(x.newEntry ? "Saved" : x.entry != null ? "Entry exists" : "Need key and value");

        }

        ret serveJSON(out);

      }

      if (eqic(uri, "/bot/approveTrustRequest")) {

        if (!signer.trusted) ret subBot_serve500("Untrusted signer");

        Signer toApprove = conceptWhere Signer(publicKey := trim(text));

        if (toApprove == null) ret subBot_serve500("Signer to approve not found");

        cset(toApprove, trusted := true, approvedBy := signer.globalID);

        ret serveJSON("Approved: " + trim(text));

      }

      ret subBot_serve500("CONFUSION");

    }

    if (eqic(uri, "/bot/post")) {

      GetEntriesAndPost x = new(db_mainConcepts());

      x.go(pageFromQ(q), params);

      ret serveJSON(x.newEntry ? "Saved" : x.entry != null ? "Entry exists" : "Need key and value");

    }

    if (eqic(uri, "/bot/entriesOnPage"))

      ret serveJSON(map(entriesOnPage(findPageFromParams(params)), entryToMap(false)));

    if (eqic(uri, "/bot/entriesForKey"))

      ret serveJSON(map(conceptsWhereIC(cc, Entry, key := params.get('key)), entryToMap(true)));

    if (eqic(uri, "/bot/lookup")) {

      S key = params.get('key);

      if (empty(key)) ret serveJSON("Need key");

      S value = getValue(findPageFromParams(params), key);

      ret serveJSON(empty(value) ? "" : litmap(+value));

    }

    if (eqic(uri, "/bot/multiLookup")) {

      S key = params.get('key);

      if (empty(key)) ret serveJSON("Need key");

      ret serveJSON(collect value(objectsWhereIC(findBackRefs(findPageFromParams(params), Entry), +key)));

    }

    if (eqic(uri, "/bot/latestEntries"))

      ret serveJSON(map(takeFirst(10, slice.idx_latestEntries.objectIterator()), entryToMap(true)));

    if (eqic(uri, "/bot/latestPages"))

      ret serveJSON(map(takeFirst(10, slice.idx_latestCreatedPages.objectIterator()), pageToMap()));

    if (eqic(uri, "/bot/latestChangedPages"))

      ret serveJSON(map(takeFirst(10, slice.idx_latestChangedPages.objectIterator()), pageToMap()));

    if (eqic(uri, "/bot/googleUsersCount"))

      ret serveJSON(countConcepts(GoogleUser));

    if (eqic(uri, "/bot/totalPageCount"))

      ret serveJSON(countConcepts(Page));

    /*if (eqic(uri, "/bot/pageWithoutPhysicalSliceCount"))

      ret serveJSON(countConceptsWhere(Page, slice := null));

    if (eqic(uri, "/bot/physicalSliceCount"))

      ret serveJSON(countConcepts(PhysicalSlice));*/

    if (eqic(uri, "/bot/trustedSignersCount"))

      ret serveJSON(countConcepts(Signer, trusted := true));

    if (eqic(uri, "/bot/valueSearch")) {

      S value = params.get('value);

      L<Entry> entries = conceptsWhereIC(cc, Entry, +value);

      ret serveJSON(map(takeFirst(100, entries), entryToMap(true)));

    }

    if (eqic(uri, "/bot/keyAndValueSearch")) {

      S key = params.get('key), value = params.get('value);

      Cl<Page> pages = pagesForKeyAndValue(key, value);

      ret servePagesToBot(pages);

    }

    if (eqic(uri, "/bot/keyValuePairsByPopularity")) {

      L<PairS> pairs = map(list(Entry), e -> pair(e.key, e.value));

      LPair<S, Int> pairs2 = multiSetTopPairs(ciMultiSet(map pairToUglyStringForCIComparison(pairs)));

      ret serveJSON(map(pairs2, p -> {

        S key, value = unpair pairFromUglyString(p.a);

        ret litorderedmap(n := p.b, +key, +value);

      }));

    }

    if (eqic(uri, "/bot/allKeys"))

      ret serveListToBot(distinctCIFieldValuesOfConcepts(cc, Entry, 'key));

    if (eqic(uri, "/bot/allKeysByPopularity"))

      ret serveListToBot(mapMultiSetByPopularity(distinctCIFieldValuesOfConcepts_multiSet(cc, Entry, 'key), (key, n) -> litorderedmap(+n, +key)));

    if (eqic(uri, "/bot/dbSize"))

      ret serveJSON(l(conceptsFile()));

    if (eqic(uri, "/bot/query"))

      ret serveBotQuery();

    if (eqic(uri, "/bot/createSlice")) {

      Slice slice = createSlice(assertNempty(params.get('name)));

      ret serveJSON(litorderedmap(id := str(slice.globalID), name := slice.name));

    }

    if (eqic(uri, "/bot/dumpAllSlices") && authed()) {

      new L<Map> out;

      for (Slice slice) try {

        out.add(litorderedmap(slice := slice.caseID, ms := returnTimeFor(() -> dumpSliceToFile(loadBackgroundSlice(slice)))));

      } catch print e {

        out.add(litorderedmap(slice.caseID, error := exceptionToStringShort(e)));

      }

      ret serveJSON(out);

    }

    if (eqic(uri, "/bot/dumpSlice")) {

      File f = dumpSliceToFile(slice);

      ret "OK, saved as: " + f2s(f);

    }

    if (eqic(uri, "/bot/diskStats"))

      ret serveJSON(litcimap(

        sizeOnDisk := sizeOnDisk!

      ));

    if (eqic(uri, "/bot/memStats")) {

      Map fg = cloneMap(loadedSlices);

      Map bg = cloneMap(backgroundSlices);

      ret serveJSON(litcimap(

        error := checkDoubleLoads(),

        centralIndexEntries := l(centralIndex),

        centralIndexMade := renderHowLongAgo(centralIndexMade),

        numLoadedForegroundSlices := l(fg),

        numLoadedBackgroundSlices := l(bg),

        loadedForegroundSlicesEstimatedSize := longSum(map estimatedSliceDataSize(keys(fg))),

        loadedBackgroundSlicesEstimatedSize := longSum(map estimatedSliceDataSize(keys(bg))),

        loadedForegroundSlices := sortedIC(keysList(fg)),

        loadedBackgroundSlices := sortedIC(keysList(bg)),

        sessionsWithGoogleUsers := countPred(list(Session), s -> s.user! instanceof GoogleUser),

        userObjects := countConcepts(User),

      ));

    }

    if (eqic(uri, "/bot/centralIndexGet")) {

      CentralIndexEntry e = centralIndex.get(q);

      ret serveJSON(e == null ? ll() : map(e.slices, sliceToMap()));

    }

    // return primary triple from each slice

    if (eqic(uri, "/bot/centralIndexGrab")) {

      CentralIndexEntry ie = centralIndex.get(q);

      new L<Map> out;

      if (ie != null) for (Slice slice : ie.slices) {

        LoadedSlice ls = loadBackgroundSlice(slice);

        WorksOnSlice wos = new(ls);

        Page page = wos.findPageFromQ(q);

        Iterable<Entry> entries = entriesOnPage(page);

        print("Page for " + q + " in slice " + ls.caseID + ": " + yesNo(page != null) + " - " + l(entries));

        Entry e = first(entries);

        if (e != null)

          out.add(litorderedmap(a := page.q, b := e.key, c := e.value, slice := slice.caseID));

      }

      ret serveJSON(out);

    }

    if (eqic(uri, "/bot/updateCentralIndex") && authed()) {

      rst_index.trigger();

      ret "OK";

    }

    if (eqic(uri, "/bot/allGoogleEmails") && authed())

      ret serveJSON(collect googleEmail(list(GoogleUser)));

    if (eqic(uri, "/bot/words2_spaces_all"))

      ret serveJSON(mapToValues_ciMap words2_spaces_cached(collect q(conceptsSortedByFieldCI(slice.cc, Page, 'q))));

    if (eqic(uri, "/bot/words2_spaces_collapse_all"))

      ret serveJSON(mapToValues_ciMap words2_spaces_collapse_cached(collect q(conceptsSortedByFieldCI(slice.cc, Page, 'q))));

    // end of bot methods

    ret subBot_serve404();

  }

  O serveBotQuery() {

    S query = params.get('query);

    ret new Query(slice).process(query);

  }

  O serveLiteralSearch() {

    S q = params.get('q);

    L<Page> searchResults = literalSearch(q);

    ret serveSearchResults("literal search" , q, searchResultsToShow, searchResults);

  }

  L<Page> literalSearch(S q, O... _) {

    int searchResultsToShow = optPar max(_, 100);

    // quick search in random order

    //L<Page> searchResults = takeFirst(searchResultsToShow+1, filterIterator(iterator(list(cc, Page)), p -> cic(p.q, q));

    // full search, order by length

    ret takeFirst(searchResultsToShow+1, pagesSortedByLength(filter(list(cc, Page), p -> cic(p.q, q))));

  }

  O serveScoredSearch() {

    S q = params.get('q);

    L<Page> searchResults = (L<Page>) dm_call('agiBlueSearch, 'search, q, +cc);

    ret serveSearchResults("scored search" , q, searchResultsToShow, searchResults);

  }

  O serveLevenSearch() {

    S q = params.get('q);

    L<Page> searchResults = levenSearch(q);

    ret serveSearchResults("leven search with distance 1" , q, searchResultsToShow, searchResults);

  }

  L<Page> levenSearch(S q, O... _) {

    int searchResultsToShow = optPar max(_, 100);

    int maxEditDistance = 1;

    new Map<Page, Int> map;

    for (Page p : list(cc, Page)) {

      int distance = leven_limitedIC(q, p.q, maxEditDistance+1);

      if (distance <= maxEditDistance) map.put(p, distance);

    }

    ret takeFirst(searchResultsToShow+1, keysSortedByValue(map));

  }

  O serveSearchResults(S searchType, S q, int searchResultsToShow, Collection<Page> searchResults) {

    S title = "agi.blue " + searchType + " for " + htmlEncode2(quote(q)) + " (" + (l(searchResults) >= searchResultsToShow ? searchResultsToShow + "+ results" : nResults(l(searchResults))) + ")";

    ret hhtml_agiBlue(hhead_title(htmldecode_dropAllTags(title))

      + hbody(hfullcenter(//top() +

        h3(title)

      + p_nemptyLines_showFirst(searchResultsToShow, map(pageToHTMLLink(), searchResults)))));

  }

  O serveQueryPage() {

    S query = params.get('query);

    if (query == null) query = loadSnippet(#1024258);

    S title = agiBlueNameHTML() + " | Execute a query script (" + targetBlank("http://code.botcompany.de/1024274", "ALQL") + ")";

    ret hhtml_miniPage(title, h3(title)

 + form(

      htextarea(query, name := 'query, cols := 80, rows := 10, autofocus := true)

      + "<br><br>"

      + hsubmit("Execute"), action := "/bot/query"

    ));

  }

  O hhtml_miniPage(S htmlTitle, O contents) {

    ret hhtml_agiBlue(hhead_title(htmldecode_dropAllTags(htmlTitle))

      + hbody(hfullcenter(contents)));

  }

  S sliceHomeURL() {

    ret slice == null ? agiBlueURL() : sliceHomeURL(slice.sliceConcept.globalID);

  }

  S sliceHomeURL(Slice slice) { ret sliceHomeURL(slice.globalID); }

  S sliceHomeURL(GlobalID slice) {

    ret agiBlueURL() + hquery(+slice);

  }

  S sliceLinkHTML(Slice slice) {

    ret slice == null ? "-": ahref(sliceHomeURL(slice), htmlEncode2(slice.name));

  }

  O serveCreateSlicePage() {

    S sliceName = trim(params.get('sliceName));

    if (eq(params.get('doIt), "1") && nempty(sliceName)) {

      // TODO: check for existing name

      Slice slice = createSlice(sliceName);

      ret hrefresh(sliceHomeURL(slice.globalID));

    }

    S title = agiBlueNameHTML() + " | Create slice";

    ret hhtml_agiBlue(hhead_title(htmldecode_dropAllTags(title))

      + hbody(hfullcenter(

        h3(title)

      + form(

          hhidden(doIt := 1)

        + "Slice name: " + htextinput(+sliceName, autofocus := true)

        + "<br><br>"

        + hsubmit("Create slice"))

      )));    

  }

  O serveDeletePage() {

    q = params.get('q);

    Page page = findPageFromQ(q);

    if (page == null) ret "Page " + quote(q) + " not found in slice " + htmlEncode(slice.name());

    bool ownage = slice.sliceConcept.owner! == session.user!;

    if (!ownage) ret "Sorry, you don't own slice " + htmlEncode(slice.name());

    deleteConcepts(findBackRefs(page, AbstractEntry));

    cdelete(page);

    ret hrefresh(1.0, sliceHomeURL()) + "Page deleted";

  }

  F1<Page, S> pageToHTMLLink(O... _) {

    optPar int displayLength = main.displayLength;

    ret func(Page p) -> S {

      S name = pageDisplayName(p);

      ret ahref(agiBlue_pageURLWithSlice(p),

        htmlEncode2(shorten(displayLength, name)),

        title := name);

    };

  }

  // and google log-in

  S sliceSelector() {

    ret htag table(tr(

      (!showGoogleLogIn ? "" : td(

        googleSignIn_signInButton(agiBlueURL() + "/google-verify", "console.log(data);", ""), style := "padding-right: 10px;")

        + td(nobr(ahref("javascript:signOut()", "Sign out")), style := "padding-right: 10px; padding-bottom: 0.15em"))

    + (!showSliceSelector ? "" : hform(td(

        "Select reality slice: "

      + hselect(availableSlices(), session.selectedSlice, name := 'slice, onchange := "this.form.submit()")

      /*+ " " + hsubmit("Go")*/

      + "   | " +

      ahref(agiBlueURL("/slices"), nSlices(countConcepts(Slice))) + " | " +

      ahref(agiBlueURL() + "/createSlice", "Create slice...")

    )))));

  }

  S sliceAsHTML() {

    if (slice.isMainSlice()) ret htmlEncode2(agiBlueName() + "'s main slice");

    if (slice.sliceConcept == null) ret "Slice ???";

    ret htmlEncode2("Slice " + quote(slice.sliceConcept.name));

  }

  Slice sliceConcept() { ret slice.sliceConcept; }

  S footer() {

    ret p(small(elapsedMS_sysNow(started) + " ms"));

  }

  S verifiedEmail() {

    if (session == null || !(session.user! instanceof GoogleUser)) null;

    ret ((GoogleUser) session.user!).googleEmail;

  }

  bool authed() {

    bool ver = eqic(verifiedEmail(), "stefan.reich.maker.of.eye@googlemail.com");

    print("Auth check " + verifiedEmail() + " -> " + ver);

    ret ver;

  }

  S googleSignInID() {

    ret eqic(domain(), "botcompany.de") ? botCompanyGoogleSignInID() : agiBlueGoogleSignInID();

  }

  S hhtml_agiBlue(S contents) {

    ret hhtml(hAddToHead_fast(contents, 

      hIncludeGoogleFont("Source Sans Pro")

      //+ [[<meta name="google-site-verification" content="oRvyeTqgSuv-FE_c-2UKM1Vp0oDqx8h9WruHYWqA-NQ" />]]

      + loadJQuery()

      + hmobilefix()

      + googleSignIn_header("", googleSignInID())

      + hstylesheet("body { font-family: Source Sans Pro }")));

  }

  O serveSlicesList() {

    S sort = getAny(params, 'sort, 'by);

    L<Slice> slices;

    S shown;

    if (eqic(sort, 'name)) {

      slices = asList(idx_slicesByName.objectIterator());

      shown = "sorted by name";

    } else {

      sort = 'date;

      slices = asList(idx_slicesByModification.objectIterator());

      shown = "last modified first";

    }

    S title = agiBlueNameHTML() + " has " + nSlices(slices) /*+ " (" + shown + ")"*/;

    new PreIncLongCounter counter;

    ret hhtml_agiBlue(hhead_title_decode(title)

      + hbody(hfullcenter(

        h1(title)

      + p(joinWithSpacedVBar(

        ahrefIf(neqic(sort, 'name), agiBlueURL() + "/slices" + hquery(sort := 'name), "List by name"),

        ahrefIf(neqic(sort, 'date), agiBlueURL() + "/slices" + hquery(sort := 'date), "List by date"),

        ahref(agiBlueURL() + "/createSlice", "Create slice...")))

      + htmlTable2(map(slices, slice ->

        litorderedmap(

          "Name" := ahref(sliceHomeURL(slice), htmlEncode2(slice.name)),

          "ID" := ahref(sliceHomeURL(slice), str(slice.globalID)),

          "Owner" := str(slice.owner!)

        )),

        htmlEncode := false)

      )));

  }

  O serveUsersList() {

    S sort = getAny(params, 'sort, 'by);

    L<User> users;

    S shown;

    if (eqic(sort, 'name))

      users = asList(idx_usersByName.objectIterator());

    else {

      sort = 'seen;

      users= asList(idx_usersBySeen.objectIterator());

    }

    S title = agiBlueNameHTML() + " has " + nUsers(users);

    new PreIncLongCounter counter;

    ret hhtml_agiBlue(hhead_title_decode(title)

      + hbody(hfullcenter(

        h1(title)

      + p(joinWithSpacedVBar(

        ahrefIf(neqic(sort, 'name), agiBlueURL() + "/users" + hquery(sort := 'name), "List by name"),

        ahrefIf(neqic(sort, 'seen), agiBlueURL() + "/users" + hquery(sort := 'seen), "List by last seen"),

        ahref(agiBlueURL() + "/createSlice", "Create slice...")))

      + htmlTable2(map(users, user ->

        litorderedmap(

          "Name" := ahref(userHomeURL(user), htmlEncode2(str(user))),

          "Last seen" := renderHowLongAgo(user.lastSeen)

        )),

        htmlEncode := false)

      )));

  }

  S sliceInfoHTML() {

    User owner = slice.sliceConcept.owner!;

    ret owner == null ? "" : p("This slice is owned by " + userHTML(owner));

  }

  S userHomeURL(User user) {

    ret user == null ? null : agiBlueURL() + "/user/" + user.globalID;

  }

  S userHTML(User user) {

    ret user == null ? "nobody" : ahref(userHomeURL(user), htmlEncode2(str(user)));

  }

  S top() {

    ret nempty(get) ? "" : hcomment("cookie: " + takeFirst(4, session.cookie))

      + hSilentComputatorWithFlag("agi.blue: " + q)

      + p(ahref(sliceHomeURL(),

        //hsnippetimg(#1101682, width := 565/5, height := 800/5, title := "Robot by Peerro @ DeviantArt")

        //hsnippetimg(#1101778, width := 96, height := 96, title := "agi.blue - a database for everything")

        hsnippetimg(#1101822, width := 314, height := 125, title := "agi.blue - Wikipedia for robots")

      ))

      + sliceInfoHTML()

      + p(small(

          agiBlueNameHTML_boldWithSize()

        + (agiBlue_isOriginal() ? "" : " " + targetBlank("http://agi.blue", "[original]"))

        + " | " + ahref(agiBlueURL("/slices"), nSlices(countConcepts(Slice)))

        + " | " + targetBlank(progLink(), "source code") + " of this web site (" + nLines(sourceCodeLines) + ") | " + targetBlank("https://gitter.im/agi-blue/community", "sponsor https?") + " | by " + targetBlank("https://BotCompany.de", "BC") + " | " + targetBlank("http://fiverr.tinybrain.de/", "Fiverr") + " | " + targetBlank("https://discordapp.com/invite/SEAjPqk", "Discord") + " | " + targetBlank("https://www.youtube.com/watch?v=b6jtRdV3Ev8", "Video")

        + " | " + targetBlank("http://code.botcompany.de/1024233", "Notes")

        + " | " + ahref(agiBlueURL() + "/query", "Query")

      ));

  }

  Slice createSlice(S name) {

    name = shorten(name, maxSliceNameLength);

    Slice slice = cnew Slice(+name, owner := session.user);

    cset(slice, caseID := slice.defaultCaseID());

    loadSlice(slice).initialSetup(slice.globalID);

    ret slice;

  }

  O serveCheckboxes() {

    set conceptsSortedByFieldCI_verbose;

    Cl<Page> pages = conceptsSortedByFieldCI(slice.cc, Page, 'q);

    S title = "Select pages";

    ret hhtml_miniPage(title, h3(title) + hpostform(

      htmlTable2(map(pages, page -> litorderedmap(

        "Page" := hcheckbox("page_" + page.globalID) + " " + pageToHTMLLink().get(page)

      )), 

      htmlEncode := false)

      + h3("Add an entry to all selected pages")

        + p("Key: " + hinputfield("key") +

          " Value: " + hinputfield("value"))

      + p(hsubmit("Add entries")), action := agiBlueURL() + "/multiAdd"));

  }

  O serveMultiAdd() {

    S key = trim(params.get('key)), value = trim(params.get('value));

    if (empty(key) || empty(value)) ret "Need key and value";

    new Matches m;

    new L<Page> pages;

    for (S a, b : params)

      if (startsWith(a, "page_", m))

        addIfNotNull(pages, conceptWhere(cc, Page, globalID := m.rest()));

    if (empty(pages)) ret "No pages selected";

    for (Page page : pages)

      new GetEntriesAndPost(cc).go(page, params);

    ret "Entry added to " + nPages(pages);

  }

} // end of Request

svoid dbLog(O... params) {

  logStructure(programFile("db.log"), ll(params));

}

static IF1<Page, Map> pageToMap(O... _) {

  optPar bool withEntries;

  bool nameOnly = eqOneOf(optPar nameOnly(_), "1", true);

  if (nameOnly) ret (IF1<Page, Map>) p -> litmap(q := p.q);

  ret (IF1<Page, Map>) p -> {

    L<Entry> entries = findBackRefs(p, Entry);

    ret litorderedmap(

      q := p.q,

      nEntries := l(entries),

      created := p.created,

      modified := p._modified,

      entries := !withEntries ? null : map(entries, entryToMap(false)));

  };

}

static IF1<Entry, Map> entryToMap(bool withPage) {

  ret (IF1<Entry, Map>) e -> litorderedmap(

    created := e.created,

    i := e.count,

    key := e.key,

    value := e.value,

    q := withPage ? e.page->q : null,

    signer := getString globalID(e.signer!));

}

static IF1<Slice, Map> sliceToMap() {

  ret (IF1<Slice, Map>) s -> litorderedmap(

    created := s.created,

    globalID := str(s.globalID),

    name := s.name,

    caseID := s.caseID);

}

sO servePageToBot(Page page, SS params) {

  if (page == null) ret serveJSON(null);

  params = asCIMap(params);

  Map map = pageToMap(withEntries := valueIs1 withEntries(params)).get(page);

  ret serveJSON(map);

}

sclass GetEntriesAndPost {

  Concepts cc;

  L<Entry> entries;

  Entry entry;

  bool newEntry;

  Signer signer;

  *(Concepts *cc) {}

  GetEntriesAndPost go(Page page, SS params) {

    S key = trim(params.get('key)), value = trim(params.get('value));

    print("GetEntriesAndPost: " + quote(page.q) + ", " + quote(key) + " := " + quote(value));

    withDBLock {

      entries = findBackRefs(page, Entry);

      if (nempty(key) && nempty(value)) {

        S ip = subBot_clientIP();

        entry = firstThat(e -> eqic(e.key, key) && eq_icIf(!allowMultipleCasesInValues, e.value, value) && eq(e.ip, ip), entries);

        if (entry == null) {

          print("SAVING");

          Entry e = cnew(cc, Entry, +page, +key, +value, +ip, count := l(entries) + 1, +signer);

          if (makeAllKeysPages) pageFromQ(cc, key);

          if (makeAllValuesPages) pageFromQ(cc, value);

          page.change(); // bump modification date

          entry = e;

          newEntry = true;

          entries.add(entry);

          dbLog("New entry", page := page.q, globalID := e.globalID, count := e.count, +key, +value);

        }

      }

    }

    sortByFieldInPlace created(entries);

    numberEntriesInConceptField count(entries);

    this;

  }

}

static SS availableSlices() {

  L<Slice> slices = asList(idx_slicesByName.objectIterator());

  ret mapToOrderedMap(s -> pair(str(s.globalID), s.name + " [ID: " + s.globalID + "]"), slices);

}

static L<Entry> entriesOnPage(Page p) {

  ret p == null ? null : sortedByField count(findBackRefs(p, Entry));

}

static L<Page> pagesSortedByLength(L<Page> l) {

  ret sortedByCalculatedField(l, p -> l(p.q));

}

sbool agiBlue_isOriginal() {

  ret amProgram(#1023558);

}

sS agiBlueURL(S subUri) { ret agiBlueURL() + prependSlash(subUri); }

sS agiBlueURL() {

  ret agiBlue_isOriginal() ? "https://agi.blue" :  "/" + psI(programID()) + "/raw";

}

sS agiBlueName() {

  ret agiBlue_isOriginal() ? "agi.blue" : "agi.blue clone " + programID();

}

svoid cleanMeUp {

  cleanUp(fan);

}

static LoadedSlice loadSlice(Slice slice) {

  ret loadSlice(slice.caseID, str(slice.globalID));

}

static LoadedSlice loadSlice(S caseID, S globalID) {

  caseID = unnull(caseID);

  lock fan.lock;

  // move slice from background to foreground

  Concepts cc = backgroundFan.getIfLoaded(caseID);

  if (cc != null) {

    LoadedSlice slice = assertNotNull(backgroundSlices.get(caseID));

    backgroundFan.loaded.remove(caseID);

    fan.loaded.put(caseID, cc);

    backgroundSlices.remove(caseID);

    loadedSlices.put(caseID, slice);

    ret slice;

  }

  fan.get(caseID, globalID);

  ret loadedSlices.get(caseID);

}

// load background slice

static LoadedSlice loadBackgroundSlice(Slice slice) {

  ret loadBackgroundSlice(slice.caseID, str(slice.globalID));

}

static LoadedSlice loadBackgroundSlice(S caseID, S globalID) {

  caseID = unnull(caseID);

  lock fan.lock;

  // check if in foreground

  if (fan.getIfLoaded(caseID) != null)

    ret loadedSlices.get(caseID);

  backgroundFan.get(caseID, globalID);

  LoadedSlice slice = assertNotNull(backgroundSlices.get(caseID));

  slice.lastAccess = sysNow();

  while (l(backgroundSlices) > maxBackgroundSlices)

    unloadLeastRecentlyAccessedBackgroundSlice();

  assertNotNull("slice.sliceConcept", slice.sliceConcept);

  ret slice;

}

// TODO: sync unloading slice with accessors

svoid unloadLeastRecentlyAccessedBackgroundSlice() {

  lock fan.lock;

  S caseID = lowestByField lastAccess(keys(backgroundSlices));

  if (caseID != null)