Not logged in.  Login/Logout/Register | List snippets | | Create snippet | Upload image | Upload data

1206
LINES

< > BotCompany Repo | #1024350 // agi.blue source with central index [dev.]

JavaX module (desktop) [tags: use-pretranspiled]

Download Jar. Uses 1658K of libraries. Click here for Pure Java version (18927L/138K).

1  
!7
2  
3  
// See #1023660 for the older 99 lines version
4  
5  
// Customizations:
6  
7  
sbool asyncSearch = false;
8  
sbool allowMultipleCasesInValues = true;
9  
sbool showSliceSelector = true;
10  
sbool showGoogleLogIn = true;
11  
sbool makeAllValuesPages = true; 
12  
sbool legacy = true; // We still have slices without applied fixes
13  
14  
static int maxSliceNameLength = 1024;
15  
static int displayLength = 140;
16  
static int sideDisplayLength = 50; // when in side bar
17  
static int searchResultsToShow = 50;
18  
sS sideSearchType = 'literal;
19  
static int maxBackgroundSlices = 1000;
20  
21  
// end of customizations
22  
23  
// in main DB
24  
concept Slice {
25  
  GlobalID globalID = aGlobalIDObject();
26  
  S caseID;
27  
  S name;
28  
  long sliceDumped; // wall time, set before dump starts
29  
  bool indexed = true;
30  
  
31  
  S defaultCaseID() {
32  
    ret uniqueFileNameUsingMD5_80_v2(name + " " + globalID);
33  
  }
34  
}
35  
36  
// in any DB
37  
concept Page {
38  
  S globalID = aGlobalID();
39  
  S url, q;
40  
}
41  
42  
abstract concept AbstractEntry {
43  
  new Ref<Page> page;
44  
  int count;
45  
  S key;
46  
  S ip;
47  
  new Ref<Signer> signer;
48  
}
49  
50  
concept Entry > AbstractEntry {
51  
  S globalID = aGlobalID(); // TODO: migrate to object
52  
  S value;
53  
}
54  
55  
concept MultiLineEntry > AbstractEntry {
56  
  GlobalID globalID = aGlobalIDObject();
57  
  S value;
58  
}
59  
60  
concept BlobEntry > AbstractEntry {
61  
  GlobalID globalID = aGlobalIDObject();
62  
  long length;
63  
  S md5;
64  
}
65  
66  
// in main DB?
67  
concept Signer {
68  
  S globalID = aGlobalID();
69  
  S publicKey;
70  
  bool trusted;
71  
  S approvedBy;
72  
}
73  
74  
// in main DB
75  
concept Session {
76  
  S cookie;
77  
  S selectedSlice; // global ID
78  
  Page slicePage; // phasing out
79  
  
80  
  long googleLogInDate;
81  
  S googleEmail, googleFirstName, googleLastName;
82  
  bool googleEmailVerified;
83  
}
84  
85  
// singleton concept in every DB
86  
concept SliceInfo {
87  
  GlobalID globalID;
88  
  LS mandatoryBotVMBusIDs;
89  
}
90  
91  
sclass CentralIndexEntry {
92  
  Set<Slice> slices = synchroLinkedHashSet();
93  
}
94  
95  
// global constants
96  
97  
static SS searchTypeToText = litmap(
98  
  leven := "Leven",
99  
  literal := "Literal",
100  
  scored := "Scored");
101  
  
102  
// global vars
103  
104  
static int sourceCodeLines;
105  
static ConceptsLoadedOnDemand fan;
106  
static ConceptsLoadedOnDemand backgroundFan;
107  
static Map<S, LoadedSlice> loadedSlices = syncMap();
108  
static Map<S, LoadedSlice> backgroundSlices = syncMap();
109  
static Map<S, CentralIndexEntry> centralIndex = ciMap();
110  
static long centralIndexMade; // wall time
111  
112  
// end of global vars
113  
114  
sclass LoadedSlice { // non-persistent class
115  
  S caseID;
116  
  Concepts cc;
117  
  Slice sliceConcept;
118  
  long lastAccess = sysNow();
119  
  
120  
  ConceptFieldIndexDesc idx_latestEntries, idx_latestCreatedPages, idx_latestChangedPages;
121  
  
122  
  *(S *caseID, Concepts *cc) {
123  
    indexThings();
124  
    
125  
    // forward changes to Slice concept
126  
    onConceptsChange(cc, r {
127  
      cset(sliceConcept, _modified := now())
128  
    });
129  
    
130  
    if (makeAllValuesPages && legacy)
131  
      for (Entry e : list(cc, Entry))
132  
        pageFromQ(e.value);
133  
  }
134  
  
135  
  void initialSetup(GlobalID globalID) {
136  
    SliceInfo info = uniq(cc, SliceInfo);
137  
    cset(info, +globalID);
138  
  }
139  
  
140  
  // create if not there
141  
  Page pageFromQ(S q) {
142  
    ret main.pageFromQ(cc, q);
143  
  }
144  
145  
  void indexThings() {
146  
    indexConceptFieldsCI(cc, Page, 'url, Page, 'q, Entry, 'key, Entry, 'value);
147  
    indexConceptFields(cc, Signer, 'publicKey);
148  
    indexConceptFields(cc, Session, 'cookie);
149  
    indexSingletonConcept(cc, SliceInfo);
150  
    idx_latestCreatedPages = new ConceptFieldIndexDesc(cc, Page, 'created);
151  
    idx_latestChangedPages = new ConceptFieldIndexDesc(cc, Page, '_modified);
152  
    idx_latestEntries = new ConceptFieldIndexDesc(cc, Entry, 'created);
153  
  }
154  
  
155  
  bool isMainSlice() { ret empty(caseID); }
156  
  GlobalID globalID() { ret sliceConcept.globalID; }
157  
}
158  
159  
p {
160  
  if (bootstrapDataFrom(#1023558)) deleteMyBackups();
161  
  thinMyBackups();
162  
  fan = slicesLoadedOnDemand(loadedSlices);
163  
  mainConcepts = fan.get("", null);
164  
  indexConceptFields(Slice, 'globalID, Slice, 'caseID);
165  
  cset(uniq_returnIfNew Slice(caseID := ""), name := "main slice");
166  
  loadedSlices.get("").initialSetup(toGlobalIDObj(mainSliceGlobalID()));
167  
  
168  
  backgroundFan = slicesLoadedOnDemand(backgroundSlices);
169  
  backgroundFan.lock = fan.lock;
170  
171  
  // legacy conversions!
172  
  //deleteConcepts(Slice, globalID := GlobalID('wftlawbagrwprywn));
173  
174  
  // Approve this machine's key
175  
  PKIKeyPair machineKey = agiBot_trustedKeyForMachine();
176  
  if (machineKey != null) {
177  
    print("Approving this machine's key: " + machineKey.publicKey);
178  
    cset(uniq_sync(Signer, publicKey := machineKey.publicKey), trusted := true, approvedBy := "local");
179  
  }
180  
  
181  
  sourceCodeLines = countLines(mySource());
182  
  
183  
  thread "Make central index" {
184  
    makeCentralIndex();
185  
  }
186  
}
187  
188  
// Serve page
189  
190  
set flag NoNanoHTTPD. html {
191  
  ret new Request().serve(uri, params);
192  
}
193  
194  
sclass WorksOnSlice {
195  
  Concepts cc;
196  
  LoadedSlice slice;
197  
  
198  
  *() {}
199  
  *(LoadedSlice *slice) { cc = slice.cc; }
200  
  
201  
  void setSlice(LoadedSlice slice) {
202  
    this.slice = slice;
203  
    cc = slice.cc;
204  
  }
205  
  
206  
  Page findPageFromParams(Map map) {
207  
    S q = getString q(map);
208  
    ret empty(q) ? null : findPageFromQ(q);
209  
  }
210  
211  
  Page findOrMakePageFromParams(Map map) {
212  
    ret pageFromQ(getString q(map));
213  
  }
214  
  
215  
  Page findPageFromQ(S q) {
216  
    ret conceptWhereCI(cc, Page, +q);
217  
  }
218  
  
219  
  Page pageFromQ(S q) {
220  
    ret slice.pageFromQ(q);
221  
  }
222  
  
223  
  Set<Page> pagesForKeyAndValue(S key, S value) {
224  
    L<Entry> entries = conceptsWhereIC(cc, Entry, +value, +key);
225  
    ret asSet(ccollect page(entries));
226  
  }
227  
  
228  
  Cl<S> keysForPageAndValue(S q, S value) {
229  
    Page page = findPageFromQ(q);
230  
    if (page == null) null;
231  
    ret collect key(conceptsWhereIC(cc, Entry, +page, +value));
232  
  }
233  
234  
  // DB functions
235  
236  
  bool hasPage(S q) { ret hasConceptWhereIC(Page, +q); }
237  
  S getValue(Page page, S key) {
238  
    ret page == null || empty(key) ? null : getString value(highestByField count(objectsWhereIC(findBackRefs(page, Entry), +key)));
239  
  }
240  
  S getValue(S page, S key) {
241  
    ret getValue(findPageFromQ(page), key);
242  
  }
243  
  S pageDisplayName(Page page) {
244  
    /*S name = getValue(page, "read as");
245  
    bool unnaturalName = nempty(name) && !eq(makeAGIDomain(name), page.url);
246  
    ret unnaturalName ? name + " " + squareBracketed(page.url) : or2(name, unpackAGIDomainOpt(page.url));*/
247  
    ret page.q;
248  
  }
249  
  
250  
  S renderThing(S s, bool forceURLDisplay) {
251  
    ret forceURLDisplay || isURL(s) || isAGIDomain(s)
252  
      ? ahref(fixAGILink(absoluteURL(s)), htmlencode2(shorten(displayLength, s)))
253  
      : ahref(agiBlue_linkForPhrase(s,
254  
          slice := slice.isMainSlice() ? null : slice.sliceConcept.globalID),
255  
        htmlencode2(shorten(displayLength, s)));
256  
  }
257  
258  
} // end of WorksOnSlice
259  
260  
sclass Request extends WorksOnSlice {
261  
  S cookie;
262  
  Session session;
263  
  S uri;
264  
  SS params;
265  
  long started = sysNow();
266  
  
267  
  // should also work for standalone
268  
  bool isHomeDomain() {
269  
    S domain = domain();
270  
    ret eqic(domain, "www.agi.blue") || !ewic(domain, ".agi.blue");
271  
  }
272  
  
273  
  O serve(S uri, SS params) {
274  
    this.uri = dropMultipleSlashes(uri);
275  
    this.params = params;
276  
    new Matches m;
277  
    
278  
    print(uri + " ? " + unnull(subBot_query()));
279  
    
280  
    // get cookie & session
281  
    
282  
    cookie = cookieFromUser();
283  
    if (cookie != null)
284  
      session = uniq_sync(Session, +cookie);
285  
    else
286  
      session = unlistedWithValues(Session);
287  
      
288  
    // check if user wants to change slices
289  
      
290  
    S selectSlice = params.get('slice);
291  
    if (isGlobalID(selectSlice))
292  
      cset(session, selectedSlice := selectSlice);
293  
    if (session.selectedSlice == null)
294  
      cset(session, selectedSlice := str(mainSliceGlobalID()));
295  
296  
    // get case ID
297  
    
298  
    S caseID = "";
299  
    Slice sliceConcept = sliceConceptForGlobalID(session.selectedSlice);
300  
    print("Selected slice: " + session.selectedSlice + ", obj? " + (sliceConcept != null));
301  
    if (sliceConcept != null) caseID = sliceConcept.caseID;
302  
    print("caseID: " + caseID);
303  
    
304  
    // load slice
305  
    
306  
    slice = assertNotNull(loadSlice(caseID, session.selectedSlice));
307  
    cc = slice.cc;
308  
    slice.sliceConcept = sliceConcept;
309  
      
310  
    // Check for special URIs
311  
312  
    if (swic(uri, "/bot/")) ret serveBot();
313  
    
314  
    // eleu appends a slash to the URI if it's a single identifier, so we drop it again
315  
    S uri2 = dropTrailingSlash(uri);
316  
    
317  
    if (eqic(uri2, "/google-verify")) {
318  
      print("Google-verify started.");
319  
      Payload payload = printStruct(googleVerifyUserToken2(googleSignInID(), params.get("token")));
320  
      if (payload == null) ret print("google-verify", "No");
321  
      cset(session,
322  
        googleLogInDate := now(),
323  
        googleEmail := payload.getEmail(),
324  
        googleEmailVerified := payload.getEmailVerified(),
325  
        googleFirstName := strOrNull(payload.get("given_name")),
326  
        googleLastName := strOrNull(payload.get("family_name")));
327  
        
328  
      ret print("google-verify", payload.getEmail() + " " + (payload.getEmailVerified() ? "(verified)" : "(not verified)"));
329  
    }
330  
    
331  
    if (eqic(uri2, "/search")) ret serveScoredSearch();
332  
    if (eqic(uri2, "/literalSearch")) ret serveLiteralSearch();
333  
    if (eqic(uri2, "/levenSearch")) ret serveLevenSearch();
334  
    
335  
    if (eqic(uri2, "/query")) ret serveQueryPage();
336  
    if (eqic(uri2, "/createSlice")) ret serveCreateSlicePage();
337  
    
338  
    S q = params.get('q);
339  
    S domain = or2(params.get('domain), domain());
340  
    
341  
    S raw = firstKeyWithValue("", params); // agi.blue?something
342  
    if (nempty(raw) && empty(q)) q = raw;
343  
    /*if (nempty(q)) {
344  
      domain = makeAGIDomain(q);
345  
      if (l(domain) > maximumDomainPartLength()) // escape with "domain="
346  
        ret hrefresh(agiBlueURL() + hquery(+domain, key := "read as", value := q));
347  
      ret hrefresh("http://" + domain + (eq(q, domain) ? "" : "/" +  hquery(key := "read as", value := q)));
348  
      //uri = "/"; replaceMapWithParams(params, key := "read as", value := q);
349  
    }*/
350  
    S url = domain + dropTrailingSlash(uri);
351  
    
352  
    // domain to query
353  
    //if (empty(q)) q = url;
354  
    if (empty(q)) q = agiBlue_urlToQuery(url);
355  
    
356  
    Page page, bool newPage = unpair uniqCI2_sync(cc, Page, +q);
357  
    if (newPage) dbLog("New page", +q);
358  
    //printStructs(+params, +raw, +q);
359  
  
360  
    Set<S> get = asCISet(nempties(subBot_paramsAsMultiMap().get('get)));
361  
    
362  
    S top = nempty(get) ? "" : hcomment("cookie: " + takeFirst(4, session.cookie))
363  
      + hSilentComputatorWithFlag("agi.blue: " + q)
364  
      + p(ahref(sliceHomeURL(),
365  
        //hsnippetimg(#1101682, width := 565/5, height := 800/5, title := "Robot by Peerro @ DeviantArt")
366  
        //hsnippetimg(#1101778, width := 96, height := 96, title := "agi.blue - a database for everything")
367  
        hsnippetimg(#1101822, width := 314, height := 125, title := "agi.blue - Wikipedia for robots")
368  
      ))
369  
      + p(small(
370  
          b(agiBlueNameHTML())
371  
        + (agiBlue_isOriginal() ? "" : " " + targetBlank("http://agi.blue", "[original]"))
372  
        + " | " + targetBlank(progLink(), "source code") + " of this web site (" + nLines(sourceCodeLines) + ") | " + targetBlank("https://gitter.im/agi-blue/community", "sponsor https?") + " | by " + targetBlank("https://BotCompany.de", "BC") + " | " + targetBlank("http://fiverr.tinybrain.de/", "Fiverr") + " | " + targetBlank("https://discordapp.com/invite/SEAjPqk", "Discord") + " | " + targetBlank("https://www.youtube.com/watch?v=b6jtRdV3Ev8", "Video")
373  
        + " | " + targetBlank("http://code.botcompany.de/1024233", "Notes")
374  
        + " | " + ahref(agiBlueURL() + "/query", "Query")
375  
      ));
376  
    
377  
    if (empty(params.get('q)) && empty(raw) && isHomeDomain()) {
378  
      L<Page> pages = sortedByFieldDesc _modified(list(cc, Page)); // TODO: use index
379  
      int start = parseInt(params.get("start")), step = 100;
380  
      S nav = pageNav2("/", l(pages), start, step, "start");
381  
      S content = hform(b("GIVE ME INPUT: &nbsp; ") + htextinput('q, autofocus := true) + " " + hsubmit("Ask"))
382  
        + h1(sliceAsHTML() + " has " + nPages(countConcepts(cc, Page)) + " and " + nConnections(countConcepts(cc, Entry)))
383  
        + p(nav)
384  
        + p_nemptyLines(map(pageToHTMLLink(), subList(pages, start, start+step)));
385  
386  
      ret hhtml_agiBlue(hhead_title("Slice " + sliceConcept().name + " | agi.blue") // SERVE SLICE HOME PAGE
387  
        + hbody(hfullcenterAndTopLeft(top + content
388  
          + footer(),
389  
          sliceSelector()
390  
        )));
391  
    }
392  
        
393  
    S key = trim(params.get('key)), value = trim(params.get('value));
394  
    L<Entry> entries = GetEntriesAndPost(cc).go(page, params).entries;
395  
    
396  
    //S get = params.get('get);
397  
    if (nempty(get))
398  
      ret serveJSON(collect value(llNotNulls(firstThat(entries, e -> get.contains(e.key)))));
399  
    
400  
    S key2 = key, value2 = value; if (nempty(key) && nempty(value)) key2 = value2 = ""; // input reset
401  
  
402  
    bool withHidden = eq(params.get('withHidden), "1");
403  
    new Set<Int> hide;
404  
    if (!withHidden) for (Entry e : entries) if (eqic(e.key, 'hide) && isSquareBracketedInt(e.value)) addAll(hide, e.count, parseInt(unSquareBracket(e.value)));
405  
    new MultiMap<Int, S> mmMeta;
406  
    for (Entry e : entries) if (isSquareBracketedInt(e.key)) mmMeta.put(parseInt(unSquareBracket(e.key)), e.value);
407  
    new MultiMap<S> mm;
408  
    for (Entry e : entries) mm.put(e.key, e.value);
409  
    
410  
    //S name = or2(/* ouch */ last(mm.get("read as")), /* end ouch */ unpackAGIDomain(page.url), page.url);
411  
    S name = page.q;
412  
    
413  
    // Find references
414  
    
415  
    L<Entry> refs = concatLists(conceptsWhereIC(cc, Entry, value := name),
416  
      conceptsWhereIC(cc, Entry, key := name));
417  
    Set<Page> refPages = asSet(ccollect page(refs));
418  
    refPages.remove(page); // don't list current page as reference
419  
    
420  
    // Search in page names (depending on default search type)
421  
    
422  
    L<Page> searchResults;
423  
    if (eq(sideSearchType, 'leven))
424  
      searchResults = levenSearch(page.q, max := 50);
425  
    else if (eq(sideSearchType, 'literal))
426  
      searchResults = literalSearch(page.q, max := 50);
427  
    else
428  
      searchResults = (L<Page>) dm_call('agiBlueSearch, 'search, page.q, maxResult := searchResultsToShow+1, agiBlueBotID := programID(), +cc);
429  
    searchResults.remove(page);
430  
431  
    S mainContents =
432  
        top + h1(ahref_unstyled(agiBlue_pageURLWithSlice(page), htmlEncode2(shorten(displayLength, name))) + (newPage ? " [huh????]" : ""))
433  
      + p_nemptyLines(map(entries, func(Entry e) -> S {
434  
        !withHidden && (hide.contains(e.count) || eqic(e.key, "read as") && eqic(e.value, name)) ? ""
435  
        : "[" + e.count + "] " +
436  
          renderThing(e.key, false) + ": " +
437  
          b(renderThing(e.value, cic(mmMeta.get(e.count), "is a URL")))
438  
        }))
439  
      + hpostform(h3("Add an entry")
440  
        + "Key: " + hinputfield(key := key2) + " Value: " + hinputfield(value := value2) + "<br><br>" + hsubmit("Add")
441  
        )
442  
        
443  
      + p(ahref(agiBlueURL() + "/literalSearch" + hquery(q := page.q), "[literal search]", title := "Search pages with a name containing this page's name literally")
444  
       + " " +
445  
       ahref(agiBlueURL() + "/levenSearch" + hquery(q := page.q), "[leven search 1]", title := "Search pages with a Levenshtein similarity of 1 containing this page's name literally")
446  
       + " " +
447  
       ahref(agiBlueURL() + "/search" + hquery(q := page.q), "[scored search]", title := "Search pages with ScoredSearch"));
448  
        
449  
    S sideContents =
450  
      hform(b("GIVE ME INPUT:") + " "
451  
        + htextinput('q) + " "
452  
        + hsubmit("Ask", onclick := "document.getElementById('newInputForm').target = '';") + " "
453  
        + hsubmit("+Tab", title := "Ask and show result in a new tab", onclick := "document.getElementById('newInputForm').target = '_blank';"),
454  
        id := 'newInputForm)
455  
      
456  
      + h3("References (" + l(refPages) + ")")
457  
      
458  
      + p_nemptyLines_showFirst(10, map(pageToHTMLLink(displayLength := sideDisplayLength), refPages))
459  
      
460  
      + h3(searchTypeToText.get(sideSearchType) + " search results (" + (l(searchResults) >= searchResultsToShow ? searchResultsToShow + "+" : str(l(searchResults))) + ")")
461  
      
462  
      + p_nemptyLines_showFirst(searchResultsToShow, map(pageToHTMLLink(displayLength := sideDisplayLength), searchResults))
463  
      
464  
      + hdiv("", id := 'extraStuff);
465  
      
466  
    // TODO: sync search delivery with WebSocket creation
467  
    if (asyncSearch)
468  
      doLater(6.0, r { dm_call('agiBlueSearch, 'searchAndPost, page.q, agiBlueBotID := programID(), +cc) });
469  
      
470  
    // serve a concept page
471  
    ret hhtml_agiBlue(hhead_title(pageDisplayName(page)) + hbody(
472  
      tag('table,
473  
        tr(
474  
          td(sliceSelector(), valign := 'top)
475  
        + td(sideContents, align := 'right, valign := 'top, rowspan := 2))
476  
      + tr(td(mainContents + footer(), align := 'center, valign := 'top)),
477  
      width := "100%", height := "100%")));
478  
  }
479  
  
480  
  O servePagesToBot(Iterable<Page> pages) {
481  
    ret serveListToBot(map(pageToMap(wrapMapAsParams(params)), pages));
482  
  }
483  
484  
  O serveListToBot(Collection l) {
485  
    if (nempty(params.get('max)))
486  
      l = takeFirst(parseInt(params.get('max)), l);
487  
    ret serveJSON(l);
488  
  }
489  
490  
  // uri starts with "/bot/"
491  
  O serveBot() {
492  
    S q = params.get('q);
493  
    
494  
    if (eqic(uri, "/bot/hello")) ret serveJSON("hello");
495  
    if (eqic(uri, "/bot/hasPage")) ret serveJSON(hasPage(q));
496  
    if (eqic(uri, "/bot/randomPageContaining")) {
497  
      assertNempty(q);
498  
      ret servePageToBot(random(filter(list(cc, Page), p -> cic(p.q, q))), params);
499  
    }
500  
    if (eqic(uri, "/bot/allPages"))
501  
      ret servePagesToBot(list(cc, Page));
502  
    if (eqic(uri, "/bot/allPagesStartingWith")) {
503  
      assertNempty(q);
504  
      ret servePagesToBot(filter(list(cc, Page), p -> swic(p.q, q)));
505  
    }
506  
    if (eqic(uri, "/bot/allPagesEndingWith")) {
507  
      assertNempty(q);
508  
      ret servePagesToBot(filter(list(cc, Page), p -> ewic(p.q, q)));
509  
    }
510  
    if (eqic(uri, "/bot/allPagesContaining")) {
511  
      assertNempty(q);
512  
      ret servePagesToBot(filter(list(cc, Page), p -> cic(p.q, q)));
513  
    }
514  
    if (eqic(uri, "/bot/allPagesContainingRegexp")) {
515  
      assertNempty(q);
516  
      Pattern pat = regexpIC(q);
517  
      ret servePagesToBot(filter(list(cc, Page), p -> regexpFindIC(pat, p.q)));
518  
    }
519  
    
520  
    if (eqicOneOf(uri, "/bot/postSigned", /*"/bot/makePhysicalSlice",*/ "/bot/approveTrustRequest")) {
521  
      S text = rtrim(params.get('text));
522  
      S key = getSignerKey(text);
523  
      if (empty(key)) ret subBot_serve500("Please include your public key");
524  
      if (!isSignedWithKey(text, key)) ret subBot_serve500("Signature didn't verify");
525  
      text = dropLastTwoLines(text); // drop signer + sig line
526  
      
527  
      Signer signer = uniq_sync Signer(publicKey := key);
528  
      
529  
      /*if (eqic(uri, "/bot/makePhysicalSlice")) {
530  
        if (!signer.trusted) ret subBot_serve500("Untrusted signer");
531  
        Page page = findPageFromParams(jsonDecodeMap(text));
532  
        if (page == null) ret subBot_serve500("Page not found");
533  
        ret serveJSON(uniq2_sync(PhysicalSlice, slicePage := page).b ? "Slice made" : "Slice exists");
534  
      }*/
535  
    
536  
      if (eqic(uri, "/bot/postSigned")) {
537  
        new L out;
538  
        for (S line : tlft(text)) {
539  
          SS map = jsonDecodeMap(line);
540  
          GetEntriesAndPost x = new(db_mainConcepts());
541  
          x.signer = signer;
542  
          Page page = findOrMakePageFromParams(map);
543  
          if (page == null) continue with out.add("Invalid page reference");
544  
          x.go(page, map);
545  
          out.add(x.newEntry ? "Saved" : x.entry != null ? "Entry exists" : "Need key and value");
546  
        }
547  
        ret serveJSON(out);
548  
      }
549  
      
550  
      if (eqic(uri, "/bot/approveTrustRequest")) {
551  
        if (!signer.trusted) ret subBot_serve500("Untrusted signer");
552  
        Signer toApprove = conceptWhere Signer(publicKey := trim(text));
553  
        if (toApprove == null) ret subBot_serve500("Signer to approve not found");
554  
        cset(toApprove, trusted := true, approvedBy := signer.globalID);
555  
        ret serveJSON("Approved: " + trim(text));
556  
      }
557  
      
558  
      ret subBot_serve500("CONFUSION");
559  
    }
560  
    
561  
    if (eqic(uri, "/bot/post")) {
562  
      GetEntriesAndPost x = new(db_mainConcepts());
563  
      x.go(pageFromQ(q), params);
564  
      ret serveJSON(x.newEntry ? "Saved" : x.entry != null ? "Entry exists" : "Need key and value");
565  
    }
566  
    
567  
    if (eqic(uri, "/bot/entriesOnPage"))
568  
      ret serveJSON(map(entriesOnPage(findPageFromParams(params)), entryToMap(false)));
569  
      
570  
    if (eqic(uri, "/bot/lookup")) {
571  
      S key = params.get('key);
572  
      if (empty(key)) ret serveJSON("Need key");
573  
      S value = getValue(findPageFromParams(params), key);
574  
      ret serveJSON(empty(value) ? "" : litmap(+value));
575  
    }
576  
  
577  
    if (eqic(uri, "/bot/latestEntries"))
578  
      ret serveJSON(map(takeFirst(10, slice.idx_latestEntries.objectIterator()), entryToMap(true)));
579  
    if (eqic(uri, "/bot/latestPages"))
580  
      ret serveJSON(map(takeFirst(10, slice.idx_latestCreatedPages.objectIterator()), pageToMap()));
581  
    if (eqic(uri, "/bot/latestChangedPages"))
582  
      ret serveJSON(map(takeFirst(10, slice.idx_latestChangedPages.objectIterator()), pageToMap()));
583  
      
584  
    if (eqic(uri, "/bot/googleUsersCount"))
585  
      ret serveJSON(l(setMinus(collectAsCISet googleEmail(list(Session)), "", null)));
586  
      
587  
    if (eqic(uri, "/bot/totalPageCount"))
588  
      ret serveJSON(countConcepts(Page));
589  
    /*if (eqic(uri, "/bot/pageWithoutPhysicalSliceCount"))
590  
      ret serveJSON(countConceptsWhere(Page, slice := null));
591  
    if (eqic(uri, "/bot/physicalSliceCount"))
592  
      ret serveJSON(countConcepts(PhysicalSlice));*/
593  
    if (eqic(uri, "/bot/trustedSignersCount"))
594  
      ret serveJSON(countConcepts(Signer, trusted := true));
595  
      
596  
    if (eqic(uri, "/bot/valueSearch")) {
597  
      S value = params.get('value);
598  
      L<Entry> entries = conceptsWhereIC(cc, Entry, +value);
599  
      ret serveJSON(map(takeFirst(100, entries), entryToMap(true)));
600  
    }
601  
      
602  
    if (eqic(uri, "/bot/keyAndValueSearch")) {
603  
      S key = params.get('key), value = params.get('value);
604  
      Cl<Page> pages = pagesForKeyAndValue(key, value);
605  
      ret servePagesToBot(pages);
606  
    }
607  
    
608  
    if (eqic(uri, "/bot/keyValuePairsByPopularity")) {
609  
      L<PairS> pairs = map(list(Entry), e -> pair(e.key, e.value));
610  
      LPair<S, Int> pairs2 = multiSetTopPairs(ciMultiSet(map pairToUglyStringForCIComparison(pairs)));
611  
      ret serveJSON(map(pairs2, p -> {
612  
        S key, value = unpair pairFromUglyString(p.a);
613  
        ret litorderedmap(n := p.b, +key, +value);
614  
      }));
615  
    }
616  
    
617  
    if (eqic(uri, "/bot/allKeys"))
618  
      ret serveListToBot(distinctCIFieldValuesOfConcepts(cc, Entry, 'key));
619  
      
620  
    if (eqic(uri, "/bot/allKeysByPopularity"))
621  
      ret serveListToBot(mapMultiSetByPopularity(distinctCIFieldValuesOfConcepts_multiSet(cc, Entry, 'key), (key, n) -> litorderedmap(+n, +key)));
622  
      
623  
    if (eqic(uri, "/bot/dbSize"))
624  
      ret serveJSON(l(conceptsFile()));
625  
      
626  
    if (eqic(uri, "/bot/query"))
627  
      ret serveBotQuery();
628  
629  
    if (eqic(uri, "/bot/createSlice")) {
630  
      Slice slice = createSlice(assertNempty(params.get('name)));
631  
      ret serveJSON(litorderedmap(id := str(slice.globalID), name := slice.name));
632  
    }
633  
    
634  
    if (eqic(uri, "/bot/dumpAllSlices") && authed()) {
635  
      new L<Map> out;
636  
      for (Slice slice) try {
637  
        out.add(litorderedmap(slice := slice.caseID, ms := returnTimeFor(() -> dumpSliceToFile(loadBackgroundSlice(slice)))));
638  
      } catch print e {
639  
        out.add(litorderedmap(slice.caseID, error := exceptionToStringShort(e)));
640  
      }
641  
      ret serveJSON(out);
642  
    }
643  
    
644  
    if (eqic(uri, "/bot/dumpSlice")) {
645  
      File f = dumpSliceToFile(slice);
646  
      ret "OK, saved as: " + f2s(f);
647  
    }
648  
    
649  
    if (eqic(uri, "/bot/memStats")) {
650  
      Map fg = cloneMap(loadedSlices);
651  
      Map bg = cloneMap(backgroundSlices);
652  
      ret serveJSON(litcimap(
653  
        error := checkDoubleLoads(),
654  
        centralIndexEntries := l(centralIndex),
655  
        centralIndexMade := renderHowLongAgo(centralIndexMade),
656  
        numLoadedForegroundSlices := l(fg),
657  
        numLoadedBackgroundSlices := l(bg),
658  
        loadedForegroundSlicesEstimatedSize := longSum(map estimatedSliceDataSize(keys(fg))),
659  
        loadedBackgroundSlicesEstimatedSize := longSum(map estimatedSliceDataSize(keys(bg))),
660  
        loadedForegroundSlices := sortedIC(keysList(fg)),
661  
        loadedBackgroundSlices := sortedIC(keysList(bg))));
662  
    }
663  
    
664  
    if (eqic(uri, "/bot/centralIndexGet")) {
665  
      CentralIndexEntry e = centralIndex.get(q);
666  
      ret serveJSON(e == null ? ll() : map(e.slices, sliceToMap()));
667  
    }
668  
    
669  
    // end of bot methods
670  
  
671  
    ret subBot_serve404();
672  
  }
673  
  
674  
  O serveBotQuery() {
675  
    S query = params.get('query);
676  
    ret new Query(slice).process(query);
677  
  }
678  
  
679  
  O serveLiteralSearch() {
680  
    S q = params.get('q);
681  
    L<Page> searchResults = literalSearch(q);
682  
    ret serveSearchResults("literal search" , q, searchResultsToShow, searchResults);
683  
  }
684  
  
685  
  L<Page> literalSearch(S q, O... _) {
686  
    int searchResultsToShow = optPar max(_, 100);
687  
    
688  
    // quick search in random order
689  
    //L<Page> searchResults = takeFirst(searchResultsToShow+1, filterIterator(iterator(list(cc, Page)), p -> cic(p.q, q));
690  
    
691  
    // full search, order by length
692  
    ret takeFirst(searchResultsToShow+1, pagesSortedByLength(filter(list(cc, Page), p -> cic(p.q, q))));
693  
  }
694  
  
695  
  O serveScoredSearch() {
696  
    S q = params.get('q);
697  
    L<Page> searchResults = (L<Page>) dm_call('agiBlueSearch, 'search, q, +cc);
698  
    ret serveSearchResults("scored search" , q, searchResultsToShow, searchResults);
699  
  }
700  
  
701  
  O serveLevenSearch() {
702  
    S q = params.get('q);
703  
    L<Page> searchResults = levenSearch(q);
704  
    ret serveSearchResults("leven search with distance 1" , q, searchResultsToShow, searchResults);
705  
  }
706  
  
707  
  L<Page> levenSearch(S q, O... _) {
708  
    int searchResultsToShow = optPar max(_, 100);
709  
    int maxEditDistance = 1;
710  
    
711  
    new Map<Page, Int> map;
712  
    for (Page p : list(cc, Page)) {
713  
      int distance = leven_limitedIC(q, p.q, maxEditDistance+1);
714  
      if (distance <= maxEditDistance) map.put(p, distance);
715  
    }
716  
    ret takeFirst(searchResultsToShow+1, keysSortedByValue(map));
717  
  }
718  
  
719  
  O serveSearchResults(S searchType, S q, int searchResultsToShow, Collection<Page> searchResults) {
720  
    S title = "agi.blue " + searchType + " for " + htmlEncode2(quote(q)) + " (" + (l(searchResults) >= searchResultsToShow ? searchResultsToShow + "+ results" : nResults(l(searchResults))) + ")";
721  
    
722  
    ret hhtml_agiBlue(hhead_title(htmldecode_dropAllTags(title))
723  
      + hbody(hfullcenter(//top +
724  
        h3(title)
725  
      + p_nemptyLines_showFirst(searchResultsToShow, map(pageToHTMLLink(), searchResults)))));
726  
  }
727  
  
728  
  O serveQueryPage() {
729  
    S query = params.get('query);
730  
    if (query == null) query = loadSnippet(#1024258);
731  
    S title = agiBlueNameHTML() + " | Execute a query script (" + targetBlank("http://code.botcompany.de/1024274", "ALQL") + ")";
732  
    ret hhtml_agiBlue(hhead_title(htmldecode_dropAllTags(title))
733  
      + hbody(hfullcenter(
734  
        h3(title)
735  
      + form(
736  
        htextarea(query, name := 'query, cols := 80, rows := 10, autofocus := true)
737  
        + "<br><br>"
738  
        + hsubmit("Execute"), action := "/bot/query")
739  
      )));    
740  
  }
741  
  
742  
  S sliceHomeURL() {
743  
    ret slice == null ? agiBlueURL() : sliceHomeURL(slice.sliceConcept.globalID);
744  
  }
745  
746  
  S sliceHomeURL(GlobalID slice) {
747  
    ret agiBlueURL() + hquery(+slice);
748  
  }
749  
  
750  
  O serveCreateSlicePage() {
751  
    S sliceName = trim(params.get('sliceName));
752  
    if (eq(params.get('doIt), "1") && nempty(sliceName)) {
753  
      // TODO: check for existing name
754  
      Slice slice = createSlice(sliceName);
755  
      ret hrefresh(sliceHomeURL(slice.globalID));
756  
    }
757  
    
758  
    S title = agiBlueNameHTML() + " | Create slice";
759  
    ret hhtml_agiBlue(hhead_title(htmldecode_dropAllTags(title))
760  
      + hbody(hfullcenter(
761  
        h3(title)
762  
      + form(
763  
          hhidden(doIt := 1)
764  
        + "Slice name: " + htextinput(+sliceName, autofocus := true)
765  
        + "<br><br>"
766  
        + hsubmit("Create slice"))
767  
      )));    
768  
  }
769  
  
770  
  F1<Page, S> pageToHTMLLink(O... _) {
771  
    optPar int displayLength = main.displayLength;
772  
    ret func(Page p) -> S {
773  
      S name = pageDisplayName(p);
774  
      ret ahref(agiBlue_pageURLWithSlice(p),
775  
        htmlEncode2(shorten(displayLength, name)),
776  
        title := name);
777  
    };
778  
  }
779  
  
780  
  // and google log-in
781  
  S sliceSelector() {
782  
    ret htag table(tr(
783  
      (!showGoogleLogIn ? "" : td(
784  
        googleSignIn_signInButton(agiBlueURL() + "/google-verify", "console.log(data);", ""), style := "padding-right: 10px;")
785  
        + td(nobr(ahref("javascript:signOut()", "Sign out")), style := "padding-right: 10px; padding-bottom: 0.15em"))
786  
    
787  
    + (!showSliceSelector ? "" : hform(td(
788  
        "Select reality slice: "
789  
      + hselect(availableSlices(), session.selectedSlice, name := 'slice, onchange := "this.form.submit()")
790  
      /*+ " " + hsubmit("Go")*/
791  
      + " &nbsp; " + ahref(agiBlueURL() + "/createSlice", "Create slice...")
792  
    )))));
793  
  }
794  
  
795  
  S sliceAsHTML() {
796  
    if (slice.isMainSlice()) ret htmlEncode2(agiBlueName() + "'s main slice");
797  
    if (slice.sliceConcept == null) ret "Slice ???";
798  
    ret htmlEncode2("Slice " + quote(slice.sliceConcept.name));
799  
  }
800  
  
801  
  Slice sliceConcept() { ret slice.sliceConcept; }
802  
  
803  
  S footer() {
804  
    ret p(small(elapsedMS_sysNow(started) + " ms");
805  
  }
806  
  
807  
  S verifiedEmail() {
808  
    ret session == null || !session.googleEmailVerified ? null
809  
      : session.googleEmail;
810  
  }
811  
  
812  
  bool authed() {
813  
    bool ver = eqic(verifiedEmail(), "stefan.reich.maker.of.eye@googlemail.com");
814  
    print("Auth check " + verifiedEmail() + " -> " + ver);
815  
    ret ver;
816  
  }
817  
  
818  
  S googleSignInID() {
819  
    ret eqic(domain(), "botcompany.de") ? botCompanyGoogleSignInID() : agiBlueGoogleSignInID();
820  
  }
821  
822  
  S hhtml_agiBlue(S contents) {
823  
    ret hhtml(hAddToHead_fast(contents, 
824  
      hIncludeGoogleFont("Source Sans Pro")
825  
      + [[<meta name="google-site-verification" content="oRvyeTqgSuv-FE_c-2UKM1Vp0oDqx8h9WruHYWqA-NQ" />]]
826  
      + loadJQuery()
827  
      + hmobilefix()
828  
      + googleSignIn_header("", googleSignInID())
829  
      + hstylesheet("body { font-family: Source Sans Pro }")));
830  
  }
831  
832  
} // end of Request
833  
834  
svoid dbLog(O... params) {
835  
  logStructure(programFile("db.log"), ll(params));
836  
}
837  
838  
static IF1<Page, Map> pageToMap(O... _) {
839  
  optPar bool withEntries;
840  
  
841  
  bool nameOnly = eqOneOf(optPar nameOnly(_), "1", true);
842  
  if (nameOnly) ret (IF1<Page, Map>) p -> litmap(q := p.q);
843  
  
844  
  ret (IF1<Page, Map>) p -> {
845  
    L<Entry> entries = findBackRefs(p, Entry);
846  
    ret litorderedmap(
847  
      q := p.q,
848  
      nEntries := l(entries),
849  
      created := p.created,
850  
      modified := p._modified,
851  
      entries := !withEntries ? null : map(entries, entryToMap(false)));
852  
  };
853  
}
854  
855  
static IF1<Entry, Map> entryToMap(bool withPage) {
856  
  ret (IF1<Entry, Map>) e -> litorderedmap(
857  
    created := e.created,
858  
    i := e.count,
859  
    key := e.key,
860  
    value := e.value,
861  
    q := withPage ? e.page->q : null,
862  
    signer := getString globalID(e.signer!));
863  
}
864  
865  
static IF1<Slice, Map> sliceToMap() {
866  
  ret (IF1<Slice, Map>) s -> litorderedmap(
867  
    created := s.created,
868  
    globalID := str(s.globalID),
869  
    name := s.name,
870  
    caseID := s.caseID);
871  
}
872  
873  
sO servePageToBot(Page page, SS params) {
874  
  if (page == null) ret serveJSON(null);
875  
  params = asCIMap(params);
876  
  Map map = pageToMap(withEntries := valueIs1 withEntries(params)).get(page);
877  
  ret serveJSON(map);
878  
}
879  
880  
sclass GetEntriesAndPost {
881  
  Concepts cc;
882  
  L<Entry> entries;
883  
  Entry entry;
884  
  bool newEntry;
885  
  Signer signer;
886  
  
887  
  *(Concepts *cc) {}
888  
  
889  
  GetEntriesAndPost go(Page page, SS params) {
890  
    S key = trim(params.get('key)), value = trim(params.get('value));
891  
    print("GetEntriesAndPost: " + quote(page.q) + ", " + quote(key) + " := " + quote(value));
892  
    withDBLock {
893  
      entries = findBackRefs(page, Entry);
894  
      if (nempty(key) && nempty(value)) {
895  
        S ip = subBot_clientIP();
896  
        entry = firstThat(e -> eqic(e.key, key) && eq_icIf(!allowMultipleCasesInValues, e.value, value) && eq(e.ip, ip), entries);
897  
        if (entry == null) {
898  
          print("SAVING");
899  
          Entry e = cnew(cc, Entry, +page, +key, +value, +ip, count := l(entries) + 1, +signer);
900  
          if (makeAllValuesPages) pageFromQ(cc, value);
901  
          page.change(); // bump modification date
902  
          entry = e;
903  
          newEntry = true;
904  
          entries.add(entry);
905  
          dbLog("New entry", page := page.q, globalID := e.globalID, count := e.count, +key, +value);
906  
        }
907  
      }
908  
    }
909  
910  
    sortByFieldInPlace created(entries);
911  
    numberEntriesInConceptField count(entries);
912  
    this;
913  
  }
914  
}
915  
916  
static SS availableSlices() {
917  
  ret mapToOrderedMap(s -> pair(str(s.globalID), s.name + " [ID: " + s.globalID + "]"), list(Slice));
918  
}
919  
920  
static L<Entry> entriesOnPage(Page p) {
921  
  ret p == null ? null : sortedByField count(findBackRefs(p, Entry));
922  
}
923  
924  
static L<Page> pagesSortedByLength(L<Page> l) {
925  
  ret sortedByCalculatedField(l, p -> l(p.q));
926  
}
927  
928  
sbool agiBlue_isOriginal() {
929  
  ret amProgram(#1023558);
930  
}
931  
932  
sS agiBlueURL() {
933  
  ret agiBlue_isOriginal() ? "https://agi.blue" :  "/" + psI(programID()) + "/raw";
934  
}
935  
936  
sS agiBlueName() {
937  
  ret agiBlue_isOriginal() ? "agi.blue" : "agi.blue clone " + programID();
938  
}
939  
940  
svoid cleanMeUp {
941  
  cleanUp(fan);
942  
}
943  
944  
static LoadedSlice loadSlice(Slice slice) {
945  
  ret loadSlice(slice.caseID, str(slice.globalID));
946  
}
947  
948  
static LoadedSlice loadSlice(S caseID, S globalID) {
949  
  caseID = unnull(caseID);
950  
  lock fan.lock;
951  
  
952  
  // move slice from background to foreground
953  
  
954  
  Concepts cc = backgroundFan.getIfLoaded(caseID);
955  
  if (cc != null) {
956  
    LoadedSlice slice = assertNotNull(backgroundSlices.get(caseID));
957  
    backgroundFan.loaded.remove(caseID);
958  
    fan.loaded.put(caseID, cc);
959  
    backgroundSlices.remove(caseID);
960  
    loadedSlices.put(caseID, slice);
961  
    ret slice;
962  
  }
963  
  
964  
  fan.get(caseID, globalID);
965  
  ret loadedSlices.get(caseID);
966  
}
967  
968  
// load background slice
969  
static LoadedSlice loadBackgroundSlice(Slice slice) {
970  
  ret loadBackgroundSlice(slice.caseID, str(slice.globalID));
971  
}
972  
973  
static LoadedSlice loadBackgroundSlice(S caseID, S globalID) {
974  
  caseID = unnull(caseID);
975  
  lock fan.lock;
976  
  
977  
  // check if in foreground
978  
  
979  
  if (fan.getIfLoaded(caseID) != null)
980  
    ret loadedSlices.get(caseID);
981  
982  
  backgroundFan.get(caseID, globalID);
983  
  LoadedSlice slice = assertNotNull(backgroundSlices.get(caseID));
984  
  slice.lastAccess = sysNow();
985  
  while (l(backgroundSlices) > maxBackgroundSlices)
986  
    unloadLeastRecentlyAccessedBackgroundSlice();
987  
  ret slice;
988  
}
989  
990  
svoid unloadLeastRecentlyAccessedBackgroundSlice() {
991  
  lock fan.lock;
992  
  S caseID = lowestByField lastAccess(keys(backgroundSlices));
993  
  if (caseID != null)
994  
    backgroundFan.unloadCase(caseID);
995  
}
996  
997  
static Slice sliceConceptForGlobalID(S globalID) {
998  
  ret sliceConceptForGlobalID(toGlobalIDObj(globalID));
999  
}
1000  
1001  
static Slice sliceConceptForGlobalID(GlobalID globalID) {
1002  
  ret conceptWhere Slice(+globalID);
1003  
}
1004  
1005  
static Slice sliceConceptForCaseID(S caseID) {
1006  
  ret conceptWhere Slice(+caseID);
1007  
}
1008  
1009  
sS agiBlueNameHTML() {
1010  
  ret ahref(agiBlueURL(), htmlEncode2(agiBlueName()));
1011  
}
1012  
1013  
static Slice createSlice(S name) {
1014  
  name = shorten(name, maxSliceNameLength);
1015  
  Slice slice = cnew Slice(+name);
1016  
  cset(slice, caseID := slice.defaultCaseID());
1017  
  loadSlice(slice).initialSetup(slice.globalID);
1018  
  ret slice;
1019  
}
1020  
1021  
static Slice sliceForName(S name) {
1022  
  ret conceptWhereCI Slice(+name);
1023  
}
1024  
1025  
sclass Query extends WorksOnSlice {
1026  
  SS vars = ciMap();
1027  
    
1028  
  *(LoadedSlice slice) { super(slice); }
1029  
  
1030  
  O process(S query) {
1031  
    try object processLines(agiBlue_parseQueryScript(query));
1032  
    ret serveJSON("No return statement");
1033  
  }
1034  
    
1035  
  O processLines(L<ALQLLine> lines) {
1036  
    for (ALQLLine line : lines) {
1037  
      if (line cast ALQLSlice) {
1038  
        Slice slice = sliceForName(line.slice);
1039  
        if (slice == null) ret serveJSON("Slice not found: " + line.slice);
1040  
        LoadedSlice oldSlice = Query.this.slice;
1041  
        try {
1042  
          setSlice(loadSlice(slice));
1043  
          try object processLines(line.contents);
1044  
        } finally {
1045  
          setSlice(oldSlice);
1046  
        }
1047  
      } else if (line cast ALQLReturn)
1048  
        ret serveJSON(ll(getOrKeep(vars, line.var)));
1049  
      else if (line cast ALQLTriple) {
1050  
        T3S t = line.triple;
1051  
        t = tripleMap(t, s -> getOrKeep(vars, s));
1052  
        Cl<Page> pages;
1053  
        S var;
1054  
        if (isDollarVar(t.c)) {
1055  
          var = t.c;
1056  
          if (isDollarVar(t.a)) {
1057  
            if (isDollarVar(t.b)) todo(t);
1058  
            Entry e = random(conceptsWhereCI(cc, Entry, key := t.b));
1059  
            if (e == null) ret serveJSON("No results for " + var);
1060  
            pages = pagesForKeyAndValue(t.b, t.c);
1061  
            vars.put(t.a, e.page->q);
1062  
            vars.put(t.c, e.value);
1063  
            continue;
1064  
          } else if (isDollarVar(t.b)) {
1065  
            Page page = findPageFromQ(t.a);
1066  
            Entry e = random(findBackRefs(page, Entry));
1067  
            if (e == null) ret serveJSON("No results for " + var);
1068  
            vars.put(t.b, e.key);
1069  
            vars.put(t.c, e.value);
1070  
            continue;
1071  
          } else {
1072  
            S val = getValue(t.a, t.b);
1073  
            if (val == null) ret serveJSON("No results for " + var);
1074  
            pages = ll(pageFromQ(val));
1075  
          }
1076  
        } else if (isDollarVar(t.b)) {
1077  
          var = t.b;
1078  
          if (isDollarVar(t.c)) todo(t);
1079  
          if (isDollarVar(t.a)) {
1080  
            L<Entry> entries = conceptsWhereCI(cc, Entry, value := t.c);
1081  
            if (empty(entries)) ret serveJSON("No results for " + var);
1082  
            Entry e = random(entries);
1083  
            vars.put(t.a, e.page->q);
1084  
            vars.put(t.b, e.key);
1085  
            continue;
1086  
          } else {
1087  
            Cl<S> keys = keysForPageAndValue(t.a, t.c);
1088  
            if (empty(keys)) ret serveJSON("No results for " + var);
1089  
            pages = map(f<S, Page> pageFromQ, keys);
1090  
          }
1091  
        } else {
1092  
          var = t.a;
1093  
          if (!isDollarVar(t.a)) todo(t);
1094  
          if (isDollarVar(t.b)) todo(t);
1095  
          if (isDollarVar(t.c)) todo(t);
1096  
          pages = pagesForKeyAndValue(t.b, t.c);
1097  
        }
1098  
        if (empty(pages)) ret serveJSON("No results for " + var);
1099  
        vars.put(var, random(pages).q);
1100  
      } else if (line cast ALQLPage) {
1101  
        if (eq(line.matchMethod, 'eqic))
1102  
          findPageFromQ(line.page);
1103  
        else if (eq(line.matchMethod, 'flexMatchDollarVarsIC_first)) {
1104  
          new L<SS> l;
1105  
          for (Page p : list(cc, Page))
1106  
            addIfNotNull(l, flexMatchDollarVarsIC_first(line.page, p.q));
1107  
          if (empty(l)) ret serveJSON("No results for " + curly(line.page));
1108  
          vars.putAll(random(l));
1109  
        } else
1110  
          fail("Unknown match method: " + line.matchMethod);
1111  
      } else
1112  
        fail("Can't interpret: " + line);
1113  
    }
1114  
    
1115  
    null;
1116  
  }
1117  
}
1118  
1119  
static Page pageFromQ(Concepts cc, S q) {
1120  
  ret empty(q) ? null : uniqCI_sync(cc, Page, +q);
1121  
}
1122  
1123  
/*sS globalIDFromCaseID(S caseID) {
1124  
  ret assertGlobalID(takeLast(globalIDLength(), caseID));
1125  
}*/
1126  
1127  
sS agiBlue_pageURLWithSlice(Page p) {
1128  
  ret p == null ? null : agiBlueURL() + hquery(slice := _get(sliceForPage(p), 'globalID), q := p.q);
1129  
}
1130  
1131  
static Slice sliceForPage(Page p) {
1132  
  if (p == null) null;
1133  
  SliceInfo info = conceptWhere(p._concepts, SliceInfo);
1134  
  ret info == null ? null : sliceConceptForGlobalID(info.globalID);
1135  
}
1136  
1137  
static GlobalID mainSliceGlobalID() {
1138  
  ret conceptWhere Slice(caseID := "").globalID;
1139  
}
1140  
1141  
static File sliceDumpFile(Slice slice) {
1142  
  ret programFile("slice-dumps/" + or2(slice.caseID, "main") + ".slice");
1143  
}
1144  
1145  
static File dumpSliceToFile(LoadedSlice slice) {
1146  
  ret withDBLock(slice.cc, func -> File {
1147  
    cset(slice.sliceConcept, sliceDumped := now());
1148  
    File file = sliceDumpFile(slice.sliceConcept);
1149  
    new LS lines;
1150  
    for (Page p : list(slice.cc, Page))
1151  
      lines.add(quote(p.q));
1152  
    for (Entry e : list(slice.cc, Entry))
1153  
      lines.add(sfu(tripleToList(entryToTriple(e))));
1154  
    saveLinesAsTextFile(file, lines);
1155  
    ret file;
1156  
  });
1157  
}
1158  
1159  
static T3S entryToTriple(Entry e) {
1160  
  ret e == null ? null : t3(e.page->q, e.key, e.value);
1161  
}
1162  
1163  
static ConceptsLoadedOnDemand slicesLoadedOnDemand(Map<S, LoadedSlice> loadedSlices) {
1164  
  new ConceptsLoadedOnDemand fan;
1165  
  fan.onCaseLoaded(voidfunc(S caseID, O globalID, Concepts concepts) {
1166  
    LoadedSlice ls = new(caseID, concepts);
1167  
    if (nempty(globalID))
1168  
      ls.initialSetup(toGlobalIDObj((S) globalID));
1169  
    loadedSlices.put(caseID, ls);
1170  
  });
1171  
  fan.onUnloadingCase(voidfunc(S caseID, Concepts concepts) {
1172  
    loadedSlices.remove(caseID);
1173  
  });
1174  
  ret fan;
1175  
}
1176  
1177  
// check if slices are loaded both in foreground and background (BAD)
1178  
sS checkDoubleLoads() {
1179  
  LS l = sharedKeys(loadedSlices, backgroundSlices);
1180  
  ret empty(l) ? null : "ERROR: Slice loaded in background and foreground: " + first(l);
1181  
}
1182  
1183  
// just use size of concepts.structure.gz
1184  
static long estimatedSliceDataSize(Slice slice) {
1185  
  ret estimatedSliceDataSize(slice.caseID);
1186  
}
1187  
1188  
static long estimatedSliceDataSize(S caseID) {
1189  
  ret l(conceptsFile(fan.dbID(caseID)));
1190  
}
1191  
1192  
static CentralIndexEntry centralIndexGet(S s) { ret getOrCreate CentralIndexEntry(centralIndex, s); }
1193  
1194  
svoid makeCentralIndex() {
1195  
  pcall {
1196  
    centralIndex.clear();
1197  
    for (Slice slice : conceptsWhere(Slice, indexed := true))
1198  
      addToCentralIndex(slice);
1199  
  }
1200  
  centralIndexMade = now();
1201  
}
1202  
1203  
svoid addToCentralIndex(Slice slice) {
1204  
  for (S s : sliceDump_pageNamesIncludingKeys(sliceDumpFile(slice)))
1205  
    centralIndexGet(s).slices.add(slice);
1206  
}

Author comment

Began life as a copy of #1023558

download  show line numbers  debug dex  old transpilations   

Travelled to 6 computer(s): bhatertpkbcr, mqqgnosmbjvj, pyentgdyhuwx, pzhvpgtvlbxg, tvejysmllsmz, vouqrxazstgt

No comments. add comment

Snippet ID: #1024350
Snippet name: agi.blue source with central index [dev.]
Eternal ID of this version: #1024350/9
Text MD5: d8fba4e0a4f99f10748a6d9812c38574
Transpilation MD5: 0053d013af2b190106f0512597f801ed
Author: stefan
Category: javax / agi.blue / the central servlet
Type: JavaX module (desktop)
Public (visible to everyone): Yes
Archived (hidden from active list): No
Created/modified: 2019-08-04 17:01:06
Source code size: 42168 bytes / 1206 lines
Pitched / IR pitched: No / No
Views / Downloads: 338 / 1596
Version history: 8 change(s)
Referenced in: [show references]