Not logged in.  Login/Logout/Register | List snippets | | Create snippet | Upload image | Upload data

1603
LINES

< > BotCompany Repo | #1024454 - agi.blue source [backup before source lib]

JavaX module (desktop) [tags: use-pretranspiled]

Uses 1658K of libraries. Click here for Pure Java version (20305L/148K).

1  
!7
2  
3  
// See #1023660 for the older 99 lines version
4  
5  
// Customizations:
6  
7  
sbool asyncSearch = false;
8  
sbool allowMultipleCasesInValues = true;
9  
sbool showSliceSelector = true;
10  
sbool showGoogleLogIn = true;
11  
sbool makeAllKeysPages = true;
12  
sbool makeAllValuesPages = true; 
13  
sbool legacy = true; // We still have slices without applied fixes
14  
sbool dumpSlicesOnAnyChange = true; // testing
15  
16  
static int maxSliceNameLength = 1024;
17  
static int displayLength = 140;
18  
static int sideDisplayLength = 50; // when in side bar
19  
static int searchResultsToShow = 50;
20  
sS sideSearchType = 'literal;
21  
static int maxBackgroundSlices = 1000;
22  
23  
// end of customizations
24  
25  
// in main DB
26  
concept Slice {
27  
  GlobalID globalID = aGlobalIDObject();
28  
  S caseID;
29  
  S name;
30  
  bool indexed = true;
31  
  long sliceDumped; // wall time, set before dump starts
32  
33  
  new Ref<User> owner;
34  
35  
  S defaultCaseID() {
36  
    ret uniqueFileNameUsingMD5_80_v2(name + " " + globalID);
37  
  }
38  
39  
  GlobalID globalID() { ret globalID; }
40  
}
41  
42  
// in any DB
43  
concept Page {
44  
  !include #1024467 // compact global ID
45  
  //S globalID = aGlobalID();
46  
  S q;
47  
48  
  void _onLoad() {
49  
    S id = cget(this, 'globalID);
50  
    if (id != null) {
51  
      setGlobalID(id);
52  
      cset(this, globalID := null);
53  
    }
54  
  }
55  
56  
  // for in-VM helper bots
57  
  void quickPost(S key, S value) {
58  
    new GetEntriesAndPost(_concepts).go(this, litmap(+key, +value));
59  
  }
60  
}
61  
62  
abstract concept AbstractEntry {
63  
  new Ref<Page> page;
64  
  int count;
65  
  S key;
66  
  S ip;
67  
  new Ref<Signer> signer;
68  
  
69  
  S q() { Page p = page!; ret p == null ? null : page->q; }
70  
}
71  
72  
concept Entry > AbstractEntry {
73  
  S globalID = aGlobalID(); // TODO: migrate to object
74  
  S value;
75  
}
76  
77  
concept MultiLineEntry > AbstractEntry {
78  
  GlobalID globalID = aGlobalIDObject();
79  
  S value;
80  
}
81  
82  
concept BlobEntry > AbstractEntry {
83  
  GlobalID globalID = aGlobalIDObject();
84  
  long length;
85  
  S md5;
86  
}
87  
88  
// in main DB?
89  
concept Signer {
90  
  S globalID = aGlobalID();
91  
  S publicKey;
92  
  bool trusted;
93  
  S approvedBy;
94  
}
95  
96  
concept User {
97  
  GlobalID globalID = aGlobalIDObject();
98  
  long lastSeen;
99  
}
100  
101  
concept CookieUser > User {
102  
  S cookie;
103  
104  
  toString { ret "[cookieUser]"; }
105  
}
106  
107  
concept GoogleUser > User {
108  
  long googleLogInDate;
109  
  S googleEmail, googleFirstName, googleLastName;
110  
  bool googleEmailVerified;
111  
112  
  toString { ret googleFirstName + " " + googleLastName; }
113  
}
114  
115  
concept MinimalUser > User {
116  
  S userName, encryptedPassword; // TODO
117  
118  
  toString { ret "[minimalUser] " + userName; }
119  
}
120  
121  
// in main DB
122  
concept Session {
123  
  S cookie;
124  
  S selectedSlice; // global ID
125  
  Page slicePage; // phasing out
126  
  new Ref<User> user;
127  
}
128  
129  
// singleton concept in every DB
130  
concept SliceInfo {
131  
  GlobalID globalID;
132  
  LS mandatoryBotVMBusIDs;
133  
}
134  
135  
sclass CentralIndexEntry {
136  
  Set<Slice> slices = synchroLinkedHashSet();
137  
}
138  
139  
// global constants
140  
141  
static SS searchTypeToText = litmap(
142  
  leven := "Leven",
143  
  literal := "Literal",
144  
  scored := "Scored");
145  
  
146  
// global vars
147  
148  
static volatile bool started, broken;
149  
static int sourceCodeLines;
150  
static ConceptsLoadedOnDemand fan;
151  
static ConceptsLoadedOnDemand backgroundFan;
152  
static Map<S, LoadedSlice> loadedSlices = syncMap();
153  
static Map<S, LoadedSlice> backgroundSlices = syncMap();
154  
static Map<S, CentralIndexEntry> centralIndex = ciMap();
155  
static Map<S, CentralIndexEntry> newCentralIndex = ciMap();
156  
static long centralIndexMade; // wall time
157  
static ReliableSingleThread rst_index = new(r makeCentralIndex);
158  
static TimedCache<Long> sizeOnDisk = new(60.0, f guessClusteredSizeOfProgramDirWithoutBackups);
159  
static ConceptFieldIndexDesc idx_slicesByModification, idx_usersBySeen;
160  
static ConceptFieldIndexCI idx_slicesByName, idx_usersByName;
161  
162  
// end of global data
163  
164  
sclass LoadedSlice { // non-persistent class
165  
  S caseID;
166  
  Concepts cc;
167  
  Slice sliceConcept;
168  
  long lastAccess = sysNow();
169  
  ReliableSingleThread rstDumpSlice = new(r {
170  
    dumpSliceToFile(LoadedSlice.this);
171  
    rst_index.trigger(); // rebuild index too - TODO: only this slice
172  
  });
173  
  bool haltSliceDumping;
174  
  
175  
  ConceptFieldIndexDesc idx_latestEntries, idx_latestCreatedPages, idx_latestChangedPages;
176  
  
177  
  *(S *caseID, Concepts *cc) {
178  
    csetAll(list(cc, Page), slice := null, url := null); // clear legacy fields
179  
    releaseEmptyFieldValuesOfAllConcepts(cc);
180  
    indexThings();
181  
    
182  
    // forward changes to Slice concept
183  
    onConceptsChange(cc, rOnceAtATimeOnly(r {
184  
      cset(sliceConcept, _modified := now())
185  
;
186  
      if (!haltSliceDumping && dumpSlicesOnAnyChange && nempty(caseID)) {
187  
        rstDumpSlice.trigger();
188  
      }
189  
    }));
190  
    
191  
    if (makeAllKeysPages && legacy)
192  
      for (Entry e : list(cc, Entry))
193  
        pageFromQ(e.key);
194  
        
195  
    if (makeAllValuesPages && legacy)
196  
      for (Entry e : list(cc, Entry))
197  
        pageFromQ(e.value);
198  
  }
199  
  
200  
  void initialSetup(GlobalID globalID) {
201  
    SliceInfo info = uniq(cc, SliceInfo);
202  
    cset(info, +globalID);
203  
  }
204  
  
205  
  // create if not there
206  
  Page pageFromQ(S q) {
207  
    ret main.pageFromQ(cc, q);
208  
  }
209  
210  
  void indexThings() {
211  
    indexConceptFieldsCI(cc, Page, 'q, Entry, 'key, Entry, 'value);
212  
    indexConceptFields(cc, Signer, 'publicKey);
213  
    indexConceptFields(cc, Session, 'cookie);
214  
    indexSingletonConcept(cc, SliceInfo);
215  
    idx_latestCreatedPages = new ConceptFieldIndexDesc(cc, Page, 'created);
216  
    idx_latestChangedPages = new ConceptFieldIndexDesc(cc, Page, '_modified);
217  
    idx_latestEntries = new ConceptFieldIndexDesc(cc, Entry, 'created);
218  
  }
219  
  
220  
  bool isMainSlice() { ret empty(caseID); }
221  
  GlobalID globalID() { ret sliceConcept.globalID; }
222  
  S name() { ret sliceConcept.name; }
223  
}
224  
225  
p {
226  
  try {
227  
    if (bootstrapDataFrom(#1023558)) deleteMyBackups();
228  
    thinMyBackups();
229  
    fan = slicesLoadedOnDemand(loadedSlices);
230  
    mainConcepts = fan.get("", null);
231  
    
232  
    // index main DB
233  
234  
    indexConceptFields(Slice, 'globalID, Slice, 'caseID);
235  
    indexConceptField(CookieUser, 'cookie);
236  
    idx_slicesByName = new ConceptFieldIndexCI(Slice, 'name);
237  
    idx_slicesByModification = new ConceptFieldIndexDesc(Slice, '_modified);
238  
    idx_usersByName = new ConceptFieldIndexCI(GoogleUser, 'googleLastName);
239  
    idx_usersBySeen = new ConceptFieldIndexDesc(User, 'lastSeen);
240  
241  
    cset(uniq_returnIfNew Slice(caseID := ""), name := "main slice");
242  
    loadedSlices.get("").initialSetup(toGlobalIDObj(mainSliceGlobalID()));
243  
    loadedSlices.get("").sliceConcept = sliceConceptForCaseID("");
244  
    
245  
    backgroundFan = slicesLoadedOnDemand(backgroundSlices);
246  
    backgroundFan.lock = fan.lock;
247  
    
248  
    // legacy conversions!
249  
    //deleteConcepts(Slice, globalID := GlobalID('wftlawbagrwprywn));
250  
  
251  
    // Approve this machine's key
252  
    PKIKeyPair machineKey = agiBot_trustedKeyForMachine();
253  
    if (machineKey != null) {
254  
      print("Approving this machine's key: " + machineKey.publicKey);
255  
      cset(uniq_sync(Signer, publicKey := machineKey.publicKey), trusted := true, approvedBy := "local");
256  
    }
257  
    
258  
    sourceCodeLines = countLines(mySource());
259  
    
260  
    rst_index.trigger();
261  
  } catch print e {
262  
    set broken;
263  
  } finally {
264  
    set started;
265  
  }
266  
  
267  
  // do stuff after we are officially started
268  
  
269  
  // calculate DB size regularly iff there were changes
270  
  sizeOnDisk.keepValueWhileCalculating = true;
271  
  doEveryAndNow(60.0, rWatcher(() -> db_mainConceptsChangeCount(), r { sizeOnDisk! }));
272  
  
273  
} // end of main program
274  
275  
// Serve page
276  
277  
set flag NoNanoHTTPD. html {
278  
  sleepWhile(() -> !started);
279  
  if (broken) ret subBot_serve500("Internal error");
280  
  ret new Request().serve(uri, params);
281  
}
282  
283  
sclass WorksOnSlice {
284  
  Concepts cc;
285  
  LoadedSlice slice;
286  
  
287  
  *() {}
288  
  *(LoadedSlice *slice) { cc = slice.cc; }
289  
  
290  
  void setSlice(LoadedSlice slice) {
291  
    this.slice = slice;
292  
    cc = slice.cc;
293  
  }
294  
  
295  
  Page findPageFromParams(Map map) {
296  
    S q = getString q(map);
297  
    ret empty(q) ? null : findPageFromQ(q);
298  
  }
299  
300  
  Page findOrMakePageFromParams(Map map) {
301  
    ret pageFromQ(getString q(map));
302  
  }
303  
  
304  
  Page findPageFromQ(S q) {
305  
    ret conceptWhereCI(cc, Page, +q);
306  
  }
307  
  
308  
  Page pageFromQ(S q) {
309  
    ret slice.pageFromQ(q);
310  
  }
311  
  
312  
  Set<Page> pagesForKeyAndValue(S key, S value) {
313  
    L<Entry> entries = conceptsWhereIC(cc, Entry, +value, +key);
314  
    ret asSet(ccollect page(entries));
315  
  }
316  
  
317  
  Cl<S> keysForPageAndValue(S q, S value) {
318  
    Page page = findPageFromQ(q);
319  
    if (page == null) null;
320  
    ret collect key(conceptsWhereIC(cc, Entry, +page, +value));
321  
  }
322  
323  
  // DB functions
324  
325  
  bool hasPage(S q) { ret hasConceptWhereIC(Page, +q); }
326  
  S getValue(Page page, S key) {
327  
    ret page == null || empty(key) ? null : getString value(highestByField count(objectsWhereIC(findBackRefs(page, Entry), +key)));
328  
  }
329  
  S getValue(S page, S key) {
330  
    ret getValue(findPageFromQ(page), key);
331  
  }
332  
  S pageDisplayName(Page page) {
333  
    /*S name = getValue(page, "read as");
334  
    bool unnaturalName = nempty(name) && !eq(makeAGIDomain(name), page.url);
335  
    ret unnaturalName ? name + " " + squareBracketed(page.url) : or2(name, unpackAGIDomainOpt(page.url));*/
336  
    ret page.q;
337  
  }
338  
  
339  
  S renderThing(S s, bool forceURLDisplay) {
340  
    ret forceURLDisplay || isURL(s) || isAGIDomain(s)
341  
      ? ahref(fixAGILink(absoluteURL(s)), htmlencode2(shorten(displayLength, s)))
342  
      : ahref(agiBlue_linkForPhrase(s,
343  
          slice := slice.isMainSlice() ? null : slice.sliceConcept.globalID),
344  
        htmlencode2(shorten(displayLength, s)));
345  
  }
346  
347  
  GlobalID sliceID() { ret slice.globalID(); }
348  
349  
} // end of WorksOnSlice
350  
351  
sclass Request extends WorksOnSlice {
352  
  S cookie;
353  
  Session session;
354  
  S uri;
355  
  SS params;
356  
  long started = sysNow();
357  
  S q, domain;
358  
  Set<S> get;
359  
  
360  
  // when serving a concept page, info that may be grabbed
361  
  // by thought bots
362  
  L<Entry> usesAsKey;
363  
  
364  
  // should also work for standalone
365  
  bool isHomeDomain() {
366  
    S domain = domain();
367  
    ret eqic(domain, "www.agi.blue") || !ewic(domain, ".agi.blue");
368  
  }
369  
  
370  
  O serve(S uri, SS params) {
371  
    this.uri = dropMultipleSlashes(uri);
372  
    this.params = params;
373  
    new Matches m;
374  
    
375  
    print(uri + " ? " + unnull(subBot_query()));
376  
    
377  
    // get cookie & session
378  
    
379  
    cookie = cookieFromUser();
380  
    if (cookie != null)
381  
 {
382  
      session = uniq_sync(Session, +cookie);
383  
      if (session.user! == null)
384  
        cset(session, user := uniq_sync CookieUser(+cookie));
385  
    } else
386  
      session = unlistedWithValues(Session);
387  
      
388  
    // check if user wants to change slices
389  
      
390  
    S selectSlice = params.get('slice);
391  
    if (isGlobalID(selectSlice))
392  
      cset(session, selectedSlice := selectSlice);
393  
    if (session.selectedSlice == null)
394  
      cset(session, selectedSlice := str(mainSliceGlobalID()));
395  
396  
    // get slice's case ID
397  
    
398  
    S caseID = "";
399  
    Slice sliceConcept = sliceConceptForGlobalID(session.selectedSlice);
400  
    print("Selected slice: " + session.selectedSlice + ", obj? " + (sliceConcept != null));
401  
    if (sliceConcept != null) caseID = sliceConcept.caseID;
402  
    print("caseID: " + caseID);
403  
    
404  
    // load slice
405  
    
406  
    slice = assertNotNull(loadSlice(caseID, session.selectedSlice));
407  
    cc = slice.cc;
408  
    
409  
    // Check for special URIs
410  
411  
    if (swic(uri, "/bot/")) ret serveBot();
412  
413  
    if (swic(uri, "/user/", m)) {
414  
      User user = conceptWhere User(globalID := toGlobalIDObj(assertGlobalID(m.rest())));
415  
      if (user == null) ret subBot_serve404("User not found");
416  
      S title = "User " + user;
417  
      ret hhtml_miniPage(title, h3(agiBlueNameHTML() + " | " + title)
418  
        + "User type: " + classShortName(user));
419  
    }
420  
421  
    // eleu appends a slash to the URI if it's a single identifier, so we drop it again
422  
    S uri2 = dropTrailingSlash(uri);
423  
    
424  
    if (eqic(uri2, "/google-verify")) {
425  
      print("Google-verify started.");
426  
      Payload payload = printStruct(googleVerifyUserToken2(googleSignInID(), params.get("token")));
427  
      if (payload == null) ret print("google-verify", "No");
428  
      S email = payload.getEmail();
429  
      if (empty(email)) ret print("google-verify", "No");
430  
      
431  
      GoogleUser user = uniqCI_sync GoogleUser(googleEmail := email);
432  
      cset(user,
433  
        googleEmailVerified := payload.getEmailVerified(),
434  
        googleFirstName := strOrNull(payload.get("given_name")),
435  
        googleLastName := strOrNull(payload.get("family_name")));
436  
      
437  
      cset(session,
438  
 +user);
439  
        
440  
      ret print("google-verify", payload.getEmail() + " " + (payload.getEmailVerified() ? "(verified)" : "(not verified)"));
441  
    }
442  
443  
    cset(session.user!, lastSeen := now());      
444  
    
445  
    if (eqic(uri2, "/users")) ret serveUsersList();
446  
    
447  
    if (eqic(uri2, "/slices")) ret serveSlicesList();
448  
    
449  
    if (eqic(uri2, "/search")) ret serveScoredSearch();
450  
    if (eqic(uri2, "/literalSearch")) ret serveLiteralSearch();
451  
    if (eqic(uri2, "/levenSearch")) ret serveLevenSearch();
452  
    
453  
    if (eqic(uri2, "/query")) ret serveQueryPage();
454  
    if (eqic(uri2, "/createSlice")) ret serveCreateSlicePage();
455  
    if (eqic(uri2, "/deletePage")) ret serveDeletePage();
456  
457  
    if (eqic(uri2, "/checkboxes")) ret serveCheckboxes();
458  
    if (eqic(uri2, "/multiAdd")) ret serveMultiAdd();
459  
    
460  
    q = params.get('q);
461  
    get = asCISet(nempties(subBot_paramsAsMultiMap().get('get)));
462  
    
463  
    domain = or2(params.get('domain), domain());
464  
    
465  
    S raw = firstKeyWithValue("", params); // agi.blue?something
466  
    if (nempty(raw) && empty(q)) q = raw;
467  
    /*if (nempty(q)) {
468  
      domain = makeAGIDomain(q);
469  
      if (l(domain) > maximumDomainPartLength()) // escape with "domain="
470  
        ret hrefresh(agiBlueURL() + hquery(+domain, key := "read as", value := q));
471  
      ret hrefresh("http://" + domain + (eq(q, domain) ? "" : "/" +  hquery(key := "read as", value := q)));
472  
      //uri = "/"; replaceMapWithParams(params, key := "read as", value := q);
473  
    }*/
474  
    S url = domain + dropTrailingSlash(uri);
475  
    
476  
    // domain to query
477  
    //if (empty(q)) q = url;
478  
    if (empty(q)) {
479  
      S qq = agiBlue_urlToQuery(url);
480  
      if (neqic(qq, "agi.blue")) q = qq;
481  
    }
482  
    
483  
    Page page, bool newPage = unpair uniqCI2_sync(cc, Page, +q);
484  
    if (newPage) dbLog("New page", +q);
485  
    //printStructs(+params, +raw, +q);
486  
  
487  
    if (empty(params.get('q)) && empty(raw) && isHomeDomain()) {
488  
      //L<Page> pages = sortedByFieldDesc _modified(list(cc, Page)); // TODO: use index
489  
      L<Page> pages = cloneList(slice.idx_latestChangedPages.objectIterator());
490  
      int start = parseInt(params.get("start")), step = 100;
491  
      S nav = pageNav2("/", l(pages), start, step, "start");
492  
      S content = hform(b("GIVE ME INPUT: &nbsp; ") + htextinput('q, autofocus := true) + " " + hsubmit("Ask"))
493  
        + h1(sliceAsHTML() + " has " + nPages(countConcepts(cc, Page)) + " and " + nConnections(countConcepts(cc, Entry)))
494  
        + p(nav)
495  
        + p_nemptyLines(map(pageToHTMLLink(), subList(pages, start, start+step)));
496  
497  
      ret hhtml_agiBlue(hhead_title("Slice " + sliceConcept().name + " | agi.blue") // SERVE SLICE HOME PAGE
498  
        + hbody(hfullcenterAndTopLeft(top() + content
499  
          + footer(),
500  
          sliceSelector()
501  
        )));
502  
    }
503  
        
504  
    S key = trim(params.get('key)), value = trim(params.get('value));
505  
    L<Entry> entries = GetEntriesAndPost(cc).go(page, params).entries;
506  
    
507  
    //S get = params.get('get);
508  
    if (nempty(get))
509  
      ret serveJSON(collect value(llNotNulls(firstThat(entries, e -> get.contains(e.key)))));
510  
    
511  
    S key2 = key, value2 = value; if (nempty(key) && nempty(value)) key2 = value2 = ""; // input reset
512  
  
513  
    bool withHidden = eq(params.get('withHidden), "1");
514  
    new Set<Int> hide;
515  
    if (!withHidden) for (Entry e : entries) if (eqic(e.key, 'hide) && isSquareBracketedInt(e.value)) addAll(hide, e.count, parseInt(unSquareBracket(e.value)));
516  
    new MultiMap<Int, S> mmMeta;
517  
    for (Entry e : entries) if (isSquareBracketedInt(e.key)) mmMeta.put(parseInt(unSquareBracket(e.key)), e.value);
518  
    new MultiMap<S> mm;
519  
    for (Entry e : entries) mm.put(e.key, e.value);
520  
    
521  
    //S name = or2(/* ouch */ last(mm.get("read as")), /* end ouch */ unpackAGIDomain(page.url), page.url);
522  
    S name = page.q;
523  
    
524  
    // Find references
525  
    
526  
    L<Entry> refs = concatLists(conceptsWhereIC(cc, Entry, value := name),
527  
      conceptsWhereIC(cc, Entry, key := name));
528  
    Set<Page> refPages = asSet(ccollect page(refs));
529  
    refPages.remove(page); // don't list current page as reference
530  
    
531  
    // Search in page names (depending on default search type)
532  
    
533  
    L<Page> searchResults;
534  
    if (eq(sideSearchType, 'leven))
535  
      searchResults = levenSearch(page.q, max := 50);
536  
    else if (eq(sideSearchType, 'literal))
537  
      searchResults = literalSearch(page.q, max := 50);
538  
    else
539  
      searchResults = (L<Page>) dm_call('agiBlueSearch, 'search, page.q, maxResult := searchResultsToShow+1, agiBlueBotID := programID(), +cc);
540  
    searchResults.remove(page);
541  
    
542  
    // Find uses as key
543  
    usesAsKey = conceptsWhereIC(cc, Entry, key := page.q);
544  
545  
    // Find page in other slices
546  
    CentralIndexEntry cie = centralIndex.get(page.q);
547  
    L<Slice> pageInOtherSlices = cie == null ? null : listMinus(cie.slices, slice.sliceConcept);
548  
    
549  
    S pageName_html = htmlEncode2(shorten(displayLength, name));
550  
551  
    S mainContents =
552  
        top() + h1(ahref_unstyled(agiBlue_pageURLWithSlice(page), pageName_html) + (newPage ? " [huh????]" : ""))
553  
      + p_nemptyLines(map(entries, func(Entry e) -> S {
554  
        !withHidden && (hide.contains(e.count) || eqic(e.key, "read as") && eqic(e.value, name)) ? ""
555  
        : "[" + e.count + "] " +
556  
          renderThing(e.key, false) + ": " +
557  
          b(renderThing(e.value, cic(mmMeta.get(e.count), "is a URL")))
558  
        }))
559  
      + hpostform(h3("Add an entry")
560  
        + "Key: " + hinputfield(key := key2) + " Value: " + hinputfield(value := value2) + "<br><br>" + hsubmit("Add")
561  
        )
562  
        
563  
      + p(ahref(agiBlueURL() + "/literalSearch" + hquery(q := page.q), "[literal search]", title := "Search pages with a name containing this page's name literally")
564  
         + " " +
565  
         ahref(agiBlueURL() + "/levenSearch" + hquery(q := page.q), "[leven search 1]", title := "Search pages with a Levenshtein similarity of 1 containing this page's name literally")
566  
         + " " +
567  
         ahref(agiBlueURL() + "/search" + hquery(q := page.q), "[scored search]", title := "Search pages with ScoredSearch")
568  
         + " " +
569  
         ahref(agiBlueURL() + "/deletePage" + hquery(slice := sliceID(), q := page.q), "[delete page]")
570  
       )
571  
       
572  
       // optional "uses as key" section
573  
       
574  
       + (empty(usesAsKey) ? "" : h3(quote(pageName_html) + " as key")
575  
       + p_nemptyLines_showFirst(50, map(usesAsKey, e ->
576  
         pageToHTMLLink().get(e.page!) + unicode_spacedRightPointingTriangle() +
577  
           pageName_html + unicode_spacedRightPointingTriangle() +
578  
         pageToHTMLLink().get(pageFromQ(e.value))
579  
       )))
580  
       
581  
       // optional "in other slices" section
582  
       
583  
       + (empty(pageInOtherSlices) ? "" : h3(quote(pageName_html) + " in other slices")
584  
       + p_nemptyLines_showFirst(50, map(slice ->
585  
         ahref(agiBlueURL() + hquery(slice := slice.globalID, q := page.q), htmlEncode2(slice.name)), pageInOtherSlices)))
586  
       ;
587  
       
588  
      // end of mainContents
589  
        
590  
    S sideContents =
591  
      hform(b("GIVE ME INPUT:") + " "
592  
        + htextinput('q) + " "
593  
        + hsubmit("Ask", onclick := "document.getElementById('newInputForm').target = '';") + " "
594  
        + hsubmit("+Tab", title := "Ask and show result in a new tab", onclick := "document.getElementById('newInputForm').target = '_blank';"),
595  
        id := 'newInputForm)
596  
      
597  
      + h3("References (" + l(refPages) + ")")
598  
      
599  
      + p_nemptyLines_showFirst(10, map(pageToHTMLLink(displayLength := sideDisplayLength), refPages))
600  
      
601  
      + h3(searchTypeToText.get(sideSearchType) + " search results (" + (l(searchResults) >= searchResultsToShow ? searchResultsToShow + "+" : str(l(searchResults))) + ")")
602  
      
603  
      + p_nemptyLines_showFirst(searchResultsToShow, map(pageToHTMLLink(displayLength := sideDisplayLength), searchResults))
604  
      
605  
      + hdiv("", id := 'extraStuff);
606  
      
607  
    // TODO: sync search delivery with WebSocket creation
608  
    if (asyncSearch)
609  
      doLater(6.0, r { dm_call('agiBlueSearch, 'searchAndPost, page.q, agiBlueBotID := programID(), +cc) });
610  
      
611  
    // notify local interested parties
612  
    vmBus_send('agiBlue_servingConceptPage, this, page);
613  
    
614  
    S iframe = params.get('render_iframe);
615  
      
616  
    // serve a concept page
617  
    ret hhtml_agiBlue(hhead_title(pageDisplayName(page)) + hbody(
618  
      tag('table,
619  
        tr(
620  
          td(sliceSelector(), valign := 'top)
621  
        + td(sideContents, align := 'right, valign := 'top, rowspan := 2))
622  
      + tr(td(mainContents +
623  
        (empty(iframe) ? "" : iframe(iframe, width := 600, height := 400)) +
624  
      footer(), align := 'center, valign := 'top)),
625  
      width := "100%", height := "100%")));
626  
  }
627  
  
628  
  O servePagesToBot(Iterable<Page> pages) {
629  
    ret serveListToBot(map(pageToMap(wrapMapAsParams(params)), pages));
630  
  }
631  
632  
  O serveListToBot(Collection l) {
633  
    if (nempty(params.get('max)))
634  
      l = takeFirst(parseInt(params.get('max)), l);
635  
    ret serveJSON(l);
636  
  }
637  
638  
  // uri starts with "/bot/"
639  
  O serveBot() {
640  
    S q = params.get('q);
641  
    
642  
    if (eqic(uri, "/bot/hello")) ret serveJSON("hello");
643  
    if (eqic(uri, "/bot/hasPage")) ret serveJSON(hasPage(q));
644  
    if (eqic(uri, "/bot/randomPageContaining")) {
645  
      assertNempty(q);
646  
      ret servePageToBot(random(filter(list(cc, Page), p -> cic(p.q, q))), params);
647  
    }
648  
    if (eqic(uri, "/bot/allPages"))
649  
      ret servePagesToBot(list(cc, Page));
650  
    if (eqic(uri, "/bot/allPagesStartingWith")) {
651  
      assertNempty(q);
652  
      ret servePagesToBot(filter(list(cc, Page), p -> swic(p.q, q)));
653  
    }
654  
    if (eqic(uri, "/bot/allPagesEndingWith")) {
655  
      assertNempty(q);
656  
      ret servePagesToBot(filter(list(cc, Page), p -> ewic(p.q, q)));
657  
    }
658  
    if (eqic(uri, "/bot/allPagesContaining")) {
659  
      assertNempty(q);
660  
      ret servePagesToBot(filter(list(cc, Page), p -> cic(p.q, q)));
661  
    }
662  
    if (eqic(uri, "/bot/allPagesContainingRegexp")) {
663  
      assertNempty(q);
664  
      Pattern pat = regexpIC(q);
665  
      ret servePagesToBot(filter(list(cc, Page), p -> regexpFindIC(pat, p.q)));
666  
    }
667  
    
668  
    if (eqicOneOf(uri, "/bot/postSigned", /*"/bot/makePhysicalSlice",*/ "/bot/approveTrustRequest")) {
669  
      S text = rtrim(params.get('text));
670  
      S key = getSignerKey(text);
671  
      if (empty(key)) ret subBot_serve500("Please include your public key");
672  
      if (!isSignedWithKey(text, key)) ret subBot_serve500("Signature didn't verify");
673  
      text = dropLastTwoLines(text); // drop signer + sig line
674  
      
675  
      Signer signer = uniq_sync Signer(publicKey := key);
676  
      
677  
      /*if (eqic(uri, "/bot/makePhysicalSlice")) {
678  
        if (!signer.trusted) ret subBot_serve500("Untrusted signer");
679  
        Page page = findPageFromParams(jsonDecodeMap(text));
680  
        if (page == null) ret subBot_serve500("Page not found");
681  
        ret serveJSON(uniq2_sync(PhysicalSlice, slicePage := page).b ? "Slice made" : "Slice exists");
682  
      }*/
683  
    
684  
      if (eqic(uri, "/bot/postSigned")) {
685  
        new L out;
686  
        for (S line : tlft(text)) {
687  
          SS map = jsonDecodeMap(line);
688  
          GetEntriesAndPost x = new(db_mainConcepts());
689  
          x.signer = signer;
690  
          Page page = findOrMakePageFromParams(map);
691  
          if (page == null) continue with out.add("Invalid page reference");
692  
          x.go(page, map);
693  
          out.add(x.newEntry ? "Saved" : x.entry != null ? "Entry exists" : "Need key and value");
694  
        }
695  
        ret serveJSON(out);
696  
      }
697  
      
698  
      if (eqic(uri, "/bot/approveTrustRequest")) {
699  
        if (!signer.trusted) ret subBot_serve500("Untrusted signer");
700  
        Signer toApprove = conceptWhere Signer(publicKey := trim(text));
701  
        if (toApprove == null) ret subBot_serve500("Signer to approve not found");
702  
        cset(toApprove, trusted := true, approvedBy := signer.globalID);
703  
        ret serveJSON("Approved: " + trim(text));
704  
      }
705  
      
706  
      ret subBot_serve500("CONFUSION");
707  
    }
708  
    
709  
    if (eqic(uri, "/bot/post")) {
710  
      GetEntriesAndPost x = new(db_mainConcepts());
711  
      x.go(pageFromQ(q), params);
712  
      ret serveJSON(x.newEntry ? "Saved" : x.entry != null ? "Entry exists" : "Need key and value");
713  
    }
714  
    
715  
    if (eqic(uri, "/bot/entriesOnPage"))
716  
      ret serveJSON(map(entriesOnPage(findPageFromParams(params)), entryToMap(false)));
717  
      
718  
    if (eqic(uri, "/bot/entriesForKey"))
719  
      ret serveJSON(map(conceptsWhereIC(cc, Entry, key := params.get('key)), entryToMap(true)));
720  
     
721  
    // look up value from page & key in current slice
722  
    if (eqic(uri, "/bot/lookup")) {
723  
      S key = params.get('key);
724  
      if (empty(key)) ret serveJSON("Need key");
725  
      S value = getValue(findPageFromParams(params), key);
726  
      ret serveJSON(empty(value) ? "" : litmap(+value));
727  
    }
728  
729  
    if (eqic(uri, "/bot/multiLookup")) {
730  
      S key = params.get('key);
731  
      if (empty(key)) ret serveJSON("Need key");
732  
      ret serveJSON(collect value(objectsWhereIC(findBackRefs(findPageFromParams(params), Entry), +key)));
733  
    }
734  
  
735  
    // look up value from page & key in all slices
736  
    if (eqic(uri, "/bot/multiLookupInAllSlices")) {
737  
      S key = params.get('key);
738  
      if (empty(key)) ret serveJSON("Need key");
739  
      Cl<Slice> slices = centralIndexGetSlices(key);
740  
      new L<Map> results;
741  
      fOr (Slice s : slices) {
742  
        WorksOnSlice wos = new(loadBackgroundSlice(s));
743  
        Page page = wos.findPageFromParams(params);
744  
        if (page != null)
745  
          fOr (Entry e : objectsWhereIC(findBackRefs(page, Entry), +key))
746  
            results.add(litorderedmap("c" := e.value, "slice" := str(s.globalID())));
747  
      }
748  
      ret serveJSON(results);
749  
    }
750  
    
751  
    if (eqic(uri, "/bot/latestEntries"))
752  
      ret serveJSON(map(takeFirst(10, slice.idx_latestEntries.objectIterator()), entryToMap(true)));
753  
    if (eqic(uri, "/bot/latestPages"))
754  
      ret serveJSON(map(takeFirst(10, slice.idx_latestCreatedPages.objectIterator()), pageToMap()));
755  
    if (eqic(uri, "/bot/latestChangedPages"))
756  
      ret serveJSON(map(takeFirst(10, slice.idx_latestChangedPages.objectIterator()), pageToMap()));
757  
      
758  
    if (eqic(uri, "/bot/googleUsersCount"))
759  
      ret serveJSON(countConcepts(GoogleUser));
760  
      
761  
    if (eqic(uri, "/bot/totalPageCount"))
762  
      ret serveJSON(countConcepts(Page));
763  
    /*if (eqic(uri, "/bot/pageWithoutPhysicalSliceCount"))
764  
      ret serveJSON(countConceptsWhere(Page, slice := null));
765  
    if (eqic(uri, "/bot/physicalSliceCount"))
766  
      ret serveJSON(countConcepts(PhysicalSlice));*/
767  
    if (eqic(uri, "/bot/trustedSignersCount"))
768  
      ret serveJSON(countConcepts(Signer, trusted := true));
769  
      
770  
    if (eqic(uri, "/bot/valueSearch")) {
771  
      S value = params.get('value);
772  
      L<Entry> entries = conceptsWhereIC(cc, Entry, +value);
773  
      ret serveJSON(map(takeFirst(100, entries), entryToMap(true)));
774  
    }
775  
      
776  
    if (eqic(uri, "/bot/keyAndValueSearch")) {
777  
      S key = params.get('key), value = params.get('value);
778  
      Cl<Page> pages = pagesForKeyAndValue(key, value);
779  
      ret servePagesToBot(pages);
780  
    }
781  
    
782  
    if (eqic(uri, "/bot/keyValuePairsByPopularity")) {
783  
      L<PairS> pairs = map(list(Entry), e -> pair(e.key, e.value));
784  
      LPair<S, Int> pairs2 = multiSetTopPairs(ciMultiSet(map pairToUglyStringForCIComparison(pairs)));
785  
      ret serveJSON(map(pairs2, p -> {
786  
        S key, value = unpair pairFromUglyString(p.a);
787  
        ret litorderedmap(n := p.b, +key, +value);
788  
      }));
789  
    }
790  
    
791  
    if (eqic(uri, "/bot/allKeys"))
792  
      ret serveListToBot(distinctCIFieldValuesOfConcepts(cc, Entry, 'key));
793  
      
794  
    if (eqic(uri, "/bot/allKeysByPopularity"))
795  
      ret serveListToBot(mapMultiSetByPopularity(distinctCIFieldValuesOfConcepts_multiSet(cc, Entry, 'key), (key, n) -> litorderedmap(+n, +key)));
796  
      
797  
    if (eqic(uri, "/bot/dbSize"))
798  
      ret serveJSON(l(conceptsFile()));
799  
      
800  
    if (eqic(uri, "/bot/query"))
801  
      ret serveBotQuery();
802  
803  
    if (eqic(uri, "/bot/createSlice")) {
804  
      Slice slice = createSlice(assertNempty(params.get('name)));
805  
      ret serveJSON(litorderedmap(id := str(slice.globalID), name := slice.name));
806  
    }
807  
    
808  
    if (eqic(uri, "/bot/dumpAllSlices") && authed()) {
809  
      new L<Map> out;
810  
      for (Slice slice) try {
811  
        out.add(litorderedmap(slice := slice.caseID, ms := returnTimeFor(() -> dumpSliceToFile(loadBackgroundSlice(slice)))));
812  
      } catch print e {
813  
        out.add(litorderedmap(slice.caseID, error := exceptionToStringShort(e)));
814  
      }
815  
      ret serveJSON(out);
816  
    }
817  
    
818  
    if (eqic(uri, "/bot/dumpSlice")) {
819  
      File f = dumpSliceToFile(slice);
820  
      ret "OK, saved as: " + f2s(f);
821  
    }
822  
    
823  
    if (eqic(uri, "/bot/diskStats"))
824  
      ret serveJSON(litcimap(
825  
        sizeOnDisk := sizeOnDisk!
826  
      ));
827  
828  
    if (eqic(uri, "/bot/memStats")) {
829  
      Map fg = cloneMap(loadedSlices);
830  
      Map bg = cloneMap(backgroundSlices);
831  
      ret serveJSON(litcimap(
832  
        error := checkDoubleLoads(),
833  
        centralIndexEntries := l(centralIndex),
834  
        centralIndexMade := renderHowLongAgo(centralIndexMade),
835  
        numLoadedForegroundSlices := l(fg),
836  
        numLoadedBackgroundSlices := l(bg),
837  
        loadedForegroundSlicesEstimatedSize := longSum(map estimatedSliceDataSize(keys(fg))),
838  
        loadedBackgroundSlicesEstimatedSize := longSum(map estimatedSliceDataSize(keys(bg))),
839  
        loadedForegroundSlices := sortedIC(keysList(fg)),
840  
        loadedBackgroundSlices := sortedIC(keysList(bg)),
841  
        sessionsWithGoogleUsers := countPred(list(Session), s -> s.user! instanceof GoogleUser),
842  
        userObjects := countConcepts(User),
843  
      ));
844  
    }
845  
    
846  
    if (eqic(uri, "/bot/centralIndexGet")) {
847  
      CentralIndexEntry e = centralIndex.get(q);
848  
      ret serveJSON(e == null ? ll() : map(e.slices, sliceToMap()));
849  
    }
850  
    
851  
    // return primary triple from each slice
852  
    if (eqic(uri, "/bot/centralIndexGrab")) {
853  
      CentralIndexEntry ie = centralIndex.get(q);
854  
      new L<Map> out;
855  
      if (ie != null) for (Slice slice : ie.slices) {
856  
        LoadedSlice ls = loadBackgroundSlice(slice);
857  
        WorksOnSlice wos = new(ls);
858  
        Page page = wos.findPageFromQ(q);
859  
        Iterable<Entry> entries = entriesOnPage(page);
860  
        print("Page for " + q + " in slice " + ls.caseID + ": " + yesNo(page != null) + " - " + l(entries));
861  
        Entry e = first(entries);
862  
        if (e != null)
863  
          out.add(litorderedmap(a := page.q, b := e.key, c := e.value, slice := slice.caseID));
864  
      }
865  
      ret serveJSON(out);
866  
    }
867  
    
868  
    if (eqic(uri, "/bot/updateCentralIndex") && authed()) {
869  
      rst_index.trigger();
870  
      ret "OK";
871  
    }
872  
873  
    if (eqic(uri, "/bot/allGoogleEmails") && authed())
874  
      ret serveJSON(collect googleEmail(list(GoogleUser)));
875  
876  
    if (eqic(uri, "/bot/words2_spaces_all"))
877  
      ret serveJSON(mapToValues_ciMap words2_spaces_cached(collect q(conceptsSortedByFieldCI(slice.cc, Page, 'q))));
878  
    
879  
    if (eqic(uri, "/bot/words2_spaces_collapse_all"))
880  
      ret serveJSON(mapToValues_ciMap words2_spaces_collapse_cached(collect q(conceptsSortedByFieldCI(slice.cc, Page, 'q))));
881  
882  
    if (eqic(uri, "/bot/makeManyPages")) {
883  
      //LS qs = nempties(subBot_paramsAsMultiMap().get('q));
884  
      LS qs = tlft(params.get("pages"));
885  
      slice.haltSliceDumping = true;
886  
      try {
887  
        fOr ping (S _q : qs)
888  
          pageFromQ(_q);
889  
      } finally {
890  
        slice.haltSliceDumping = true;
891  
        slice.rstDumpSlice.trigger();
892  
      }
893  
      ret "OK (" + l(qs) + ")";
894  
    }
895  
896  
    // end of bot methods
897  
  
898  
    ret subBot_serve404();
899  
  }
900  
  
901  
  O serveBotQuery() {
902  
    S query = params.get('query);
903  
    ret new Query(slice).process(query);
904  
  }
905  
  
906  
  O serveLiteralSearch() {
907  
    S q = params.get('q);
908  
    L<Page> searchResults = literalSearch(q);
909  
    ret serveSearchResults("literal search" , q, searchResultsToShow, searchResults);
910  
  }
911  
  
912  
  L<Page> literalSearch(S q, O... _) {
913  
    int searchResultsToShow = optPar max(_, 100);
914  
    
915  
    // quick search in random order
916  
    //L<Page> searchResults = takeFirst(searchResultsToShow+1, filterIterator(iterator(list(cc, Page)), p -> cic(p.q, q));
917  
    
918  
    // full search, order by length
919  
    ret takeFirst(searchResultsToShow+1, pagesSortedByLength(filter(list(cc, Page), p -> cic(p.q, q))));
920  
  }
921  
  
922  
  O serveScoredSearch() {
923  
    S q = params.get('q);
924  
    L<Page> searchResults = (L<Page>) dm_call('agiBlueSearch, 'search, q, +cc);
925  
    ret serveSearchResults("scored search" , q, searchResultsToShow, searchResults);
926  
  }
927  
  
928  
  O serveLevenSearch() {
929  
    S q = params.get('q);
930  
    L<Page> searchResults = levenSearch(q);
931  
    ret serveSearchResults("leven search with distance 1" , q, searchResultsToShow, searchResults);
932  
  }
933  
  
934  
  L<Page> levenSearch(S q, O... _) {
935  
    int searchResultsToShow = optPar max(_, 100);
936  
    int maxEditDistance = 1;
937  
    
938  
    new Map<Page, Int> map;
939  
    for (Page p : list(cc, Page)) {
940  
      int distance = leven_limitedIC(q, p.q, maxEditDistance+1);
941  
      if (distance <= maxEditDistance) map.put(p, distance);
942  
    }
943  
    ret takeFirst(searchResultsToShow+1, keysSortedByValue(map));
944  
  }
945  
  
946  
  O serveSearchResults(S searchType, S q, int searchResultsToShow, Collection<Page> searchResults) {
947  
    S title = "agi.blue " + searchType + " for " + htmlEncode2(quote(q)) + " (" + (l(searchResults) >= searchResultsToShow ? searchResultsToShow + "+ results" : nResults(l(searchResults))) + ")";
948  
    
949  
    ret hhtml_agiBlue(hhead_title(htmldecode_dropAllTags(title))
950  
      + hbody(hfullcenter(//top() +
951  
        h3(title)
952  
      + p_nemptyLines_showFirst(searchResultsToShow, map(pageToHTMLLink(), searchResults)))));
953  
  }
954  
  
955  
  O serveQueryPage() {
956  
    S query = params.get('query);
957  
    if (query == null) query = loadSnippet(#1024258);
958  
    S title = agiBlueNameHTML() + " | Execute a query script (" + targetBlank("http://code.botcompany.de/1024274", "ALQL") + ")";
959  
    ret hhtml_miniPage(title, h3(title)
960  
 + form(
961  
      htextarea(query, name := 'query, cols := 80, rows := 10, autofocus := true)
962  
      + "<br><br>"
963  
      + hsubmit("Execute"), action := "/bot/query"
964  
    ));
965  
  }
966  
967  
  O hhtml_miniPage(S htmlTitle, O contents) {
968  
    ret hhtml_agiBlue(hhead_title(htmldecode_dropAllTags(htmlTitle))
969  
      + hbody(hfullcenter(contents)));
970  
  }
971  
  
972  
  S sliceHomeURL() {
973  
    ret slice == null ? agiBlueURL() : sliceHomeURL(slice.sliceConcept.globalID);
974  
  }
975  
  
976  
  S sliceHomeURL(Slice slice) { ret sliceHomeURL(slice.globalID); }
977  
978  
  S sliceHomeURL(GlobalID slice) {
979  
    ret agiBlueURL() + hquery(+slice);
980  
  }
981  
982  
  S sliceLinkHTML(Slice slice) {
983  
    ret slice == null ? "-": ahref(sliceHomeURL(slice), htmlEncode2(slice.name));
984  
  }
985  
  
986  
  O serveCreateSlicePage() {
987  
    S sliceName = trim(params.get('sliceName));
988  
    if (eq(params.get('doIt), "1") && nempty(sliceName)) {
989  
      // TODO: check for existing name
990  
      Slice slice = createSlice(sliceName);
991  
      ret hrefresh(sliceHomeURL(slice.globalID));
992  
    }
993  
    
994  
    S title = agiBlueNameHTML() + " | Create slice";
995  
    ret hhtml_agiBlue(hhead_title(htmldecode_dropAllTags(title))
996  
      + hbody(hfullcenter(
997  
        h3(title)
998  
      + form(
999  
          hhidden(doIt := 1)
1000  
        + "Slice name: " + htextinput(+sliceName, autofocus := true)
1001  
        + "<br><br>"
1002  
        + hsubmit("Create slice"))
1003  
      )));    
1004  
  }
1005  
1006  
  O serveDeletePage() {
1007  
    q = params.get('q);
1008  
    Page page = findPageFromQ(q);
1009  
    if (page == null) ret "Page " + quote(q) + " not found in slice " + htmlEncode(slice.name());
1010  
    bool ownage = slice.sliceConcept.owner! == session.user!;
1011  
    if (!ownage) ret "Sorry, you don't own slice " + htmlEncode(slice.name());
1012  
    deleteConcepts(findBackRefs(page, AbstractEntry));
1013  
    cdelete(page);
1014  
    ret hrefresh(1.0, sliceHomeURL()) + "Page deleted";
1015  
  }
1016  
  
1017  
  F1<Page, S> pageToHTMLLink(O... _) {
1018  
    optPar int displayLength = main.displayLength;
1019  
    ret func(Page p) -> S {
1020  
      S name = pageDisplayName(p);
1021  
      ret ahref(agiBlue_pageURLWithSlice(p),
1022  
        htmlEncode2(shorten(displayLength, name)),
1023  
        title := name);
1024  
    };
1025  
  }
1026  
  
1027  
  // and google log-in
1028  
  S sliceSelector() {
1029  
    ret htag table(tr(
1030  
      (!showGoogleLogIn ? "" : td(
1031  
        googleSignIn_signInButton(agiBlueURL() + "/google-verify", "console.log(data);", ""), style := "padding-right: 10px;")
1032  
        + td(nobr(ahref("javascript:signOut()", "Sign out")), style := "padding-right: 10px; padding-bottom: 0.15em"))
1033  
    
1034  
    + (!showSliceSelector ? "" : hform(td(
1035  
        "Select reality slice: "
1036  
      + hselect(availableSlices(), session.selectedSlice, name := 'slice, onchange := "this.form.submit()")
1037  
      /*+ " " + hsubmit("Go")*/
1038  
      + " &nbsp; | " +
1039  
      ahref(agiBlueURL("/slices"), nSlices(countConcepts(Slice))) + " | " +
1040  
      ahref(agiBlueURL() + "/createSlice", "Create slice...")
1041  
    )))));
1042  
  }
1043  
  
1044  
  S sliceAsHTML() {
1045  
    if (slice.isMainSlice()) ret htmlEncode2(agiBlueName() + "'s main slice");
1046  
    if (slice.sliceConcept == null) ret "Slice ???";
1047  
    ret htmlEncode2("Slice " + quote(slice.sliceConcept.name));
1048  
  }
1049  
  
1050  
  Slice sliceConcept() { ret slice.sliceConcept; }
1051  
  
1052  
  S footer() {
1053  
    ret p(small(elapsedMS_sysNow(started) + " ms"));
1054  
  }
1055  
  
1056  
  S verifiedEmail() {
1057  
    if (session == null || !(session.user! instanceof GoogleUser)) null;
1058  
    ret ((GoogleUser) session.user!).googleEmail;
1059  
  }
1060  
  
1061  
  bool authed() {
1062  
    bool ver = eqic(verifiedEmail(), "stefan.reich.maker.of.eye@googlemail.com");
1063  
    print("Auth check " + verifiedEmail() + " -> " + ver);
1064  
    ret ver;
1065  
  }
1066  
  
1067  
  S googleSignInID() {
1068  
    ret eqic(domain(), "botcompany.de") ? botCompanyGoogleSignInID() : agiBlueGoogleSignInID();
1069  
  }
1070  
1071  
  S hhtml_agiBlue(S contents) {
1072  
    ret hhtml(hAddToHead_fast(contents, 
1073  
      hIncludeGoogleFont("Source Sans Pro")
1074  
      //+ [[<meta name="google-site-verification" content="oRvyeTqgSuv-FE_c-2UKM1Vp0oDqx8h9WruHYWqA-NQ" />]]
1075  
      + loadJQuery()
1076  
      + hmobilefix()
1077  
      + googleSignIn_header("", googleSignInID())
1078  
      + hstylesheet("body { font-family: Source Sans Pro }")));
1079  
  }
1080  
  
1081  
  O serveSlicesList() {
1082  
    S sort = getAny(params, 'sort, 'by);
1083  
    L<Slice> slices;
1084  
    S shown;
1085  
    if (eqic(sort, 'name)) {
1086  
      slices = asList(idx_slicesByName.objectIterator());
1087  
      shown = "sorted by name";
1088  
    } else {
1089  
      sort = 'date;
1090  
      slices = asList(idx_slicesByModification.objectIterator());
1091  
      shown = "last modified first";
1092  
    }
1093  
      
1094  
    S title = agiBlueNameHTML() + " has " + nSlices(slices) /*+ " (" + shown + ")"*/;
1095  
    new PreIncLongCounter counter;
1096  
    ret hhtml_agiBlue(hhead_title_decode(title)
1097  
      + hbody(hfullcenter(
1098  
        h1(title)
1099  
      + p(joinWithSpacedVBar(
1100  
        ahrefIf(neqic(sort, 'name), agiBlueURL() + "/slices" + hquery(sort := 'name), "List by name"),
1101  
        ahrefIf(neqic(sort, 'date), agiBlueURL() + "/slices" + hquery(sort := 'date), "List by date"),
1102  
        ahref(agiBlueURL() + "/createSlice", "Create slice...")))
1103  
1104  
      + htmlTable2(map(slices, slice ->
1105  
        litorderedmap(
1106  
          "Name" := ahref(sliceHomeURL(slice), htmlEncode2(slice.name)),
1107  
          "ID" := ahref(sliceHomeURL(slice), str(slice.globalID)),
1108  
          "Owner" := str(slice.owner!)
1109  
        )),
1110  
        htmlEncode := false)
1111  
      )));
1112  
  }
1113  
1114  
  O serveUsersList() {
1115  
    S sort = getAny(params, 'sort, 'by);
1116  
    L<User> users;
1117  
    S shown;
1118  
    if (eqic(sort, 'name))
1119  
      users = asList(idx_usersByName.objectIterator());
1120  
    else {
1121  
      sort = 'seen;
1122  
      users= asList(idx_usersBySeen.objectIterator());
1123  
    }
1124  
      
1125  
    S title = agiBlueNameHTML() + " has " + nUsers(users);
1126  
    new PreIncLongCounter counter;
1127  
    ret hhtml_agiBlue(hhead_title_decode(title)
1128  
      + hbody(hfullcenter(
1129  
        h1(title)
1130  
      + p(joinWithSpacedVBar(
1131  
        ahrefIf(neqic(sort, 'name), agiBlueURL() + "/users" + hquery(sort := 'name), "List by name"),
1132  
        ahrefIf(neqic(sort, 'seen), agiBlueURL() + "/users" + hquery(sort := 'seen), "List by last seen"),
1133  
        ahref(agiBlueURL() + "/createSlice", "Create slice...")))
1134  
1135  
      + htmlTable2(map(users, user ->
1136  
        litorderedmap(
1137  
          "Name" := ahref(userHomeURL(user), htmlEncode2(str(user))),
1138  
          "Last seen" := renderHowLongAgo(user.lastSeen)
1139  
        )),
1140  
        htmlEncode := false)
1141  
      )));
1142  
  }
1143  
1144  
  S sliceInfoHTML() {
1145  
    User owner = slice.sliceConcept.owner!;
1146  
    ret owner == null ? "" : p("This slice is owned by " + userHTML(owner));
1147  
  }
1148  
1149  
  S userHomeURL(User user) {
1150  
    ret user == null ? null : agiBlueURL() + "/user/" + user.globalID;
1151  
  }
1152  
  
1153  
  S userHTML(User user) {
1154  
    ret user == null ? "nobody" : ahref(userHomeURL(user), htmlEncode2(str(user)));
1155  
  }
1156  
  
1157  
  S top() {
1158  
    ret nempty(get) ? "" : hcomment("cookie: " + takeFirst(4, session.cookie))
1159  
      + hSilentComputatorWithFlag("agi.blue: " + q)
1160  
      + p(ahref(sliceHomeURL(),
1161  
        //hsnippetimg(#1101682, width := 565/5, height := 800/5, title := "Robot by Peerro @ DeviantArt")
1162  
        //hsnippetimg(#1101778, width := 96, height := 96, title := "agi.blue - a database for everything")
1163  
        hsnippetimg(#1101822, width := 314, height := 125, title := "agi.blue - Wikipedia for robots")
1164  
      ))
1165  
      + sliceInfoHTML()
1166  
      + p(small(
1167  
          agiBlueNameHTML_boldWithSize()
1168  
        + (agiBlue_isOriginal() ? "" : " " + targetBlank("http://agi.blue", "[original]"))
1169  
        + " | " + ahref(agiBlueURL("/slices"), nSlices(countConcepts(Slice)))
1170  
        + " | " + targetBlank(progLink(), "source code") + " of this web site (" + nLines(sourceCodeLines) + ") | " + targetBlank("https://gitter.im/agi-blue/community", "sponsor https?") + " | by " + targetBlank("https://BotCompany.de", "BC") + " | " + targetBlank("http://fiverr.tinybrain.de/", "Fiverr") + " | " + targetBlank("https://discordapp.com/invite/SEAjPqk", "Discord") + " | " + targetBlank("https://www.youtube.com/watch?v=b6jtRdV3Ev8", "Video")
1171  
        + " | " + targetBlank("http://code.botcompany.de/1024233", "Notes")
1172  
        + " | " + ahref(agiBlueURL() + "/query", "Query")
1173  
      ));
1174  
  }
1175  
1176  
  Slice createSlice(S name) {
1177  
    name = shorten(name, maxSliceNameLength);
1178  
    Slice slice = cnew Slice(+name, owner := session.user);
1179  
    cset(slice, caseID := slice.defaultCaseID());
1180  
    loadSlice(slice).initialSetup(slice.globalID);
1181  
    ret slice;
1182  
  }
1183  
1184  
  O serveCheckboxes() {
1185  
    set conceptsSortedByFieldCI_verbose;
1186  
    Cl<Page> pages = conceptsSortedByFieldCI(slice.cc, Page, 'q);
1187  
    S title = "Select pages";
1188  
    ret hhtml_miniPage(title, h3(title) + hpostform(
1189  
      htmlTable2(map(pages, page -> litorderedmap(
1190  
        "Page" := hcheckbox("page_" + page.globalID()) + " " + pageToHTMLLink().get(page)
1191  
      )), 
1192  
      htmlEncode := false)
1193  
      + h3("Add an entry to all selected pages")
1194  
        + p("Key: " + hinputfield("key") +
1195  
          " Value: " + hinputfield("value"))
1196  
      + p(hsubmit("Add entries")), action := agiBlueURL() + "/multiAdd"));
1197  
  }
1198  
1199  
  O serveMultiAdd() {
1200  
    S key = trim(params.get('key)), value = trim(params.get('value));
1201  
    if (empty(key) || empty(value)) ret "Need key and value";
1202  
    new Matches m;
1203  
    new L<Page> pages;
1204  
    for (S a, b : params)
1205  
      if (startsWith(a, "page_", m))
1206  
        addIfNotNull(pages, conceptWhere(cc, Page, globalID := m.rest()));
1207  
    if (empty(pages)) ret "No pages selected";
1208  
    for (Page page : pages)
1209  
      new GetEntriesAndPost(cc).go(page, params);
1210  
    ret "Entry added to " + nPages(pages);
1211  
  }
1212  
1213  
} // end of Request
1214  
1215  
svoid dbLog(O... params) {
1216  
  logStructure(programFile("db.log"), ll(params));
1217  
}
1218  
1219  
static IF1<Page, Map> pageToMap(O... _) {
1220  
  optPar bool withEntries;
1221  
  
1222  
  bool nameOnly = eqOneOf(optPar nameOnly(_), "1", true);
1223  
  if (nameOnly) ret (IF1<Page, Map>) p -> litmap(q := p.q);
1224  
  
1225  
  ret (IF1<Page, Map>) p -> {
1226  
    L<Entry> entries = findBackRefs(p, Entry);
1227  
    ret litorderedmap(
1228  
      q := p.q,
1229  
      nEntries := l(entries),
1230  
      created := p.created,
1231  
      modified := p._modified,
1232  
      entries := !withEntries ? null : map(entries, entryToMap(false)));
1233  
  };
1234  
}
1235  
1236  
static IF1<Entry, Map> entryToMap(bool withPage) {
1237  
  ret (IF1<Entry, Map>) e -> litorderedmap(
1238  
    created := e.created,
1239  
    i := e.count,
1240  
    key := e.key,
1241  
    value := e.value,
1242  
    q := withPage ? e.page->q : null,
1243  
    signer := getString globalID(e.signer!));
1244  
}
1245  
1246  
static IF1<Slice, Map> sliceToMap() {
1247  
  ret (IF1<Slice, Map>) s -> litorderedmap(
1248  
    created := s.created,
1249  
    globalID := str(s.globalID),
1250  
    name := s.name,
1251  
    caseID := s.caseID);
1252  
}
1253  
1254  
sO servePageToBot(Page page, SS params) {
1255  
  if (page == null) ret serveJSON(null);
1256  
  params = asCIMap(params);
1257  
  Map map = pageToMap(withEntries := valueIs1 withEntries(params)).get(page);
1258  
  ret serveJSON(map);
1259  
}
1260  
1261  
sclass GetEntriesAndPost {
1262  
  Concepts cc;
1263  
  L<Entry> entries;
1264  
  Entry entry;
1265  
  bool newEntry;
1266  
  Signer signer;
1267  
  
1268  
  *(Concepts *cc) {}
1269  
  
1270  
  GetEntriesAndPost go(Page page, SS params) {
1271  
    S key = trim(params.get('key)), value = trim(params.get('value));
1272  
    print("GetEntriesAndPost: " + quote(page.q) + ", " + quote(key) + " := " + quote(value));
1273  
    withDBLock {
1274  
      entries = findBackRefs(page, Entry);
1275  
      if (nempty(key) && nempty(value)) {
1276  
        S ip = subBot_clientIP();
1277  
        entry = firstThat(e -> eqic(e.key, key) && eq_icIf(!allowMultipleCasesInValues, e.value, value) && eq(e.ip, ip), entries);
1278  
        if (entry == null) {
1279  
          print("SAVING");
1280  
          Entry e = cnew(cc, Entry, +page, +key, +value, +ip, count := l(entries) + 1, +signer);
1281  
          if (makeAllKeysPages) pageFromQ(cc, key);
1282  
          if (makeAllValuesPages) pageFromQ(cc, value);
1283  
          page.change(); // bump modification date
1284  
          entry = e;
1285  
          newEntry = true;
1286  
          entries.add(entry);
1287  
          dbLog("New entry", page := page.q, globalID := e.globalID, count := e.count, +key, +value);
1288  
        }
1289  
      }
1290  
    }
1291  
1292  
    sortByFieldInPlace created(entries);
1293  
    numberEntriesInConceptField count(entries);
1294  
    this;
1295  
  }
1296  
}
1297  
1298  
static SS availableSlices() {
1299  
  L<Slice> slices = asList(idx_slicesByName.objectIterator());
1300  
  ret mapToOrderedMap(s -> pair(str(s.globalID), s.name + " [ID: " + s.globalID + "]"), slices);
1301  
}
1302  
1303  
static L<Entry> entriesOnPage(Page p) {
1304  
  ret p == null ? null : sortedByField count(findBackRefs(p, Entry));
1305  
}
1306  
1307  
static L<Page> pagesSortedByLength(L<Page> l) {
1308  
  ret sortedByCalculatedField(l, p -> l(p.q));
1309  
}
1310  
1311  
sbool agiBlue_isOriginal() {
1312  
  ret amProgram(#1023558);
1313  
}
1314  
1315  
sS agiBlueURL(S subUri) { ret agiBlueURL() + prependSlash(subUri); }
1316  
1317  
sS agiBlueURL() {
1318  
  ret agiBlue_isOriginal() ? "https://agi.blue" :  "/" + psI(programID()) + "/raw";
1319  
}
1320  
1321  
sS agiBlueName() {
1322  
  ret agiBlue_isOriginal() ? "agi.blue" : "agi.blue clone " + programID();
1323  
}
1324  
1325  
svoid cleanMeUp {
1326  
  cleanUp(fan);
1327  
}
1328  
1329  
static LoadedSlice loadSlice(Slice slice) {
1330  
  ret loadSlice(slice.caseID, str(slice.globalID));
1331  
}
1332  
1333  
static LoadedSlice loadSlice(S caseID, S globalID) {
1334  
  caseID = unnull(caseID);
1335  
  lock fan.lock;
1336  
  
1337  
  // move slice from background to foreground
1338  
  
1339  
  Concepts cc = backgroundFan.getIfLoaded(caseID);
1340  
  if (cc != null) {
1341  
    LoadedSlice slice = assertNotNull(backgroundSlices.get(caseID));
1342  
    backgroundFan.loaded.remove(caseID);
1343  
    fan.loaded.put(caseID, cc);
1344  
    backgroundSlices.remove(caseID);
1345  
    loadedSlices.put(caseID, slice);
1346  
    ret slice;
1347  
  }
1348  
  
1349  
  fan.get(caseID, globalID);
1350  
  ret loadedSlices.get(caseID);
1351  
}
1352  
1353  
// load background slice
1354  
static LoadedSlice loadBackgroundSlice(Slice slice) {
1355  
  ret loadBackgroundSlice(slice.caseID, str(slice.globalID));
1356  
}
1357  
1358  
static LoadedSlice loadBackgroundSlice(S caseID, S globalID) {
1359  
  caseID = unnull(caseID);
1360  
  lock fan.lock;
1361  
  
1362  
  // check if in foreground
1363  
  
1364  
  if (fan.getIfLoaded(caseID) != null)
1365  
    ret loadedSlices.get(caseID);
1366  
1367  
  backgroundFan.get(caseID, globalID);
1368  
  LoadedSlice slice = assertNotNull(backgroundSlices.get(caseID));
1369  
  slice.lastAccess = sysNow();
1370  
  while (l(backgroundSlices) > maxBackgroundSlices)
1371  
    unloadLeastRecentlyAccessedBackgroundSlice();
1372  
  assertNotNull("slice.sliceConcept", slice.sliceConcept);
1373  
  ret slice;
1374  
}
1375  
1376  
// TODO: sync unloading slice with accessors
1377  
svoid unloadLeastRecentlyAccessedBackgroundSlice() {
1378  
  lock fan.lock;
1379  
  S caseID = lowestByField lastAccess(keys(backgroundSlices));
1380  
  if (caseID != null)
1381  
    backgroundFan.unloadCase(caseID);
1382  
}
1383  
1384  
static Slice sliceConceptForGlobalID(S globalID) {
1385  
  ret sliceConceptForGlobalID(toGlobalIDObj(globalID));
1386  
}
1387  
1388  
static Slice sliceConceptForGlobalID(GlobalID globalID) {
1389  
  ret conceptWhere Slice(+globalID);
1390  
}
1391  
1392  
static Slice sliceConceptForCaseID(S caseID) {
1393  
  ret conceptWhere Slice(+caseID);
1394  
}
1395  
1396  
sS agiBlueNameHTML() {
1397  
  ret ahref(agiBlueURL(), htmlEncode2(agiBlueName()));
1398  
}
1399  
1400  
sS agiBlueNameHTML_boldWithSize() {
1401  
  Long size = sizeOnDisk.peek();
1402  
  ret b(agiBlueNameHTML()) + (size == null ? "" : " " + spanTitle("Size of agi.blue's database on disk", "(" + toM(size) + " MB)"));
1403  
}
1404  
1405  
static Slice sliceForName(S name) {
1406  
  ret conceptWhereCI Slice(+name);
1407  
}
1408  
1409  
sclass Query extends WorksOnSlice {
1410  
  SS vars = ciMap();
1411  
    
1412  
  *(LoadedSlice slice) { super(slice); }
1413  
  
1414  
  O process(S query) {
1415  
    try object processLines(agiBlue_parseQueryScript(query));
1416  
    ret serveJSON("No return statement");
1417  
  }
1418  
    
1419  
  O processLines(L<ALQLLine> lines) {
1420  
    for (ALQLLine line : lines) {
1421  
      if (line cast ALQLSlice) {
1422  
        Slice slice = sliceForName(line.slice);
1423  
        if (slice == null) ret serveJSON("Slice not found: " + line.slice);
1424  
        LoadedSlice oldSlice = Query.this.slice;
1425  
        try {
1426  
          setSlice(loadSlice(slice));
1427  
          try object processLines(line.contents);
1428  
        } finally {
1429  
          setSlice(oldSlice);
1430  
        }
1431  
      } else if (line cast ALQLReturn)
1432  
        ret serveJSON(ll(getOrKeep(vars, line.var)));
1433  
      else if (line cast ALQLTriple) {
1434  
        T3S t = line.triple;
1435  
        t = tripleMap(t, s -> getOrKeep(vars, s));
1436  
        Cl<Page> pages;
1437  
        S var;
1438  
        if (isDollarVar(t.c)) {
1439  
          var = t.c;
1440  
          if (isDollarVar(t.a)) {
1441  
            if (isDollarVar(t.b)) todo(t);
1442  
            Entry e = random(conceptsWhereCI(cc, Entry, key := t.b));
1443  
            if (e == null) ret serveJSON("No results for " + var);
1444  
            pages = pagesForKeyAndValue(t.b, t.c);
1445  
            vars.put(t.a, e.page->q);
1446  
            vars.put(t.c, e.value);
1447  
            continue;
1448  
          } else if (isDollarVar(t.b)) {
1449  
            Page page = findPageFromQ(t.a);
1450  
            Entry e = random(findBackRefs(page, Entry));
1451  
            if (e == null) ret serveJSON("No results for " + var);
1452  
            vars.put(t.b, e.key);
1453  
            vars.put(t.c, e.value);
1454  
            continue;
1455  
          } else {
1456  
            S val = getValue(t.a, t.b);
1457  
            if (val == null) ret serveJSON("No results for " + var);
1458  
            pages = ll(pageFromQ(val));
1459  
          }
1460  
        } else if (isDollarVar(t.b)) {
1461  
          var = t.b;
1462  
          if (isDollarVar(t.c)) todo(t);
1463  
          if (isDollarVar(t.a)) {
1464  
            L<Entry> entries = conceptsWhereCI(cc, Entry, value := t.c);
1465  
            if (empty(entries)) ret serveJSON("No results for " + var);
1466  
            Entry e = random(entries);
1467  
            vars.put(t.a, e.page->q);
1468  
            vars.put(t.b, e.key);
1469  
            continue;
1470  
          } else {
1471  
            Cl<S> keys = keysForPageAndValue(t.a, t.c);
1472  
            if (empty(keys)) ret serveJSON("No results for " + var);
1473  
            pages = map(f<S, Page> pageFromQ, keys);
1474  
          }
1475  
        } else {
1476  
          var = t.a;
1477  
          if (!isDollarVar(t.a)) todo(t);
1478  
          if (isDollarVar(t.b)) todo(t);
1479  
          if (isDollarVar(t.c)) todo(t);
1480  
          pages = pagesForKeyAndValue(t.b, t.c);
1481  
        }
1482  
        if (empty(pages)) ret serveJSON("No results for " + var);
1483  
        vars.put(var, random(pages).q);
1484  
      } else if (line cast ALQLPage) {
1485  
        if (eq(line.matchMethod, 'eqic))
1486  
          findPageFromQ(line.page);
1487  
        else if (eq(line.matchMethod, 'flexMatchDollarVarsIC_first)) {
1488  
          new L<SS> l;
1489  
          for (Page p : list(cc, Page))
1490  
            addIfNotNull(l, flexMatchDollarVarsIC_first(line.page, p.q));
1491  
          if (empty(l)) ret serveJSON("No results for " + curly(line.page));
1492  
          vars.putAll(random(l));
1493  
        } else
1494  
          fail("Unknown match method: " + line.matchMethod);
1495  
      } else
1496  
        fail("Can't interpret: " + line);
1497  
    }
1498  
    
1499  
    null;
1500  
  }
1501  
}
1502  
1503  
static Page pageFromQ(Concepts cc, S q) {
1504  
  ret empty(q) ? null : uniqCI_sync(cc, Page, +q);
1505  
}
1506  
1507  
/*sS globalIDFromCaseID(S caseID) {
1508  
  ret assertGlobalID(takeLast(globalIDLength(), caseID));
1509  
}*/
1510  
1511  
sS agiBlue_pageURLWithSlice(Page p) {
1512  
  ret p == null ? null : agiBlueURL() + hquery(slice := _get(sliceForPage(p), 'globalID), q := p.q);
1513  
}
1514  
1515  
static Slice sliceForPage(Page p) {
1516  
  if (p == null) null;
1517  
  SliceInfo info = conceptWhere(p._concepts, SliceInfo);
1518  
  ret info == null ? null : sliceConceptForGlobalID(info.globalID);
1519  
}
1520  
1521  
static GlobalID mainSliceGlobalID() {
1522  
  ret conceptWhere Slice(caseID := "").globalID;
1523  
}
1524  
1525  
static File sliceDumpFile(Slice slice) {
1526  
  ret programFile("slice-dumps/" + or2(slice.caseID, "main") + ".slice");
1527  
}
1528  
1529  
static File dumpSliceToFile(LoadedSlice slice) {
1530  
  ret withDBLock(slice.cc, func -> File {
1531  
    cset(slice.sliceConcept, sliceDumped := now());
1532  
    File file = sliceDumpFile(slice.sliceConcept);
1533  
    new LS lines;
1534  
    for (Page p : list(slice.cc, Page))
1535  
      lines.add(quote(p.q));
1536  
    for (Entry e : list(slice.cc, Entry))
1537  
      lines.add(sfu(tripleToList(entryToTriple(e))));
1538  
    saveLinesAsTextFile(file, lines);
1539  
    ret file;
1540  
  });
1541  
}
1542  
1543  
static T3S entryToTriple(Entry e) {
1544  
  ret e == null ? null : t3(e.page->q, e.key, e.value);
1545  
}
1546  
1547  
static ConceptsLoadedOnDemand slicesLoadedOnDemand(Map<S, LoadedSlice> loadedSlices) {
1548  
  new ConceptsLoadedOnDemand fan;
1549  
  fan.onCaseLoaded(voidfunc(S caseID, O globalID, Concepts concepts) {
1550  
    LoadedSlice ls = new(caseID, concepts);
1551  
    ls.sliceConcept = sliceConceptForCaseID(caseID);
1552  
1553  
    if (nempty(globalID))
1554  
      ls.initialSetup(toGlobalIDObj((S) globalID));
1555  
    loadedSlices.put(caseID, ls);
1556  
  });
1557  
  fan.onUnloadingCase(voidfunc(S caseID, Concepts concepts) {
1558  
    loadedSlices.remove(caseID);
1559  
  });
1560  
  ret fan;
1561  
}
1562  
1563  
// check if slices are loaded both in foreground and background (BAD)
1564  
sS checkDoubleLoads() {
1565  
  LS l = sharedKeys(loadedSlices, backgroundSlices);
1566  
  ret empty(l) ? null : "ERROR: Slice loaded in background and foreground: " + first(l);
1567  
}
1568  
1569  
// just use size of concepts.structure.gz
1570  
static long estimatedSliceDataSize(Slice slice) {
1571  
  ret estimatedSliceDataSize(slice.caseID);
1572  
}
1573  
1574  
static long estimatedSliceDataSize(S caseID) {
1575  
  ret l(conceptsFile(fan.dbID(caseID)));
1576  
}
1577  
1578  
static CentralIndexEntry centralIndexGet(S s) { ret getOrCreate CentralIndexEntry(centralIndex, s); }
1579  
1580  
static CentralIndexEntry newCentralIndexGet(S s) { ret getOrCreate CentralIndexEntry(newCentralIndex, s); }
1581  
1582  
static Cl<Slice> centralIndexGetSlices(S s) {
1583  
  CentralIndexEntry e = centralIndexGet(s);
1584  
  ret e == null ? null : e.slices;
1585  
}
1586  
1587  
// executed by rst_index
1588  
svoid makeCentralIndex() {
1589  
  newCentralIndex.clear();
1590  
  pcall {
1591  
    for (Slice slice : conceptsWhere(Slice, indexed := true))
1592  
      addToNewCentralIndex(slice);
1593  
  }
1594  
  centralIndex.putAll(newCentralIndex);
1595  
  removeAllKeysNotIn(centralIndex, newCentralIndex);
1596  
  newCentralIndex.clear();
1597  
  centralIndexMade = now();
1598  
}
1599  
1600  
svoid addToNewCentralIndex(Slice slice) {
1601  
  for (S s : sliceDump_pageNamesIncludingKeys(sliceDumpFile(slice)))
1602  
    newCentralIndexGet(s).slices.add(slice);
1603  
}

Author comment

Began life as a copy of #1023558

download  show line numbers  debug dex   

Travelled to 4 computer(s): bhatertpkbcr, mqqgnosmbjvj, pzhvpgtvlbxg, tvejysmllsmz

No comments. add comment

Snippet ID: #1024454
Snippet name: agi.blue source [backup before source lib]
Eternal ID of this version: #1024454/2
Text MD5: 6cbc45b79bd93f390dff6468dcaf2dc9
Transpilation MD5: 5cfefbbb6e66e686bb7b2b827aafb20c
Author: stefan
Category: javax / html
Type: JavaX module (desktop)
Public (visible to everyone): Yes
Archived (hidden from active list): No
Created/modified: 2019-08-13 14:36:41
Source code size: 56527 bytes / 1603 lines
Pitched / IR pitched: No / No
Views / Downloads: 16 / 28
Version history: 1 change(s)
Referenced in: [show references]

Formerly at http://tinybrain.de/1024454 & http://1024454.tinybrain.de