Not logged in.  Login/Logout/Register | List snippets | | Create snippet | Upload image | Upload data

1505
LINES

< > BotCompany Repo | #1024512 // agi.blue productive [LIVE]

JavaX module (desktop) [tags: use-pretranspiled]

Download Jar. Uses 1658K of libraries. Click here for Pure Java version (24905L/147K).

1  
!7
2  
3  
// See #1023660 for the older 99 lines version
4  
5  
!include once #1024472 // agi.blue core db
6  
7  
// global constants
8  
9  
static SS searchTypeToText = litmap(
10  
  leven := "Leven",
11  
  literal := "Literal",
12  
  scored := "Scored");
13  
  
14  
// global vars
15  
16  
static int sourceCodeLines;
17  
18  
static RealAGIBlue agiBlue;
19  
static Renderer renderer = new Renderer1;
20  
21  
set flag AllPublic.
22  
23  
sS classesToShare = [[
24  
  DynamicObject Concept Concepts Page Entry Slice LoadedSlice
25  
  IConceptIndex IFieldIndex IConceptCounter ConceptFieldIndexCI ConceptFieldIndexDesc
26  
  AbstractEntry Signer Session User GoogleUser MinimalUser CookieUser
27  
  BlobEntry
28  
  SliceInfo CentralIndexEntry AGIBlue RealAGIBlue TimedCache
29  
  GlobalID Renderer
30  
  IterableIterator CloseableIterableIterator
31  
]];
32  
33  
// end of global data
34  
35  
p {
36  
  agiBlue = new RealAGIBlue;
37  
}
38  
39  
svoid _onLoad_sourceCodeLines {
40  
  sourceCodeLines = countLines(mySource());
41  
}
42  
43  
svoid cleanMeUp {
44  
  cleanUp(agiBlue);
45  
  agiBlue = null;
46  
}
47  
48  
sinterface Renderer {
49  
  O html(S uri, SS params);
50  
}
51  
52  
set flag NoNanoHTTPD. html {
53  
  printWithTime("Starting request");
54  
  O o = renderer.html(uri, params);
55  
  printWithTime("Done with request");
56  
  ret o;
57  
}
58  
59  
sclass RealAGIBlue > AGIBlue {
60  
  volatile bool started, broken;
61  
  TimedCache<Long> sizeOnDisk = new(60.0, f guessClusteredSizeOfProgramDirWithoutBackups);
62  
  
63  
  *() {
64  
    
65  
    try {
66  
      if (bootstrapDataFrom(#1023558)) deleteMyBackups();
67  
      thinMyBackups();
68  
      fan = slicesLoadedOnDemand(loadedSlices);
69  
      mainConcepts = fan.get("", null);
70  
      
71  
      // index main DB
72  
  
73  
      indexConceptFields(Slice, 'globalID, Slice, 'caseID);
74  
      indexConceptField(CookieUser, 'cookie);
75  
      idx_slicesByName = new ConceptFieldIndexCI(Slice, 'name);
76  
      idx_slicesByModification = new ConceptFieldIndexDesc(Slice, '_modified);
77  
      idx_usersByName = new ConceptFieldIndexCI(GoogleUser, 'googleLastName);
78  
      idx_usersBySeen = new ConceptFieldIndexDesc(User, 'lastSeen);
79  
  
80  
      cset(uniq_returnIfNew Slice(caseID := ""), name := "main slice");
81  
      loadedSlices.get("").initialSetup(toGlobalIDObj(mainSliceGlobalID()));
82  
      loadedSlices.get("").sliceConcept = sliceConceptForCaseID("");
83  
      
84  
      backgroundFan = slicesLoadedOnDemand(backgroundSlices);
85  
      backgroundFan.lock = fan.lock;
86  
      
87  
      // legacy conversions!
88  
      //deleteConcepts(Slice, globalID := GlobalID('wftlawbagrwprywn));
89  
    
90  
      // Approve this machine's key
91  
      PKIKeyPair machineKey = agiBot_trustedKeyForMachine();
92  
      if (machineKey != null) {
93  
        print("Approving this machine's key: " + machineKey.publicKey);
94  
        cset(uniq_sync(Signer, publicKey := machineKey.publicKey), trusted := true, approvedBy := "local");
95  
      }
96  
      
97  
      rst_index.trigger();
98  
    } catch print e {
99  
      set broken;
100  
    } finally {
101  
      set started;
102  
    }
103  
    
104  
    // do stuff after we are officially started
105  
    
106  
    // calculate DB size regularly iff there were changes
107  
    sizeOnDisk.keepValueWhileCalculating = true;
108  
    doEveryAndNow(60.0, rWatcher(() -> db_mainConceptsChangeCount(), r { sizeOnDisk! }));
109  
    
110  
  } // end of main program
111  
112  
  L<Entry> entriesOnPage(Page p) {
113  
    ret p == null ? null : sortedByField count(findBackRefs(p, Entry));
114  
  }
115  
  
116  
  L<Page> pagesSortedByLength(L<Page> l) {
117  
    ret sortedByCalculatedField(l, p -> l(p.q));
118  
  }
119  
  
120  
  sbool agiBlue_isOriginal() {
121  
    ret amProgram(#1023558);
122  
  }
123  
  
124  
  S agiBlueURL(S subUri) { ret agiBlueURL() + prependSlash(subUri); }
125  
  
126  
  S agiBlueURL() {
127  
    ret agiBlue_isOriginal() ? "https://agi.blue" :  "/" + psI(programID()) + "/raw";
128  
  }
129  
130  
  S agiBlueName() {
131  
    ret agiBlue_isOriginal()
132  
      ? "agi.blue"
133  
      : isProgramID(#1024512) ? "agi.blue productive"
134  
      : "agi.blue clone " + programID();
135  
  }
136  
  
137  
  void cleanMeUp {
138  
    cleanUp(fan);
139  
  }
140  
  
141  
  LoadedSlice loadSlice(Slice slice) {
142  
    ret loadSlice(slice.caseID, str(slice.globalID));
143  
  }
144  
  
145  
  LoadedSlice loadSlice(S caseID, S globalID) {
146  
    caseID = unnull(caseID);
147  
    lock fan.lock;
148  
    
149  
    // move slice from background to foreground
150  
    
151  
    Concepts cc = backgroundFan.getIfLoaded(caseID);
152  
    if (cc != null) {
153  
      LoadedSlice slice = assertNotNull(backgroundSlices.get(caseID));
154  
      backgroundFan.loaded.remove(caseID);
155  
      fan.loaded.put(caseID, cc);
156  
      backgroundSlices.remove(caseID);
157  
      loadedSlices.put(caseID, slice);
158  
      ret slice;
159  
    }
160  
    
161  
    fan.get(caseID, globalID);
162  
    ret loadedSlices.get(caseID);
163  
  }
164  
  
165  
  void unloadSlice(S caseID) {
166  
    fan.unloadCase(caseID);
167  
    backgroundFan.unloadCase(caseID);
168  
  }
169  
  
170  
  // load background slice
171  
  LoadedSlice loadBackgroundSlice(Slice slice) {
172  
    ret loadBackgroundSlice(slice.caseID, str(slice.globalID));
173  
  }
174  
  
175  
  LoadedSlice loadBackgroundSlice(S caseID, S globalID) {
176  
    caseID = unnull(caseID);
177  
    lock fan.lock;
178  
    
179  
    // check if in foreground
180  
    
181  
    if (fan.getIfLoaded(caseID) != null)
182  
      ret loadedSlices.get(caseID);
183  
  
184  
    backgroundFan.get(caseID, globalID);
185  
    LoadedSlice slice = assertNotNull(backgroundSlices.get(caseID));
186  
    slice.lastAccess = sysNow();
187  
    while (l(backgroundSlices) > maxBackgroundSlices)
188  
      unloadLeastRecentlyAccessedBackgroundSlice();
189  
    assertNotNull("slice.sliceConcept", slice.sliceConcept);
190  
    ret slice;
191  
  }
192  
  
193  
  // TODO: sync unloading slice with accessors
194  
  void unloadLeastRecentlyAccessedBackgroundSlice() {
195  
    lock fan.lock;
196  
    S caseID = lowestByField lastAccess(keys(backgroundSlices));
197  
    if (caseID != null)
198  
      backgroundFan.unloadCase(caseID);
199  
  }
200  
  
201  
  Slice sliceConceptForGlobalID(S globalID) {
202  
    ret sliceConceptForGlobalID(toGlobalIDObj(globalID));
203  
  }
204  
  
205  
  Slice sliceConceptForGlobalID(GlobalID globalID) {
206  
    ret conceptWhere Slice(+globalID);
207  
  }
208  
  
209  
  Slice sliceConceptForCaseID(S caseID) {
210  
    ret conceptWhere Slice(+caseID);
211  
  }
212  
  
213  
  Slice sliceForName(S name) {
214  
    ret conceptWhereCI Slice(+name);
215  
  }
216  
  
217  
  class Query extends WorksOnSlice {
218  
    SS vars = ciMap();
219  
      
220  
    *(LoadedSlice slice) { super(slice); }
221  
    
222  
    O process(S query) {
223  
      try object processLines(agiBlue_parseQueryScript(query));
224  
      ret serveJSON("No return statement");
225  
    }
226  
      
227  
    O processLines(L<ALQLLine> lines) {
228  
      for (ALQLLine line : lines) {
229  
        if (line cast ALQLSlice) {
230  
          Slice slice = sliceForName(line.slice);
231  
          if (slice == null) ret serveJSON("Slice not found: " + line.slice);
232  
          LoadedSlice oldSlice = Query.this.slice;
233  
          try {
234  
            setSlice(loadSlice(slice));
235  
            try object processLines(line.contents);
236  
          } finally {
237  
            setSlice(oldSlice);
238  
          }
239  
        } else if (line cast ALQLReturn)
240  
          ret serveJSON(ll(getOrKeep(vars, line.var)));
241  
        else if (line cast ALQLTriple) {
242  
          T3S t = line.triple;
243  
          t = tripleMap(t, s -> getOrKeep(vars, s));
244  
          Cl<Page> pages;
245  
          S var;
246  
          if (isDollarVar(t.c)) {
247  
            var = t.c;
248  
            if (isDollarVar(t.a)) {
249  
              if (isDollarVar(t.b)) todo(t);
250  
              Entry e = random(conceptsWhereCI(cc, Entry, key := t.b));
251  
              if (e == null) ret serveJSON("No results for " + var);
252  
              pages = pagesForKeyAndValue(t.b, t.c);
253  
              vars.put(t.a, e.page->q);
254  
              vars.put(t.c, e.value);
255  
              continue;
256  
            } else if (isDollarVar(t.b)) {
257  
              Page page = findPageFromQ(t.a);
258  
              Entry e = random(findBackRefs(page, Entry));
259  
              if (e == null) ret serveJSON("No results for " + var);
260  
              vars.put(t.b, e.key);
261  
              vars.put(t.c, e.value);
262  
              continue;
263  
            } else {
264  
              S val = getValue(t.a, t.b);
265  
              if (val == null) ret serveJSON("No results for " + var);
266  
              pages = ll(pageFromQ(val));
267  
            }
268  
          } else if (isDollarVar(t.b)) {
269  
            var = t.b;
270  
            if (isDollarVar(t.c)) todo(t);
271  
            if (isDollarVar(t.a)) {
272  
              L<Entry> entries = conceptsWhereCI(cc, Entry, value := t.c);
273  
              if (empty(entries)) ret serveJSON("No results for " + var);
274  
              Entry e = random(entries);
275  
              vars.put(t.a, e.page->q);
276  
              vars.put(t.b, e.key);
277  
              continue;
278  
            } else {
279  
              Cl<S> keys = keysForPageAndValue(t.a, t.c);
280  
              if (empty(keys)) ret serveJSON("No results for " + var);
281  
              pages = map(f<S, Page> pageFromQ, keys);
282  
            }
283  
          } else {
284  
            var = t.a;
285  
            if (!isDollarVar(t.a)) todo(t);
286  
            if (isDollarVar(t.b)) todo(t);
287  
            if (isDollarVar(t.c)) todo(t);
288  
            pages = pagesForKeyAndValue(t.b, t.c);
289  
          }
290  
          if (empty(pages)) ret serveJSON("No results for " + var);
291  
          vars.put(var, random(pages).q);
292  
        } else if (line cast ALQLPage) {
293  
          if (eq(line.matchMethod, 'eqic))
294  
            findPageFromQ(line.page);
295  
          else if (eq(line.matchMethod, 'flexMatchDollarVarsIC_first)) {
296  
            new L<SS> l;
297  
            for (Page p : list(cc, Page))
298  
              addIfNotNull(l, flexMatchDollarVarsIC_first(line.page, p.q));
299  
            if (empty(l)) ret serveJSON("No results for " + curly(line.page));
300  
            vars.putAll(random(l));
301  
          } else
302  
            fail("Unknown match method: " + line.matchMethod);
303  
        } else
304  
          fail("Can't interpret: " + line);
305  
      }
306  
      
307  
      null;
308  
    }
309  
  }
310  
  
311  
  /*S globalIDFromCaseID(S caseID) {
312  
    ret assertGlobalID(takeLast(globalIDLength(), caseID));
313  
  }*/
314  
  
315  
  S agiBlue_pageURLWithSlice(Page p) {
316  
    ret p == null ? null : agiBlueURL() + hquery(slice := _get(sliceForPage(p), 'globalID), q := p.q);
317  
  }
318  
  
319  
  Slice sliceForPage(Page p) {
320  
    if (p == null) null;
321  
    SliceInfo info = conceptWhere(p._concepts, SliceInfo);
322  
    ret info == null ? null : sliceConceptForGlobalID(info.globalID);
323  
  }
324  
  
325  
  GlobalID mainSliceGlobalID() {
326  
    ret conceptWhere Slice(caseID := "").globalID;
327  
  }
328  
  
329  
  ConceptsLoadedOnDemand slicesLoadedOnDemand(Map<S, LoadedSlice> loadedSlices) {
330  
    new ConceptsLoadedOnDemand fan;
331  
    fan.onCaseLoaded(voidfunc(S caseID, O globalID, Concepts concepts) {
332  
      LoadedSlice ls = new(caseID, concepts);
333  
      concepts.miscMap = mapPutOrCreate_multi(concepts.miscMap,
334  
        agiBlue := RealAGIBlue.this,
335  
        loadedSlice := ls);
336  
      ls.sliceConcept = sliceConceptForCaseID(caseID);
337  
  
338  
      if (nempty(globalID))
339  
        ls.initialSetup(toGlobalIDObj((S) globalID));
340  
      loadedSlices.put(caseID, ls);
341  
    });
342  
    fan.onUnloadingCase(voidfunc(S caseID, Concepts concepts) {
343  
      loadedSlices.remove(caseID);
344  
    });
345  
    ret fan;
346  
  }
347  
  
348  
  // check if slices are loaded both in foreground and background (BAD)
349  
  S checkDoubleLoads() {
350  
    LS l = sharedKeys(loadedSlices, backgroundSlices);
351  
    ret empty(l) ? null : "ERROR: Slice loaded in background and foreground: " + first(l);
352  
  }
353  
  
354  
  // just use size of concepts.structure.gz
355  
  long estimatedSliceDataSize(Slice slice) {
356  
    ret estimatedSliceDataSize(slice.caseID);
357  
  }
358  
  
359  
  long estimatedSliceDataSize(S caseID) {
360  
    ret l(conceptsFile(fan.dbID(caseID)));
361  
  }
362  
  
363  
  // restart magic!
364  
  void softRestart { // TODO
365  
    print("Cloning...");
366  
    cloningSince = sysNow();
367  
    Class c = hotwireDependent(programID());
368  
369  
    // New strategy: Share AGIBlue & just grab the new renderer :)
370  
371  
    copyFields(mc(), c, 'agiBlue, 'mainConcepts, 'creator_class);
372  
    renderer = (Renderer) get renderer(c);
373  
    print("Got new renderer.");
374  
    
375  
    // TODO: any clean up of this instance?
376  
  }
377  
} // end of RealAGIBlue
378  
379  
sclass WorksOnSlice {
380  
  delegate LoadedSlice to AGIBlue.
381  
    
382  
  Concepts cc;
383  
  LoadedSlice slice;
384  
  
385  
  *() {}
386  
  *(LoadedSlice *slice) { cc = slice.cc; }
387  
  
388  
  void setSlice(LoadedSlice slice) {
389  
    this.slice = slice;
390  
    cc = slice.cc;
391  
  }
392  
  
393  
  Page findPageFromParams(Map map) {
394  
    S q = getString q(map);
395  
    ret empty(q) ? null : findPageFromQ(q);
396  
  }
397  
398  
  Page findOrMakePageFromParams(Map map) {
399  
    ret pageFromQ(getString q(map));
400  
  }
401  
  
402  
  Page findPageFromQ(S q) {
403  
    ret conceptWhereCI(cc, Page, +q);
404  
  }
405  
  
406  
  Page pageFromQ(S q) {
407  
    ret slice.pageFromQ(q);
408  
  }
409  
  
410  
  Set<Page> pagesForKeyAndValue(S key, S value) {
411  
    L<Entry> entries = conceptsWhereIC(cc, Entry, +value, +key);
412  
    ret asSet(ccollect page(entries));
413  
  }
414  
  
415  
  Cl<S> keysForPageAndValue(S q, S value) {
416  
    Page page = findPageFromQ(q);
417  
    if (page == null) null;
418  
    ret collect key(conceptsWhereIC(cc, Entry, +page, +value));
419  
  }
420  
421  
  // DB functions
422  
423  
  bool hasPage(S q) { ret hasConceptWhereIC(Page, +q); }
424  
  S getValue(Page page, S key) {
425  
    ret page == null || empty(key) ? null : getString value(highestByField count(objectsWhereIC(findBackRefs(page, Entry), +key)));
426  
  }
427  
  S getValue(S page, S key) {
428  
    ret getValue(findPageFromQ(page), key);
429  
  }
430  
  S pageDisplayName(Page page) {
431  
    /*S name = getValue(page, "read as");
432  
    bool unnaturalName = nempty(name) && !eq(makeAGIDomain(name), page.url);
433  
    ret unnaturalName ? name + " " + squareBracketed(page.url) : or2(name, unpackAGIDomainOpt(page.url));*/
434  
    ret page.q;
435  
  }
436  
  
437  
  S renderThing(S s, bool forceURLDisplay) {
438  
    ret forceURLDisplay || isURL(s) || isAGIDomain(s)
439  
      ? ahref(fixAGILink(absoluteURL(s)), htmlencode2(shorten(agiBlue.displayLength, s)))
440  
      : ahref(agiBlue_linkForPhrase(s,
441  
          slice := slice.isMainSlice() ? null : slice.sliceConcept.globalID),
442  
        htmlencode2(shorten(agiBlue.displayLength, s)));
443  
  }
444  
445  
  GlobalID sliceID() { ret slice.globalID(); }
446  
447  
  SliceInfo sliceInfo() { ret slice.sliceInfo; }
448  
449  
} // end of WorksOnSlice
450  
451  
452  
sclass Renderer1 implements Renderer {
453  
  // Serve page
454  
  
455  
  O html(S uri, SS params) {
456  
    sleepWhile(() -> !agiBlue.started);
457  
    if (agiBlue.broken) ret subBot_serve500("Internal error");
458  
    ret new Request().serve(uri, params);
459  
  }
460  
461  
  // forward things to AGIBlue object
462  
  delegate agiBlue_pageURLWithSlice to agiBlue.
463  
  delegate newCentralIndexGet to agiBlue.
464  
  delegate centralIndexMade to agiBlue.
465  
  delegate agiBlueURL to agiBlue.
466  
  delegate dumpSliceToFile to agiBlue.
467  
  delegate checkDoubleLoads to agiBlue.
468  
  delegate asyncSearch to agiBlue.
469  
  delegate centralIndexGetSlices to agiBlue.
470  
  delegate dbLog to agiBlue.
471  
  delegate entryToTriple to agiBlue.
472  
  delegate loadSlice to agiBlue.
473  
  delegate agiBlueName to agiBlue.
474  
  delegate idx_slicesByModification to agiBlue.
475  
  delegate idx_slicesByName to agiBlue.
476  
  delegate idx_usersBySeen to agiBlue.
477  
  delegate idx_usersByName to agiBlue.
478  
  delegate agiBlue_isOriginal to agiBlue.
479  
  delegate loadBackgroundSlice to agiBlue.
480  
  delegate maxSliceNameLength to agiBlue.
481  
  delegate pagesSortedByLength to agiBlue.
482  
  delegate entriesOnPage to agiBlue.
483  
  delegate centralIndex to agiBlue.
484  
  delegate showGoogleLogIn to agiBlue.
485  
  delegate rst_index to agiBlue.
486  
  delegate LoadedSlice to AGIBlue.
487  
  delegate centralIndexMadeIn to agiBlue.
488  
  delegate sideSearchType to agiBlue.
489  
  delegate sideDisplayLength to agiBlue.
490  
  delegate sliceConceptForGlobalID to agiBlue.
491  
  delegate mainSliceGlobalID to agiBlue.
492  
  
493  
  class Request extends WorksOnSlice {
494  
    S cookie;
495  
    Session session;
496  
    Slice sliceConcept;
497  
    S uri;
498  
    SS params;
499  
    long started = sysNow();
500  
    S q, domain;
501  
    Set<S> get;
502  
    
503  
    // when serving a concept page, info that may be grabbed
504  
    // by thought bots
505  
    L<Entry> usesAsKey;
506  
    
507  
    // should also work for standalone
508  
    bool isHomeDomain() {
509  
      S domain = domain();
510  
      ret eqic(domain, "www.agi.blue") || !ewic(domain, ".agi.blue");
511  
    }
512  
    
513  
    O serve(S uri, SS params) {
514  
      this.uri = dropMultipleSlashes(uri);
515  
      this.params = params;
516  
      new Matches m;
517  
      
518  
      print(uri + " ? " + unnull(subBot_query()));
519  
      
520  
      // get cookie & session
521  
      
522  
      cookie = cookieFromUser();
523  
      if (cookie != null)
524  
 {
525  
        session = uniq_sync(Session, +cookie);
526  
        if (session.user! == null)
527  
          cset(session, user := uniq_sync CookieUser(+cookie));
528  
      } else
529  
        session = unlistedWithValues(Session);
530  
        
531  
      // check if user wants to change slices
532  
        
533  
      S selectSlice = params.get('slice);
534  
      if (isGlobalID(selectSlice))
535  
        cset(session, selectedSlice := selectSlice);
536  
      if (session.selectedSlice == null)
537  
        cset(session, selectedSlice := str(mainSliceGlobalID()));
538  
  
539  
      // get slice's case ID
540  
      
541  
      S caseID = "";
542  
      sliceConcept = sliceConceptForGlobalID(session.selectedSlice);
543  
      print("Selected slice: " + session.selectedSlice + ", obj? " + (sliceConcept != null));
544  
      if (sliceConcept != null) caseID = sliceConcept.caseID;
545  
      print("caseID: " + caseID);
546  
      
547  
      // load slice
548  
      
549  
      slice = assertNotNull(loadSlice(caseID, session.selectedSlice));
550  
      cc = slice.cc;
551  
      
552  
      // Check for special URIs
553  
  
554  
      if (swic(uri, "/bot/")) ret serveBot();
555  
  
556  
      if (swic(uri, "/user/", m)) {
557  
        User user = conceptWhere User(globalID := toGlobalIDObj(assertGlobalID(m.rest())));
558  
        if (user == null) ret subBot_serve404("User not found");
559  
        S title = "User " + user;
560  
        ret hhtml_miniPage(title, h3(agiBlueNameHTML() + " | " + title)
561  
          + "User type: " + classShortName(user));
562  
      }
563  
  
564  
      // eleu appends a slash to the URI if it's a single identifier, so we drop it again
565  
      S uri2 = dropTrailingSlash(uri);
566  
      
567  
      if (eqic(uri2, "/google-verify")) {
568  
        print("Google-verify started.");
569  
        Payload payload = printStruct(googleVerifyUserToken2(googleSignInID(), params.get("token")));
570  
        if (payload == null) ret print("google-verify", "No");
571  
        S email = payload.getEmail();
572  
        if (empty(email)) ret print("google-verify", "No");
573  
        
574  
        GoogleUser user = uniqCI_sync GoogleUser(googleEmail := email);
575  
        cset(user,
576  
          googleEmailVerified := payload.getEmailVerified(),
577  
          googleFirstName := strOrNull(payload.get("given_name")),
578  
          googleLastName := strOrNull(payload.get("family_name")));
579  
        
580  
        cset(session,
581  
 +user);
582  
          
583  
        ret print("google-verify", payload.getEmail() + " " + (payload.getEmailVerified() ? "(verified)" : "(not verified)"));
584  
      }
585  
  
586  
      cset(session.user!, lastSeen := now());      
587  
      
588  
      if (eqic(uri2, "/users")) ret serveUsersList();
589  
      
590  
      if (eqic(uri2, "/slices")) ret serveSlicesList();
591  
      
592  
      if (eqic(uri2, "/search")) ret serveScoredSearch();
593  
      if (eqic(uri2, "/literalSearch")) ret serveLiteralSearch();
594  
      if (eqic(uri2, "/levenSearch")) ret serveLevenSearch();
595  
      
596  
      if (eqic(uri2, "/query")) ret serveQueryPage();
597  
      if (eqic(uri2, "/createSlice")) ret serveCreateSlicePage();
598  
      if (eqic(uri2, "/deletePage")) ret serveDeletePage();
599  
      if (eqic(uri2, "/deleteSlice")) ret serveDeleteSlice();
600  
  
601  
      if (eqic(uri2, "/checkboxes")) ret serveCheckboxes();
602  
      if (eqic(uri2, "/multiAdd")) ret serveMultiAdd();
603  
604  
      if (eqic(uri2, "/version")) ret myTranspilationDate();
605  
606  
      if (eqic(uri2, "/restart") && authed()) {
607  
        agiBlue.softRestart();
608  
        ret "OK";
609  
      }
610  
      
611  
      q = params.get('q);
612  
      get = asCISet(nempties(subBot_paramsAsMultiMap().get('get)));
613  
      
614  
      domain = or2(params.get('domain), domain());
615  
      
616  
      S raw = firstKeyWithValue("", params); // agi.blue?something
617  
      if (nempty(raw) && empty(q)) q = raw;
618  
      /*if (nempty(q)) {
619  
        domain = makeAGIDomain(q);
620  
        if (l(domain) > maximumDomainPartLength()) // escape with "domain="
621  
          ret hrefresh(agiBlueURL() + hquery(+domain, key := "read as", value := q));
622  
        ret hrefresh("http://" + domain + (eq(q, domain) ? "" : "/" +  hquery(key := "read as", value := q)));
623  
        //uri = "/"; replaceMapWithParams(params, key := "read as", value := q);
624  
      }*/
625  
      S url = domain + dropTrailingSlash(uri);
626  
      
627  
      // domain to query
628  
      //if (empty(q)) q = url;
629  
      if (empty(q)) {
630  
        S qq = agiBlue_urlToQuery(url);
631  
        if (neqic(qq, "agi.blue")) q = qq;
632  
      }
633  
      
634  
      Page page, bool newPage = unpair uniqCI2_sync(cc, Page, +q);
635  
      if (newPage) dbLog("New page", +q);
636  
      //printStructs(+params, +raw, +q);
637  
    
638  
      if (empty(params.get('q)) && empty(raw) && isHomeDomain()) {
639  
        //L<Page> pages = sortedByFieldDesc _modified(list(cc, Page)); // TODO: use index
640  
        L<Page> pages = cloneList(slice.idx_latestChangedPages.objectIterator());
641  
        int start = parseInt(params.get("start")), step = 100;
642  
        S nav = pageNav2("/", l(pages), start, step, "start");
643  
        S content = hform(b("GIVE ME INPUT: &nbsp; ") + htextinput('q, autofocus := true) + " " + hsubmit("Ask"))
644  
          + h1(sliceAsHTML() + " has " + nPages(countConcepts(cc, Page)) + " and " + nConnections(countConcepts(cc, Entry)))
645  
          + p(nav)
646  
          + p_nemptyLines(map(pageToHTMLLink(), subList(pages, start, start+step)));
647  
  
648  
        ret hhtml_agiBlue(hhead_title("Slice " + sliceConcept().name + " | agi.blue") // SERVE SLICE HOME PAGE
649  
          + hbody(hfullcenterAndTopLeft(top() + content
650  
            + footer(),
651  
            sliceSelector()
652  
          )));
653  
      }
654  
          
655  
      S key = trim(params.get('key)), value = trim(params.get('value));
656  
      L<Entry> entries = agiBlue.new GetEntriesAndPost(cc).go(page, params).entries;
657  
      
658  
      //S get = params.get('get);
659  
      if (nempty(get))
660  
        ret serveJSON(collect value(llNotNulls(firstThat(entries, e -> get.contains(e.key)))));
661  
      
662  
      S key2 = key, value2 = value; if (nempty(key) && nempty(value)) key2 = value2 = ""; // input reset
663  
    
664  
      bool withHidden = eq(params.get('withHidden), "1");
665  
      new Set<Int> hide;
666  
      if (!withHidden) for (Entry e : entries) if (eqic(e.key, 'hide) && isSquareBracketedInt(e.value)) addAll(hide, e.count, parseInt(unSquareBracket(e.value)));
667  
      new MultiMap<Int, S> mmMeta;
668  
      for (Entry e : entries) if (isSquareBracketedInt(e.key)) mmMeta.put(parseInt(unSquareBracket(e.key)), e.value);
669  
      new MultiMap<S> mm;
670  
      for (Entry e : entries) mm.put(e.key, e.value);
671  
      
672  
      //S name = or2(/* ouch */ last(mm.get("read as")), /* end ouch */ unpackAGIDomain(page.url), page.url);
673  
      S name = page.q;
674  
      
675  
      // Find references
676  
      
677  
      L<Entry> refs = concatLists(conceptsWhereIC(cc, Entry, value := name),
678  
        conceptsWhereIC(cc, Entry, key := name));
679  
      Set<Page> refPages = asSet(ccollect page(refs));
680  
      refPages.remove(page); // don't list current page as reference
681  
      
682  
      // Search in page names (depending on default search type)
683  
      
684  
      L<Page> searchResults;
685  
      int totalResults = 0, searchResultsToShow = 50;
686  
      if (eq(sideSearchType, 'leven))
687  
        searchResults = levenSearch(page.q, max := searchResultsToShow);
688  
      else if (eq(sideSearchType, 'literal))
689  
 {
690  
        searchResults = literalSearch(page.q, max := maxInt());
691  
        print("totalResults", totalResults = l(searchResults));
692  
        searchResults = takeFirst(searchResultsToShow, searchResults);
693  
      } else
694  
        searchResults = (L<Page>) dm_call('agiBlueSearch, 'search, page.q, maxResult := clipIntPlus(searchResultsToShow, 1), agiBlueBotID := programID(), +cc);
695  
      searchResults.remove(page);
696  
      
697  
      // Find uses as key
698  
      usesAsKey = conceptsWhereIC(cc, Entry, key := page.q);
699  
  
700  
      // Find page in other slices
701  
      CentralIndexEntry cie = centralIndex.get(page.q);
702  
      L<Slice> pageInOtherSlices = cie == null ? null : listMinus(cie.slices, slice.sliceConcept);
703  
      
704  
      S pageName_html = htmlEncode2(shorten(agiBlue.displayLength, name));
705  
  
706  
      S mainContents =
707  
          top() + h1(ahref_unstyled(agiBlue_pageURLWithSlice(page), pageName_html) + (newPage ? " [huh????]" : ""))
708  
        + p_nemptyLines(map(entries, func(Entry e) -> S {
709  
          !withHidden && (hide.contains(e.count) || eqic(e.key, "read as") && eqic(e.value, name)) ? ""
710  
          : "[" + e.count + "] " +
711  
            renderThing(e.key, false) + ": " +
712  
            b(renderThing(e.value, cic(mmMeta.get(e.count), "is a URL")))
713  
          }))
714  
        + hpostform(h3("Add an entry")
715  
          + "Key: " + hinputfield(key := key2) + " Value: " + hinputfield(value := value2) + "<br><br>" + hsubmit("Add")
716  
          )
717  
          
718  
        + p(ahref(agiBlueURL() + "/literalSearch" + hquery(q := page.q), "[literal search]", title := "Search pages with a name containing this page's name literally")
719  
           + " " +
720  
           ahref(agiBlueURL() + "/levenSearch" + hquery(q := page.q), "[leven search 1]", title := "Search pages with a Levenshtein similarity of 1 containing this page's name literally")
721  
           + " " +
722  
           ahref(agiBlueURL() + "/search" + hquery(q := page.q), "[scored search]", title := "Search pages with ScoredSearch")
723  
           + " " +
724  
           ahref(agiBlueURL() + "/deletePage" + hquery(slice := sliceID(), q := page.q), "[delete page]")
725  
         )
726  
         
727  
         // optional "uses as key" section
728  
         
729  
         + (empty(usesAsKey) ? "" : h3(quote(pageName_html) + " as key")
730  
         + p_nemptyLines_showFirst(50, map(usesAsKey, e ->
731  
           pageToHTMLLink().get(e.page!) + unicode_spacedRightPointingTriangle() +
732  
             pageName_html + unicode_spacedRightPointingTriangle() +
733  
           pageToHTMLLink().get(pageFromQ(e.value))
734  
         )))
735  
         
736  
         // optional "in other slices" section
737  
         
738  
         + (empty(pageInOtherSlices) ? "" : h3(quote(pageName_html) + " in other slices")
739  
         + p_nemptyLines_showFirst(50, map(slice ->
740  
           ahref(agiBlueURL() + hquery(slice := slice.globalID, q := page.q), htmlEncode2(slice.name)), pageInOtherSlices)))
741  
         ;
742  
         
743  
        // end of mainContents
744  
          
745  
      S sideContents =
746  
        hform(b("GIVE ME INPUT:") + " "
747  
          + htextinput('q) + " "
748  
          + hsubmit("Ask", onclick := "document.getElementById('newInputForm').target = '';") + " "
749  
          + hsubmit("+Tab", title := "Ask and show result in a new tab", onclick := "document.getElementById('newInputForm').target = '_blank';"),
750  
          id := 'newInputForm)
751  
        
752  
        + h3("References (" + l(refPages) + ")")
753  
        
754  
        + p_nemptyLines_showFirst(10, map(pageToHTMLLink(displayLength := sideDisplayLength), refPages))
755  
        
756  
        + h3(searchTypeToText.get(sideSearchType) + " search results ("
757  
          + (l(searchResults) >= searchResultsToShow ? searchResultsToShow + "+" : str(l(searchResults)))
758  
          + (totalResults > l(searchResults) ? " of " + n2(totalResults) : "") + ")")
759  
        
760  
        + p_nemptyLines_showFirst(searchResultsToShow, map(pageToHTMLLink(displayLength := sideDisplayLength), searchResults))
761  
        
762  
        + hdiv("", id := 'extraStuff);
763  
        
764  
      // TODO: sync search delivery with WebSocket creation
765  
      if (asyncSearch)
766  
        doLater(6.0, r { dm_call('agiBlueSearch, 'searchAndPost, page.q, agiBlueBotID := programID(), +cc) });
767  
        
768  
      // notify local interested parties
769  
      vmBus_send('agiBlue_servingConceptPage, this, page);
770  
      
771  
      S iframe = params.get('render_iframe);
772  
        
773  
      // serve a concept page
774  
      ret hhtml_agiBlue(hhead_title(pageDisplayName(page)) + hbody(
775  
        tag('table,
776  
          tr(
777  
            td(sliceSelector(), valign := 'top)
778  
          + td(sideContents, align := 'right, valign := 'top, rowspan := 2))
779  
        + tr(td(mainContents +
780  
          (empty(iframe) ? "" : iframe(iframe, width := 600, height := 400)) +
781  
        footer(), align := 'center, valign := 'top)),
782  
        width := "100%", height := "100%")));
783  
    }
784  
    
785  
    O servePagesToBot(Iterable<Page> pages) {
786  
      ret serveListToBot(map(pageToMap(wrapMapAsParams(params)), pages));
787  
    }
788  
  
789  
    O serveListToBot(Collection l) {
790  
      if (nempty(params.get('max)))
791  
        l = takeFirst(parseInt(params.get('max)), l);
792  
      ret serveJSON(l);
793  
    }
794  
  
795  
    // uri starts with "/bot/"
796  
    O serveBot() {
797  
      S q = params.get('q);
798  
      
799  
      if (eqic(uri, "/bot/hello")) ret serveJSON("hello");
800  
      if (eqic(uri, "/bot/hasPage")) ret serveJSON(hasPage(q));
801  
      if (eqic(uri, "/bot/randomPageContaining")) {
802  
        assertNempty(q);
803  
        ret servePageToBot(random(filter(list(cc, Page), p -> cic(p.q, q))), params);
804  
      }
805  
      if (eqic(uri, "/bot/allPages"))
806  
        ret servePagesToBot(list(cc, Page));
807  
      if (eqic(uri, "/bot/allPagesStartingWith")) {
808  
        assertNempty(q);
809  
        ret servePagesToBot(filter(list(cc, Page), p -> swic(p.q, q)));
810  
      }
811  
      if (eqic(uri, "/bot/allPagesEndingWith")) {
812  
        assertNempty(q);
813  
        ret servePagesToBot(filter(list(cc, Page), p -> ewic(p.q, q)));
814  
      }
815  
      if (eqic(uri, "/bot/allPagesContaining")) {
816  
        assertNempty(q);
817  
        ret servePagesToBot(filter(list(cc, Page), p -> cic(p.q, q)));
818  
      }
819  
      if (eqic(uri, "/bot/allPagesContainingRegexp")) {
820  
        assertNempty(q);
821  
        Pattern pat = regexpIC(q);
822  
        ret servePagesToBot(filter(list(cc, Page), p -> regexpFindIC(pat, p.q)));
823  
      }
824  
      
825  
      if (eqicOneOf(uri, "/bot/postSigned", /*"/bot/makePhysicalSlice",*/ "/bot/approveTrustRequest")) {
826  
        S text = rtrim(params.get('text));
827  
        S key = getSignerKey(text);
828  
        if (empty(key)) ret subBot_serve500("Please include your public key");
829  
        if (!isSignedWithKey(text, key)) ret subBot_serve500("Signature didn't verify");
830  
        text = dropLastTwoLines(text); // drop signer + sig line
831  
        
832  
        Signer signer = uniq_sync Signer(publicKey := key);
833  
        
834  
        /*if (eqic(uri, "/bot/makePhysicalSlice")) {
835  
          if (!signer.trusted) ret subBot_serve500("Untrusted signer");
836  
          Page page = findPageFromParams(jsonDecodeMap(text));
837  
          if (page == null) ret subBot_serve500("Page not found");
838  
          ret serveJSON(uniq2_sync(PhysicalSlice, slicePage := page).b ? "Slice made" : "Slice exists");
839  
        }*/
840  
      
841  
        if (eqic(uri, "/bot/postSigned")) {
842  
          new L out;
843  
          for (S line : tlft(text)) {
844  
            SS map = jsonDecodeMap(line);
845  
            AGIBlue.GetEntriesAndPost x = agiBlue.new GetEntriesAndPost(db_mainConcepts());
846  
            x.signer = signer;
847  
            Page page = findOrMakePageFromParams(map);
848  
            if (page == null) continue with out.add("Invalid page reference");
849  
            x.go(page, map);
850  
            out.add(x.newEntry ? "Saved" : x.entry != null ? "Entry exists" : "Need key and value");
851  
          }
852  
          ret serveJSON(out);
853  
        }
854  
        
855  
        if (eqic(uri, "/bot/approveTrustRequest")) {
856  
          if (!signer.trusted) ret subBot_serve500("Untrusted signer");
857  
          Signer toApprove = conceptWhere Signer(publicKey := trim(text));
858  
          if (toApprove == null) ret subBot_serve500("Signer to approve not found");
859  
          cset(toApprove, trusted := true, approvedBy := signer.globalID);
860  
          ret serveJSON("Approved: " + trim(text));
861  
        }
862  
        
863  
        ret subBot_serve500("CONFUSION");
864  
      }
865  
      
866  
      if (eqic(uri, "/bot/post")) {
867  
        AGIBlue.GetEntriesAndPost x = agiBlue.new GetEntriesAndPost(cc);
868  
        x.go(pageFromQ(q), params);
869  
        ret serveJSON(x.newEntry ? "Saved" : x.entry != null ? "Entry exists" : "Need key and value");
870  
      }
871  
      
872  
      if (eqic(uri, "/bot/entriesOnPage"))
873  
        ret serveJSON(map(entriesOnPage(findPageFromParams(params)), entryToMap(false)));
874  
        
875  
      if (eqic(uri, "/bot/entriesForKey"))
876  
        ret serveJSON(map(conceptsWhereIC(cc, Entry, key := params.get('key)), entryToMap(true)));
877  
       
878  
      // look up value from page & key in current slice
879  
      if (eqic(uri, "/bot/lookup")) {
880  
        S key = params.get('key);
881  
        if (empty(key)) ret serveJSON("Need key");
882  
        S value = getValue(findPageFromParams(params), key);
883  
        ret serveJSON(empty(value) ? "" : litmap(+value));
884  
      }
885  
  
886  
      if (eqic(uri, "/bot/multiLookup")) {
887  
        S key = params.get('key);
888  
        if (empty(key)) ret serveJSON("Need key");
889  
        ret serveJSON(collect value(objectsWhereIC(findBackRefs(findPageFromParams(params), Entry), +key)));
890  
      }
891  
    
892  
      // look up value from page & key in all slices
893  
      if (eqic(uri, "/bot/multiLookupInAllSlices")) {
894  
        S key = params.get('key);
895  
        if (empty(key)) ret serveJSON("Need key");
896  
        Cl<Slice> slices = centralIndexGetSlices(key);
897  
        new L<Map> results;
898  
        fOr (Slice s : slices) {
899  
          WorksOnSlice wos = new(loadBackgroundSlice(s));
900  
          Page page = wos.findPageFromParams(params);
901  
          if (page != null)
902  
            fOr (Entry e : objectsWhereIC(findBackRefs(page, Entry), +key))
903  
              results.add(litorderedmap("c" := e.value, "slice" := str(s.globalID())));
904  
        }
905  
        ret serveJSON(results);
906  
      }
907  
      
908  
      if (eqic(uri, "/bot/latestEntries"))
909  
        ret serveJSON(map(takeFirst(10, slice.idx_latestEntries.objectIterator()), entryToMap(true)));
910  
      if (eqic(uri, "/bot/latestPages"))
911  
        ret serveJSON(map(takeFirst(10, slice.idx_latestCreatedPages.objectIterator()), pageToMap()));
912  
      if (eqic(uri, "/bot/latestChangedPages"))
913  
        ret serveJSON(map(takeFirst(10, slice.idx_latestChangedPages.objectIterator()), pageToMap()));
914  
        
915  
      if (eqic(uri, "/bot/googleUsersCount"))
916  
        ret serveJSON(countConcepts(GoogleUser));
917  
        
918  
      if (eqic(uri, "/bot/totalPageCount"))
919  
        ret serveJSON(countConcepts(Page));
920  
      /*if (eqic(uri, "/bot/pageWithoutPhysicalSliceCount"))
921  
        ret serveJSON(countConceptsWhere(Page, slice := null));
922  
      if (eqic(uri, "/bot/physicalSliceCount"))
923  
        ret serveJSON(countConcepts(PhysicalSlice));*/
924  
      if (eqic(uri, "/bot/trustedSignersCount"))
925  
        ret serveJSON(countConcepts(Signer, trusted := true));
926  
        
927  
      if (eqic(uri, "/bot/valueSearch")) {
928  
        S value = params.get('value);
929  
        L<Entry> entries = conceptsWhereIC(cc, Entry, +value);
930  
        ret serveJSON(map(takeFirst(100, entries), entryToMap(true)));
931  
      }
932  
        
933  
      if (eqic(uri, "/bot/keyAndValueSearch")) {
934  
        S key = params.get('key), value = params.get('value);
935  
        Cl<Page> pages = pagesForKeyAndValue(key, value);
936  
        ret servePagesToBot(pages);
937  
      }
938  
      
939  
      if (eqic(uri, "/bot/aLookup")) {
940  
        S key = params.get('key), value = params.get('value);
941  
        Cl<Page> pages = pagesForKeyAndValue(key, value);
942  
        ret serveJSON(collect q(pages));
943  
      }
944  
      
945  
      if (eqic(uri, "/bot/bLookup")) {
946  
        S value = params.get('value);
947  
        ret serveJSON(keysForPageAndValue(q, value));
948  
      }
949  
      
950  
      if (eqic(uri, "/bot/keyValuePairsByPopularity")) {
951  
        L<PairS> pairs = map(list(Entry), e -> pair(e.key, e.value));
952  
        LPair<S, Int> pairs2 = multiSetTopPairs(ciMultiSet(map pairToUglyStringForCIComparison(pairs)));
953  
        ret serveJSON(map(pairs2, p -> {
954  
          S key, value = unpair pairFromUglyString(p.a);
955  
          ret litorderedmap(n := p.b, +key, +value);
956  
        }));
957  
      }
958  
      
959  
      if (eqic(uri, "/bot/allKeys"))
960  
        ret serveListToBot(distinctCIFieldValuesOfConcepts(cc, Entry, 'key));
961  
        
962  
      if (eqic(uri, "/bot/allKeysByPopularity"))
963  
        ret serveListToBot(mapMultiSetByPopularity(distinctCIFieldValuesOfConcepts_multiSet(cc, Entry, 'key), (key, n) -> litorderedmap(+n, +key)));
964  
        
965  
      if (eqic(uri, "/bot/dbSize"))
966  
        ret serveJSON(l(conceptsFile()));
967  
        
968  
      if (eqic(uri, "/bot/query"))
969  
        ret serveBotQuery();
970  
  
971  
      if (eqic(uri, "/bot/createSlice")) {
972  
        Slice slice = createSlice(assertNempty(params.get('name)));
973  
        ret serveJSON(litorderedmap(id := str(slice.globalID), name := slice.name));
974  
      }
975  
      
976  
      if (eqic(uri, "/bot/dumpAllSlices") && authed()) {
977  
        new L<Map> out;
978  
        for (Slice slice) try {
979  
          out.add(litorderedmap(slice := slice.caseID, ms := returnTimeFor(() -> dumpSliceToFile(loadBackgroundSlice(slice)))));
980  
        } catch print e {
981  
          out.add(litorderedmap(slice.caseID, error := exceptionToStringShort(e)));
982  
        }
983  
        ret serveJSON(out);
984  
      }
985  
      
986  
      if (eqic(uri, "/bot/dumpSlice")) {
987  
        File f = dumpSliceToFile(slice);
988  
        ret "OK, saved as: " + f2s(f);
989  
      }
990  
      
991  
      if (eqic(uri, "/bot/diskStats"))
992  
        ret serveJSON(litcimap(
993  
          sizeOnDisk := agiBlue.sizeOnDisk!,
994  
          freeDiskSpace := freeDiskSpace()
995  
        ));
996  
  
997  
      if (eqic(uri, "/bot/memStats")) {
998  
        Map<S, ?> fg = cloneMap(agiBlue.loadedSlices);
999  
        Map<S, ?> bg = cloneMap(agiBlue.backgroundSlices);
1000  
        ret serveJSON(litcimap(
1001  
          error := checkDoubleLoads(),
1002  
          centralIndexEntries := l(centralIndex),
1003  
          centralIndexMade := renderHowLongAgo(centralIndexMade),
1004  
          centralIndexMadeInMS := centralIndexMadeIn,
1005  
          numLoadedForegroundSlices := l(fg),
1006  
          numLoadedBackgroundSlices := l(bg),
1007  
          loadedForegroundSlicesEstimatedSize := longSum(map(s -> agiBlue.estimatedSliceDataSize(s), keys(fg))),
1008  
          loadedBackgroundSlicesEstimatedSize := longSum(map(s -> agiBlue.estimatedSliceDataSize(s), keys(bg))),
1009  
          loadedForegroundSlices := sortedIC(keysList(fg)),
1010  
          loadedBackgroundSlices := sortedIC(keysList(bg)),
1011  
          sessionsWithGoogleUsers := countPred(list(Session), s -> s.user! instanceof GoogleUser),
1012  
          userObjects := countConcepts(User),
1013  
          processSize := getOpt(vmBus_query('processSize), 'processSize)
1014  
        ));
1015  
      }
1016  
      
1017  
      if (eqic(uri, "/bot/centralIndexGet")) {
1018  
        CentralIndexEntry e = centralIndex.get(q);
1019  
        ret serveJSON(e == null ? ll() : map(e.slices, sliceToMap()));
1020  
      }
1021  
      
1022  
      // return primary triple from each slice
1023  
      if (eqic(uri, "/bot/centralIndexGrab")) {
1024  
        CentralIndexEntry ie = centralIndex.get(q);
1025  
        new L<Map> out;
1026  
        if (ie != null) for (Slice slice : ie.slices) {
1027  
          LoadedSlice ls = loadBackgroundSlice(slice);
1028  
          WorksOnSlice wos = new(ls);
1029  
          Page page = wos.findPageFromQ(q);
1030  
          Iterable<Entry> entries = entriesOnPage(page);
1031  
          print("Page for " + q + " in slice " + ls.caseID + ": " + yesNo(page != null) + " - " + l(entries));
1032  
          Entry e = first(entries);
1033  
          if (e != null)
1034  
            out.add(litorderedmap(a := page.q, b := e.key, c := e.value, slice := slice.caseID));
1035  
        }
1036  
        ret serveJSON(out);
1037  
      }
1038  
      
1039  
      if (eqic(uri, "/bot/updateCentralIndex") && authed()) {
1040  
        rst_index.trigger();
1041  
        ret "OK";
1042  
      }
1043  
  
1044  
      if (eqic(uri, "/bot/allGoogleEmails") && authed())
1045  
        ret serveJSON(collect googleEmail(list(GoogleUser)));
1046  
  
1047  
      if (eqic(uri, "/bot/words2_spaces_all"))
1048  
        ret serveJSON(mapToValues_ciMap words2_spaces_cached(collect q(conceptsSortedByFieldCI(slice.cc, Page, 'q))));
1049  
      
1050  
      if (eqic(uri, "/bot/words2_spaces_collapse_all"))
1051  
        ret serveJSON(mapToValues_ciMap words2_spaces_collapse_cached(collect q(conceptsSortedByFieldCI(slice.cc, Page, 'q))));
1052  
  
1053  
      if (eqic(uri, "/bot/makeManyPages")) {
1054  
        //LS qs = nempties(subBot_paramsAsMultiMap().get('q));
1055  
        LS qs = tlft(params.get("pages"));
1056  
        slice.haltSliceDumping = true;
1057  
        try {
1058  
          fOr ping (S _q : qs)
1059  
            pageFromQ(_q);
1060  
        } finally {
1061  
          slice.haltSliceDumping = true;
1062  
          slice.rstDumpSlice.trigger();
1063  
        }
1064  
        ret "OK (" + l(qs) + ")";
1065  
      }
1066  
1067  
      if (eqic(uri, "/bot/sliceNamesMap"))
1068  
        ret serveJSON(mapToMap(list(Slice), s -> pair(str(s.globalID), s.name)));
1069  
1070  
      if (eqic(uri, "/bot/allowOpenPosting")) {
1071  
        try object errorIfNotOwnerOfSlice();
1072  
        bool allow = eq("1", params.get('allow));
1073  
        cset(sliceInfo(), openPosting := allow);
1074  
        ret "Set openPosting to " + allow;
1075  
      }
1076  
      
1077  
      if (eqic(uri, "/bot/createUnusedNumberedPage")) {
1078  
        int n = 1;
1079  
        // TODO: sync
1080  
        while (findPageFromQ(q + n) != null) ++n;
1081  
        ret serveJSON(litmap(q := pageFromQ(q + n).q));
1082  
      }
1083  
  
1084  
      // end of bot methods
1085  
    
1086  
      ret subBot_serve404();
1087  
    }
1088  
    
1089  
    O serveBotQuery() {
1090  
      S query = params.get('query);
1091  
      ret agiBlue.new Query(slice).process(query);
1092  
    }
1093  
1094  
    int searchResultsToShow() { ret agiBlue.searchResultsToShow; }
1095  
    
1096  
    O serveLiteralSearch() {
1097  
      S q = params.get('q);
1098  
      L<Page> searchResults = literalSearch(q);
1099  
      ret serveSearchResults("literal search" , q, searchResultsToShow(), searchResults);
1100  
    }
1101  
    
1102  
    L<Page> literalSearch(S q, O... _) {
1103  
      int searchResultsToShow = optPar max(_, 100);
1104  
      
1105  
      // quick search in random order
1106  
      //L<Page> searchResults = takeFirst(clipIntPlus(searchResultsToShow, 1), filterIterator(iterator(list(cc, Page)), p -> cic(p.q, q));
1107  
      
1108  
      // full search, order by length
1109  
      ret takeFirst(clipIntPlus(searchResultsToShow, 1), pagesSortedByLength(filter(list(cc, Page), p -> cic(p.q, q))));
1110  
    }
1111  
    
1112  
    O serveScoredSearch() {
1113  
      S q = params.get('q);
1114  
      L<Page> searchResults = (L<Page>) dm_call('agiBlueSearch, 'search, q, +cc);
1115  
      ret serveSearchResults("scored search" , q, searchResultsToShow(), searchResults);
1116  
    }
1117  
    
1118  
    O serveLevenSearch() {
1119  
      S q = params.get('q);
1120  
      L<Page> searchResults = levenSearch(q);
1121  
      ret serveSearchResults("leven search with distance 1" , q, searchResultsToShow(), searchResults);
1122  
    }
1123  
    
1124  
    L<Page> levenSearch(S q, O... _) {
1125  
      int searchResultsToShow = optPar max(_, 100);
1126  
      int maxEditDistance = 1;
1127  
      
1128  
      new Map<Page, Int> map;
1129  
      for (Page p : list(cc, Page)) {
1130  
        int distance = leven_limitedIC(q, p.q, maxEditDistance+1);
1131  
        if (distance <= maxEditDistance) map.put(p, distance);
1132  
      }
1133  
      ret takeFirst(clipIntPlus(searchResultsToShow, 1), keysSortedByValue(map));
1134  
    }
1135  
    
1136  
    O serveSearchResults(S searchType, S q, int searchResultsToShow, Collection<Page> searchResults) {
1137  
      S title = "agi.blue " + searchType + " for " + htmlEncode2(quote(q)) + " (" + (l(searchResults) >= searchResultsToShow ? searchResultsToShow + "+ results" : nResults(l(searchResults))) + ")";
1138  
      
1139  
      ret hhtml_agiBlue(hhead_title(htmldecode_dropAllTags(title))
1140  
        + hbody(hfullcenter(//top() +
1141  
          h3(title)
1142  
        + p_nemptyLines_showFirst(searchResultsToShow, map(pageToHTMLLink(), searchResults)))));
1143  
    }
1144  
    
1145  
    O serveQueryPage() {
1146  
      S query = params.get('query);
1147  
      if (query == null) query = loadSnippet(#1024258);
1148  
      S title = agiBlueNameHTML() + " | Execute a query script (" + targetBlank("http://code.botcompany.de/1024274", "ALQL") + ")";
1149  
      ret hhtml_miniPage(title, h3(title)
1150  
 + form(
1151  
        htextarea(query, name := 'query, cols := 80, rows := 10, autofocus := true)
1152  
        + "<br><br>"
1153  
        + hsubmit("Execute"), action := "/bot/query"
1154  
      ));
1155  
    }
1156  
  
1157  
    O hhtml_miniPage(S htmlTitle, O contents) {
1158  
      ret hhtml_agiBlue(hhead_title(htmldecode_dropAllTags(htmlTitle))
1159  
        + hbody(hfullcenter(contents)));
1160  
    }
1161  
    
1162  
    S sliceHomeURL() {
1163  
      ret slice == null ? agiBlueURL() : sliceHomeURL(slice.sliceConcept.globalID);
1164  
    }
1165  
    
1166  
    S sliceHomeURL(Slice slice) { ret sliceHomeURL(slice.globalID); }
1167  
  
1168  
    S sliceHomeURL(GlobalID slice) {
1169  
      ret agiBlueURL() + hquery(+slice);
1170  
    }
1171  
  
1172  
    S sliceLinkHTML(Slice slice) {
1173  
      ret slice == null ? "-": ahref(sliceHomeURL(slice), htmlEncode2(slice.name));
1174  
    }
1175  
    
1176  
    O serveCreateSlicePage() {
1177  
      S sliceName = trim(params.get('sliceName));
1178  
      if (eq(params.get('doIt), "1") && nempty(sliceName)) {
1179  
        // TODO: check for existing name
1180  
        Slice slice = createSlice(sliceName);
1181  
        ret hrefresh(sliceHomeURL(slice.globalID));
1182  
      }
1183  
      
1184  
      S title = agiBlueNameHTML() + " | Create slice";
1185  
      ret hhtml_agiBlue(hhead_title(htmldecode_dropAllTags(title))
1186  
        + hbody(hfullcenter(
1187  
          h3(title)
1188  
        + form(
1189  
            hhidden(doIt := 1)
1190  
          + "Slice name: " + htextinput(+sliceName, autofocus := true)
1191  
          + "<br><br>"
1192  
          + hsubmit("Create slice"))
1193  
        )));    
1194  
    }
1195  
  
1196  
    O serveDeletePage() {
1197  
      q = params.get('q);
1198  
      Page page = findPageFromQ(q);
1199  
      if (page == null) ret "Page " + quote(q) + " not found in slice " + htmlEncode(slice.name());
1200  
      try object errorIfNotOwnerOfSlice();
1201  
      deleteConcepts(findBackRefs(page, AbstractEntry));
1202  
      cdelete(page);
1203  
      ret hrefresh(1.0, sliceHomeURL()) + "Page deleted";
1204  
    }
1205  
1206  
    // TODO: delete without loading
1207  
    O serveDeleteSlice() {
1208  
      try object errorIfNotOwnerOfSlice();
1209  
      S caseID = sliceConcept.caseID;
1210  
      agiBlue.unloadSlice(caseID);
1211  
      File dir = sliceDir(sliceConcept);
1212  
      if (!dirExists(dir)) ret "Dir not found: " + f2s(dir);
1213  
      moveDirectory(dir, programFile("deletedSlices/" + sliceConcept.caseID));
1214  
      cset(session, selectedSlice := null);
1215  
      cdelete(sliceConcept);
1216  
      ret serveText("Slice " + caseID + " deleted (& backed up)");
1217  
    }
1218  
1219  
    O errorIfNotOwnerOfSlice() {
1220  
      bool ownage = slice.sliceConcept.owner! == session.user!;
1221  
      if (!ownage)
1222  
        ret subBot_serve403("Sorry, you don't own slice " + htmlEncode(slice.name()));
1223  
      null;
1224  
    }
1225  
    
1226  
    F1<Page, S> pageToHTMLLink(O... _) {
1227  
      optPar int displayLength = agiBlue.displayLength;
1228  
      ret func(Page p) -> S {
1229  
        S name = pageDisplayName(p);
1230  
        ret ahref(agiBlue_pageURLWithSlice(p),
1231  
          htmlEncode2(shorten(displayLength, name)),
1232  
          title := name);
1233  
      };
1234  
    }
1235  
    
1236  
    // and google log-in
1237  
    S sliceSelector() {
1238  
      ret htag table(tr(
1239  
        (!showGoogleLogIn ? "" : td(
1240  
          googleSignIn_signInButton(agiBlueURL() + "/google-verify", "console.log(data);", ""), style := "padding-right: 10px;")
1241  
          + td(nobr(ahref("javascript:signOut()", "Sign out")), style := "padding-right: 10px; padding-bottom: 0.15em"))
1242  
      
1243  
      + (!agiBlue.showSliceSelector ? "" : hform(td(
1244  
          "Select reality slice: "
1245  
        + hselect(availableSlices(), session.selectedSlice, name := 'slice, onchange := "this.form.submit()")
1246  
        /*+ " " + hsubmit("Go")*/
1247  
        + " &nbsp; | " +
1248  
        ahref(agiBlueURL("/slices"), nSlices(countConcepts(Slice))) + " | " +
1249  
        ahref(agiBlueURL() + "/createSlice", "Create slice...")
1250  
      )))));
1251  
    }
1252  
    
1253  
    S sliceAsHTML() {
1254  
      if (slice.isMainSlice()) ret htmlEncode2(agiBlueName() + "'s main slice");
1255  
      if (slice.sliceConcept == null) ret "Slice ???";
1256  
      ret htmlEncode2("Slice " + quote(slice.sliceConcept.name));
1257  
    }
1258  
    
1259  
    Slice sliceConcept() { ret slice.sliceConcept; }
1260  
    
1261  
    S footer() {
1262  
      ret p(small(elapsedMS_sysNow(started) + " ms"));
1263  
    }
1264  
    
1265  
    S verifiedEmail() {
1266  
      if (session == null || !(session.user! instanceof GoogleUser)) null;
1267  
      ret ((GoogleUser) session.user!).googleEmail;
1268  
    }
1269  
    
1270  
    bool authed() {
1271  
      bool ver = eqic(verifiedEmail(), "stefan.reich.maker.of.eye@googlemail.com");
1272  
      print("Auth check " + verifiedEmail() + " -> " + ver);
1273  
      ret ver;
1274  
    }
1275  
    
1276  
    S googleSignInID() {
1277  
      ret eqic(domain(), "botcompany.de") ? botCompanyGoogleSignInID() : agiBlueGoogleSignInID();
1278  
    }
1279  
  
1280  
    S hhtml_agiBlue(S contents) {
1281  
      ret hhtml(hAddToHead_fast(contents, 
1282  
        hIncludeGoogleFont("Source Sans Pro")
1283  
        //+ [[<meta name="google-site-verification" content="oRvyeTqgSuv-FE_c-2UKM1Vp0oDqx8h9WruHYWqA-NQ" />]]
1284  
        + loadJQuery()
1285  
        + hmobilefix()
1286  
        + googleSignIn_header("", googleSignInID())
1287  
        + hstylesheet("body { font-family: Source Sans Pro }")));
1288  
    }
1289  
    
1290  
    O serveSlicesList() {
1291  
      S sort = getAny(params, 'sort, 'by);
1292  
      L<Slice> slices;
1293  
      S shown;
1294  
      if (eqic(sort, 'name)) {
1295  
        slices = asList(idx_slicesByName.objectIterator());
1296  
        shown = "sorted by name";
1297  
      } else {
1298  
        sort = 'date;
1299  
        slices = asList(idx_slicesByModification.objectIterator());
1300  
        shown = "last modified first";
1301  
      }
1302  
        
1303  
      S title = agiBlueNameHTML() + " has " + nSlices(slices) /*+ " (" + shown + ")"*/;
1304  
      new PreIncLongCounter counter;
1305  
      ret hhtml_agiBlue(hhead_title_decode(title)
1306  
        + hbody(hfullcenter(
1307  
          h1(title)
1308  
        + p(joinWithSpacedVBar(
1309  
          ahrefIf(neqic(sort, 'name), agiBlueURL() + "/slices" + hquery(sort := 'name), "List by name"),
1310  
          ahrefIf(neqic(sort, 'date), agiBlueURL() + "/slices" + hquery(sort := 'date), "List by date"),
1311  
          ahref(agiBlueURL() + "/createSlice", "Create slice...")))
1312  
  
1313  
        + htmlTable2(map(slices, slice ->
1314  
          litorderedmap(
1315  
            "Name" := ahref(sliceHomeURL(slice), htmlEncode2(slice.name)),
1316  
            "ID" := ahref(sliceHomeURL(slice), str(slice.globalID)),
1317  
            "Owner" := str(slice.owner!)
1318  
          )),
1319  
          htmlEncode := false)
1320  
        )));
1321  
    }
1322  
  
1323  
    O serveUsersList() {
1324  
      S sort = getAny(params, 'sort, 'by);
1325  
      L<User> users;
1326  
      S shown;
1327  
      if (eqic(sort, 'name))
1328  
        users = asList(idx_usersByName.objectIterator());
1329  
      else {
1330  
        sort = 'seen;
1331  
        users= asList(idx_usersBySeen.objectIterator());
1332  
      }
1333  
        
1334  
      S title = agiBlueNameHTML() + " has " + nUsers(users);
1335  
      new PreIncLongCounter counter;
1336  
      ret hhtml_agiBlue(hhead_title_decode(title)
1337  
        + hbody(hfullcenter(
1338  
          h1(title)
1339  
        + p(joinWithSpacedVBar(
1340  
          ahrefIf(neqic(sort, 'name), agiBlueURL() + "/users" + hquery(sort := 'name), "List by name"),
1341  
          ahrefIf(neqic(sort, 'seen), agiBlueURL() + "/users" + hquery(sort := 'seen), "List by last seen"),
1342  
          ahref(agiBlueURL() + "/createSlice", "Create slice...")))
1343  
  
1344  
        + htmlTable2(map(users, user ->
1345  
          litorderedmap(
1346  
            "Name" := ahref(userHomeURL(user), htmlEncode2(str(user))),
1347  
            "Last seen" := renderHowLongAgo(user.lastSeen)
1348  
          )),
1349  
          htmlEncode := false)
1350  
        )));
1351  
    }
1352  
  
1353  
    S sliceInfoHTML() {
1354  
      User owner = slice.sliceConcept.owner!;
1355  
      ret owner == null ? "" : p("This slice is owned by " + userHTML(owner));
1356  
    }
1357  
  
1358  
    S userHomeURL(User user) {
1359  
      ret user == null ? null : agiBlueURL() + "/user/" + user.globalID;
1360  
    }
1361  
    
1362  
    S userHTML(User user) {
1363  
      ret user == null ? "nobody" : ahref(userHomeURL(user), htmlEncode2(str(user)));
1364  
    }
1365  
    
1366  
    S top() {
1367  
      ret nempty(get) ? "" : hcomment("cookie: " + takeFirst(4, session.cookie))
1368  
        + hSilentComputatorWithFlag("agi.blue: " + q)
1369  
        + p(ahref(sliceHomeURL(),
1370  
          //hsnippetimg(#1101682, width := 565/5, height := 800/5, title := "Robot by Peerro @ DeviantArt")
1371  
          //hsnippetimg(#1101778, width := 96, height := 96, title := "agi.blue - a database for everything")
1372  
          hsnippetimg(#1101822, width := 314, height := 125, title := "agi.blue - Wikipedia for robots")
1373  
        ))
1374  
        + h2(ahref_unstyled(sliceHomeURL(), htmlEncode2(slice.sliceConcept.name), style := "color: yellow"))
1375  
        + sliceInfoHTML()
1376  
        + p(small(
1377  
            agiBlueNameHTML_boldWithSize()
1378  
          + (agiBlue_isOriginal() ? "" : " " + targetBlank("http://agi.blue", "[original]"))
1379  
          + " | " + ahref(agiBlueURL("/slices"), nSlices(countConcepts(Slice)))
1380  
          + " | " + targetBlank(progLink(), "source code") + " of this web site (" + nLines(sourceCodeLines) + ") | " + targetBlank("https://gitter.im/agi-blue/community", "sponsor https?") + " | by " + targetBlank("https://BotCompany.de", "BC") + " | " + targetBlank("http://fiverr.tinybrain.de/", "Fiverr") + " | " + targetBlank("https://discordapp.com/invite/SEAjPqk", "Discord") + " | " + targetBlank("https://www.youtube.com/watch?v=b6jtRdV3Ev8", "Video")
1381  
          + " | " + targetBlank("http://code.botcompany.de/1024233", "Notes")
1382  
          + " | " + ahref(agiBlueURL() + "/query", "Query")
1383  
        ));
1384  
    }
1385  
  
1386  
    Slice createSlice(S name) {
1387  
      name = shorten(name, maxSliceNameLength);
1388  
      Slice slice = cnew Slice(+name, owner := session.user);
1389  
      cset(slice, caseID := slice.defaultCaseID());
1390  
      loadSlice(slice).initialSetup(slice.globalID);
1391  
      ret slice;
1392  
    }
1393  
  
1394  
    O serveCheckboxes() {
1395  
      set conceptsSortedByFieldCI_verbose;
1396  
      int showMax = 1000;
1397  
      Cl<Page> pages = takeFirst(showMax, conceptsSortedByFieldCI(slice.cc, Page, 'q));
1398  
      S title = "Select pages";
1399  
      ret hhtml_miniPage(title, h3(title) + hpostform(
1400  
        htmlTable2(map(pages, page -> litorderedmap(
1401  
          "Page" := hcheckbox("page_" + page.globalID()) + " " + pageToHTMLLink().get(page)
1402  
        )), 
1403  
        htmlEncode := false)
1404  
        + h3("Add an entry to all selected pages")
1405  
          + p("Key: " + hinputfield("key") +
1406  
            " Value: " + hinputfield("value"))
1407  
        + p(hsubmit("Add entries")), action := agiBlueURL() + "/multiAdd"));
1408  
    }
1409  
  
1410  
    O serveMultiAdd() {
1411  
      S key = trim(params.get('key)), value = trim(params.get('value));
1412  
      if (empty(key) || empty(value)) ret "Need key and value";
1413  
      new Matches m;
1414  
      new L<Page> pages;
1415  
      for (S a, b : params)
1416  
        if (startsWith(a, "page_", m))
1417  
          addIfNotNull(pages, conceptWhere(cc, Page, globalID := m.rest()));
1418  
      if (empty(pages)) ret "No pages selected";
1419  
      for (Page page : pages)
1420  
        agiBlue.new GetEntriesAndPost(cc).go(page, params);
1421  
      ret "Entry added to " + nPages(pages);
1422  
    }
1423  
  
1424  
    S agiBlueNameHTML() {
1425  
      ret ahref(agiBlueURL(), htmlEncode2(agiBlueName()));
1426  
    }
1427  
  
1428  
    S agiBlueNameHTML_boldWithSize() {
1429  
      Long size = agiBlue.sizeOnDisk.peek();
1430  
      ret b(agiBlueNameHTML()) + (size == null ? "" : " " + spanTitle("Size of agi.blue's database on disk", "(" + toM(size) + " MB)"));
1431  
    }
1432  
1433  
    IF1<Page, Map> pageToMap(O... _) {
1434  
      optPar bool withEntries;
1435  
      
1436  
      bool nameOnly = eqOneOf(optPar nameOnly(_), "1", true);
1437  
      if (nameOnly) ret (IF1<Page, Map>) p -> litmap(q := p.q);
1438  
      
1439  
      ret (IF1<Page, Map>) p -> {
1440  
        L<Entry> entries = findBackRefs(p, Entry);
1441  
        ret litorderedmap(
1442  
          q := p.q,
1443  
          nEntries := l(entries),
1444  
          created := p.created,
1445  
          modified := p._modified,
1446  
          entries := !withEntries ? null : map(entries, entryToMap(false)));
1447  
      };
1448  
    }
1449  
    
1450  
    IF1<Entry, Map> entryToMap(bool withPage) {
1451  
      ret (IF1<Entry, Map>) e -> litorderedmap(
1452  
        created := e.created,
1453  
        i := e.count,
1454  
        key := e.key,
1455  
        value := e.value,
1456  
        q := withPage ? e.page->q : null,
1457  
        signer := getString globalID(e.signer!));
1458  
    }
1459  
    
1460  
    IF1<Slice, Map> sliceToMap() {
1461  
      ret (IF1<Slice, Map>) s -> litorderedmap(
1462  
        created := s.created,
1463  
        globalID := str(s.globalID),
1464  
        name := s.name,
1465  
        caseID := s.caseID);
1466  
    }
1467  
    
1468  
    O servePageToBot(Page page, SS params) {
1469  
      if (page == null) ret serveJSON(null);
1470  
      params = asCIMap(params);
1471  
      Map map = pageToMap(withEntries := valueIs1 withEntries(params)).get(page);
1472  
      ret serveJSON(map);
1473  
    }
1474  
    
1475  
    SS availableSlices() {
1476  
      L<Slice> slices = asList(idx_slicesByName.objectIterator());
1477  
      ret mapToOrderedMap(s -> pair(str(s.globalID), s.name + " [ID: " + s.globalID + "]"), slices);
1478  
    }
1479  
    
1480  
  } // end of Request
1481  
  
1482  
} // end of renderer
1483  
1484  
set flag hotwire_here.
1485  
1486  
// share ISpec interface with sub-modules
1487  
static JavaXClassLoader hotwire_makeClassLoader(L<File> files) {
1488  
  ClassLoader cl = myClassLoader();
1489  
  
1490  
  // Avoid class loader chaining, always reference base class loader
1491  
  // (TODO)
1492  
  /*if (agiBlue().isClone && agiBlue().cloningSince != 0) {
1493  
    ClassLoader parent = cast getOpt(cl, 'virtualParent);
1494  
    print("Cloned class loader. " + parent);
1495  
    if (parent != null) cl = parent;
1496  
  }*/
1497  
  
1498  
  ret new JavaXClassLoaderWithParent2(null, files, cl, parseClassesToShareList(classesToShare));
1499  
}
1500  
1501  
static AGIBlue agiBlue() { ret agiBlue; }
1502  
1503  
static File sliceDir(Slice slice) {
1504  
  ret programFile(slice.caseID);
1505  
}

download  show line numbers  debug dex  old transpilations   

Travelled to 6 computer(s): bhatertpkbcr, mqqgnosmbjvj, pyentgdyhuwx, pzhvpgtvlbxg, tvejysmllsmz, vouqrxazstgt

No comments. add comment

Snippet ID: #1024512
Snippet name: agi.blue productive [LIVE]
Eternal ID of this version: #1024512/5
Text MD5: a60036f2298c2ec909585d8d216c8b79
Transpilation MD5: a06c334ec3a0902d4996e071d91a72b9
Author: stefan
Category:
Type: JavaX module (desktop)
Public (visible to everyone): Yes
Archived (hidden from active list): No
Created/modified: 2019-08-17 20:27:18
Source code size: 57263 bytes / 1505 lines
Pitched / IR pitched: No / No
Views / Downloads: 382 / 1714
Version history: 4 change(s)
Referenced in: [show references]