Download Jar. Uses 1658K of libraries. Click here for Pure Java version (21171L/157K).
1 | !7 |
2 | |
3 | // See #1023660 for the older 99 lines version |
4 | |
5 | // Customizations: |
6 | |
7 | sbool asyncSearch = false; |
8 | sbool allowMultipleCasesInValues = true; |
9 | sbool showSliceSelector = true; |
10 | sbool showGoogleLogIn = true; |
11 | sbool makeAllKeysPages = true; |
12 | sbool makeAllValuesPages = true; |
13 | sbool legacy = true; // We still have slices without applied fixes |
14 | |
15 | static int maxSliceNameLength = 1024; |
16 | static int displayLength = 140; |
17 | static int sideDisplayLength = 50; // when in side bar |
18 | static int searchResultsToShow = 50; |
19 | sS sideSearchType = 'literal; |
20 | static int maxBackgroundSlices = 1000; |
21 | |
22 | // end of customizations |
23 | |
24 | // in main DB |
25 | concept Slice { |
26 | GlobalID globalID = aGlobalIDObject(); |
27 | S caseID; |
28 | S name; |
29 | long sliceDumped; // wall time, set before dump starts |
30 | bool indexed = true; |
31 | |
32 | new Ref<User> owner; |
33 | |
34 | S defaultCaseID() { |
35 | ret uniqueFileNameUsingMD5_80_v2(name + " " + globalID); |
36 | } |
37 | } |
38 | |
39 | // in any DB |
40 | concept Page { |
41 | S globalID = aGlobalID(); |
42 | S url, q; |
43 | |
44 | // for in-VM helper bots |
45 | void quickPost(S key, S value) { |
46 | new GetEntriesAndPost(_concepts).go(this, litmap(+key, +value)); |
47 | } |
48 | } |
49 | |
50 | abstract concept AbstractEntry { |
51 | new Ref<Page> page; |
52 | int count; |
53 | S key; |
54 | S ip; |
55 | new Ref<Signer> signer; |
56 | |
57 | S q() { Page p = page!; ret p == null ? null : page->q; } |
58 | } |
59 | |
60 | concept Entry > AbstractEntry { |
61 | S globalID = aGlobalID(); // TODO: migrate to object |
62 | S value; |
63 | } |
64 | |
65 | concept MultiLineEntry > AbstractEntry { |
66 | GlobalID globalID = aGlobalIDObject(); |
67 | S value; |
68 | } |
69 | |
70 | concept BlobEntry > AbstractEntry { |
71 | GlobalID globalID = aGlobalIDObject(); |
72 | long length; |
73 | S md5; |
74 | } |
75 | |
76 | // in main DB? |
77 | concept Signer { |
78 | S globalID = aGlobalID(); |
79 | S publicKey; |
80 | bool trusted; |
81 | S approvedBy; |
82 | } |
83 | |
84 | concept User { |
85 | GlobalID globalID = aGlobalIDObject(); |
86 | } |
87 | |
88 | concept GoogleUser > User { |
89 | long googleLogInDate; |
90 | S googleEmail, googleFirstName, googleLastName; |
91 | bool googleEmailVerified; |
92 | } |
93 | |
94 | // in main DB |
95 | concept Session { |
96 | S cookie; |
97 | S selectedSlice; // global ID |
98 | Page slicePage; // phasing out |
99 | |
100 | long googleLogInDate; |
101 | S googleEmail, googleFirstName, googleLastName; |
102 | bool googleEmailVerified; |
103 | } |
104 | |
105 | // singleton concept in every DB |
106 | concept SliceInfo { |
107 | GlobalID globalID; |
108 | LS mandatoryBotVMBusIDs; |
109 | } |
110 | |
111 | sclass CentralIndexEntry { |
112 | Set<Slice> slices = synchroLinkedHashSet(); |
113 | } |
114 | |
115 | // global constants |
116 | |
117 | static SS searchTypeToText = litmap( |
118 | leven := "Leven", |
119 | literal := "Literal", |
120 | scored := "Scored"); |
121 | |
122 | // global vars |
123 | |
124 | static volatile bool started, broken; |
125 | static int sourceCodeLines; |
126 | static ConceptsLoadedOnDemand fan; |
127 | static ConceptsLoadedOnDemand backgroundFan; |
128 | static Map<S, LoadedSlice> loadedSlices = syncMap(); |
129 | static Map<S, LoadedSlice> backgroundSlices = syncMap(); |
130 | static Map<S, CentralIndexEntry> centralIndex = ciMap(); |
131 | static Map<S, CentralIndexEntry> newCentralIndex = ciMap(); |
132 | static long centralIndexMade; // wall time |
133 | static ReliableSingleThread rst_index = new(r makeCentralIndex); |
134 | |
135 | // end of global vars |
136 | |
137 | sclass LoadedSlice { // non-persistent class |
138 | S caseID; |
139 | Concepts cc; |
140 | Slice sliceConcept; |
141 | long lastAccess = sysNow(); |
142 | |
143 | ConceptFieldIndexDesc idx_latestEntries, idx_latestCreatedPages, idx_latestChangedPages; |
144 | |
145 | *(S *caseID, Concepts *cc) { |
146 | indexThings(); |
147 | |
148 | // forward changes to Slice concept |
149 | onConceptsChange(cc, r { |
150 | cset(sliceConcept, _modified := now()) |
151 | }); |
152 | |
153 | if (makeAllKeysPages && legacy) |
154 | for (Entry e : list(cc, Entry)) |
155 | pageFromQ(e.key); |
156 | |
157 | if (makeAllValuesPages && legacy) |
158 | for (Entry e : list(cc, Entry)) |
159 | pageFromQ(e.value); |
160 | } |
161 | |
162 | void initialSetup(GlobalID globalID) { |
163 | SliceInfo info = uniq(cc, SliceInfo); |
164 | cset(info, +globalID); |
165 | } |
166 | |
167 | // create if not there |
168 | Page pageFromQ(S q) { |
169 | ret main.pageFromQ(cc, q); |
170 | } |
171 | |
172 | void indexThings() { |
173 | indexConceptFieldsCI(cc, Page, 'url, Page, 'q, Entry, 'key, Entry, 'value); |
174 | indexConceptFields(cc, Signer, 'publicKey); |
175 | indexConceptFields(cc, Session, 'cookie); |
176 | indexSingletonConcept(cc, SliceInfo); |
177 | idx_latestCreatedPages = new ConceptFieldIndexDesc(cc, Page, 'created); |
178 | idx_latestChangedPages = new ConceptFieldIndexDesc(cc, Page, '_modified); |
179 | idx_latestEntries = new ConceptFieldIndexDesc(cc, Entry, 'created); |
180 | } |
181 | |
182 | bool isMainSlice() { ret empty(caseID); } |
183 | GlobalID globalID() { ret sliceConcept.globalID; } |
184 | } |
185 | |
186 | p { |
187 | try { |
188 | if (bootstrapDataFrom(#1023558)) deleteMyBackups(); |
189 | thinMyBackups(); |
190 | fan = slicesLoadedOnDemand(loadedSlices); |
191 | mainConcepts = fan.get("", null); |
192 | indexConceptFields(Slice, 'globalID, Slice, 'caseID); |
193 | cset(uniq_returnIfNew Slice(caseID := ""), name := "main slice"); |
194 | loadedSlices.get("").initialSetup(toGlobalIDObj(mainSliceGlobalID())); |
195 | |
196 | backgroundFan = slicesLoadedOnDemand(backgroundSlices); |
197 | backgroundFan.lock = fan.lock; |
198 | |
199 | // legacy conversions! |
200 | //deleteConcepts(Slice, globalID := GlobalID('wftlawbagrwprywn)); |
201 | |
202 | // Approve this machine's key |
203 | PKIKeyPair machineKey = agiBot_trustedKeyForMachine(); |
204 | if (machineKey != null) { |
205 | print("Approving this machine's key: " + machineKey.publicKey); |
206 | cset(uniq_sync(Signer, publicKey := machineKey.publicKey), trusted := true, approvedBy := "local"); |
207 | } |
208 | |
209 | sourceCodeLines = countLines(mySource()); |
210 | |
211 | rst_index.trigger(); |
212 | } catch print e { |
213 | set broken; |
214 | } finally { |
215 | set started; |
216 | } |
217 | } |
218 | |
219 | // Serve page |
220 | |
221 | set flag NoNanoHTTPD. html { |
222 | sleepWhile(() -> !started); |
223 | if (broken) ret serve500("Internal error"); |
224 | ret new Request().serve(uri, params); |
225 | } |
226 | |
227 | sclass WorksOnSlice { |
228 | Concepts cc; |
229 | LoadedSlice slice; |
230 | |
231 | *() {} |
232 | *(LoadedSlice *slice) { cc = slice.cc; } |
233 | |
234 | void setSlice(LoadedSlice slice) { |
235 | this.slice = slice; |
236 | cc = slice.cc; |
237 | } |
238 | |
239 | Page findPageFromParams(Map map) { |
240 | S q = getString q(map); |
241 | ret empty(q) ? null : findPageFromQ(q); |
242 | } |
243 | |
244 | Page findOrMakePageFromParams(Map map) { |
245 | ret pageFromQ(getString q(map)); |
246 | } |
247 | |
248 | Page findPageFromQ(S q) { |
249 | ret conceptWhereCI(cc, Page, +q); |
250 | } |
251 | |
252 | Page pageFromQ(S q) { |
253 | ret slice.pageFromQ(q); |
254 | } |
255 | |
256 | Set<Page> pagesForKeyAndValue(S key, S value) { |
257 | L<Entry> entries = conceptsWhereIC(cc, Entry, +value, +key); |
258 | ret asSet(ccollect page(entries)); |
259 | } |
260 | |
261 | Cl<S> keysForPageAndValue(S q, S value) { |
262 | Page page = findPageFromQ(q); |
263 | if (page == null) null; |
264 | ret collect key(conceptsWhereIC(cc, Entry, +page, +value)); |
265 | } |
266 | |
267 | // DB functions |
268 | |
269 | bool hasPage(S q) { ret hasConceptWhereIC(Page, +q); } |
270 | S getValue(Page page, S key) { |
271 | ret page == null || empty(key) ? null : getString value(highestByField count(objectsWhereIC(findBackRefs(page, Entry), +key))); |
272 | } |
273 | S getValue(S page, S key) { |
274 | ret getValue(findPageFromQ(page), key); |
275 | } |
276 | S pageDisplayName(Page page) { |
277 | /*S name = getValue(page, "read as"); |
278 | bool unnaturalName = nempty(name) && !eq(makeAGIDomain(name), page.url); |
279 | ret unnaturalName ? name + " " + squareBracketed(page.url) : or2(name, unpackAGIDomainOpt(page.url));*/ |
280 | ret page.q; |
281 | } |
282 | |
283 | S renderThing(S s, bool forceURLDisplay) { |
284 | ret forceURLDisplay || isURL(s) || isAGIDomain(s) |
285 | ? ahref(fixAGILink(absoluteURL(s)), htmlencode2(shorten(displayLength, s))) |
286 | : ahref(agiBlue_linkForPhrase(s, |
287 | slice := slice.isMainSlice() ? null : slice.sliceConcept.globalID), |
288 | htmlencode2(shorten(displayLength, s))); |
289 | } |
290 | |
291 | } // end of WorksOnSlice |
292 | |
293 | sclass Request extends WorksOnSlice { |
294 | S cookie; |
295 | Session session; |
296 | S uri; |
297 | SS params; |
298 | long started = sysNow(); |
299 | |
300 | // when serving a concept page, info that may be grabbed |
301 | // by thought bots |
302 | L<Entry> usesAsKey; |
303 | |
304 | // should also work for standalone |
305 | bool isHomeDomain() { |
306 | S domain = domain(); |
307 | ret eqic(domain, "www.agi.blue") || !ewic(domain, ".agi.blue"); |
308 | } |
309 | |
310 | O serve(S uri, SS params) { |
311 | this.uri = dropMultipleSlashes(uri); |
312 | this.params = params; |
313 | new Matches m; |
314 | |
315 | print(uri + " ? " + unnull(subBot_query())); |
316 | |
317 | // get cookie & session |
318 | |
319 | cookie = cookieFromUser(); |
320 | if (cookie != null) |
321 | session = uniq_sync(Session, +cookie); |
322 | else |
323 | session = unlistedWithValues(Session); |
324 | |
325 | // check if user wants to change slices |
326 | |
327 | S selectSlice = params.get('slice); |
328 | if (isGlobalID(selectSlice)) |
329 | cset(session, selectedSlice := selectSlice); |
330 | if (session.selectedSlice == null) |
331 | cset(session, selectedSlice := str(mainSliceGlobalID())); |
332 | |
333 | // get case ID |
334 | |
335 | S caseID = ""; |
336 | Slice sliceConcept = sliceConceptForGlobalID(session.selectedSlice); |
337 | print("Selected slice: " + session.selectedSlice + ", obj? " + (sliceConcept != null)); |
338 | if (sliceConcept != null) caseID = sliceConcept.caseID; |
339 | print("caseID: " + caseID); |
340 | |
341 | // load slice |
342 | |
343 | slice = assertNotNull(loadSlice(caseID, session.selectedSlice)); |
344 | cc = slice.cc; |
345 | slice.sliceConcept = sliceConcept; |
346 | |
347 | // Check for special URIs |
348 | |
349 | if (swic(uri, "/bot/")) ret serveBot(); |
350 | |
351 | // eleu appends a slash to the URI if it's a single identifier, so we drop it again |
352 | S uri2 = dropTrailingSlash(uri); |
353 | |
354 | if (eqic(uri2, "/google-verify")) { |
355 | print("Google-verify started."); |
356 | Payload payload = printStruct(googleVerifyUserToken2(googleSignInID(), params.get("token"))); |
357 | if (payload == null) ret print("google-verify", "No"); |
358 | cset(session, |
359 | googleLogInDate := now(), |
360 | googleEmail := payload.getEmail(), |
361 | googleEmailVerified := payload.getEmailVerified(), |
362 | googleFirstName := strOrNull(payload.get("given_name")), |
363 | googleLastName := strOrNull(payload.get("family_name"))); |
364 | |
365 | ret print("google-verify", payload.getEmail() + " " + (payload.getEmailVerified() ? "(verified)" : "(not verified)")); |
366 | } |
367 | |
368 | if (eqic(uri2, "/search")) ret serveScoredSearch(); |
369 | if (eqic(uri2, "/literalSearch")) ret serveLiteralSearch(); |
370 | if (eqic(uri2, "/levenSearch")) ret serveLevenSearch(); |
371 | |
372 | if (eqic(uri2, "/query")) ret serveQueryPage(); |
373 | if (eqic(uri2, "/createSlice")) ret serveCreateSlicePage(); |
374 | |
375 | S q = params.get('q); |
376 | S domain = or2(params.get('domain), domain()); |
377 | |
378 | S raw = firstKeyWithValue("", params); // agi.blue?something |
379 | if (nempty(raw) && empty(q)) q = raw; |
380 | /*if (nempty(q)) { |
381 | domain = makeAGIDomain(q); |
382 | if (l(domain) > maximumDomainPartLength()) // escape with "domain=" |
383 | ret hrefresh(agiBlueURL() + hquery(+domain, key := "read as", value := q)); |
384 | ret hrefresh("http://" + domain + (eq(q, domain) ? "" : "/" + hquery(key := "read as", value := q))); |
385 | //uri = "/"; replaceMapWithParams(params, key := "read as", value := q); |
386 | }*/ |
387 | S url = domain + dropTrailingSlash(uri); |
388 | |
389 | // domain to query |
390 | //if (empty(q)) q = url; |
391 | if (empty(q)) q = agiBlue_urlToQuery(url); |
392 | |
393 | Page page, bool newPage = unpair uniqCI2_sync(cc, Page, +q); |
394 | if (newPage) dbLog("New page", +q); |
395 | //printStructs(+params, +raw, +q); |
396 | |
397 | Set<S> get = asCISet(nempties(subBot_paramsAsMultiMap().get('get))); |
398 | |
399 | S top = nempty(get) ? "" : hcomment("cookie: " + takeFirst(4, session.cookie)) |
400 | + hSilentComputatorWithFlag("agi.blue: " + q) |
401 | + p(ahref(sliceHomeURL(), |
402 | //hsnippetimg(#1101682, width := 565/5, height := 800/5, title := "Robot by Peerro @ DeviantArt") |
403 | //hsnippetimg(#1101778, width := 96, height := 96, title := "agi.blue - a database for everything") |
404 | hsnippetimg(#1101822, width := 314, height := 125, title := "agi.blue - Wikipedia for robots") |
405 | )) |
406 | + p(small( |
407 | b(agiBlueNameHTML()) |
408 | + (agiBlue_isOriginal() ? "" : " " + targetBlank("http://agi.blue", "[original]")) |
409 | + " | " + targetBlank(progLink(), "source code") + " of this web site (" + nLines(sourceCodeLines) + ") | " + targetBlank("https://gitter.im/agi-blue/community", "sponsor https?") + " | by " + targetBlank("https://BotCompany.de", "BC") + " | " + targetBlank("http://fiverr.tinybrain.de/", "Fiverr") + " | " + targetBlank("https://discordapp.com/invite/SEAjPqk", "Discord") + " | " + targetBlank("https://www.youtube.com/watch?v=b6jtRdV3Ev8", "Video") |
410 | + " | " + targetBlank("http://code.botcompany.de/1024233", "Notes") |
411 | + " | " + ahref(agiBlueURL() + "/query", "Query") |
412 | )); |
413 | |
414 | if (empty(params.get('q)) && empty(raw) && isHomeDomain()) { |
415 | //L<Page> pages = sortedByFieldDesc _modified(list(cc, Page)); // TODO: use index |
416 | L<Page> pages = cloneList(slice.idx_latestChangedPages.objectIterator()); |
417 | int start = parseInt(params.get("start")), step = 100; |
418 | S nav = pageNav2("/", l(pages), start, step, "start"); |
419 | S content = hform(b("GIVE ME INPUT: ") + htextinput('q, autofocus := true) + " " + hsubmit("Ask")) |
420 | + h1(sliceAsHTML() + " has " + nPages(countConcepts(cc, Page)) + " and " + nConnections(countConcepts(cc, Entry))) |
421 | + p(nav) |
422 | + p_nemptyLines(map(pageToHTMLLink(), subList(pages, start, start+step))); |
423 | |
424 | ret hhtml_agiBlue(hhead_title("Slice " + sliceConcept().name + " | agi.blue") // SERVE SLICE HOME PAGE |
425 | + hbody(hfullcenterAndTopLeft(top + content |
426 | + footer(), |
427 | sliceSelector() |
428 | ))); |
429 | } |
430 | |
431 | S key = trim(params.get('key)), value = trim(params.get('value)); |
432 | L<Entry> entries = GetEntriesAndPost(cc).go(page, params).entries; |
433 | |
434 | //S get = params.get('get); |
435 | if (nempty(get)) |
436 | ret serveJSON(collect value(llNotNulls(firstThat(entries, e -> get.contains(e.key))))); |
437 | |
438 | S key2 = key, value2 = value; if (nempty(key) && nempty(value)) key2 = value2 = ""; // input reset |
439 | |
440 | bool withHidden = eq(params.get('withHidden), "1"); |
441 | new Set<Int> hide; |
442 | if (!withHidden) for (Entry e : entries) if (eqic(e.key, 'hide) && isSquareBracketedInt(e.value)) addAll(hide, e.count, parseInt(unSquareBracket(e.value))); |
443 | new MultiMap<Int, S> mmMeta; |
444 | for (Entry e : entries) if (isSquareBracketedInt(e.key)) mmMeta.put(parseInt(unSquareBracket(e.key)), e.value); |
445 | new MultiMap<S> mm; |
446 | for (Entry e : entries) mm.put(e.key, e.value); |
447 | |
448 | //S name = or2(/* ouch */ last(mm.get("read as")), /* end ouch */ unpackAGIDomain(page.url), page.url); |
449 | S name = page.q; |
450 | |
451 | // Find references |
452 | |
453 | L<Entry> refs = concatLists(conceptsWhereIC(cc, Entry, value := name), |
454 | conceptsWhereIC(cc, Entry, key := name)); |
455 | Set<Page> refPages = asSet(ccollect page(refs)); |
456 | refPages.remove(page); // don't list current page as reference |
457 | |
458 | // Search in page names (depending on default search type) |
459 | |
460 | L<Page> searchResults; |
461 | if (eq(sideSearchType, 'leven)) |
462 | searchResults = levenSearch(page.q, max := 50); |
463 | else if (eq(sideSearchType, 'literal)) |
464 | searchResults = literalSearch(page.q, max := 50); |
465 | else |
466 | searchResults = (L<Page>) dm_call('agiBlueSearch, 'search, page.q, maxResult := searchResultsToShow+1, agiBlueBotID := programID(), +cc); |
467 | searchResults.remove(page); |
468 | |
469 | // Find uses as key |
470 | usesAsKey = conceptsWhereIC(cc, Entry, key := page.q); |
471 | |
472 | S pageName_html = htmlEncode2(shorten(displayLength, name)); |
473 | |
474 | S mainContents = |
475 | top + h1(ahref_unstyled(agiBlue_pageURLWithSlice(page), pageName_html) + (newPage ? " [huh????]" : "")) |
476 | + p_nemptyLines(map(entries, func(Entry e) -> S { |
477 | !withHidden && (hide.contains(e.count) || eqic(e.key, "read as") && eqic(e.value, name)) ? "" |
478 | : "[" + e.count + "] " + |
479 | renderThing(e.key, false) + ": " + |
480 | b(renderThing(e.value, cic(mmMeta.get(e.count), "is a URL"))) |
481 | })) |
482 | + hpostform(h3("Add an entry") |
483 | + "Key: " + hinputfield(key := key2) + " Value: " + hinputfield(value := value2) + "<br><br>" + hsubmit("Add") |
484 | ) |
485 | |
486 | + p(ahref(agiBlueURL() + "/literalSearch" + hquery(q := page.q), "[literal search]", title := "Search pages with a name containing this page's name literally") |
487 | + " " + |
488 | ahref(agiBlueURL() + "/levenSearch" + hquery(q := page.q), "[leven search 1]", title := "Search pages with a Levenshtein similarity of 1 containing this page's name literally") |
489 | + " " + |
490 | ahref(agiBlueURL() + "/search" + hquery(q := page.q), "[scored search]", title := "Search pages with ScoredSearch")) |
491 | |
492 | + (empty(usesAsKey) ? "" : h3(quote(pageName_html) + " as key") |
493 | + p_nemptyLines_showFirst(50, map(usesAsKey, e -> |
494 | pageToHTMLLink().get(e.page!) + unicode_spacedRightPointingTriangle() + |
495 | pageName_html + unicode_spacedRightPointingTriangle() + |
496 | pageToHTMLLink().get(pageFromQ(e.value)) |
497 | ))); |
498 | |
499 | // end of mainContents |
500 | |
501 | S sideContents = |
502 | hform(b("GIVE ME INPUT:") + " " |
503 | + htextinput('q) + " " |
504 | + hsubmit("Ask", onclick := "document.getElementById('newInputForm').target = '';") + " " |
505 | + hsubmit("+Tab", title := "Ask and show result in a new tab", onclick := "document.getElementById('newInputForm').target = '_blank';"), |
506 | id := 'newInputForm) |
507 | |
508 | + h3("References (" + l(refPages) + ")") |
509 | |
510 | + p_nemptyLines_showFirst(10, map(pageToHTMLLink(displayLength := sideDisplayLength), refPages)) |
511 | |
512 | + h3(searchTypeToText.get(sideSearchType) + " search results (" + (l(searchResults) >= searchResultsToShow ? searchResultsToShow + "+" : str(l(searchResults))) + ")") |
513 | |
514 | + p_nemptyLines_showFirst(searchResultsToShow, map(pageToHTMLLink(displayLength := sideDisplayLength), searchResults)) |
515 | |
516 | + hdiv("", id := 'extraStuff); |
517 | |
518 | // TODO: sync search delivery with WebSocket creation |
519 | if (asyncSearch) |
520 | doLater(6.0, r { dm_call('agiBlueSearch, 'searchAndPost, page.q, agiBlueBotID := programID(), +cc) }); |
521 | |
522 | // notify local interested parties |
523 | vmBus_send('agiBlue_servingConceptPage, this, page); |
524 | |
525 | S iframe = params.get('render_iframe); |
526 | |
527 | // serve a concept page |
528 | ret hhtml_agiBlue(hhead_title(pageDisplayName(page)) + hbody( |
529 | tag('table, |
530 | tr( |
531 | td(sliceSelector(), valign := 'top) |
532 | + td(sideContents, align := 'right, valign := 'top, rowspan := 2)) |
533 | + tr(td(mainContents + |
534 | (empty(iframe) ? "" : iframe(iframe, width := 600, height := 400)) + |
535 | footer(), align := 'center, valign := 'top)), |
536 | width := "100%", height := "100%"))); |
537 | } |
538 | |
539 | O servePagesToBot(Iterable<Page> pages) { |
540 | ret serveListToBot(map(pageToMap(wrapMapAsParams(params)), pages)); |
541 | } |
542 | |
543 | O serveListToBot(Collection l) { |
544 | if (nempty(params.get('max))) |
545 | l = takeFirst(parseInt(params.get('max)), l); |
546 | ret serveJSON(l); |
547 | } |
548 | |
549 | // uri starts with "/bot/" |
550 | O serveBot() { |
551 | S q = params.get('q); |
552 | |
553 | if (eqic(uri, "/bot/hello")) ret serveJSON("hello"); |
554 | if (eqic(uri, "/bot/hasPage")) ret serveJSON(hasPage(q)); |
555 | if (eqic(uri, "/bot/randomPageContaining")) { |
556 | assertNempty(q); |
557 | ret servePageToBot(random(filter(list(cc, Page), p -> cic(p.q, q))), params); |
558 | } |
559 | if (eqic(uri, "/bot/allPages")) |
560 | ret servePagesToBot(list(cc, Page)); |
561 | if (eqic(uri, "/bot/allPagesStartingWith")) { |
562 | assertNempty(q); |
563 | ret servePagesToBot(filter(list(cc, Page), p -> swic(p.q, q))); |
564 | } |
565 | if (eqic(uri, "/bot/allPagesEndingWith")) { |
566 | assertNempty(q); |
567 | ret servePagesToBot(filter(list(cc, Page), p -> ewic(p.q, q))); |
568 | } |
569 | if (eqic(uri, "/bot/allPagesContaining")) { |
570 | assertNempty(q); |
571 | ret servePagesToBot(filter(list(cc, Page), p -> cic(p.q, q))); |
572 | } |
573 | if (eqic(uri, "/bot/allPagesContainingRegexp")) { |
574 | assertNempty(q); |
575 | Pattern pat = regexpIC(q); |
576 | ret servePagesToBot(filter(list(cc, Page), p -> regexpFindIC(pat, p.q))); |
577 | } |
578 | |
579 | if (eqicOneOf(uri, "/bot/postSigned", /*"/bot/makePhysicalSlice",*/ "/bot/approveTrustRequest")) { |
580 | S text = rtrim(params.get('text)); |
581 | S key = getSignerKey(text); |
582 | if (empty(key)) ret subBot_serve500("Please include your public key"); |
583 | if (!isSignedWithKey(text, key)) ret subBot_serve500("Signature didn't verify"); |
584 | text = dropLastTwoLines(text); // drop signer + sig line |
585 | |
586 | Signer signer = uniq_sync Signer(publicKey := key); |
587 | |
588 | /*if (eqic(uri, "/bot/makePhysicalSlice")) { |
589 | if (!signer.trusted) ret subBot_serve500("Untrusted signer"); |
590 | Page page = findPageFromParams(jsonDecodeMap(text)); |
591 | if (page == null) ret subBot_serve500("Page not found"); |
592 | ret serveJSON(uniq2_sync(PhysicalSlice, slicePage := page).b ? "Slice made" : "Slice exists"); |
593 | }*/ |
594 | |
595 | if (eqic(uri, "/bot/postSigned")) { |
596 | new L out; |
597 | for (S line : tlft(text)) { |
598 | SS map = jsonDecodeMap(line); |
599 | GetEntriesAndPost x = new(db_mainConcepts()); |
600 | x.signer = signer; |
601 | Page page = findOrMakePageFromParams(map); |
602 | if (page == null) continue with out.add("Invalid page reference"); |
603 | x.go(page, map); |
604 | out.add(x.newEntry ? "Saved" : x.entry != null ? "Entry exists" : "Need key and value"); |
605 | } |
606 | ret serveJSON(out); |
607 | } |
608 | |
609 | if (eqic(uri, "/bot/approveTrustRequest")) { |
610 | if (!signer.trusted) ret subBot_serve500("Untrusted signer"); |
611 | Signer toApprove = conceptWhere Signer(publicKey := trim(text)); |
612 | if (toApprove == null) ret subBot_serve500("Signer to approve not found"); |
613 | cset(toApprove, trusted := true, approvedBy := signer.globalID); |
614 | ret serveJSON("Approved: " + trim(text)); |
615 | } |
616 | |
617 | ret subBot_serve500("CONFUSION"); |
618 | } |
619 | |
620 | if (eqic(uri, "/bot/post")) { |
621 | GetEntriesAndPost x = new(db_mainConcepts()); |
622 | x.go(pageFromQ(q), params); |
623 | ret serveJSON(x.newEntry ? "Saved" : x.entry != null ? "Entry exists" : "Need key and value"); |
624 | } |
625 | |
626 | if (eqic(uri, "/bot/entriesOnPage")) |
627 | ret serveJSON(map(entriesOnPage(findPageFromParams(params)), entryToMap(false))); |
628 | |
629 | if (eqic(uri, "/bot/entriesForKey")) |
630 | ret serveJSON(map(conceptsWhereIC(cc, Entry, key := params.get('key)), entryToMap(true))); |
631 | |
632 | if (eqic(uri, "/bot/lookup")) { |
633 | S key = params.get('key); |
634 | if (empty(key)) ret serveJSON("Need key"); |
635 | S value = getValue(findPageFromParams(params), key); |
636 | ret serveJSON(empty(value) ? "" : litmap(+value)); |
637 | } |
638 | |
639 | if (eqic(uri, "/bot/latestEntries")) |
640 | ret serveJSON(map(takeFirst(10, slice.idx_latestEntries.objectIterator()), entryToMap(true))); |
641 | if (eqic(uri, "/bot/latestPages")) |
642 | ret serveJSON(map(takeFirst(10, slice.idx_latestCreatedPages.objectIterator()), pageToMap())); |
643 | if (eqic(uri, "/bot/latestChangedPages")) |
644 | ret serveJSON(map(takeFirst(10, slice.idx_latestChangedPages.objectIterator()), pageToMap())); |
645 | |
646 | if (eqic(uri, "/bot/googleUsersCount")) |
647 | ret serveJSON(l(setMinus(collectAsCISet googleEmail(list(Session)), "", null))); |
648 | |
649 | if (eqic(uri, "/bot/totalPageCount")) |
650 | ret serveJSON(countConcepts(Page)); |
651 | /*if (eqic(uri, "/bot/pageWithoutPhysicalSliceCount")) |
652 | ret serveJSON(countConceptsWhere(Page, slice := null)); |
653 | if (eqic(uri, "/bot/physicalSliceCount")) |
654 | ret serveJSON(countConcepts(PhysicalSlice));*/ |
655 | if (eqic(uri, "/bot/trustedSignersCount")) |
656 | ret serveJSON(countConcepts(Signer, trusted := true)); |
657 | |
658 | if (eqic(uri, "/bot/valueSearch")) { |
659 | S value = params.get('value); |
660 | L<Entry> entries = conceptsWhereIC(cc, Entry, +value); |
661 | ret serveJSON(map(takeFirst(100, entries), entryToMap(true))); |
662 | } |
663 | |
664 | if (eqic(uri, "/bot/keyAndValueSearch")) { |
665 | S key = params.get('key), value = params.get('value); |
666 | Cl<Page> pages = pagesForKeyAndValue(key, value); |
667 | ret servePagesToBot(pages); |
668 | } |
669 | |
670 | if (eqic(uri, "/bot/keyValuePairsByPopularity")) { |
671 | L<PairS> pairs = map(list(Entry), e -> pair(e.key, e.value)); |
672 | LPair<S, Int> pairs2 = multiSetTopPairs(ciMultiSet(map pairToUglyStringForCIComparison(pairs))); |
673 | ret serveJSON(map(pairs2, p -> { |
674 | S key, value = unpair pairFromUglyString(p.a); |
675 | ret litorderedmap(n := p.b, +key, +value); |
676 | })); |
677 | } |
678 | |
679 | if (eqic(uri, "/bot/allKeys")) |
680 | ret serveListToBot(distinctCIFieldValuesOfConcepts(cc, Entry, 'key)); |
681 | |
682 | if (eqic(uri, "/bot/allKeysByPopularity")) |
683 | ret serveListToBot(mapMultiSetByPopularity(distinctCIFieldValuesOfConcepts_multiSet(cc, Entry, 'key), (key, n) -> litorderedmap(+n, +key))); |
684 | |
685 | if (eqic(uri, "/bot/dbSize")) |
686 | ret serveJSON(l(conceptsFile())); |
687 | |
688 | if (eqic(uri, "/bot/query")) |
689 | ret serveBotQuery(); |
690 | |
691 | if (eqic(uri, "/bot/createSlice")) { |
692 | Slice slice = createSlice(assertNempty(params.get('name))); |
693 | ret serveJSON(litorderedmap(id := str(slice.globalID), name := slice.name)); |
694 | } |
695 | |
696 | if (eqic(uri, "/bot/dumpAllSlices") && authed()) { |
697 | new L<Map> out; |
698 | for (Slice slice) try { |
699 | out.add(litorderedmap(slice := slice.caseID, ms := returnTimeFor(() -> dumpSliceToFile(loadBackgroundSlice(slice))))); |
700 | } catch print e { |
701 | out.add(litorderedmap(slice.caseID, error := exceptionToStringShort(e))); |
702 | } |
703 | ret serveJSON(out); |
704 | } |
705 | |
706 | if (eqic(uri, "/bot/dumpSlice")) { |
707 | File f = dumpSliceToFile(slice); |
708 | ret "OK, saved as: " + f2s(f); |
709 | } |
710 | |
711 | if (eqic(uri, "/bot/memStats")) { |
712 | Map fg = cloneMap(loadedSlices); |
713 | Map bg = cloneMap(backgroundSlices); |
714 | ret serveJSON(litcimap( |
715 | error := checkDoubleLoads(), |
716 | centralIndexEntries := l(centralIndex), |
717 | centralIndexMade := renderHowLongAgo(centralIndexMade), |
718 | numLoadedForegroundSlices := l(fg), |
719 | numLoadedBackgroundSlices := l(bg), |
720 | loadedForegroundSlicesEstimatedSize := longSum(map estimatedSliceDataSize(keys(fg))), |
721 | loadedBackgroundSlicesEstimatedSize := longSum(map estimatedSliceDataSize(keys(bg))), |
722 | loadedForegroundSlices := sortedIC(keysList(fg)), |
723 | loadedBackgroundSlices := sortedIC(keysList(bg)))); |
724 | } |
725 | |
726 | if (eqic(uri, "/bot/centralIndexGet")) { |
727 | CentralIndexEntry e = centralIndex.get(q); |
728 | ret serveJSON(e == null ? ll() : map(e.slices, sliceToMap())); |
729 | } |
730 | |
731 | // return primary triple from each slice |
732 | if (eqic(uri, "/bot/centralIndexGrab")) { |
733 | CentralIndexEntry ie = centralIndex.get(q); |
734 | new L<Map> out; |
735 | if (ie != null) for (Slice slice : ie.slices) { |
736 | LoadedSlice ls = loadBackgroundSlice(slice); |
737 | WorksOnSlice wos = new(ls); |
738 | Page page = wos.findPageFromQ(q); |
739 | Iterable<Entry> entries = entriesOnPage(page); |
740 | print("Page for " + q + " in slice " + ls.caseID + ": " + yesNo(page != null) + " - " + l(entries)); |
741 | Entry e = first(entries); |
742 | if (e != null) |
743 | out.add(litorderedmap(a := page.q, b := e.key, c := e.value, slice := slice.caseID)); |
744 | } |
745 | ret serveJSON(out); |
746 | } |
747 | |
748 | if (eqic(uri, "/bot/updateCentralIndex") && authed()) { |
749 | rst_index.trigger(); |
750 | ret "OK"; |
751 | } |
752 | |
753 | // end of bot methods |
754 | |
755 | ret subBot_serve404(); |
756 | } |
757 | |
758 | O serveBotQuery() { |
759 | S query = params.get('query); |
760 | ret new Query(slice).process(query); |
761 | } |
762 | |
763 | O serveLiteralSearch() { |
764 | S q = params.get('q); |
765 | L<Page> searchResults = literalSearch(q); |
766 | ret serveSearchResults("literal search" , q, searchResultsToShow, searchResults); |
767 | } |
768 | |
769 | L<Page> literalSearch(S q, O... _) { |
770 | int searchResultsToShow = optPar max(_, 100); |
771 | |
772 | // quick search in random order |
773 | //L<Page> searchResults = takeFirst(searchResultsToShow+1, filterIterator(iterator(list(cc, Page)), p -> cic(p.q, q)); |
774 | |
775 | // full search, order by length |
776 | ret takeFirst(searchResultsToShow+1, pagesSortedByLength(filter(list(cc, Page), p -> cic(p.q, q)))); |
777 | } |
778 | |
779 | O serveScoredSearch() { |
780 | S q = params.get('q); |
781 | L<Page> searchResults = (L<Page>) dm_call('agiBlueSearch, 'search, q, +cc); |
782 | ret serveSearchResults("scored search" , q, searchResultsToShow, searchResults); |
783 | } |
784 | |
785 | O serveLevenSearch() { |
786 | S q = params.get('q); |
787 | L<Page> searchResults = levenSearch(q); |
788 | ret serveSearchResults("leven search with distance 1" , q, searchResultsToShow, searchResults); |
789 | } |
790 | |
791 | L<Page> levenSearch(S q, O... _) { |
792 | int searchResultsToShow = optPar max(_, 100); |
793 | int maxEditDistance = 1; |
794 | |
795 | new Map<Page, Int> map; |
796 | for (Page p : list(cc, Page)) { |
797 | int distance = leven_limitedIC(q, p.q, maxEditDistance+1); |
798 | if (distance <= maxEditDistance) map.put(p, distance); |
799 | } |
800 | ret takeFirst(searchResultsToShow+1, keysSortedByValue(map)); |
801 | } |
802 | |
803 | O serveSearchResults(S searchType, S q, int searchResultsToShow, Collection<Page> searchResults) { |
804 | S title = "agi.blue " + searchType + " for " + htmlEncode2(quote(q)) + " (" + (l(searchResults) >= searchResultsToShow ? searchResultsToShow + "+ results" : nResults(l(searchResults))) + ")"; |
805 | |
806 | ret hhtml_agiBlue(hhead_title(htmldecode_dropAllTags(title)) |
807 | + hbody(hfullcenter(//top + |
808 | h3(title) |
809 | + p_nemptyLines_showFirst(searchResultsToShow, map(pageToHTMLLink(), searchResults))))); |
810 | } |
811 | |
812 | O serveQueryPage() { |
813 | S query = params.get('query); |
814 | if (query == null) query = loadSnippet(#1024258); |
815 | S title = agiBlueNameHTML() + " | Execute a query script (" + targetBlank("http://code.botcompany.de/1024274", "ALQL") + ")"; |
816 | ret hhtml_agiBlue(hhead_title(htmldecode_dropAllTags(title)) |
817 | + hbody(hfullcenter( |
818 | h3(title) |
819 | + form( |
820 | htextarea(query, name := 'query, cols := 80, rows := 10, autofocus := true) |
821 | + "<br><br>" |
822 | + hsubmit("Execute"), action := "/bot/query") |
823 | ))); |
824 | } |
825 | |
826 | S sliceHomeURL() { |
827 | ret slice == null ? agiBlueURL() : sliceHomeURL(slice.sliceConcept.globalID); |
828 | } |
829 | |
830 | S sliceHomeURL(GlobalID slice) { |
831 | ret agiBlueURL() + hquery(+slice); |
832 | } |
833 | |
834 | O serveCreateSlicePage() { |
835 | S sliceName = trim(params.get('sliceName)); |
836 | if (eq(params.get('doIt), "1") && nempty(sliceName)) { |
837 | // TODO: check for existing name |
838 | Slice slice = createSlice(sliceName); |
839 | ret hrefresh(sliceHomeURL(slice.globalID)); |
840 | } |
841 | |
842 | S title = agiBlueNameHTML() + " | Create slice"; |
843 | ret hhtml_agiBlue(hhead_title(htmldecode_dropAllTags(title)) |
844 | + hbody(hfullcenter( |
845 | h3(title) |
846 | + form( |
847 | hhidden(doIt := 1) |
848 | + "Slice name: " + htextinput(+sliceName, autofocus := true) |
849 | + "<br><br>" |
850 | + hsubmit("Create slice")) |
851 | ))); |
852 | } |
853 | |
854 | F1<Page, S> pageToHTMLLink(O... _) { |
855 | optPar int displayLength = main.displayLength; |
856 | ret func(Page p) -> S { |
857 | S name = pageDisplayName(p); |
858 | ret ahref(agiBlue_pageURLWithSlice(p), |
859 | htmlEncode2(shorten(displayLength, name)), |
860 | title := name); |
861 | }; |
862 | } |
863 | |
864 | // and google log-in |
865 | S sliceSelector() { |
866 | ret htag table(tr( |
867 | (!showGoogleLogIn ? "" : td( |
868 | googleSignIn_signInButton(agiBlueURL() + "/google-verify", "console.log(data);", ""), style := "padding-right: 10px;") |
869 | + td(nobr(ahref("javascript:signOut()", "Sign out")), style := "padding-right: 10px; padding-bottom: 0.15em")) |
870 | |
871 | + (!showSliceSelector ? "" : hform(td( |
872 | "Select reality slice: " |
873 | + hselect(availableSlices(), session.selectedSlice, name := 'slice, onchange := "this.form.submit()") |
874 | /*+ " " + hsubmit("Go")*/ |
875 | + " " + ahref(agiBlueURL() + "/createSlice", "Create slice...") |
876 | ))))); |
877 | } |
878 | |
879 | S sliceAsHTML() { |
880 | if (slice.isMainSlice()) ret htmlEncode2(agiBlueName() + "'s main slice"); |
881 | if (slice.sliceConcept == null) ret "Slice ???"; |
882 | ret htmlEncode2("Slice " + quote(slice.sliceConcept.name)); |
883 | } |
884 | |
885 | Slice sliceConcept() { ret slice.sliceConcept; } |
886 | |
887 | S footer() { |
888 | ret p(small(elapsedMS_sysNow(started) + " ms"); |
889 | } |
890 | |
891 | S verifiedEmail() { |
892 | ret session == null || !session.googleEmailVerified ? null |
893 | : session.googleEmail; |
894 | } |
895 | |
896 | bool authed() { |
897 | bool ver = eqic(verifiedEmail(), "stefan.reich.maker.of.eye@googlemail.com"); |
898 | print("Auth check " + verifiedEmail() + " -> " + ver); |
899 | ret ver; |
900 | } |
901 | |
902 | S googleSignInID() { |
903 | ret eqic(domain(), "botcompany.de") ? botCompanyGoogleSignInID() : agiBlueGoogleSignInID(); |
904 | } |
905 | |
906 | S hhtml_agiBlue(S contents) { |
907 | ret hhtml(hAddToHead_fast(contents, |
908 | hIncludeGoogleFont("Source Sans Pro") |
909 | //+ [[<meta name="google-site-verification" content="oRvyeTqgSuv-FE_c-2UKM1Vp0oDqx8h9WruHYWqA-NQ" />]] |
910 | + loadJQuery() |
911 | + hmobilefix() |
912 | + googleSignIn_header("", googleSignInID()) |
913 | + hstylesheet("body { font-family: Source Sans Pro }"))); |
914 | } |
915 | |
916 | } // end of Request |
917 | |
918 | svoid dbLog(O... params) { |
919 | logStructure(programFile("db.log"), ll(params)); |
920 | } |
921 | |
922 | static IF1<Page, Map> pageToMap(O... _) { |
923 | optPar bool withEntries; |
924 | |
925 | bool nameOnly = eqOneOf(optPar nameOnly(_), "1", true); |
926 | if (nameOnly) ret (IF1<Page, Map>) p -> litmap(q := p.q); |
927 | |
928 | ret (IF1<Page, Map>) p -> { |
929 | L<Entry> entries = findBackRefs(p, Entry); |
930 | ret litorderedmap( |
931 | q := p.q, |
932 | nEntries := l(entries), |
933 | created := p.created, |
934 | modified := p._modified, |
935 | entries := !withEntries ? null : map(entries, entryToMap(false))); |
936 | }; |
937 | } |
938 | |
939 | static IF1<Entry, Map> entryToMap(bool withPage) { |
940 | ret (IF1<Entry, Map>) e -> litorderedmap( |
941 | created := e.created, |
942 | i := e.count, |
943 | key := e.key, |
944 | value := e.value, |
945 | q := withPage ? e.page->q : null, |
946 | signer := getString globalID(e.signer!)); |
947 | } |
948 | |
949 | static IF1<Slice, Map> sliceToMap() { |
950 | ret (IF1<Slice, Map>) s -> litorderedmap( |
951 | created := s.created, |
952 | globalID := str(s.globalID), |
953 | name := s.name, |
954 | caseID := s.caseID); |
955 | } |
956 | |
957 | sO servePageToBot(Page page, SS params) { |
958 | if (page == null) ret serveJSON(null); |
959 | params = asCIMap(params); |
960 | Map map = pageToMap(withEntries := valueIs1 withEntries(params)).get(page); |
961 | ret serveJSON(map); |
962 | } |
963 | |
964 | sclass GetEntriesAndPost { |
965 | Concepts cc; |
966 | L<Entry> entries; |
967 | Entry entry; |
968 | bool newEntry; |
969 | Signer signer; |
970 | |
971 | *(Concepts *cc) {} |
972 | |
973 | GetEntriesAndPost go(Page page, SS params) { |
974 | S key = trim(params.get('key)), value = trim(params.get('value)); |
975 | print("GetEntriesAndPost: " + quote(page.q) + ", " + quote(key) + " := " + quote(value)); |
976 | withDBLock { |
977 | entries = findBackRefs(page, Entry); |
978 | if (nempty(key) && nempty(value)) { |
979 | S ip = subBot_clientIP(); |
980 | entry = firstThat(e -> eqic(e.key, key) && eq_icIf(!allowMultipleCasesInValues, e.value, value) && eq(e.ip, ip), entries); |
981 | if (entry == null) { |
982 | print("SAVING"); |
983 | Entry e = cnew(cc, Entry, +page, +key, +value, +ip, count := l(entries) + 1, +signer); |
984 | if (makeAllKeysPages) pageFromQ(cc, key); |
985 | if (makeAllValuesPages) pageFromQ(cc, value); |
986 | page.change(); // bump modification date |
987 | entry = e; |
988 | newEntry = true; |
989 | entries.add(entry); |
990 | dbLog("New entry", page := page.q, globalID := e.globalID, count := e.count, +key, +value); |
991 | } |
992 | } |
993 | } |
994 | |
995 | sortByFieldInPlace created(entries); |
996 | numberEntriesInConceptField count(entries); |
997 | this; |
998 | } |
999 | } |
1000 | |
1001 | static SS availableSlices() { |
1002 | ret mapToOrderedMap(s -> pair(str(s.globalID), s.name + " [ID: " + s.globalID + "]"), list(Slice)); |
1003 | } |
1004 | |
1005 | static L<Entry> entriesOnPage(Page p) { |
1006 | ret p == null ? null : sortedByField count(findBackRefs(p, Entry)); |
1007 | } |
1008 | |
1009 | static L<Page> pagesSortedByLength(L<Page> l) { |
1010 | ret sortedByCalculatedField(l, p -> l(p.q)); |
1011 | } |
1012 | |
1013 | sbool agiBlue_isOriginal() { |
1014 | ret amProgram(#1023558); |
1015 | } |
1016 | |
1017 | sS agiBlueURL() { |
1018 | ret agiBlue_isOriginal() ? "https://agi.blue" : "/" + psI(programID()) + "/raw"; |
1019 | } |
1020 | |
1021 | sS agiBlueName() { |
1022 | ret agiBlue_isOriginal() ? "agi.blue" : "agi.blue clone " + programID(); |
1023 | } |
1024 | |
1025 | svoid cleanMeUp { |
1026 | cleanUp(fan); |
1027 | } |
1028 | |
1029 | static LoadedSlice loadSlice(Slice slice) { |
1030 | ret loadSlice(slice.caseID, str(slice.globalID)); |
1031 | } |
1032 | |
1033 | static LoadedSlice loadSlice(S caseID, S globalID) { |
1034 | caseID = unnull(caseID); |
1035 | lock fan.lock; |
1036 | |
1037 | // move slice from background to foreground |
1038 | |
1039 | Concepts cc = backgroundFan.getIfLoaded(caseID); |
1040 | if (cc != null) { |
1041 | LoadedSlice slice = assertNotNull(backgroundSlices.get(caseID)); |
1042 | backgroundFan.loaded.remove(caseID); |
1043 | fan.loaded.put(caseID, cc); |
1044 | backgroundSlices.remove(caseID); |
1045 | loadedSlices.put(caseID, slice); |
1046 | ret slice; |
1047 | } |
1048 | |
1049 | fan.get(caseID, globalID); |
1050 | ret loadedSlices.get(caseID); |
1051 | } |
1052 | |
1053 | // load background slice |
1054 | static LoadedSlice loadBackgroundSlice(Slice slice) { |
1055 | ret loadBackgroundSlice(slice.caseID, str(slice.globalID)); |
1056 | } |
1057 | |
1058 | static LoadedSlice loadBackgroundSlice(S caseID, S globalID) { |
1059 | caseID = unnull(caseID); |
1060 | lock fan.lock; |
1061 | |
1062 | // check if in foreground |
1063 | |
1064 | if (fan.getIfLoaded(caseID) != null) |
1065 | ret loadedSlices.get(caseID); |
1066 | |
1067 | backgroundFan.get(caseID, globalID); |
1068 | LoadedSlice slice = assertNotNull(backgroundSlices.get(caseID)); |
1069 | slice.lastAccess = sysNow(); |
1070 | while (l(backgroundSlices) > maxBackgroundSlices) |
1071 | unloadLeastRecentlyAccessedBackgroundSlice(); |
1072 | ret slice; |
1073 | } |
1074 | |
1075 | // TODO: sync unloading slice with accessors |
1076 | svoid unloadLeastRecentlyAccessedBackgroundSlice() { |
1077 | lock fan.lock; |
1078 | S caseID = lowestByField lastAccess(keys(backgroundSlices)); |
1079 | if (caseID != null) |
1080 | backgroundFan.unloadCase(caseID); |
1081 | } |
1082 | |
1083 | static Slice sliceConceptForGlobalID(S globalID) { |
1084 | ret sliceConceptForGlobalID(toGlobalIDObj(globalID)); |
1085 | } |
1086 | |
1087 | static Slice sliceConceptForGlobalID(GlobalID globalID) { |
1088 | ret conceptWhere Slice(+globalID); |
1089 | } |
1090 | |
1091 | static Slice sliceConceptForCaseID(S caseID) { |
1092 | ret conceptWhere Slice(+caseID); |
1093 | } |
1094 | |
1095 | sS agiBlueNameHTML() { |
1096 | ret ahref(agiBlueURL(), htmlEncode2(agiBlueName())); |
1097 | } |
1098 | |
1099 | static Slice createSlice(S name) { |
1100 | name = shorten(name, maxSliceNameLength); |
1101 | Slice slice = cnew Slice(+name); |
1102 | cset(slice, caseID := slice.defaultCaseID()); |
1103 | loadSlice(slice).initialSetup(slice.globalID); |
1104 | ret slice; |
1105 | } |
1106 | |
1107 | static Slice sliceForName(S name) { |
1108 | ret conceptWhereCI Slice(+name); |
1109 | } |
1110 | |
1111 | sclass Query extends WorksOnSlice { |
1112 | SS vars = ciMap(); |
1113 | |
1114 | *(LoadedSlice slice) { super(slice); } |
1115 | |
1116 | O process(S query) { |
1117 | try object processLines(agiBlue_parseQueryScript(query)); |
1118 | ret serveJSON("No return statement"); |
1119 | } |
1120 | |
1121 | O processLines(L<ALQLLine> lines) { |
1122 | for (ALQLLine line : lines) { |
1123 | if (line cast ALQLSlice) { |
1124 | Slice slice = sliceForName(line.slice); |
1125 | if (slice == null) ret serveJSON("Slice not found: " + line.slice); |
1126 | LoadedSlice oldSlice = Query.this.slice; |
1127 | try { |
1128 | setSlice(loadSlice(slice)); |
1129 | try object processLines(line.contents); |
1130 | } finally { |
1131 | setSlice(oldSlice); |
1132 | } |
1133 | } else if (line cast ALQLReturn) |
1134 | ret serveJSON(ll(getOrKeep(vars, line.var))); |
1135 | else if (line cast ALQLTriple) { |
1136 | T3S t = line.triple; |
1137 | t = tripleMap(t, s -> getOrKeep(vars, s)); |
1138 | Cl<Page> pages; |
1139 | S var; |
1140 | if (isDollarVar(t.c)) { |
1141 | var = t.c; |
1142 | if (isDollarVar(t.a)) { |
1143 | if (isDollarVar(t.b)) todo(t); |
1144 | Entry e = random(conceptsWhereCI(cc, Entry, key := t.b)); |
1145 | if (e == null) ret serveJSON("No results for " + var); |
1146 | pages = pagesForKeyAndValue(t.b, t.c); |
1147 | vars.put(t.a, e.page->q); |
1148 | vars.put(t.c, e.value); |
1149 | continue; |
1150 | } else if (isDollarVar(t.b)) { |
1151 | Page page = findPageFromQ(t.a); |
1152 | Entry e = random(findBackRefs(page, Entry)); |
1153 | if (e == null) ret serveJSON("No results for " + var); |
1154 | vars.put(t.b, e.key); |
1155 | vars.put(t.c, e.value); |
1156 | continue; |
1157 | } else { |
1158 | S val = getValue(t.a, t.b); |
1159 | if (val == null) ret serveJSON("No results for " + var); |
1160 | pages = ll(pageFromQ(val)); |
1161 | } |
1162 | } else if (isDollarVar(t.b)) { |
1163 | var = t.b; |
1164 | if (isDollarVar(t.c)) todo(t); |
1165 | if (isDollarVar(t.a)) { |
1166 | L<Entry> entries = conceptsWhereCI(cc, Entry, value := t.c); |
1167 | if (empty(entries)) ret serveJSON("No results for " + var); |
1168 | Entry e = random(entries); |
1169 | vars.put(t.a, e.page->q); |
1170 | vars.put(t.b, e.key); |
1171 | continue; |
1172 | } else { |
1173 | Cl<S> keys = keysForPageAndValue(t.a, t.c); |
1174 | if (empty(keys)) ret serveJSON("No results for " + var); |
1175 | pages = map(f<S, Page> pageFromQ, keys); |
1176 | } |
1177 | } else { |
1178 | var = t.a; |
1179 | if (!isDollarVar(t.a)) todo(t); |
1180 | if (isDollarVar(t.b)) todo(t); |
1181 | if (isDollarVar(t.c)) todo(t); |
1182 | pages = pagesForKeyAndValue(t.b, t.c); |
1183 | } |
1184 | if (empty(pages)) ret serveJSON("No results for " + var); |
1185 | vars.put(var, random(pages).q); |
1186 | } else if (line cast ALQLPage) { |
1187 | if (eq(line.matchMethod, 'eqic)) |
1188 | findPageFromQ(line.page); |
1189 | else if (eq(line.matchMethod, 'flexMatchDollarVarsIC_first)) { |
1190 | new L<SS> l; |
1191 | for (Page p : list(cc, Page)) |
1192 | addIfNotNull(l, flexMatchDollarVarsIC_first(line.page, p.q)); |
1193 | if (empty(l)) ret serveJSON("No results for " + curly(line.page)); |
1194 | vars.putAll(random(l)); |
1195 | } else |
1196 | fail("Unknown match method: " + line.matchMethod); |
1197 | } else |
1198 | fail("Can't interpret: " + line); |
1199 | } |
1200 | |
1201 | null; |
1202 | } |
1203 | } |
1204 | |
1205 | static Page pageFromQ(Concepts cc, S q) { |
1206 | ret empty(q) ? null : uniqCI_sync(cc, Page, +q); |
1207 | } |
1208 | |
1209 | /*sS globalIDFromCaseID(S caseID) { |
1210 | ret assertGlobalID(takeLast(globalIDLength(), caseID)); |
1211 | }*/ |
1212 | |
1213 | sS agiBlue_pageURLWithSlice(Page p) { |
1214 | ret p == null ? null : agiBlueURL() + hquery(slice := _get(sliceForPage(p), 'globalID), q := p.q); |
1215 | } |
1216 | |
1217 | static Slice sliceForPage(Page p) { |
1218 | if (p == null) null; |
1219 | SliceInfo info = conceptWhere(p._concepts, SliceInfo); |
1220 | ret info == null ? null : sliceConceptForGlobalID(info.globalID); |
1221 | } |
1222 | |
1223 | static GlobalID mainSliceGlobalID() { |
1224 | ret conceptWhere Slice(caseID := "").globalID; |
1225 | } |
1226 | |
1227 | static File sliceDumpFile(Slice slice) { |
1228 | ret programFile("slice-dumps/" + or2(slice.caseID, "main") + ".slice"); |
1229 | } |
1230 | |
1231 | static File dumpSliceToFile(LoadedSlice slice) { |
1232 | ret withDBLock(slice.cc, func -> File { |
1233 | cset(slice.sliceConcept, sliceDumped := now()); |
1234 | File file = sliceDumpFile(slice.sliceConcept); |
1235 | new LS lines; |
1236 | for (Page p : list(slice.cc, Page)) |
1237 | lines.add(quote(p.q)); |
1238 | for (Entry e : list(slice.cc, Entry)) |
1239 | lines.add(sfu(tripleToList(entryToTriple(e)))); |
1240 | saveLinesAsTextFile(file, lines); |
1241 | ret file; |
1242 | }); |
1243 | } |
1244 | |
1245 | static T3S entryToTriple(Entry e) { |
1246 | ret e == null ? null : t3(e.page->q, e.key, e.value); |
1247 | } |
1248 | |
1249 | static ConceptsLoadedOnDemand slicesLoadedOnDemand(Map<S, LoadedSlice> loadedSlices) { |
1250 | new ConceptsLoadedOnDemand fan; |
1251 | fan.onCaseLoaded(voidfunc(S caseID, O globalID, Concepts concepts) { |
1252 | LoadedSlice ls = new(caseID, concepts); |
1253 | if (nempty(globalID)) |
1254 | ls.initialSetup(toGlobalIDObj((S) globalID)); |
1255 | loadedSlices.put(caseID, ls); |
1256 | }); |
1257 | fan.onUnloadingCase(voidfunc(S caseID, Concepts concepts) { |
1258 | loadedSlices.remove(caseID); |
1259 | }); |
1260 | ret fan; |
1261 | } |
1262 | |
1263 | // check if slices are loaded both in foreground and background (BAD) |
1264 | sS checkDoubleLoads() { |
1265 | LS l = sharedKeys(loadedSlices, backgroundSlices); |
1266 | ret empty(l) ? null : "ERROR: Slice loaded in background and foreground: " + first(l); |
1267 | } |
1268 | |
1269 | // just use size of concepts.structure.gz |
1270 | static long estimatedSliceDataSize(Slice slice) { |
1271 | ret estimatedSliceDataSize(slice.caseID); |
1272 | } |
1273 | |
1274 | static long estimatedSliceDataSize(S caseID) { |
1275 | ret l(conceptsFile(fan.dbID(caseID))); |
1276 | } |
1277 | |
1278 | static CentralIndexEntry centralIndexGet(S s) { ret getOrCreate CentralIndexEntry(centralIndex, s); } |
1279 | |
1280 | static CentralIndexEntry newCentralIndexGet(S s) { ret getOrCreate CentralIndexEntry(newCentralIndex, s); } |
1281 | |
1282 | // executed by rst_index |
1283 | svoid makeCentralIndex() { |
1284 | newCentralIndex.clear(); |
1285 | pcall { |
1286 | for (Slice slice : conceptsWhere(Slice, indexed := true)) |
1287 | addToNewCentralIndex(slice); |
1288 | } |
1289 | centralIndex.putAll(newCentralIndex); |
1290 | removeAllKeysNotIn(centralIndex, newCentralIndex); |
1291 | newCentralIndex.clear(); |
1292 | centralIndexMade = now(); |
1293 | } |
1294 | |
1295 | svoid addToNewCentralIndex(Slice slice) { |
1296 | for (S s : sliceDump_pageNamesIncludingKeys(sliceDumpFile(slice))) |
1297 | newCentralIndexGet(s).slices.add(slice); |
1298 | } |
Began life as a copy of #1023558
download show line numbers debug dex old transpilations
Travelled to 6 computer(s): bhatertpkbcr, mqqgnosmbjvj, pyentgdyhuwx, pzhvpgtvlbxg, tvejysmllsmz, vouqrxazstgt
No comments. add comment
Snippet ID: | #1024374 |
Snippet name: | agi.blue source [backup 2019/08/08] |
Eternal ID of this version: | #1024374/1 |
Text MD5: | 07f2766b05c90ef881b0eeee61c03492 |
Transpilation MD5: | 5d36b38d353eb71d4f777274f322d618 |
Author: | stefan |
Category: | javax / html |
Type: | JavaX module (desktop) |
Public (visible to everyone): | Yes |
Archived (hidden from active list): | No |
Created/modified: | 2019-08-08 13:24:27 |
Source code size: | 45411 bytes / 1298 lines |
Pitched / IR pitched: | No / No |
Views / Downloads: | 262 / 859 |
Referenced in: | [show references] |