Download Jar. Uses 1658K of libraries. Click here for Pure Java version (18755L/137K).
1 | !7 |
2 | |
3 | // See #1023660 for the older 99 lines version |
4 | |
5 | // Customizations: |
6 | |
7 | sbool asyncSearch = false; |
8 | sbool allowMultipleCasesInValues = true; |
9 | sbool showSliceSelector = true; |
10 | sbool showGoogleLogIn = true; |
11 | sbool makeAllValuesPages = true; |
12 | sbool legacy = true; // We still have slices without applied fixes |
13 | |
14 | static int displayLength = 140; |
15 | static int sideDisplayLength = 50; // when in side bar |
16 | static int searchResultsToShow = 50; |
17 | sS sideSearchType = 'literal; |
18 | static int maxBackgroundSlices = 1000; |
19 | |
20 | // end of customizations |
21 | |
22 | // in main DB |
23 | concept Slice { |
24 | GlobalID globalID = aGlobalIDObject(); |
25 | S caseID; |
26 | S name; |
27 | long sliceDumped; // wall time |
28 | |
29 | S defaultCaseID() { |
30 | ret uniqueFileNameUsingMD5_80_v2(name + " " + globalID); |
31 | } |
32 | } |
33 | |
34 | // in any DB |
35 | concept Page { |
36 | S globalID = aGlobalID(); |
37 | S url, q; |
38 | } |
39 | |
40 | abstract concept AbstractEntry { |
41 | new Ref<Page> page; |
42 | int count; |
43 | S key; |
44 | S ip; |
45 | new Ref<Signer> signer; |
46 | } |
47 | |
48 | concept Entry > AbstractEntry { |
49 | S globalID = aGlobalID(); // TODO: migrate to object |
50 | S value; |
51 | } |
52 | |
53 | concept MultiLineEntry > AbstractEntry { |
54 | GlobalID globalID = aGlobalIDObject(); |
55 | S value; |
56 | } |
57 | |
58 | concept BlobEntry > AbstractEntry { |
59 | GlobalID globalID = aGlobalIDObject(); |
60 | long length; |
61 | S md5; |
62 | } |
63 | |
64 | // in main DB? |
65 | concept Signer { |
66 | S globalID = aGlobalID(); |
67 | S publicKey; |
68 | bool trusted; |
69 | S approvedBy; |
70 | } |
71 | |
72 | // in main DB |
73 | concept Session { |
74 | S cookie; |
75 | S selectedSlice; // global ID |
76 | Page slicePage; // phasing out |
77 | |
78 | long googleLogInDate; |
79 | S googleEmail, googleFirstName, googleLastName; |
80 | bool googleEmailVerified; |
81 | } |
82 | |
83 | // singleton concept in every DB |
84 | concept SliceInfo { |
85 | GlobalID globalID; |
86 | LS mandatoryBotVMBusIDs; |
87 | } |
88 | |
89 | static SS searchTypeToText = litmap( |
90 | leven := "Leven", |
91 | literal := "Literal", |
92 | scored := "Scored"); |
93 | |
94 | static int sourceCodeLines; |
95 | static ConceptsLoadedOnDemand fan; |
96 | static ConceptsLoadedOnDemand backgroundFan; |
97 | static Map<S, LoadedSlice> loadedSlices = syncMap(); |
98 | static Map<S, LoadedSlice> backgroundSlices = syncMap(); |
99 | |
100 | sclass LoadedSlice { // non-persistent class |
101 | S caseID; |
102 | Concepts cc; |
103 | Slice sliceConcept; |
104 | long lastAccess = sysNow(); |
105 | |
106 | ConceptFieldIndexDesc idx_latestEntries, idx_latestCreatedPages, idx_latestChangedPages; |
107 | |
108 | *(S *caseID, Concepts *cc) { |
109 | indexThings(); |
110 | |
111 | // forward changes to Slice concept |
112 | onConceptsChange(cc, r { |
113 | cset(sliceConcept, _modified := now()) |
114 | }); |
115 | |
116 | if (makeAllValuesPages && legacy) |
117 | for (Entry e : list(cc, Entry)) |
118 | pageFromQ(e.value); |
119 | } |
120 | |
121 | void initialSetup(GlobalID globalID) { |
122 | SliceInfo info = uniq(cc, SliceInfo); |
123 | cset(info, +globalID); |
124 | } |
125 | |
126 | // create if not there |
127 | Page pageFromQ(S q) { |
128 | ret main.pageFromQ(cc, q); |
129 | } |
130 | |
131 | void indexThings() { |
132 | indexConceptFieldsCI(cc, Page, 'url, Page, 'q, Entry, 'key, Entry, 'value); |
133 | indexConceptFields(cc, Signer, 'publicKey); |
134 | indexConceptFields(cc, Session, 'cookie); |
135 | indexSingletonConcept(cc, SliceInfo); |
136 | idx_latestCreatedPages = new ConceptFieldIndexDesc(cc, Page, 'created); |
137 | idx_latestChangedPages = new ConceptFieldIndexDesc(cc, Page, '_modified); |
138 | idx_latestEntries = new ConceptFieldIndexDesc(cc, Entry, 'created); |
139 | } |
140 | |
141 | bool isMainSlice() { ret empty(caseID); } |
142 | GlobalID globalID() { ret sliceConcept.globalID; } |
143 | } |
144 | |
145 | p { |
146 | if (bootstrapDataFrom(#1023558)) deleteMyBackups(); |
147 | thinMyBackups(); |
148 | fan = slicesLoadedOnDemand(loadedSlices); |
149 | mainConcepts = fan.get("", null); |
150 | indexConceptFields(Slice, 'globalID, Slice, 'caseID); |
151 | cset(uniq_returnIfNew Slice(caseID := ""), name := "main slice"); |
152 | loadedSlices.get("").initialSetup(toGlobalIDObj(mainSliceGlobalID())); |
153 | |
154 | backgroundFan = slicesLoadedOnDemand(backgroundSlices); |
155 | backgroundFan.lock = fan.lock; |
156 | |
157 | // legacy conversions! |
158 | //deleteConcepts(Slice, globalID := GlobalID('wftlawbagrwprywn)); |
159 | |
160 | // Approve this machine's key |
161 | PKIKeyPair machineKey = agiBot_trustedKeyForMachine(); |
162 | if (machineKey != null) { |
163 | print("Approving this machine's key: " + machineKey.publicKey); |
164 | cset(uniq_sync(Signer, publicKey := machineKey.publicKey), trusted := true, approvedBy := "local"); |
165 | } |
166 | |
167 | sourceCodeLines = countLines(mySource()); |
168 | } |
169 | |
170 | // Serve page |
171 | |
172 | set flag NoNanoHTTPD. html { |
173 | ret new Request().serve(uri, params); |
174 | } |
175 | |
176 | sclass WorksOnSlice { |
177 | Concepts cc; |
178 | LoadedSlice slice; |
179 | |
180 | *() {} |
181 | *(LoadedSlice *slice) { cc = slice.cc; } |
182 | |
183 | void setSlice(LoadedSlice slice) { |
184 | this.slice = slice; |
185 | cc = slice.cc; |
186 | } |
187 | |
188 | Page findPageFromParams(Map map) { |
189 | S q = getString q(map); |
190 | ret empty(q) ? null : findPageFromQ(q); |
191 | } |
192 | |
193 | Page findOrMakePageFromParams(Map map) { |
194 | ret pageFromQ(getString q(map)); |
195 | } |
196 | |
197 | Page findPageFromQ(S q) { |
198 | ret conceptWhereCI(cc, Page, +q); |
199 | } |
200 | |
201 | Page pageFromQ(S q) { |
202 | ret slice.pageFromQ(q); |
203 | } |
204 | |
205 | Set<Page> pagesForKeyAndValue(S key, S value) { |
206 | L<Entry> entries = conceptsWhereIC(cc, Entry, +value, +key); |
207 | ret asSet(ccollect page(entries)); |
208 | } |
209 | |
210 | Cl<S> keysForPageAndValue(S q, S value) { |
211 | Page page = findPageFromQ(q); |
212 | if (page == null) null; |
213 | ret collect key(conceptsWhereIC(cc, Entry, +page, +value)); |
214 | } |
215 | |
216 | // DB functions |
217 | |
218 | bool hasPage(S q) { ret hasConceptWhereIC(Page, +q); } |
219 | S getValue(Page page, S key) { |
220 | ret page == null || empty(key) ? null : getString value(highestByField count(objectsWhereIC(findBackRefs(page, Entry), +key))); |
221 | } |
222 | S getValue(S page, S key) { |
223 | ret getValue(findPageFromQ(page), key); |
224 | } |
225 | S pageDisplayName(Page page) { |
226 | /*S name = getValue(page, "read as"); |
227 | bool unnaturalName = nempty(name) && !eq(makeAGIDomain(name), page.url); |
228 | ret unnaturalName ? name + " " + squareBracketed(page.url) : or2(name, unpackAGIDomainOpt(page.url));*/ |
229 | ret page.q; |
230 | } |
231 | |
232 | S renderThing(S s, bool forceURLDisplay) { |
233 | ret forceURLDisplay || isURL(s) || isAGIDomain(s) |
234 | ? ahref(fixAGILink(absoluteURL(s)), htmlencode2(shorten(displayLength, s))) |
235 | : ahref(agiBlue_linkForPhrase(s, |
236 | slice := slice.isMainSlice() ? null : slice.sliceConcept.globalID), |
237 | htmlencode2(shorten(displayLength, s))); |
238 | } |
239 | |
240 | } // end of WorksOnSlice |
241 | |
242 | sclass Request extends WorksOnSlice { |
243 | S cookie; |
244 | Session session; |
245 | S uri; |
246 | SS params; |
247 | long started = sysNow(); |
248 | |
249 | // should also work for standalone |
250 | bool isHomeDomain() { |
251 | S domain = domain(); |
252 | ret eqic(domain, "www.agi.blue") || !ewic(domain, ".agi.blue"); |
253 | } |
254 | |
255 | O serve(S uri, SS params) { |
256 | this.uri = dropMultipleSlashes(uri); |
257 | this.params = params; |
258 | new Matches m; |
259 | |
260 | print(uri + " ? " + unnull(subBot_query())); |
261 | |
262 | // get cookie & session |
263 | |
264 | cookie = cookieFromUser(); |
265 | if (cookie != null) |
266 | session = uniq_sync(Session, +cookie); |
267 | else |
268 | session = unlistedWithValues(Session); |
269 | |
270 | // check if user wants to change slices |
271 | |
272 | S selectSlice = params.get('slice); |
273 | if (isGlobalID(selectSlice)) |
274 | cset(session, selectedSlice := selectSlice); |
275 | if (session.selectedSlice == null) |
276 | cset(session, selectedSlice := str(mainSliceGlobalID())); |
277 | |
278 | // get case ID |
279 | |
280 | S caseID = ""; |
281 | Slice sliceConcept = sliceConceptForGlobalID(session.selectedSlice); |
282 | print("Selected slice: " + session.selectedSlice + ", obj? " + (sliceConcept != null)); |
283 | if (sliceConcept != null) caseID = sliceConcept.caseID; |
284 | print("caseID: " + caseID); |
285 | |
286 | // load slice |
287 | |
288 | slice = assertNotNull(loadSlice(caseID, session.selectedSlice)); |
289 | cc = slice.cc; |
290 | slice.sliceConcept = sliceConcept; |
291 | |
292 | // Check for special URIs |
293 | |
294 | if (swic(uri, "/bot/")) ret serveBot(); |
295 | |
296 | // eleu appends a slash to the URI if it's a single identifier, so we drop it again |
297 | S uri2 = dropTrailingSlash(uri); |
298 | |
299 | if (eqic(uri2, "/google-verify")) { |
300 | print("Google-verify started."); |
301 | Payload payload = printStruct(googleVerifyUserToken2(googleSignInID(), params.get("token"))); |
302 | if (payload == null) ret print("google-verify", "No"); |
303 | cset(session, |
304 | googleLogInDate := now(), |
305 | googleEmail := payload.getEmail(), |
306 | googleEmailVerified := payload.getEmailVerified(), |
307 | googleFirstName := strOrNull(payload.get("given_name")), |
308 | googleLastName := strOrNull(payload.get("family_name"))); |
309 | |
310 | ret print("google-verify", payload.getEmail() + " " + (payload.getEmailVerified() ? "(verified)" : "(not verified)")); |
311 | } |
312 | |
313 | if (eqic(uri2, "/search")) ret serveScoredSearch(); |
314 | if (eqic(uri2, "/literalSearch")) ret serveLiteralSearch(); |
315 | if (eqic(uri2, "/levenSearch")) ret serveLevenSearch(); |
316 | |
317 | if (eqic(uri2, "/query")) ret serveQueryPage(); |
318 | if (eqic(uri2, "/createSlice")) ret serveCreateSlicePage(); |
319 | |
320 | S q = params.get('q); |
321 | S domain = or2(params.get('domain), domain()); |
322 | |
323 | S raw = firstKeyWithValue("", params); // agi.blue?something |
324 | if (nempty(raw) && empty(q)) q = raw; |
325 | /*if (nempty(q)) { |
326 | domain = makeAGIDomain(q); |
327 | if (l(domain) > maximumDomainPartLength()) // escape with "domain=" |
328 | ret hrefresh(agiBlueURL() + hquery(+domain, key := "read as", value := q)); |
329 | ret hrefresh("http://" + domain + (eq(q, domain) ? "" : "/" + hquery(key := "read as", value := q))); |
330 | //uri = "/"; replaceMapWithParams(params, key := "read as", value := q); |
331 | }*/ |
332 | S url = domain + dropTrailingSlash(uri); |
333 | |
334 | // domain to query |
335 | //if (empty(q)) q = url; |
336 | if (empty(q)) q = agiBlue_urlToQuery(url); |
337 | |
338 | Page page, bool newPage = unpair uniqCI2_sync(cc, Page, +q); |
339 | if (newPage) dbLog("New page", +q); |
340 | //printStructs(+params, +raw, +q); |
341 | |
342 | Set<S> get = asCISet(nempties(subBot_paramsAsMultiMap().get('get))); |
343 | |
344 | S top = nempty(get) ? "" : hcomment("cookie: " + takeFirst(4, session.cookie)) |
345 | + hSilentComputatorWithFlag("agi.blue: " + q) |
346 | + p(ahref(sliceHomeURL(), |
347 | //hsnippetimg(#1101682, width := 565/5, height := 800/5, title := "Robot by Peerro @ DeviantArt") |
348 | //hsnippetimg(#1101778, width := 96, height := 96, title := "agi.blue - a database for everything") |
349 | hsnippetimg(#1101822, width := 314, height := 125, title := "agi.blue - Wikipedia for robots") |
350 | )) |
351 | + p(small( |
352 | b(agiBlueNameHTML()) |
353 | + (agiBlue_isOriginal() ? "" : " " + targetBlank("http://agi.blue", "[original]")) |
354 | + " | " + targetBlank(progLink(), "source code") + " of this web site (" + nLines(sourceCodeLines) + ") | " + targetBlank("https://gitter.im/agi-blue/community", "sponsor https?") + " | by " + targetBlank("https://BotCompany.de", "BC") + " | " + targetBlank("http://fiverr.tinybrain.de/", "Fiverr") + " | " + targetBlank("https://discordapp.com/invite/SEAjPqk", "Discord") + " | " + targetBlank("https://www.youtube.com/watch?v=b6jtRdV3Ev8", "Video") |
355 | + " | " + targetBlank("http://code.botcompany.de/1024233", "Notes") |
356 | + " | " + ahref(agiBlueURL() + "/query", "Query") |
357 | )); |
358 | |
359 | if (empty(params.get('q)) && empty(raw) && isHomeDomain()) { |
360 | L<Page> pages = sortedByFieldDesc _modified(list(cc, Page)); // TODO: use index |
361 | int start = parseInt(params.get("start")), step = 100; |
362 | S nav = pageNav2("/", l(pages), start, step, "start"); |
363 | S content = hform(b("GIVE ME INPUT: ") + htextinput('q, autofocus := true) + " " + hsubmit("Ask")) |
364 | + h1(sliceAsHTML() + " has " + nPages(countConcepts(cc, Page)) + " and " + nConnections(countConcepts(cc, Entry))) |
365 | + p(nav) |
366 | + p_nemptyLines(map(pageToHTMLLink(), subList(pages, start, start+step))); |
367 | |
368 | ret hhtml_agiBlue(hhead_title("Slice " + sliceConcept().name + " | agi.blue") // SERVE SLICE HOME PAGE |
369 | + hbody(hfullcenterAndTopLeft(top + content |
370 | + footer(), |
371 | sliceSelector() |
372 | ))); |
373 | } |
374 | |
375 | S key = trim(params.get('key)), value = trim(params.get('value)); |
376 | L<Entry> entries = GetEntriesAndPost(cc).go(page, params).entries; |
377 | |
378 | //S get = params.get('get); |
379 | if (nempty(get)) |
380 | ret serveJSON(collect value(llNotNulls(firstThat(entries, e -> get.contains(e.key))))); |
381 | |
382 | S key2 = key, value2 = value; if (nempty(key) && nempty(value)) key2 = value2 = ""; // input reset |
383 | |
384 | bool withHidden = eq(params.get('withHidden), "1"); |
385 | new Set<Int> hide; |
386 | if (!withHidden) for (Entry e : entries) if (eqic(e.key, 'hide) && isSquareBracketedInt(e.value)) addAll(hide, e.count, parseInt(unSquareBracket(e.value))); |
387 | new MultiMap<Int, S> mmMeta; |
388 | for (Entry e : entries) if (isSquareBracketedInt(e.key)) mmMeta.put(parseInt(unSquareBracket(e.key)), e.value); |
389 | new MultiMap<S> mm; |
390 | for (Entry e : entries) mm.put(e.key, e.value); |
391 | |
392 | //S name = or2(/* ouch */ last(mm.get("read as")), /* end ouch */ unpackAGIDomain(page.url), page.url); |
393 | S name = page.q; |
394 | |
395 | // Find references |
396 | |
397 | L<Entry> refs = concatLists(conceptsWhereIC(cc, Entry, value := name), |
398 | conceptsWhereIC(cc, Entry, key := name)); |
399 | Set<Page> refPages = asSet(ccollect page(refs)); |
400 | refPages.remove(page); // don't list current page as reference |
401 | |
402 | // Search in page names (depending on default search type) |
403 | |
404 | L<Page> searchResults; |
405 | if (eq(sideSearchType, 'leven)) |
406 | searchResults = levenSearch(page.q, max := 50); |
407 | else if (eq(sideSearchType, 'literal)) |
408 | searchResults = literalSearch(page.q, max := 50); |
409 | else |
410 | searchResults = (L<Page>) dm_call('agiBlueSearch, 'search, page.q, maxResult := searchResultsToShow+1, agiBlueBotID := programID(), +cc); |
411 | searchResults.remove(page); |
412 | |
413 | S mainContents = |
414 | top + h1(ahref_unstyled(agiBlue_pageURLWithSlice(page), htmlEncode2(shorten(displayLength, name))) + (newPage ? " [huh????]" : "")) |
415 | + p_nemptyLines(map(entries, func(Entry e) -> S { |
416 | !withHidden && (hide.contains(e.count) || eqic(e.key, "read as") && eqic(e.value, name)) ? "" |
417 | : "[" + e.count + "] " + |
418 | renderThing(e.key, false) + ": " + |
419 | b(renderThing(e.value, cic(mmMeta.get(e.count), "is a URL"))) |
420 | })) |
421 | + hpostform(h3("Add an entry") |
422 | + "Key: " + hinputfield(key := key2) + " Value: " + hinputfield(value := value2) + "<br><br>" + hsubmit("Add") |
423 | ) |
424 | |
425 | + p(ahref(agiBlueURL() + "/literalSearch" + hquery(q := page.q), "[literal search]", title := "Search pages with a name containing this page's name literally") |
426 | + " " + |
427 | ahref(agiBlueURL() + "/levenSearch" + hquery(q := page.q), "[leven search 1]", title := "Search pages with a Levenshtein similarity of 1 containing this page's name literally") |
428 | + " " + |
429 | ahref(agiBlueURL() + "/search" + hquery(q := page.q), "[scored search]", title := "Search pages with ScoredSearch")); |
430 | |
431 | S sideContents = |
432 | hform(b("GIVE ME INPUT:") + " " |
433 | + htextinput('q) + " " |
434 | + hsubmit("Ask", onclick := "document.getElementById('newInputForm').target = '';") + " " |
435 | + hsubmit("+Tab", title := "Ask and show result in a new tab", onclick := "document.getElementById('newInputForm').target = '_blank';"), |
436 | id := 'newInputForm) |
437 | |
438 | + h3("References (" + l(refPages) + ")") |
439 | |
440 | + p_nemptyLines_showFirst(10, map(pageToHTMLLink(displayLength := sideDisplayLength), refPages)) |
441 | |
442 | + h3(searchTypeToText.get(sideSearchType) + " search results (" + (l(searchResults) >= searchResultsToShow ? searchResultsToShow + "+" : str(l(searchResults))) + ")") |
443 | |
444 | + p_nemptyLines_showFirst(searchResultsToShow, map(pageToHTMLLink(displayLength := sideDisplayLength), searchResults)) |
445 | |
446 | + hdiv("", id := 'extraStuff); |
447 | |
448 | // TODO: sync search delivery with WebSocket creation |
449 | if (asyncSearch) |
450 | doLater(6.0, r { dm_call('agiBlueSearch, 'searchAndPost, page.q, agiBlueBotID := programID(), +cc) }); |
451 | |
452 | // serve a concept page |
453 | ret hhtml_agiBlue(hhead_title(pageDisplayName(page)) + hbody( |
454 | tag('table, |
455 | tr( |
456 | td(sliceSelector(), valign := 'top) |
457 | + td(sideContents, align := 'right, valign := 'top, rowspan := 2)) |
458 | + tr(td(mainContents + footer(), align := 'center, valign := 'top)), |
459 | width := "100%", height := "100%"))); |
460 | } |
461 | |
462 | O servePagesToBot(Iterable<Page> pages) { |
463 | ret serveListToBot(map(pageToMap(wrapMapAsParams(params)), pages)); |
464 | } |
465 | |
466 | O serveListToBot(Collection l) { |
467 | if (nempty(params.get('max))) |
468 | l = takeFirst(parseInt(params.get('max)), l); |
469 | ret serveJSON(l); |
470 | } |
471 | |
472 | // uri starts with "/bot/" |
473 | O serveBot() { |
474 | S q = params.get('q); |
475 | |
476 | if (eqic(uri, "/bot/hello")) ret serveJSON("hello"); |
477 | if (eqic(uri, "/bot/hasPage")) ret serveJSON(hasPage(q)); |
478 | if (eqic(uri, "/bot/randomPageContaining")) { |
479 | assertNempty(q); |
480 | ret servePageToBot(random(filter(list(cc, Page), p -> cic(p.q, q))), params); |
481 | } |
482 | if (eqic(uri, "/bot/allPages")) |
483 | ret servePagesToBot(list(cc, Page)); |
484 | if (eqic(uri, "/bot/allPagesStartingWith")) { |
485 | assertNempty(q); |
486 | ret servePagesToBot(filter(list(cc, Page), p -> swic(p.q, q))); |
487 | } |
488 | if (eqic(uri, "/bot/allPagesEndingWith")) { |
489 | assertNempty(q); |
490 | ret servePagesToBot(filter(list(cc, Page), p -> ewic(p.q, q))); |
491 | } |
492 | if (eqic(uri, "/bot/allPagesContaining")) { |
493 | assertNempty(q); |
494 | ret servePagesToBot(filter(list(cc, Page), p -> cic(p.q, q))); |
495 | } |
496 | if (eqic(uri, "/bot/allPagesContainingRegexp")) { |
497 | assertNempty(q); |
498 | Pattern pat = regexpIC(q); |
499 | ret servePagesToBot(filter(list(cc, Page), p -> regexpFindIC(pat, p.q))); |
500 | } |
501 | |
502 | if (eqicOneOf(uri, "/bot/postSigned", /*"/bot/makePhysicalSlice",*/ "/bot/approveTrustRequest")) { |
503 | S text = rtrim(params.get('text)); |
504 | S key = getSignerKey(text); |
505 | if (empty(key)) ret subBot_serve500("Please include your public key"); |
506 | if (!isSignedWithKey(text, key)) ret subBot_serve500("Signature didn't verify"); |
507 | text = dropLastTwoLines(text); // drop signer + sig line |
508 | |
509 | Signer signer = uniq_sync Signer(publicKey := key); |
510 | |
511 | /*if (eqic(uri, "/bot/makePhysicalSlice")) { |
512 | if (!signer.trusted) ret subBot_serve500("Untrusted signer"); |
513 | Page page = findPageFromParams(jsonDecodeMap(text)); |
514 | if (page == null) ret subBot_serve500("Page not found"); |
515 | ret serveJSON(uniq2_sync(PhysicalSlice, slicePage := page).b ? "Slice made" : "Slice exists"); |
516 | }*/ |
517 | |
518 | if (eqic(uri, "/bot/postSigned")) { |
519 | new L out; |
520 | for (S line : tlft(text)) { |
521 | SS map = jsonDecodeMap(line); |
522 | GetEntriesAndPost x = new(db_mainConcepts()); |
523 | x.signer = signer; |
524 | Page page = findOrMakePageFromParams(map); |
525 | if (page == null) continue with out.add("Invalid page reference"); |
526 | x.go(page, map); |
527 | out.add(x.newEntry ? "Saved" : x.entry != null ? "Entry exists" : "Need key and value"); |
528 | } |
529 | ret serveJSON(out); |
530 | } |
531 | |
532 | if (eqic(uri, "/bot/approveTrustRequest")) { |
533 | if (!signer.trusted) ret subBot_serve500("Untrusted signer"); |
534 | Signer toApprove = conceptWhere Signer(publicKey := trim(text)); |
535 | if (toApprove == null) ret subBot_serve500("Signer to approve not found"); |
536 | cset(toApprove, trusted := true, approvedBy := signer.globalID); |
537 | ret serveJSON("Approved: " + trim(text)); |
538 | } |
539 | |
540 | ret subBot_serve500("CONFUSION"); |
541 | } |
542 | |
543 | if (eqic(uri, "/bot/post")) { |
544 | GetEntriesAndPost x = new(db_mainConcepts()); |
545 | x.go(pageFromQ(q), params); |
546 | ret serveJSON(x.newEntry ? "Saved" : x.entry != null ? "Entry exists" : "Need key and value"); |
547 | } |
548 | |
549 | if (eqic(uri, "/bot/entriesOnPage")) |
550 | ret serveJSON(map(entriesOnPage(findPageFromParams(params)), entryToMap(false))); |
551 | |
552 | if (eqic(uri, "/bot/lookup")) { |
553 | S key = params.get('key); |
554 | if (empty(key)) ret serveJSON("Need key"); |
555 | S value = getValue(findPageFromParams(params), key); |
556 | ret serveJSON(empty(value) ? "" : litmap(+value)); |
557 | } |
558 | |
559 | if (eqic(uri, "/bot/latestEntries")) |
560 | ret serveJSON(map(takeFirst(10, slice.idx_latestEntries.objectIterator()), entryToMap(true))); |
561 | if (eqic(uri, "/bot/latestPages")) |
562 | ret serveJSON(map(takeFirst(10, slice.idx_latestCreatedPages.objectIterator()), pageToMap())); |
563 | if (eqic(uri, "/bot/latestChangedPages")) |
564 | ret serveJSON(map(takeFirst(10, slice.idx_latestChangedPages.objectIterator()), pageToMap())); |
565 | |
566 | if (eqic(uri, "/bot/googleUsersCount")) |
567 | ret serveJSON(l(setMinus(collectAsCISet googleEmail(list(Session)), "", null))); |
568 | |
569 | if (eqic(uri, "/bot/totalPageCount")) |
570 | ret serveJSON(countConcepts(Page)); |
571 | /*if (eqic(uri, "/bot/pageWithoutPhysicalSliceCount")) |
572 | ret serveJSON(countConceptsWhere(Page, slice := null)); |
573 | if (eqic(uri, "/bot/physicalSliceCount")) |
574 | ret serveJSON(countConcepts(PhysicalSlice));*/ |
575 | if (eqic(uri, "/bot/trustedSignersCount")) |
576 | ret serveJSON(countConcepts(Signer, trusted := true)); |
577 | |
578 | if (eqic(uri, "/bot/valueSearch")) { |
579 | S value = params.get('value); |
580 | L<Entry> entries = conceptsWhereIC(cc, Entry, +value); |
581 | ret serveJSON(map(takeFirst(100, entries), entryToMap(true))); |
582 | } |
583 | |
584 | if (eqic(uri, "/bot/keyAndValueSearch")) { |
585 | S key = params.get('key), value = params.get('value); |
586 | Cl<Page> pages = pagesForKeyAndValue(key, value); |
587 | ret servePagesToBot(pages); |
588 | } |
589 | |
590 | if (eqic(uri, "/bot/keyValuePairsByPopularity")) { |
591 | L<PairS> pairs = map(list(Entry), e -> pair(e.key, e.value)); |
592 | LPair<S, Int> pairs2 = multiSetTopPairs(ciMultiSet(map pairToUglyStringForCIComparison(pairs))); |
593 | ret serveJSON(map(pairs2, p -> { |
594 | S key, value = unpair pairFromUglyString(p.a); |
595 | ret litorderedmap(n := p.b, +key, +value); |
596 | })); |
597 | } |
598 | |
599 | if (eqic(uri, "/bot/allKeys")) |
600 | ret serveListToBot(distinctCIFieldValuesOfConcepts(cc, Entry, 'key)); |
601 | |
602 | if (eqic(uri, "/bot/allKeysByPopularity")) |
603 | ret serveListToBot(mapMultiSetByPopularity(distinctCIFieldValuesOfConcepts_multiSet(cc, Entry, 'key), (key, n) -> litorderedmap(+n, +key))); |
604 | |
605 | if (eqic(uri, "/bot/dbSize")) |
606 | ret serveJSON(l(conceptsFile())); |
607 | |
608 | if (eqic(uri, "/bot/query")) |
609 | ret serveBotQuery(); |
610 | |
611 | if (eqic(uri, "/bot/createSlice")) { |
612 | Slice slice = createSlice(assertNempty(params.get('name))); |
613 | ret serveJSON(litorderedmap(id := str(slice.globalID), name := slice.name)); |
614 | } |
615 | |
616 | if (eqic(uri, "/bot/dumpAllSlices") && authed()) { |
617 | new L<Map> out; |
618 | for (Slice slice) try { |
619 | out.add(litorderedmap(slice := slice.caseID, ms := returnTimeFor(() -> dumpSliceToFile(loadBackgroundSlice(slice))))); |
620 | } catch print e { |
621 | out.add(litorderedmap(slice.caseID, error := exceptionToStringShort(e))); |
622 | } |
623 | ret serveJSON(out); |
624 | } |
625 | |
626 | if (eqic(uri, "/bot/dumpSlice")) { |
627 | File f = dumpSliceToFile(slice); |
628 | ret "OK, saved as: " + f2s(f); |
629 | } |
630 | |
631 | if (eqic(uri, "/bot/memStats")) { |
632 | Map fg = cloneMap(loadedSlices); |
633 | Map bg = cloneMap(backgroundSlices); |
634 | ret serveJSON(litcimap( |
635 | error := checkDoubleLoads(), |
636 | numLoadedForegroundSlices := l(fg), |
637 | numLoadedBackgroundSlices := l(bg), |
638 | loadedForegroundSlicesEstimatedSize := longSum(map estimatedSliceDataSize(keys(fg))), |
639 | loadedBackgroundSlicesEstimatedSize := longSum(map estimatedSliceDataSize(keys(bg))), |
640 | loadedForegroundSlices := sortedIC(keysList(fg)), |
641 | loadedBackgroundSlices := sortedIC(keysList(bg)))); |
642 | } |
643 | |
644 | // end of bot methods |
645 | |
646 | ret subBot_serve404(); |
647 | } |
648 | |
649 | O serveBotQuery() { |
650 | S query = params.get('query); |
651 | ret new Query(slice).process(query); |
652 | } |
653 | |
654 | O serveLiteralSearch() { |
655 | S q = params.get('q); |
656 | L<Page> searchResults = literalSearch(q); |
657 | ret serveSearchResults("literal search" , q, searchResultsToShow, searchResults); |
658 | } |
659 | |
660 | L<Page> literalSearch(S q, O... _) { |
661 | int searchResultsToShow = optPar max(_, 100); |
662 | |
663 | // quick search in random order |
664 | //L<Page> searchResults = takeFirst(searchResultsToShow+1, filterIterator(iterator(list(cc, Page)), p -> cic(p.q, q)); |
665 | |
666 | // full search, order by length |
667 | ret takeFirst(searchResultsToShow+1, pagesSortedByLength(filter(list(cc, Page), p -> cic(p.q, q)))); |
668 | } |
669 | |
670 | O serveScoredSearch() { |
671 | S q = params.get('q); |
672 | L<Page> searchResults = (L<Page>) dm_call('agiBlueSearch, 'search, q, +cc); |
673 | ret serveSearchResults("scored search" , q, searchResultsToShow, searchResults); |
674 | } |
675 | |
676 | O serveLevenSearch() { |
677 | S q = params.get('q); |
678 | L<Page> searchResults = levenSearch(q); |
679 | ret serveSearchResults("leven search with distance 1" , q, searchResultsToShow, searchResults); |
680 | } |
681 | |
682 | L<Page> levenSearch(S q, O... _) { |
683 | int searchResultsToShow = optPar max(_, 100); |
684 | int maxEditDistance = 1; |
685 | |
686 | new Map<Page, Int> map; |
687 | for (Page p : list(cc, Page)) { |
688 | int distance = leven_limitedIC(q, p.q, maxEditDistance+1); |
689 | if (distance <= maxEditDistance) map.put(p, distance); |
690 | } |
691 | ret takeFirst(searchResultsToShow+1, keysSortedByValue(map)); |
692 | } |
693 | |
694 | O serveSearchResults(S searchType, S q, int searchResultsToShow, Collection<Page> searchResults) { |
695 | S title = "agi.blue " + searchType + " for " + htmlEncode2(quote(q)) + " (" + (l(searchResults) >= searchResultsToShow ? searchResultsToShow + "+ results" : nResults(l(searchResults))) + ")"; |
696 | |
697 | ret hhtml_agiBlue(hhead_title(htmldecode_dropAllTags(title)) |
698 | + hbody(hfullcenter(//top + |
699 | h3(title) |
700 | + p_nemptyLines_showFirst(searchResultsToShow, map(pageToHTMLLink(), searchResults))))); |
701 | } |
702 | |
703 | O serveQueryPage() { |
704 | S query = params.get('query); |
705 | if (query == null) query = loadSnippet(#1024258); |
706 | S title = agiBlueNameHTML() + " | Execute a query script (" + targetBlank("http://code.botcompany.de/1024274", "ALQL") + ")"; |
707 | ret hhtml_agiBlue(hhead_title(htmldecode_dropAllTags(title)) |
708 | + hbody(hfullcenter( |
709 | h3(title) |
710 | + form( |
711 | htextarea(query, name := 'query, cols := 80, rows := 10, autofocus := true) |
712 | + "<br><br>" |
713 | + hsubmit("Execute"), action := "/bot/query") |
714 | ))); |
715 | } |
716 | |
717 | S sliceHomeURL() { |
718 | ret slice == null ? agiBlueURL() : sliceHomeURL(slice.sliceConcept.globalID); |
719 | } |
720 | |
721 | S sliceHomeURL(GlobalID slice) { |
722 | ret agiBlueURL() + hquery(+slice); |
723 | } |
724 | |
725 | O serveCreateSlicePage() { |
726 | S sliceName = trim(params.get('sliceName)); |
727 | if (eq(params.get('doIt), "1") && nempty(sliceName)) { |
728 | // TODO: check for existing name |
729 | Slice slice = createSlice(sliceName); |
730 | ret hrefresh(sliceHomeURL(slice.globalID)); |
731 | } |
732 | |
733 | S title = agiBlueNameHTML() + " | Create slice"; |
734 | ret hhtml_agiBlue(hhead_title(htmldecode_dropAllTags(title)) |
735 | + hbody(hfullcenter( |
736 | h3(title) |
737 | + form( |
738 | hhidden(doIt := 1) |
739 | + "Slice name: " + htextinput(+sliceName, autofocus := true) |
740 | + "<br><br>" |
741 | + hsubmit("Create slice")) |
742 | ))); |
743 | } |
744 | |
745 | F1<Page, S> pageToHTMLLink(O... _) { |
746 | optPar int displayLength = main.displayLength; |
747 | ret func(Page p) -> S { |
748 | S name = pageDisplayName(p); |
749 | ret ahref(agiBlue_pageURLWithSlice(p), |
750 | htmlEncode2(shorten(displayLength, name)), |
751 | title := name); |
752 | }; |
753 | } |
754 | |
755 | // and google log-in |
756 | S sliceSelector() { |
757 | ret htag table(tr( |
758 | (!showGoogleLogIn ? "" : td( |
759 | googleSignIn_signInButton(agiBlueURL() + "/google-verify", "console.log(data);", ""), style := "padding-right: 10px;") |
760 | + td(nobr(ahref("javascript:signOut()", "Sign out")), style := "padding-right: 10px; padding-bottom: 0.15em")) |
761 | |
762 | + (!showSliceSelector ? "" : hform(td( |
763 | "Select reality slice: " |
764 | + hselect(availableSlices(), session.selectedSlice, name := 'slice, onchange := "this.form.submit()") |
765 | /*+ " " + hsubmit("Go")*/ |
766 | + " " + ahref(agiBlueURL() + "/createSlice", "Create slice...") |
767 | ))))); |
768 | } |
769 | |
770 | S sliceAsHTML() { |
771 | if (slice.isMainSlice()) ret htmlEncode2(agiBlueName() + "'s main slice"); |
772 | if (slice.sliceConcept == null) ret "Slice ???"; |
773 | ret htmlEncode2("Slice " + quote(slice.sliceConcept.name)); |
774 | } |
775 | |
776 | Slice sliceConcept() { ret slice.sliceConcept; } |
777 | |
778 | S footer() { |
779 | ret p(small(elapsedMS_sysNow(started) + " ms"); |
780 | } |
781 | |
782 | S verifiedEmail() { |
783 | ret session == null || !session.googleEmailVerified ? null |
784 | : session.googleEmail; |
785 | } |
786 | |
787 | bool authed() { |
788 | bool ver = eqic(verifiedEmail(), "stefan.reich.maker.of.eye@googlemail.com"); |
789 | print("Auth check " + verifiedEmail() + " -> " + ver); |
790 | ret ver; |
791 | } |
792 | |
793 | S googleSignInID() { |
794 | ret eqic(domain(), "botcompany.de") ? botCompanyGoogleSignInID() : agiBlueGoogleSignInID(); |
795 | } |
796 | |
797 | S hhtml_agiBlue(S contents) { |
798 | ret hhtml(hAddToHead_fast(contents, |
799 | hIncludeGoogleFont("Source Sans Pro") |
800 | + [[<meta name="google-site-verification" content="oRvyeTqgSuv-FE_c-2UKM1Vp0oDqx8h9WruHYWqA-NQ" />]] |
801 | + loadJQuery() |
802 | + hmobilefix() |
803 | + googleSignIn_header("", googleSignInID()) |
804 | + hstylesheet("body { font-family: Source Sans Pro }"))); |
805 | } |
806 | |
807 | } // end of Request |
808 | |
809 | svoid dbLog(O... params) { |
810 | logStructure(programFile("db.log"), ll(params)); |
811 | } |
812 | |
813 | static IF1<Page, Map> pageToMap(O... _) { |
814 | optPar bool withEntries; |
815 | |
816 | bool nameOnly = eqOneOf(optPar nameOnly(_), "1", true); |
817 | if (nameOnly) ret (IF1<Page, Map>) p -> litmap(q := p.q); |
818 | |
819 | ret (IF1<Page, Map>) p -> { |
820 | L<Entry> entries = findBackRefs(p, Entry); |
821 | ret litorderedmap( |
822 | q := p.q, |
823 | nEntries := l(entries), |
824 | created := p.created, |
825 | modified := p._modified, |
826 | entries := !withEntries ? null : map(entries, entryToMap(false))); |
827 | }; |
828 | } |
829 | |
830 | static IF1<Entry, Map> entryToMap(bool withPage) { |
831 | ret (IF1<Entry, Map>) e -> litorderedmap( |
832 | created := e.created, |
833 | i := e.count, |
834 | key := e.key, |
835 | value := e.value, |
836 | q := withPage ? e.page->q : null, |
837 | signer := getString globalID(e.signer!)); |
838 | } |
839 | |
840 | sO servePageToBot(Page page, SS params) { |
841 | if (page == null) ret serveJSON(null); |
842 | params = asCIMap(params); |
843 | Map map = pageToMap(withEntries := valueIs1 withEntries(params)).get(page); |
844 | ret serveJSON(map); |
845 | } |
846 | |
847 | sclass GetEntriesAndPost { |
848 | Concepts cc; |
849 | L<Entry> entries; |
850 | Entry entry; |
851 | bool newEntry; |
852 | Signer signer; |
853 | |
854 | *(Concepts *cc) {} |
855 | |
856 | GetEntriesAndPost go(Page page, SS params) { |
857 | S key = trim(params.get('key)), value = trim(params.get('value)); |
858 | print("GetEntriesAndPost: " + quote(page.q) + ", " + quote(key) + " := " + quote(value)); |
859 | withDBLock { |
860 | entries = findBackRefs(page, Entry); |
861 | if (nempty(key) && nempty(value)) { |
862 | S ip = subBot_clientIP(); |
863 | entry = firstThat(e -> eqic(e.key, key) && eq_icIf(!allowMultipleCasesInValues, e.value, value) && eq(e.ip, ip), entries); |
864 | if (entry == null) { |
865 | print("SAVING"); |
866 | Entry e = cnew(cc, Entry, +page, +key, +value, +ip, count := l(entries) + 1, +signer); |
867 | if (makeAllValuesPages) pageFromQ(cc, value); |
868 | page.change(); // bump modification date |
869 | entry = e; |
870 | newEntry = true; |
871 | entries.add(entry); |
872 | dbLog("New entry", page := page.q, globalID := e.globalID, count := e.count, +key, +value); |
873 | } |
874 | } |
875 | } |
876 | |
877 | sortByFieldInPlace created(entries); |
878 | numberEntriesInConceptField count(entries); |
879 | this; |
880 | } |
881 | } |
882 | |
883 | static SS availableSlices() { |
884 | ret mapToOrderedMap(s -> pair(str(s.globalID), s.name + " [ID: " + s.globalID + "]"), list(Slice)); |
885 | } |
886 | |
887 | static L<Entry> entriesOnPage(Page p) { |
888 | ret p == null ? null : sortedByField count(findBackRefs(p, Entry)); |
889 | } |
890 | |
891 | static L<Page> pagesSortedByLength(L<Page> l) { |
892 | ret sortedByCalculatedField(l, p -> l(p.q)); |
893 | } |
894 | |
895 | sbool agiBlue_isOriginal() { |
896 | ret amProgram(#1023558); |
897 | } |
898 | |
899 | sS agiBlueURL() { |
900 | ret agiBlue_isOriginal() ? "https://agi.blue" : "/" + psI(programID()) + "/raw"; |
901 | } |
902 | |
903 | sS agiBlueName() { |
904 | ret agiBlue_isOriginal() ? "agi.blue" : "agi.blue clone " + programID(); |
905 | } |
906 | |
907 | svoid cleanMeUp { |
908 | cleanUp(fan); |
909 | } |
910 | |
911 | static LoadedSlice loadSlice(Slice slice) { |
912 | ret loadSlice(slice.caseID, str(slice.globalID)); |
913 | } |
914 | |
915 | static LoadedSlice loadSlice(S caseID, S globalID) { |
916 | caseID = unnull(caseID); |
917 | lock fan.lock; |
918 | |
919 | // move slice from background to foreground |
920 | |
921 | Concepts cc = backgroundFan.getIfLoaded(caseID); |
922 | if (cc != null) { |
923 | LoadedSlice slice = assertNotNull(backgroundSlices.get(caseID)); |
924 | backgroundFan.loaded.remove(caseID); |
925 | fan.loaded.put(caseID, cc); |
926 | backgroundSlices.remove(caseID); |
927 | loadedSlices.put(caseID, slice); |
928 | ret slice; |
929 | } |
930 | |
931 | fan.get(caseID, globalID); |
932 | ret loadedSlices.get(caseID); |
933 | } |
934 | |
935 | // load background slice |
936 | static LoadedSlice loadBackgroundSlice(Slice slice) { |
937 | ret loadBackgroundSlice(slice.caseID, str(slice.globalID)); |
938 | } |
939 | |
940 | static LoadedSlice loadBackgroundSlice(S caseID, S globalID) { |
941 | caseID = unnull(caseID); |
942 | lock fan.lock; |
943 | |
944 | // check if in foreground |
945 | |
946 | if (fan.getIfLoaded(caseID) != null) |
947 | ret loadedSlices.get(caseID); |
948 | |
949 | backgroundFan.get(caseID, globalID); |
950 | LoadedSlice slice = assertNotNull(backgroundSlices.get(caseID)); |
951 | slice.lastAccess = sysNow(); |
952 | while (l(backgroundSlices) > maxBackgroundSlices) |
953 | unloadLeastRecentlyAccessedBackgroundSlice(); |
954 | ret slice; |
955 | } |
956 | |
957 | svoid unloadLeastRecentlyAccessedBackgroundSlice() { |
958 | lock fan.lock; |
959 | S caseID = lowestByField lastAccess(keys(backgroundSlices)); |
960 | if (caseID != null) |
961 | backgroundFan.unloadCase(caseID); |
962 | } |
963 | |
964 | static Slice sliceConceptForGlobalID(S globalID) { |
965 | ret sliceConceptForGlobalID(toGlobalIDObj(globalID)); |
966 | } |
967 | |
968 | static Slice sliceConceptForGlobalID(GlobalID globalID) { |
969 | ret conceptWhere Slice(+globalID); |
970 | } |
971 | |
972 | static Slice sliceConceptForCaseID(S caseID) { |
973 | ret conceptWhere Slice(+caseID); |
974 | } |
975 | |
976 | sS agiBlueNameHTML() { |
977 | ret ahref(agiBlueURL(), htmlEncode2(agiBlueName())); |
978 | } |
979 | |
980 | static Slice createSlice(S name) { |
981 | Slice slice = cnew Slice(+name); |
982 | cset(slice, caseID := slice.defaultCaseID()); |
983 | loadSlice(slice).initialSetup(slice.globalID); |
984 | ret slice; |
985 | } |
986 | |
987 | static Slice sliceForName(S name) { |
988 | ret conceptWhereCI Slice(+name); |
989 | } |
990 | |
991 | sclass Query extends WorksOnSlice { |
992 | SS vars = ciMap(); |
993 | |
994 | *(LoadedSlice slice) { super(slice); } |
995 | |
996 | O process(S query) { |
997 | try object processLines(agiBlue_parseQueryScript(query)); |
998 | ret serveJSON("No return statement"); |
999 | } |
1000 | |
1001 | O processLines(L<ALQLLine> lines) { |
1002 | for (ALQLLine line : lines) { |
1003 | if (line cast ALQLSlice) { |
1004 | Slice slice = sliceForName(line.slice); |
1005 | if (slice == null) ret serveJSON("Slice not found: " + line.slice); |
1006 | LoadedSlice oldSlice = Query.this.slice; |
1007 | try { |
1008 | setSlice(loadSlice(slice)); |
1009 | try object processLines(line.contents); |
1010 | } finally { |
1011 | setSlice(oldSlice); |
1012 | } |
1013 | } else if (line cast ALQLReturn) |
1014 | ret serveJSON(ll(getOrKeep(vars, line.var))); |
1015 | else if (line cast ALQLTriple) { |
1016 | T3S t = line.triple; |
1017 | t = tripleMap(t, s -> getOrKeep(vars, s)); |
1018 | Cl<Page> pages; |
1019 | S var; |
1020 | if (isDollarVar(t.c)) { |
1021 | var = t.c; |
1022 | if (isDollarVar(t.a)) { |
1023 | if (isDollarVar(t.b)) todo(t); |
1024 | Entry e = random(conceptsWhereCI(cc, Entry, key := t.b)); |
1025 | if (e == null) ret serveJSON("No results for " + var); |
1026 | pages = pagesForKeyAndValue(t.b, t.c); |
1027 | vars.put(t.a, e.page->q); |
1028 | vars.put(t.c, e.value); |
1029 | continue; |
1030 | } else if (isDollarVar(t.b)) { |
1031 | Page page = findPageFromQ(t.a); |
1032 | Entry e = random(findBackRefs(page, Entry)); |
1033 | if (e == null) ret serveJSON("No results for " + var); |
1034 | vars.put(t.b, e.key); |
1035 | vars.put(t.c, e.value); |
1036 | continue; |
1037 | } else { |
1038 | S val = getValue(t.a, t.b); |
1039 | if (val == null) ret serveJSON("No results for " + var); |
1040 | pages = ll(pageFromQ(val)); |
1041 | } |
1042 | } else if (isDollarVar(t.b)) { |
1043 | var = t.b; |
1044 | if (isDollarVar(t.c)) todo(t); |
1045 | if (isDollarVar(t.a)) { |
1046 | L<Entry> entries = conceptsWhereCI(cc, Entry, value := t.c); |
1047 | if (empty(entries)) ret serveJSON("No results for " + var); |
1048 | Entry e = random(entries); |
1049 | vars.put(t.a, e.page->q); |
1050 | vars.put(t.b, e.key); |
1051 | continue; |
1052 | } else { |
1053 | Cl<S> keys = keysForPageAndValue(t.a, t.c); |
1054 | if (empty(keys)) ret serveJSON("No results for " + var); |
1055 | pages = map(f<S, Page> pageFromQ, keys); |
1056 | } |
1057 | } else { |
1058 | var = t.a; |
1059 | if (!isDollarVar(t.a)) todo(t); |
1060 | if (isDollarVar(t.b)) todo(t); |
1061 | if (isDollarVar(t.c)) todo(t); |
1062 | pages = pagesForKeyAndValue(t.b, t.c); |
1063 | } |
1064 | if (empty(pages)) ret serveJSON("No results for " + var); |
1065 | vars.put(var, random(pages).q); |
1066 | } else if (line cast ALQLPage) { |
1067 | if (eq(line.matchMethod, 'eqic)) |
1068 | findPageFromQ(line.page); |
1069 | else if (eq(line.matchMethod, 'flexMatchDollarVarsIC_first)) { |
1070 | new L<SS> l; |
1071 | for (Page p : list(cc, Page)) |
1072 | addIfNotNull(l, flexMatchDollarVarsIC_first(line.page, p.q)); |
1073 | if (empty(l)) ret serveJSON("No results for " + curly(line.page)); |
1074 | vars.putAll(random(l)); |
1075 | } else |
1076 | fail("Unknown match method: " + line.matchMethod); |
1077 | } else |
1078 | fail("Can't interpret: " + line); |
1079 | } |
1080 | |
1081 | null; |
1082 | } |
1083 | } |
1084 | |
1085 | static Page pageFromQ(Concepts cc, S q) { |
1086 | ret empty(q) ? null : uniqCI_sync(cc, Page, +q); |
1087 | } |
1088 | |
1089 | /*sS globalIDFromCaseID(S caseID) { |
1090 | ret assertGlobalID(takeLast(globalIDLength(), caseID)); |
1091 | }*/ |
1092 | |
1093 | sS agiBlue_pageURLWithSlice(Page p) { |
1094 | ret p == null ? null : agiBlueURL() + hquery(slice := _get(sliceForPage(p), 'globalID), q := p.q); |
1095 | } |
1096 | |
1097 | static Slice sliceForPage(Page p) { |
1098 | if (p == null) null; |
1099 | SliceInfo info = conceptWhere(p._concepts, SliceInfo); |
1100 | ret info == null ? null : sliceConceptForGlobalID(info.globalID); |
1101 | } |
1102 | |
1103 | static GlobalID mainSliceGlobalID() { |
1104 | ret conceptWhere Slice(caseID := "").globalID; |
1105 | } |
1106 | |
1107 | static File dumpSliceToFile(LoadedSlice slice) { |
1108 | ret withDBLock(slice.cc, func -> File { |
1109 | File file = programFile("slice-dumps/" + slice.caseID + ".slice"); |
1110 | new LS lines; |
1111 | for (Page p : list(slice.cc, Page)) |
1112 | lines.add(quote(p.q)); |
1113 | for (Entry e : list(slice.cc, Entry)) |
1114 | lines.add(sfu(tripleToList(entryToTriple(e)))); |
1115 | saveLinesAsTextFile(file, lines); |
1116 | ret file; |
1117 | }); |
1118 | } |
1119 | |
1120 | static T3S entryToTriple(Entry e) { |
1121 | ret e == null ? null : t3(e.page->q, e.key, e.value); |
1122 | } |
1123 | |
1124 | static ConceptsLoadedOnDemand slicesLoadedOnDemand(Map<S, LoadedSlice> loadedSlices) { |
1125 | new ConceptsLoadedOnDemand fan; |
1126 | fan.onCaseLoaded(voidfunc(S caseID, O globalID, Concepts concepts) { |
1127 | LoadedSlice ls = new(caseID, concepts); |
1128 | if (nempty(globalID)) |
1129 | ls.initialSetup(toGlobalIDObj((S) globalID)); |
1130 | loadedSlices.put(caseID, ls); |
1131 | }); |
1132 | fan.onUnloadingCase(voidfunc(S caseID, Concepts concepts) { |
1133 | loadedSlices.remove(caseID); |
1134 | }); |
1135 | ret fan; |
1136 | } |
1137 | |
1138 | // check if slices are loaded both in foreground and background (BAD) |
1139 | sS checkDoubleLoads() { |
1140 | LS l = sharedKeys(loadedSlices, backgroundSlices); |
1141 | ret empty(l) ? null : "ERROR: Slice loaded in background and foreground: " + first(l); |
1142 | } |
1143 | |
1144 | // just use size of concepts.structure.gz |
1145 | static long estimatedSliceDataSize(Slice slice) { |
1146 | ret estimatedSliceDataSize(slice.caseID); |
1147 | } |
1148 | |
1149 | static long estimatedSliceDataSize(S caseID) { |
1150 | ret l(conceptsFile(fan.dbID(caseID))); |
1151 | } |
Began life as a copy of #1023558
download show line numbers debug dex old transpilations
Travelled to 6 computer(s): bhatertpkbcr, mqqgnosmbjvj, pyentgdyhuwx, pzhvpgtvlbxg, tvejysmllsmz, vouqrxazstgt
No comments. add comment
Snippet ID: | #1024334 |
Snippet name: | agi.blue source [with background-loading slices, merged into main] |
Eternal ID of this version: | #1024334/26 |
Text MD5: | d4786b9c0c11d5ef0cf4a8f135d4bc2a |
Transpilation MD5: | ca63e89994bc634c4e814619c48cb480 |
Author: | stefan |
Category: | javax / html |
Type: | JavaX module (desktop) |
Public (visible to everyone): | Yes |
Archived (hidden from active list): | No |
Created/modified: | 2019-08-04 15:24:46 |
Source code size: | 40610 bytes / 1151 lines |
Pitched / IR pitched: | No / No |
Views / Downloads: | 2649 / 3397 |
Version history: | 25 change(s) |
Referenced in: | [show references] |