1 | !7 |
2 | |
3 | // See #1023660 for the older 99 lines version |
4 | |
5 | // Customizations: |
6 | |
7 | sbool asyncSearch = false; |
8 | sbool allowMultipleCasesInValues = true; |
9 | sbool showSliceSelector = true; |
10 | sbool showGoogleLogIn = true; |
11 | sbool makeAllKeysPages = true; |
12 | sbool makeAllValuesPages = true; |
13 | sbool legacy = true; // We still have slices without applied fixes |
14 | sbool dumpSlicesOnAnyChange = true; // testing |
15 | |
16 | static int maxSliceNameLength = 1024; |
17 | static int displayLength = 140; |
18 | static int sideDisplayLength = 50; // when in side bar |
19 | static int searchResultsToShow = 50; |
20 | sS sideSearchType = 'literal; |
21 | static int maxBackgroundSlices = 1000; |
22 | |
23 | // end of customizations |
24 | |
25 | !include once #1024472 // agi.blue core db |
26 | |
27 | // global constants |
28 | |
29 | static SS searchTypeToText = litmap( |
30 | leven := "Leven", |
31 | literal := "Literal", |
32 | scored := "Scored"); |
33 | |
34 | // global vars |
35 | |
36 | static volatile bool started, broken; |
37 | static int sourceCodeLines; |
38 | static TimedCache<Long> sizeOnDisk = new(60.0, f guessClusteredSizeOfProgramDirWithoutBackups); |
39 | |
40 | // end of global data |
41 | |
42 | sclass LoadedSlice { // non-persistent class |
43 | S caseID; |
44 | Concepts cc; |
45 | Slice sliceConcept; |
46 | long lastAccess = sysNow(); |
47 | ReliableSingleThread rstDumpSlice = new(r { |
48 | dumpSliceToFile(LoadedSlice.this); |
49 | rst_index.trigger(); // rebuild index too - TODO: only this slice |
50 | }); |
51 | bool haltSliceDumping; |
52 | |
53 | ConceptFieldIndexDesc idx_latestEntries, idx_latestCreatedPages, idx_latestChangedPages; |
54 | |
55 | *(S *caseID, Concepts *cc) { |
56 | csetAll(list(cc, Page), slice := null, url := null); // clear legacy fields |
57 | releaseEmptyFieldValuesOfAllConcepts(cc); |
58 | indexThings(); |
59 | |
60 | // forward changes to Slice concept |
61 | onConceptsChange(cc, rOnceAtATimeOnly(r { |
62 | cset(sliceConcept, _modified := now()) |
63 | ; |
64 | if (!haltSliceDumping && dumpSlicesOnAnyChange && nempty(caseID)) { |
65 | rstDumpSlice.trigger(); |
66 | } |
67 | })); |
68 | |
69 | if (makeAllKeysPages && legacy) |
70 | for (Entry e : list(cc, Entry)) |
71 | pageFromQ(e.key); |
72 | |
73 | if (makeAllValuesPages && legacy) |
74 | for (Entry e : list(cc, Entry)) |
75 | pageFromQ(e.value); |
76 | } |
77 | |
78 | void initialSetup(GlobalID globalID) { |
79 | SliceInfo info = uniq(cc, SliceInfo); |
80 | cset(info, +globalID); |
81 | } |
82 | |
83 | // create if not there |
84 | Page pageFromQ(S q) { |
85 | ret main.pageFromQ(cc, q); |
86 | } |
87 | |
88 | void indexThings() { |
89 | indexConceptFieldsCI(cc, Page, 'q, Entry, 'key, Entry, 'value); |
90 | indexConceptFields(cc, Signer, 'publicKey); |
91 | indexConceptFields(cc, Session, 'cookie); |
92 | indexSingletonConcept(cc, SliceInfo); |
93 | idx_latestCreatedPages = new ConceptFieldIndexDesc(cc, Page, 'created); |
94 | idx_latestChangedPages = new ConceptFieldIndexDesc(cc, Page, '_modified); |
95 | idx_latestEntries = new ConceptFieldIndexDesc(cc, Entry, 'created); |
96 | } |
97 | |
98 | bool isMainSlice() { ret empty(caseID); } |
99 | GlobalID globalID() { ret sliceConcept.globalID; } |
100 | S name() { ret sliceConcept.name; } |
101 | } |
102 | |
103 | p { |
104 | try { |
105 | if (bootstrapDataFrom(#1023558)) deleteMyBackups(); |
106 | thinMyBackups(); |
107 | fan = slicesLoadedOnDemand(loadedSlices); |
108 | mainConcepts = fan.get("", null); |
109 | |
110 | // index main DB |
111 | |
112 | indexConceptFields(Slice, 'globalID, Slice, 'caseID); |
113 | indexConceptField(CookieUser, 'cookie); |
114 | idx_slicesByName = new ConceptFieldIndexCI(Slice, 'name); |
115 | idx_slicesByModification = new ConceptFieldIndexDesc(Slice, '_modified); |
116 | idx_usersByName = new ConceptFieldIndexCI(GoogleUser, 'googleLastName); |
117 | idx_usersBySeen = new ConceptFieldIndexDesc(User, 'lastSeen); |
118 | |
119 | cset(uniq_returnIfNew Slice(caseID := ""), name := "main slice"); |
120 | loadedSlices.get("").initialSetup(toGlobalIDObj(mainSliceGlobalID())); |
121 | loadedSlices.get("").sliceConcept = sliceConceptForCaseID(""); |
122 | |
123 | backgroundFan = slicesLoadedOnDemand(backgroundSlices); |
124 | backgroundFan.lock = fan.lock; |
125 | |
126 | // legacy conversions! |
127 | //deleteConcepts(Slice, globalID := GlobalID('wftlawbagrwprywn)); |
128 | |
129 | // Approve this machine's key |
130 | PKIKeyPair machineKey = agiBot_trustedKeyForMachine(); |
131 | if (machineKey != null) { |
132 | print("Approving this machine's key: " + machineKey.publicKey); |
133 | cset(uniq_sync(Signer, publicKey := machineKey.publicKey), trusted := true, approvedBy := "local"); |
134 | } |
135 | |
136 | sourceCodeLines = countLines(mySource()); |
137 | |
138 | rst_index.trigger(); |
139 | } catch print e { |
140 | set broken; |
141 | } finally { |
142 | set started; |
143 | } |
144 | |
145 | // do stuff after we are officially started |
146 | |
147 | // calculate DB size regularly iff there were changes |
148 | sizeOnDisk.keepValueWhileCalculating = true; |
149 | doEveryAndNow(60.0, rWatcher(() -> db_mainConceptsChangeCount(), r { sizeOnDisk! })); |
150 | |
151 | } // end of main program |
152 | |
153 | // Serve page |
154 | |
155 | set flag NoNanoHTTPD. html { |
156 | sleepWhile(() -> !started); |
157 | if (broken) ret subBot_serve500("Internal error"); |
158 | ret new Request().serve(uri, params); |
159 | } |
160 | |
161 | sclass WorksOnSlice { |
162 | Concepts cc; |
163 | LoadedSlice slice; |
164 | |
165 | *() {} |
166 | *(LoadedSlice *slice) { cc = slice.cc; } |
167 | |
168 | void setSlice(LoadedSlice slice) { |
169 | this.slice = slice; |
170 | cc = slice.cc; |
171 | } |
172 | |
173 | Page findPageFromParams(Map map) { |
174 | S q = getString q(map); |
175 | ret empty(q) ? null : findPageFromQ(q); |
176 | } |
177 | |
178 | Page findOrMakePageFromParams(Map map) { |
179 | ret pageFromQ(getString q(map)); |
180 | } |
181 | |
182 | Page findPageFromQ(S q) { |
183 | ret conceptWhereCI(cc, Page, +q); |
184 | } |
185 | |
186 | Page pageFromQ(S q) { |
187 | ret slice.pageFromQ(q); |
188 | } |
189 | |
190 | Set<Page> pagesForKeyAndValue(S key, S value) { |
191 | L<Entry> entries = conceptsWhereIC(cc, Entry, +value, +key); |
192 | ret asSet(ccollect page(entries)); |
193 | } |
194 | |
195 | Cl<S> keysForPageAndValue(S q, S value) { |
196 | Page page = findPageFromQ(q); |
197 | if (page == null) null; |
198 | ret collect key(conceptsWhereIC(cc, Entry, +page, +value)); |
199 | } |
200 | |
201 | // DB functions |
202 | |
203 | bool hasPage(S q) { ret hasConceptWhereIC(Page, +q); } |
204 | S getValue(Page page, S key) { |
205 | ret page == null || empty(key) ? null : getString value(highestByField count(objectsWhereIC(findBackRefs(page, Entry), +key))); |
206 | } |
207 | S getValue(S page, S key) { |
208 | ret getValue(findPageFromQ(page), key); |
209 | } |
210 | S pageDisplayName(Page page) { |
211 | /*S name = getValue(page, "read as"); |
212 | bool unnaturalName = nempty(name) && !eq(makeAGIDomain(name), page.url); |
213 | ret unnaturalName ? name + " " + squareBracketed(page.url) : or2(name, unpackAGIDomainOpt(page.url));*/ |
214 | ret page.q; |
215 | } |
216 | |
217 | S renderThing(S s, bool forceURLDisplay) { |
218 | ret forceURLDisplay || isURL(s) || isAGIDomain(s) |
219 | ? ahref(fixAGILink(absoluteURL(s)), htmlencode2(shorten(displayLength, s))) |
220 | : ahref(agiBlue_linkForPhrase(s, |
221 | slice := slice.isMainSlice() ? null : slice.sliceConcept.globalID), |
222 | htmlencode2(shorten(displayLength, s))); |
223 | } |
224 | |
225 | GlobalID sliceID() { ret slice.globalID(); } |
226 | |
227 | } // end of WorksOnSlice |
228 | |
229 | sclass Request extends WorksOnSlice { |
230 | S cookie; |
231 | Session session; |
232 | S uri; |
233 | SS params; |
234 | long started = sysNow(); |
235 | S q, domain; |
236 | Set<S> get; |
237 | |
238 | // when serving a concept page, info that may be grabbed |
239 | // by thought bots |
240 | L<Entry> usesAsKey; |
241 | |
242 | // should also work for standalone |
243 | bool isHomeDomain() { |
244 | S domain = domain(); |
245 | ret eqic(domain, "www.agi.blue") || !ewic(domain, ".agi.blue"); |
246 | } |
247 | |
248 | O serve(S uri, SS params) { |
249 | this.uri = dropMultipleSlashes(uri); |
250 | this.params = params; |
251 | new Matches m; |
252 | |
253 | print(uri + " ? " + unnull(subBot_query())); |
254 | |
255 | // get cookie & session |
256 | |
257 | cookie = cookieFromUser(); |
258 | if (cookie != null) |
259 | { |
260 | session = uniq_sync(Session, +cookie); |
261 | if (session.user! == null) |
262 | cset(session, user := uniq_sync CookieUser(+cookie)); |
263 | } else |
264 | session = unlistedWithValues(Session); |
265 | |
266 | // check if user wants to change slices |
267 | |
268 | S selectSlice = params.get('slice); |
269 | if (isGlobalID(selectSlice)) |
270 | cset(session, selectedSlice := selectSlice); |
271 | if (session.selectedSlice == null) |
272 | cset(session, selectedSlice := str(mainSliceGlobalID())); |
273 | |
274 | // get slice's case ID |
275 | |
276 | S caseID = ""; |
277 | Slice sliceConcept = sliceConceptForGlobalID(session.selectedSlice); |
278 | print("Selected slice: " + session.selectedSlice + ", obj? " + (sliceConcept != null)); |
279 | if (sliceConcept != null) caseID = sliceConcept.caseID; |
280 | print("caseID: " + caseID); |
281 | |
282 | // load slice |
283 | |
284 | slice = assertNotNull(loadSlice(caseID, session.selectedSlice)); |
285 | cc = slice.cc; |
286 | |
287 | // Check for special URIs |
288 | |
289 | if (swic(uri, "/bot/")) ret serveBot(); |
290 | |
291 | if (swic(uri, "/user/", m)) { |
292 | User user = conceptWhere User(globalID := toGlobalIDObj(assertGlobalID(m.rest()))); |
293 | if (user == null) ret subBot_serve404("User not found"); |
294 | S title = "User " + user; |
295 | ret hhtml_miniPage(title, h3(agiBlueNameHTML() + " | " + title) |
296 | + "User type: " + classShortName(user)); |
297 | } |
298 | |
299 | // eleu appends a slash to the URI if it's a single identifier, so we drop it again |
300 | S uri2 = dropTrailingSlash(uri); |
301 | |
302 | if (eqic(uri2, "/google-verify")) { |
303 | print("Google-verify started."); |
304 | Payload payload = printStruct(googleVerifyUserToken2(googleSignInID(), params.get("token"))); |
305 | if (payload == null) ret print("google-verify", "No"); |
306 | S email = payload.getEmail(); |
307 | if (empty(email)) ret print("google-verify", "No"); |
308 | |
309 | GoogleUser user = uniqCI_sync GoogleUser(googleEmail := email); |
310 | cset(user, |
311 | googleEmailVerified := payload.getEmailVerified(), |
312 | googleFirstName := strOrNull(payload.get("given_name")), |
313 | googleLastName := strOrNull(payload.get("family_name"))); |
314 | |
315 | cset(session, |
316 | +user); |
317 | |
318 | ret print("google-verify", payload.getEmail() + " " + (payload.getEmailVerified() ? "(verified)" : "(not verified)")); |
319 | } |
320 | |
321 | cset(session.user!, lastSeen := now()); |
322 | |
323 | if (eqic(uri2, "/users")) ret serveUsersList(); |
324 | |
325 | if (eqic(uri2, "/slices")) ret serveSlicesList(); |
326 | |
327 | if (eqic(uri2, "/search")) ret serveScoredSearch(); |
328 | if (eqic(uri2, "/literalSearch")) ret serveLiteralSearch(); |
329 | if (eqic(uri2, "/levenSearch")) ret serveLevenSearch(); |
330 | |
331 | if (eqic(uri2, "/query")) ret serveQueryPage(); |
332 | if (eqic(uri2, "/createSlice")) ret serveCreateSlicePage(); |
333 | if (eqic(uri2, "/deletePage")) ret serveDeletePage(); |
334 | |
335 | if (eqic(uri2, "/checkboxes")) ret serveCheckboxes(); |
336 | if (eqic(uri2, "/multiAdd")) ret serveMultiAdd(); |
337 | |
338 | q = params.get('q); |
339 | get = asCISet(nempties(subBot_paramsAsMultiMap().get('get))); |
340 | |
341 | domain = or2(params.get('domain), domain()); |
342 | |
343 | S raw = firstKeyWithValue("", params); // agi.blue?something |
344 | if (nempty(raw) && empty(q)) q = raw; |
345 | /*if (nempty(q)) { |
346 | domain = makeAGIDomain(q); |
347 | if (l(domain) > maximumDomainPartLength()) // escape with "domain=" |
348 | ret hrefresh(agiBlueURL() + hquery(+domain, key := "read as", value := q)); |
349 | ret hrefresh("http://" + domain + (eq(q, domain) ? "" : "/" + hquery(key := "read as", value := q))); |
350 | //uri = "/"; replaceMapWithParams(params, key := "read as", value := q); |
351 | }*/ |
352 | S url = domain + dropTrailingSlash(uri); |
353 | |
354 | // domain to query |
355 | //if (empty(q)) q = url; |
356 | if (empty(q)) { |
357 | S qq = agiBlue_urlToQuery(url); |
358 | if (neqic(qq, "agi.blue")) q = qq; |
359 | } |
360 | |
361 | Page page, bool newPage = unpair uniqCI2_sync(cc, Page, +q); |
362 | if (newPage) dbLog("New page", +q); |
363 | //printStructs(+params, +raw, +q); |
364 | |
365 | if (empty(params.get('q)) && empty(raw) && isHomeDomain()) { |
366 | //L<Page> pages = sortedByFieldDesc _modified(list(cc, Page)); // TODO: use index |
367 | L<Page> pages = cloneList(slice.idx_latestChangedPages.objectIterator()); |
368 | int start = parseInt(params.get("start")), step = 100; |
369 | S nav = pageNav2("/", l(pages), start, step, "start"); |
370 | S content = hform(b("GIVE ME INPUT: ") + htextinput('q, autofocus := true) + " " + hsubmit("Ask")) |
371 | + h1(sliceAsHTML() + " has " + nPages(countConcepts(cc, Page)) + " and " + nConnections(countConcepts(cc, Entry))) |
372 | + p(nav) |
373 | + p_nemptyLines(map(pageToHTMLLink(), subList(pages, start, start+step))); |
374 | |
375 | ret hhtml_agiBlue(hhead_title("Slice " + sliceConcept().name + " | agi.blue") // SERVE SLICE HOME PAGE |
376 | + hbody(hfullcenterAndTopLeft(top() + content |
377 | + footer(), |
378 | sliceSelector() |
379 | ))); |
380 | } |
381 | |
382 | S key = trim(params.get('key)), value = trim(params.get('value)); |
383 | L<Entry> entries = GetEntriesAndPost(cc).go(page, params).entries; |
384 | |
385 | //S get = params.get('get); |
386 | if (nempty(get)) |
387 | ret serveJSON(collect value(llNotNulls(firstThat(entries, e -> get.contains(e.key))))); |
388 | |
389 | S key2 = key, value2 = value; if (nempty(key) && nempty(value)) key2 = value2 = ""; // input reset |
390 | |
391 | bool withHidden = eq(params.get('withHidden), "1"); |
392 | new Set<Int> hide; |
393 | if (!withHidden) for (Entry e : entries) if (eqic(e.key, 'hide) && isSquareBracketedInt(e.value)) addAll(hide, e.count, parseInt(unSquareBracket(e.value))); |
394 | new MultiMap<Int, S> mmMeta; |
395 | for (Entry e : entries) if (isSquareBracketedInt(e.key)) mmMeta.put(parseInt(unSquareBracket(e.key)), e.value); |
396 | new MultiMap<S> mm; |
397 | for (Entry e : entries) mm.put(e.key, e.value); |
398 | |
399 | //S name = or2(/* ouch */ last(mm.get("read as")), /* end ouch */ unpackAGIDomain(page.url), page.url); |
400 | S name = page.q; |
401 | |
402 | // Find references |
403 | |
404 | L<Entry> refs = concatLists(conceptsWhereIC(cc, Entry, value := name), |
405 | conceptsWhereIC(cc, Entry, key := name)); |
406 | Set<Page> refPages = asSet(ccollect page(refs)); |
407 | refPages.remove(page); // don't list current page as reference |
408 | |
409 | // Search in page names (depending on default search type) |
410 | |
411 | L<Page> searchResults; |
412 | int totalResults = 0, searchResultsToShow = 50; |
413 | if (eq(sideSearchType, 'leven)) |
414 | searchResults = levenSearch(page.q, max := searchResultsToShow); |
415 | else if (eq(sideSearchType, 'literal)) |
416 | { |
417 | searchResults = literalSearch(page.q, max := maxInt()); |
418 | print("totalResults", totalResults = l(searchResults)); |
419 | searchResults = takeFirst(searchResultsToShow, searchResults); |
420 | } else |
421 | searchResults = (L<Page>) dm_call('agiBlueSearch, 'search, page.q, maxResult := clipIntPlus(searchResultsToShow, 1), agiBlueBotID := programID(), +cc); |
422 | searchResults.remove(page); |
423 | |
424 | // Find uses as key |
425 | usesAsKey = conceptsWhereIC(cc, Entry, key := page.q); |
426 | |
427 | // Find page in other slices |
428 | CentralIndexEntry cie = centralIndex.get(page.q); |
429 | L<Slice> pageInOtherSlices = cie == null ? null : listMinus(cie.slices, slice.sliceConcept); |
430 | |
431 | S pageName_html = htmlEncode2(shorten(displayLength, name)); |
432 | |
433 | S mainContents = |
434 | top() + h1(ahref_unstyled(agiBlue_pageURLWithSlice(page), pageName_html) + (newPage ? " [huh????]" : "")) |
435 | + p_nemptyLines(map(entries, func(Entry e) -> S { |
436 | !withHidden && (hide.contains(e.count) || eqic(e.key, "read as") && eqic(e.value, name)) ? "" |
437 | : "[" + e.count + "] " + |
438 | renderThing(e.key, false) + ": " + |
439 | b(renderThing(e.value, cic(mmMeta.get(e.count), "is a URL"))) |
440 | })) |
441 | + hpostform(h3("Add an entry") |
442 | + "Key: " + hinputfield(key := key2) + " Value: " + hinputfield(value := value2) + "<br><br>" + hsubmit("Add") |
443 | ) |
444 | |
445 | + p(ahref(agiBlueURL() + "/literalSearch" + hquery(q := page.q), "[literal search]", title := "Search pages with a name containing this page's name literally") |
446 | + " " + |
447 | ahref(agiBlueURL() + "/levenSearch" + hquery(q := page.q), "[leven search 1]", title := "Search pages with a Levenshtein similarity of 1 containing this page's name literally") |
448 | + " " + |
449 | ahref(agiBlueURL() + "/search" + hquery(q := page.q), "[scored search]", title := "Search pages with ScoredSearch") |
450 | + " " + |
451 | ahref(agiBlueURL() + "/deletePage" + hquery(slice := sliceID(), q := page.q), "[delete page]") |
452 | ) |
453 | |
454 | // optional "uses as key" section |
455 | |
456 | + (empty(usesAsKey) ? "" : h3(quote(pageName_html) + " as key") |
457 | + p_nemptyLines_showFirst(50, map(usesAsKey, e -> |
458 | pageToHTMLLink().get(e.page!) + unicode_spacedRightPointingTriangle() + |
459 | pageName_html + unicode_spacedRightPointingTriangle() + |
460 | pageToHTMLLink().get(pageFromQ(e.value)) |
461 | ))) |
462 | |
463 | // optional "in other slices" section |
464 | |
465 | + (empty(pageInOtherSlices) ? "" : h3(quote(pageName_html) + " in other slices") |
466 | + p_nemptyLines_showFirst(50, map(slice -> |
467 | ahref(agiBlueURL() + hquery(slice := slice.globalID, q := page.q), htmlEncode2(slice.name)), pageInOtherSlices))) |
468 | ; |
469 | |
470 | // end of mainContents |
471 | |
472 | S sideContents = |
473 | hform(b("GIVE ME INPUT:") + " " |
474 | + htextinput('q) + " " |
475 | + hsubmit("Ask", onclick := "document.getElementById('newInputForm').target = '';") + " " |
476 | + hsubmit("+Tab", title := "Ask and show result in a new tab", onclick := "document.getElementById('newInputForm').target = '_blank';"), |
477 | id := 'newInputForm) |
478 | |
479 | + h3("References (" + l(refPages) + ")") |
480 | |
481 | + p_nemptyLines_showFirst(10, map(pageToHTMLLink(displayLength := sideDisplayLength), refPages)) |
482 | |
483 | + h3(searchTypeToText.get(sideSearchType) + " search results (" |
484 | + (l(searchResults) >= searchResultsToShow ? searchResultsToShow + "+" : str(l(searchResults))) |
485 | + (totalResults > l(searchResults) ? " of " + n2(totalResults) : "") + ")") |
486 | |
487 | + p_nemptyLines_showFirst(searchResultsToShow, map(pageToHTMLLink(displayLength := sideDisplayLength), searchResults)) |
488 | |
489 | + hdiv("", id := 'extraStuff); |
490 | |
491 | // TODO: sync search delivery with WebSocket creation |
492 | if (asyncSearch) |
493 | doLater(6.0, r { dm_call('agiBlueSearch, 'searchAndPost, page.q, agiBlueBotID := programID(), +cc) }); |
494 | |
495 | // notify local interested parties |
496 | vmBus_send('agiBlue_servingConceptPage, this, page); |
497 | |
498 | S iframe = params.get('render_iframe); |
499 | |
500 | // serve a concept page |
501 | ret hhtml_agiBlue(hhead_title(pageDisplayName(page)) + hbody( |
502 | tag('table, |
503 | tr( |
504 | td(sliceSelector(), valign := 'top) |
505 | + td(sideContents, align := 'right, valign := 'top, rowspan := 2)) |
506 | + tr(td(mainContents + |
507 | (empty(iframe) ? "" : iframe(iframe, width := 600, height := 400)) + |
508 | footer(), align := 'center, valign := 'top)), |
509 | width := "100%", height := "100%"))); |
510 | } |
511 | |
512 | O servePagesToBot(Iterable<Page> pages) { |
513 | ret serveListToBot(map(pageToMap(wrapMapAsParams(params)), pages)); |
514 | } |
515 | |
516 | O serveListToBot(Collection l) { |
517 | if (nempty(params.get('max))) |
518 | l = takeFirst(parseInt(params.get('max)), l); |
519 | ret serveJSON(l); |
520 | } |
521 | |
522 | // uri starts with "/bot/" |
523 | O serveBot() { |
524 | S q = params.get('q); |
525 | |
526 | if (eqic(uri, "/bot/hello")) ret serveJSON("hello"); |
527 | if (eqic(uri, "/bot/hasPage")) ret serveJSON(hasPage(q)); |
528 | if (eqic(uri, "/bot/randomPageContaining")) { |
529 | assertNempty(q); |
530 | ret servePageToBot(random(filter(list(cc, Page), p -> cic(p.q, q))), params); |
531 | } |
532 | if (eqic(uri, "/bot/allPages")) |
533 | ret servePagesToBot(list(cc, Page)); |
534 | if (eqic(uri, "/bot/allPagesStartingWith")) { |
535 | assertNempty(q); |
536 | ret servePagesToBot(filter(list(cc, Page), p -> swic(p.q, q))); |
537 | } |
538 | if (eqic(uri, "/bot/allPagesEndingWith")) { |
539 | assertNempty(q); |
540 | ret servePagesToBot(filter(list(cc, Page), p -> ewic(p.q, q))); |
541 | } |
542 | if (eqic(uri, "/bot/allPagesContaining")) { |
543 | assertNempty(q); |
544 | ret servePagesToBot(filter(list(cc, Page), p -> cic(p.q, q))); |
545 | } |
546 | if (eqic(uri, "/bot/allPagesContainingRegexp")) { |
547 | assertNempty(q); |
548 | Pattern pat = regexpIC(q); |
549 | ret servePagesToBot(filter(list(cc, Page), p -> regexpFindIC(pat, p.q))); |
550 | } |
551 | |
552 | if (eqicOneOf(uri, "/bot/postSigned", /*"/bot/makePhysicalSlice",*/ "/bot/approveTrustRequest")) { |
553 | S text = rtrim(params.get('text)); |
554 | S key = getSignerKey(text); |
555 | if (empty(key)) ret subBot_serve500("Please include your public key"); |
556 | if (!isSignedWithKey(text, key)) ret subBot_serve500("Signature didn't verify"); |
557 | text = dropLastTwoLines(text); // drop signer + sig line |
558 | |
559 | Signer signer = uniq_sync Signer(publicKey := key); |
560 | |
561 | /*if (eqic(uri, "/bot/makePhysicalSlice")) { |
562 | if (!signer.trusted) ret subBot_serve500("Untrusted signer"); |
563 | Page page = findPageFromParams(jsonDecodeMap(text)); |
564 | if (page == null) ret subBot_serve500("Page not found"); |
565 | ret serveJSON(uniq2_sync(PhysicalSlice, slicePage := page).b ? "Slice made" : "Slice exists"); |
566 | }*/ |
567 | |
568 | if (eqic(uri, "/bot/postSigned")) { |
569 | new L out; |
570 | for (S line : tlft(text)) { |
571 | SS map = jsonDecodeMap(line); |
572 | GetEntriesAndPost x = new(db_mainConcepts()); |
573 | x.signer = signer; |
574 | Page page = findOrMakePageFromParams(map); |
575 | if (page == null) continue with out.add("Invalid page reference"); |
576 | x.go(page, map); |
577 | out.add(x.newEntry ? "Saved" : x.entry != null ? "Entry exists" : "Need key and value"); |
578 | } |
579 | ret serveJSON(out); |
580 | } |
581 | |
582 | if (eqic(uri, "/bot/approveTrustRequest")) { |
583 | if (!signer.trusted) ret subBot_serve500("Untrusted signer"); |
584 | Signer toApprove = conceptWhere Signer(publicKey := trim(text)); |
585 | if (toApprove == null) ret subBot_serve500("Signer to approve not found"); |
586 | cset(toApprove, trusted := true, approvedBy := signer.globalID); |
587 | ret serveJSON("Approved: " + trim(text)); |
588 | } |
589 | |
590 | ret subBot_serve500("CONFUSION"); |
591 | } |
592 | |
593 | if (eqic(uri, "/bot/post")) { |
594 | GetEntriesAndPost x = new(db_mainConcepts()); |
595 | x.go(pageFromQ(q), params); |
596 | ret serveJSON(x.newEntry ? "Saved" : x.entry != null ? "Entry exists" : "Need key and value"); |
597 | } |
598 | |
599 | if (eqic(uri, "/bot/entriesOnPage")) |
600 | ret serveJSON(map(entriesOnPage(findPageFromParams(params)), entryToMap(false))); |
601 | |
602 | if (eqic(uri, "/bot/entriesForKey")) |
603 | ret serveJSON(map(conceptsWhereIC(cc, Entry, key := params.get('key)), entryToMap(true))); |
604 | |
605 | // look up value from page & key in current slice |
606 | if (eqic(uri, "/bot/lookup")) { |
607 | S key = params.get('key); |
608 | if (empty(key)) ret serveJSON("Need key"); |
609 | S value = getValue(findPageFromParams(params), key); |
610 | ret serveJSON(empty(value) ? "" : litmap(+value)); |
611 | } |
612 | |
613 | if (eqic(uri, "/bot/multiLookup")) { |
614 | S key = params.get('key); |
615 | if (empty(key)) ret serveJSON("Need key"); |
616 | ret serveJSON(collect value(objectsWhereIC(findBackRefs(findPageFromParams(params), Entry), +key))); |
617 | } |
618 | |
619 | // look up value from page & key in all slices |
620 | if (eqic(uri, "/bot/multiLookupInAllSlices")) { |
621 | S key = params.get('key); |
622 | if (empty(key)) ret serveJSON("Need key"); |
623 | Cl<Slice> slices = centralIndexGetSlices(key); |
624 | new L<Map> results; |
625 | fOr (Slice s : slices) { |
626 | WorksOnSlice wos = new(loadBackgroundSlice(s)); |
627 | Page page = wos.findPageFromParams(params); |
628 | if (page != null) |
629 | fOr (Entry e : objectsWhereIC(findBackRefs(page, Entry), +key)) |
630 | results.add(litorderedmap("c" := e.value, "slice" := str(s.globalID()))); |
631 | } |
632 | ret serveJSON(results); |
633 | } |
634 | |
635 | if (eqic(uri, "/bot/latestEntries")) |
636 | ret serveJSON(map(takeFirst(10, slice.idx_latestEntries.objectIterator()), entryToMap(true))); |
637 | if (eqic(uri, "/bot/latestPages")) |
638 | ret serveJSON(map(takeFirst(10, slice.idx_latestCreatedPages.objectIterator()), pageToMap())); |
639 | if (eqic(uri, "/bot/latestChangedPages")) |
640 | ret serveJSON(map(takeFirst(10, slice.idx_latestChangedPages.objectIterator()), pageToMap())); |
641 | |
642 | if (eqic(uri, "/bot/googleUsersCount")) |
643 | ret serveJSON(countConcepts(GoogleUser)); |
644 | |
645 | if (eqic(uri, "/bot/totalPageCount")) |
646 | ret serveJSON(countConcepts(Page)); |
647 | /*if (eqic(uri, "/bot/pageWithoutPhysicalSliceCount")) |
648 | ret serveJSON(countConceptsWhere(Page, slice := null)); |
649 | if (eqic(uri, "/bot/physicalSliceCount")) |
650 | ret serveJSON(countConcepts(PhysicalSlice));*/ |
651 | if (eqic(uri, "/bot/trustedSignersCount")) |
652 | ret serveJSON(countConcepts(Signer, trusted := true)); |
653 | |
654 | if (eqic(uri, "/bot/valueSearch")) { |
655 | S value = params.get('value); |
656 | L<Entry> entries = conceptsWhereIC(cc, Entry, +value); |
657 | ret serveJSON(map(takeFirst(100, entries), entryToMap(true))); |
658 | } |
659 | |
660 | if (eqic(uri, "/bot/keyAndValueSearch")) { |
661 | S key = params.get('key), value = params.get('value); |
662 | Cl<Page> pages = pagesForKeyAndValue(key, value); |
663 | ret servePagesToBot(pages); |
664 | } |
665 | |
666 | if (eqic(uri, "/bot/keyValuePairsByPopularity")) { |
667 | L<PairS> pairs = map(list(Entry), e -> pair(e.key, e.value)); |
668 | LPair<S, Int> pairs2 = multiSetTopPairs(ciMultiSet(map pairToUglyStringForCIComparison(pairs))); |
669 | ret serveJSON(map(pairs2, p -> { |
670 | S key, value = unpair pairFromUglyString(p.a); |
671 | ret litorderedmap(n := p.b, +key, +value); |
672 | })); |
673 | } |
674 | |
675 | if (eqic(uri, "/bot/allKeys")) |
676 | ret serveListToBot(distinctCIFieldValuesOfConcepts(cc, Entry, 'key)); |
677 | |
678 | if (eqic(uri, "/bot/allKeysByPopularity")) |
679 | ret serveListToBot(mapMultiSetByPopularity(distinctCIFieldValuesOfConcepts_multiSet(cc, Entry, 'key), (key, n) -> litorderedmap(+n, +key))); |
680 | |
681 | if (eqic(uri, "/bot/dbSize")) |
682 | ret serveJSON(l(conceptsFile())); |
683 | |
684 | if (eqic(uri, "/bot/query")) |
685 | ret serveBotQuery(); |
686 | |
687 | if (eqic(uri, "/bot/createSlice")) { |
688 | Slice slice = createSlice(assertNempty(params.get('name))); |
689 | ret serveJSON(litorderedmap(id := str(slice.globalID), name := slice.name)); |
690 | } |
691 | |
692 | if (eqic(uri, "/bot/dumpAllSlices") && authed()) { |
693 | new L<Map> out; |
694 | for (Slice slice) try { |
695 | out.add(litorderedmap(slice := slice.caseID, ms := returnTimeFor(() -> dumpSliceToFile(loadBackgroundSlice(slice))))); |
696 | } catch print e { |
697 | out.add(litorderedmap(slice.caseID, error := exceptionToStringShort(e))); |
698 | } |
699 | ret serveJSON(out); |
700 | } |
701 | |
702 | if (eqic(uri, "/bot/dumpSlice")) { |
703 | File f = dumpSliceToFile(slice); |
704 | ret "OK, saved as: " + f2s(f); |
705 | } |
706 | |
707 | if (eqic(uri, "/bot/diskStats")) |
708 | ret serveJSON(litcimap( |
709 | sizeOnDisk := sizeOnDisk! |
710 | )); |
711 | |
712 | if (eqic(uri, "/bot/memStats")) { |
713 | Map fg = cloneMap(loadedSlices); |
714 | Map bg = cloneMap(backgroundSlices); |
715 | ret serveJSON(litcimap( |
716 | error := checkDoubleLoads(), |
717 | centralIndexEntries := l(centralIndex), |
718 | centralIndexMade := renderHowLongAgo(centralIndexMade), |
719 | centralIndexMadeInMS := centralIndexMadeIn, |
720 | numLoadedForegroundSlices := l(fg), |
721 | numLoadedBackgroundSlices := l(bg), |
722 | loadedForegroundSlicesEstimatedSize := longSum(map estimatedSliceDataSize(keys(fg))), |
723 | loadedBackgroundSlicesEstimatedSize := longSum(map estimatedSliceDataSize(keys(bg))), |
724 | loadedForegroundSlices := sortedIC(keysList(fg)), |
725 | loadedBackgroundSlices := sortedIC(keysList(bg)), |
726 | sessionsWithGoogleUsers := countPred(list(Session), s -> s.user! instanceof GoogleUser), |
727 | userObjects := countConcepts(User), |
728 | )); |
729 | } |
730 | |
731 | if (eqic(uri, "/bot/centralIndexGet")) { |
732 | CentralIndexEntry e = centralIndex.get(q); |
733 | ret serveJSON(e == null ? ll() : map(e.slices, sliceToMap())); |
734 | } |
735 | |
736 | // return primary triple from each slice |
737 | if (eqic(uri, "/bot/centralIndexGrab")) { |
738 | CentralIndexEntry ie = centralIndex.get(q); |
739 | new L<Map> out; |
740 | if (ie != null) for (Slice slice : ie.slices) { |
741 | LoadedSlice ls = loadBackgroundSlice(slice); |
742 | WorksOnSlice wos = new(ls); |
743 | Page page = wos.findPageFromQ(q); |
744 | Iterable<Entry> entries = entriesOnPage(page); |
745 | print("Page for " + q + " in slice " + ls.caseID + ": " + yesNo(page != null) + " - " + l(entries)); |
746 | Entry e = first(entries); |
747 | if (e != null) |
748 | out.add(litorderedmap(a := page.q, b := e.key, c := e.value, slice := slice.caseID)); |
749 | } |
750 | ret serveJSON(out); |
751 | } |
752 | |
753 | if (eqic(uri, "/bot/updateCentralIndex") && authed()) { |
754 | rst_index.trigger(); |
755 | ret "OK"; |
756 | } |
757 | |
758 | if (eqic(uri, "/bot/allGoogleEmails") && authed()) |
759 | ret serveJSON(collect googleEmail(list(GoogleUser))); |
760 | |
761 | if (eqic(uri, "/bot/words2_spaces_all")) |
762 | ret serveJSON(mapToValues_ciMap words2_spaces_cached(collect q(conceptsSortedByFieldCI(slice.cc, Page, 'q)))); |
763 | |
764 | if (eqic(uri, "/bot/words2_spaces_collapse_all")) |
765 | ret serveJSON(mapToValues_ciMap words2_spaces_collapse_cached(collect q(conceptsSortedByFieldCI(slice.cc, Page, 'q)))); |
766 | |
767 | if (eqic(uri, "/bot/makeManyPages")) { |
768 | //LS qs = nempties(subBot_paramsAsMultiMap().get('q)); |
769 | LS qs = tlft(params.get("pages")); |
770 | slice.haltSliceDumping = true; |
771 | try { |
772 | fOr ping (S _q : qs) |
773 | pageFromQ(_q); |
774 | } finally { |
775 | slice.haltSliceDumping = true; |
776 | slice.rstDumpSlice.trigger(); |
777 | } |
778 | ret "OK (" + l(qs) + ")"; |
779 | } |
780 | |
781 | // end of bot methods |
782 | |
783 | ret subBot_serve404(); |
784 | } |
785 | |
786 | O serveBotQuery() { |
787 | S query = params.get('query); |
788 | ret new Query(slice).process(query); |
789 | } |
790 | |
791 | O serveLiteralSearch() { |
792 | S q = params.get('q); |
793 | L<Page> searchResults = literalSearch(q); |
794 | ret serveSearchResults("literal search" , q, searchResultsToShow, searchResults); |
795 | } |
796 | |
797 | L<Page> literalSearch(S q, O... _) { |
798 | int searchResultsToShow = optPar max(_, 100); |
799 | |
800 | // quick search in random order |
801 | //L<Page> searchResults = takeFirst(clipIntPlus(searchResultsToShow, 1), filterIterator(iterator(list(cc, Page)), p -> cic(p.q, q)); |
802 | |
803 | // full search, order by length |
804 | ret takeFirst(clipIntPlus(searchResultsToShow, 1), pagesSortedByLength(filter(list(cc, Page), p -> cic(p.q, q)))); |
805 | } |
806 | |
807 | O serveScoredSearch() { |
808 | S q = params.get('q); |
809 | L<Page> searchResults = (L<Page>) dm_call('agiBlueSearch, 'search, q, +cc); |
810 | ret serveSearchResults("scored search" , q, searchResultsToShow, searchResults); |
811 | } |
812 | |
813 | O serveLevenSearch() { |
814 | S q = params.get('q); |
815 | L<Page> searchResults = levenSearch(q); |
816 | ret serveSearchResults("leven search with distance 1" , q, searchResultsToShow, searchResults); |
817 | } |
818 | |
819 | L<Page> levenSearch(S q, O... _) { |
820 | int searchResultsToShow = optPar max(_, 100); |
821 | int maxEditDistance = 1; |
822 | |
823 | new Map<Page, Int> map; |
824 | for (Page p : list(cc, Page)) { |
825 | int distance = leven_limitedIC(q, p.q, maxEditDistance+1); |
826 | if (distance <= maxEditDistance) map.put(p, distance); |
827 | } |
828 | ret takeFirst(clipIntPlus(searchResultsToShow, 1), keysSortedByValue(map)); |
829 | } |
830 | |
831 | O serveSearchResults(S searchType, S q, int searchResultsToShow, Collection<Page> searchResults) { |
832 | S title = "agi.blue " + searchType + " for " + htmlEncode2(quote(q)) + " (" + (l(searchResults) >= searchResultsToShow ? searchResultsToShow + "+ results" : nResults(l(searchResults))) + ")"; |
833 | |
834 | ret hhtml_agiBlue(hhead_title(htmldecode_dropAllTags(title)) |
835 | + hbody(hfullcenter(//top() + |
836 | h3(title) |
837 | + p_nemptyLines_showFirst(searchResultsToShow, map(pageToHTMLLink(), searchResults))))); |
838 | } |
839 | |
840 | O serveQueryPage() { |
841 | S query = params.get('query); |
842 | if (query == null) query = loadSnippet(#1024258); |
843 | S title = agiBlueNameHTML() + " | Execute a query script (" + targetBlank("http://code.botcompany.de/1024274", "ALQL") + ")"; |
844 | ret hhtml_miniPage(title, h3(title) |
845 | + form( |
846 | htextarea(query, name := 'query, cols := 80, rows := 10, autofocus := true) |
847 | + "<br><br>" |
848 | + hsubmit("Execute"), action := "/bot/query" |
849 | )); |
850 | } |
851 | |
852 | O hhtml_miniPage(S htmlTitle, O contents) { |
853 | ret hhtml_agiBlue(hhead_title(htmldecode_dropAllTags(htmlTitle)) |
854 | + hbody(hfullcenter(contents))); |
855 | } |
856 | |
857 | S sliceHomeURL() { |
858 | ret slice == null ? agiBlueURL() : sliceHomeURL(slice.sliceConcept.globalID); |
859 | } |
860 | |
861 | S sliceHomeURL(Slice slice) { ret sliceHomeURL(slice.globalID); } |
862 | |
863 | S sliceHomeURL(GlobalID slice) { |
864 | ret agiBlueURL() + hquery(+slice); |
865 | } |
866 | |
867 | S sliceLinkHTML(Slice slice) { |
868 | ret slice == null ? "-": ahref(sliceHomeURL(slice), htmlEncode2(slice.name)); |
869 | } |
870 | |
871 | O serveCreateSlicePage() { |
872 | S sliceName = trim(params.get('sliceName)); |
873 | if (eq(params.get('doIt), "1") && nempty(sliceName)) { |
874 | // TODO: check for existing name |
875 | Slice slice = createSlice(sliceName); |
876 | ret hrefresh(sliceHomeURL(slice.globalID)); |
877 | } |
878 | |
879 | S title = agiBlueNameHTML() + " | Create slice"; |
880 | ret hhtml_agiBlue(hhead_title(htmldecode_dropAllTags(title)) |
881 | + hbody(hfullcenter( |
882 | h3(title) |
883 | + form( |
884 | hhidden(doIt := 1) |
885 | + "Slice name: " + htextinput(+sliceName, autofocus := true) |
886 | + "<br><br>" |
887 | + hsubmit("Create slice")) |
888 | ))); |
889 | } |
890 | |
891 | O serveDeletePage() { |
892 | q = params.get('q); |
893 | Page page = findPageFromQ(q); |
894 | if (page == null) ret "Page " + quote(q) + " not found in slice " + htmlEncode(slice.name()); |
895 | bool ownage = slice.sliceConcept.owner! == session.user!; |
896 | if (!ownage) ret "Sorry, you don't own slice " + htmlEncode(slice.name()); |
897 | deleteConcepts(findBackRefs(page, AbstractEntry)); |
898 | cdelete(page); |
899 | ret hrefresh(1.0, sliceHomeURL()) + "Page deleted"; |
900 | } |
901 | |
902 | F1<Page, S> pageToHTMLLink(O... _) { |
903 | optPar int displayLength = main.displayLength; |
904 | ret func(Page p) -> S { |
905 | S name = pageDisplayName(p); |
906 | ret ahref(agiBlue_pageURLWithSlice(p), |
907 | htmlEncode2(shorten(displayLength, name)), |
908 | title := name); |
909 | }; |
910 | } |
911 | |
912 | // and google log-in |
913 | S sliceSelector() { |
914 | ret htag table(tr( |
915 | (!showGoogleLogIn ? "" : td( |
916 | googleSignIn_signInButton(agiBlueURL() + "/google-verify", "console.log(data);", ""), style := "padding-right: 10px;") |
917 | + td(nobr(ahref("javascript:signOut()", "Sign out")), style := "padding-right: 10px; padding-bottom: 0.15em")) |
918 | |
919 | + (!showSliceSelector ? "" : hform(td( |
920 | "Select reality slice: " |
921 | + hselect(availableSlices(), session.selectedSlice, name := 'slice, onchange := "this.form.submit()") |
922 | /*+ " " + hsubmit("Go")*/ |
923 | + " | " + |
924 | ahref(agiBlueURL("/slices"), nSlices(countConcepts(Slice))) + " | " + |
925 | ahref(agiBlueURL() + "/createSlice", "Create slice...") |
926 | ))))); |
927 | } |
928 | |
929 | S sliceAsHTML() { |
930 | if (slice.isMainSlice()) ret htmlEncode2(agiBlueName() + "'s main slice"); |
931 | if (slice.sliceConcept == null) ret "Slice ???"; |
932 | ret htmlEncode2("Slice " + quote(slice.sliceConcept.name)); |
933 | } |
934 | |
935 | Slice sliceConcept() { ret slice.sliceConcept; } |
936 | |
937 | S footer() { |
938 | ret p(small(elapsedMS_sysNow(started) + " ms")); |
939 | } |
940 | |
941 | S verifiedEmail() { |
942 | if (session == null || !(session.user! instanceof GoogleUser)) null; |
943 | ret ((GoogleUser) session.user!).googleEmail; |
944 | } |
945 | |
946 | bool authed() { |
947 | bool ver = eqic(verifiedEmail(), "stefan.reich.maker.of.eye@googlemail.com"); |
948 | print("Auth check " + verifiedEmail() + " -> " + ver); |
949 | ret ver; |
950 | } |
951 | |
952 | S googleSignInID() { |
953 | ret eqic(domain(), "botcompany.de") ? botCompanyGoogleSignInID() : agiBlueGoogleSignInID(); |
954 | } |
955 | |
956 | S hhtml_agiBlue(S contents) { |
957 | ret hhtml(hAddToHead_fast(contents, |
958 | hIncludeGoogleFont("Source Sans Pro") |
959 | //+ [[<meta name="google-site-verification" content="oRvyeTqgSuv-FE_c-2UKM1Vp0oDqx8h9WruHYWqA-NQ" />]] |
960 | + loadJQuery() |
961 | + hmobilefix() |
962 | + googleSignIn_header("", googleSignInID()) |
963 | + hstylesheet("body { font-family: Source Sans Pro }"))); |
964 | } |
965 | |
966 | O serveSlicesList() { |
967 | S sort = getAny(params, 'sort, 'by); |
968 | L<Slice> slices; |
969 | S shown; |
970 | if (eqic(sort, 'name)) { |
971 | slices = asList(idx_slicesByName.objectIterator()); |
972 | shown = "sorted by name"; |
973 | } else { |
974 | sort = 'date; |
975 | slices = asList(idx_slicesByModification.objectIterator()); |
976 | shown = "last modified first"; |
977 | } |
978 | |
979 | S title = agiBlueNameHTML() + " has " + nSlices(slices) /*+ " (" + shown + ")"*/; |
980 | new PreIncLongCounter counter; |
981 | ret hhtml_agiBlue(hhead_title_decode(title) |
982 | + hbody(hfullcenter( |
983 | h1(title) |
984 | + p(joinWithSpacedVBar( |
985 | ahrefIf(neqic(sort, 'name), agiBlueURL() + "/slices" + hquery(sort := 'name), "List by name"), |
986 | ahrefIf(neqic(sort, 'date), agiBlueURL() + "/slices" + hquery(sort := 'date), "List by date"), |
987 | ahref(agiBlueURL() + "/createSlice", "Create slice..."))) |
988 | |
989 | + htmlTable2(map(slices, slice -> |
990 | litorderedmap( |
991 | "Name" := ahref(sliceHomeURL(slice), htmlEncode2(slice.name)), |
992 | "ID" := ahref(sliceHomeURL(slice), str(slice.globalID)), |
993 | "Owner" := str(slice.owner!) |
994 | )), |
995 | htmlEncode := false) |
996 | ))); |
997 | } |
998 | |
999 | O serveUsersList() { |
1000 | S sort = getAny(params, 'sort, 'by); |
1001 | L<User> users; |
1002 | S shown; |
1003 | if (eqic(sort, 'name)) |
1004 | users = asList(idx_usersByName.objectIterator()); |
1005 | else { |
1006 | sort = 'seen; |
1007 | users= asList(idx_usersBySeen.objectIterator()); |
1008 | } |
1009 | |
1010 | S title = agiBlueNameHTML() + " has " + nUsers(users); |
1011 | new PreIncLongCounter counter; |
1012 | ret hhtml_agiBlue(hhead_title_decode(title) |
1013 | + hbody(hfullcenter( |
1014 | h1(title) |
1015 | + p(joinWithSpacedVBar( |
1016 | ahrefIf(neqic(sort, 'name), agiBlueURL() + "/users" + hquery(sort := 'name), "List by name"), |
1017 | ahrefIf(neqic(sort, 'seen), agiBlueURL() + "/users" + hquery(sort := 'seen), "List by last seen"), |
1018 | ahref(agiBlueURL() + "/createSlice", "Create slice..."))) |
1019 | |
1020 | + htmlTable2(map(users, user -> |
1021 | litorderedmap( |
1022 | "Name" := ahref(userHomeURL(user), htmlEncode2(str(user))), |
1023 | "Last seen" := renderHowLongAgo(user.lastSeen) |
1024 | )), |
1025 | htmlEncode := false) |
1026 | ))); |
1027 | } |
1028 | |
1029 | S sliceInfoHTML() { |
1030 | User owner = slice.sliceConcept.owner!; |
1031 | ret owner == null ? "" : p("This slice is owned by " + userHTML(owner)); |
1032 | } |
1033 | |
1034 | S userHomeURL(User user) { |
1035 | ret user == null ? null : agiBlueURL() + "/user/" + user.globalID; |
1036 | } |
1037 | |
1038 | S userHTML(User user) { |
1039 | ret user == null ? "nobody" : ahref(userHomeURL(user), htmlEncode2(str(user))); |
1040 | } |
1041 | |
1042 | S top() { |
1043 | ret nempty(get) ? "" : hcomment("cookie: " + takeFirst(4, session.cookie)) |
1044 | + hSilentComputatorWithFlag("agi.blue: " + q) |
1045 | + p(ahref(sliceHomeURL(), |
1046 | //hsnippetimg(#1101682, width := 565/5, height := 800/5, title := "Robot by Peerro @ DeviantArt") |
1047 | //hsnippetimg(#1101778, width := 96, height := 96, title := "agi.blue - a database for everything") |
1048 | hsnippetimg(#1101822, width := 314, height := 125, title := "agi.blue - Wikipedia for robots") |
1049 | )) |
1050 | + sliceInfoHTML() |
1051 | + p(small( |
1052 | agiBlueNameHTML_boldWithSize() |
1053 | + (agiBlue_isOriginal() ? "" : " " + targetBlank("http://agi.blue", "[original]")) |
1054 | + " | " + ahref(agiBlueURL("/slices"), nSlices(countConcepts(Slice))) |
1055 | + " | " + targetBlank(progLink(), "source code") + " of this web site (" + nLines(sourceCodeLines) + ") | " + targetBlank("https://gitter.im/agi-blue/community", "sponsor https?") + " | by " + targetBlank("https://BotCompany.de", "BC") + " | " + targetBlank("http://fiverr.tinybrain.de/", "Fiverr") + " | " + targetBlank("https://discordapp.com/invite/SEAjPqk", "Discord") + " | " + targetBlank("https://www.youtube.com/watch?v=b6jtRdV3Ev8", "Video") |
1056 | + " | " + targetBlank("http://code.botcompany.de/1024233", "Notes") |
1057 | + " | " + ahref(agiBlueURL() + "/query", "Query") |
1058 | )); |
1059 | } |
1060 | |
1061 | Slice createSlice(S name) { |
1062 | name = shorten(name, maxSliceNameLength); |
1063 | Slice slice = cnew Slice(+name, owner := session.user); |
1064 | cset(slice, caseID := slice.defaultCaseID()); |
1065 | loadSlice(slice).initialSetup(slice.globalID); |
1066 | ret slice; |
1067 | } |
1068 | |
1069 | O serveCheckboxes() { |
1070 | set conceptsSortedByFieldCI_verbose; |
1071 | int showMax = 1000; |
1072 | Cl<Page> pages = takeFirst(showMax, conceptsSortedByFieldCI(slice.cc, Page, 'q)); |
1073 | S title = "Select pages"; |
1074 | ret hhtml_miniPage(title, h3(title) + hpostform( |
1075 | htmlTable2(map(pages, page -> litorderedmap( |
1076 | "Page" := hcheckbox("page_" + page.globalID()) + " " + pageToHTMLLink().get(page) |
1077 | )), |
1078 | htmlEncode := false) |
1079 | + h3("Add an entry to all selected pages") |
1080 | + p("Key: " + hinputfield("key") + |
1081 | " Value: " + hinputfield("value")) |
1082 | + p(hsubmit("Add entries")), action := agiBlueURL() + "/multiAdd")); |
1083 | } |
1084 | |
1085 | O serveMultiAdd() { |
1086 | S key = trim(params.get('key)), value = trim(params.get('value)); |
1087 | if (empty(key) || empty(value)) ret "Need key and value"; |
1088 | new Matches m; |
1089 | new L<Page> pages; |
1090 | for (S a, b : params) |
1091 | if (startsWith(a, "page_", m)) |
1092 | addIfNotNull(pages, conceptWhere(cc, Page, globalID := m.rest())); |
1093 | if (empty(pages)) ret "No pages selected"; |
1094 | for (Page page : pages) |
1095 | new GetEntriesAndPost(cc).go(page, params); |
1096 | ret "Entry added to " + nPages(pages); |
1097 | } |
1098 | |
1099 | } // end of Request |
1100 | |
1101 | svoid dbLog(O... params) { |
1102 | logStructure(programFile("db.log"), ll(params)); |
1103 | } |
1104 | |
1105 | static IF1<Page, Map> pageToMap(O... _) { |
1106 | optPar bool withEntries; |
1107 | |
1108 | bool nameOnly = eqOneOf(optPar nameOnly(_), "1", true); |
1109 | if (nameOnly) ret (IF1<Page, Map>) p -> litmap(q := p.q); |
1110 | |
1111 | ret (IF1<Page, Map>) p -> { |
1112 | L<Entry> entries = findBackRefs(p, Entry); |
1113 | ret litorderedmap( |
1114 | q := p.q, |
1115 | nEntries := l(entries), |
1116 | created := p.created, |
1117 | modified := p._modified, |
1118 | entries := !withEntries ? null : map(entries, entryToMap(false))); |
1119 | }; |
1120 | } |
1121 | |
1122 | static IF1<Entry, Map> entryToMap(bool withPage) { |
1123 | ret (IF1<Entry, Map>) e -> litorderedmap( |
1124 | created := e.created, |
1125 | i := e.count, |
1126 | key := e.key, |
1127 | value := e.value, |
1128 | q := withPage ? e.page->q : null, |
1129 | signer := getString globalID(e.signer!)); |
1130 | } |
1131 | |
1132 | static IF1<Slice, Map> sliceToMap() { |
1133 | ret (IF1<Slice, Map>) s -> litorderedmap( |
1134 | created := s.created, |
1135 | globalID := str(s.globalID), |
1136 | name := s.name, |
1137 | caseID := s.caseID); |
1138 | } |
1139 | |
1140 | sO servePageToBot(Page page, SS params) { |
1141 | if (page == null) ret serveJSON(null); |
1142 | params = asCIMap(params); |
1143 | Map map = pageToMap(withEntries := valueIs1 withEntries(params)).get(page); |
1144 | ret serveJSON(map); |
1145 | } |
1146 | |
1147 | sclass GetEntriesAndPost { |
1148 | Concepts cc; |
1149 | L<Entry> entries; |
1150 | Entry entry; |
1151 | bool newEntry; |
1152 | Signer signer; |
1153 | |
1154 | *(Concepts *cc) {} |
1155 | |
1156 | GetEntriesAndPost go(Page page, SS params) { |
1157 | S key = trim(params.get('key)), value = trim(params.get('value)); |
1158 | print("GetEntriesAndPost: " + quote(page.q) + ", " + quote(key) + " := " + quote(value)); |
1159 | withDBLock { |
1160 | entries = findBackRefs(page, Entry); |
1161 | if (nempty(key) && nempty(value)) { |
1162 | S ip = subBot_clientIP(); |
1163 | entry = firstThat(e -> eqic(e.key, key) && eq_icIf(!allowMultipleCasesInValues, e.value, value) && eq(e.ip, ip), entries); |
1164 | if (entry == null) { |
1165 | print("SAVING"); |
1166 | Entry e = cnew(cc, Entry, +page, +key, +value, +ip, count := l(entries) + 1, +signer); |
1167 | if (makeAllKeysPages) pageFromQ(cc, key); |
1168 | if (makeAllValuesPages) pageFromQ(cc, value); |
1169 | page.change(); // bump modification date |
1170 | entry = e; |
1171 | newEntry = true; |
1172 | entries.add(entry); |
1173 | dbLog("New entry", page := page.q, globalID := e.globalID, count := e.count, +key, +value); |
1174 | } |
1175 | } |
1176 | } |
1177 | |
1178 | sortByFieldInPlace created(entries); |
1179 | numberEntriesInConceptField count(entries); |
1180 | this; |
1181 | } |
1182 | } |
1183 | |
1184 | static SS availableSlices() { |
1185 | L<Slice> slices = asList(idx_slicesByName.objectIterator()); |
1186 | ret mapToOrderedMap(s -> pair(str(s.globalID), s.name + " [ID: " + s.globalID + "]"), slices); |
1187 | } |
1188 | |
1189 | static L<Entry> entriesOnPage(Page p) { |
1190 | ret p == null ? null : sortedByField count(findBackRefs(p, Entry)); |
1191 | } |
1192 | |
1193 | static L<Page> pagesSortedByLength(L<Page> l) { |
1194 | ret sortedByCalculatedField(l, p -> l(p.q)); |
1195 | } |
1196 | |
1197 | sbool agiBlue_isOriginal() { |
1198 | ret amProgram(#1023558); |
1199 | } |
1200 | |
1201 | sS agiBlueURL(S subUri) { ret agiBlueURL() + prependSlash(subUri); } |
1202 | |
1203 | sS agiBlueURL() { |
1204 | ret agiBlue_isOriginal() ? "https://agi.blue" : "/" + psI(programID()) + "/raw"; |
1205 | } |
1206 | |
1207 | sS agiBlueName() { |
1208 | ret agiBlue_isOriginal() ? "agi.blue" : "agi.blue clone " + programID(); |
1209 | } |
1210 | |
1211 | svoid cleanMeUp { |
1212 | cleanUp(fan); |
1213 | } |
1214 | |
1215 | static LoadedSlice loadSlice(Slice slice) { |
1216 | ret loadSlice(slice.caseID, str(slice.globalID)); |
1217 | } |
1218 | |
1219 | static LoadedSlice loadSlice(S caseID, S globalID) { |
1220 | caseID = unnull(caseID); |
1221 | lock fan.lock; |
1222 | |
1223 | // move slice from background to foreground |
1224 | |
1225 | Concepts cc = backgroundFan.getIfLoaded(caseID); |
1226 | if (cc != null) { |
1227 | LoadedSlice slice = assertNotNull(backgroundSlices.get(caseID)); |
1228 | backgroundFan.loaded.remove(caseID); |
1229 | fan.loaded.put(caseID, cc); |
1230 | backgroundSlices.remove(caseID); |
1231 | loadedSlices.put(caseID, slice); |
1232 | ret slice; |
1233 | } |
1234 | |
1235 | fan.get(caseID, globalID); |
1236 | ret loadedSlices.get(caseID); |
1237 | } |
1238 | |
1239 | // load background slice |
1240 | static LoadedSlice loadBackgroundSlice(Slice slice) { |
1241 | ret loadBackgroundSlice(slice.caseID, str(slice.globalID)); |
1242 | } |
1243 | |
1244 | static LoadedSlice loadBackgroundSlice(S caseID, S globalID) { |
1245 | caseID = unnull(caseID); |
1246 | lock fan.lock; |
1247 | |
1248 | // check if in foreground |
1249 | |
1250 | if (fan.getIfLoaded(caseID) != null) |
1251 | ret loadedSlices.get(caseID); |
1252 | |
1253 | backgroundFan.get(caseID, globalID); |
1254 | LoadedSlice slice = assertNotNull(backgroundSlices.get(caseID)); |
1255 | slice.lastAccess = sysNow(); |
1256 | while (l(backgroundSlices) > maxBackgroundSlices) |
1257 | unloadLeastRecentlyAccessedBackgroundSlice(); |
1258 | assertNotNull("slice.sliceConcept", slice.sliceConcept); |
1259 | ret slice; |
1260 | } |
1261 | |
1262 | // TODO: sync unloading slice with accessors |
1263 | svoid unloadLeastRecentlyAccessedBackgroundSlice() { |
1264 | lock fan.lock; |
1265 | S caseID = lowestByField lastAccess(keys(backgroundSlices)); |
1266 | if (caseID != null) |
1267 | backgroundFan.unloadCase(caseID); |
1268 | } |
1269 | |
1270 | static Slice sliceConceptForGlobalID(S globalID) { |
1271 | ret sliceConceptForGlobalID(toGlobalIDObj(globalID)); |
1272 | } |
1273 | |
1274 | static Slice sliceConceptForGlobalID(GlobalID globalID) { |
1275 | ret conceptWhere Slice(+globalID); |
1276 | } |
1277 | |
1278 | static Slice sliceConceptForCaseID(S caseID) { |
1279 | ret conceptWhere Slice(+caseID); |
1280 | } |
1281 | |
1282 | sS agiBlueNameHTML() { |
1283 | ret ahref(agiBlueURL(), htmlEncode2(agiBlueName())); |
1284 | } |
1285 | |
1286 | sS agiBlueNameHTML_boldWithSize() { |
1287 | Long size = sizeOnDisk.peek(); |
1288 | ret b(agiBlueNameHTML()) + (size == null ? "" : " " + spanTitle("Size of agi.blue's database on disk", "(" + toM(size) + " MB)")); |
1289 | } |
1290 | |
1291 | static Slice sliceForName(S name) { |
1292 | ret conceptWhereCI Slice(+name); |
1293 | } |
1294 | |
1295 | sclass Query extends WorksOnSlice { |
1296 | SS vars = ciMap(); |
1297 | |
1298 | *(LoadedSlice slice) { super(slice); } |
1299 | |
1300 | O process(S query) { |
1301 | try object processLines(agiBlue_parseQueryScript(query)); |
1302 | ret serveJSON("No return statement"); |
1303 | } |
1304 | |
1305 | O processLines(L<ALQLLine> lines) { |
1306 | for (ALQLLine line : lines) { |
1307 | if (line cast ALQLSlice) { |
1308 | Slice slice = sliceForName(line.slice); |
1309 | if (slice == null) ret serveJSON("Slice not found: " + line.slice); |
1310 | LoadedSlice oldSlice = Query.this.slice; |
1311 | try { |
1312 | setSlice(loadSlice(slice)); |
1313 | try object processLines(line.contents); |
1314 | } finally { |
1315 | setSlice(oldSlice); |
1316 | } |
1317 | } else if (line cast ALQLReturn) |
1318 | ret serveJSON(ll(getOrKeep(vars, line.var))); |
1319 | else if (line cast ALQLTriple) { |
1320 | T3S t = line.triple; |
1321 | t = tripleMap(t, s -> getOrKeep(vars, s)); |
1322 | Cl<Page> pages; |
1323 | S var; |
1324 | if (isDollarVar(t.c)) { |
1325 | var = t.c; |
1326 | if (isDollarVar(t.a)) { |
1327 | if (isDollarVar(t.b)) todo(t); |
1328 | Entry e = random(conceptsWhereCI(cc, Entry, key := t.b)); |
1329 | if (e == null) ret serveJSON("No results for " + var); |
1330 | pages = pagesForKeyAndValue(t.b, t.c); |
1331 | vars.put(t.a, e.page->q); |
1332 | vars.put(t.c, e.value); |
1333 | continue; |
1334 | } else if (isDollarVar(t.b)) { |
1335 | Page page = findPageFromQ(t.a); |
1336 | Entry e = random(findBackRefs(page, Entry)); |
1337 | if (e == null) ret serveJSON("No results for " + var); |
1338 | vars.put(t.b, e.key); |
1339 | vars.put(t.c, e.value); |
1340 | continue; |
1341 | } else { |
1342 | S val = getValue(t.a, t.b); |
1343 | if (val == null) ret serveJSON("No results for " + var); |
1344 | pages = ll(pageFromQ(val)); |
1345 | } |
1346 | } else if (isDollarVar(t.b)) { |
1347 | var = t.b; |
1348 | if (isDollarVar(t.c)) todo(t); |
1349 | if (isDollarVar(t.a)) { |
1350 | L<Entry> entries = conceptsWhereCI(cc, Entry, value := t.c); |
1351 | if (empty(entries)) ret serveJSON("No results for " + var); |
1352 | Entry e = random(entries); |
1353 | vars.put(t.a, e.page->q); |
1354 | vars.put(t.b, e.key); |
1355 | continue; |
1356 | } else { |
1357 | Cl<S> keys = keysForPageAndValue(t.a, t.c); |
1358 | if (empty(keys)) ret serveJSON("No results for " + var); |
1359 | pages = map(f<S, Page> pageFromQ, keys); |
1360 | } |
1361 | } else { |
1362 | var = t.a; |
1363 | if (!isDollarVar(t.a)) todo(t); |
1364 | if (isDollarVar(t.b)) todo(t); |
1365 | if (isDollarVar(t.c)) todo(t); |
1366 | pages = pagesForKeyAndValue(t.b, t.c); |
1367 | } |
1368 | if (empty(pages)) ret serveJSON("No results for " + var); |
1369 | vars.put(var, random(pages).q); |
1370 | } else if (line cast ALQLPage) { |
1371 | if (eq(line.matchMethod, 'eqic)) |
1372 | findPageFromQ(line.page); |
1373 | else if (eq(line.matchMethod, 'flexMatchDollarVarsIC_first)) { |
1374 | new L<SS> l; |
1375 | for (Page p : list(cc, Page)) |
1376 | addIfNotNull(l, flexMatchDollarVarsIC_first(line.page, p.q)); |
1377 | if (empty(l)) ret serveJSON("No results for " + curly(line.page)); |
1378 | vars.putAll(random(l)); |
1379 | } else |
1380 | fail("Unknown match method: " + line.matchMethod); |
1381 | } else |
1382 | fail("Can't interpret: " + line); |
1383 | } |
1384 | |
1385 | null; |
1386 | } |
1387 | } |
1388 | |
1389 | static Page pageFromQ(Concepts cc, S q) { |
1390 | ret empty(q) ? null : uniqCI_sync(cc, Page, +q); |
1391 | } |
1392 | |
1393 | /*sS globalIDFromCaseID(S caseID) { |
1394 | ret assertGlobalID(takeLast(globalIDLength(), caseID)); |
1395 | }*/ |
1396 | |
1397 | sS agiBlue_pageURLWithSlice(Page p) { |
1398 | ret p == null ? null : agiBlueURL() + hquery(slice := _get(sliceForPage(p), 'globalID), q := p.q); |
1399 | } |
1400 | |
1401 | static Slice sliceForPage(Page p) { |
1402 | if (p == null) null; |
1403 | SliceInfo info = conceptWhere(p._concepts, SliceInfo); |
1404 | ret info == null ? null : sliceConceptForGlobalID(info.globalID); |
1405 | } |
1406 | |
1407 | static GlobalID mainSliceGlobalID() { |
1408 | ret conceptWhere Slice(caseID := "").globalID; |
1409 | } |
1410 | |
1411 | static File sliceDumpFile(Slice slice) { |
1412 | ret programFile("slice-dumps/" + or2(slice.caseID, "main") + ".slice"); |
1413 | } |
1414 | |
1415 | static File dumpSliceToFile(LoadedSlice slice) { |
1416 | ret withDBLock(slice.cc, func -> File { |
1417 | cset(slice.sliceConcept, sliceDumped := now()); |
1418 | File file = sliceDumpFile(slice.sliceConcept); |
1419 | new LS lines; |
1420 | for (Page p : list(slice.cc, Page)) |
1421 | lines.add(quote(p.q)); |
1422 | for (Entry e : list(slice.cc, Entry)) |
1423 | lines.add(sfu(tripleToList(entryToTriple(e)))); |
1424 | saveLinesAsTextFile(file, lines); |
1425 | ret file; |
1426 | }); |
1427 | } |
1428 | |
1429 | static T3S entryToTriple(Entry e) { |
1430 | ret e == null ? null : t3(e.page->q, e.key, e.value); |
1431 | } |
1432 | |
1433 | static ConceptsLoadedOnDemand slicesLoadedOnDemand(Map<S, LoadedSlice> loadedSlices) { |
1434 | new ConceptsLoadedOnDemand fan; |
1435 | fan.onCaseLoaded(voidfunc(S caseID, O globalID, Concepts concepts) { |
1436 | LoadedSlice ls = new(caseID, concepts); |
1437 | ls.sliceConcept = sliceConceptForCaseID(caseID); |
1438 | |
1439 | if (nempty(globalID)) |
1440 | ls.initialSetup(toGlobalIDObj((S) globalID)); |
1441 | loadedSlices.put(caseID, ls); |
1442 | }); |
1443 | fan.onUnloadingCase(voidfunc(S caseID, Concepts concepts) { |
1444 | loadedSlices.remove(caseID); |
1445 | }); |
1446 | ret fan; |
1447 | } |
1448 | |
1449 | // check if slices are loaded both in foreground and background (BAD) |
1450 | sS checkDoubleLoads() { |
1451 | LS l = sharedKeys(loadedSlices, backgroundSlices); |
1452 | ret empty(l) ? null : "ERROR: Slice loaded in background and foreground: " + first(l); |
1453 | } |
1454 | |
1455 | // just use size of concepts.structure.gz |
1456 | static long estimatedSliceDataSize(Slice slice) { |
1457 | ret estimatedSliceDataSize(slice.caseID); |
1458 | } |
1459 | |
1460 | static long estimatedSliceDataSize(S caseID) { |
1461 | ret l(conceptsFile(fan.dbID(caseID))); |
1462 | } |
1463 | |
1464 | static CentralIndexEntry centralIndexGet(S s) { ret getOrCreate CentralIndexEntry(centralIndex, s); } |
1465 | |
1466 | static CentralIndexEntry newCentralIndexGet(S s) { ret getOrCreate CentralIndexEntry(newCentralIndex, s); } |
1467 | |
1468 | static Cl<Slice> centralIndexGetSlices(S s) { |
1469 | CentralIndexEntry e = centralIndexGet(s); |
1470 | ret e == null ? null : e.slices; |
1471 | } |
1472 | |
1473 | // executed by rst_index |
1474 | svoid makeCentralIndex() { |
1475 | newCentralIndex.clear(); |
1476 | long time = now(); |
1477 | pcall { |
1478 | for (Slice slice : conceptsWhere(Slice, indexed := true)) |
1479 | addToNewCentralIndex(slice); |
1480 | } |
1481 | centralIndex.putAll(newCentralIndex); |
1482 | removeAllKeysNotIn(centralIndex, newCentralIndex); |
1483 | newCentralIndex.clear(); |
1484 | centralIndexMade = now(); |
1485 | centralIndexMadeIn = now()-time; |
1486 | } |
1487 | |
1488 | svoid addToNewCentralIndex(Slice slice) { |
1489 | for (S s : sliceDump_pageNamesIncludingKeys(sliceDumpFile(slice))) |
1490 | newCentralIndexGet(s).slices.add(slice); |
1491 | } |
Began life as a copy of #1023558
download show line numbers debug dex old transpilations
Travelled to 6 computer(s): bhatertpkbcr, mqqgnosmbjvj, pyentgdyhuwx, pzhvpgtvlbxg, tvejysmllsmz, vouqrxazstgt
No comments. add comment
Snippet ID: | #1024477 |
Snippet name: | agi.blue source [backup before moving to class] |
Eternal ID of this version: | #1024477/1 |
Text MD5: | 43b4c96787b0c3f52661171b14749fa8 |
Author: | stefan |
Category: | javax / html |
Type: | JavaX module (desktop) |
Public (visible to everyone): | Yes |
Archived (hidden from active list): | No |
Created/modified: | 2019-08-13 17:42:28 |
Source code size: | 54323 bytes / 1491 lines |
Pitched / IR pitched: | No / No |
Views / Downloads: | 278 / 427 |
Referenced in: | [show references] |