sbool maintainPositionalIndex = true; // in main DB concept Slice { GlobalID globalID = aGlobalIDObject(); S caseID; S name; bool indexed = true; long sliceDumped; // wall time, set before dump starts new Ref owner; S defaultCaseID() { ret uniqueFileNameUsingMD5_80_v2(name + " " + globalID); } GlobalID globalID() { ret globalID; } } // in any DB concept Page { !include #1024467 // compact global ID short flags; //S globalID = aGlobalID(); S q; void _onLoad() { S id = cget(this, 'globalID); if (id != null) { setGlobalID(id); cset(this, globalID := null); } } // for in-VM helper bots void quickPost(S key, S value) { agiBlue().new GetEntriesAndPost(_concepts).go(this, litmap(+key, +value)); } AGIBlue agiBlue() { ret (AGIBlue) mapGet(_concepts.miscMap, 'agiBlue); } AGIBlue.LoadedSlice loadedSlice() { ret (AGIBlue.LoadedSlice) mapGet(_concepts.miscMap, 'loadedSlice); } } abstract concept AbstractEntry { new Ref page; int count; S key; S ip; new Ref signer; S q() { Page p = page!; ret p == null ? null : page->q; } } concept Entry > AbstractEntry { S globalID = aGlobalID(); // TODO: migrate to object S value; } /*concept MultiLineEntry > AbstractEntry { GlobalID globalID = aGlobalIDObject(); S value; }*/ concept BlobEntry > AbstractEntry { GlobalID globalID = aGlobalIDObject(); long length; S md5; S contentType; GlobalID globalID() { ret globalID; } } // in main DB? concept Signer { S globalID = aGlobalID(); S publicKey; bool trusted; S approvedBy; } concept User { GlobalID globalID = aGlobalIDObject(); long lastSeen; } concept CookieUser > User { S cookie; toString { ret "[cookieUser]"; } } concept GoogleUser > User { long googleLogInDate; S googleEmail, googleFirstName, googleLastName; bool googleEmailVerified; toString { ret googleFirstName + " " + googleLastName; } } concept MinimalUser > User { S userName, encryptedPassword; // TODO toString { ret "[minimalUser] " + userName; } } // supply password with _pass concept JustPasswordUser > User { S password; } // in main DB concept Session { S cookie; S selectedSlice; // global ID Page slicePage; // phasing out new Ref user; } // singleton concept in every DB concept SliceInfo { GlobalID globalID; LS mandatoryBotVMBusIDs; bool openPosting = true; // allow posting by anyone } sclass CentralIndexEntry { Set slices = synchroLinkedHashSet(); } abstract sclass AGIBlue { // Customizations: bool asyncSearch = false; bool allowMultipleCasesInValues = true; bool showSliceSelector = true; bool showGoogleLogIn = true; bool makeAllKeysPages = true; bool makeAllValuesPages = true; bool legacy = true; // We still have slices without applied fixes bool dumpSlicesOnAnyChange = true; // testing int maxSliceNameLength = 1024; int displayLength = 140; int sideDisplayLength = 50; // when in side bar int searchResultsToShow = 50; S sideSearchType = 'literal; int maxBackgroundSlices = 1000; // end of customizations long cloningSince, clonedSince; bool isClone; // general methods File sliceDumpFile(Slice slice) { ret programFile("slice-dumps/" + or2(slice.caseID, "main") + ".slice"); } File dumpSliceToFile(LoadedSlice slice) { ret withDBLock(slice.cc, func -> File { cset(slice.sliceConcept, sliceDumped := now()); File file = sliceDumpFile(slice.sliceConcept); new LS lines; for (Page p : list(slice.cc, Page)) lines.add(quote(p.q)); for (Entry e : list(slice.cc, Entry)) lines.add(sfu(tripleToList(entryToTriple(e)))); saveLinesAsTextFile(file, lines); ret file; }); } Page pageFromQ(Concepts cc, S q) { ret empty(q) ? null : uniqCI_sync(cc, Page, +q); } // executed by rst_index void makeCentralIndex() { newCentralIndex.clear(); long time = now(); pcall { for (Slice slice : conceptsWhere(Slice, indexed := true)) addToNewCentralIndex(slice); } centralIndex.putAll(newCentralIndex); removeAllKeysNotIn(centralIndex, newCentralIndex); newCentralIndex.clear(); centralIndexMade = now(); centralIndexMadeIn = now()-time; } void addToNewCentralIndex(Slice slice) { for (S s : sliceDump_pageNamesIncludingKeys(sliceDumpFile(slice))) newCentralIndexGet(s).slices.add(slice); } CentralIndexEntry centralIndexGet(S s) { ret getOrCreate CentralIndexEntry(centralIndex, s); } CentralIndexEntry newCentralIndexGet(S s) { ret getOrCreate CentralIndexEntry(newCentralIndex, s); } Cl centralIndexGetSlices(S s) { CentralIndexEntry e = centralIndexGet(s); ret e == null ? null : e.slices; } void dbLog(O... params) { logStructure(programFile("db.log"), ll(params)); } class LoadedSlice { // non-persistent class S caseID; Concepts cc; Slice sliceConcept; SliceInfo sliceInfo; long lastAccess = sysNow(); ReliableSingleThread rstDumpSlice = new(r { dumpSliceToFile(LoadedSlice.this); rst_index.trigger(); // rebuild index too - TODO: only this slice }); bool haltSliceDumping; // indices ConceptFieldIndexDesc idx_latestEntries, idx_latestCreatedPages, idx_latestChangedPages; ReliableSingleThread rstMakePositionalIndex = new(r makePositionalIndex_impl); PositionalTokenIndex2 positionalIndex; void makePositionalIndex_impl { time "Making positional index" { positionalIndex = positionalTokenIndex2_wordTok(collect q(list(cc, Page))); } } *(S *caseID, Concepts *cc) { csetAll(list(cc, Page), slice := null, url := null); // clear legacy fields releaseEmptyFieldValuesOfAllConcepts(cc); indexThings(); // forward changes to Slice concept onConceptsChange(cc, rOnceAtATimeOnly(r { cset(sliceConcept, _modified := now()) ; if (!haltSliceDumping && dumpSlicesOnAnyChange && nempty(caseID)) { rstDumpSlice.trigger(); } if (maintainPositionalIndex) rstMakePositionalIndex.trigger(); })); if (makeAllKeysPages && legacy) for (Entry e : list(cc, Entry)) pageFromQ(e.key); if (makeAllValuesPages && legacy) for (Entry e : list(cc, Entry)) pageFromQ(e.value); } void initialSetup(GlobalID globalID) { sliceInfo = uniq(cc, SliceInfo); cset(sliceInfo, +globalID); } SliceInfo sliceInfo() { ret sliceInfo; } // create if not there Page pageFromQ(S q) { ret AGIBlue.this.pageFromQ(cc, q); } void indexThings() { indexConceptFieldsCI(cc, Page, 'q, Entry, 'key, Entry, 'value); indexConceptFields(cc, Signer, 'publicKey); indexConceptFields(cc, Session, 'cookie); indexSingletonConcept(cc, SliceInfo); idx_latestCreatedPages = new ConceptFieldIndexDesc(cc, Page, 'created); idx_latestChangedPages = new ConceptFieldIndexDesc(cc, Page, '_modified); idx_latestEntries = new ConceptFieldIndexDesc(cc, Entry, 'created); if (maintainPositionalIndex) rstMakePositionalIndex.trigger(); } bool isMainSlice() { ret empty(caseID); } GlobalID globalID() { ret sliceConcept.globalID; } S name() { ret sliceConcept.name; } } // global vars ConceptsLoadedOnDemand fan; ConceptsLoadedOnDemand backgroundFan; Map loadedSlices = syncMap(); Map backgroundSlices = syncMap(); Map centralIndex = ciMap(); Map newCentralIndex = ciMap(); long centralIndexMade; // wall time long centralIndexMadeIn; ReliableSingleThread rst_index = new(r makeCentralIndex); ConceptFieldIndexDesc idx_slicesByModification, idx_usersBySeen; ConceptFieldIndexCI idx_slicesByName, idx_usersByName; class GetEntriesAndPost { Concepts cc; L entries; Entry entry; bool newEntry; Signer signer; Session session; *(Concepts *cc, Session *session) {} // migrate to this constructor *(Concepts *cc) {} GetEntriesAndPost go(Page page, SS params) { S key = trim(params.get('key)), value = trim(params.get('value)); print("GetEntriesAndPost: " + quote(page.q) + ", " + quote(key) + " := " + quote(value)); withDBLock { entries = findBackRefs(page, Entry); if (nempty(key) && nempty(value)) { Slice slice = page.loadedSlice().sliceConcept; SliceInfo sliceInfo = page.loadedSlice().sliceInfo(); if (!sliceInfo.openPosting && session != null && session.user! != slice.owner!) fail("Not owner of slice"); S ip = subBot_clientIP(); entry = firstThat(e -> eqic(e.key, key) && eq_icIf(!allowMultipleCasesInValues, e.value, value) && eq(e.ip, ip), entries); if (entry == null) { print("SAVING"); Entry e = cnew(cc, Entry, +page, +key, +value, +ip, count := l(entries) + 1, +signer); if (makeAllKeysPages) pageFromQ(cc, key); if (makeAllValuesPages) pageFromQ(cc, value); page.change(); // bump modification date entry = e; newEntry = true; entries.add(entry); dbLog("New entry", page := page.q, globalID := e.globalID, count := e.count, +key, +value); } } } sortByFieldInPlace created(entries); numberEntriesInConceptField count(entries); this; } } T3S entryToTriple(Entry e) { ret e == null ? null : t3(e.page->q, e.key, e.value); } } // end of AGIBlue