sbool maintainPositionalIndex = true;

// in main DB

concept Slice {

  GlobalID globalID = aGlobalIDObject();

  S caseID;

  S name;

  bool indexed = true;

  long sliceDumped; // wall time, set before dump starts

  new Ref<User> owner;

  S defaultCaseID() {

    ret uniqueFileNameUsingMD5_80_v2(name + " " + globalID);

  }

  GlobalID globalID() { ret globalID; }

}

// in any DB

concept Page {

  !include #1024467 // compact global ID

  short flags;

  //S globalID = aGlobalID();

  S q;

  void _onLoad() {

    S id = cget(this, 'globalID);

    if (id != null) {

      setGlobalID(id);

      cset(this, globalID := null);

    }

  }

  // for in-VM helper bots

  void quickPost(S key, S value) {

    agiBlue().new GetEntriesAndPost(_concepts).go(this, litmap(+key, +value));

  }

  AGIBlue agiBlue() { ret (AGIBlue) mapGet(_concepts.miscMap, 'agiBlue); }

  AGIBlue.LoadedSlice loadedSlice() { ret (AGIBlue.LoadedSlice) mapGet(_concepts.miscMap, 'loadedSlice); }

}

abstract concept AbstractEntry {

  new Ref<Page> page;

  int count;

  S key;

  S ip;

  new Ref<Signer> signer;

  S q() { Page p = page!; ret p == null ? null : page->q; }

}

concept Entry > AbstractEntry {

  S globalID = aGlobalID(); // TODO: migrate to object

  S value;

}

/*concept MultiLineEntry > AbstractEntry {

  GlobalID globalID = aGlobalIDObject();

  S value;

}*/

concept BlobEntry > AbstractEntry {

  GlobalID globalID = aGlobalIDObject();

  long length;

  S md5;

  S contentType;

  GlobalID globalID() { ret globalID; }

}

// in main DB?

concept Signer {

  S globalID = aGlobalID();

  S publicKey;

  bool trusted;

  S approvedBy;

}

concept User {

  GlobalID globalID = aGlobalIDObject();

  long lastSeen;

}

concept CookieUser > User {

  S cookie;

  toString { ret "[cookieUser]"; }

}

concept GoogleUser > User {

  long googleLogInDate;

  S googleEmail, googleFirstName, googleLastName;

  bool googleEmailVerified;

  toString { ret googleFirstName + " " + googleLastName; }

}

concept MinimalUser > User {

  S userName, encryptedPassword; // TODO

  toString { ret "[minimalUser] " + userName; }

}

// supply password with _pass

concept JustPasswordUser > User {

  S password;

}

// in main DB

concept Session {

  S cookie;

  S selectedSlice; // global ID

  Page slicePage; // phasing out

  new Ref<User> user;

}

// singleton concept in every DB

concept SliceInfo {

  GlobalID globalID;

  LS mandatoryBotVMBusIDs;

  bool openPosting = true; // allow posting by anyone

}

sclass CentralIndexEntry {

  Set<Slice> slices = synchroLinkedHashSet();

}

abstract sclass AGIBlue {

  // Customizations:

  bool asyncSearch = false;

  bool allowMultipleCasesInValues = true;

  bool showSliceSelector = true;

  bool showGoogleLogIn = true;

  bool makeAllKeysPages = true;

  bool makeAllValuesPages = true; 

  bool legacy = true; // We still have slices without applied fixes

  bool dumpSlicesOnAnyChange = true; // testing

  int maxSliceNameLength = 1024;

  int displayLength = 140;

  int sideDisplayLength = 50; // when in side bar

  int searchResultsToShow = 50;

  S sideSearchType = 'literal;

  int maxBackgroundSlices = 1000;

  // end of customizations

  long cloningSince, clonedSince;

  bool isClone;

  // general methods

  File sliceDumpFile(Slice slice) {

    ret programFile("slice-dumps/" + or2(slice.caseID, "main") + ".slice");

  }

  File dumpSliceToFile(LoadedSlice slice) {

    ret withDBLock(slice.cc, func -> File {

      cset(slice.sliceConcept, sliceDumped := now());

      File file = sliceDumpFile(slice.sliceConcept);

      new LS lines;

      for (Page p : list(slice.cc, Page))

        lines.add(quote(p.q));

      for (Entry e : list(slice.cc, Entry))

        lines.add(sfu(tripleToList(entryToTriple(e))));

      saveLinesAsTextFile(file, lines);

      ret file;

    });

  }

  Page pageFromQ(Concepts cc, S q) {

    ret empty(q) ? null : uniqCI_sync(cc, Page, +q);

  }

    // executed by rst_index

  void makeCentralIndex() {

    newCentralIndex.clear();

    long time = now();

    pcall {

      for (Slice slice : conceptsWhere(Slice, indexed := true))

        addToNewCentralIndex(slice);

    }

    centralIndex.putAll(newCentralIndex);

    removeAllKeysNotIn(centralIndex, newCentralIndex);

    newCentralIndex.clear();

    centralIndexMade = now();

    centralIndexMadeIn = now()-time;

  }

  void addToNewCentralIndex(Slice slice) {

    for (S s : sliceDump_pageNamesIncludingKeys(sliceDumpFile(slice)))

      newCentralIndexGet(s).slices.add(slice);

  }

  CentralIndexEntry centralIndexGet(S s) { ret getOrCreate CentralIndexEntry(centralIndex, s); }

  CentralIndexEntry newCentralIndexGet(S s) { ret getOrCreate CentralIndexEntry(newCentralIndex, s); }

  Cl<Slice> centralIndexGetSlices(S s) {

    CentralIndexEntry e = centralIndexGet(s);

    ret e == null ? null : e.slices;

  }

  void dbLog(O... params) {

    logStructure(programFile("db.log"), ll(params));

  }

  class LoadedSlice { // non-persistent class

    S caseID;

    Concepts cc;

    Slice sliceConcept;

    SliceInfo sliceInfo;

    long lastAccess = sysNow();

    ReliableSingleThread rstDumpSlice = new(r {

      dumpSliceToFile(LoadedSlice.this);

      rst_index.trigger(); // rebuild index too - TODO: only this slice

    });

    bool haltSliceDumping;

    // indices

    ConceptFieldIndexDesc idx_latestEntries, idx_latestCreatedPages, idx_latestChangedPages;

    ReliableSingleThread rstMakePositionalIndex = new(r makePositionalIndex_impl);

    PositionalTokenIndex2 positionalIndex;

    void makePositionalIndex_impl {

      time "Making positional index" {

        positionalIndex = positionalTokenIndex2_wordTok(collect q(list(cc, Page)));

      }

    }

    *(S *caseID, Concepts *cc) {

      csetAll(list(cc, Page), slice := null, url := null); // clear legacy fields

      releaseEmptyFieldValuesOfAllConcepts(cc);

      indexThings();

      // forward changes to Slice concept

      onConceptsChange(cc, rOnceAtATimeOnly(r {

        cset(sliceConcept, _modified := now())

  ;

        if (!haltSliceDumping && dumpSlicesOnAnyChange && nempty(caseID)) {

          rstDumpSlice.trigger();

        }

        if (maintainPositionalIndex) rstMakePositionalIndex.trigger();

      }));

      if (makeAllKeysPages && legacy)

        for (Entry e : list(cc, Entry))

          pageFromQ(e.key);

      if (makeAllValuesPages && legacy)

        for (Entry e : list(cc, Entry))

          pageFromQ(e.value);

    }

    void initialSetup(GlobalID globalID) {

      sliceInfo = uniq(cc, SliceInfo);

      cset(sliceInfo, +globalID);

    }

    SliceInfo sliceInfo() { ret sliceInfo; }

    // create if not there

    Page pageFromQ(S q) {

      ret AGIBlue.this.pageFromQ(cc, q);

    }

    void indexThings() {

      indexConceptFieldsCI(cc, Page, 'q, Entry, 'key, Entry, 'value);

      indexConceptFields(cc, Signer, 'publicKey);

      indexConceptFields(cc, Session, 'cookie);

      indexSingletonConcept(cc, SliceInfo);

      idx_latestCreatedPages = new ConceptFieldIndexDesc(cc, Page, 'created);

      idx_latestChangedPages = new ConceptFieldIndexDesc(cc, Page, '_modified);

      idx_latestEntries = new ConceptFieldIndexDesc(cc, Entry, 'created);

      if (maintainPositionalIndex) rstMakePositionalIndex.trigger();

    }

    bool isMainSlice() { ret empty(caseID); }

    GlobalID globalID() { ret sliceConcept.globalID; }

    S name() { ret sliceConcept.name; }

  }

  // global vars

  ConceptsLoadedOnDemand fan;

  ConceptsLoadedOnDemand backgroundFan;

  Map<S, LoadedSlice> loadedSlices = syncMap();

  Map<S, LoadedSlice> backgroundSlices = syncMap();

  Map<S, CentralIndexEntry> centralIndex = ciMap();

  Map<S, CentralIndexEntry> newCentralIndex = ciMap();

  long centralIndexMade; // wall time

  long centralIndexMadeIn;

  ReliableSingleThread rst_index = new(r makeCentralIndex);

  ConceptFieldIndexDesc idx_slicesByModification, idx_usersBySeen;

  ConceptFieldIndexCI idx_slicesByName, idx_usersByName;

  class GetEntriesAndPost {

    Concepts cc;

    L<Entry> entries;

    Entry entry;

    bool newEntry;

    Signer signer;

    Session session;

    *(Concepts *cc, Session *session) {} // migrate to this constructor

    *(Concepts *cc) {}

    GetEntriesAndPost go(Page page, SS params) {

      S key = trim(params.get('key)), value = trim(params.get('value));

      print("GetEntriesAndPost: " + quote(page.q) + ", " + quote(key) + " := " + quote(value));

      withDBLock {

        entries = findBackRefs(page, Entry);

        if (nempty(key) && nempty(value)) {

          Slice slice = page.loadedSlice().sliceConcept;

          SliceInfo sliceInfo = page.loadedSlice().sliceInfo(); 

          if (!sliceInfo.openPosting && session != null && session.user! != slice.owner!)

            fail("Not owner of slice");

          S ip = subBot_clientIP();

          entry = firstThat(e -> eqic(e.key, key) && eq_icIf(!allowMultipleCasesInValues, e.value, value) && eq(e.ip, ip), entries);

          if (entry == null) {

            print("SAVING");

            Entry e = cnew(cc, Entry, +page, +key, +value, +ip, count := l(entries) + 1, +signer);

            if (makeAllKeysPages) pageFromQ(cc, key);

            if (makeAllValuesPages) pageFromQ(cc, value);

            page.change(); // bump modification date

            entry = e;

            newEntry = true;

            entries.add(entry);

            dbLog("New entry", page := page.q, globalID := e.globalID, count := e.count, +key, +value);

          }

        }

      }

      sortByFieldInPlace created(entries);

      numberEntriesInConceptField count(entries);

      this;

    }

  }

  T3S entryToTriple(Entry e) {

    ret e == null ? null : t3(e.page->q, e.key, e.value);

  }

} // end of AGIBlue