sbool maintainPositionalIndex = true;

// in main DB
concept Slice {
  GlobalID globalID = aGlobalIDObject();
  S caseID;
  S name;
  bool indexed = true;
  long sliceDumped; // wall time, set before dump starts

  new Ref<User> owner;

  S defaultCaseID() {
    ret uniqueFileNameUsingMD5_80_v2(name + " " + globalID);
  }

  GlobalID globalID() { ret globalID; }
}

// in any DB
concept Page {
  !include #1024467 // compact global ID
  short flags;
  
  //S globalID = aGlobalID();
  S q;

  void _onLoad() {
    S id = cget(this, 'globalID);
    if (id != null) {
      setGlobalID(id);
      cset(this, globalID := null);
    }
  }

  // for in-VM helper bots
  void quickPost(S key, S value) {
    agiBlue().new GetEntriesAndPost(_concepts).go(this, litmap(+key, +value));
  }
  
  AGIBlue agiBlue() { ret (AGIBlue) mapGet(_concepts.miscMap, 'agiBlue); }
  AGIBlue.LoadedSlice loadedSlice() { ret (AGIBlue.LoadedSlice) mapGet(_concepts.miscMap, 'loadedSlice); }
}

abstract concept AbstractEntry {
  new Ref<Page> page;
  int count;
  S key;
  S ip;
  new Ref<Signer> signer;
  
  S q() { Page p = page!; ret p == null ? null : page->q; }
}

concept Entry > AbstractEntry {
  S globalID = aGlobalID(); // TODO: migrate to object
  S value;
}

/*concept MultiLineEntry > AbstractEntry {
  GlobalID globalID = aGlobalIDObject();
  S value;
}*/

concept BlobEntry > AbstractEntry {
  GlobalID globalID = aGlobalIDObject();
  long length;
  S md5;
  S contentType;
  
  GlobalID globalID() { ret globalID; }
}

// in main DB?
concept Signer {
  S globalID = aGlobalID();
  S publicKey;
  bool trusted;
  S approvedBy;
}

concept User {
  GlobalID globalID = aGlobalIDObject();
  long lastSeen;
}

concept CookieUser > User {
  S cookie;

  toString { ret "[cookieUser]"; }
}

concept GoogleUser > User {
  long googleLogInDate;
  S googleEmail, googleFirstName, googleLastName;
  bool googleEmailVerified;

  toString { ret googleFirstName + " " + googleLastName; }
}

concept MinimalUser > User {
  S userName, encryptedPassword; // TODO

  toString { ret "[minimalUser] " + userName; }
}

// supply password with _pass
concept JustPasswordUser > User {
  S password;
}

// in main DB
concept Session {
  S cookie;
  S selectedSlice; // global ID
  Page slicePage; // phasing out
  new Ref<User> user;
}

// singleton concept in every DB
concept SliceInfo {
  GlobalID globalID;
  LS mandatoryBotVMBusIDs;
  bool openPosting = true; // allow posting by anyone
}

sclass CentralIndexEntry {
  Set<Slice> slices = synchroLinkedHashSet();
}

abstract sclass AGIBlue {
  // Customizations:
  
  bool asyncSearch = false;
  bool allowMultipleCasesInValues = true;
  bool showSliceSelector = true;
  bool showGoogleLogIn = true;
  bool makeAllKeysPages = true;
  bool makeAllValuesPages = true; 
  bool legacy = true; // We still have slices without applied fixes
  bool dumpSlicesOnAnyChange = true; // testing
  
  int maxSliceNameLength = 1024;
  int displayLength = 140;
  int sideDisplayLength = 50; // when in side bar
  int searchResultsToShow = 50;
  S sideSearchType = 'literal;
  int maxBackgroundSlices = 1000;
  
  // end of customizations
  
  long cloningSince, clonedSince;
  bool isClone;
  
  // general methods

  File sliceDumpFile(Slice slice) {
    ret programFile("slice-dumps/" + or2(slice.caseID, "main") + ".slice");
  }
  
  File dumpSliceToFile(LoadedSlice slice) {
    ret withDBLock(slice.cc, func -> File {
      cset(slice.sliceConcept, sliceDumped := now());
      File file = sliceDumpFile(slice.sliceConcept);
      new LS lines;
      for (Page p : list(slice.cc, Page))
        lines.add(quote(p.q));
      for (Entry e : list(slice.cc, Entry))
        lines.add(sfu(tripleToList(entryToTriple(e))));
      saveLinesAsTextFile(file, lines);
      ret file;
    });
  }
  
  Page pageFromQ(Concepts cc, S q) {
    ret empty(q) ? null : uniqCI_sync(cc, Page, +q);
  }
  
    // executed by rst_index
  void makeCentralIndex() {
    newCentralIndex.clear();
    long time = now();
    pcall {
      for (Slice slice : conceptsWhere(Slice, indexed := true))
        addToNewCentralIndex(slice);
    }
    centralIndex.putAll(newCentralIndex);
    removeAllKeysNotIn(centralIndex, newCentralIndex);
    newCentralIndex.clear();
    centralIndexMade = now();
    centralIndexMadeIn = now()-time;
  }
  
  void addToNewCentralIndex(Slice slice) {
    for (S s : sliceDump_pageNamesIncludingKeys(sliceDumpFile(slice)))
      newCentralIndexGet(s).slices.add(slice);
  }

  CentralIndexEntry centralIndexGet(S s) { ret getOrCreate CentralIndexEntry(centralIndex, s); }
  
  CentralIndexEntry newCentralIndexGet(S s) { ret getOrCreate CentralIndexEntry(newCentralIndex, s); }
  
  Cl<Slice> centralIndexGetSlices(S s) {
    CentralIndexEntry e = centralIndexGet(s);
    ret e == null ? null : e.slices;
  }
  
  void dbLog(O... params) {
    logStructure(programFile("db.log"), ll(params));
  }
  
  class LoadedSlice { // non-persistent class
    S caseID;
    Concepts cc;
    Slice sliceConcept;
    SliceInfo sliceInfo;
    long lastAccess = sysNow();
    ReliableSingleThread rstDumpSlice = new(r {
      dumpSliceToFile(LoadedSlice.this);
      rst_index.trigger(); // rebuild index too - TODO: only this slice
    });
    bool haltSliceDumping;

    // indices
    
    ConceptFieldIndexDesc idx_latestEntries, idx_latestCreatedPages, idx_latestChangedPages;

    ReliableSingleThread rstMakePositionalIndex = new(r makePositionalIndex_impl);
    PositionalTokenIndex2 positionalIndex;
    
    void makePositionalIndex_impl {
      time "Making positional index" {
        positionalIndex = positionalTokenIndex2_wordTok(collect q(list(cc, Page)));
      }
    }
    
    *(S *caseID, Concepts *cc) {
      csetAll(list(cc, Page), slice := null, url := null); // clear legacy fields
      releaseEmptyFieldValuesOfAllConcepts(cc);
      indexThings();
      
      // forward changes to Slice concept
      onConceptsChange(cc, rOnceAtATimeOnly(r {
        cset(sliceConcept, _modified := now())
  ;
        if (!haltSliceDumping && dumpSlicesOnAnyChange && nempty(caseID)) {
          rstDumpSlice.trigger();
        }
        if (maintainPositionalIndex) rstMakePositionalIndex.trigger();
      }));
      
      if (makeAllKeysPages && legacy)
        for (Entry e : list(cc, Entry))
          pageFromQ(e.key);
          
      if (makeAllValuesPages && legacy)
        for (Entry e : list(cc, Entry))
          pageFromQ(e.value);
    }
    
    void initialSetup(GlobalID globalID) {
      sliceInfo = uniq(cc, SliceInfo);
      cset(sliceInfo, +globalID);
    }
    
    SliceInfo sliceInfo() { ret sliceInfo; }
    
    // create if not there
    Page pageFromQ(S q) {
      ret AGIBlue.this.pageFromQ(cc, q);
    }
  
    void indexThings() {
      indexConceptFieldsCI(cc, Page, 'q, Entry, 'key, Entry, 'value);
      indexConceptFields(cc, Signer, 'publicKey);
      indexConceptFields(cc, Session, 'cookie);
      indexSingletonConcept(cc, SliceInfo);
      idx_latestCreatedPages = new ConceptFieldIndexDesc(cc, Page, 'created);
      idx_latestChangedPages = new ConceptFieldIndexDesc(cc, Page, '_modified);
      idx_latestEntries = new ConceptFieldIndexDesc(cc, Entry, 'created);
      
      if (maintainPositionalIndex) rstMakePositionalIndex.trigger();
    }
    
    bool isMainSlice() { ret empty(caseID); }
    GlobalID globalID() { ret sliceConcept.globalID; }
    S name() { ret sliceConcept.name; }
  }
  
  // global vars
  
  ConceptsLoadedOnDemand fan;
  ConceptsLoadedOnDemand backgroundFan;
  Map<S, LoadedSlice> loadedSlices = syncMap();
  Map<S, LoadedSlice> backgroundSlices = syncMap();
  Map<S, CentralIndexEntry> centralIndex = ciMap();
  Map<S, CentralIndexEntry> newCentralIndex = ciMap();
  long centralIndexMade; // wall time
  long centralIndexMadeIn;
  ReliableSingleThread rst_index = new(r makeCentralIndex);
  ConceptFieldIndexDesc idx_slicesByModification, idx_usersBySeen;
  ConceptFieldIndexCI idx_slicesByName, idx_usersByName;
  
  class GetEntriesAndPost {
    Concepts cc;
    L<Entry> entries;
    Entry entry;
    bool newEntry;
    Signer signer;
    Session session;
    
    *(Concepts *cc, Session *session) {} // migrate to this constructor
    *(Concepts *cc) {}
    
    GetEntriesAndPost go(Page page, SS params) {
      S key = trim(params.get('key)), value = trim(params.get('value));
      print("GetEntriesAndPost: " + quote(page.q) + ", " + quote(key) + " := " + quote(value));
      withDBLock {
        entries = findBackRefs(page, Entry);
        if (nempty(key) && nempty(value)) {
          Slice slice = page.loadedSlice().sliceConcept;
          SliceInfo sliceInfo = page.loadedSlice().sliceInfo(); 
          if (!sliceInfo.openPosting && session != null && session.user! != slice.owner!)
            fail("Not owner of slice");
          S ip = subBot_clientIP();
          entry = firstThat(e -> eqic(e.key, key) && eq_icIf(!allowMultipleCasesInValues, e.value, value) && eq(e.ip, ip), entries);
          if (entry == null) {
            print("SAVING");
            Entry e = cnew(cc, Entry, +page, +key, +value, +ip, count := l(entries) + 1, +signer);
            if (makeAllKeysPages) pageFromQ(cc, key);
            if (makeAllValuesPages) pageFromQ(cc, value);
            page.change(); // bump modification date
            entry = e;
            newEntry = true;
            entries.add(entry);
            dbLog("New entry", page := page.q, globalID := e.globalID, count := e.count, +key, +value);
          }
        }
      }
  
      sortByFieldInPlace created(entries);
      numberEntriesInConceptField count(entries);
      this;
    }
  }

  T3S entryToTriple(Entry e) {
    ret e == null ? null : t3(e.page->q, e.key, e.value);
  }
  
} // end of AGIBlue