// A database system for only slightly modified Java objects.

// Recent changes:
// -minimal crash recovery disabled for now (could reenable)
// -idCounter.structure file no longer used
// -allDynamic/safeLoad disabled, it doesn't really make any sense
// -conceptsFile can be in any directory now

// Functions that should always be there for child processes:
please include function dbLock.

static int concepts_internStringsLongerThan = 10;

static new ThreadLocal<Bool> concepts_unlisted;

// BREAKING CHANGE 2021/6/7 set to true
sbool concepts_unlistedByDefault = true; // true = we can create instances of concepts with "new" without registering them automatically

interface IConceptIndex {
  void update(Concept c); // also for adding
  void remove(Concept c);
}

interface IFieldIndex<A extends Concept, Val> {
  Cl<A> getAll(Val val);
  L<Val> allValues(); // returns a cloned list
  MultiSet<Val> allValues_multiSet();
  ItIt<A> objectIterator();
}

// Approach to persisting the Concepts object itself (in normal
// DB operation, this is not done): For simplification, speed and
// compactness, we make almost all the fields transient and store only // the concepts and the idCounter. To unstructure the Concepts object,
// use unstructureConcepts() or postUnstructureConcepts(), then
// re-set up any indices, listeners etc.

// base class indicating nothing
sclass ConceptsChange {}

// change of a single concept
srecord ConceptCreate(Concept c) extends ConceptsChange {}

// change of a single concept
srecord ConceptChange(Concept c) extends ConceptsChange {}

// removal of a single concept
// c.id is going to become 0 at some point, so we pass the
// id separately
srecord ConceptDelete(long id, Concept c) extends ConceptsChange {}

// unknown change anywhere in concepts; consider it all dirty
// (this one should not be used except for batch jobs)
srecord FullChange extends ConceptsChange {}

sclass Concepts implements AutoCloseable {
  NavigableMap<Long, Concept> concepts = synchroTreeMap();
  
  // ID of last created concept
  gettable long idCounter;
  
  transient HashMap<Class, O> perClassData;
  transient Map miscMap; // don't use directly, call miscMap... methods to access
  
  // set to "-" for non-persistent (possibly not implemented)
  // also, can include a case ID ("#123/1")
  // TODO: phase out (we have conceptsFile field now)
  transient S programID;
  transient File conceptsFile;
  
  transient Concepts parent; // new mechanism
  transient volatile long changes, changesWritten, lastChange;
  transient volatile java.util.Timer autoSaver;
  transient volatile bool dontSave;
  transient volatile bool savingConcepts, noXFullGrab;
  transient bool vmBusSend = true;
  transient bool initialSave = false; // set to false to avoid initial useless saving
  transient int autoSaveInterval = -1000; // 1 second + wait logic
  transient bool useGZIP = true, quietSave;
  transient ReentrantLock lock = new ReentrantLock(true);
  transient ReentrantLock saverLock = new ReentrantLock(true);
  transient long lastSaveTook = -1, lastSaveWas, loadTook, uncompressedSize;
  transient float maxAutoSavePercentage = 10;
  transient L<IConceptIndex> conceptIndices;
  transient Map<Class<? extends Concept>, Map<S, IFieldIndex>> fieldIndices;
  transient Map<Class<? extends Concept>, Map<S, IFieldIndex>> ciFieldIndices;
  //transient L saveActions = synchroList();
  transient L<Runnable> preSave;
  transient O classFinder = _defaultClassFinder();
  transient L onAllChanged = synchroList(); // list of runnables
  transient new Set<IVF1> onChange;
  transient O saveWrapper; // VF1<Runnable>, to profile saving
  settable transient bool modifyOnCreate; // set _modified == created initially
  transient bool modifyOnBackRef; // set modified if back refs change
  transient bool useFileLock = true; // instead of locking by bot
  settable transient bool grabThroughSocket = true; // probably never want to do this again actually
  // OLD - not done anymore. transient bool collectForwardRefs
  transient FileBasedLock fileLock;
  transient bool storeBaseClassesInStructure; // helps with schema evolution when concept subclasses disappear
  transient bool useBackRefsForSearches; // assume backRefs are sane in order to speed up searches
  transient bool defunct;
  transient int newBackupEveryXMinutes = 60;
  settable transient bool closeConceptsOnExit;
  settable transient Throwable saveError;
  event fireSaveError(Throwable error);
  
  // add more fields for Concepts here
  
  *() {}
  *(S *programID) {}
  *(File *conceptsFile) {}
  
  synchronized long internalID() {
    do {
      ++idCounter;
    } while (hasConcept(idCounter));
    ret idCounter;
  }
  
  synchronized HashMap<Class, O> perClassData() {
    if (perClassData == null) perClassData = new HashMap;
    ret perClassData;
  }
  
  void initProgramID() {
    if (programID == null)
      programID = getDBProgramID();
  }
  
  // load from structure
  Concepts load(S structure, bool allDynamic default false) {
    clearConcepts();
    Map<Long, Concept> map = unstructureMap(structure, allDynamic, classFinder);
    concepts.putAll(map);
    assignConceptsToUs();
    calcIdCounter();
    this;
  }
  
  Concepts load() {
    initProgramID();
    
    // try custom grabber
    O dbGrabber = miscMapGet("dbGrabber");
    if (dbGrabber != null && !isFalse(callF(dbGrabber)))
      this;

    try {
      if (grabThroughSocket && tryToGrab()) ret this;
    } catch e {
      if (!exceptionMessageContains(e, "no xfullgrab"))
        printShortException(e);
      print("xfullgrab failed - loading DB of " + programID + " from disk");
    }
    ret loadFromDisk();
  }
  
  Concepts loadFromDisk() {
    if (nempty(concepts)) clearConcepts();
    
    // minimal crash recovery (disabled for now)
    //restoreLatestBackupIfConceptsFileEmpty(programID, doIt := true);

    long time = now();
    Map<Long, Concept> _concepts = cast unstructureGZFile(conceptsFile(), toIF1(classFinder));
    putAll(concepts, _concepts);
    assignConceptsToUs();
    loadTook = now()-time;
    done("Loaded " + n2(l(concepts), "concept"), time);
    calcIdCounter();
    this;
  }
  
  Concepts loadConcepts() { ret load(); }
  
  bool tryToGrab() {
    if (sameSnippetID(programID, getDBProgramID())) false;
    temp RemoteDB db = connectToDBOpt(programID);
    if (db != null) {
      loadGrab(db.fullgrab());
      true;
    }
    false;
  }
  
  Concepts loadGrab(S grab) {
    clearConcepts();
    DynamicObject_loading.set(true);
    try {
      Map<Long, Concept> map = (Map) unstructure(grab, false, classFinder);
      concepts.putAll(map);
      assignConceptsToUs();
      for (long l : map.keySet())
        idCounter = max(idCounter, l);
    } finally {
      DynamicObject_loading.set(null);
    }
    //XXX allChanged(); // Nobody is listening at this point anyway
    ret this;
  }
  
  void assignConceptsToUs() {
    // fix unstructure bugs
    
    for (Pair<Long, O> p: mapToPairs((Map<Long, O>) (Map) concepts))
      if (!(p.b instanceof Concept)) {
       print("DROPPING non-existant concept " + p.a + ": " + dynShortName(p.b));
       concepts.remove(p.a);
    }
    
    for (Concept c : values(concepts)) c._concepts = this;
    for (Concept c : values(concepts))
      c._doneLoading2(); // doneLoading2 is called on all concepts after all concepts are loaded
  }

  S progID() {
    ret programID == null ? getDBProgramID() : programID;
  }
  
  Concept getConcept(S id) {
    ret empty(id) ? null : getConcept(parseLong(id));
  }
  
  Concept getConcept(long id) {
    ret (Concept) concepts.get((long) id);
  }
  
  Concept getConcept(RC ref) {
    ret ref == null ? null : getConcept(ref.longID());
  }
  
  bool hasConcept(long id) {
    ret concepts.containsKey((long) id);
  }
  
  void deleteConcept(long id) {
    Concept c = getConcept(id);
    if (c == null)
      print("Concept " + id + " not found");
    else
      c.delete();
  }
  
  void calcIdCounter() {
    Long lastID = lastKey(concepts);
    idCounter = lastID == null ? 1 : lastID+1;
  }
  
  File conceptsDir() { ret dirOfFile(conceptsFile()); }
  
  selfType conceptsFile(File conceptsFile) { this.conceptsFile = conceptsFile; this; }
  
  File conceptsFile() {
    if (conceptsFile != null) ret conceptsFile;
    
    ret getProgramFile(programID, useGZIP ? "concepts.structure.gz" : "concepts.structure");
  }
  
  // used for locking when useFileLock is activated
  File lockFile() {
    ret newFile(conceptsDir(), "concepts.lock");
  }
  
  FileBasedLock fileLock() {
    if (fileLock == null)
      fileLock = new FileBasedLock(lockFile());
    ret fileLock;
  }
  
  void saveConceptsIfDirty() { saveConcepts(); }
  void save() { saveConcepts(); }

  void saveConcepts() {
    vmBus_send saveConceptsCalled(Concepts.this);
    if (dontSave) ret;
    initProgramID();
    saverLock.lock();
    savingConcepts = true;
    long start = now(), time;
    try {
      S s = null;
      //synchronized(main.class) {
      long _changes = changes;
      if (_changes == changesWritten) ret;
      
      File f = conceptsFile();
      
      lock.lock();
      long fullTime = now();
      try {
        if (useGZIP) {
          vmBus_send callingSaveWrapper(Concepts.this, saveWrapper);
          callRunnableWithWrapper(saveWrapper, r {
            vmBus_send callingPreSave(Concepts.this, preSave);
            callFAll(preSave);
            vmBus_send writingFile(Concepts.this, f);
            uncompressedSize = saveGZStructureToFile(f, cloneMap(concepts), makeStructureData());
            vmBus_send gzFileSaved(Concepts.this, f, uncompressedSize);
          });
          newFile(conceptsDir(), "concepts.structure").delete();
        } else
          s = fullStructure();
      } finally {
        lock.unlock();
      }
      
      /*while (nempty(saveActions))
        pcallF(popFirst(saveActions));*/

      changesWritten = _changes; // only update when structure didn't fail
      
      if (!useGZIP) {
        time = now()-start;
        if (!quietSave)
          print("Saving " + toM(l(s)) + "M chars (" /*+ changesWritten + ", "*/ + time + " ms)");
        start = now();
        saveTextFile(f, javaTokWordWrap(s));
        newFile(conceptsDir(), "concepts.structure.gz").delete();
      }
      
      File conceptsFile = conceptsFile();
      File backupFile = newFile(conceptsDir(), "backups/" + fileName(conceptsFile) + ".backup" + ymd() + "-" + formatInt(hours(), 2)
        + (newBackupEveryXMinutes >= 60 ? "" : formatInt(roundDownTo_rev(minutes(), newBackupEveryXMinutes), 2)));
      // TODO: get rid of this
      copyFile(f, backupFile);
      
      time = now()-start;
      if (!quietSave)
        print("Saved " + toK(f.length()) + " K, " + n(concepts, "concepts") + " (" + time + " ms)");
      lastSaveWas = fullTime;
      lastSaveTook = now()-fullTime;
      saveError(null);
    } finally {
      savingConcepts = false;
      saverLock.unlock();
    }
  }
  
  void _autoSaveConcepts() {
    if (autoSaveInterval < 0 && maxAutoSavePercentage != 0) {
      long pivotTime = Math.round(lastSaveWas+lastSaveTook*100.0/maxAutoSavePercentage);
      if (now() < pivotTime) {
        //print("Skipping auto-save (last save took " + lastSaveTook + ")");
        ret;
      }
    }
    try {
      saveConcepts();
    } catch e {
      saveError(e);
      print("Concept save failed, will try again");
      printStackTrace(e);
      fireSaveError(e);
    }
  }
  
  S fullStructure() {
    ret structure(cloneMap(concepts), makeStructureData());
  }
  
  swappable structure_Data makeStructureData() {
    ret finishStructureData(new structure_Data);
  }
  
  structure_Data finishStructureData(structure_Data data) {
    if (storeBaseClassesInStructure)
      data.storeBaseClasses = true;
    ret data;
  }
  
  void clearConcepts() {
    for (Concept c : allConcepts()) c.delete();
    //concepts.clear();
    //allChanged();
  }
  
  void fireLegacyChangeEvent() {
    synchronized(this) { ++changes; lastChange = sysNow(); }
    if (vmBusSend) vmBus_send conceptsChanged(this);
    pcallFAll(onAllChanged);
  }
  
  // auto-save every second if dirty
  synchronized void autoSaveConcepts() {
    if (autoSaver == null) {
      if (isTransient()) fail("Can't persist transient database");
      autoSaver = doEvery_daemon("Concepts Saver for " + conceptsDir(),
        abs(autoSaveInterval), r { _autoSaveConcepts() });
      // print("Installed auto-saver (" + autoSaveInterval + " ms, " + progID() + ")");
    }
  }
  
  public void close { cleanMeUp(); }
  
  void cleanMeUp() {
    pcall {
      if (closeConceptsOnExit)
        closeAllOpt(allConcepts());
        
      set defunct;
      bool shouldSave = autoSaver != null;
      if (autoSaver != null) {
        autoSaver.cancel();
        autoSaver = null;
      }
      while (savingConcepts) sleepInCleanUp(10);
      if (shouldSave)
        saveConceptsIfDirty();
    }
    dispose fileLock;
  }
  
  Map<Long, S> getIDsAndNames() {
    new Map<Long, S> map;
    Map<Long, Concept> cloned = cloneMap(concepts);
    for (long id : keys(cloned)) 
      map.put(id, cloned.get(id).className);
    ret map;
  }
  
  void deleteConcepts(L l) {
    ping();
    if (l != null) for (O o : cloneList(l))
      if (o instanceof Long) {
        Concept c = concepts.get(o);
        if (c != null) c.delete();
      } else if (o cast Concept)
        o.delete();
      else
        warn("Can't delete " + getClassName(o));
  }
  
  <A extends Concept> A conceptOfType(Class<A> type) {
    IConceptCounter counter = conceptCounterForClass(type);
    if (counter != null) ret (A) first(counter.allConcepts());
    ret firstOfType(allConcepts(), type);
  }
  
  <A extends Concept> L<A> conceptsOfType(Class<A> type) {
    L<A> l = conceptsOfType_noParent(type);
    if (parent == null) ret l;
    ret concatLists_conservative(l, parent.conceptsOfType(type));
  }
  
  <A extends Concept> L<A> conceptsOfType_noParent(Class<A> type) {
    ping();
    IConceptCounter counter = conceptCounterForClass(type);
    if (counter != null) ret (L<A>) cloneList(counter.allConcepts());
    ret filterByType(allConcepts(), type);
  }
  
  <A extends Concept> L<A> listConcepts(Class<A> type) {
    ret conceptsOfType(type);
  }
  
  <A extends Concept> L<A> list(Class<A> type) {
    ret conceptsOfType(type);
  }
  
  <A extends Concept> L<A> list_noParent(Class<A> type) {
    ret conceptsOfType_noParent(type);
  }
  
  // TODO: would be better to make this Cl (indices may return sets)
  L<Concept> list(S type) {
    ret conceptsOfType(type);
  }
  
  L<Concept> conceptsOfType(S type) {
    ret filterByDynamicType(allConcepts(), "main$" + type);
  }
  
  bool hasConceptOfType(Class<? extends Concept> type) {
    ret hasType(allConcepts(), type);
  }
  
  void persistConcepts() {
    loadConcepts();
    autoSaveConcepts();
  }
  
  // We love synonyms
  void conceptPersistence() { persistConcepts(); }
  
  Concepts persist() { persistConcepts(); ret this; }
  void persist(Int interval) {
    if (interval != null) autoSaveInterval = interval;
    persist();
  }
    
  // Runs r if there is no concept of that type
  <A extends Concept> A ensureHas(Class<A> c, Runnable r) {
    A a = conceptOfType(c);
    if (a == null) {
      r.run();
      a = conceptOfType(c);
      if (a == null)
        fail("Concept not made by " + r + ": " + shortClassName(c));
    }
    ret a;
  }
  
  // Ensures that every concept of type c1 is ref'd by a concept of
  // type c2.
  // Type of func: voidfunc(concept)
  void ensureHas(Class<? extends Concept> c1, Class<? extends Concept> c2, O func) {
    for (Concept a : conceptsOfType(c1)) {
      Concept b = findBackRef(a, c2);
      if (b == null) {
        callF(func, a);
        b = findBackRef(a, c2);
        if (b == null)
          fail("Concept not made by " + func + ": " + shortClassName(c2));
      }
    }
  }
  
  // Type of func: voidfunc(concept)
  void forEvery(Class<? extends Concept> type, O func) {
    for (Concept c : conceptsOfType(type))
      callF(func, c);
  }
  
  int deleteAll(Class<? extends Concept> type) {
    L<Concept> l = (L) conceptsOfType(type);
    for (Concept c : l) c.delete();
    ret l(l);
  }
  
  // always returns a new list (callers depend on this)
  Collection<Concept> allConcepts() {
    synchronized(concepts) {
      ret new L(values(concepts));
    }
  }
  
  IConceptCounter conceptCounterForClass(Class<? extends Concept> c) {
    for (IFieldIndex idx : values(mapGet(fieldIndices, c)))
      if (idx cast IConceptCounter) ret idx;
    for (IFieldIndex idx : values(mapGet(ciFieldIndices, c)))
      if (idx cast IConceptCounter) ret idx;
    null;
  }
  

  <A extends Concept> int countConcepts(Class<A> c, O... params) {
    int n = countConcepts_noParent(c, params);
    if (parent == null) ret n;
    ret n+parent.countConcepts(c, params);
  }
  
  <A extends Concept> int countConcepts_noParent(Class<A> c, O... params) {
    ping();
    if (empty(params)) {
      IConceptCounter counter = conceptCounterForClass(c);
      if (counter != null) ret counter.countConcepts();
      ret l(list_noParent(c));
    }
    int n = 0;
    for (A x : list_noParent(c)) if (checkConceptFields(x, params)) ++n;
    ret n;
  }

  int countConcepts(S c, O... params) {
    ping();
    if (empty(params)) ret l(list(c));
    int n = 0;
    for (Concept x : list(c)) if (checkConceptFields(x, params)) ++n;
    ret n;
  }

  int countConcepts() {
    ret l(concepts);
  }
  
  synchronized L<IConceptIndex> clonedConceptIndices() {
    ret cloneList(conceptIndices);
  }
  
  synchronized void addConceptIndex(IConceptIndex index) {
    if (conceptIndices == null)
      conceptIndices = new L;
    conceptIndices.add(index);
  }
  
  synchronized void removeConceptIndex(IConceptIndex index) {
    if (conceptIndices == null) ret;
    conceptIndices.remove(index);
    if (empty(conceptIndices)) conceptIndices = null;
  }
  
  synchronized void addFieldIndex(Class<? extends Concept> c, S field, IFieldIndex index) {
    if (fieldIndices == null)
      fieldIndices = new HashMap;
    Map<S, IFieldIndex> map = fieldIndices.get(c);
    if (map == null)
      fieldIndices.put(c, map = new HashMap);
    map.put(field, index);
  }
  
  synchronized void removeFieldIndex(Class<? extends Concept> c, S field, IFieldIndex index) {
    Map<S, IFieldIndex> map = mapGet(fieldIndices, c);
    mapRemove(map, field);
  }
  
  synchronized IFieldIndex getFieldIndex(Class<? extends Concept> c, S field) {
    if (fieldIndices == null) null;
    Map<S, IFieldIndex> map = fieldIndices.get(c);
    ret map == null ? null : map.get(field);
  }
  
  synchronized IFieldIndex getAnyIndexForClass(Class<? extends Concept> c) {
    ret first(allIndicesForClass(c));
  }
  
  synchronized Cl<IFieldIndex> allIndicesForClass(Class<? extends Concept> c) {
    ret concatLists(
      values(fieldIndices?.get(c)),
      values(ciFieldIndices?.get(c)));
  }
  
  synchronized void addCIFieldIndex(Class<? extends Concept> c, S field, IFieldIndex index) {
    if (ciFieldIndices == null)
      ciFieldIndices = new HashMap;
    Map<S, IFieldIndex> map = ciFieldIndices.get(c);
    if (map == null)
      ciFieldIndices.put(c, map = new HashMap);
    map.put(field, index);
  }
  
  synchronized void removeCIFieldIndex(Class<? extends Concept> c, S field) {
    Map<S, IFieldIndex> map = mapGet(ciFieldIndices, c);
    mapRemove(map, field);
  }
  
  synchronized IFieldIndex getCIFieldIndex(Class<? extends Concept> c, S field) {
    if (ciFieldIndices == null) null;
    Map<S, IFieldIndex> map = ciFieldIndices.get(c);
    ret map == null ? null : map.get(field);
  }
  
  // inter-process methods
  
  RC xnew(S name, O... values) {
    ret new RC(cnew(name, values));
  }
  
  void xset(long id, S field, O value) {
    xset(new RC(id), field, value);
  }
  
  void xset(RC c, S field, O value) {
    if (value instanceof RC)
      value = getConcept((RC) value);
    cset(getConcept(c), field, value);
  }
  
  O xget(long id, S field) {
    ret xget(new RC(id), field);
  }
  
  O xget(RC c, S field) {
    ret xgetPost(cget(getConcept(c), field));
  }
  
  O xgetPost(O o) {
    o = deref(o);
    if (o instanceof Concept)
      ret new RC((Concept) o);
    ret o;
  }
  
  void xdelete(long id) {
    xdelete(new RC(id));
  }
  
  void xdelete(RC c) {
    getConcept(c).delete();
  }
  
  void xdelete(L<RC> l) {
    for (RC c : l)
      xdelete(c);
  }
  
  L<RC> xlist() {
    ret map("toPassRef", allConcepts());
  }
  
  L<RC> xlist(S className) {
    ret map("toPassRef", conceptsOfType(className));
  }
  
  bool isTransient() { ret eq(programID, "-"); }
  
  S xfullgrab() {
    if (noXFullGrab) fail("no xfullgrab (DB too large)");
    lock lock();
    if (changes == changesWritten && !isTransient())
      ret loadConceptsStructure(programID);
    ret fullStructure();
  }
  
  /* dev.
  Either<File, byte[]> xfullgrabGZipped() {
    lock lock();
    if (changes == changesWritten && !isTransient())
      ret loadConceptsStructure(programID);
    ret fullStructure();
  }*/
  
  void xshutdown() {
    // Killing whole VM if someone wants this DB to shut down
    cleanKillVM();
  }
  
  long xchangeCount() { ret changes; }
  int xcount() { ret countConcepts(); }
  
  void register(Concept c) {
    ping();
    if (c._concepts == this) ret;
    if (c._concepts != null) fail("Can't re-register");
    c.id = internalID();
    c.created = now();
    if (modifyOnCreate) c._setModified(c.created);
    register_phase2(c);
    vmBus_send conceptCreated(c);
    fireChange(new ConceptCreate(c));
  }
  
  // also called by replaceConceptAndUpdateRefs
  void register_phase2(Concept c) {
    c._concepts = this;
    concepts.put((long) c.id, c);
    for (Concept.Ref r : c._refs())
      r.index();
    c.change();
    c._onRegistered();
  }
  
  void registerKeepingID(Concept c) {
    if (c._concepts == this) ret;
    if (c._concepts != null) fail("Can't re-register");
    c._concepts = this;
    concepts.put((long) c.id, c);
    c.change();
  }
  
  void conceptChanged(Concept c) {
    fireChange(new ConceptChange(c));
    if (conceptIndices != null)
      for (IConceptIndex index : clonedConceptIndices())
        index.update(c);
  }
  
  bool hasUnsavedData() {
    ret changes != changesWritten || savingConcepts;
  }
  
  synchronized O miscMapGet(O key) {
    ret mapGet(miscMap, key);
  }
  
  synchronized O miscMapPut(O key, O value) {
    if (miscMap == null) miscMap = new Map;
    ret miscMap.put(key, value);
  }
  
  synchronized void miscMapRemove(O key) {
    mapRemove(miscMap, key);
  }
  
  // Note: auto-typing can fool you, make sure create returns
  // a wide enough type
  synchronized <A> A miscMapGetOrCreate(O key, IF0<A> create) {
    if (containsKey(miscMap, key)) ret (A) miscMap.get(key);
    A value = create!;
    miscMapPut(key, value);
    ret value;
  }
  
  void setParent(Concepts parent) {
    this.parent = parent;
  }
  
  void fireChange(ConceptsChange change) {
    if (change == null) ret;
    pcallFAll(onChange, change);
    fireLegacyChangeEvent();
  }
  
  void addChangeListener aka onChange(IVF1<ConceptsChange> l) {
    syncAdd(onChange, l);
  }
  
  void removeChangeListener(IVF1<ConceptsChange> l) {
    syncRemove(onChange, l);
  }
  
  void addPreSave(Runnable r) {
    preSave = syncAddOrCreate(preSave, r);
  }
  
  toString {
    ret nConcepts(concepts) + " (" + conceptsDir() + ", hash: " + identityHashCode(this) + ")";
  }
} // end of Concepts

// TODO: Move everyone over to this so we can have concepts that
// don't subclass Concept. It's hard though because of Concept.Ref
interface IConcept {
  public long _conceptID();
  public Concepts concepts();
}

sclass Concept extends DynamicObject is IConcept, ChangeTriggerable {
  transient Concepts _concepts; // Where we belong
  long id;
  long created, _modified;
  ifdef ConceptRefsField
  L<Ref> refs;
  endifdef
  L<Ref> backRefs;
  
  // used only internally (cnew)
  *(S className) {
    super(className);
    _created();
  }
  
  *() {
    if (!_loading()) {
      //className = shortClassName(this); // XXX - necessary?
      //print("New concept of type " + className);
      _created();
    }
  }
  
  *(bool unlisted) {
    if (!unlisted) _created();
  }
  
  bool includeZeroIDInToString() { false; }
  
  toString {
    S s = shortDynamicClassName(this);
    long id = this.id;
    if (id != 0 || includeZeroIDInToString())
      s += " " + id;
    ret s;
  }
  
  static bool loading() { ret _loading(); }
  static bool _loading() { ret dynamicObjectIsLoading(); }

  void _created() {
    if (!concepts_unlistedByDefault && !eq(concepts_unlisted!, true))
      db_mainConcepts().register(this);
  }
  
  // base class + required interface. experimental
  persistable class TypedRef<A extends Concept, B> extends Ref<A> {
    //Class<B> aType;
    Class<B> bType;
    
    *(Class<B> *bType) {}
    *(Class<B> *bType, B value) { set((A) value); }
    *(B value) { set((A) value); }
    
    public bool set(A a) {
      ret super.set(checkValue(a));
    }
    
    void check { checkValue(get()); }
    
    <C> C checkValue(C a) {
      if (bType != null && a != null)
        assertIsInstance(a, bType);
      ret a;
    }
    
    B b() { ret (B) value; }
  }
  
  class Ref<A extends Concept> implements IRef<A> {
    A value;
    
    *() {
      if (!dynamicObjectIsLoading())
        registerRef();
    }
    
    void registerRef {
      vmBus_send registeringConceptRef(this);
    }
    
    *(A *value) {
      registerRef();
      index();
    }
    
    // get owning concept (source)
    Concept concept() {
      ret Concept.this;
    }
    
    // get target
    public A get() { ret value; }
    public bool has() { ret value != null; }
    
    bool set(A a) {
      if (a == value) false;
      unindex();
      value = a;
      index();
      change();
      true;
    }
    
    void setIfEmpty(A a) {
      if (!has()) set(a);
    }
    
    public void set(Ref<A> ref) { set(ref.get()); }
    public void clear() { set((A) null); }
    
    bool validRef() {
      ret value != null && _concepts != null && _concepts == value._concepts;
    }
    
    // TODO: sync all the indexing and unindexing!?
    void index() { 
      if (validRef()) {
        value._addBackRef(this);
        change();
      }
    }

    Ref<A> unindex() {
      if (validRef()) {
        value._removeBackRef(this);
        change();
      }
      this;
    }
    
    void unindexAndDrop {
      unindex();
      _removeRef(this);
    }
    
    void change() {
      Concept.this.change();
    }
    
    toString { ret ifdef ConceptRef_markInToString "Concept.Ref " + endifdef
    str(value); }
  }
  
  class RefL<A extends Concept> extends AbstractList<A> {
    new L<Ref<A>> l;
    
    *() {}
    *(L<A> l) { replaceWithList(l); }
    
    public void clear {
      while (!isEmpty()) main removeLast(this);
    }
    
    public void replaceWithList(L<A> l) {
      clear();
      fOr (A a : l) add(a);
    }
    
    public A set(int i, A o) {
      Ref<A> ref = syncGet(l, i);
      A prev = ref!;
      ref.set(o);
      ret prev;
    }
    
    public void add(int i, A o) {
      syncAdd(l, i, new Ref(o));
    }
    
    public A get(int i) {
      ret syncGet(l, i)!;
    }
    
    public A remove(int i) {
      ret syncRemove(l, i)!;
    }
    
    public int size() {
      ret syncL(l);
    }
    
    public bool contains(O o) {
      if (o instanceof Concept)
        for (Ref<A> r : l) if (eq(r!, o)) true;
      ret super.contains(o);
    }
  }
  
  void delete() {
    //name = "[defunct " + name + "]";
    //defunct = true;
    //energy = 0;
    
    // clean refs
    
    fOr (Ref r : _refs())
      r.unindex();
    ifdef ConceptRefsField
    refs = null;
    endifdef
    
    // set back refs to null
    
    for (Ref r : cloneList(backRefs))
      r.set((Concept) null);
    backRefs = null;
    
    var _concepts = this._concepts;
    if (_concepts != null) {
      _concepts.concepts.remove(id);
      _concepts.fireChange(new ConceptDelete(id, this));
      if (_concepts.conceptIndices != null)
        for (IConceptIndex index : _concepts.conceptIndices)
          index.remove(this);
      this._concepts = null;
    }
    id = 0;
  }
  
  BaseXRef export() {
    ret new BaseXRef(_concepts.progID(), id);
  }
  
  // notice system of a change in this object
  public void change aka _change() {
    _setModified(now());
    _change_withoutUpdatingModifiedField();
  }
  
  void _setModified(long modified) {
    _modified = modified;
  }
  
  final void _change_withoutUpdatingModifiedField() {
    _onChange();
    if (_concepts != null) _concepts.conceptChanged(this);
  }
  
  // overridable
  void _onChange {}
  
  S _programID() {
    ret _concepts == null ? getDBProgramID() : _concepts.progID();
  }
  
  // overridable
  
  void _addBackRef(Concept.Ref ref) {
    backRefs = addDyn_quickSync(backRefs, ref);
    _backRefsModified();
  }
  
  void _backRefsModified {
    if (_concepts != null && _concepts.modifyOnBackRef) change();
  }
  
  void _removeBackRef(Concept.Ref ref) {
    backRefs = removeDyn_quickSync(backRefs, ref);
    _backRefsModified();
  }
  
  void _removeRef(Concept.Ref ref) {
    ifdef ConceptRefsField
    refs = removeDyn_quickSync(refs, ref);
    endifdef
  }
  
  int _backRefCount() { ret syncL(backRefs); }
  
  // convenience methods
  
  void _setField aka setField(S field, O value) {
    cset(this, field, value);
  }
  
  bool setField_trueIfChanged(S field, O value) {
    ret cset(this, field, value) != 0;
  }
  
  <A> A setFieldAndReturn(S field, A value) {
    setField(field, value);
    ret value;
  }
  
  void _setFields aka setFields(O... values) {
    cset(this, values);
  }
  
  public Concepts concepts() { ret _concepts; }
  
  bool isDeleted() { ret id == 0; }
  
  // Called directly after concept is unstructured
  void _doneLoading() {}
  
  // Called after all concepts are unstructured and added to Concepts
  void _doneLoading2() {
    Map<S, FieldMigration> map = _fieldMigrations();
    if (map != null) for (S oldField, FieldMigration m : map)
      crenameField_noOverwrite(this, oldField, m.newField);
  }
  
  srecord FieldMigration(S newField) {}
  
  // value is 
  Map<S, FieldMigration> _fieldMigrations() { null; }
  
  // new wrapper to get a copy of the refs list
  // so we can eventually drop the refs field
  Cl<Ref> _refs() {
    ret scanConceptForRefs(this);
  }
  
  ifdef ConceptRefsField
  // for debugging
  L<Ref> _rawRefsField() {
    ret refs;
  }
  endifdef
  
  Concepts _concepts() { ret _concepts; }
  
  bool _conceptsDefunct() { ret _concepts != null && _concepts.defunct; }
  bool _conceptsDefunctOrUnregistered() { ret _concepts == null || _concepts.defunct; }
  
  // allow refs to do magic stuff?
  void _onRegistered {
    /*for (Ref ref : _refs())
      refs._onRegistered();*/
  }
  
  !include #1031423 // Concept convenience methods
  
  File conceptsDir() {
    var concepts = concepts();
    ret concepts?.conceptsDir();
  }
  
  File fileInConceptsDir(S name) {
    var dir = conceptsDir();
    ret dir == null ?: newFile(dir, name);
  }
  
  public long _conceptID() { ret id; }
} // end of Concept

// remote reference (for inter-process communication or
// external databases). Formerly "PassRef".
// prepared for string ids if we do them later
sclass RC {
  transient O owner;
  S id;
  
  *() {} // make serialisation happy
  *(long id) { this.id = str(id); }
  *(O owner, long id) { this.id = str(id); this.owner = owner; }
  *(Concept c) { this(c.id); }
  long longID() { ret parseLong(id); }
  
  public S toString() {
    ret id;
  }
} // end of RC

// Reference to a concept in another program
sclass BaseXRef {
  S programID;
  long id;
    
  *() {}
  *(S *programID, long *id) {}
  
  public bool equals(O o) {
    if (!(o instanceof BaseXRef)) false;
    BaseXRef r = cast o;
    ret eq(programID, r.programID) && eq(id, r.id);
  }
  
  public int hashCode() {
    ret programID.hashCode() + (int) id;
  }
}

// BaseXRef as a concept
sclass XRef extends Concept {
  BaseXRef ref;
  
  *() {}
  *(BaseXRef *ref) { _doneLoading2(); }
  
  // after we have been added to concepts
  void _doneLoading2() {
    getIndex().put(ref, this);
  }
    
  HashMap<BaseXRef, XRef> getIndex() {
    ret getXRefIndex(_concepts);
  }
}

static synchronized HashMap<BaseXRef, XRef> getXRefIndex(Concepts concepts) {
  HashMap cache = (HashMap) concepts.perClassData().get(XRef.class);
  if (cache == null)
    concepts.perClassData.put(XRef.class, cache = new HashMap);
  ret cache;
}

// uses mainConcepts
static XRef lookupOrCreateXRef(BaseXRef ref) {
  XRef xref = getXRefIndex(db_mainConcepts()).get(ref);
  if (xref == null)
    xref = new XRef(ref);
  ret xref;
}

// define standard concept functions to use main concepts

// Now in db_mainConcepts()
/*static void cleanMeUp_concepts() {
  if (db_mainConcepts() != null) db_mainConcepts().cleanMeUp();
  // mainConcepts = null; // TODO
}*/

svoid loadAndAutoSaveConcepts {
  db_mainConcepts().persist();
}

svoid loadAndAutoSaveConcepts(int interval) {
  db_mainConcepts().persist(interval);
}

static RC toPassRef(Concept c) {
  ret new RC(c);
}

// so we can instantiate the program to run as a bare DB bot
please include function bareDBMode.

// so unstructuring Concepts works
please include function dynamicObjectIsLoading_threadLocal.

svoid concepts_setUnlistedByDefault(bool b) {
  concepts_unlistedByDefault = b;
}

Began life as a copy of #1004681