// A database system for only slightly modified Java objects.

// Recent changes:

// -minimal crash recovery disabled for now (could reenable)

// -idCounter.structure file no longer used

// -allDynamic/safeLoad disabled, it doesn't really make any sense

// -conceptsFile can be in any directory now

// Functions that should always be there for child processes:

please include function dbLock.

static int concepts_internStringsLongerThan = 10;

static new ThreadLocal<Bool> concepts_unlisted;

// BREAKING CHANGE 2021/6/7 set to true

sbool concepts_unlistedByDefault = true; // true = we can create instances of concepts with "new" without registering them automatically

interface IConceptIndex {

  void update(Concept c); // also for adding

  void remove(Concept c);

}

interface IFieldIndex<A extends Concept, Val> {

  Cl<A> getAll(Val val);

  L<Val> allValues(); // returns a cloned list

  MultiSet<Val> allValues_multiSet();

  ItIt<A> objectIterator();

}

// Approach to persisting the Concepts object itself (in normal

// DB operation, this is not done): For simplification, speed and

// compactness, we make almost all the fields transient and store only // the concepts and the idCounter. To unstructure the Concepts object,

// use unstructureConcepts() or postUnstructureConcepts(), then

// re-set up any indices, listeners etc.

// base class indicating nothing

sclass ConceptsChange {}

// change of a single concept

srecord ConceptCreate(Concept c) extends ConceptsChange {}

// change of a single concept

srecord ConceptChange(Concept c) extends ConceptsChange {}

// removal of a single concept

// c.id is going to become 0 at some point, so we pass the

// id separately

srecord ConceptDelete(long id, Concept c) extends ConceptsChange {}

// unknown change anywhere in concepts; consider it all dirty

// (this one should not be used except for batch jobs)

srecord FullChange extends ConceptsChange {}

sclass Concepts implements AutoCloseable {

  NavigableMap<Long, Concept> concepts = synchroTreeMap();

  // ID of last created concept

  gettable long idCounter;

  transient HashMap<Class, O> perClassData;

  transient Map miscMap; // don't use directly, call miscMap... methods to access

  // set to "-" for non-persistent (possibly not implemented)

  // also, can include a case ID ("#123/1")

  // TODO: phase out (we have conceptsFile field now)

  transient S programID;

  transient File conceptsFile;

  transient Concepts parent; // new mechanism

  transient volatile long changes, changesWritten, lastChange;

  transient volatile java.util.Timer autoSaver;

  transient volatile bool dontSave;

  transient volatile bool savingConcepts, noXFullGrab;

  transient bool vmBusSend = true;

  transient bool initialSave = false; // set to false to avoid initial useless saving

  transient int autoSaveInterval = -1000; // 1 second + wait logic

  transient bool useGZIP = true, quietSave;

  transient ReentrantLock lock = new ReentrantLock(true);

  transient ReentrantLock saverLock = new ReentrantLock(true);

  transient long lastSaveTook = -1, lastSaveWas, loadTook, uncompressedSize;

  transient float maxAutoSavePercentage = 10;

  transient L<IConceptIndex> conceptIndices;

  transient Map<Class<? extends Concept>, Map<S, IFieldIndex>> fieldIndices;

  transient Map<Class<? extends Concept>, Map<S, IFieldIndex>> ciFieldIndices;

  //transient L saveActions = synchroList();

  transient L<Runnable> preSave;

  transient O classFinder = _defaultClassFinder();

  transient L onAllChanged = synchroList(); // list of runnables

  transient new Set<IVF1> onChange;

  transient O saveWrapper; // VF1<Runnable>, to profile saving

  settable transient bool modifyOnCreate; // set _modified == created initially

  transient bool modifyOnBackRef; // set modified if back refs change

  transient bool useFileLock = true; // instead of locking by bot

  settable transient bool grabThroughSocket = true; // probably never want to do this again actually

  // OLD - not done anymore. transient bool collectForwardRefs

  transient FileBasedLock fileLock;

  transient bool storeBaseClassesInStructure; // helps with schema evolution when concept subclasses disappear

  transient bool useBackRefsForSearches; // assume backRefs are sane in order to speed up searches

  transient bool defunct;

  transient int newBackupEveryXMinutes = 60;

  settable transient bool closeConceptsOnExit;

  settable transient Throwable saveError;

  event fireSaveError(Throwable error);

  // add more fields for Concepts here

  *() {}

  *(S *programID) {}

  *(File *conceptsFile) {}

  synchronized long internalID() {

    do {

      ++idCounter;

    } while (hasConcept(idCounter));

    ret idCounter;

  }

  synchronized HashMap<Class, O> perClassData() {

    if (perClassData == null) perClassData = new HashMap;

    ret perClassData;

  }

  void initProgramID() {

    if (programID == null)

      programID = getDBProgramID();

  }

  // load from structure

  Concepts load(S structure, bool allDynamic default false) {

    clearConcepts();

    Map<Long, Concept> map = unstructureMap(structure, allDynamic, classFinder);

    concepts.putAll(map);

    assignConceptsToUs();

    calcIdCounter();

    this;

  }

  Concepts load() {

    initProgramID();

    // try custom grabber

    O dbGrabber = miscMapGet("dbGrabber");

    if (dbGrabber != null && !isFalse(callF(dbGrabber)))

      this;

    try {

      if (grabThroughSocket && tryToGrab()) ret this;

    } catch e {

      if (!exceptionMessageContains(e, "no xfullgrab"))

        printShortException(e);

      print("xfullgrab failed - loading DB of " + programID + " from disk");

    }

    ret loadFromDisk();

  }

  Concepts loadFromDisk() {

    if (nempty(concepts)) clearConcepts();

    // minimal crash recovery (disabled for now)

    //restoreLatestBackupIfConceptsFileEmpty(programID, doIt := true);

    long time = now();

    Map<Long, Concept> _concepts = cast unstructureGZFile(conceptsFile(), toIF1(classFinder));

    putAll(concepts, _concepts);

    assignConceptsToUs();

    loadTook = now()-time;

    done("Loaded " + n2(l(concepts), "concept"), time);

    calcIdCounter();

    this;

  }

  Concepts loadConcepts() { ret load(); }

  bool tryToGrab() {

    if (sameSnippetID(programID, getDBProgramID())) false;

    temp RemoteDB db = connectToDBOpt(programID);

    if (db != null) {

      loadGrab(db.fullgrab());

      true;

    }

    false;

  }

  Concepts loadGrab(S grab) {

    clearConcepts();

    DynamicObject_loading.set(true);

    try {

      Map<Long, Concept> map = (Map) unstructure(grab, false, classFinder);

      concepts.putAll(map);

      assignConceptsToUs();

      for (long l : map.keySet())

        idCounter = max(idCounter, l);

    } finally {

      DynamicObject_loading.set(null);

    }

    //XXX allChanged(); // Nobody is listening at this point anyway

    ret this;

  }

  void assignConceptsToUs() {

    // fix unstructure bugs

    for (Pair<Long, O> p: mapToPairs((Map<Long, O>) (Map) concepts))

      if (!(p.b instanceof Concept)) {

       print("DROPPING non-existant concept " + p.a + ": " + dynShortName(p.b));

       concepts.remove(p.a);

    }

    for (Concept c : values(concepts)) c._concepts = this;

    for (Concept c : values(concepts))

      c._doneLoading2(); // doneLoading2 is called on all concepts after all concepts are loaded

  }

  S progID() {

    ret programID == null ? getDBProgramID() : programID;

  }

  Concept getConcept(S id) {

    ret empty(id) ? null : getConcept(parseLong(id));

  }

  Concept getConcept(long id) {

    ret (Concept) concepts.get((long) id);

  }

  Concept getConcept(RC ref) {

    ret ref == null ? null : getConcept(ref.longID());

  }

  bool hasConcept(long id) {

    ret concepts.containsKey((long) id);

  }

  void deleteConcept(long id) {

    Concept c = getConcept(id);

    if (c == null)

      print("Concept " + id + " not found");

    else

      c.delete();

  }

  void calcIdCounter() {

    Long lastID = lastKey(concepts);

    idCounter = lastID == null ? 1 : lastID+1;

  }

  File conceptsDir() { ret dirOfFile(conceptsFile()); }

  selfType conceptsFile(File conceptsFile) { this.conceptsFile = conceptsFile; this; }

  File conceptsFile() {

    if (conceptsFile != null) ret conceptsFile;

    ret getProgramFile(programID, useGZIP ? "concepts.structure.gz" : "concepts.structure");

  }

  // used for locking when useFileLock is activated

  File lockFile() {

    ret newFile(conceptsDir(), "concepts.lock");

  }

  FileBasedLock fileLock() {

    if (fileLock == null)

      fileLock = new FileBasedLock(lockFile());

    ret fileLock;

  }

  void saveConceptsIfDirty() { saveConcepts(); }

  void save() { saveConcepts(); }

  void saveConcepts() {

    vmBus_send saveConceptsCalled(Concepts.this);

    if (dontSave) ret;

    initProgramID();

    saverLock.lock();

    savingConcepts = true;

    long start = now(), time;

    try {

      S s = null;

      //synchronized(main.class) {

      long _changes = changes;

      if (_changes == changesWritten) ret;

      File f = conceptsFile();

      lock.lock();

      long fullTime = now();

      try {

        if (useGZIP) {

          vmBus_send callingSaveWrapper(Concepts.this, saveWrapper);

          callRunnableWithWrapper(saveWrapper, r {

            vmBus_send callingPreSave(Concepts.this, preSave);

            callFAll(preSave);

            vmBus_send writingFile(Concepts.this, f);

            uncompressedSize = saveGZStructureToFile(f, cloneMap(concepts), makeStructureData());

            vmBus_send gzFileSaved(Concepts.this, f, uncompressedSize);

          });

          newFile(conceptsDir(), "concepts.structure").delete();

        } else

          s = fullStructure();

      } finally {

        lock.unlock();

      }

      /*while (nempty(saveActions))

        pcallF(popFirst(saveActions));*/

      changesWritten = _changes; // only update when structure didn't fail

      if (!useGZIP) {

        time = now()-start;

        if (!quietSave)

          print("Saving " + toM(l(s)) + "M chars (" /*+ changesWritten + ", "*/ + time + " ms)");

        start = now();

        saveTextFile(f, javaTokWordWrap(s));

        newFile(conceptsDir(), "concepts.structure.gz").delete();

      }

      File conceptsFile = conceptsFile();

      File backupFile = newFile(conceptsDir(), "backups/" + fileName(conceptsFile) + ".backup" + ymd() + "-" + formatInt(hours(), 2)

        + (newBackupEveryXMinutes >= 60 ? "" : formatInt(roundDownTo_rev(minutes(), newBackupEveryXMinutes), 2)));

      // TODO: get rid of this

      copyFile(f, backupFile);

      time = now()-start;

      if (!quietSave)

        print("Saved " + toK(f.length()) + " K, " + n(concepts, "concepts") + " (" + time + " ms)");

      lastSaveWas = fullTime;

      lastSaveTook = now()-fullTime;

      saveError(null);

    } finally {

      savingConcepts = false;

      saverLock.unlock();

    }

  }

  void _autoSaveConcepts() {

    if (autoSaveInterval < 0 && maxAutoSavePercentage != 0) {

      long pivotTime = Math.round(lastSaveWas+lastSaveTook*100.0/maxAutoSavePercentage);

      if (now() < pivotTime) {

        //print("Skipping auto-save (last save took " + lastSaveTook + ")");

        ret;

      }

    }

    try {

      saveConcepts();

    } catch e {

      saveError(e);

      print("Concept save failed, will try again");

      printStackTrace(e);

      fireSaveError(e);

    }

  }

  S fullStructure() {

    ret structure(cloneMap(concepts), makeStructureData());

  }

  swappable structure_Data makeStructureData() {

    ret finishStructureData(new structure_Data);

  }

  structure_Data finishStructureData(structure_Data data) {

    if (storeBaseClassesInStructure)

      data.storeBaseClasses = true;

    ret data;

  }

  void clearConcepts() {

    for (Concept c : allConcepts()) c.delete();

    //concepts.clear();

    //allChanged();

  }

  void fireLegacyChangeEvent() {

    synchronized(this) { ++changes; lastChange = sysNow(); }

    if (vmBusSend) vmBus_send conceptsChanged(this);

    pcallFAll(onAllChanged);

  }

  // auto-save every second if dirty

  synchronized void autoSaveConcepts() {

    if (autoSaver == null) {

      if (isTransient()) fail("Can't persist transient database");

      autoSaver = doEvery_daemon("Concepts Saver for " + conceptsDir(),

        abs(autoSaveInterval), r { _autoSaveConcepts() });

      // print("Installed auto-saver (" + autoSaveInterval + " ms, " + progID() + ")");

    }

  }

  public void close { cleanMeUp(); }

  void cleanMeUp() {

    pcall {

      if (closeConceptsOnExit)

        closeAllOpt(allConcepts());

      set defunct;

      bool shouldSave = autoSaver != null;

      if (autoSaver != null) {

        autoSaver.cancel();

        autoSaver = null;

      }

      while (savingConcepts) sleepInCleanUp(10);

      if (shouldSave)

        saveConceptsIfDirty();

    }

    dispose fileLock;

  }

  Map<Long, S> getIDsAndNames() {

    new Map<Long, S> map;

    Map<Long, Concept> cloned = cloneMap(concepts);

    for (long id : keys(cloned)) 

      map.put(id, cloned.get(id).className);

    ret map;

  }

  void deleteConcepts(L l) {

    ping();

    if (l != null) for (O o : cloneList(l))

      if (o instanceof Long) {

        Concept c = concepts.get(o);

        if (c != null) c.delete();

      } else if (o cast Concept)

        o.delete();

      else

        warn("Can't delete " + getClassName(o));

  }

  <A extends Concept> A conceptOfType(Class<A> type) {

    IConceptCounter counter = conceptCounterForClass(type);

    if (counter != null) ret (A) first(counter.allConcepts());

    ret firstOfType(allConcepts(), type);

  }

  <A extends Concept> L<A> conceptsOfType(Class<A> type) {

    L<A> l = conceptsOfType_noParent(type);

    if (parent == null) ret l;

    ret concatLists_conservative(l, parent.conceptsOfType(type));

  }

  <A extends Concept> L<A> conceptsOfType_noParent(Class<A> type) {

    ping();

    IConceptCounter counter = conceptCounterForClass(type);

    if (counter != null) ret (L<A>) cloneList(counter.allConcepts());

    ret filterByType(allConcepts(), type);

  }

  <A extends Concept> L<A> listConcepts(Class<A> type) {

    ret conceptsOfType(type);

  }

  <A extends Concept> L<A> list(Class<A> type) {

    ret conceptsOfType(type);

  }

  <A extends Concept> L<A> list_noParent(Class<A> type) {

    ret conceptsOfType_noParent(type);

  }

  // TODO: would be better to make this Cl (indices may return sets)

  L<Concept> list(S type) {

    ret conceptsOfType(type);

  }

  L<Concept> conceptsOfType(S type) {

    ret filterByDynamicType(allConcepts(), "main$" + type);

  }

  bool hasConceptOfType(Class<? extends Concept> type) {

    ret hasType(allConcepts(), type);

  }

  void persistConcepts() {

    loadConcepts();

    autoSaveConcepts();

  }

  // We love synonyms

  void conceptPersistence() { persistConcepts(); }

  Concepts persist() { persistConcepts(); ret this; }

  void persist(Int interval) {

    if (interval != null) autoSaveInterval = interval;

    persist();

  }

  // Runs r if there is no concept of that type

  <A extends Concept> A ensureHas(Class<A> c, Runnable r) {

    A a = conceptOfType(c);

    if (a == null) {

      r.run();

      a = conceptOfType(c);

      if (a == null)

        fail("Concept not made by " + r + ": " + shortClassName(c));

    }

    ret a;

  }

  // Ensures that every concept of type c1 is ref'd by a concept of

  // type c2.

  // Type of func: voidfunc(concept)

  void ensureHas(Class<? extends Concept> c1, Class<? extends Concept> c2, O func) {

    for (Concept a : conceptsOfType(c1)) {

      Concept b = findBackRef(a, c2);

      if (b == null) {

        callF(func, a);

        b = findBackRef(a, c2);

        if (b == null)

          fail("Concept not made by " + func + ": " + shortClassName(c2));

      }

    }

  }

  // Type of func: voidfunc(concept)

  void forEvery(Class<? extends Concept> type, O func) {

    for (Concept c : conceptsOfType(type))

      callF(func, c);

  }

  int deleteAll(Class<? extends Concept> type) {

    L<Concept> l = (L) conceptsOfType(type);

    for (Concept c : l) c.delete();

    ret l(l);

  }

  // always returns a new list (callers depend on this)

  Collection<Concept> allConcepts() {

    synchronized(concepts) {

      ret new L(values(concepts));

    }

  }

  IConceptCounter conceptCounterForClass(Class<? extends Concept> c) {

    for (IFieldIndex idx : values(mapGet(fieldIndices, c)))

      if (idx cast IConceptCounter) ret idx;

    for (IFieldIndex idx : values(mapGet(ciFieldIndices, c)))

      if (idx cast IConceptCounter) ret idx;

    null;

  }

  <A extends Concept> int countConcepts(Class<A> c, O... params) {

    int n = countConcepts_noParent(c, params);

    if (parent == null) ret n;

    ret n+parent.countConcepts(c, params);

  }

  <A extends Concept> int countConcepts_noParent(Class<A> c, O... params) {

    ping();

    if (empty(params)) {

      IConceptCounter counter = conceptCounterForClass(c);

      if (counter != null) ret counter.countConcepts();

      ret l(list_noParent(c));

    }

    int n = 0;

    for (A x : list_noParent(c)) if (checkConceptFields(x, params)) ++n;

    ret n;

  }

  int countConcepts(S c, O... params) {

    ping();

    if (empty(params)) ret l(list(c));

    int n = 0;

    for (Concept x : list(c)) if (checkConceptFields(x, params)) ++n;

    ret n;

  }

  int countConcepts() {

    ret l(concepts);

  }

  synchronized L<IConceptIndex> clonedConceptIndices() {

    ret cloneList(conceptIndices);

  }

  synchronized void addConceptIndex(IConceptIndex index) {

    if (conceptIndices == null)

      conceptIndices = new L;

    conceptIndices.add(index);

  }

  synchronized void removeConceptIndex(IConceptIndex index) {

    if (conceptIndices == null) ret;

    conceptIndices.remove(index);

    if (empty(conceptIndices)) conceptIndices = null;

  }

  synchronized void addFieldIndex(Class<? extends Concept> c, S field, IFieldIndex index) {

    if (fieldIndices == null)

      fieldIndices = new HashMap;

    Map<S, IFieldIndex> map = fieldIndices.get(c);

    if (map == null)

      fieldIndices.put(c, map = new HashMap);

    map.put(field, index);

  }

  synchronized void removeFieldIndex(Class<? extends Concept> c, S field, IFieldIndex index) {

    Map<S, IFieldIndex> map = mapGet(fieldIndices, c);

    mapRemove(map, field);

  }

  synchronized IFieldIndex getFieldIndex(Class<? extends Concept> c, S field) {

    if (fieldIndices == null) null;

    Map<S, IFieldIndex> map = fieldIndices.get(c);

    ret map == null ? null : map.get(field);

  }

  synchronized IFieldIndex getAnyIndexForClass(Class<? extends Concept> c) {

    ret first(allIndicesForClass(c));

  }

  synchronized Cl<IFieldIndex> allIndicesForClass(Class<? extends Concept> c) {

    ret concatLists(

      values(fieldIndices?.get(c)),

      values(ciFieldIndices?.get(c)));

  }

  synchronized void addCIFieldIndex(Class<? extends Concept> c, S field, IFieldIndex index) {

    if (ciFieldIndices == null)

      ciFieldIndices = new HashMap;

    Map<S, IFieldIndex> map = ciFieldIndices.get(c);

    if (map == null)

      ciFieldIndices.put(c, map = new HashMap);

    map.put(field, index);

  }

  synchronized void removeCIFieldIndex(Class<? extends Concept> c, S field) {

    Map<S, IFieldIndex> map = mapGet(ciFieldIndices, c);

    mapRemove(map, field);

  }

  synchronized IFieldIndex getCIFieldIndex(Class<? extends Concept> c, S field) {

    if (ciFieldIndices == null) null;

    Map<S, IFieldIndex> map = ciFieldIndices.get(c);

    ret map == null ? null : map.get(field);

  }

  // inter-process methods

  RC xnew(S name, O... values) {

    ret new RC(cnew(name, values));

  }

  void xset(long id, S field, O value) {

    xset(new RC(id), field, value);

  }

  void xset(RC c, S field, O value) {

    if (value instanceof RC)

      value = getConcept((RC) value);

    cset(getConcept(c), field, value);

  }

  O xget(long id, S field) {

    ret xget(new RC(id), field);

  }

  O xget(RC c, S field) {

    ret xgetPost(cget(getConcept(c), field));

  }

  O xgetPost(O o) {

    o = deref(o);

    if (o instanceof Concept)

      ret new RC((Concept) o);

    ret o;

  }

  void xdelete(long id) {

    xdelete(new RC(id));

  }

  void xdelete(RC c) {

    getConcept(c).delete();

  }

  void xdelete(L<RC> l) {

    for (RC c : l)

      xdelete(c);

  }

  L<RC> xlist() {

    ret map("toPassRef", allConcepts());

  }

  L<RC> xlist(S className) {

    ret map("toPassRef", conceptsOfType(className));

  }

  bool isTransient() { ret eq(programID, "-"); }

  S xfullgrab() {

    if (noXFullGrab) fail("no xfullgrab (DB too large)");

    lock lock();

    if (changes == changesWritten && !isTransient())

      ret loadConceptsStructure(programID);

    ret fullStructure();

  }

  /* dev.

  Either<File, byte[]> xfullgrabGZipped() {

    lock lock();

    if (changes == changesWritten && !isTransient())

      ret loadConceptsStructure(programID);

    ret fullStructure();

  }*/

  void xshutdown() {

    // Killing whole VM if someone wants this DB to shut down

    cleanKillVM();

  }

  long xchangeCount() { ret changes; }

  int xcount() { ret countConcepts(); }

  void register(Concept c) {

    ping();

    if (c._concepts == this) ret;

    if (c._concepts != null) fail("Can't re-register");

    c.id = internalID();

    c.created = now();

    if (modifyOnCreate) c._setModified(c.created);

    register_phase2(c);

    vmBus_send conceptCreated(c);

    fireChange(new ConceptCreate(c));

  }

  // also called by replaceConceptAndUpdateRefs

  void register_phase2(Concept c) {

    c._concepts = this;

    concepts.put((long) c.id, c);

    for (Concept.Ref r : c._refs())

      r.index();

    c.change();

    c._onRegistered();

  }

  void registerKeepingID(Concept c) {

    if (c._concepts == this) ret;

    if (c._concepts != null) fail("Can't re-register");

    c._concepts = this;

    concepts.put((long) c.id, c);

    c.change();

  }

  void conceptChanged(Concept c) {

    fireChange(new ConceptChange(c));

    if (conceptIndices != null)

      for (IConceptIndex index : clonedConceptIndices())

        index.update(c);

  }

  bool hasUnsavedData() {

    ret changes != changesWritten || savingConcepts;

  }

  synchronized O miscMapGet(O key) {

    ret mapGet(miscMap, key);

  }

  synchronized O miscMapPut(O key, O value) {

    if (miscMap == null) miscMap = new Map;

    ret miscMap.put(key, value);

  }

  synchronized void miscMapRemove(O key) {

    mapRemove(miscMap, key);

  }

  // Note: auto-typing can fool you, make sure create returns

  // a wide enough type

  synchronized <A> A miscMapGetOrCreate(O key, IF0<A> create) {

    if (containsKey(miscMap, key)) ret (A) miscMap.get(key);

    A value = create!;

    miscMapPut(key, value);

    ret value;

  }

  void setParent(Concepts parent) {

    this.parent = parent;

  }

  void fireChange(ConceptsChange change) {

    if (change == null) ret;

    pcallFAll(onChange, change);

    fireLegacyChangeEvent();

  }

  void addChangeListener aka onChange(IVF1<ConceptsChange> l) {

    syncAdd(onChange, l);

  }

  void removeChangeListener(IVF1<ConceptsChange> l) {

    syncRemove(onChange, l);

  }

  void addPreSave(Runnable r) {

    preSave = syncAddOrCreate(preSave, r);

  }

  toString {

    ret nConcepts(concepts) + " (" + conceptsDir() + ", hash: " + identityHashCode(this) + ")";

  }

} // end of Concepts

// TODO: Move everyone over to this so we can have concepts that

// don't subclass Concept. It's hard though because of Concept.Ref

interface IConcept {

  public long _conceptID();

  public Concepts concepts();

}

sclass Concept extends DynamicObject is IConcept, ChangeTriggerable {

  transient Concepts _concepts; // Where we belong

  long id;

  long created, _modified;

  ifdef ConceptRefsField

  L<Ref> refs;

  endifdef

  L<Ref> backRefs;

  // used only internally (cnew)

  *(S className) {

    super(className);

    _created();

  }

  *() {

    if (!_loading()) {

      //className = shortClassName(this); // XXX - necessary?

      //print("New concept of type " + className);

      _created();

    }

  }

  *(bool unlisted) {

    if (!unlisted) _created();

  }

  bool includeZeroIDInToString() { false; }

  toString {

    S s = shortDynamicClassName(this);

    long id = this.id;

    if (id != 0 || includeZeroIDInToString())

      s += " " + id;

    ret s;

  }

  static bool loading() { ret _loading(); }

  static bool _loading() { ret dynamicObjectIsLoading(); }

  void _created() {

    if (!concepts_unlistedByDefault && !eq(concepts_unlisted!, true))

      db_mainConcepts().register(this);

  }

  // base class + required interface. experimental

  persistable class TypedRef<A extends Concept, B> extends Ref<A> {

    //Class<B> aType;

    Class<B> bType;

    *(Class<B> *bType) {}

    *(Class<B> *bType, B value) { set((A) value); }

    *(B value) { set((A) value); }

    public bool set(A a) {

      ret super.set(checkValue(a));

    }

    void check { checkValue(get()); }

    <C> C checkValue(C a) {

      if (bType != null && a != null)

        assertIsInstance(a, bType);

      ret a;

    }

    B b() { ret (B) value; }

  }

  class Ref<A extends Concept> implements IRef<A> {

    A value;

    *() {

      if (!dynamicObjectIsLoading())

        registerRef();

    }

    void registerRef {

      vmBus_send registeringConceptRef(this);

    }

    *(A *value) {

      registerRef();

      index();

    }

    // get owning concept (source)

    Concept concept() {

      ret Concept.this;

    }

    // get target

    public A get() { ret value; }

    public bool has() { ret value != null; }

    bool set(A a) {

      if (a == value) false;

      unindex();

      value = a;

      index();

      change();

      true;

    }

    void setIfEmpty(A a) {

      if (!has()) set(a);

    }

    public void set(Ref<A> ref) { set(ref.get()); }

    public void clear() { set((A) null); }

    bool validRef() {

      ret value != null && _concepts != null && _concepts == value._concepts;

    }

    // TODO: sync all the indexing and unindexing!?

    void index() { 

      if (validRef()) {

        value._addBackRef(this);

        change();

      }

    }

    Ref<A> unindex() {

      if (validRef()) {

        value._removeBackRef(this);

        change();

      }

      this;

    }

    void unindexAndDrop {

      unindex();

      _removeRef(this);

    }

    void change() {

      Concept.this.change();

    }

    toString { ret ifdef ConceptRef_markInToString "Concept.Ref " + endifdef

    str(value); }

  }

  class RefL<A extends Concept> extends AbstractList<A> {

    new L<Ref<A>> l;

    *() {}

    *(L<A> l) { replaceWithList(l); }

    public void clear {

      while (!isEmpty()) main removeLast(this);

    }

    public void replaceWithList(L<A> l) {

      clear();

      fOr (A a : l) add(a);

    }

    public A set(int i, A o) {

      Ref<A> ref = syncGet(l, i);

      A prev = ref!;

      ref.set(o);

      ret prev;

    }

    public void add(int i, A o) {

      syncAdd(l, i, new Ref(o));

    }

    public A get(int i) {

      ret syncGet(l, i)!;

    }

    public A remove(int i) {

      ret syncRemove(l, i)!;

    }

    public int size() {

      ret syncL(l);

    }

    public bool contains(O o) {

      if (o instanceof Concept)

        for (Ref<A> r : l) if (eq(r!, o)) true;

      ret super.contains(o);

    }

  }

  void delete() {

    //name = "[defunct " + name + "]";

    //defunct = true;

    //energy = 0;

    // clean refs

    fOr (Ref r : _refs())

      r.unindex();

    ifdef ConceptRefsField

    refs = null;

    endifdef

    // set back refs to null

    for (Ref r : cloneList(backRefs))

      r.set((Concept) null);

    backRefs = null;

    var _concepts = this._concepts;

    if (_concepts != null) {

      _concepts.concepts.remove(id);

      _concepts.fireChange(new ConceptDelete(id, this));

      if (_concepts.conceptIndices != null)

        for (IConceptIndex index : _concepts.conceptIndices)

          index.remove(this);

      this._concepts = null;

    }

    id = 0;

  }

  BaseXRef export() {

    ret new BaseXRef(_concepts.progID(), id);

  }

  // notice system of a change in this object

  public void change aka _change() {

    _setModified(now());

    _change_withoutUpdatingModifiedField();

  }

  void _setModified(long modified) {

    _modified = modified;

  }

  final void _change_withoutUpdatingModifiedField() {

    _onChange();

    if (_concepts != null) _concepts.conceptChanged(this);

  }

  // overridable

  void _onChange {}

  S _programID() {

    ret _concepts == null ? getDBProgramID() : _concepts.progID();

  }

  // overridable

  void _addBackRef(Concept.Ref ref) {

    backRefs = addDyn_quickSync(backRefs, ref);

    _backRefsModified();

  }

  void _backRefsModified {

    if (_concepts != null && _concepts.modifyOnBackRef) change();

  }

  void _removeBackRef(Concept.Ref ref) {

    backRefs = removeDyn_quickSync(backRefs, ref);

    _backRefsModified();

  }

  void _removeRef(Concept.Ref ref) {

    ifdef ConceptRefsField

    refs = removeDyn_quickSync(refs, ref);

    endifdef

  }

  int _backRefCount() { ret syncL(backRefs); }

  // convenience methods

  void _setField aka setField(S field, O value) {

    cset(this, field, value);

  }

  bool setField_trueIfChanged(S field, O value) {

    ret cset(this, field, value) != 0;

  }

  <A> A setFieldAndReturn(S field, A value) {

    setField(field, value);

    ret value;

  }

  void _setFields aka setFields(O... values) {

    cset(this, values);

  }

  public Concepts concepts() { ret _concepts; }

  bool isDeleted() { ret id == 0; }

  // Called directly after concept is unstructured

  void _doneLoading() {}

  // Called after all concepts are unstructured and added to Concepts

  void _doneLoading2() {

    Map<S, FieldMigration> map = _fieldMigrations();

    if (map != null) for (S oldField, FieldMigration m : map)

      crenameField_noOverwrite(this, oldField, m.newField);

  }

  srecord FieldMigration(S newField) {}

  // value is 

  Map<S, FieldMigration> _fieldMigrations() { null; }

  // new wrapper to get a copy of the refs list

  // so we can eventually drop the refs field

  Cl<Ref> _refs() {

    ret scanConceptForRefs(this);

  }

  ifdef ConceptRefsField

  // for debugging

  L<Ref> _rawRefsField() {

    ret refs;

  }

  endifdef

  Concepts _concepts() { ret _concepts; }

  bool _conceptsDefunct() { ret _concepts != null && _concepts.defunct; }

  bool _conceptsDefunctOrUnregistered() { ret _concepts == null || _concepts.defunct; }

  // allow refs to do magic stuff?

  void _onRegistered {

    /*for (Ref ref : _refs())

      refs._onRegistered();*/

  }

  !include #1031423 // Concept convenience methods

  File conceptsDir() {

    var concepts = concepts();

    ret concepts?.conceptsDir();

  }

  File fileInConceptsDir(S name) {

    var dir = conceptsDir();

    ret dir == null ?: newFile(dir, name);

  }

  public long _conceptID() { ret id; }

} // end of Concept

// remote reference (for inter-process communication or

// external databases). Formerly "PassRef".

// prepared for string ids if we do them later

sclass RC {

  transient O owner;

  S id;

  *() {} // make serialisation happy

  *(long id) { this.id = str(id); }

  *(O owner, long id) { this.id = str(id); this.owner = owner; }

  *(Concept c) { this(c.id); }

  long longID() { ret parseLong(id); }

  public S toString() {

    ret id;

  }

} // end of RC

// Reference to a concept in another program

sclass BaseXRef {

  S programID;

  long id;

  *() {}

  *(S *programID, long *id) {}

  public bool equals(O o) {

    if (!(o instanceof BaseXRef)) false;

    BaseXRef r = cast o;

    ret eq(programID, r.programID) && eq(id, r.id);

  }

  public int hashCode() {

    ret programID.hashCode() + (int) id;

  }

}

// BaseXRef as a concept

sclass XRef extends Concept {

  BaseXRef ref;

  *() {}

  *(BaseXRef *ref) { _doneLoading2(); }

  // after we have been added to concepts

  void _doneLoading2() {

    getIndex().put(ref, this);

  }

  HashMap<BaseXRef, XRef> getIndex() {

    ret getXRefIndex(_concepts);

  }

}

static synchronized HashMap<BaseXRef, XRef> getXRefIndex(Concepts concepts) {

  HashMap cache = (HashMap) concepts.perClassData().get(XRef.class);

  if (cache == null)

    concepts.perClassData.put(XRef.class, cache = new HashMap);

  ret cache;

}

// uses mainConcepts

static XRef lookupOrCreateXRef(BaseXRef ref) {

  XRef xref = getXRefIndex(db_mainConcepts()).get(ref);

  if (xref == null)

    xref = new XRef(ref);

  ret xref;

}

// define standard concept functions to use main concepts

// Now in db_mainConcepts()

/*static void cleanMeUp_concepts() {

  if (db_mainConcepts() != null) db_mainConcepts().cleanMeUp();

  // mainConcepts = null; // TODO

}*/

svoid loadAndAutoSaveConcepts {

  db_mainConcepts().persist();

}

svoid loadAndAutoSaveConcepts(int interval) {

  db_mainConcepts().persist(interval);

}

static RC toPassRef(Concept c) {

  ret new RC(c);

}

// so we can instantiate the program to run as a bare DB bot

please include function bareDBMode.

// so unstructuring Concepts works

please include function dynamicObjectIsLoading_threadLocal.

svoid concepts_setUnlistedByDefault(bool b) {

  concepts_unlistedByDefault = b;

}