// A concept should be an object, not just a string.

// Functions that should always be there for child processes:

please include function dbLock.

static int concepts_internStringsLongerThan = 10;

static new ThreadLocal<Bool> concepts_unlisted;

sbool concepts_unlistedByDefault; // true = we can create instances of concepts with "new" without registering them automatically

static interface Derefable {

  Concept get();

}

static interface IConceptIndex {

  void update(Concept c); // also for adding

  void remove(Concept c);

}

static interface IFieldIndex<A extends Concept, Val> {

  Cl<A> getAll(Val val);

  L<Val> allValues(); // returns a cloned list

  MultiSet<Val> allValues_multiSet();

}

// Approach to persisting the Concepts object itself (in normal

// DB operation, this is not done): For simplification, speed and

// compactness, we make almost all the fields transient and store only // the concepts and the idCounter. To unstructure the Concepts object,

// use unstructureConcepts() or postUnstructureConcepts(), then

// re-set up any indices, listeners etc.

sclass Concepts implements AutoCloseable {

  Map<Long, Concept> concepts = synchroTreeMap();

  long idCounter;

  transient HashMap<Class, O> perClassData;

  transient Map miscMap; // don't use directly, call miscMap... methods to access

  // set to "-" for non-persistent (possibly not implemented)

  // also, can include a case ID ("#123/1")

  // TODO: have an actual directory instead

  transient S programID;

  transient volatile long changes, changesWritten, lastChange;

  transient volatile java.util.Timer autoSaver;

  transient volatile bool dontSave;

  transient volatile bool savingConcepts, noXFullGrab;

  transient bool vmBusSend = true;

  transient bool initialSave = true; // set to false to avoid initial useless saving

  transient int autoSaveInterval = -1000; // 1 second + wait logic

  transient bool useGZIP = true, quietSave;

  transient ReentrantLock lock = new ReentrantLock(true);

  transient ReentrantLock saverLock = new ReentrantLock(true);

  transient long lastSaveTook = -1, lastSaveWas, loadTook, uncompressedSize;

  transient float maxAutoSavePercentage = 10;

  transient L<IConceptIndex> conceptIndices;

  transient Map<Class<? extends Concept>, Map<S, IFieldIndex>> fieldIndices;

  transient Map<Class<? extends Concept>, Map<S, IFieldIndex>> ciFieldIndices;

  transient L saveActions = synchroList();

  transient O classFinder = _defaultClassFinder();

  transient L onAllChanged = synchroList(); // list of runnables

  transient O saveWrapper; // VF1<Runnable>, to profile saving

  transient bool modifyOnCreate; // set _modified == created initially

  transient bool modifyOnBackRef; // set modified if back refs change

  transient bool useFileLock = true; // instead of locking by bot

  transient FileBasedLock fileLock;

  *() {}

  *(S *programID) {}

  synchronized long internalID() {

    do {

      ++idCounter;

    } while (hasConcept(idCounter));

    ret idCounter;

  }

  synchronized HashMap<Class, O> perClassData () {

    if (perClassData == null) perClassData = new HashMap;

    ret perClassData;

  }

  void initProgramID() {

    if (programID == null)

      programID = getDBProgramID();

  }

  // Now tries to load from bot first, then go to disk.

  Concepts load() {

    ret load(false);

  }

  Concepts safeLoad() {

    ret load(true);

  }

  Concepts load(bool allDynamic) {

    initProgramID();

    // try custom grabber

    O dbGrabber = miscMapGet("dbGrabber");

    if (dbGrabber != null && !isFalse(callF(dbGrabber)))

      this;

    try {

      if (tryToGrab(allDynamic)) ret this;

    } catch e {

      if (!exceptionMessageContains(e, "no xfullgrab"))

        printShortException(e);

      print("xfullgrab failed - loading DB of " + programID + " from disk");

    }

    ret loadFromDisk(allDynamic);

  }

  Concepts loadFromDisk() { ret loadFromDisk(false); }

  Concepts loadFromDisk(bool allDynamic) {

    if (nempty(concepts)) clearConcepts();

    //DynamicObject_loading.set(true); // now done in unstructure()

    //try {

      // minimal crash recovery

      restoreLatestBackupIfConceptsFileEmpty(programID, doIt := true);

      long time = now();

      Map<Long, Concept> _concepts = concepts; // empty map

      readLocally2_allDynamic.set(allDynamic);

      temp tempSetTL(readLocally2_classFinder, classFinder);

      readLocally2(this, programID, "concepts");

      Map<Long, Concept> __concepts = concepts;

      concepts = _concepts;

      concepts.putAll(__concepts);

      int l = readLocally_stringLength;

      int tokrefs = unstructure_tokrefs;

      assignConceptsToUs();

      loadTook = now()-time;

      done("Loaded " + n(l(concepts), "concepts"), time);

      try {

        if (fileSize(getProgramFile(programID, "idCounter.structure")) != 0)

          readLocally2(this, programID, "idCounter");

        else

          calcIdCounter();

      } catch print e {

        calcIdCounter();

      }

    /*} finally {

      DynamicObject_loading.set(null);

    }*/

    if (initialSave) allChanged();

    ret this;

  }

  Concepts loadConcepts() { ret load(); }

  bool tryToGrab(bool allDynamic) {

    if (sameSnippetID(programID, getDBProgramID())) false;

    temp RemoteDB db = connectToDBOpt(programID);

    if (db != null) {

      loadGrab(db.fullgrab(), allDynamic);

      true;

    }

    false;

  }

  Concepts load(S grab) {

    ret loadGrab(grab, false);

  }

  Concepts safeLoad(S grab) {

    ret loadGrab(grab, true);

  }

  Concepts loadGrab(S grab, bool allDynamic) {

    clearConcepts();

    DynamicObject_loading.set(true);

    try {

      Map<Long, Concept> map = (Map) unstructure(grab, allDynamic, classFinder);

      concepts.putAll(map);

      assignConceptsToUs();

      for (long l : map.keySet())

        idCounter = max(idCounter, l);

    } finally {

      DynamicObject_loading.set(null);

    }

    allChanged();

    ret this;

  }

  void assignConceptsToUs() {

    // fix unstructure bugs

    for (Pair<Long, O> p: mapToPairs((Map<Long, O>) (Map) concepts))

      if (!(p.b instanceof Concept)) {

       print("DROPPING non-existant concept " + p.a + ": " + dynShortName(p.b));

       concepts.remove(p.a);

    }

    for (Concept c : values(concepts)) c._concepts = this;

    for (Concept c : values(concepts))

      c._doneLoading2(); // doneLoading2 is called on all concepts after all concepts are loaded

  }

  S progID() {

    ret programID == null ? getDBProgramID() : programID;

  }

  Concept getConcept(S id) {

    ret empty(id) ? null : getConcept(parseLong(id));

  }

  Concept getConcept(long id) {

    ret (Concept) concepts.get((long) id);

  }

  Concept getConcept(RC ref) {

    ret ref == null ? null : getConcept(ref.longID());

  }

  bool hasConcept(long id) {

    ret concepts.containsKey((long) id);

  }

  void deleteConcept(long id) {

    Concept c = getConcept(id);

    if (c == null)

      print("Concept " + id + " not found");

    else

      c.delete();

  }

  void calcIdCounter() {

    long id_ = 0;

    for (long id : keys(concepts))

      id_ = max(id_, id);

    idCounter = id_+1;

    saveLocally2(this, programID, "idCounter");

  }

  File conceptsFile() {

    ret getProgramFile(programID, useGZIP ? "concepts.structure.gz" : "concepts.structure");

  }

  // used for locking when useFileLock is activated

  File lockFile() {

    ret getProgramFile(programID, "concepts.lock");

  }

  FileBasedLock fileLock() {

    if (fileLock == null)

      fileLock = new FileBasedLock(lockFile());

    ret fileLock;

  }

  void saveConceptsIfDirty() { saveConcepts(); }

  void save() { saveConcepts(); }

  void saveConcepts() {

    if (dontSave) ret;

    initProgramID();

    saverLock.lock();

    savingConcepts = true;

    long start = now(), time;

    try {

      S s = null;

      //synchronized(main.class) {

      long _changes = changes;

      if (_changes == changesWritten) ret;

      File f = conceptsFile();

      lock.lock();

      long fullTime = now();

      try {

        saveLocally2(this, programID, "idCounter");

        if (useGZIP) {

          callRunnableWithWrapper(saveWrapper, r {

            uncompressedSize = saveGZStructureToFile(f, cloneMap(concepts));

          });

          getProgramFile(programID, "concepts.structure").delete();

        } else

          s = fullStructure();

      } finally {

        lock.unlock();

      }

      while (nempty(saveActions))

        pcallF(popFirst(saveActions));

      changesWritten = _changes; // only update when structure didn't fail

      if (!useGZIP) {

        time = now()-start;

        if (!quietSave)

          print("Saving " + toM(l(s)) + "M chars (" /*+ changesWritten + ", "*/ + time + " ms)");

        start = now();

        saveTextFile(f, javaTokWordWrap(s));

        getProgramFile(programID, "concepts.structure.gz").delete();

      }

      copyFile(f, getProgramFile(programID, "backups/concepts.structure" + (useGZIP ? ".gz" : "") + ".backup" + ymd() + "-" + formatInt(hours(), 2)));

      time = now()-start;

      if (!quietSave)

        print(programID + ": Saved " + toK(f.length()) + " K, " + n(concepts, "concepts") + " (" + time + " ms)");

      lastSaveWas = fullTime;

      lastSaveTook = now()-fullTime;

    } finally {

      savingConcepts = false;

      saverLock.unlock();

    }

  }

  void _autoSaveConcepts() {

    if (autoSaveInterval < 0 && maxAutoSavePercentage != 0) {

      long pivotTime = Math.round(lastSaveWas+lastSaveTook*100.0/maxAutoSavePercentage);

      if (now() < pivotTime) {

        //print("Skipping auto-save (last save took " + lastSaveTook + ")");

        ret;

      }

    }

    try {

      saveConcepts();

    } catch e {

      print("Concept save failed, will try again: " + e);

    }

  }

  S fullStructure() {

    ret structure(cloneMap(concepts));

  }

  void clearConcepts() {

    concepts.clear();

    allChanged();

  }

  void allChanged() {

    synchronized(this) { ++changes; lastChange = sysNow(); }

    if (vmBusSend) vmBus_send('conceptsChanged, this);

    pcallFAll(onAllChanged);

  }

  // auto-save every second if dirty

  synchronized void autoSaveConcepts() {

    if (autoSaver == null) {

      if (isTransient()) fail("Can't persist transient database");

      autoSaver = doEvery_daemon(abs(autoSaveInterval), r { _autoSaveConcepts() });

      // print("Installed auto-saver (" + autoSaveInterval + " ms, " + progID() + ")");

    }

  }

  public void close { cleanMeUp(); }

  void cleanMeUp() {

    pcall {

      bool shouldSave = autoSaver != null;

      if (autoSaver != null) {

        autoSaver.cancel();

        autoSaver = null;

      }

      while (savingConcepts) sleepInCleanUp(10);

      if (shouldSave)

        saveConceptsIfDirty();

    }

    dispose fileLock;

  }

  Map<Long, S> getIDsAndNames() {

    new Map<Long, S> map;

    Map<Long, Concept> cloned = cloneMap(concepts);

    for (long id : keys(cloned)) 

      map.put(id, cloned.get(id).className);

    ret map;

  }

  void deleteConcepts(L l) {

    ping();

    if (l != null) for (O o : cloneList(l))

      if (o instanceof Long) {

        Concept c = concepts.get(o);

        if (c != null) c.delete();

      } else if (o cast Concept)

        o.delete();

      else

        warn("Can't delete " + getClassName(o));

  }

  <A extends Concept> A conceptOfType(Class<A> type) {

    IConceptCounter counter = conceptCounterForClass(type);

    if (counter != null) ret (A) first(counter.allConcepts());

    ret firstOfType(allConcepts(), type);

  }

  <A extends Concept> L<A> conceptsOfType(Class<A> type) {

    ping();

    IConceptCounter counter = conceptCounterForClass(type);

    if (counter != null) ret (L<A>) cloneList(counter.allConcepts());

    ret filterByType(allConcepts(), type);

  }

  <A extends Concept> L<A> listConcepts(Class<A> type) {

    ret conceptsOfType(type);

  }

  <A extends Concept> L<A> list(Class<A> type) {

    ret conceptsOfType(type);

  }

  // TODO: would be better to make this Cl (indices may return sets)

  L<Concept> list(S type) {

    ret conceptsOfType(type);

  }

  L<Concept> conceptsOfType(S type) {

    ret filterByDynamicType(allConcepts(), "main$" + type);

  }

  bool hasConceptOfType(Class<? extends Concept> type) {

    ret hasType(allConcepts(), type);

  }

  void persistConcepts() {

    loadConcepts();

    autoSaveConcepts();

  }

  // We love synonyms

  void conceptPersistence() { persistConcepts(); }

  Concepts persist() { persistConcepts(); ret this; }

  void persist(Int interval) {

    if (interval != null) autoSaveInterval = interval;

    persist();

  }

  // Runs r if there is no concept of that type

  <A extends Concept> A ensureHas(Class<A> c, Runnable r) {

    A a = conceptOfType(c);

    if (a == null) {

      r.run();

      a = conceptOfType(c);

      if (a == null)

        fail("Concept not made by " + r + ": " + shortClassName(c));

    }

    ret a;

  }

  // Ensures that every concept of type c1 is ref'd by a concept of

  // type c2.

  // Type of func: voidfunc(concept)

  void ensureHas(Class<? extends Concept> c1, Class<? extends Concept> c2, O func) {

    for (Concept a : conceptsOfType(c1)) {

      Concept b = findBackRef(a, c2);

      if (b == null) {

        callF(func, a);

        b = findBackRef(a, c2);

        if (b == null)

          fail("Concept not made by " + func + ": " + shortClassName(c2));

      }

    }

  }

  // Type of func: voidfunc(concept)

  void forEvery(Class<? extends Concept> type, O func) {

    for (Concept c : conceptsOfType(type))

      callF(func, c);

  }

  int deleteAll(Class<? extends Concept> type) {

    L<Concept> l = (L) conceptsOfType(type);

    for (Concept c : l) c.delete();

    ret l(l);

  }

  Collection<Concept> allConcepts() {

    synchronized(concepts) {

      ret new L(values(concepts));

    }

  }

  IConceptCounter conceptCounterForClass(Class<? extends Concept> c) {

    for (IFieldIndex idx : values(mapGet(fieldIndices, c)))

      if (idx cast IConceptCounter) ret idx;

    for (IFieldIndex idx : values(mapGet(ciFieldIndices, c)))

      if (idx cast IConceptCounter) ret idx;

    null;

  }

  <A extends Concept> int countConcepts(Class<A> c, O... params) {

    ping();

    if (empty(params)) {

      IConceptCounter counter = conceptCounterForClass(c);

      if (counter != null) ret counter.countConcepts();

      ret l(list(c));

    }

    int n = 0;

    for (A x : list(c)) if (checkConceptFields(x, params)) ++n;

    ret n;

  }

  int countConcepts(S c, O... params) {

    ping();

    if (empty(params)) ret l(list(c));

    int n = 0;

    for (Concept x : list(c)) if (checkConceptFields(x, params)) ++n;

    ret n;

  }

  int countConcepts() {

    ret l(concepts);

  }

  synchronized void addConceptIndex(IConceptIndex index) {

    if (conceptIndices == null)

      conceptIndices = new L;

    conceptIndices.add(index);

  }

  synchronized void removeConceptIndex(IConceptIndex index) {

    if (conceptIndices == null) ret;

    conceptIndices.remove(index);

    if (empty(conceptIndices)) conceptIndices = null;

  }

  synchronized void addFieldIndex(Class<? extends Concept> c, S field, IFieldIndex index) {

    if (fieldIndices == null)

      fieldIndices = new HashMap;

    Map<S, IFieldIndex> map = fieldIndices.get(c);

    if (map == null)

      fieldIndices.put(c, map = new HashMap);

    map.put(field, index);

  }

  synchronized IFieldIndex getFieldIndex(Class<? extends Concept> c, S field) {

    if (fieldIndices == null) null;

    Map<S, IFieldIndex> map = fieldIndices.get(c);

    ret map == null ? null : map.get(field);

  }

  synchronized void addCIFieldIndex(Class<? extends Concept> c, S field, IFieldIndex index) {

    if (ciFieldIndices == null)

      ciFieldIndices = new HashMap;

    Map<S, IFieldIndex> map = ciFieldIndices.get(c);

    if (map == null)

      ciFieldIndices.put(c, map = new HashMap);

    map.put(field, index);

  }

  synchronized IFieldIndex getCIFieldIndex(Class<? extends Concept> c, S field) {

    if (ciFieldIndices == null) null;

    Map<S, IFieldIndex> map = ciFieldIndices.get(c);

    ret map == null ? null : map.get(field);

  }

  // inter-process methods

  RC xnew(S name, O... values) {

    ret new RC(cnew(name, values));

  }

  void xset(long id, S field, O value) {

    xset(new RC(id), field, value);

  }

  void xset(RC c, S field, O value) {

    if (value instanceof RC)

      value = getConcept((RC) value);

    cset(getConcept(c), field, value);

  }

  O xget(long id, S field) {

    ret xget(new RC(id), field);

  }

  O xget(RC c, S field) {

    ret xgetPost(cget(getConcept(c), field));

  }

  O xgetPost(O o) {

    o = deref(o);

    if (o instanceof Concept)

      ret new RC((Concept) o);

    ret o;

  }

  void xdelete(long id) {

    xdelete(new RC(id));

  }

  void xdelete(RC c) {

    getConcept(c).delete();

  }

  void xdelete(L<RC> l) {

    for (RC c : l)

      xdelete(c);

  }

  L<RC> xlist() {

    ret map("toPassRef", allConcepts());

  }

  L<RC> xlist(S className) {

    ret map("toPassRef", conceptsOfType(className));

  }

  bool isTransient() { ret eq(programID, "-"); }

  S xfullgrab() {

    if (noXFullGrab) fail("no xfullgrab (DB too large)");

    lock lock();

    if (changes == changesWritten && !isTransient())

      ret loadConceptsStructure(programID);

    ret fullStructure();

  }

  /* dev.

  Either<File, byte[]> xfullgrabGZipped() {

    lock lock();

    if (changes == changesWritten && !isTransient())

      ret loadConceptsStructure(programID);

    ret fullStructure();

  }*/

  void xshutdown() {

    // Killing whole VM if someone wants this DB to shut down

    cleanKillVM();

  }

  long xchangeCount() { ret changes; }

  int xcount() { ret countConcepts(); }

  void register(Concept c) {

    ping();

    if (c._concepts == this) ret;

    if (c._concepts != null) fail("Can't re-register");

    c.id = internalID();

    c.created = now();

    if (modifyOnCreate) c._setModified(c.created);

    register_phase2(c);

  }

  void register_phase2(Concept c) {

    c._concepts = this;

    concepts.put((long) c.id, c);

    for (Concept.Ref r : unnull(c.refs))

      r.index();

    c.change();

  }

  void registerKeepingID(Concept c) {

    if (c._concepts == this) ret;

    if (c._concepts != null) fail("Can't re-register");

    c._concepts = this;

    concepts.put((long) c.id, c);

    c.change();

  }

  void conceptChanged(Concept c) {

    allChanged();

    if (conceptIndices != null)

      for (IConceptIndex index : conceptIndices)

        index.update(c);

  }

  bool hasUnsavedData() {

    ret changes != changesWritten || savingConcepts;

  }

  synchronized O miscMapGet(O key) {

    ret mapGet(miscMap, key);

  }

  synchronized O miscMapPut(O key, O value) {

    if (miscMap == null) miscMap = new Map;

    ret miscMap.put(key, value);

  }

  synchronized void miscMapRemove(O key) {

    mapRemove(miscMap, key);

  }

  // Note: auto-typing can fool you, make sure create returns

  // a wide enough type

  synchronized <A> A miscMapGetOrCreate(O key, IF0<A> create) {

    if (containsKey(miscMap, key)) ret (A) miscMap.get(key);

    A value = create!;

    miscMapPut(key, value);

    ret value;

  }

} // end of Concepts

sclass Concept extends DynamicObject {

  transient Concepts _concepts; // Where we belong

  long id;

  long created, _modified;

  L<Ref> refs;

  L<Ref> backRefs;

  // used only internally (cnew)

  *(S className) {

    super(className);

    _created();

  }

  *() {

    if (!_loading()) {

      //className = shortClassName(this); // XXX - necessary?

      //print("New concept of type " + className);

      _created();

    }

  }

  *(bool unlisted) {

    if (!unlisted) _created();

  }

  toString {

    ret shortDynamicClassName(this) + " " + id;

  }

  static bool loading() { ret _loading(); }

  static bool _loading() { ret dynamicObjectIsLoading(); }

  void _created() {

    if (!concepts_unlistedByDefault && !eq(concepts_unlisted!, true))

      db_mainConcepts().register(this);

  }

  /*void put(S field, O value) {

    fieldValues.put(field, value);

    change();

  }

  O get(S field) {

    ret fieldValues.get(field);

  }*/

  class Ref<A extends Concept> implements Derefable, IF0<A> {

    A value;

    *() {

      if (!dynamicObjectIsLoading()) refs = addDyn_quickSync(refs, this);

    }

    *(A *value) {

      refs = addDyn_quickSync(refs, this);

      index();

    }

    // get owning concept (source)

    Concept concept() {

      ret Concept.this;

    }

    // get target

    public A get() { ret value; }

    bool has() { ret value != null; }

    bool set(A a) {

      if (a == value) false;

      unindex();

      value = a;

      index();

      true;

    }

    void set(Ref<A> ref) { set(ref.get()); }

    void clear() { set((A) null); }

    bool validRef() {

      ret value != null && _concepts != null && _concepts == value._concepts;

    }

    // TODO: sync all the indexing and unindexing!?

    void index() { 

      if (validRef()) {

        value._addBackRef(this);

        change();

      }

    }

    Ref<A> unindex() {

      if (validRef()) {

        value._removeBackRef(this);

        change();

      }

      this;

    }

    void unindexAndDrop {

      unindex();

      _removeRef(this);

    }

    void change() {

      Concept.this.change();

    }

    toString { ret str(value); }

  }

  class RefL<A extends Concept> extends AbstractList<A> {

    new L<Ref<A>> l;

    public void clear {

      while (!isEmpty()) removeLast(this);

    }

    public void replaceWithList(L<A> l) {

      clear();

      fOr (A a : l) add(a);

    }

    public A set(int i, A o) {

      Ref<A> ref = syncGet(l, i);

      A prev = ref!;

      ref.set(o);

      ret prev;

    }

    public void add(int i, A o) {

      syncAdd(l, i, new Ref(o));

    }

    public A get(int i) {

      ret syncGet(l, i)!;

    }

    public A remove(int i) {

      ret syncRemove(l, i)!;

    }

    public int size() {

      ret syncL(l);

    }

    public bool contains(O o) {

      if (o instanceof Concept)

        for (Ref<A> r : l) if (eq(r!, o)) true;

      ret super.contains(o);

    }

  }

  void delete() {

    //name = "[defunct " + name + "]";

    //defunct = true;

    //energy = 0;

    // clean refs

    for (Ref r : unnull(refs))

      r.unindex();

    refs = null;

    // set back refs to null

    for (Ref r : cloneList(backRefs))

      r.set((Concept) null);

    backRefs = null;

    if (_concepts != null) {

      _concepts.concepts.remove((long) id);

      _concepts.allChanged();

      if (_concepts.conceptIndices != null)

        for (IConceptIndex index : _concepts.conceptIndices)

          index.remove(this);

      _concepts = null;

    }

    id = 0;

  }

  BaseXRef export() {

    ret new BaseXRef(_concepts.progID(), id);

  }

  // notice system of a change in this object

  void change() {

    _setModified(now());

    _change_withoutUpdatingModifiedField();

  }

  void _setModified(long modified) {

    _modified = modified;

  }

  void _change_withoutUpdatingModifiedField() {

    if (_concepts != null) _concepts.conceptChanged(this);

  }

  void _change() { change(); }

  S _programID() {

    ret _concepts == null ? getDBProgramID() : _concepts.progID();

  }

  // overridable

  void _addBackRef(Concept.Ref ref) {

    backRefs = addDyn_quickSync(backRefs, ref);

    _backRefsModified();

  }

  void _backRefsModified {

    if (_concepts != null && _concepts.modifyOnBackRef) change();

  }

  void _removeBackRef(Concept.Ref ref) {

    backRefs = removeDyn_quickSync(backRefs, ref);

    _backRefsModified();

  }

  void _removeRef(Concept.Ref ref) {

    refs = removeDyn_quickSync(refs, ref);

  }

  // convenience methods

  void _setField(S field, O value) {

    cset(this, field, value);

  }

  void _setFields(O... values) {

    cset(this, values);

  }

  Concepts concepts() { ret _concepts; }

  bool isDeleted() { ret id == 0; }

  void _doneLoading2() {

    Map<S, FieldMigration> map = _fieldMigrations();

    if (map != null) for (S oldField, FieldMigration m : map)

      crenameField_noOverwrite(this, oldField, m.newField);

  }

  srecord FieldMigration(S newField) {}

  // value is 

  Map<S, FieldMigration> _fieldMigrations() { null; }

} // end of Concept

// remote reference (for inter-process communication or

// external databases). Formerly "PassRef".

// prepared for string ids if we do them later

sclass RC {

  transient O owner;

  S id;

  *() {} // make serialisation happy

  *(long id) { this.id = str(id); }

  *(O owner, long id) { this.id = str(id); this.owner = owner; }

  *(Concept c) { this(c.id); }

  long longID() { ret parseLong(id); }

  public S toString() {

    ret id;

  }

}

// Reference to a concept in another program

sclass BaseXRef {

  S programID;

  long id;

  *() {}

  *(S *programID, long *id) {}

  public bool equals(O o) {

    if (!(o instanceof BaseXRef)) false;

    BaseXRef r = cast o;

    ret eq(programID, r.programID) && eq(id, r.id);

  }

  public int hashCode() {

    ret programID.hashCode() + (int) id;

  }

}

// BaseXRef as a concept

sclass XRef extends Concept {

  BaseXRef ref;

  *() {}

  *(BaseXRef *ref) { _doneLoading2(); }

  // after we have been added to concepts

  void _doneLoading2() {

    getIndex().put(ref, this);

  }

  HashMap<BaseXRef, XRef> getIndex() {

    ret getXRefIndex(_concepts);

  }

}

static synchronized HashMap<BaseXRef, XRef> getXRefIndex(Concepts concepts) {

  HashMap cache = (HashMap) concepts.perClassData().get(XRef.class);

  if (cache == null)

    concepts.perClassData.put(XRef.class, cache = new HashMap);

  ret cache;

}

// uses mainConcepts

static XRef lookupOrCreateXRef(BaseXRef ref) {

  XRef xref = getXRefIndex(db_mainConcepts()).get(ref);

  if (xref == null)

    xref = new XRef(ref);

  ret xref;

}

// define standard concept functions to use main concepts

// Now in db_mainConcepts()

/*static void cleanMeUp_concepts() {

  if (db_mainConcepts() != null) db_mainConcepts().cleanMeUp();

  // mainConcepts = null; // TODO

}*/

svoid loadAndAutoSaveConcepts {

  db_mainConcepts().persist();

}

svoid loadAndAutoSaveConcepts(int interval) {

  db_mainConcepts().persist(interval);

}

static RC toPassRef(Concept c) {

  ret new RC(c);

}

// so we can instantiate the program to run as a bare DB bot

please include function bareDBMode.

// so unstructuring Concepts works

please include function dynamicObjectIsLoading_threadLocal.

svoid concepts_setUnlistedByDefault(bool b) {

  concepts_unlistedByDefault = b;

}