// A concept should be an object, not just a string.

static int concepts_internStringsLongerThan = 10;
static new ThreadLocal<Bool> concepts_unlisted;

static interface Derefable {
  Concept get();
}

static interface IConceptIndex {
  void update(Concept c); // also for adding
  void remove(Concept c);
}

sclass Concepts {
  Map<Long, Concept> concepts = synchroTreeMap();
  new HashMap<Class, O> perClassData;
  S programID; // set to "-" for non-persistent (possibly not implemented)
  long idCounter;
  volatile long changes = 1, changesWritten;
  volatile java.util.Timer autoSaver;
  volatile bool savingConcepts;
  int autoSaveInterval = -1000; // 1 second + wait logic
  bool useGZIP = true, quietSave;
  ReentrantLock lock = new ReentrantLock(true);
  ReentrantLock saverLock = new ReentrantLock(true);
  long lastSaveTook, lastSaveWas;
  float maxAutoSavePercentage = 10;
  L<IConceptIndex> conceptIndices;
  
  *() {}
  *(S *programID) {}
  
  synchronized long internalID() {
    do {
      ++idCounter;
    } while (hasConcept(idCounter));
    ret idCounter;
  }
  
  void initProgramID() {
    if (programID == null)
      programID = getDBProgramID();
  }
  
  // Now tries to load from bot first, then go to disk.
  Concepts load() {
    ret load(false);
  }
  
  Concepts safeLoad() {
    ret load(true);
  }
  
  Concepts load(bool allDynamic) {
    initProgramID();
    if (tryToGrab(allDynamic)) ret this;
    ret loadFromDisk(allDynamic);
  }
  
  Concepts loadFromDisk() { ret loadFromDisk(false); }
  
  Concepts loadFromDisk(bool allDynamic) {
    clearConcepts();
    DynamicObject_loading.set(true);
    try {
      long time = now();
      Map<Long, Concept> _concepts = concepts; // empty map
      readLocally2_allDynamic.set(allDynamic);
      readLocally2(this, programID, "concepts");
      Map<Long, Concept> __concepts = concepts;
      concepts = _concepts;
      concepts.putAll(__concepts);
      int l = readLocally_stringLength;
      int tokrefs = unstructure_tokrefs;
      assignConceptsToUs();
      done("Loaded " + n(l(concepts), "concepts"), time);
      if (fileSize(getProgramFile(programID, "idCounter.structure")) != 0)
        readLocally2(this, programID, "idCounter");
      else
        calcIdCounter();
    } finally {
      DynamicObject_loading.set(null);
    }
    allChanged();
    ret this;
  }
  
  Concepts loadConcepts() { ret load(); }
  
  bool tryToGrab(bool allDynamic) {
    if (sameSnippetID(programID, getDBProgramID())) false;
    RemoteDB db = connectToDBOpt(programID);
    try {
      if (db != null) {
        loadGrab(db.fullgrab(), allDynamic);
        true;
      }
    } finally {
      if (db != null) db.close();
    }
    false;
  }
  
  Concepts load(S grab) {
    ret loadGrab(grab, false);
  }
  
  Concepts safeLoad(S grab) {
    ret loadGrab(grab, true);
  }
  
  Concepts loadGrab(S grab, bool allDynamic) {
    clearConcepts();
    DynamicObject_loading.set(true);
    try {
      Map<Long, Concept> map = (Map)
        (allDynamic ? safeUnstructure(grab) : unstructure(grab));
      concepts.putAll(map);
      assignConceptsToUs();
      for (long l : map.keySet())
        idCounter = max(idCounter, l);
    } finally {
      DynamicObject_loading.set(null);
    }
    allChanged();
    ret this;
  }
  
  void assignConceptsToUs() {
    for (Concept c : values(concepts)) {
      c._concepts = this;
      callOpt_noArgs(c, "_doneLoading2");
    }
  }

  S progID() {
    ret programID == null ? getDBProgramID() : programID;
  }
  
  Concept getConcept(S id) {
    ret empty(id) ? null : getConcept(parseLong(id));
  }
  
  Concept getConcept(long id) {
    ret (Concept) concepts.get((long) id);
  }
  
  Concept getConcept(RC ref) {
    ret ref == null ? null : getConcept(ref.longID());
  }
  
  bool hasConcept(long id) {
    ret concepts.containsKey((long) id);
  }
  
  void deleteConcept(long id) {
    Concept c = getConcept(id);
    if (c == null)
      print("Concept " + id + " not found");
    else
      c.delete();
  }
  
  void calcIdCounter() {
    long id_ = 0;
    for (long id : keys(concepts))
      id_ = max(id_, id);
    idCounter = id_+1;
    saveLocally2(this, programID, "idCounter");
  }
  
  void saveConceptsIfDirty() { saveConcepts(); }
  void save() { saveConcepts(); }

  void saveConcepts() {
    initProgramID();
    saverLock.lock();
    savingConcepts = true;
    long start = now(), time;
    try {
      S s = null;
      //synchronized(main.class) {
      File f = getProgramFile(programID, useGZIP ? "concepts.structure.gz" : "concepts.structure");
      long _changes = changes;
      if (_changes == changesWritten) ret;
      
      lock.lock();
      long fullTime = now();
      try {
        saveLocally2(this, programID, "idCounter");
        
        if (useGZIP) {
          saveGZStructureToFile(f, cloneMap(concepts));
          getProgramFile(programID, "concepts.structure").delete();
        } else
          s = structure(cloneMap(concepts));
      } finally {
        lock.unlock();
      }

      changesWritten = _changes; // only update when structure didn't fail
      
      if (!useGZIP) {
        time = now()-start;
        if (!quietSave)
          print("Saving " + toM(l(s)) + "M chars (" /*+ changesWritten + ", "*/ + time + " ms)");
        start = now();
        saveTextFile(f, javaTokWordWrap(s));
        getProgramFile(programID, "concepts.structure.gz").delete();
      }
      
      copyFile(f, getProgramFile(programID, "concepts.structure" + (useGZIP ? ".gz" : "") + ".backup" + ymd() + "-" + formatInt(hours(), 2)));
      time = now()-start;
      if (!quietSave)
        print(programID + ": Saved " + toK(f.length()) + " K, " + n(concepts, "concepts") + " (" + time + " ms)");
      lastSaveWas = fullTime;
      lastSaveTook = now()-fullTime;
    } finally {
      savingConcepts = false;
      saverLock.unlock();
    }
  }
  
  void _autoSaveConcepts() {
    if (autoSaveInterval < 0 && maxAutoSavePercentage != 0) {
      long pivotTime = Math.round(lastSaveWas+lastSaveTook*100.0/maxAutoSavePercentage);
      if (now() < pivotTime) {
        //print("Skipping auto-save (last save took " + lastSaveTook + ")");
        ret;
      }
    }
    saveConcepts();
  }
  
  void clearConcepts() {
    concepts.clear();
    allChanged();
  }
  
  synchronized void allChanged() {
    ++changes;
  }
  
  // auto-save every second if dirty
  synchronized void autoSaveConcepts() {
    if (autoSaver == null) {
      if (isTransient()) fail("Can't persist transient database");
      autoSaver = doEvery_daemon(abs(autoSaveInterval), r { _autoSaveConcepts() });
      // print("Installed auto-saver (" + autoSaveInterval + " ms, " + progID() + ")");
    }
  }
  
  void cleanMeUp() {
    if (autoSaver != null) {
      autoSaver.cancel();
      autoSaver = null;
      while (savingConcepts) sleepInCleanUp(10);
      saveConceptsIfDirty();
    }
  }
  
  Map<Long, S> getIDsAndNames() {
    new Map<Long, S> map;
    Map<Long, Concept> cloned = cloneMap(concepts);
    for (long id : keys(cloned)) 
      map.put(id, cloned.get(id).className);
    ret map;
  }
  
  void deleteConcepts(L l) {
    for (O o : l)
      if (o instanceof Long)
        concepts.remove((Long) o);
      else if (o instanceof Concept)
        ((Concept) o).delete();
      else
        warn("Can't delete " + getClassName(o));
  }
  
  <A extends Concept> A conceptOfType(Class<A> type) {
    ret firstOfType(allConcepts(), type);
  }
  
  <A extends Concept> L<A> conceptsOfType(Class<A> type) {
    ret filterByType(allConcepts(), type);
  }
  
  <A extends Concept> L<A> listConcepts(Class<A> type) {
    ret conceptsOfType(type);
  }
  
  <A extends Concept> L<A> list(Class<A> type) {
    ret conceptsOfType(type);
  }
  
  L<Concept> list(S type) {
    ret conceptsOfType(type);
  }
  
  L<Concept> conceptsOfType(S type) {
    ret filterByDynamicType(allConcepts(), "main$" + type);
  }
  
  bool hasConceptOfType(Class<? extends Concept> type) {
    ret hasType(allConcepts(), type);
  }
  
  void persistConcepts() {
    loadConcepts();
    autoSaveConcepts();
  }
  
  // We love synonyms
  void conceptPersistence() {
    persistConcepts();
  }
  
  void persist() { persistConcepts(); }
  void persist(int interval) { autoSaveInterval = interval; persist(); }
    
  // Runs r if there is no concept of that type
  <A extends Concept> A ensureHas(Class<A> c, Runnable r) {
    A a = conceptOfType(c);
    if (a == null) {
      r.run();
      a = conceptOfType(c);
      if (a == null)
        fail("Concept not made by " + r + ": " + shortClassName(c));
    }
    ret a;
  }
  
  // Ensures that every concept of type c1 is ref'd by a concept of
  // type c2.
  // Type of func: voidfunc(concept)
  void ensureHas(Class<? extends Concept> c1, Class<? extends Concept> c2, O func) {
    for (Concept a : conceptsOfType(c1)) {
      Concept b = findBackRef(a, c2);
      if (b == null) {
        callF(func, a);
        b = findBackRef(a, c2);
        if (b == null)
          fail("Concept not made by " + func + ": " + shortClassName(c2));
      }
    }
  }
  
  // Type of func: voidfunc(concept)
  void forEvery(Class<? extends Concept> type, O func) {
    for (Concept c : conceptsOfType(type))
      callF(func, c);
  }
  
  int deleteAll(Class<? extends Concept> type) {
    L<Concept> l = (L) conceptsOfType(type);
    for (Concept c : l) c.delete();
    ret l(l);
  }
  
  Collection<Concept> allConcepts() {
    synchronized(concepts) {
      ret new L(values(concepts));
    }
  }
  
  <A extends Concept> int countConcepts(Class<A> c, O... params) {
    int n = 0;
    for (A x : list(c)) if (checkConceptFields(x, params)) ++n;
    ret n;
  }

  int countConcepts(S c, O... params) {
    int n = 0;
    for (Concept x : list(c)) if (checkConceptFields(x, params)) ++n;
    ret n;
  }

  int countConcepts() {
    ret l(concepts);
  }  
  
  // inter-process methods
  
  RC xnew(S name, O... values) {
    ret new RC(cnew(name, values));
  }
  
  void xset(long id, S field, O value) {
    xset(new RC(id), field, value);
  }
  
  void xset(RC c, S field, O value) {
    if (value instanceof RC)
      value = getConcept((RC) value);
    cset(getConcept(c), field, value);
  }
  
  O xget(long id, S field) {
    ret xget(new RC(id), field);
  }
  
  O xget(RC c, S field) {
    ret xgetPost(cget(getConcept(c), field));
  }
  
  O xgetPost(O o) {
    o = deref(o);
    if (o instanceof Concept)
      ret new RC((Concept) o);
    ret o;
  }
  
  void xdelete(long id) {
    xdelete(new RC(id));
  }
  
  void xdelete(RC c) {
    getConcept(c).delete();
  }
  
  void xdelete(L<RC> l) {
    for (RC c : l)
      xdelete(c);
  }
  
  L<RC> xlist() {
    ret map("toPassRef", allConcepts());
  }
  
  L<RC> xlist(S className) {
    ret map("toPassRef", conceptsOfType(className));
  }
  
  bool isTransient() { ret eq(programID, "-"); }
  
  S xfullgrab() {
    lock.lock();
    try {
      if (changes == changesWritten && !isTransient())
        ret loadConceptsStructure(programID);
      ret structure(cloneMap(concepts));
    } finally {
      lock.unlock();
    }
  }
  
  void xshutdown() {
    // Killing whole VM if someone wants this DB to shut down
    cleanKillVM();
  }
  
  long xchangeCount() { ret changes; }
  int xcount() { ret countConcepts(); }
  
  void register(Concept c) {
    if (c._concepts == this) ret;
    if (c._concepts != null) fail("Can't re-register");
    c._concepts = this;
    c.id = internalID();
    c.created = now();
    concepts.put((long) c.id, c);
    c.change();
  }
  
} // class Concepts

static volatile new Concepts mainConcepts; // Where we create new concepts

sclass Concept extends DynamicObject {
  transient Concepts _concepts; // Where we belong
  long id;
  //O madeBy;
  //double energy;
  //bool defunct;
  long created;
  
  // used only internally (cnew)
  *(S className) {
    super(className);
    _created();
  }
  
  *() {
    if (!_loading()) {
      //className = shortClassName(this); // XXX - necessary?
      //print("New concept of type " + className);
      _created();
    }
  }
  
  L<Ref> refs;
  L<Ref> backRefs;
  
  static bool loading() { ret _loading(); }
  static bool _loading() { ret isTrue(DynamicObject_loading.get()); }

  void _created() {
    if (!isTrue(concepts_unlisted.get())) mainConcepts.register(this);
  }
  
  /*void put(S field, O value) {
    fieldValues.put(field, value);
    change();
  }
  
  O get(S field) {
    ret fieldValues.get(field);
  }*/
  
  class Ref<A extends Concept> {
    A value;
    
    *() {
      if (!isTrue(DynamicObject_loading.get())) refs = addDyn(refs, this);
    }
    
    *(A *value) {
      refs = addDyn(refs, this);
      index();
    }
    
    // get owning concept (source)
    Concept concept() {
      ret Concept.this;
    }
    
    // get target
    A get() { ret value; }
    bool has() { ret value != null; }
    
    void set(A a) {
      if (a == value) ret;
      unindex();
      value = a;
      index();
    }
    
    void set(Ref<A> ref) { set(ref.get()); }
    void clear() { set((A) null); }
    
    void index() { 
      if (value != null)
        value.backRefs = addDyn(value.backRefs, this);
      change();
    }
    
    void unindex() {
      if (value != null)
        value.backRefs = removeDyn(value.backRefs, this);
    }
    
    void change() {
      Concept.this.change();
    }
  }
  
  class RefL<A extends Concept> extends AbstractList<A> {
    new L<Ref<A>> l;
    
    public A set(int i, A o) {
      A prev = l.get(i).get();
      l.get(i).set(o);
      ret prev;
    }
    
    public void add(int i, A o) {
      l.add(i, new Ref(o));
    }
    
    public A get(int i) {
      ret l.get(i).get();
    }
    
    public A remove(int i) {
      ret l.remove(i).get();
    }
    
    public int size() {
      ret l.size();
    }
    
    public bool contains(O o) {
      if (o instanceof Concept)
        for (Ref<A> r : l) if (eq(r!, o)) true;
      ret super.contains(o);
    }
  }
  
  void delete() {
    //name = "[defunct " + name + "]";
    //defunct = true;
    //energy = 0;
    for (Ref r : unnull(refs))
      r.unindex();
    refs = null;
    for (Ref r : cloneList(backRefs))
      r.set((Concept) null);
    backRefs = null;
    
    if (_concepts != null) {
      _concepts.concepts.remove((long) id);
      _concepts.allChanged();
      if (_concepts.conceptIndices != null)
        for (IConceptIndex index : _concepts.conceptIndices)
          index.remove(this);
      _concepts = null;
    }
    id = 0;
  }
  
  BaseXRef export() {
    ret new BaseXRef(_concepts.progID(), id);
  }
  
  // notice system of a change in this object
  void change() {
    if (_concepts != null) {
      _concepts.allChanged();
      if (_concepts.conceptIndices != null)
        for (IConceptIndex index : _concepts.conceptIndices)
          index.update(this);
    }
  }
  
  S _programID() {
    ret _concepts == null ? getDBProgramID() : _concepts.progID();
  }
} // class Concept

// remote reference (for inter-process communication or
// external databases). Formerly "PassRef".
// prepared for string ids if we do them later
sclass RC {
  transient O owner;
  S id;
  
  *() {} // make serialisation happy
  *(long id) { this.id = str(id); }
  *(O owner, long id) { this.id = str(id); this.owner = owner; }
  *(Concept c) { this(c.id); }
  long longID() { ret parseLong(id); }
  
  public S toString() {
    ret id;
  }
}

// Reference to a concept in another program
sclass BaseXRef {
  S programID;
  long id;
    
  *() {}
  *(S *programID, long *id) {}
  
  public bool equals(O o) {
    if (!(o instanceof BaseXRef)) false;
    BaseXRef r = cast o;
    ret eq(programID, r.programID) && eq(id, r.id);
  }
  
  public int hashCode() {
    ret programID.hashCode() + (int) id;
  }
}

// BaseXRef as a concept
sclass XRef extends Concept {
  BaseXRef ref;
  
  *() {}
  *(BaseXRef *ref) { _doneLoading2(); }
  
  // after we have been added to concepts
  void _doneLoading2() {
    getIndex().put(ref, this);
  }
    
  HashMap<BaseXRef, XRef> getIndex() {
    ret getXRefIndex(_concepts);
  }
}

static synchronized HashMap<BaseXRef, XRef> getXRefIndex(Concepts concepts) {
  HashMap cache = (HashMap) concepts.perClassData.get(XRef.class);
  if (cache == null)
    concepts.perClassData.put(XRef.class, cache = new HashMap);
  ret cache;
}

// uses mainConcepts
static XRef lookupOrCreateXRef(BaseXRef ref) {
  XRef xref = getXRefIndex(mainConcepts).get(ref);
  if (xref == null)
    xref = new XRef(ref);
  ret xref;
}

// define standard concept functions to use main concepts

static <A extends Concept> L<A> list(Class<A> type) {
  ret mainConcepts.list(type);
}

static <A extends Concept> L<A> list(Concepts concepts, Class<A> type) {
  ret concepts.list(type);
}

static L<Concept> list(S type) {
  ret mainConcepts.list(type);
}

static L<Concept> list(Concepts concepts, S type) {
  ret concepts.list(type);
}

static int csetAll(Concept c, O... values) {
  ret cset(c, values);
}

// returns number of changes
static int cset(Concept c, O... values) ctex {
  int changes = 0;
  values = expandParams(c.getClass(), values);
  warnIfOddCount(values);
  for (int i = 0; i+1 < l(values); i += 2) {
    S field = (S) values[i];
    O value = values[i+1];
    Field f = setOpt_findField(c.getClass(), field);
    //print("cset: " + c.id + " " + field + " " + struct(value) + " " + f);
    if (value instanceof RC) value = c._concepts.getConcept((RC) value);
    value = deref(value);
    
    if (value instanceof S && l((S) value) >= concepts_internStringsLongerThan) value = ((S) value).intern();
    
    if (f == null) {
      // TODO: keep ref if it exists
      mapPut2(c.fieldValues, field, value instanceof Concept ? c.new Ref((Concept) value) : value);
      c.change();
    } else if (isSubtypeOf(f.getType(), Concept.Ref.class)) {
      ((Concept.Ref) f.get(c)).set((Concept) derefRef(value));
      c.change(); ++changes;
    } else {
      O old = f.get(c);
      if (neq(value, old)) {
        f.set(c, value);
        c.change(); ++changes;
      }
    }
  }
  ret changes;
}

static void cleanMeUp_concepts() {
  mainConcepts.cleanMeUp();
}

svoid loadAndAutoSaveConcepts {
  mainConcepts.persist();
}

svoid loadAndAutoSaveConcepts(int interval) {
  mainConcepts.persist(interval);
}

svoid loadConceptsFrom(S progID) {
  mainConcepts.programID = progID;
  mainConcepts.load();
}

static L<Concept> conceptsOfType(S type) {
  ret mainConcepts.conceptsOfType(type);
}

static long changeCount() {
  ret mainConcepts.changes;
}

static L<S> exposedDBMethods = ll("xlist", "xnew", "xset", "xdelete", "xget", "xclass", "xfullgrab", "xshutdown", "xchangeCount", "xcount");

static RC toPassRef(Concept c) {
  ret new RC(c);
}

// so we can instantiate the program to run as a bare DB bot
please include function bareDBMode.