Not logged in.  Login/Logout/Register | List snippets | | Create snippet | Upload image | Upload data

252
LINES

< > BotCompany Repo | #1012330 // TripleIndex with direct link to triples [LIVE]

JavaX fragment (include)

sclass TripleIndex extends VirtualNodeIndex {
  static bool useExactIndex; // experimental
  bool activated; // are we the main index (see tripleIndex())
  
  // index of word in any position - not used anymore
  SyncListMultiMap<Symbol, TripleWeb> index;// = symbolSyncListMultiMap();
  
  // count per word
  MultiSet<Symbol> wordIndex =
    //caseInsensitiveCompactMultiSet(); // not a NavigableMap
    caseInsensitiveMultiSet();
  
  // word in position
  SyncListMultiMap<Symbol, TripleWeb>[] positionalIndices = new SyncListMultiMap[] {
    symbolSyncListMultiMap(),
    symbolSyncListMultiMap(),
    symbolSyncListMultiMap() };
    
  // all three words
  ExactTripleIndex exactIndex = useExactIndex ? exactTripleIndex() : null;
  
  // triples by ID
  new CompactHashSet<TripleWeb> websByID;
  
  // first word -> relation -> triple
  Map<Symbol, SyncListMultiMap<Symbol, TripleWeb>> oneTwoIndex; // = (Map) symbolMap();
  // SyncListMultiMap<Pair<Symbol>, TripleWeb> oneTwoIndex2 = new SyncListMultiMap; // works, but wastes space
  SyncListMultiMap<Pair<Symbol>, TripleWeb> oneTwoIndex2 = new SyncListMultiMap(new CompactPairKeyHashMap); // try compact version

  int size() { ret numWebs(); }
  int numWebs() { ret l(websByID); }
  int numTerms() { ret index != null ? index.keysSize() : wordIndex.uniqueSize(); }
  
  Set<Symbol> mainKeys() {
    ret index != null ? keys(index) : wordIndex.asSet();
  }
  
  NavigableSet<Symbol> navigableMainKeys() {
    ret index != null ? navigableKeys(index) : wordIndex.navigableSet();
  }
  
  // for synchronizing, e.g. in fullIndexedTermsStartingWith
  Map mainIndexMap() {
    ret index != null ? index.data : wordIndex.map;
  }
  
  Collection<Symbol> indexedTerms() { ret mainKeys(); }
  
  // query is shortened term
  L<WebNode> get(CharSequence _query) {
    Symbol query = symbol(_query);
    L<TripleWeb> triples = index != null ? index.get(query)
      : combineLists3(getTriples(query, 0), getTriples(query, 1), getTriples(query, 2));
    ret ai_triplesToWebNodes_lazyList(query, triples);
  }
  
  // terms are NOT shortened
  /*TripleWeb getExact(Symbol a, Symbol b, Symbol c) {
    if (exactIndex == null) null;
    ret exactIndex.find(dummyTripleWebWithEntries(a, b, c));
  }*/

  // query is shortened term
  L<TripleRef<Symbol>> getTripleRefs(Symbol query) {
    if (index == null) fail("no index");
    ret ai_triplesToTripleRefs_lazyList(query, index.get(query));
  }
  
  L<TripleRef<Symbol>> getTripleRefs(Symbol query, int position) {
    ret ai_triplesToTripleRefs_lazyList(query, getTriples(query, position));
  }
  
  // query is shortened term
  L<TripleWeb> getTriples(Symbol query) {
    if (index == null) fail("no index");
    ret index.get(query);
  }
  
  // query is shortened term
  L<TripleWeb> getTriples(Symbol query, int position) {
    ret positionalIndices[position].get(query);
  }
  
  bool hasShortTerm(Symbol s) {
    ret index != null ? index.containsKey(s) : wordIndex.contains(s); }

  Web getWeb(GlobalID id) {
    ret webFromTriple(getTriple(id));
  }
  
  TripleWeb getTriple(GlobalID id) {
    ret websByID.find(dummyTripleWebWithGlobalID(id));
  }
  
  void addWeb(Web web) {
    if (web == null) ret;
    TripleWeb w = ai_webToTripleWeb(web);
    if (w == null)
      fail("Skipping non-tripelizable web: " + webToStringShort(web));
    addTriple(w);
  }
  
  void addTriple(TripleWeb w) {
    if (w == null) ret;
    if (index != null)
      for (Symbol s : sortedInPlace(ll(w.a, w.b, w.c)))
        index.put(s, w);
    if (wordIndex != null) {
      wordIndex.add(w.a);
      wordIndex.add(w.b);
      wordIndex.add(w.c);
    }
    positionalIndices[0].put(w.a, w);
    positionalIndices[1].put(w.b, w);
    positionalIndices[2].put(w.c, w);
    if (exactIndex != null) exactIndex.add(w);
    if (oneTwoIndex != null) synchronized(oneTwoIndex) {
      SyncListMultiMap<Symbol, TripleWeb> map = oneTwoIndex.get(w.a);
      if (map == null) oneTwoIndex.put(w.a, map = symbolSyncListMultiMap());
      map.put(w.b, w);
    }
    if (oneTwoIndex2 != null) oneTwoIndex2.put(pair(w.a, w.b), w);
    websByID.add(w);
    if (activated)
      ai_fireNewTriple(w);
  }
  
  void removeWeb(Web web) {
    if (web != null)
      removeTriple(ai_webToTripleWeb(web));
  }    
    
  void removeTriples(Collection<TripleWeb> l) {
    new MultiMap<Symbol, TripleWeb> mm;
    for (TripleWeb w : l) {
      mm.put(ai_shortenForIndex(w.a), w);
      mm.put(ai_shortenForIndex(w.b), w);
      mm.put(ai_shortenForIndex(w.c), w);
      removeFromSets(w);
    }
    for (Symbol term : keys(mm)) {
      HashSet<TripleWeb> webs = asHashSet(mm.get(term));
      if (index != null) indexRemoveMulti(index, term, webs);
      for i to 3:
        indexRemoveMulti(positionalIndices[i], term, webs);
    }
    if (activated)
      for (TripleWeb w : l)
        ai_fireForgottenTriple(w);
  }
  
  // internal
  void removeFromSets(TripleWeb w) {
    websByID.remove(w);
    if (exactIndex != null) exactIndex.remove(w);
    if (oneTwoIndex != null) synchronized(oneTwoIndex) {
      SyncListMultiMap<Symbol, TripleWeb> map = oneTwoIndex.get(w.a);
      if (map != null) {
        map.remove(w.a, w);
        if (empty(map)) oneTwoIndex.remove(w.a);
      }
    }
    if (oneTwoIndex2 != null) oneTwoIndex2.remove(pair(w.a, w.b), w);
    if (wordIndex != null) {
      wordIndex.remove(w.a);
      wordIndex.remove(w.b);
      wordIndex.remove(w.c);
    }
  }
  
  void removeTriple(TripleWeb w) {
    if (w == null) ret;
    removeFromSets(w);
    GlobalID id = w.globalID();
    indexRemove(0, ai_shortenForIndex(w.a), id);
    indexRemove(1, ai_shortenForIndex(w.b), id);
    indexRemove(2, ai_shortenForIndex(w.c), id);
    if (activated)
      ai_fireForgottenTriple(w);
  }
  
  // internal
  void indexRemove(int position, Symbol term, GlobalID globalID) {
    if (index != null) indexRemove2(index, term, globalID);
    indexRemove2(positionalIndices[position], term, globalID);
  }
    
  // remove from a single list
  void indexRemove2(MultiMap<Symbol, TripleWeb> mm, Symbol term, GlobalID globalID) {
    L<TripleWeb> l = mm.get(term);
    for i over l:
      if (eq(l.get(i).globalID(), globalID)) {
        mm.remove(term, l.get(i));
        ret;
      }
  }
  
  // internal
  void indexRemoveMulti(MultiMap<Symbol, TripleWeb> mm, Symbol term, Set<TripleWeb> set) {
    L<TripleWeb> l = mm.get(term);
    synchronized(l) {
      removeSetFromListQuickly(l, set);
    }
  }
  
  void clear() {
    if (index != null) index.clear();
    if (wordIndex != null) wordIndex.clear();
    websByID.clear();
    if (exactIndex != null) exactIndex.clear();
    if (oneTwoIndex != null) oneTwoIndex.clear();
    if (oneTwoIndex2 != null) oneTwoIndex2.clear();
  }
  
  void activate {
    if (!activated) {
      activated = true;
      ai_onNewOrRemovedWeb(func(Web web) {
        addWeb(web);
        false;
      }, func(Web web) {
        removeWeb(web);
        false;
      });
    }
  }
  
  Collection<TripleWeb> allTriples() { ret websByID; }
  
  void trimToSize() {
    if (index != null) trimToSizeAll(index.allLists());
    for i to 3: trimToSizeAll(positionalIndices[i].allLists());
    for (SyncListMultiMap<Symbol, TripleWeb> map : values(oneTwoIndex))       trimToSizeAll(map.allLists());
  }
  
  void sortAllLists() {
    if (index != null) sortLists(index.allLists());
    for i to 3: sortLists(positionalIndices[i].allLists());
    for (SyncListMultiMap<Symbol, TripleWeb> map : values(oneTwoIndex))       sortLists(map.allLists());
  }
  
  void sortLists(Collection<L<TripleWeb>> ll) {
    for (L<TripleWeb> l : ll)
      ai_sortTriplesListByPriority_inPlace(l);
  }
  
  L<TripleWeb> getOneTwo(Symbol a, Symbol b) {
    if (oneTwoIndex2 != null)
      ret oneTwoIndex2.get(pair(a, b));
    ret shortestList2(getTriples(a), getTriples(b));
  }
}

Author comment

Began life as a copy of #1012323

download  show line numbers  debug dex  old transpilations   

Travelled to 14 computer(s): aoiabmzegqzx, bhatertpkbcr, cbybwowwnfue, cfunsshuasjs, gwrvuhgaqvyk, ishqpsrjomds, lpdgvwnxivlt, mqqgnosmbjvj, onxytkatvevr, pyentgdyhuwx, pzhvpgtvlbxg, tslmcundralx, tvejysmllsmz, vouqrxazstgt

No comments. add comment

Snippet ID: #1012330
Snippet name: TripleIndex with direct link to triples [LIVE]
Eternal ID of this version: #1012330/83
Text MD5: b9186e3ea62e6ea5e48051ed98d66fa2
Author: stefan
Category: javax / a.i
Type: JavaX fragment (include)
Public (visible to everyone): Yes
Archived (hidden from active list): No
Created/modified: 2018-01-15 16:59:10
Source code size: 8105 bytes / 252 lines
Pitched / IR pitched: No / No
Views / Downloads: 700 / 1284
Version history: 82 change(s)
Referenced in: #1034167 - Standard Classes + Interfaces (LIVE, continuation of #1003674)