TripleIndex with direct link to triples [LIVE] [1012330]

sclass TripleIndex extends VirtualNodeIndex {
  static bool useExactIndex; // experimental
  bool activated; // are we the main index (see tripleIndex())
  
  // index of word in any position - not used anymore
  SyncListMultiMap<Symbol, TripleWeb> index;// = symbolSyncListMultiMap();
  
  // count per word
  MultiSet<Symbol> wordIndex =
    //caseInsensitiveCompactMultiSet(); // not a NavigableMap
    caseInsensitiveMultiSet();
  
  // word in position
  SyncListMultiMap<Symbol, TripleWeb>[] positionalIndices = new SyncListMultiMap[] {
    symbolSyncListMultiMap(),
    symbolSyncListMultiMap(),
    symbolSyncListMultiMap() };
    
  // all three words
  ExactTripleIndex exactIndex = useExactIndex ? exactTripleIndex() : null;
  
  // triples by ID
  new CompactHashSet<TripleWeb> websByID;
  
  // first word -> relation -> triple
  Map<Symbol, SyncListMultiMap<Symbol, TripleWeb>> oneTwoIndex; // = (Map) symbolMap();
  // SyncListMultiMap<Pair<Symbol>, TripleWeb> oneTwoIndex2 = new SyncListMultiMap; // works, but wastes space
  SyncListMultiMap<Pair<Symbol>, TripleWeb> oneTwoIndex2 = new SyncListMultiMap(new CompactPairKeyHashMap); // try compact version

  int size() { ret numWebs(); }
  int numWebs() { ret l(websByID); }
  int numTerms() { ret index != null ? index.keysSize() : wordIndex.uniqueSize(); }
  
  Set<Symbol> mainKeys() {
    ret index != null ? keys(index) : wordIndex.asSet();
  }
  
  NavigableSet<Symbol> navigableMainKeys() {
    ret index != null ? navigableKeys(index) : wordIndex.navigableSet();
  }
  
  // for synchronizing, e.g. in fullIndexedTermsStartingWith
  Map mainIndexMap() {
    ret index != null ? index.data : wordIndex.map;
  }
  
  Collection<Symbol> indexedTerms() { ret mainKeys(); }
  
  // query is shortened term
  L<WebNode> get(CharSequence _query) {
    Symbol query = symbol(_query);
    L<TripleWeb> triples = index != null ? index.get(query)
      : combineLists3(getTriples(query, 0), getTriples(query, 1), getTriples(query, 2));
    ret ai_triplesToWebNodes_lazyList(query, triples);
  }
  
  // terms are NOT shortened
  /*TripleWeb getExact(Symbol a, Symbol b, Symbol c) {
    if (exactIndex == null) null;
    ret exactIndex.find(dummyTripleWebWithEntries(a, b, c));
  }*/

  // query is shortened term
  L<TripleRef<Symbol>> getTripleRefs(Symbol query) {
    if (index == null) fail("no index");
    ret ai_triplesToTripleRefs_lazyList(query, index.get(query));
  }
  
  L<TripleRef<Symbol>> getTripleRefs(Symbol query, int position) {
    ret ai_triplesToTripleRefs_lazyList(query, getTriples(query, position));
  }
  
  // query is shortened term
  L<TripleWeb> getTriples(Symbol query) {
    if (index == null) fail("no index");
    ret index.get(query);
  }
  
  // query is shortened term
  L<TripleWeb> getTriples(Symbol query, int position) {
    ret positionalIndices[position].get(query);
  }
  
  bool hasShortTerm(Symbol s) {
    ret index != null ? index.containsKey(s) : wordIndex.contains(s); }

  Web getWeb(GlobalID id) {
    ret webFromTriple(getTriple(id));
  }
  
  TripleWeb getTriple(GlobalID id) {
    ret websByID.find(dummyTripleWebWithGlobalID(id));
  }
  
  void addWeb(Web web) {
    if (web == null) ret;
    TripleWeb w = ai_webToTripleWeb(web);
    if (w == null)
      fail("Skipping non-tripelizable web: " + webToStringShort(web));
    addTriple(w);
  }
  
  void addTriple(TripleWeb w) {
    if (w == null) ret;
    if (index != null)
      for (Symbol s : sortedInPlace(ll(w.a, w.b, w.c)))
        index.put(s, w);
    if (wordIndex != null) {
      wordIndex.add(w.a);
      wordIndex.add(w.b);
      wordIndex.add(w.c);
    }
    positionalIndices[0].put(w.a, w);
    positionalIndices[1].put(w.b, w);
    positionalIndices[2].put(w.c, w);
    if (exactIndex != null) exactIndex.add(w);
    if (oneTwoIndex != null) synchronized(oneTwoIndex) {
      SyncListMultiMap<Symbol, TripleWeb> map = oneTwoIndex.get(w.a);
      if (map == null) oneTwoIndex.put(w.a, map = symbolSyncListMultiMap());
      map.put(w.b, w);
    }
    if (oneTwoIndex2 != null) oneTwoIndex2.put(pair(w.a, w.b), w);
    websByID.add(w);
    if (activated)
      ai_fireNewTriple(w);
  }
  
  void removeWeb(Web web) {
    if (web != null)
      removeTriple(ai_webToTripleWeb(web));
  }    
    
  void removeTriples(Collection<TripleWeb> l) {
    new MultiMap<Symbol, TripleWeb> mm;
    for (TripleWeb w : l) {
      mm.put(ai_shortenForIndex(w.a), w);
      mm.put(ai_shortenForIndex(w.b), w);
      mm.put(ai_shortenForIndex(w.c), w);
      removeFromSets(w);
    }
    for (Symbol term : keys(mm)) {
      HashSet<TripleWeb> webs = asHashSet(mm.get(term));
      if (index != null) indexRemoveMulti(index, term, webs);
      for i to 3:
        indexRemoveMulti(positionalIndices[i], term, webs);
    }
    if (activated)
      for (TripleWeb w : l)
        ai_fireForgottenTriple(w);
  }
  
  // internal
  void removeFromSets(TripleWeb w) {
    websByID.remove(w);
    if (exactIndex != null) exactIndex.remove(w);
    if (oneTwoIndex != null) synchronized(oneTwoIndex) {
      SyncListMultiMap<Symbol, TripleWeb> map = oneTwoIndex.get(w.a);
      if (map != null) {
        map.remove(w.a, w);
        if (empty(map)) oneTwoIndex.remove(w.a);
      }
    }
    if (oneTwoIndex2 != null) oneTwoIndex2.remove(pair(w.a, w.b), w);
    if (wordIndex != null) {
      wordIndex.remove(w.a);
      wordIndex.remove(w.b);
      wordIndex.remove(w.c);
    }
  }
  
  void removeTriple(TripleWeb w) {
    if (w == null) ret;
    removeFromSets(w);
    GlobalID id = w.globalID();
    indexRemove(0, ai_shortenForIndex(w.a), id);
    indexRemove(1, ai_shortenForIndex(w.b), id);
    indexRemove(2, ai_shortenForIndex(w.c), id);
    if (activated)
      ai_fireForgottenTriple(w);
  }
  
  // internal
  void indexRemove(int position, Symbol term, GlobalID globalID) {
    if (index != null) indexRemove2(index, term, globalID);
    indexRemove2(positionalIndices[position], term, globalID);
  }
    
  // remove from a single list
  void indexRemove2(MultiMap<Symbol, TripleWeb> mm, Symbol term, GlobalID globalID) {
    L<TripleWeb> l = mm.get(term);
    for i over l:
      if (eq(l.get(i).globalID(), globalID)) {
        mm.remove(term, l.get(i));
        ret;
      }
  }
  
  // internal
  void indexRemoveMulti(MultiMap<Symbol, TripleWeb> mm, Symbol term, Set<TripleWeb> set) {
    L<TripleWeb> l = mm.get(term);
    synchronized(l) {
      removeSetFromListQuickly(l, set);
    }
  }
  
  void clear() {
    if (index != null) index.clear();
    if (wordIndex != null) wordIndex.clear();
    websByID.clear();
    if (exactIndex != null) exactIndex.clear();
    if (oneTwoIndex != null) oneTwoIndex.clear();
    if (oneTwoIndex2 != null) oneTwoIndex2.clear();
  }
  
  void activate {
    if (!activated) {
      activated = true;
      ai_onNewOrRemovedWeb(func(Web web) {
        addWeb(web);
        false;
      }, func(Web web) {
        removeWeb(web);
        false;
      });
    }
  }
  
  Collection<TripleWeb> allTriples() { ret websByID; }
  
  void trimToSize() {
    if (index != null) trimToSizeAll(index.allLists());
    for i to 3: trimToSizeAll(positionalIndices[i].allLists());
    for (SyncListMultiMap<Symbol, TripleWeb> map : values(oneTwoIndex))       trimToSizeAll(map.allLists());
  }
  
  void sortAllLists() {
    if (index != null) sortLists(index.allLists());
    for i to 3: sortLists(positionalIndices[i].allLists());
    for (SyncListMultiMap<Symbol, TripleWeb> map : values(oneTwoIndex))       sortLists(map.allLists());
  }
  
  void sortLists(Collection<L<TripleWeb>> ll) {
    for (L<TripleWeb> l : ll)
      ai_sortTriplesListByPriority_inPlace(l);
  }
  
  L<TripleWeb> getOneTwo(Symbol a, Symbol b) {
    if (oneTwoIndex2 != null)
      ret oneTwoIndex2.get(pair(a, b));
    ret shortestList2(getTriples(a), getTriples(b));
  }
}

Travelled to 14 computer(s): aoiabmzegqzx, bhatertpkbcr, cbybwowwnfue, cfunsshuasjs, gwrvuhgaqvyk, ishqpsrjomds, lpdgvwnxivlt, mqqgnosmbjvj, onxytkatvevr, pyentgdyhuwx, pzhvpgtvlbxg, tslmcundralx, tvejysmllsmz, vouqrxazstgt

Snippet ID:	#1012330
Snippet name:	TripleIndex with direct link to triples [LIVE]
Eternal ID of this version:	#1012330/83
Text MD5:	b9186e3ea62e6ea5e48051ed98d66fa2
Author:	stefan
Category:	javax / a.i
Type:	JavaX fragment (include)
Public (visible to everyone):	Yes
Archived (hidden from active list):	No
Created/modified:	2018-01-15 16:59:10
Source code size:	8105 bytes / 252 lines
Pitched / IR pitched:	No / No
Views / Downloads:	634 / 1216
Version history:	82 change(s)
Referenced in:	[show references]

< > BotCompany Repo | #1012330 // TripleIndex with direct link to triples [LIVE]

JavaX fragment (include)

Author comment

1	sclass TripleIndex extends VirtualNodeIndex {
2	static bool useExactIndex; // experimental
3	bool activated; // are we the main index (see tripleIndex())
4
5	// index of word in any position - not used anymore
6	SyncListMultiMap<Symbol, TripleWeb> index;// = symbolSyncListMultiMap();
7
8	// count per word
9	MultiSet<Symbol> wordIndex =
10	//caseInsensitiveCompactMultiSet(); // not a NavigableMap
11	caseInsensitiveMultiSet();
12
13	// word in position
14	SyncListMultiMap<Symbol, TripleWeb>[] positionalIndices = new SyncListMultiMap[] {
15	symbolSyncListMultiMap(),
16	symbolSyncListMultiMap(),
17	symbolSyncListMultiMap() };
18
19	// all three words
20	ExactTripleIndex exactIndex = useExactIndex ? exactTripleIndex() : null;
21
22	// triples by ID
23	new CompactHashSet<TripleWeb> websByID;
24
25	// first word -> relation -> triple
26	Map<Symbol, SyncListMultiMap<Symbol, TripleWeb>> oneTwoIndex; // = (Map) symbolMap();
27	// SyncListMultiMap<Pair<Symbol>, TripleWeb> oneTwoIndex2 = new SyncListMultiMap; // works, but wastes space
28	SyncListMultiMap<Pair<Symbol>, TripleWeb> oneTwoIndex2 = new SyncListMultiMap(new CompactPairKeyHashMap); // try compact version
29
30	int size() { ret numWebs(); }
31	int numWebs() { ret l(websByID); }
32	int numTerms() { ret index != null ? index.keysSize() : wordIndex.uniqueSize(); }
33
34	Set<Symbol> mainKeys() {
35	ret index != null ? keys(index) : wordIndex.asSet();
36	}
37
38	NavigableSet<Symbol> navigableMainKeys() {
39	ret index != null ? navigableKeys(index) : wordIndex.navigableSet();
40	}
41
42	// for synchronizing, e.g. in fullIndexedTermsStartingWith
43	Map mainIndexMap() {
44	ret index != null ? index.data : wordIndex.map;
45	}
46
47	Collection<Symbol> indexedTerms() { ret mainKeys(); }
48
49	// query is shortened term
50	L<WebNode> get(CharSequence _query) {
51	Symbol query = symbol(_query);
52	L<TripleWeb> triples = index != null ? index.get(query)
53	: combineLists3(getTriples(query, 0), getTriples(query, 1), getTriples(query, 2));
54	ret ai_triplesToWebNodes_lazyList(query, triples);
55	}
56
57	// terms are NOT shortened
58	/*TripleWeb getExact(Symbol a, Symbol b, Symbol c) {
59	if (exactIndex == null) null;
60	ret exactIndex.find(dummyTripleWebWithEntries(a, b, c));
61	}*/
62
63	// query is shortened term
64	L<TripleRef<Symbol>> getTripleRefs(Symbol query) {
65	if (index == null) fail("no index");
66	ret ai_triplesToTripleRefs_lazyList(query, index.get(query));
67	}
68
69	L<TripleRef<Symbol>> getTripleRefs(Symbol query, int position) {
70	ret ai_triplesToTripleRefs_lazyList(query, getTriples(query, position));
71	}
72
73	// query is shortened term
74	L<TripleWeb> getTriples(Symbol query) {
75	if (index == null) fail("no index");
76	ret index.get(query);
77	}
78
79	// query is shortened term
80	L<TripleWeb> getTriples(Symbol query, int position) {
81	ret positionalIndices[position].get(query);
82	}
83
84	bool hasShortTerm(Symbol s) {
85	ret index != null ? index.containsKey(s) : wordIndex.contains(s); }
86
87	Web getWeb(GlobalID id) {
88	ret webFromTriple(getTriple(id));
89	}
90
91	TripleWeb getTriple(GlobalID id) {
92	ret websByID.find(dummyTripleWebWithGlobalID(id));
93	}
94
95	void addWeb(Web web) {
96	if (web == null) ret;
97	TripleWeb w = ai_webToTripleWeb(web);
98	if (w == null)
99	fail("Skipping non-tripelizable web: " + webToStringShort(web));
100	addTriple(w);
101	}
102
103	void addTriple(TripleWeb w) {
104	if (w == null) ret;
105	if (index != null)
106	for (Symbol s : sortedInPlace(ll(w.a, w.b, w.c)))
107	index.put(s, w);
108	if (wordIndex != null) {
109	wordIndex.add(w.a);
110	wordIndex.add(w.b);
111	wordIndex.add(w.c);
112	}
113	positionalIndices[0].put(w.a, w);
114	positionalIndices[1].put(w.b, w);
115	positionalIndices[2].put(w.c, w);
116	if (exactIndex != null) exactIndex.add(w);
117	if (oneTwoIndex != null) synchronized(oneTwoIndex) {
118	SyncListMultiMap<Symbol, TripleWeb> map = oneTwoIndex.get(w.a);
119	if (map == null) oneTwoIndex.put(w.a, map = symbolSyncListMultiMap());
120	map.put(w.b, w);
121	}
122	if (oneTwoIndex2 != null) oneTwoIndex2.put(pair(w.a, w.b), w);
123	websByID.add(w);
124	if (activated)
125	ai_fireNewTriple(w);
126	}
127
128	void removeWeb(Web web) {
129	if (web != null)
130	removeTriple(ai_webToTripleWeb(web));
131	}
132
133	void removeTriples(Collection<TripleWeb> l) {
134	new MultiMap<Symbol, TripleWeb> mm;
135	for (TripleWeb w : l) {
136	mm.put(ai_shortenForIndex(w.a), w);
137	mm.put(ai_shortenForIndex(w.b), w);
138	mm.put(ai_shortenForIndex(w.c), w);
139	removeFromSets(w);
140	}
141	for (Symbol term : keys(mm)) {
142	HashSet<TripleWeb> webs = asHashSet(mm.get(term));
143	if (index != null) indexRemoveMulti(index, term, webs);
144	for i to 3:
145	indexRemoveMulti(positionalIndices[i], term, webs);
146	}
147	if (activated)
148	for (TripleWeb w : l)
149	ai_fireForgottenTriple(w);
150	}
151
152	// internal
153	void removeFromSets(TripleWeb w) {
154	websByID.remove(w);
155	if (exactIndex != null) exactIndex.remove(w);
156	if (oneTwoIndex != null) synchronized(oneTwoIndex) {
157	SyncListMultiMap<Symbol, TripleWeb> map = oneTwoIndex.get(w.a);
158	if (map != null) {
159	map.remove(w.a, w);
160	if (empty(map)) oneTwoIndex.remove(w.a);
161	}
162	}
163	if (oneTwoIndex2 != null) oneTwoIndex2.remove(pair(w.a, w.b), w);
164	if (wordIndex != null) {
165	wordIndex.remove(w.a);
166	wordIndex.remove(w.b);
167	wordIndex.remove(w.c);
168	}
169	}
170
171	void removeTriple(TripleWeb w) {
172	if (w == null) ret;
173	removeFromSets(w);
174	GlobalID id = w.globalID();
175	indexRemove(0, ai_shortenForIndex(w.a), id);
176	indexRemove(1, ai_shortenForIndex(w.b), id);
177	indexRemove(2, ai_shortenForIndex(w.c), id);
178	if (activated)
179	ai_fireForgottenTriple(w);
180	}
181
182	// internal
183	void indexRemove(int position, Symbol term, GlobalID globalID) {
184	if (index != null) indexRemove2(index, term, globalID);
185	indexRemove2(positionalIndices[position], term, globalID);
186	}
187
188	// remove from a single list
189	void indexRemove2(MultiMap<Symbol, TripleWeb> mm, Symbol term, GlobalID globalID) {
190	L<TripleWeb> l = mm.get(term);
191	for i over l:
192	if (eq(l.get(i).globalID(), globalID)) {
193	mm.remove(term, l.get(i));
194	ret;
195	}
196	}
197
198	// internal
199	void indexRemoveMulti(MultiMap<Symbol, TripleWeb> mm, Symbol term, Set<TripleWeb> set) {
200	L<TripleWeb> l = mm.get(term);
201	synchronized(l) {
202	removeSetFromListQuickly(l, set);
203	}
204	}
205
206	void clear() {
207	if (index != null) index.clear();
208	if (wordIndex != null) wordIndex.clear();
209	websByID.clear();
210	if (exactIndex != null) exactIndex.clear();
211	if (oneTwoIndex != null) oneTwoIndex.clear();
212	if (oneTwoIndex2 != null) oneTwoIndex2.clear();
213	}
214
215	void activate {
216	if (!activated) {
217	activated = true;
218	ai_onNewOrRemovedWeb(func(Web web) {
219	addWeb(web);
220	false;
221	}, func(Web web) {
222	removeWeb(web);
223	false;
224	});
225	}
226	}
227
228	Collection<TripleWeb> allTriples() { ret websByID; }
229
230	void trimToSize() {
231	if (index != null) trimToSizeAll(index.allLists());
232	for i to 3: trimToSizeAll(positionalIndices[i].allLists());
233	for (SyncListMultiMap<Symbol, TripleWeb> map : values(oneTwoIndex)) trimToSizeAll(map.allLists());
234	}
235
236	void sortAllLists() {
237	if (index != null) sortLists(index.allLists());
238	for i to 3: sortLists(positionalIndices[i].allLists());
239	for (SyncListMultiMap<Symbol, TripleWeb> map : values(oneTwoIndex)) sortLists(map.allLists());
240	}
241
242	void sortLists(Collection<L<TripleWeb>> ll) {
243	for (L<TripleWeb> l : ll)
244	ai_sortTriplesListByPriority_inPlace(l);
245	}
246
247	L<TripleWeb> getOneTwo(Symbol a, Symbol b) {
248	if (oneTwoIndex2 != null)
249	ret oneTwoIndex2.get(pair(a, b));
250	ret shortestList2(getTriples(a), getTriples(b));
251	}
252	}