transient sclass DeepWordIndex<A> {
  S regexp = "\\w+";
  bool useHashMaps = true; // makes it case-sensitive and doesn't allow partial word searches
  new Map<A, Entry<A>> entries;
  MultiSetMap<S, Entry<A>> entriesByWord = ciMultiSetMap();
  
  sclass Entry<A> extends Var<A> implements Comparable<Entry<A>> {
    Map<S, int[]> wordPositions = ciMap(); // int array is sorted
    
    *(A id) { super(id); }
    
    public int compareTo(Entry<A> e) {
      ret ((Comparable) get()).compareTo(e!);
    }
    
    public bool equals(O o) {
      ret o instanceof Entry && get().equals(((Entry) o)!);
    }
  }
  
  void useHashMaps() {
    set useHashMaps;
    entriesByWord = new MultiSetMap;
  }
  
  L<IntRange> wordRanges(S text) {
    ret regexpFindRanges(regexp, text);
  }
   
  void add(A a, S text) {
    Entry<A> e = new Entry<A>(a);
    if (useHashMaps) {
      e.wordPositions = new HashMap;
      text = upper(text);
    }
    if (entries.put(a, e) != null) fail("Double insertion");
    MultiMap<S, Int> wordPositions = ciMultiMap();
    for (IntRange r : wordRanges(text)) {
      S word = substring(text, r);
      wordPositions.put(word, r.start);
      entriesByWord.put(word, e);
    }
    for (S word : keys(wordPositions))
      e.wordPositions.put(word, toIntArray(wordPositions.get(word)));
  }
  
  Set<Entry<A>> get(S word) { ret entriesByWord.get(word); }
  
  int numWords() { ret entriesByWord.keysSize(); }
  
  Iterable<Pair<A, Cl<Int>>>lookupString_withPositions(S query, O... _) {
    optPar bool debug;
    if (useHashMaps) query = upper(query);
    S _query = query;
    L<IntRange> ranges = wordRanges(query);
    if (empty(ranges)) null;
    int nRanges = l(ranges);
    int iFirstComplete = first(ranges).start == 0 ? 1 : 0;
    int iLastComplete = last(ranges).end == l(query) ? nRanges-1 : nRanges;
    
    LS words = map(ranges, r -> substring(_query, r));
    L<Set<Entry<A>>> entriesAtIndex = map(words, word -> entriesByWord.get(word));
      
    if (iLastComplete >= iFirstComplete+1) {
      int shortest = iFirstComplete, nBest = l(entriesAtIndex.get(shortest));
      if (nBest == 0) ret emptyList();
      for (int iWord = iFirstComplete+1; iWord < iLastComplete; iWord++) {
        int n = l(entriesAtIndex.get(iWord));
        if (n == 0) ret emptyList();
        if (n < nBest) {
          shortest = iWord;
          nBest = n;
        }
      }
      
      /*if (debug)*/ print("words: " + zipTwoLists(words, lmap l(entriesAtIndex)));
      
      Set<Entry<A>> entries = entriesAtIndex.get(shortest);
      int startShortest = ranges.get(shortest).start;
      S shortestWord = words.get(shortest); // not the shortest word, but the word with the shortest result list
      new IntBuffer intBuffer;
      
      new LPair<A, Cl<Int>> out;
      entrySearch: for (Entry<A> entry : entries) {
        int[] positions = entry.wordPositions.get(shortestWord);

        for (int iWord = iFirstComplete; iWord < iLastComplete; iWord++) {
          continue if iWord == shortest;
          IntRange r2 = ranges.get(iWord);
          S word2 = words.get(iWord);
          int[] positions2 = entry.wordPositions.get(word2);
          if (positions2 == null) continue entrySearch;
          int ofs = startShortest-r2.start;
          //if (debug) print("Intersecting " + asList(positions) + "/" + asList(positions2) + " with ofs " + ofs);
          positions = intersectSortedIntArrays_ofs_optimized2(positions, positions2, ofs, intBuffer);
          //if (debug) print("Got " + asList(positions));
          if (empty(positions)) continue entrySearch;
        }
        
        out.add(pair(entry!, wrapIntArrayAsImmutableList_ofs(positions, -startShortest)));
      }
      ret out;
    }
    
    null;
  }
}