!7

cprint SnippetsDeepBitSetWordIndex {
  transient ChunkedDeepBitSetWordIndex<S> wordIndex; // string = snippet ID
  switchable S regexp = "\\w+";
  switchable int chunkSize = 16384;

  start-thread {
    dm_reloadOnFieldChange("regexp", "chunkSize");
    time "Make bit-set word index" {
      print("Making index");
      new ChunkedDeepBitSetWordIndex<S> wordIndex;
      wordIndex.chunkLength = chunkSize;
      wordIndex.regexp = regexp;
      for (virtual CSnippet sn : dm_allSnippets()) {
        S snippetID = (S) rcall snippetID(sn);
        S text = cast rcall text(sn);
        wordIndex.add(snippetID, text);
      }
      wordIndex.doneAdding();
      setField(+wordIndex);
    }
    infoBox("Indexed " + nWords(wordIndex.numWords()));
  }
  
  // API
  
  Iterable<S> snippetPreSearch(S query, O... _) {
    long nanos = nanoTime();
    Iterable<S> l;
    //time "lookup" {
      l = wordIndex == null ? null : wordIndex.preSearch(query, _);
    //}
    //print((nanoTime()-nanos) + " nanos in pre");
    ret l;
  }
}

Began life as a copy of #1029084