// texts are split into chunks of a maximum size
transient sclass ChunkedDeepBitSetWordIndex<A> {
  class Chunk { A a; int nr; *(A *a, int *nr) {} }
  
  int chunkLength = 32768;
  int maxQueryLength = 32; // max query length = overlap between chunks
  S regexp = "\\w+";
  new Map<Chunk, SingleTextWordIndex> singleTextIndices;
  new ElementInstanceMatrix<Chunk, S> mainIndex;

  void add(A a, S text) {
    LS chunks = textToChunks(text);
    for i over chunks: {
      S chunkText = chunks.get(i);
      Chunk c = new Chunk(a, i);
      singleTextIndices.put(c, new SingleTextWordIndex(regexp, chunkText));
      mainIndex.add(c, mapToSet upper(regexpExtractAll(regexp, chunkText)));
    }
  }
  
  LS textToChunks(S text) {
    ret stringToChunksWithOverlap(text, chunkLength, maxQueryLength);
  }
  
  void doneAdding {
    mainIndex.doneAdding();
  }
  
  LPair<S, Int> wordsAndOffsets(S text) {
    ret map(regexpFindRanges(regexp, text),
      r -> pair(upper(substring(text, r)), r.start));
  }

  // assumes word boundaries left and right of query
  Cl<Chunk> preSearch_chunks(S query, O... _) {
    optPar bool debug;
    LPair<S, Int> l = wordsAndOffsets(query);
    Cl<Chunk> candidates = mainIndex.instancesContainingAllElements(pairsA(l));
    if (debug) {
      L<Int> lengths = map(candidates, a -> singleTextIndices.get(a).length);
      print(nCandidates(candidates) + ", total length: " + n2(intSum(lengths)) + ", lengths: " + lengths);
    }
    ret /*parallelFilter*/filter(candidates, a -> nempty(singleTextIndices.get(a).indicesOfWordCombination(l)));
  }
  
  Cl<A> preSearch(S query, O... _) {
    ret mapToSet(c -> c.a, preSearch_chunks(query, _));
  }
  
  int numWords() { ret mainIndex.numElements(); }
}

Began life as a copy of #1029082