// texts are split into chunks of a maximum size
transient sclass ChunkedDeepBitSetWordIndex<A> {
  record Chunk(A a, int nr) {}
  
  int chunkLength = 65536;
  int maxQueryLength = 32; // max query length = overlap between chunks
  S regexp = "\\w+";
  new Map<Chunk, SingleTextWordIndex> singleTextIndices;
  new ElementInstanceMatrix<Chunk, S> mainIndex;

  void add(A a, S text) {
    LS chunks = textToChunks(text);
    for i over chunks: {
      chunkText = chunks.get(i);
      Chunk c = new Chunk(a, i);
      singleTextIndices.put(c, new SingleTextWordIndex(regexp, chunkText));
      mainIndex.add(c, mapToSet upper(regexpExtractAll(regexp, chunkText)));
    }
  }
  
  LS textToChunks(S text) {
    ret stringToChunksWithOverlap(text, chunkLength, overlap);
  }
  
  void doneAdding {
    mainIndex.doneAdding();
  }
  
  LPair<S, Int> wordsAndOffsets(S text) {
    ret map(regexpFindRanges(regexp, text),
      r -> pair(upper(substring(text, r)), r.start));
  }

  // assumes word boundaries left and right of query
  Cl<Chunk> preSearch_chunks(S query, O... _) {
    optPar bool debug;
    LPair<S, Int> l = wordsAndOffsets(query);
    Cl<Chunk> candidates = mainIndex.instancesContainingAllElements(pairsA(l));
    if (debug) {
      L<Int> lengths = map(candidates, a -> singleTextIndices.get(a).length);
      print(nCandidates(candidates) + ", total length: " + n2(intSum(lengths)) + ", lengths: " + lengths);
    }
    ret filter(candidates, a -> nempty(singleTextIndices.get(a).indicesOfWordCombination(l)));
  }
  
  Cl<A> preSearch(S query, O... _) {
    ret pairsASet(preSearch_chunks(query, _));
  }
  
  int numWords() { ret mainIndex.numElements(); }
}