// clever index of tokenized strings
// maintaining one CI-sorted list per token position
sclass PositionalTokenIndex2 {
  new L<PositionalTokenIndex> indices; // index = token count

  *() {}
  *(Iterable<LS> toks) { addAll(toks); }
  
  void addAll(Iterable<LS> toks) {
    fOr (LS tok : toks)
      add(tok);
  }
  
  void add(LS tok) {
    listGetOrCreate_full PositionalTokenIndex(indices, l(tok)/2).add(tok);
  }
  
  // return all tokenizations with token t in position i
  // having tokenCount code tokens
  // may return null
  LLS byToken(int i, S t, int tokenCount) {
    PositionalTokenIndex idx = get(indices, tokenCount);
    ret idx == null ? null : idx.byToken(i, t);
  }
  
  PositionalTokenIndex indexForTokenCount(int tokenCount) {
    ret get(indices, tokenCount);
  }
  
  void clean() {
    for (PositionalTokenIndex idx : indices) idx.clean();
  }
}