!7

static new LL<S> allTokens;

p-experiment {
  loadSnippetsDBForAnalysis();
  print("Tokenizing.");
  time {
    for (CSnippet s)
      if (nempty(s.text))
        allTokens.add(javaTok(s.text));
  }
  print(totalLength(allTokens) + " tokens");
}