!7

static LL<S> allTokens;

p-experiment {
  restartWith1GBHeap();
  loadSnippetsDBForAnalysis();
  print("Tokenizing.");
  allTokens = new L;
  time {
    for (CSnippet s)
      if (nempty(s.text))
        allTokens.add(javaTok(s.text()));
  }
  print(n2(totalLength(allTokens)) + " tokens");
}