!7

static LL<S> allTokens;

p-experiment {
  preload("allTokens", r {
    loadSnippetsDBForAnalysis();
    print("Tokenizing.");
    allTokens = new L;
    time {
      for (CSnippet s)
        if (nempty(s.text)) {
          L<S> tokens = internAll(javaTok(s.text()));
          allTokens.add(tokens);
          s.text = tokens;
        }
    }
    print(n2(totalLength(allTokens)) + " tokens");
  }, r {
    time {
      int sum;
      for (L<S> tokens : allTokens)
        sum += tokens.contains("if") ? 1 : 0;
      print(sum);
    }
  });
}