!7

set flag DynModule.

sclass Entry {
  S function;
  S tokenized;
}

cmodule InputTokenizer > DynBigInputAndSections {
  transient LS functions = ll(
    f javaTok,
    f javaTokWithBrackets,
    f javaTokNPunct,
    f javaTokWithAngleBrackets);
  
  void makeSections_impl() {
    S text = this.text;
    JTextPane tp = fontSizePlus(7, jTextPane());
    bool first = true;
    for (S f : functions) pcall {
      LS tok = cast callAndMake(function, text));
      if (!first) appendToTextPane(tp, "\n"); else first = false;
      appendToTextPane(tp, f + " => ");
      ai_tokenizationToTextPane(tp, tok);
    }
    addSection("Tokenizations", tp);
  }
}