!7

set flag DynModule.

sclass Entry {
  S function;
  S tokenized;
}

cmodule InputTokenizer > DynBigInputAndList<Entry> {
  transient LS functions = ll(
    f javaTok,
    f javaTokWithBrackets,
    f javaTokNPunct,
    f javaTokWithAngleBrackets);
  
  void makeList_impl() {
    S text = this.text;
    setList(map_pcall(functions, function ->
      nu Entry(+function, tokenized := sfu(callAndMake(function, text)))));
  }
}