!include once #1016609 // LinkedToken + Tokenization

static Tokenization makeTokenization(L<S> tokens) {
  new Tokenization tok;
  for (S t : tokens) {
    LinkedToken lt = new(t);
    lt.t = t;
    if (tok.first == null)
      tok.first = lt;
    else {
      lt.prev = tok.last;
      tok.last.next = lt;
    }
    tok.last = lt;
    lt.prevIdentical = tok.lastByContent.get(t);
    if (lt.prevIdentical == null)
      tok.firstByContent.put(t, lt);
    else {
      lt.prevIdentical.nextIdentical = lt;
    }
    tok.lastByContent.put(t, lt);
  }
  ret tok;
}