!1006722

static bool autoQuine = true;
static int maxQuineLength = 80;

static L<S> functionsToAlwaysInclude = ll("_registerThread");

!include #1001496 // Matches
!include #1000882 // EGDiff
!include #1000883 // BlockDiffer
!include #1001065 // DialoGIO
!include #1004045 // IndexedList2
//include #1001296 // MultiMap
!include #1000988 // MultiSet
!include #1004543 // DynamicObject
!include #1005575 // CompilerBot

!include #1002662 // isTrue

static int varCount;
static new Map<S, S> snippetCache;
static bool useIndexedList = true, opt_javaTok = true;

static new HashSet<Long> included;
static new HashSet<S> definitions;
static new HashSet<S> shouldNotIncludeFunction, shouldNotIncludeClass;

p {
  if (useIndexedList) findCodeTokens_debug = true;
  javaTok_opt = opt_javaTok;
  findCodeTokens_indexed = findCodeTokens_unindexed = 0;
  findCodeTokens_bails = findCodeTokens_nonbails = 0;
  javaTok_n = javaTok_elements = 0;
  S in = loadMainJava();
  
  print("759 STARTING " + identityHashCode(main.class));
  included.clear();
  shouldNotIncludeFunction.clear();
  shouldNotIncludeClass.clear();
  varCount = 0;
  
  //L ts = findTranslators(toLines(join(tok)));
  //print("Translators in source at start: " + structure(ts));
  
  // "duplicate" statement
  
  L<S> lines = toLines(in);
  call(getJavaX(), "findTranslators", lines);
  in = fromLines(lines);
  new Matches m;
  if (match("duplicate *", in, m)) {
    // actual copying - unused
    // tok = jtok(loadSnippet(m.get(0)));
    
    // reference by include()
    in = "m { p { callMain(include(" + quote(m.get(0)) + ")); } }";
  }
  
  L<S> tok = jtok(in);

  // add m { }
  
  if (!hasCodeTokens(tok, "m", "{") && !hasCodeTokens(tok, "main", "{") && !hasCodeTokens(tok, "class", "main")) {
    //tok = jtok(moveImportsUp("m {\n" + in + "\n}"));
    replaceTokens_reTok(tok, 1, 2, "m {\n" + tok.get(1));
    replaceTokens_reTok(tok, l(tok)-2, l(tok)-1, tok.get(l(tok)-2) + "}");
    tok_moveImportsUp(tok);
  }
  
  // standard translate
  
  //ts = findTranslators(toLines(join(tok)));
  //print("Translators in source: " + structure(ts));
  tok = jtok(defaultTranslate(join(tok)));
  
  //print("end of default translate");
  //print(join(tok));

  //tok_autoCloseBrackets(tok);    
  tok = processIncludes(tok); // before standard functions
  processConceptsDot(tok);
  tok = processIncludes(tok);
  earlyStuff(tok);
  
  int safety = 0;
  boolean same;
  do {
    S before = join(tok);
    
    // shortened method declarations BEFORE standardFunctions
    jreplace(tok, "svoid", "static void");
    jreplace(tok, "void <id> {", "$1 $2() {");
    jreplace(tok, "String <id> {", "$1 $2() {");
    jreplace(tok, "Object <id> {", "$1 $2() {");
    jreplace(tok, "List <id> {", "$1 $2() {");
    
    tok_definitions(tok);
    tok_ifndef(tok);
    tok_ifdef(tok);
    tok = standardFunctions(tok);
    tok = stdstuff(tok); // standard functions and all the keywords
    S diff;
    long startTime = now();
    //diff = unidiff(before, join(tok));
    //print("unidiff: " + (now()-startTime) + " ms");
    //same = eq(diff, "");
    same = eq(before, join(tok)); // << could be sped up
    if (!same) {
      print("Not same " + safety + ".");
      //print(indent(2, diff));
    }
    if (safety++ >= 10) {
      //print(unidiff(before, join(tok)));
      print("----");
      print(join(tok));
      print("----");
      fail("safety 10 error!");
    }
  } while (!same);
  
  // POST-PROCESSING after stdstuff loop
  
  for (S clazz : ll("Pair", "Either")) {
    jreplace(tok, clazz + "< <id> >", clazz + "<$3, $3>");
    jreplace(tok, clazz + "< <id><<id>> >", clazz + "<$3<$5>, $3<$5>>");
    jreplace(tok, clazz + "< <id><<id>,<id>> >", clazz + "<$3<$5,$7>, $3<$5,$7>>");
  }

  quicknew2(tok);
  //tok = jtok(quicknew(join(tok)));
  tok = extendClasses(tok);
  libs(tok);
  sourceCodeLine(tok);
  throwFail(tok);
  innerClassesVar(tok);
  tok_ifclass(tok);

  // Stuff that depends on the list of inner classes (haveClasses)
  HashSet<S> haveClasses = haveClasses(tok);
  expandClassReferences(tok, haveClasses);
  slashCasts(tok, haveClasses);
  newWithoutNew(tok, haveClasses);
  
  tok = autoImports(tok); // faster to do it at the end
  
  /*if (useIndexedList)
    print("Indexed/unindexed lookups: " + findCodeTokens_indexed + "/" + findCodeTokens_unindexed + ", lists made: " + IndexedList2.instances);
  print("findCodeToken bails: " + findCodeTokens_bails + "/" + findCodeTokens_nonbails);
  print("javaToks: " + javaTok_n + "/" + javaTok_elements);*/
  
  if (tok.contains("package"))
    splitJavaFiles(tok);
  else
    saveMainJava(tok);
}

static L<S> stdstuff(L<S> tok) {
  //if (++level >= 10) fail("woot? 10");
  
  print("stdstuff!");
  int i;
  
  if (jfind(tok, "!<int>") >= 0) {
    L<S> lines = toLines(join(tok));
    L ts = findTranslators(lines);
    tok = jtok(fromLines(lines));
    print("DROPPING TRANSLATORS: " + structure(ts));
  }
  
  tok = quickmain(tok);
  tok = processIncludes(tok);
  processConceptsDot(tok);
  tok = processIncludes(tok);
  earlyStuff(tok);
  tok = multilineStrings(tok);
  tok_singleQuoteIdentifiersToStringConstants(tok);
  inStringEvals(tok);
  listComprehensions(tok);
  doubleFor_simple(tok);
  forPing(tok);
  directSnippetRefs(tok);
  quicknu(tok);
  //tok_juxtaposeCalls(tok);
  
  jreplace(tok, "synced <id>", "synchronized $2");
  
  replaceKeywordBlock(tok, "answer",
    "static S answer(S s) {\nfinal new Matches m;\n",
    "\nret null;\n}");
    
  replaceKeywordBlock(tok, "static-pcall",
    "static { pcall {",
    "}}");
    
  replaceKeywordBlock(tok, "loading",
    "{ JWindow _loading_window = showLoadingAnimation(); try { /* loading try */ ",
    "/* loading try */ } finally { disposeWindow(_loading_window); }\n /* loading end */ } /* after loading */ \n");

  replaceKeywordBlock(tok, "html",
    "static O html(S uri, fMap<S, S> params) ctex " + "{\n", "}");
  
  // "static sync" => static synchronized
  jreplace(tok, "static sync", "static synchronized");

  // "sclass" => static class
  jreplace(tok, "sclass", "static class");

  // "asclass" => abstract static class
  jreplace(tok, "asclass", "abstract static class");

  // "sinterface" => static interface
  jreplace(tok, "sinterface", "static interface");

  // "ssynchronized" => static synchronized
  jreplace(tok, "ssynchronized", "static synchronized");

  jreplace(tok, "ssvoid", "static synchronized void");
  jreplace(tok, "sbool", "static bool");
  jreplace(tok, "sint", "static int");
  jreplace(tok, "snew", "static new");
  jreplace(tok, "sv <id>", "static void $2");
  jreplace(tok, "pvoid", "public void");

  // "sS" => static S
  jreplace(tok, "sS", "static S");

  // "sO" => static O
  jreplace(tok, "sO", "static O");

  // "sL" => static L
  jreplace(tok, "sL", "static L");

  // "toString {" => "public S toString() {"
  jreplace(tok, "toString {", "public S toString() {");
  
  jreplace(tok, "Int", "Integer");
  jreplace(tok, "Bool", "Boolean");
  jreplace(tok, "BigInt", "BigInteger");
  jreplace(tok, "Char", "Character");
  
  // I REALLY wanted to avoid this, but eh...
  jreplace(tok, "SS", "Map<S, S>");

  // "on fail {" => "catch (Throwable _e) { ... rethrow(_e); }"
  replaceKeywordBlock(tok, "on fail",
    "catch (Throwable _e) {",
    "\nthrow rethrow(_e); }");

  // "catch {" => "catch (Throwable _e) {"
  jreplace(tok, "catch {", "catch (Throwable _e) {");

  // "catch X e {" => "catch (X e) {"
  jreplace(tok, "catch <id> <id> {", "catch ($2 $3) {");

  // "catch e {" => "catch (Throwable e) {" (if e is lowercase)
  jreplace(tok, "catch <id> {", "catch (Throwable $2) {", new O() {
    bool get(L<S> tok, int i) {
      S word = tok.get(i+3);
      ret startsWithLowerCaseOrUnderscore(word);
    }
  });
  
  jreplace(tok, "+ +", "+", new O() {
    bool get(L<S> tok, int i) {
      //printStructure("++: ", subList(tok, i-1, i+6));
      if (empty(_get(tok, i+2))) ret false; // no space between the pluses
      if (empty(_get(tok, i)) && eq("+", _get(tok, i-1))) ret false;  // an actual "++" at the left
      if (empty(_get(tok, i+4)) && eq("+", _get(tok, i+5))) ret false;  // an actual "++" at the right
      //print("doing it");
      ret true;
    }
  });

  // some crazy fancy syntax
  jreplace(tok, "set <id>;", "$2 = true;");
  
  // [stdEq] -> implementation of equals() and hashCode()
  jreplace(tok, "[stdEq]",
    "public bool equals(O o) { ret stdEq2(this, o); }\n" +
    "public int hashCode() { ret stdHash2(this); }");
  
  // [concepts] "concept.field!" for dereferencing references
  
  jreplace(tok, "<id>!", "$1.get()", func(L<S> tok, int i) {
    if (tok.get(i+2).contains("\n")) false; // no line break between <id> and !
    if (nempty(tok.get(i+4))) true; // space after = ok
    S t = _get(tok, i+5);
    if (t == null) ret false;
    if (isIdentifier(t) || eqOneOf(t, "=", "(")) false;
    true;
  });
  
  // [concepts] "field := value" for defining fields e.g. in "uniq"
  while ((i = jfind(tok, "<id> :=")) >= 0) {
    tok.set(i, quote(tok.get(i)));
    tok.set(i+2, ",");
    tok.set(i+4, "");
    reTok(tok, i, i+5);
  }
  
  // "quoted" := value
  while ((i = jfind(tok, "<quoted> :=")) >= 0) {
    tok.set(i, tok.get(i));
    tok.set(i+2, ",");
    tok.set(i+4, "");
    reTok(tok, i, i+5);
  }
  
  // more shortening

  jreplace(tok, "fS", "final S");
  jreplace(tok, "fO", "final O");
  jreplace(tok, "fL", "final L");
  jreplace(tok, "fMap", "final Map");
  jreplace(tok, "fRunnable", "final Runnable");
  jreplace(tok, "f int", "final int");
  
  // "continue unless"
  
  while ((i = jfind(tok, "continue unless")) >= 0) {
    int j = scanOverExpression(tok, getBracketMap(tok), i+4, ";");
    replaceTokens(tok, i, i+4, "{ if (!(");
    tok.set(j, ")) continue; }");
    reTok(tok, i, j+1);
  }
  
  // "continue if"
  
  while ((i = jfind(tok, "continue if")) >= 0) {
    int j = scanOverExpression(tok, getBracketMap(tok), i+4, ";");
    replaceTokens(tok, i, i+4, "{ if (");
    tok.set(j, ") continue; }");
    reTok(tok, i, j+1);
  }
  
  // "return if"
  
  while ((i = jfind(tok, "return if")) >= 0) {
    int j = scanOverExpression(tok, getBracketMap(tok), i+4, ";");
    replaceTokens(tok, i, i+4, "{ if (");
    tok.set(j, ") return; }");
    reTok(tok, i, j+1);
  }
  
  // while not null ()
  
  while ((i = jfind(tok, "while not null (")) >= 0) {
    int closingBracket = findEndOfBracketPart(tok, i+6)-1;
    replaceTokens(tok, i+2, i+6, "(");
    tok.set(closingBracket, ") != null)");
    reTok(tok, i, closingBracket+1);
  }
  
  // Replace $1 with m.unq(0) etc. - caveat: this blocks identifiers $1, $2, ...
  for (i = 1; i < l(tok); i += 2) {
    S s = tok.get(i);
    if (s.startsWith("$")) {
      s = substring(s, 1);
      if (isInteger(s)) {
        tok.set(i, "m.unq(" + (parseInt(s)-1) + ")");
        reTok(tok, i);
      }
    }
  }

  // instanceof trickery
  
  jreplace(tok, "is a <id>", "instanceof $3");
  jreplace(tok, "!<id> instanceof <id>.<id>", "!($2 instanceof $4.$6)");
  jreplace(tok, "!<id> instanceof <id>", "!($2 instanceof $4)");
  jreplace(tok, "<id> !instanceof <id>", "!($1 instanceof $4)");
  
  // map funcname(...) => map(f funcname, ...)
  
  jreplace(tok, "map <id>(", "map(f $2,");

  // func keyword for lambdas - now automatically quines toString() if enabled
  
  while ((i = jfind(tok, "func(")) >= 0) {
    int argsFrom = i+4, argsTo = findCodeTokens(tok, i, false, ")");
    int idx = findCodeTokens(tok, argsTo, false, "{");
    int j = findEndOfBracketPart(tok, idx);
    L<S> contents = subList(tok, idx+1, j-1);
    
    S returnType = null;
    if (eq(tok.get(argsTo+2), "-") && eq(tok.get(argsTo+4), ">"))
      returnType = join(subList(tok, argsTo+6, idx-1));
      
    S toString = autoQuine ? "  public S toString() { ret " + quote(shorten(maxQuineLength, trim(join(contents)))) + "; }" : "";
    
    // TODO: infer argument types
    /* if (nempty(returnType))
      replaceTokens(tok, i, j, "new F1<?, " + returnType + ">{ " + returnType + " get(" + join(subList(tok, argsFrom, argsTo-1)) + ") ctex { " + tok_addReturn(contents) + " }\n" +*/
    replaceTokens(tok, i, j, "new O { O get(" + join(subList(tok, argsFrom, argsTo-1)) + ") ctex { " + tok_addReturn(contents) + " }\n" +
     + toString + "}");
    reTok(tok, i, j);
  }
  
  while ((i = jfind(tok, "voidfunc(")) >= 0) {
    int argsFrom = i+4, argsTo = findCodeTokens(tok, i, false, ")");
    int idx = findCodeTokens(tok, argsTo, false, "{");
    int j = findEndOfBracketPart(tok, idx);
    L<S> contents = subList(tok, idx+1, j-1);
    replaceTokens(tok, i, j, "new O { void get(" + join(subList(tok, argsFrom, argsTo-1)) + ") ct" + "ex { " + tok_addSemicolon(contents) + " }\n" +
    (autoQuine ? "  public S toString() { ret " + quote(shorten(maxQuineLength, trim(join(contents)))) + "; }" : "") + "}");
    reTok(tok, i, j);
  }
  
  for (S keyword : ll("f", "func")) {
    while ((i = jfind(tok, keyword + " {")) >= 0) {
      int idx = findCodeTokens(tok, i, false, "{");
      int j = findEndOfBracketPart(tok, idx+2); // XXX - not idx???
      L<S> contents = subList(tok, idx+1, j-1);
      replaceTokens(tok, i, j, "new O { O get() ct" + "ex { " + tok_addReturn(contents) + " }\n" +
      (autoQuine ? "  public S toString() { ret " + quote(shorten(maxQuineLength, trim(join(contents)))) + "; }" : "") + "}");
      reTok(tok, i, j);
    }
  
    while ((i = jfind(tok, keyword + " ->")) >= 0) {
      int idx = findCodeTokens(tok, i, false, "{");
      int j = findEndOfBracketPart(tok, idx);
      S returnType = join(subList(tok, i+6, idx-1));
      L<S> contents = subList(tok, idx+1, j-1);
      replaceTokens(tok, i, j, "new F0<" + returnType + ">() { " + returnType + " get() ctex { " + tok_addReturn(contents) + " }\n" +
      (autoQuine ? "  public S toString() { ret " + quote(shorten(maxQuineLength, trim(join(contents)))) + "; }" : "") + "}");
      reTok(tok, i, j);
    }
  }

  // "ref -> bla" for dereferencing Concept.Ref or ThreadLocal or other
  //jreplace(tok, "<id> ->", "$1.get().");
  jreplace(tok, "->", ".get().", func(L<S> tok, int i) {
    empty(tok.get(i+2))
      && !(empty(_get(tok, i)) && eq(_get(tok, i-1), "-")) // i-->0
  });
  
  // shortened subconcept declaration (before star constructors!)
  shortenedSubconcepts(tok);
  
  // "case" as a variable name ( => _case)
  
  caseAsVariableName(tok);
  
  // "do" as a function name ( => dO)
  
  tok_doAsMethodName(tok);
  
  // "continue" as a function name ( => _continue)
  continueAsFunctionName(tok);
  
  // Do this BEFORE awt replacement! ("p-awt" contains "awt" token)
  if (hasCodeTokens(tok, "p", "-")) {
    jreplace(tok, "p-pretty {", "p-noconsole {");
    replaceKeywordBlock(tok, "p-awt-noconsole", "p-awt {", "\nhideConsole(); }");
    replaceKeywordBlock(tok, "p-substance-noconsole", "p-substance {", "\nhideConsole(); }");
    replaceKeywordBlock(tok, "p-nimbus-noconsole", "p-nimbus {", "\nhideConsole(); }");
    replaceKeywordBlock(tok, "p-subst-noconsole", "p-subst {", "\nhideConsole(); }");
    replaceKeywordBlock(tok, "p-noconsole", "p-subst {", "\nhideConsole(); }");
    replaceKeywordBlock(tok, "p-subst", "p-substance-thread {", "}");
    replaceKeywordBlock(tok, "p-substance-thread", "p { substance();", "}");
    replaceKeywordBlock(tok, "p-magellan-thread", "p { magellan();", "}");
    replaceKeywordBlock(tok, "p-substance", "p-awt { substance();", "}");
    replaceKeywordBlock(tok, "p-nimbus", "p-awt { nimbus();", "}");
    jreplace(tok, "p-type {", "p-typewriter {");
    jreplace(tok, "p-tt {", "p-typewriter {");
    replaceKeywordBlock(tok, "p-awt", "p { swing {", "}}");
    replaceKeywordBlock(tok, "p-typewriter", "p { typeWriterConsole();", "}");
    replaceKeywordBlock(tok, "p-lowprio", "p { lowPriorityThread(r " + "{", "}); }");
    tok_p_repeatWithSleep(tok);
  }

  replaceKeywordBlock(tok,
    "awt",
    "swingLater(r " + "{", // for #711
    "});");

  replaceKeywordBlock(tok,
    "swing",
    "{ swingAndWait(r " + "{", // for #711
    "}); }");
    
  unswing(tok);
  lockBlocks(tok);

  // crazy stuff

  jreplace (tok, "for <id> over <id>:", "for (int $2 = 0; $2 < l($4); $2++)");
  jreplace (tok, "for <id>, <id> <id> over <id>: {", "for (int $2 = 0; $2 < l($7); $2++) { $4 $5 = $7.get($2);");
  jreplace (tok, "for <id> to <id>:", "for (int $2 = 0; $2 < $4; $2++)");
  jreplace (tok, "for <id> to <int>:", "for (int $2 = 0; $2 < $4; $2++)");
  
  conceptDeclarations(tok);
  
  // * constructors
  if (hasCodeTokens(tok, "\\*", "("))
    tok = expandStarConstructors(tok);
    
  // STANDARD CLASSES & INTERFACES

  S sc = cacheGet("#1003674");   
  new L<S> lclasses;
  for (S line : toLinesFullTrim(sc)) {
    line = javaDropComments(line).trim();
    int idx = line.indexOf('/');
    lclasses.addAll(ll(line.substring(0, idx), line.substring(idx+1)));
  }
  
  final Set<S> haveClasses = addStandardClasses(tok, toStringArray(lclasses));
  
  if (tok.contains("Triple") && !haveClasses.contains("Triple"))
    jreplace(tok, "Triple", "T3");
  
  // "x << X" or "x >> X" => "x instanceof X"
  for (S op : ll("<<", ">>"))
    jreplace(tok, "<id> " + op + " <id>", "$1 instanceof $4", func(L<S> tok, int i) {
      haveClasses.contains(tok.get(i+7))
    });

  expandVarCopies(tok);
  
  // concept-related stuff
  
  // auto-import concepts
  bool _a = tok_hasClassRef2(tok, /*"extends",*/ "Concept") || tok_hasClassRef2(tok, "Concepts"), _b = !haveClasses.contains("Concept");
  //print("auto-import: " + _a + ", " + _b);
  if (_a && _b) {
    print("Auto-including concepts.");
    if (shouldNotIncludeClass.contains("Concepts")) {
      print(join(tok));
      fail("Unwanted concepts import");
    }
    printStruct(haveClasses);
    tok = includeInMainLoaded(tok, "concepts.");
    reTok(tok, l(tok)-1, l(tok));
    //processConceptsDot(tok);
  }
  
  jreplace(tok, "for (<id> <id>)", "for ($3 $4 : list($3))");
  jreplace(tok, "for (final <id> <id>)", "for (final $4 $5 : list($4))");

  // the infamous missing functions (usually caused by class Matches)
  // maybe not needed anymore?
  if (!hasCodeTokens(tok, "String", "unquote") && containsToken(tok, "unquote")) {
    print("Adding unquote");
    tok = includeInMain(tok, "#1001735");
  }
    
  if (!hasCodeTokens(tok, "String", "formatSnippetID") && containsToken(tok, "formatSnippetID")) {
    print("Adding formatSnippetID");
    tok = includeInMain(tok, "#1000709");
  }
  
  tok = expandShortTypes(tok);
    
  if (containsToken(tok, "cast")) {
    S s = join(tok);
    s = s.replaceAll("(\\w+<[\\w\\s,\\[\\]]+>|\\w+|\\w+\\[\\]|\\w+\\[\\]\\[\\])\\s+(\\w+)\\s*=\\s*cast(\\W[^;]*);", "$1 $2 = ($1) ($3);");
    tok = jtok(s);
  }
  
  replaceKeywordBlock(tok, "r-thread", "runnableThread(r " + "{", "})");
  rNamedThread(tok);
  
  // runnable and r - now also with automatic toString if enabled
  for (S keyword : ll("runnable", "r"))
    while ((i = jfind(tok, keyword + " {")) >= 0) {
      int idx = findCodeTokens(tok, i, false, "{");
      int j = findEndOfBracketPart(tok, idx);
      L<S> contents = subList(tok, idx+1, j-1);
      //print("r contents: " + structure(contents));
      replaceTokens(tok, i, j+1, "new Runnable() { public void run() { try { " + tok_addSemicolon(contents) + 
        "\n} catch (Exception __e) { throw rethrow(__e); } }" +
    (autoQuine ? "  public S toString() { return " + quote(shorten(maxQuineLength, trim(join(contents)))) + "; }" : "") +
        "}");
      reTok(tok, i, j+1);
    }
  
  replaceKeywordBlock(tok,
    "expectException",
    "{ bool __ok = false; try {",
    "} catch { __ok = true; } assertTrue(\"expected exception\", __ok); }");
    
  while ((i = tok.indexOf("tex")) >= 0) {
    tok.set(i, "throws Exception");
    tok = jtok(tok);
  }
  
  // shorter & smarter whiles
  
  jreplace(tok, "while true", "while (true)");
  jreplace(tok, "while licensed", "while (licensed())");
  jreplace(tok, "repeat {", "while (licensed()) {");
  tok_repeatWithSleep(tok);
  
  // null; => return null; etc.

  O cond = func(L<S> tok, int i) {
    tok_tokenBeforeLonelyReturnValue(_get(tok, i-1))
  };
  jreplace(tok, "null;", "return null;", cond);
  jreplace(tok, "false;", "return false;", cond);
  jreplace(tok, "true;", "return true;", cond);
  jreplace(tok, "this;", "return this;", cond);

  // "myFunction;" instead of "myFunction();" - quite rough
  cond = new O() {
    bool get(L<S> tok, int i) {
      S word = tok.get(i+3);
      //print("single word: " + word);
      ret !litlist("break", "continue", "return", "else").contains(word);
    }
  };
  for (S pre : litlist("}", ";"))
    jreplace(tok, pre + " <id>;", "$1 $2();", cond);

  // shorter match syntax for answer methods
  
  jreplace(tok, "if <quoted> || <quoted>",
    "if (matchOneOf(s, m, $2, $5))");
  // "bla..."
  jreplace(tok, "if <quoted>", "if (matchStartX($2, s, m))",
    new O() { bool get(L<S> tok, int i) {
      ret unquote(tok.get(i+3)).endsWith("...");
    }});
    
  // "bla * bla | blubb * blubb"
  jreplace_dyn(tok, "if <quoted>", func(L<S> tok, int cIdx) {
    S s = unquote(tok.get(cIdx+2));
    //print("multimatch: " + quote(s));
    new L<S> l;
    for (S pat : splitAtJavaToken(s, "|")) {
      //print("multimatch part: " + quote(pat));
      if (javaTok(pat).contains("*"))
        l.add("match(" + quote(trim(pat)) + ", s, m)");
      else
        l.add("match(" + quote(trim(pat)) + ", s)");
    }
    ret "if (" + join(" || ", l) + ")";
  }, func(L<S> tok, int i) {
    javaTokC(unquote(tok.get(i+3))).contains("|")
  });
  
  // "bla * bla"
  jreplace(tok, "if <quoted>", "if (match($2, s))",
    new O() { bool get(L<S> tok, int i) {
      ret !javaTokC(unquote(tok.get(i+3))).contains("*");
    }});
  // "bla"
  jreplace(tok, "if <quoted>", "if (match($2, s, m))");
  jreplace(tok, "if match <quoted>", "if (match($3, s, m))");

  // extra commas ("litlist(1, 2,)")
  
  jreplace(tok, ",)", ")");
  
  // additional translations (if necessary)
  
  replaceKeywordBlock(tok,
    "pcall",
    "try {",
    "} catch (Throwable __e) { printStackTrace2(__e); }");

  replaceKeywordBlock(tok,
    "pcall-short",
    "try {",
    "} catch (Throwable __e) { print(exceptionToStringShort(__e)); }");

  replaceKeywordBlock(tok,
    "pcall-silent",
    "try {",
    "} catch (Throwable __e) { silentException(__e); }");

  replaceKeywordBlock(tok,
    "pcall-messagebox",
    "try {",
    "} catch __e { messageBox(__e); }");

  replaceKeywordBlock(tok,
    "pcall-infobox",
    "try {",
    "} catch __e { infoBox(__e); }");

  tok = dialogHandler(tok);
  
  replaceKeywordBlock(tok, "exceptionToUser",
    "try {",
    "} catch (Throwable __e) { ret exceptionToUser(__e); }"); 

  if (hasCodeTokens(tok, "twice", "{"))
    replaceKeywordBlock(tok, "twice",
      "for (int __twice = 0; __twice < 2; __twice++) {",
      "}"); 

  while ((i = findCodeTokens(tok, "repeat", "*", "{")) >= 0) {
    S v = makeVar("repeat");
    tok.set(i, "for (int " + v + " = 0; " + v + " < " + tok.get(i+2) + "; " + v + "++)");
    tok.set(i+2, "");
    tok = jtok(tok);
  }

  replaceKeywordBlockDyn(tok,
    "time",
    new O() { S[] get() {
      S var = makeVar("startTime");
      ret new S[] {
        "{ long " + var + " = sysNow(); try { ",
        "} finally { " + var + " = sysNow()-" + var + "; saveTiming(" + var + "); } }"};
    }});
  
  // version without { }
  replaceKeywordBlockDyn(tok,
    "time2",
    new O() { S[] get() {
      S var = makeVar("startTime");
      ret new S[] {
        "long " + var + " = sysNow(); ",
        " " + var + " = sysNow()-" + var + "; saveTiming(" + var + "); "};
    }});
  
  // time "bla" {
  replaceKeywordPlusQuotedBlock(tok,
    "time",
    func(L<S> tok, int i) {
      S var = makeVar("startTime");
      ret new S[] {
        "long " + var + " = sysNow(); ",
        " done2(" + tok.get(i+2) + ", " + var + "); "};
    });
  
  if (hasCodeTokens(tok, "assertFail", "{")) {
    S var = makeVar("oops");
    
    replaceKeywordBlock(tok,
      "assertFail",
      "boolean " + var + " = false; try {",
      "\n" + var + " = true; } catch (Exception e) { /* ok */ } assertFalse(" + var + ");"); 
  }
  
  replaceKeywordBlock(tok,
    "yo",
    "try {",
    "} catch (Exception " + makeVar("e") + ") { ret false; }");

  replaceKeywordBlock(tok,
    "awtIfNecessary",
    "swingNowOrLater(r " + "{",
    "});");
    
  ctex(tok);
    
  replaceKeywordBlock(tok,
    "actionListener",
    "new java.awt.event.ActionListener() { " +
      "public void actionPerformed(java.awt.event.ActionEvent _evt) {",
    "}}");
    
  namedThreads(tok);
  threads(tok);

  // try answer
  while ((i = findCodeTokens(tok, "try", "answer")) >= 0) {
    int j = findEndOfStatement(tok, i);
    S v = makeVar("a");
    tok.set(i, "{ S " + v);
    tok.set(i+2, "=");
    tok.set(j-1, "; if (!empty(" + v + ")) ret " + v + "; }");
    tok = jtok(tok);
  }

  // return optional (return if not null)
  while ((i = jfind(tok, "return optional <id> =")) >= 0) {
    int j = findEndOfStatement(tok, i);
    S v = tok.get(i+4);
    clearTokens(tok, i+2, i+4);
    tok.set(i, "{");
    tok.set(j-1, "; if (" + v + " != null) ret " + v + "; }");
    tok = jtok(tok);
  }
  
  functionReferences(tok);

  tok_moveImportsUp(tok);
  /*S moved = moveImportsUp2(join(tok));
  if (moved != null) tok = jtok(moved);*/
    
  ret tok;
} // end of stdStuff!

static L<S> multilineStrings(L<S> tok) {
  for (int i = 1; i < tok.size(); i += 2) {
    S t = tok.get(i);
    if (isQuoted(t))
      if (t.startsWith("[") || t.contains("\r") || t.contains("\n"))
        tok.set(i, quote(unquote(t)));
  }
  ret tok;
}

static void inStringEvals(L<S> tok) {
  bool change = false;
  for (int i = 1; i < tok.size(); i += 2) {
    S t = tok.get(i);
    if (!isQuoted(t)) continue;
    if (t.contains("\\*") && !t.contains("\\\\")) { // << rough
      tok.set(i, inStringEval(t));
      change = true;
    }
  }
  if (change) reTok(tok);
}

static S inStringEval(S t) {
  t = dropPrefix("\"", dropSuffix("\"", t));
  new L<S> l;
  int idx;
  while ((idx = t.indexOf("\\*")) >= 0) {
    int j = indexOf(t, idx, "*/");
    if (j < 0) break;
    if (idx > 0)
      l.add("\"" + substring(t, 0, idx) + "\"");
    l.add("(" + trim(substring(t, idx+2, j)) + ")");
    t = substring(t, j+2);
  }
  if (nempty(t))
    l.add("\"" + t + "\"");
  ret "(" + join(" + ", l) + ")";
}

static L<S> quickmain(L<S> tok) {
  bool change = false;
  
  int i = findCodeTokens(tok, "main", "{");
  if (i < 0) i = findCodeTokens(tok, "m", "{");
  if (i >= 0 && !(i-2 > 0 && tok.get(i-2).equals("class"))) {
    tok.set(i, "class main");
    change = true;
  }
    
  i = findCodeTokens(tok, "psvm", "{");
  if (i < 0) i = findCodeTokens(tok, "p", "{");
  if (i >= 0) {
    int idx = i+2;
    int j = findEndOfBracketPart(tok, idx);
    L<S> contents = subList(tok, idx+1, j-1);
    tok.set(i, "public static void main(final String[] args) throws Exception");
    replaceTokens(tok, idx+1, j-1, tok_addSemicolon(contents));
    change = true;
  }
    
  ret change ? jtok(tok) : tok;
}

static S makeVar(S name) {
  ret "_" + name + "_" + varCount++;
}

static S makeVar() { ret makeVar(""); }

/*static L<S> standardFunctions(L<S> tok) {
  ret rtq(tok, "#1002474");
}*/

static L<S> rtq(L<S> tok, S id) {
  ret runTranslatorQuick(tok, id);
}

static L<S> expandShortTypes(L<S> tok) {
  // replace <int> with <Integer>
  for (int i = 1; i+4 < tok.size(); i += 2)
    if (tok.get(i).equals("<")
      && litlist(">", ",").contains(tok.get(i+4))) {
      String type = tok.get(i+2);
      if (type.equals("int")) type = "Integer";
      else if (type.equals("long")) type = "Long";
      tok.set(i+2, type);
    }
    
  // O = Object, S = String, ret = return
  for (int i = 1; i < tok.size(); i += 2) {
    String t = tok.get(i);
    if (t.equals("O")) t = "Object";
    if (t.equals("S")) t = "String";
    else if (t.equals("L")) t = "List";
    //else if (t.equals("F")) t = "Function";
    else if (t.equals("ret")) t = "return";
    else if (t.equals("bool") && i+2 < tok.size() && neq(tok.get(i+2), "(")) t = "boolean"; // bool -> boolean if it's not a function name
    tok.set(i, t);
  }
  
  jreplace(tok, "LL< <id> >", "L<L<$3>>");
  
  ret tok;
}

static L<S> autoImports(L<S> tok) {
  S s = join(tok);
  List<String> imports = findImports(s);
  new StringBuilder buf;
  for (String c : standardImports)
    if (!(imports.contains(c)))
      buf.append("import " + c + ";\n");
  if (buf.length() == 0) ret tok;
  ret jtok(buf+s);
}

static String[] standardImports = {
  "java.util.*",
  "java.util.zip.*",
  "java.util.List",
  "java.util.regex.*",
  "java.util.concurrent.*",
  "java.util.concurrent.atomic.*",
  "java.util.concurrent.locks.*",
  "javax.swing.*",
  "javax.swing.event.*",
  "javax.swing.text.*",
  "javax.swing.table.*",
  "java.io.*",
  "java.net.*",
  "java.lang.reflect.*",
  "java.lang.ref.*",
  "java.lang.management.*",
  "java.security.*",
  "java.security.spec.*",
  "java.awt.*",
  "java.awt.event.*",
  "java.awt.image.*",
  "javax.imageio.*",
  "java.math.*"
};

static L<S> expandStarConstructors(L<S> tok) {
  mainLoop: for (int i = 3; i+6 < tok.size(); i += 2) {
    String t = tok.get(i), l = tok.get(i-2);
    if (!t.equals("*"))
      continue;
    if (!tok.get(i+2).equals("("))
      continue;
    if (!eqOneOf(l, "}", "public", "private", "protected", ";", "{", "endif") && neq(get(tok, i-4), "ifclass")) // is this correct...??
      continue;
      
    // ok, it seems like a constructor declaration.
    // Now find class name by going backwards.
    
    int j = i, level = 1;
    while (j > 0 && level > 0) {
      t = tok.get(j);
      if (t.equals("}")) ++level;
      if (t.equals("{")) --level;
      j -= 2;
    }
    
    while (j > 0) {
      t = tok.get(j);
      if (t.equals("class")) {
        String className = tok.get(j+2);
        tok.set(i, className); // exchange constructor name!
        
        // now for the parameters.
        // Syntax: *(Learner *learner) {
        // We will surely add type inference here in time... :)
        
        j = i+2;
        while (!tok.get(j).equals("{"))
          j += 2;
        int block = j+1;
        for (int k = i+2; k < block-1; k += 2)
          if (tok.get(k).equals("*")) {
            tok.remove(k);
            tok.remove(k);
            block -= 2;
            String name = tok.get(k);
            tok.addAll(block, Arrays.asList(new String[] {
              "\n  ", "this", "", ".", "", name, " ", "=", " ", name, "", ";" }));
          }
        
        continue mainLoop;
      }
      j -= 2;
    }
  }
  ret tok;
}

static L<S> processIncludes(L<S> tok) {
  int safety = 0;
  while (hasCodeTokens(tok, "!", "include") && ++safety < 100)
    tok = processIncludesSingle(tok);
  
  //tok_autoCloseBrackets(tok);
  ret tok;
}

static L<S> processIncludesSingle(L<S> tok) {
  int i;
  while ((i = jfind(tok, "!include #<int>")) >= 0) {
    S id = tok.get(i+6);
    included.add(parseLong(id));
    replaceTokens(tok, i, i+8, "\n" + loadSnippet(id) + "\n");
    reTok(tok, i, i+8);
  }
  while ((i = jfind(tok, "!include once #<int>")) >= 0) {
    S id = tok.get(i+8);
    bool isNew = included.add(parseLong(id));
    replaceTokens(tok, i, i+10, 
      isNew ? "\n" + loadSnippet(id) + "\n" : "");
    reTok(tok, i, i+10);
  }
  ret tok;
}

static void ctex(L<S> tok) {
  replaceKeywordBlock(tok, "ctex",
    "{ try {",
    "} catch (Exception __e) { throw rethrow(__e); } }");
  replaceKeywordBlock(tok, "null on exception",
    "{ try {",
    "} catch (Throwable __e) { return null; } }");
  replaceKeywordBlock(tok, "false on exception",
    "{ try {",
    "} catch (Throwable __e) { return false; } }");
}
  
static L<S> dialogHandler(L<S> tok) {
  ret replaceKeywordBlock(tok,
    "dialogHandler",
    "new DialogHandler() {\n" +
      "public void run(final DialogIO io) {",
    "}}");
}

static void quicknew2(L<S> tok) {
  jreplace(tok, "new <id> <id>;", "$2 $3 = new $2;");
  jreplace(tok, "new <id><<id>> <id>;", "$2<$4> $6 = new $2;");
  jreplace(tok, "new <id><<id>> <id>, <id>;", "$2<$4> $6 = new $2, $8 = new $2;");
  jreplace(tok, "new <id><<id>,<id>> <id>;", "$2<$4,$6> $8 = new $2;");
  jreplace(tok, "new <id><<id><<id>>> <id>;", "$2 $3 $4 $5 $6 $7 $8 $9 = new $2;");
  jreplace(tok, "new <id>< <id><<id>,<id>> > <id>;", "$2 $3 $4 $5 $6 $7 $8 $9 $10 $11 = new $2;");
  
  // [abandoned, confusing, looks like a function definition] with arguments - new A a(...); => A a = new A(...);
  //jreplace(tok, "new <id> <id>(", "$2 $3 = new $2(");

  jreplace(tok, "for args " + "{", "for (int i = 0; i < args.length; i++) { final String arg = args[i];");
  
  // Constructor calls without parentheses
  // So you can say something like: predictors.add(new P1);
  
  jreplace1(tok, "new <id>", "new $2()", func(L<S> tok, int i) {
    eqOneOf(_get(tok, i+5), "{", ",", ")", ";", ":")
  });
  
  jreplace(tok, "new List(", "new ArrayList(");
  jreplace(tok, "new Map(", "new HashMap(");
  jreplace(tok, "new Set(", "new HashSet(");
  jreplace(tok, "\\*<id>[<id>] <id>;", "$2[] $6 = new $2[$4];");
  
  // X x = new(...)  =>  X x = new X(...)
  // X x = new       =>  X x = new
  jreplace(tok, "<id> <id> = new", "$1 $2 = new $1", func(L<S> tok, int i) {
    eqOneOf(_get(tok, i+9), "(", ";", ",", ")")
  });
  jreplace(tok, "<id> <id> = new \\*", "$1 $2 = new $1");
  jreplace(tok, "\\* <id> = new <id>", "$5 $2 = new $5");
  jreplace(tok, "<id><<id>> <id> = new", "$1 $2 $3 $4 $5 = new $1", func(L<S> tok, int i) {
    eqOneOf(_get(tok, i+9+3*2), "(", ";", ",", ")")
  });
  jreplace(tok, "<id><<id>,<id>> <id> = new", "$1 $2 $3 $4 $5 $6 $7 = new $1", func(L<S> tok, int i) {
    eqOneOf(_get(tok, i+9+5*2), "(", ";", ",", ")")
  });
}

static L<S> extendClasses(L<S> tok) {
  int i;
  while ((i = jfind(tok, "extend <id> {")) >= 0) {
    S className = tok.get(i+2);
    int idx = findCodeTokens(tok, i, false, "{");
    int j = findEndOfBracketPart(tok, idx+2);
    S content = join(subList(tok, idx+1, j-1));
    L<S> c = findInnerClassOfMain(tok, className);
    //print("Extending class " + className + " ==> " + join(c));
    clearTokens(tok.subList(i, j+1));
    if (c == null) {
      print("Warning: Can't extend class " + className + ", not found");
      continue;
    }
    int startOfClass = indexOfSubList(tok, c); // magicIndexOfSubList is broken
    int endOfClass = startOfClass + l(c)-1;
    //print("Extending class " + className + " ==> " + join(subList(tok, startOfClass, endOfClass)));
    while (neq(tok.get(endOfClass), "}")) --endOfClass;
    //print("Extending class " + className + " ==> " + join(subList(tok, startOfClass, endOfClass)));
    tok.set(endOfClass, content + "\n" + tok.get(endOfClass));
    reTok(tok); // changed in 2 places, let's retok it all
  }
  ret tok;
}

static void listComprehensions(L<S> tok) {
  int i;
  for (S op : ll(":", "in"))
    while ((i = jfind(tok, "[<id> <id> " + op)) >= 0) {
      Map<Integer, Integer> bracketMap = getBracketMap(tok); // XXX - optimize
      S type = tok.get(i+2), id = tok.get(i+4);
      int j = scanOverExpression(tok, bracketMap, i+8, "|");
      S exp = join(tok.subList(i+8, j));
      j += 2;
      int k = scanOverExpression(tok, bracketMap, j, "]");
      S where = join(tok.subList(j, k));
      ++k;
      
      S code = "filter(" + exp + ", func(" + type + " " + id + ") { " + where + " })";
      replaceTokens(tok, i, k, code);
      reTok(tok, i, k);
    }
}

/* complicated version - TODO */
// for (S key, S value : map)
/*static void doubleFor(L<S> tok) {
  int i;
  while ((i = jfind(tok, "for (<id> <id>, <id> <id> :")) >= 0) {
    int iComma = indexOf(tok, ",", i);
    S valType = tok.get(iComma+2);
    S valName = tok.get(iComma+4);
    int expStart = indexOf(tok, ":", i)+2;
    int expEnd = findEndOfBracketPart(expStart-2);
    assertEquals(")", tok.get(expEnd));
    assertEquals("{", tok.get(expEnd+2));
    replaceTokens(tok, iComma, expStart, ": keys(");
    tok.set(expEnd, "))");
    tok.set(expEnd+2, "{ " + valType + " " + valName + " = " + 
    S type = tok.get(i+2), id = tok.get(i+4);
      int j = scanOverExpression(tok, bracketMap, i+8, "|");
      S exp = join(tok.subList(i+8, j));
      j += 2;
      int k = scanOverExpression(tok, bracketMap, j, "]");
      S where = join(tok.subList(j, k));
      ++k;
      
      S code = "filter(" + exp + ", func(" + type + " " + id + ") { " + where + " })";
      replaceTokens(tok, i, k, code);
      reTok(tok, i, k);
    }
}*/

static void doubleFor_simple(L<S> tok) {
  jreplace(tok, "for (<id> <id>, <id> <id> : <id>) {",
    "for ($3 $4 : keys($9)) { $6 $7 = $9.get($4);");
  jreplace(tok, "for (<id> <id>, <id> : <id>) {",
    "for ($3 $4 : keys($8)) { $3 $6 = $8.get($4);");
}

static void forPing(L<S> tok) {
  int i, safety = 0;
  while ((i = jfind(tok, "for ping (")) >= 0) {
    if (++safety >= 10) fail("safety");
    int bracketEnd = findEndOfBracketPart(tok, i+4)-1;
    if (neq("{", tok.get(bracketEnd+2)))
      fail("Please use { with 'for ping' : " + joinSubList(tok, i+4, bracketEnd+3));
    tokSet(tok, bracketEnd+2, "{ ping();");
    tokSet(tok, i+2, "");
  }
}

// lib 123 => !123
static void libs(L<S> tok) {
  new TreeSet<S> libs;
  int i;
  while ((i = jfind(tok, "lib <int>")) >= 0) {
    S id = tok.get(i+2);
    print("lib " + id);
    if (!libs.contains(id)) {
      libs.add(id);
      tok.set(i, "!");
      tok.set(i+1, "");
    } else {
      print("...ignoring (duplicate)");
      clearAllTokens(tok, i, i+3);
      reTok(tok, i, i+3);
    }
  }
}

// sourceCodeLine() => 1234
static void sourceCodeLine(L<S> tok) {
  int i ;
  while ((i = jfind(tok, "sourceCodeLine()")) >= 0) {
    replaceTokens(tok, i, i+5, str(countChar(join(subList(tok, 0, i)), '\n')+1));
    reTok(tok, i, i+5);
  }
}

// done before any other processing
static void earlyStuff(L<S> tok) {
  int i;
  
  tok_autosemi(tok);
  tok_autoCloseBrackets(tok);
  jreplace(tok, "°", "()");
  
  // Note: this makes the word "quine" a special operator
  // (unusable as a function name)
  
  while ((i = jfind(tok, "quine(")) >= 0) {
    int idx = findCodeTokens(tok, i, false, "(");
    int j = findEndOfBracketPart(tok, idx+2);
    tok.set(i, "new Quine");
    tok.set(idx, "(" + quote(join(subList(tok, idx+1, j-1))) + ", ");
    reTok(tok, i, idx+1);
  }
}

static void throwFail(L<S> tok) {
  bool anyChange = false;
  for (int i = 1; i+2 < l(tok); i += 2)
    if (eq(get(tok, i+2), "fail") && !eqOneOf(get(tok, i), "throw", "RuntimeException", "return")) {
      tok.set(i+2, "throw fail");
      anyChange = true;
    }
  if (anyChange)
    reTok(tok);
}

static void namedThreads(L<S> tok) {
  for (int i = 0; i < 100; i++) {
    int idx = findCodeTokens(tok, "thread", "<quoted>", "{");
    if (idx < 0) idx = findCodeTokens(tok, "thread", "<id>", "{");
    if (idx < 0)
      break;
    int j = findEndOfBracketPart(tok, idx+4);
    S tName = tok.get(idx+2);
    
    S var = "_t_" + i;
    S pre = "{ Thread " + var + " = new Thread(" + tName + ") {\n" +
      "public void run() { pcall {\n";
    S post = "} }\n};\n" +
      "startThread(" + var + "); }";

    tok.set(idx, pre);
    tok.set(idx+2, "");
    tok.set(idx+4, "");
    tok.set(j-1, post);
    //print(join(subList(tok, idx, j)));
    reTok(tok, idx, j);
  }
}

static void rNamedThread(L<S> tok) {
  for (int i = 0; i < 100; i++) {
    int idx = findCodeTokens(tok, "r", "-", "thread", "<quoted>", "{");
    if (idx < 0) idx = findCodeTokens(tok, "r", "-", "thread", "<id>", "{");
    if (idx < 0)
      break;
    int j = findEndOfBracketPart(tok, idx+8);
    S tName = tok.get(idx+6);
    
    S pre = "r { thread " + tName + " {";
    S post = "}}";

    replaceTokens(tok, idx, idx+9, pre);
    tok.set(j-1, post);
    reTok(tok, idx, j);
  }
}

static void threads(L<S> tok) {
  for (bool daemon : litlist(false, true))
    for (int i = 0; i < 100; i++) {
      int idx = findCodeTokens(tok, daemon ? "daemon" : "thread", "{");
      if (idx < 0)
        break;
      int j = findEndOfBracketPart(tok, idx+2);

      S var = "_t_" + i;
      S pre = "{ Thread " + var + " = new Thread() {\n" +
        "public void run() { pcall\n";
      S post = "} }\n};\n" +
        (daemon ? var + ".setDaemon(true);\n" : "") +
        "startThread(" + var + "); }";

      tok.set(idx, pre);
      tok.set(j-1, post);
      reTok(tok, idx, j);
    }
}

static Map<S, S> sf;
static Bool _761ok;

static L<S> standardFunctions(L<S> tok) {
  if (sf == null) {
    S _761 = cacheGet("#761");
    if (_761ok == null)
      _761ok = isBracketHygienic(_761);
    assertTrue("Whoa! #761 truncated?", _761ok);
    L<S> standardFunctions = concatLists(
      (L) loadVariableDefinition(_761, "standardFunctions"),
      (L) loadVariableDefinition(cacheGet("#1006654"), "standardFunctions"));

    sf = new HashMap();
    for (String x : standardFunctions) {
      String[] f = x.split("/");
      sf.put(f[1], f[0]);
    }
  }
  
  for (int i = 0; ; i++) {
    Set<String> defd = new HashSet(findFunctions(tok));
    
    int j;
    while ((j = jfind(tok, "should not include function *.")) >= 0) {
      S fname = tok.get(j+8);
      shouldNotIncludeFunction.add(fname);
      clearAllTokens(tok.subList(j, j+12));
    }

    // changes tok
    Set<String> invocations = findFunctionInvocations(tok, sf);
    invocations.addAll(functionsToAlwaysInclude);

    //print("Functions invoked: " + structure(invocations));
    List<String> needed = diff(invocations, defd);
    if (needed.isEmpty())
      break;
    print("Adding functions: " + join(" " , needed));
    
    Collection<S> bad = setIntersection(new HashSet(needed), shouldNotIncludeFunction);
    if (nempty(bad)) {
      S msg = "INCLUDING BAD FUNCTIONS: " + sfu(bad);
      print(msg);
      print(join(tok));
      fail(msg);
    }
      
    new L<S> added;
    new StringBuilder buf;
    new L<S> preload;
    
    for (S x : needed)
      if (sf.containsKey(x))
        preload.add(sf.get(x));
    cachePreload(preload);
    
    for (String x : cloneList(needed)) {
      if (defd.contains(x)) continue;
      
      String id = sf.get(x);
      if (id == null) {
        print("Standard function " + x + " not found.");
        needed.remove(x);
        continue;
      }
      //print("Adding function: " + x + " (" + id + ")");
       
      S function = cacheGet(id);
      if (("\n" + function).contains("\n!")) print("Warning: " + id + " contains translators.");
      
      buf.append(function).append("\n");
      added.add(x);
      defd.addAll(findFunctionDefs(javaTok(function)));
    }
    tok = includeInMainLoaded(tok, str(buf));
    // defd = new HashSet(findFunctions(tok));
    //print("Functions added: " + structure(added));
    
    for (String x : needed)
      if (!defd.contains(x)) {
        print(join(tok));
        fail("Function not defined properly: " + x);
      }
    //print("Iteration " + (i+2));
    
    tok_definitions(tok);
    tok_ifndef(tok);
    tok_ifdef(tok);
    
    if (i >= 1000) fail("Too many iterations");
  }
  
  ret tok;
}

static L<S> findFunctions(L<S> tok) {
  //ret findFunctionDefinitions(join(findMainClass(tok)));
  ret findFunctionDefs(findMainClass(tok));
}

static S cacheGet(S snippetID) {
  snippetID = formatSnippetID(snippetID);
  S text = snippetCache.get(snippetID);
  if (text == null)
    snippetCache.put(snippetID, text = loadSnippet(snippetID));
  ret text;
}

static void cachePreload(L<S> ids) {
  new L<S> needed;
  for (S id : ids)
    if (!snippetCache.containsKey(formatSnippetID(id)))
      needed.add(formatSnippetID(id));
  if (l(needed) > 1) {
    L<S> texts = loadSnippets(needed);
    for (int i = 0; i < l(needed); i++)
      if (texts.get(i) != null)
        snippetCache.put(needed.get(i), texts.get(i));
  }
}

static L<S> jtok(L<S> tok) {
  ret jtok(join(tok));
}

static L<S> jtok(S s) {
  L<S> l = javaTok(s);
  ret useIndexedList ? new IndexedList2(l) : l;
}

static HashSet<S> haveClasses(L<S> tok) {
  new HashSet<S> have;
  for (L<S> c : innerClassesOfMain(tok))
    have.add(getClassDeclarationName(c));
  have.addAll(tok_importedClassNames(tok));
  have.add("String");
  ret have;
}

// works on Java level (no "sclass" etc)
// returns list of classes we have (useful for other processing)
static Set<S> addStandardClasses(L<S> tok, S... data) {
  HashSet<S> have = haveClasses(tok);

  int j;
  while ((j = jfind(tok, "should not include class *.")) >= 0) {
    S cname = tok.get(j+8);
    shouldNotIncludeClass.add(cname);
    clearAllTokens(tok.subList(j, j+12));
  }

  //L<S> idx = IndexedList2.ensureIndexed(tok);
  Set<S> idx = tokenIndexWithoutIfclass(tok);
  for (int i = 0; i+1 < l(data); i++) {
    S className = data[i], snippetID = data[i+1];
    if (idx.contains(className)
      && !have.contains(className)) {
      
      if (shouldNotIncludeClass.contains(className)) {
        S msg = "INCLUDING BAD CLASS: " + className;
        print(msg);
        print(join(tok));
        fail(msg);
      }

      print("Adding class " + className + " / " + snippetID);
      snippetID = formatSnippetID(snippetID);
      S text = cacheGet(snippetID);
      includeInMainLoaded(tok, text);
      L<S> ct = javaTok(text);
      shortenedSubconcepts(ct);
      conceptDeclarations(ct);
      jreplace(ct, "sclass", "static class");
      jreplace(ct, "sinterface", "static interface");
      //tok_autoCloseBrackets(ct);
      for (L<S> c : allClasses(ct))
        have.add(getClassDeclarationName(c));
      if (!have.contains(className))
        fail("Wrongly defined class: " + className + " / " + snippetID);
    }
  }
  
  ret have;
}

static Set<S> expandableClassNames = lithashset("BigInteger");

// magically append ".class" to class name references
static void expandClassReferences(L<S> tok, Set<S> classNames) {
  bool change = false;
  for (int i = 3; i+2 < l(tok); i += 2) {
    S t = tok.get(i);
    if (classNames.contains(t) || expandableClassNames.contains(t)) {
      S s = tok.get(i-2); t = tok.get(i+2);
      if (eqOneOf(s, "instanceof", "new", ".", "<", ">", "implements", "throws", "extends", "/", "nu")) continue;
      if (isIdentifier(s) || isInteger(s)) continue;
      if (eq(s, ",") && isIdentifier(get(tok, i-4)) && eqGet(tok, i-6, "<")) continue; // e.g. T3<S, S, S>
      if (eq(s, ",") && eqOneOf(_get(tok, i-6), "implements", "throws")) continue;
      // TODO: longer lists
      
      // check for cast
      if (eq(s, "(") && eq(t, ")") && i >= 5) {
        if (!eqOneOf(get(tok, i+4), "{", ";")) {
          S x = tok.get(i-4);
          if (!isIdentifier(x)) continue;
          if (eqOneOf(x, "ret", "return")) continue;
        }
      }
      if (eqOneOf(t, ",", ")", ";", ":")) {
        tok.set(i, tok.get(i) + ".class");
        change = true;
      }
    }
  }
  if (change) reTok(tok);
}

// "<id>/<ClassName>" => "((ClassName) <id>)"
static void slashCasts(L<S> tok, final Set<S> classNames) {
  jreplace(tok, "<id>/<id>", "(($3) $1)", new O() {
    O get(L<S> tok, int i) {
      ret classNames.contains(tok.get(i+5));
    }
  });
}

// experimental - "<ClassName>(...)" => "new <ClassName>(...)"
// doesn't work at beginning of statement as we can't easily
// distinguish it from a constructor declaration.
static void newWithoutNew(L<S> tok, final Set<S> classNames) {
  jreplace(tok, "<id>(", "new $1(", func(L<S> tok, int i) {
    if (!classNames.contains(tok.get(i+1))) false;
    bool ok = neqOneOf(_get(tok, i-1), "new", ";", "}", "{", "public", "protected", "private", ".");
    //print("newWithoutNew: checking " + struct(subList(tok, i-1, i+2)) + " => " + ok);
    ret ok;
  });
}

// +var => "var", +var
static void expandVarCopies(L<S> tok) {
  bool change = false;
  for (int i = 3; i+2 < l(tok); i += 2) {
    if (!eq(tok.get(i), "+")) continue;
    if (!eqOneOf(tok.get(i-2), "(", ",", "{")) continue;
    S s = tok.get(i+2);
    if (!isIdentifier(s)) continue;
    tok.set(i, quote(s) + ", ");
    change = true;
  }
  if (change) reTok(tok);
}

static void processConceptsDot(L<S> tok) {
  bool change;
  do {
    change = false;
    for (int i : jfindAll(tok, "concepts."))
      if (contains(get(tok, i+3), "\n")) {
        replaceTokens(tok, i, i+3, "!" + "include #1004863 // Dynamic Concepts");
        reTok(tok, i, i+3);
        change = true;
        break;
      }
  } while (change);
}

static void addFieldOrder(L<S> tok, int i) {
  int idx = findCodeTokens(tok, i, false, "{");
  if (idx < 0) ret;
  int j = findEndOfBracketPart(tok, idx);
  L<S> vars = allVarNames(subList(tok, idx+1, j-1));
  print("addFieldOrder " + struct(vars));
  if (!vars.contains("_fieldOrder")
    && !isSortedList(vars)) {
    print("Adding field order");
    tok.set(idx+2, "static String _fieldOrder = " + quote(join(" ", vars)) + ";\n  " + tok.get(idx+2));
    // reTok has to be done by caller
  }
}

static void caseAsVariableName(L<S> tok) {
  if (!tok.contains("case")) ret;
  for (int i = 1; i+2 < l(tok); i += 2) {
    S t = tok.get(i+2);
    if (tok.get(i).equals("case")
      && !(t.startsWith("'") || isInteger(t) || isIdentifier(t)))
      tok.set(i, "_case");
  }
}

static void continueAsFunctionName(L<S> tok) {
  jreplace(tok, "continue(", "_continue(");
}

// f bla => "bla" - and "please include function bla."
static void functionReferences(L<S> tok) {
  int i;
  S keyword = "f";
  while ((i = jfind(tok, keyword + " <id>", new O() {
    O get(L<S> tok, int i) {
      ret !eq(tok.get(i+3), "instanceof");
    }
  })) >= 0) {
    S f = tok.get(i+2);
    clearTokens(tok, i, i+2);
    tok.set(i+2, quote(f));
    reTok(tok, i, i+2);
    tok.set(l(tok)-1, last(tok) + "\nplease include function " + f + ".");
    reTok(tok, l(tok)-1, l(tok));
  }
}

// # 123 => "#123"
static void directSnippetRefs(L<S> tok) {
  int i;
  while ((i = jfind(tok, "#<int>", new O() {
    bool get(L<S> tok, int i) {
      ret neq(_get(tok, i-1), "include");
    }
  })) >= 0) {
    S id = tok.get(i+2);
    clearTokens(tok, i+1, i+3);
    tok.set(i, quote("#" + id));
    reTok(tok, i, i+3);
  }
}

static void quicknu(L<S> tok) {
  jreplace(tok, "nu <id>(", "nu($2.class, ");
  jreplace(tok, "nu <id>", "new $2");
}

// fill variable innerClasses_list
static void innerClassesVar(L<S> tok) {
  if (!tok.contains("myInnerClasses_list")) ret;
  
  L<S> have = classDeclarationNames(innerClassesOfMain(tok));
  int i = jfind(tok, ">myInnerClasses_list;");
  if (i < 0) ret;
  tok.set(i+4, "=litlist(\n" + joinQuoted(", ", have) + ");");
  reTok(tok, i+4, i+5);
}

// process ifclass x ... endif blocks
static void tok_ifclass(L<S> tok) {
  if (!tok.contains("ifclass")) ret;
  
  L<S> have = classDeclarationNames(innerClassesOfMain(tok));
  int i;
  while ((i = rjfind(tok, "ifclass <id>")) >= 0) {
    int j = jfind(tok, i+4, "endif");
    if (j < 0) j = l(tok)-1;
    bool has = have.contains(tok.get(i+2));
    clearTokens(tok, i, i+3);
    clearTokens(tok, j, j+1);
    if (!has) clearTokens(tok, i+3, j);
    reTok(tok, i, j+1);
  }
}

// set flag *.
static void tok_definitions(L<S> tok) {
  int i;
  while ((i = jfind(tok, "set flag <id>.")) >= 0) {
    String fname = tok.get(i+4);
    definitions.add(fname);
    clearAllTokens(tok.subList(i, i+8));
  }
}

// process ifndef x ... endifndef blocks
static void tok_ifndef(L<S> tok) {
  if (!tok.contains("ifndef")) ret;
  
  int i;
  while ((i = rjfind(tok, "ifndef <id>")) >= 0) {
    int j = jfind(tok, i+4, "endifndef");
    if (j < 0) j = l(tok)-1;
    S fname = tok.get(i+2);
    bool has = !definitions.contains(fname);
    print("ifndef " + fname + " => " + has);
    clearTokens(tok, i, i+3);
    clearTokens(tok, j, j+1);
    if (!has) clearTokens(tok, i+3, j);
    reTok(tok, i, j+1);
  }
}

// process ifdef x ... endifdef blocks
static void tok_ifdef(L<S> tok) {
  if (!tok.contains("ifdef")) ret;
  
  int i;
  while ((i = rjfind(tok, "ifdef <id>")) >= 0) {
    int j = jfind(tok, i+4, "endifdef");
    if (j < 0) j = l(tok)-1;
    S fname = tok.get(i+2);
    bool has = definitions.contains(fname);
    print("ifdef " + fname + " => " + has);
    clearTokens(tok, i, i+3);
    clearTokens(tok, j, j+1);
    if (!has) clearTokens(tok, i+3, j);
    reTok(tok, i, j+1);
  }
}

svoid conceptDeclarations(L<S> tok) {
  for (S kw : ll("concept", "sconcept")) {
    O cond = new O() {
      O get(L<S> tok, int i) {
        addFieldOrder(tok, i+1);
        ret true;
      }
    };
    bool re = false;
    if (jreplace(tok, kw + " <id> {", "static class $2 extends Concept {", cond)) re = true;
    if (jreplace(tok, kw + " <id> implements", "static class $2 extends Concept implements", cond)) re = true;
    if (jreplace(tok, kw + " <id>", "static class $2", cond)) re = true;
    if (re) reTok(tok);
  }
}

svoid shortenedSubconcepts(L<S> tok) {
  jreplace(tok, "<id> > <id> {", "concept $3 extends $1 {", new O() { O get(L<S> tok, int i) {
    bool b = (i == 0 || tok.get(i).contains("\n")) || eq(_get(tok, i-1), "abstract"); // only at beginning of line or after "abstract"
    //print("subconcept " + b + ": " + structure(subList(tok, i-1, i+5)));
    ret b;
  }});
}

// -slightly experimental
// -do calculation in another thread, then return to AWT thread
// -must be placed in a block
// -transforms rest of block 
svoid unswing(L<S> tok) {
  int i;
  while ((i = jfind(tok, "unswing {")) >= 0) {
    int idx = i+2;
    int closingBracket = findEndOfBracketPart(tok, idx)-1;
    int endOfOuterBlock = findEndOfBracketPart(tok, closingBracket)-1;
    tok.set(i, "thread");
    tok.set(closingBracket, " awt {");
    tok.set(endOfOuterBlock, "}}}");
    reTok(tok, closingBracket-1, endOfOuterBlock+1);
  }
}

// -slightly experimental
// -Syntax: lock theLock;
// -lock a lock, unlock at end of current block with finally
svoid lockBlocks(L<S> tok) {
  int i;
  while ((i = jfind(tok, "lock <id>", func(L<S> tok, int i) { neq(tok.get(i+3), "instanceof") })) >= 0) {
    int semicolon = findEndOfStatement(tok, i)-1;
    S var = makeVar("lock");
    int endOfOuterBlock = findEndOfBracketPart(tok, semicolon)-1;
    replaceTokens(tok, i, semicolon+1,
      "Lock " + var + " = " + joinSubList(tok, i+2, semicolon-1) + "; lock(" + var + "); try {");
    tok.set(endOfOuterBlock, "} finally { " + var + ".unlock(); } }");
    reTok(tok, i, endOfOuterBlock+1);
  }
}

Began life as a copy of #759