!747

m {

  !include #1001496 // Matches

  !include #1000882 // EGDiff

  !include #1000883 // BlockDiffer

  !include #1001065 // DialoGIO

  !include #1002662 // isTrue

  !include #1003674 // Standard Classes List

  static int varCount;

  static new Map<S, S> snippetCache;

  p {

    print("759 STARTING " + identityHashCode(main.class));

    varCount = 0;

    L<S> tok = tokMainJava();

    L ts = findTranslators(toLines(join(tok)));

    print("Translators in source at start: " + structure(ts));

    // "duplicate" statement

    L<S> lines = toLines(join(tok));

    call(getJavaX(), "findTranslators", lines);

    new Matches m;

    if (match("duplicate *", fromLines(lines), m)) {

      // actual copying - unused

      // tok = javaTok(loadSnippet(m.get(0)));

      // reference by include()

      tok = javaTok("m { p { callMain(include(" + quote(m.get(0)) + ")); } }");

    }

    // add m { }

    if (!hasCodeTokens(tok, "m", "{") && !hasCodeTokens(tok, "main", "{") && !hasCodeTokens(tok, "class", "main"))

      tok = javaTok(moveImportsUp("m {\n" + join(tok) + "\n}"));

    // standard translate

    ts = findTranslators(toLines(join(tok)));

    print("Translators in source: " + structure(ts));

    tok = javaTok(defaultTranslate(join(tok)));

    print("end of default translate");

    //print(join(tok));

    tok = processIncludes(tok); // before standard functions

    earlyStuff(tok);

    int safety = 0;

    boolean same;

    do {

      S before = join(tok);

      tok = standardFunctions(tok);

      tok = stdstuff(tok); // standard functions and all the keywords

      S diff;

      long startTime = now();

      diff = unidiff(before, join(tok));

      print("unidiff: " + (now()-startTime) + " ms");

      same = eq(diff, "");

      if (!same) {

        print("Not same " + safety + ".");

        //print(indent(2, diff));

      }

      if (safety++ >= 10) fail("safety 10 error!");

    } while (!same);

    // POST-PROCESSING after stdstuff loop

    tok = javaTok(quicknew(join(tok)));

    tok = extendClasses(tok);

    libs(tok);

    sourceCodeLine(tok);

    throwFail(tok);

    tok = autoImports(tok); // faster to do it at the end

    saveMainJava(tok);

  }

  // now also for interfaces

  static L<S> loadClass(L<S> tok, S className, S snippetID) {

    if (containsToken(tok, className)

      && !hasCodeTokens(tok, "class", className) && !hasCodeTokens(tok, "interface", className)

      && !tok_importsClassNamed(tok, className)) {

      snippetID = formatSnippetID(snippetID);

      S text = cacheGet(snippetID);

      tok = includeInMainLoaded(tok, text);

    }

    ret tok;

  }

  static L<S> stdstuff(L<S> tok) {

    //if (++level >= 10) fail("woot? 10");

    print("stdstuff!");

    L<S> lines = toLines(join(tok));

    L ts = findTranslators(lines);

    tok = javaTok(fromLines(lines));

    if (nempty(ts)) {

      //print("Warning: Contains translators.");

      print("DROPPING TRANSLATORS: " + structure(ts));

    }

    tok = quickmain(tok);

    tok = processIncludes(tok);

    earlyStuff(tok);

    tok = multilineStrings(tok);

    listComprehensions(tok);

    tok = replaceKeywordBlock(tok, "answer",

      "static S answer(S s) {\nnew Matches m;\n",

      "\nret null;\n}");

    tok = replaceKeywordBlock(tok, "loading",

      "{ JWindow _loading_window = showLoadingAnimation(); try {",

      "} finally { disposeWindow(_loading_window); }}");

    tok = replaceKeywordBlock(tok, "html",

      "static O html(S uri, fMap<S, S> params) {\n", "}");

    // "static sync" => static synchronized

    jreplace(tok, "static sync", "static synchronized");

    // "sclass" => static class

    jreplace(tok, "sclass", "static class");

    // "asclass" => abstract static class

    jreplace(tok, "asclass", "abstract static class");

    // "sinterface" => static interface

    jreplace(tok, "sinterface", "static interface");

    // "ssynchronized" => static synchronized

    jreplace(tok, "ssynchronized", "static synchronized");

    jreplace(tok, "ssvoid", "static synchronized void");

    jreplace(tok, "svoid", "static void");

    jreplace(tok, "sbool", "static bool");

    jreplace(tok, "sint", "static int");

    jreplace(tok, "snew", "static new");

    jreplace(tok, "sv <id>", "static void $2");

    jreplace(tok, "pvoid", "public void");

    // "sS" => static S

    jreplace(tok, "sS", "static S");

    // "sO" => static O

    jreplace(tok, "sO", "static O");

    // "sL" => static L

    jreplace(tok, "sL", "static L");

    // "toString {" => "public S toString() {"

    jreplace(tok, "toString {", "public S toString() {");

    jreplace(tok, "Int", "Integer");

    jreplace(tok, "Bool", "Boolean");

    // "catch {" => "catch (Throwable _e) {"

    jreplace(tok, "catch {", "catch (Throwable _e) {");

    // "catch X e {" => "catch (X e) {"

    jreplace(tok, "catch <id> <id> {", "catch ($2 $3) {");

    // "catch e {" => "catch (Throwable e) {" (if e is lowercase)

    jreplace(tok, "catch <id> {", "catch (Throwable $2) {", new O() {

      bool check(L<S> tok, int i) {

        S word = tok.get(i+3);

        ret startsWithLowerCase(word);

      }

    });

    // some crazy fancy syntax

    jreplace(tok, "set <id>;", "$2 = true;");

    jreplace(tok, "fS", "final S");

    jreplace(tok, "fO", "final O");

    jreplace(tok, "fL", "final L");

    jreplace(tok, "fMap", "final Map");

    jreplace(tok, "fRunnable", "final Runnable");

    jreplace(tok, "f int", "final int");

    jreplace(tok, "new <id> {", "new $2() {");

    // shortened method declarations - hopefully that works

    jreplace(tok, "void <id> {", "$1 $2() {");

    jreplace(tok, "String <id> {", "$1 $2() {");

    jreplace(tok, "Object <id> {", "$1 $2() {");

    jreplace(tok, "List <id> {", "$1 $2() {");

    // "is a X" => instanceof X

    jreplace(tok, "is a <id>", "instanceof $3");

    // func keyword for lambdas - now automatically quines toString()

    int i;

    while ((i = jfind(tok, "func(")) >= 0) {

      int argsFrom = i+4, argsTo = findCodeTokens(tok, i, false, ")");

      int idx = findCodeTokens(tok, argsTo, false, "{");

      int j = findEndOfBracketPart(tok, idx+2);

      L<S> contents = subList(tok, idx+1, j-1);

      replaceTokens(tok, i, j, "new O { O get(" + join(subList(tok, argsFrom, argsTo-1)) + ") { " + tok_addReturn(contents) + " }\n" +

      "  public S toString() { ret " + quote(trim(join(contents))) + "; }}");

      reTok(tok);

    }

    while ((i = jfind(tok, "voidfunc(")) >= 0) {

      int argsFrom = i+4, argsTo = findCodeTokens(tok, i, false, ")");

      int idx = findCodeTokens(tok, argsTo, false, "{");

      int j = findEndOfBracketPart(tok, idx+2);

      L<S> contents = subList(tok, idx+1, j-1);

      replaceTokens(tok, i, j, "new O { void get(" + join(subList(tok, argsFrom, argsTo-1)) + ") { " + join(contents) + " }\n" +

      "  public S toString() { ret " + quote(trim(join(contents))) + "; }}");

      reTok(tok);

    }

    while ((i = jfind(tok, "func {")) >= 0) {

      int idx = findCodeTokens(tok, i, false, "{");

      int j = findEndOfBracketPart(tok, idx+2);

      L<S> contents = subList(tok, idx+1, j-1);

      replaceTokens(tok, i, j, "new O { O get() { " + tok_addReturn(contents) + " }\n" +

      "  public S toString() { ret " + quote(trim(join(contents))) + "; }}");

      reTok(tok);

    }

    /*tok = replaceKeywordBlock(tok, "func",

      "new O { O get() { ret",

      ";}}");*/

    // * constructors

    if (hasCodeTokens(tok, "\\*", "("))

      tok = expandStarConstructors(tok);

    // Do this BEFORE awt replacement! ("p-awt" contains "awt" token)

    tok = replaceKeywordBlock(tok, "p-substance", "p-awt { substance();", "}");

    tok = replaceKeywordBlock(tok, "p-awt", "p { awt {", "}}");

    tok = replaceKeywordBlock(tok,

      "awt",

      "swingLater(r {",

      "});");

    // crazy stuff

    jreplace (tok, "for <id> over <id>:", "for (int $2 = 0; $2 < l($4); $2++)");

    // STANDARD CLASSES & INTERFACES

    for (S line : toLinesFullTrim(standardClasses)) {

      int idx = line.indexOf('/');

      tok = loadClass(tok, line.substring(0, idx), line.substring(idx+1));

    }

    tok = loadClass(tok, "Quine", "#1003601");

    tok = loadClass(tok, "Var", "#1003550");

    tok = loadClass(tok, "TableFinder", "#1000850");

    tok = loadClass(tok, "Snippet", "#1003519");

    tok = loadClass(tok, "GermanDateAdapter", "#1003510");

    tok = loadClass(tok, "BF", "#1003489");

    tok = loadClass(tok, "BotChat", "#1003499");

    tok = loadClass(tok, "StrictTreeMap", "#1003482");

    tok = loadClass(tok, "E", "#1003474");

    tok = loadClass(tok, "Dialog", "#1003356");

    tok = loadClass(tok, "OccTree2", "#1003407");

    tok = loadClass(tok, "OccTree", "#1003331");

    tok = loadClass(tok, "EGDiff", "#1000882");

    tok = loadClass(tok, "BlockDiffer", "#1000883");

    tok = loadClass(tok, "Q", "#1000934");

    tok = loadClass(tok, "F", "#1003033");

    tok = loadClass(tok, "LoadEnv", "#1002955");

    tok = loadClass(tok, "CIString", "#1002873");

    tok = loadClass(tok, "PersistentCIMap", "#1002872");

    tok = loadClass(tok, "EvalTransform", "#1002867");

    tok = loadClass(tok, "Prolog", "#1002884"); // Yay!

    tok = loadClass(tok, "IEngine", "#1002809");

    tok = loadClass(tok, "Native", "#1002774");

    tok = loadClass(tok, "ProgramScan", "#1000804");

    tok = loadClass(tok, "SNLMatches", "#1002748");

    tok = loadClass(tok, "SNLInfo", "#1002744");

    tok = loadClass(tok, "Lisp", "#1002696");

    tok = loadClass(tok, "Explain", "#1002529");

    tok = loadClass(tok, "NanoHTTPD", "#1001651");

    tok = loadClass(tok, "Source", "#1002461");

    tok = loadClass(tok, "ShowBigText", "#1001425");

    tok = loadClass(tok, "DialogIO", "#1001065");

    tok = loadClass(tok, "Matches", "#1001496");

    tok = loadClass(tok, "MultiMap", "#1001296");

    tok = loadClass(tok, "MultiSet", "#1000988");

    tok = loadClass(tok, "MultiHashSet", "#1003597");

    tok = loadClass(tok, "PersistentLog", "#1001972");

    tok = loadClass(tok, "PersistentMap", "#1002063");

    tok = loadClass(tok, "PersistentTreeMap", "#1003529");

    tok = loadClass(tok, "Rat", "#1002052");

    tok = loadClass(tok, "DiskTextMap", "#1002087");

    // image classes (now static)

    tok = loadClass(tok, "ImageSurface", "#1002470");

    tok = loadClass(tok, "RGB", "#1002470");

    tok = loadClass(tok, "RGBImage", "#1002470");

    // the infamous missing functions (usually caused by class Matches)

    if (!hasCodeTokens(tok, "String", "unquote") && containsToken(tok, "unquote")) {

      print("Adding unquote");

      tok = includeInMain(tok, "#1001735");

    }

    if (!hasCodeTokens(tok, "String", "formatSnippetID") && containsToken(tok, "formatSnippetID")) {

      print("Adding formatSnippetID");

      tok = includeInMain(tok, "#1000709");

    }

    tok = expandShortTypes(tok);

    if (containsToken(tok, "cast")) {

      S s = join(tok);

      s = s.replaceAll("(\\w+<[\\w\\s,\\[\\]]+>|\\w+|\\w+\\[\\]|\\w+\\[\\]\\[\\])\\s+(\\w+)\\s*=\\s*cast(\\W[^;]*);", "$1 $2 = ($1) ($3);");

      tok = javaTok(s);

    }

    for (S keyword : ll("runnable", "r"))

      tok = replaceKeywordBlock(tok,

        keyword,

        "new Runnable() { public void run() { try {",

        "} catch (Exception __e) { throw __e instanceof RuntimeException ? (RuntimeException) __e : new RuntimeException(__e); }}}");

    tok = replaceKeywordBlock(tok,

      "expectException",

      "{ bool __ok = false; try {",

      "} catch { __ok = true; } assertTrue(\"expected exception\", __ok); }");

    while ((i = tok.indexOf("tex")) >= 0) {

      tok.set(i, "throws Exception");

      tok = javaTok(tok);

    }

    while ((i = findCodeTokens(tok, "while", "true")) >= 0) {

      tok.set(i+2, "(true)");

      tok = javaTok(tok);

    }

    // "null;" instead of "return null;"

    jreplace(tok, "{ null;", "$1 return null;");

    jreplace(tok, "} null;", "$1 return null;");

    jreplace(tok, "; null;", "$1 return null;");

    jreplace(tok, ") null;", "$1 return null;");

    // "false;" instead of "return false;"

    jreplace(tok, "} false;", "} return false;");

    jreplace(tok, "; false;", "; return false;");

    jreplace(tok, ") false;", ") return false;");

    // "true;" instead of "return true;"

    jreplace(tok, "} true;", "} return true;");

    jreplace(tok, "; true;", "; return true;");

    jreplace(tok, ") true;", ") return true;");

    // "myFunction;" instead of "myFunction();" - quite rough

    O cond = new O() {

      bool check(L<S> tok, int i) {

        S word = tok.get(i+3);

        //print("single word: " + word);

        ret !litlist("break", "continue", "return").contains(word);

      }

    };

    for (S pre : litlist("}", ";"))

      jreplace(tok, pre + " <id>;", "$1 $2();", cond);

    // shorter match syntax for answer methods

    jreplace(tok, "if <quoted>", "if (match($2, s, m))");

    jreplace(tok, "if match <quoted>", "if (match($3, s, m))");

    // extra commas ("litlist(1, 2,)")

    jreplace(tok, ",)", ")");

    // additional translations (if necessary)

    if (hasCodeTokens(tok, "pcall", "{"))

      tok = replaceKeywordBlock(tok,

        "pcall",

        "try {",

        "} catch (Throwable __e) { printStackTrace(__e); }");

    tok = dialogHandler(tok);

    tok = replaceKeywordBlock(tok, "exceptionToUser",

      "try {",

      "} catch (Throwable __e) { ret exceptionToUser(__e); }"); 

    if (hasCodeTokens(tok, "twice", "{"))

      tok = replaceKeywordBlock(tok, "twice",

        "for (int __twice = 0; __twice < 2; __twice++) {",

        "}"); 

    while ((i = findCodeTokens(tok, "repeat", "*", "{")) >= 0) {

      S v = makeVar("repeat");

      tok.set(i, "for (int " + v + " = 0; " + v + " < " + tok.get(i+2) + "; " + v + "++)");

      tok.set(i+2, "");

      tok = javaTok(tok);

    }

    tok = replaceKeywordBlockDyn(tok,

      "time",

      new O() { S[] get() {

        S var = makeVar("startTime");

        ret new S[] {

          "{ long " + var + " = now(); try { ",

          "} finally { " + var + " = now()-" + var + "; saveTiming(" + var + "); } }"};

      }});

    if (hasCodeTokens(tok, "assertFail", "{")) {

      S var = makeVar("oops");

      tok = replaceKeywordBlock(tok,

        "assertFail",

        "boolean " + var + " = false; try {",

        "\n" + var + " = true; } catch (Exception e) { /* ok */ } assertFalse(" + var + ");"); 

    }

    tok = replaceKeywordBlock(tok,

      "yo",

      "try {",

      "} catch (Exception " + makeVar("e") + ") { ret false; }");

    tok = replaceKeywordBlock(tok,

      "awtIfNecessary",

      "swingNowOrLater(r {",

      "});");

    if (hasCodeTokens(tok, "ctex"))

      tok = ctex(tok);

    tok = replaceKeywordBlock(tok,

      "actionListener",

      "new java.awt.event.ActionListener() { " +

        "public void actionPerformed(java.awt.event.ActionEvent _evt) {",

      "}}");

    namedThreads(tok);

    threads(tok);

    // try answer

    while ((i = findCodeTokens(tok, "try", "answer")) >= 0) {

      int j = findEndOfStatement(tok, i);

      S v = makeVar("a");

      tok.set(i, "S " + v);

      tok.set(i+2, "=");

      tok.set(j-1, "; if (!empty(" + v + ")) ret " + v + ";");

      tok = javaTok(tok);

    }

    tok = javaTok(moveImportsUp(join(tok)));

    ret tok;

  }

  static L<S> multilineStrings(L<S> tok) {

    for (int i = 1; i < tok.size(); i += 2) {

      S t = tok.get(i);

      if (t.startsWith("[") && isQuoted (t))

        tok.set(i, quote(unquote(t)));  

    }

    ret tok;

  }

  static L<S> quickmain(L<S> tok) {

    int i = findCodeTokens(tok, "main", "{");

    if (i < 0) i = findCodeTokens(tok, "m", "{");

    if (i >= 0 && !(i-2 > 0 && tok.get(i-2).equals("class")))

      tok.set(i, "public class main");

    i = findCodeTokens(tok, "psvm", "{");

    if (i < 0) i = findCodeTokens(tok, "p", "{");

    if (i >= 0)

      tok.set(i, "public static void main(String[] args) throws Exception");

    ret javaTok(tok);

  }

  static S makeVar(S name) {

    ret "_" + name + "_" + varCount++;

  }

  static S makeVar() { ret makeVar(""); }

  /*static L<S> standardFunctions(L<S> tok) {

    ret rtq(tok, "#1002474");

  }*/

  static L<S> rtq(L<S> tok, S id) {

    ret runTranslatorQuick(tok, id);

  }

  static L<S> expandShortTypes(L<S> tok) {

    // replace <int> with <Integer>

    for (int i = 1; i+4 < tok.size(); i += 2)

      if (tok.get(i).equals("<")

        && litlist(">", ",").contains(tok.get(i+4))) {

        String type = tok.get(i+2);

        if (type.equals("int")) type = "Integer";

        else if (type.equals("long")) type = "Long";

        tok.set(i+2, type);

      }

    // O = Object, S = String, ret = return

    for (int i = 1; i < tok.size(); i += 2) {

      String t = tok.get(i);

      if (t.equals("O")) t = "Object";

      if (t.equals("S")) t = "String";

      else if (t.equals("L")) t = "List";

      else if (t.equals("F")) t = "Function";

      else if (t.equals("ret")) t = "return";

      else if (t.equals("bool") && i+2 < tok.size() && neq(tok.get(i+2), "(")) t = "boolean"; // bool -> boolean if it's not a function name

      tok.set(i, t);

    }

    ret tok;

  }

  static L<S> autoImports(L<S> tok) {

    S s = join(tok);

    List<String> imports = findImports(s);

    new StringBuilder buf;

    for (String c : standardImportClasses)

      if (!(imports.contains(c)))

        buf.append("import " + c + ";\n");

    if (buf.length() == 0) ret tok;

    ret javaTok(buf+s);

  }

  static String[] standardImportClasses = {

    "java.util.*",

    "java.util.zip.*",

    "java.util.List",

    "java.util.regex.*",

    "java.util.concurrent.*",

    "java.util.concurrent.atomic.*",

    "javax.swing.*",

    "javax.swing.event.*",

    "javax.swing.text.*",

    "javax.swing.table.*",

    "java.io.*",

    "java.net.*",

    "java.lang.reflect.*",

    "java.lang.ref.*",

    "java.lang.management.*",

    "java.security.*",

    "java.security.spec.*",

    "java.awt.*",

    "java.awt.event.*",

    "java.awt.image.*",

    "javax.imageio.*",

    "java.math.*"

  };

  static L<S> expandStarConstructors(L<S> tok) {

    mainLoop: for (int i = 3; i+6 < tok.size(); i += 2) {

      String t = tok.get(i), l = tok.get(i-2);

      if (!t.equals("*"))

        continue;

      if (!tok.get(i+2).equals("("))

        continue;

      if (!eqOneOf(l, "}", "public", "private", "protected", ";", "{")) // is this correct...??

        continue;

      // ok, it seems like a constructor declaration.

      // Now find class name by going backwards.

      int j = i, level = 1;

      while (j > 0 && level > 0) {

        t = tok.get(j);

        if (t.equals("}")) ++level;

        if (t.equals("{")) --level;

        j -= 2;

      }

      while (j > 0) {

        t = tok.get(j);

        if (t.equals("class")) {

          String className = tok.get(j+2);

          tok.set(i, className); // exchange constructor name!

          // now for the parameters.

          // Syntax: *(Learner *learner) {

          // We will surely add type inference here in time... :)

          j = i+2;

          while (!tok.get(j).equals("{"))

            j += 2;

          int block = j+1;

          for (int k = i+2; k < block-1; k += 2)

            if (tok.get(k).equals("*")) {

              tok.remove(k);

              tok.remove(k);

              block -= 2;

              String name = tok.get(k);

              tok.addAll(block, Arrays.asList(new String[] {

                "\n  ", "this", "", ".", "", name, " ", "=", " ", name, "", ";" }));

            }

          continue mainLoop;

        }

        j -= 2;

      }

    }

    ret tok;

  }

  static L<S> processIncludes(L<S> tok) {

    int safety = 0;

    while (hasCodeTokens(tok, "!", "include") && ++safety < 100)

      tok = processIncludesSingle(tok);

    ret tok;

  }

  static L<S> processIncludesSingle(L<S> tok) {

    S s = join(tok);

    Matcher m = Pattern.compile("\n\\s*!include (#\\d+)").matcher(s);

    StringBuffer buf = new StringBuffer();

    while (m.find()) {

      String includedSrc = loadSnippet(m.group(1));

      m.appendReplacement(buf, m.quoteReplacement("\n" + includedSrc));

    }

    m.appendTail(buf);

    ret javaTok(str(buf));

  }

  static L<S> ctex(L<S> tok) {

    S s = join(tok);

    Pattern regex = Pattern.compile("\\s+(no\\s+exceptions|ctex|null on exception)\\s*\\{");

    for (int i = 0; i < 100; i++) {

      Matcher matcher = regex.matcher(s);

      if (!matcher.find())

        break;

      String kind = matcher.group(1);

      System.out.println("Iteration " + (i+1));

      int start = matcher.start(), end = matcher.end();

      int endOfBlock = ctex_findEndOfBlock(s, end);

      String catchBlock, catchWhat;

      if (kind.startsWith("null")) {

        catchBlock = "return null;";

        catchWhat = "Throwable";

      } else {

        catchBlock = "throw __e instanceof RuntimeException ? (RuntimeException) __e : new RuntimeException(__e);";

        catchWhat = "Throwable";

      }

      String tryBlock = " { try {\n " +

        s.substring(end, endOfBlock) + "\n} catch (" + catchWhat + " __e) { " + catchBlock + " }";

      s = s.substring(0, start) + tryBlock + s.substring(endOfBlock);

    }

    ret javaTok(s);

  }

  // start is the index AFTER the opening bracket

  // returns index OF closing bracket

  static int ctex_findEndOfBlock(String s, int start) {

    int level = 1;

    for (int i = start; i < s.length(); i++) {

      if (s.charAt(i) == '{') ++level;

      else if (s.charAt(i) == '}') --level;

      if (level == 0)

        return i;

    }

    return s.length();

  }

  static L<S> dialogHandler(L<S> tok) {

    ret replaceKeywordBlock(tok,

      "dialogHandler",

      "new DialogHandler() {\n" +

        "public void run(final DialogIO io) {",

      "}}");

  }

  /*static S defaultTrans(S s) {

    findTranslators...

  }*/

  static S quicknew(S s) {

    // TODO: switch to tokenizing!

    s = s.replaceAll("new\\s+(\\w+)\\s+(\\w+);", "$1 $2 = new $1();");

    s = s.replaceAll("new\\s+L<([\\w\\[\\]<>,\\s]+)>\\s+(\\w+);", "List<$1> $2 = new ArrayList<$1>();");

    s = s.replaceAll("new\\s+List<([\\w\\[\\]<>,\\s]+)>\\s+(\\w+);", "List<$1> $2 = new ArrayList<$1>();");

    s = s.replaceAll("new\\s+\\(Hash\\)Set<(\\w+)>\\s+(\\w+);", "Set<$1> $2 = new HashSet<$1>();");

    s = s.replaceAll("new\\s+\\(Tree\\)Set<(\\w+)>\\s+(\\w+);", "Set<$1> $2 = new TreeSet<$1>();");

    s = s.replaceAll("new\\s+Set<(\\w+)>\\s+(\\w+);", "Set<$1> $2 = new TreeSet<$1>();"); // TreeSet now default - pay attention to explicitly say HashSet if you need it.

    s = s.replaceAll("new\\s+\\(Hash\\)Map<([\\w\\s,]+)>\\s+(\\w+);", "Map<$1> $2 = new HashMap<$1>();");

    s = s.replaceAll("new\\s+\\(Tree\\)Map<([\\w\\s,]+)>\\s+(\\w+);", "Map<$1> $2 = new TreeMap<$1>();");

    // TreeMap when string as key

    s = s.replaceAll("new\\s+Map<(S,[\\w\\s,]+)>\\s+(\\w+);", "Map<$1> $2 = new TreeMap<$1>();");

    s = s.replaceAll("new\\s+Map<(String,[\\w\\s,]+)>\\s+(\\w+);", "Map<$1> $2 = new TreeMap<$1>();");

    // HashMap is default for everything else

    s = s.replaceAll("new\\s+Map<([\\w\\s,]+)>\\s+(\\w+);", "Map<$1> $2 = new HashMap<$1>();");

    s = s.replaceAll("new\\s+(\\w+<[\\w\\s,]+>)\\s+(\\w+);", "$1 $2 = new $1();");

    // Constructor calls without parantheses

    // So you can say something like: predictors.add(new P1);

    s = s.replaceAll("new\\s+(\\w+)\\s*([,\\);])", "new $1()$2");

    // replace "new List" with "new ArrayList"

    s = s.replaceAll("new\\s+List\\s*\\(", "new ArrayList(");

    // for args

    s = s.replace("for args {", "for (int i = 0; i < args.length; i++) { final String arg = args[i];");

    ret s;

  }

  static L<S> extendClasses(L<S> tok) {

    int i;

    while ((i = jfind(tok, "extend <id> {")) >= 0) {

      S className = tok.get(i+2);

      int idx = findCodeTokens(tok, i, false, "{");

      int j = findEndOfBracketPart(tok, idx+2);

      S content = join(subList(tok, idx+1, j-1));

      L<S> c = findInnerClassOfMain(tok, className);

      clearTokens(tok.subList(i, j+1));

      if (c == null) {

        print("Warning: Can't extend class " + className + ", not found");

        continue;

      }

      int endOfClass = magicIndexOfSubList(tok, c) + l(c)-1;

      while (neq(tok.get(endOfClass), "}")) --endOfClass;

      tok.set(endOfClass, content + "\n" + tok.get(endOfClass));

      reTok(tok);

    }

    ret tok;

  }

  static void listComprehensions(L<S> tok) {

    int i;

    while ((i = jfind(tok, "[<id> <id> in")) >= 0) {

      Map<Integer, Integer> bracketMap = getBracketMap(tok);

      S type = tok.get(i+2), id = tok.get(i+4);

      int j = scanOverExpression(tok, bracketMap, i+8, "|");

      S exp = join(tok.subList(i+8, j));

      j += 2;

      int k = scanOverExpression(tok, bracketMap, j, "]");

      S where = join(tok.subList(j, k));

      ++k;

      S code = "filter(" + exp + ", func(" + type + " " + id + ") { " + where + " })";

      replaceTokens(tok, i, k, code);

      reTok(tok);

    }

  }

  // lib 123 => !123

  static void libs(L<S> tok) {

    new Set<S> libs;

    int i;

    while ((i = jfind(tok, "lib <int>")) >= 0) {

      S id = tok.get(i+2);

      print("lib " + id);

      if (!libs.contains(id)) {

        libs.add(id);

        tok.set(i, "!");

        tok.set(i+1, "");

      } else {

        print("...ignoring (duplicate)");

        clearAllTokens(tok, i, i+3);

        reTok(tok);

      }

    }

  }

  // sourceCodeLine() => 1234

  static void sourceCodeLine(L<S> tok) {

    int i ;

    while ((i = jfind(tok, "sourceCodeLine()")) >= 0) {

      replaceTokens(tok, i, i+5, str(countChar(join(subList(tok, 0, i)), '\n')+1));

      reTok(tok);

    }

  }

  // done before any other processing

  static void earlyStuff(L<S> tok) {

    int i;

    // Note: this makes the word "quine" a special operator

    // (unusable as a function name)

    while ((i = jfind(tok, "quine(")) >= 0) {

      int idx = findCodeTokens(tok, i, false, "(");

      int j = findEndOfBracketPart(tok, idx+2);

      tok.set(i, "new Quine");

      tok.set(idx, "(" + quote(join(subList(tok, idx+1, j-1))) + ", ");

      reTok(tok);

    }

  }

  static void throwFail(L<S> tok) {

    bool anyChange = false;

    for (int i = 1; i+2 < l(tok); i += 2)

      if (eq(get(tok, i+2), "fail") && !eqOneOf(get(tok, i), "throw", "RuntimeException", "return")) {

        tok.set(i+2, "throw fail");

        anyChange = true;

      }

    if (anyChange)

      reTok(tok);

  }

  static void namedThreads(L<S> tok) {

    bool change = false;

    for (int i = 0; i < 100; i++) {

      int idx = findCodeTokens(tok, "thread", "*", "{");

      if (idx < 0)

        break;

      int j = findEndOfBracketPart(tok, idx+4);

      S tName = tok.get(idx+2);

      S var = "_t_" + i;

      S pre = "{ Thread " + var + " = new Thread(" + tName + ") {\n" +

        "public void run() {\n" +

        "try";

      S post = "} catch (Exception _e) {\n" +

          "  throw _e instanceof RuntimeException ? (RuntimeException) _e : new RuntimeException(_e); } }\n};\n" +

        var + ".start(); }";

      tok.set(idx, pre);

      tok.set(idx+2, "");

      tok.set(j-1, post);

      change = true;

    }

    if (change) reTok(tok);

  }

  static void threads(L<S> tok) {

    bool change = false;

    for (bool daemon : litlist(false, true))

      for (int i = 0; i < 100; i++) {

        int idx = findCodeTokens(tok, daemon ? "daemon" : "thread", "{");

        if (idx < 0)

          break;

        int j = findEndOfBracketPart(tok, idx+2);

        S var = "_t_" + i;

        S pre = "{ Thread " + var + " = new Thread() {\n" +

          "public void run() {\n" +

          "try";

        S post = "} catch (Exception _e) {\n" +

            "  throw _e instanceof RuntimeException ? (RuntimeException) _e : new RuntimeException(_e); } }\n};\n" +

          (daemon ? var + ".setDaemon(true);\n" : "") +

          var + ".start(); }";

        tok.set(idx, pre);

        tok.set(j-1, post);

        change = true;

      }

    if (change) reTok(tok);

  }

  static Map<S, S> sf;

  static L<S> standardFunctions(L<S> tok) {

    if (sf == null) {

      L<S> standardFunctions = (L) loadVariableDefinition(

        loadSnippet("#761"), "standardFunctions");

      sf = new HashMap();

      for (String x : standardFunctions) {

        String[] f = x.split("/");

        sf.put(f[1], f[0]);

      }

    }

    for (int i = 0; ; i++) {

      Set<String> defd = new HashSet(findFunctions(tok));

      // changes tok

      Set<String> invocations = findFunctionInvocations(tok, sf);

      print("Functions invoked: " + structure(invocations));

      List<String> needed = diff(invocations, defd);

      print("Functions needed: " + structure(needed));

      if (needed.isEmpty())

        break;

      new L<S> added;

      new StringBuilder buf;

      for (String x : needed) {

        if (defd.contains(x)) continue;

        String id = sf.get(x);

        //print("Adding function: " + x + " (" + id + ")");

        S function = cacheGet(id);

        if (("\n" + function).contains("\n!")) print("Warning: " + id + " contains translators.");

        buf.append(function).append("\n");

        added.add(x);

        defd.addAll(findFunctionDefinitions(function));

      }

      tok = includeInMainLoaded(tok, str(buf));

      defd = new HashSet(findFunctions(tok));

      print("Functions added: " + structure(added));

      for (String x : needed)

        if (!defd.contains(x))

          fail("Function not defined properly: " + x);

      print("Iteration " + (i+2));

      if (i >= 1000) fail("Too many iterations");

    }

    ret tok;

  }

  static L<S> findFunctions(L<S> tok) {

    ret findFunctionDefinitions(join(findMainClass(tok)));

  }

  static S cacheGet(S snippetID) {

    S text = snippetCache.get(snippetID);

    if (text == null)

      snippetCache.put(snippetID, text = loadSnippet(snippetID));

    ret text;

  }

}