!747

m {

  !include #1001496 // Matches

  !include #1000882 // EGDiff

  !include #1000883 // BlockDiffer

  !include #1001065 // DialoGIO

  !include #1004045 // IndexedList2

  //include #1001296 // MultiMap

  !include #1000988 // MultiSet

  !include #1002662 // isTrue

  !include #1003674 // Standard Classes List

  static int varCount;

  static new Map<S, S> snippetCache;

  static bool useIndexedList = false, opt_javaTok = true;

  p {

    if (useIndexedList) findCodeTokens_debug = true;

    javaTok_opt = opt_javaTok;

    findCodeTokens_indexed = findCodeTokens_unindexed = 0;

    javaTok_n = javaTok_elements = 0;

    print("759 STARTING " + identityHashCode(main.class));

    varCount = 0;

    L<S> tok = jtok(loadMainJava());

    L ts = findTranslators(toLines(join(tok)));

    print("Translators in source at start: " + structure(ts));

    // "duplicate" statement

    L<S> lines = toLines(join(tok));

    call(getJavaX(), "findTranslators", lines);

    new Matches m;

    if (match("duplicate *", fromLines(lines), m)) {

      // actual copying - unused

      // tok = jtok(loadSnippet(m.get(0)));

      // reference by include()

      tok = jtok("m { p { callMain(include(" + quote(m.get(0)) + ")); } }");

    }

    // add m { }

    if (!hasCodeTokens(tok, "m", "{") && !hasCodeTokens(tok, "main", "{") && !hasCodeTokens(tok, "class", "main"))

      tok = jtok(moveImportsUp("m {\n" + join(tok) + "\n}"));

    // standard translate

    ts = findTranslators(toLines(join(tok)));

    print("Translators in source: " + structure(ts));

    tok = jtok(defaultTranslate(join(tok)));

    print("end of default translate");

    //print(join(tok));

    tok = processIncludes(tok); // before standard functions

    earlyStuff(tok);

    int safety = 0;

    boolean same;

    do {

      S before = join(tok);

      tok = standardFunctions(tok);

      tok = stdstuff(tok); // standard functions and all the keywords

      S diff;

      long startTime = now();

      //diff = unidiff(before, join(tok));

      //print("unidiff: " + (now()-startTime) + " ms");

      //same = eq(diff, "");

      same = eq(before, join(tok)); // << could be sped up

      if (!same) {

        print("Not same " + safety + ".");

        //print(indent(2, diff));

      }

      if (safety++ >= 10) fail("safety 10 error!");

    } while (!same);

    // POST-PROCESSING after stdstuff loop

    tok = jtok(quicknew(join(tok)));

    tok = extendClasses(tok);

    libs(tok);

    sourceCodeLine(tok);

    throwFail(tok);

    tok = autoImports(tok); // faster to do it at the end

    if (useIndexedList)

      print("Indexed/unindexed lookups: " + findCodeTokens_indexed + "/" + findCodeTokens_unindexed + ", lists made: " + IndexedList2.instances);

    print("javaToks: " + javaTok_n + "/" + javaTok_elements);

    saveMainJava(tok);

  }

  static L<S> stdstuff(L<S> tok) {

    //if (++level >= 10) fail("woot? 10");

    print("stdstuff!");

    int i;

    if (jfind(tok, "!<int>") >= 0) {

      L<S> lines = toLines(join(tok));

      L ts = findTranslators(lines);

      tok = jtok(fromLines(lines));

      print("DROPPING TRANSLATORS: " + structure(ts));

    }

    tok = quickmain(tok);

    tok = processIncludes(tok);

    earlyStuff(tok);

    tok = multilineStrings(tok);

    listComprehensions(tok);

    tok = replaceKeywordBlock(tok, "answer",

      "static S answer(S s) {\nnew Matches m;\n",

      "\nret null;\n}");

    tok = replaceKeywordBlock(tok, "loading",

      "{ JWindow _loading_window = showLoadingAnimation(); try {",

      "} finally { disposeWindow(_loading_window); }}");

    tok = replaceKeywordBlock(tok, "html",

      "static O html(S uri, fMap<S, S> params) {\n", "}");

    // "static sync" => static synchronized

    jreplace(tok, "static sync", "static synchronized");

    // "sclass" => static class

    jreplace(tok, "sclass", "static class");

    // "asclass" => abstract static class

    jreplace(tok, "asclass", "abstract static class");

    // "sinterface" => static interface

    jreplace(tok, "sinterface", "static interface");

    // "ssynchronized" => static synchronized

    jreplace(tok, "ssynchronized", "static synchronized");

    jreplace(tok, "ssvoid", "static synchronized void");

    jreplace(tok, "svoid", "static void");

    jreplace(tok, "sbool", "static bool");

    jreplace(tok, "sint", "static int");

    jreplace(tok, "snew", "static new");

    jreplace(tok, "sv <id>", "static void $2");

    jreplace(tok, "pvoid", "public void");

    // "sS" => static S

    jreplace(tok, "sS", "static S");

    // "sO" => static O

    jreplace(tok, "sO", "static O");

    // "sL" => static L

    jreplace(tok, "sL", "static L");

    // "toString {" => "public S toString() {"

    jreplace(tok, "toString {", "public S toString() {");

    jreplace(tok, "Int", "Integer");

    jreplace(tok, "Bool", "Boolean");

    // "catch {" => "catch (Throwable _e) {"

    jreplace(tok, "catch {", "catch (Throwable _e) {");

    // "catch X e {" => "catch (X e) {"

    jreplace(tok, "catch <id> <id> {", "catch ($2 $3) {");

    // "catch e {" => "catch (Throwable e) {" (if e is lowercase)

    jreplace(tok, "catch <id> {", "catch (Throwable $2) {", new O() {

      bool check(L<S> tok, int i) {

        S word = tok.get(i+3);

        ret startsWithLowerCase(word);

      }

    });

    // some crazy fancy syntax

    jreplace(tok, "set <id>;", "$2 = true;");

    // more shortening

    jreplace(tok, "fS", "final S");

    jreplace(tok, "fO", "final O");

    jreplace(tok, "fL", "final L");

    jreplace(tok, "fMap", "final Map");

    jreplace(tok, "fRunnable", "final Runnable");

    jreplace(tok, "f int", "final int");

    jreplace(tok, "new <id> {", "new $2() {");

    // "continue unless"

    while ((i = jfind(tok, "continue unless")) >= 0) {

      int j = scanOverExpression(tok, getBracketMap(tok), i+4, ";");

      replaceTokens(tok, i, i+4, "{ if (!(");

      tok.set(j, ")) continue; }");

      reTok(tok, i, j+1);

    }

    // "continue if"

    while ((i = jfind(tok, "continue if")) >= 0) {

      int j = scanOverExpression(tok, getBracketMap(tok), i+4, ";");

      replaceTokens(tok, i, i+4, "{ if (");

      tok.set(j, ") continue; }");

      reTok(tok, i, j+1);

    }

    // shortened method declarations - hopefully that works

    jreplace(tok, "void <id> {", "$1 $2() {");

    jreplace(tok, "String <id> {", "$1 $2() {");

    jreplace(tok, "Object <id> {", "$1 $2() {");

    jreplace(tok, "List <id> {", "$1 $2() {");

    // "is a X" => instanceof X

    jreplace(tok, "is a <id>", "instanceof $3");

    // !a instanceof X => !(a instanceof X)

    jreplace(tok, "!<id> instanceof <id>", "!($2 instanceof $4)");

    // func keyword for lambdas - now automatically quines toString()

    while ((i = jfind(tok, "func(")) >= 0) {

      int argsFrom = i+4, argsTo = findCodeTokens(tok, i, false, ")");

      int idx = findCodeTokens(tok, argsTo, false, "{");

      int j = findEndOfBracketPart(tok, idx+2);

      L<S> contents = subList(tok, idx+1, j-1);

      replaceTokens(tok, i, j, "new O { O get(" + join(subList(tok, argsFrom, argsTo-1)) + ") { " + tok_addReturn(contents) + " }\n" +

      "  public S toString() { ret " + quote(trim(join(contents))) + "; }}");

      reTok(tok, i, j);

    }

    while ((i = jfind(tok, "voidfunc(")) >= 0) {

      int argsFrom = i+4, argsTo = findCodeTokens(tok, i, false, ")");

      int idx = findCodeTokens(tok, argsTo, false, "{");

      int j = findEndOfBracketPart(tok, idx+2);

      L<S> contents = subList(tok, idx+1, j-1);

      replaceTokens(tok, i, j, "new O { void get(" + join(subList(tok, argsFrom, argsTo-1)) + ") { " + join(contents) + " }\n" +

      "  public S toString() { ret " + quote(trim(join(contents))) + "; }}");

      reTok(tok, i, j);

    }

    while ((i = jfind(tok, "func {")) >= 0) {

      int idx = findCodeTokens(tok, i, false, "{");

      int j = findEndOfBracketPart(tok, idx+2);

      L<S> contents = subList(tok, idx+1, j-1);

      replaceTokens(tok, i, j, "new O { O get() { " + tok_addReturn(contents) + " }\n" +

      "  public S toString() { ret " + quote(trim(join(contents))) + "; }}");

      reTok(tok, i, j);

    }

    /*tok = replaceKeywordBlock(tok, "func",

      "new O { O get() { ret",

      ";}}");*/

    // * constructors

    if (hasCodeTokens(tok, "\\*", "("))

      tok = expandStarConstructors(tok);

    // Do this BEFORE awt replacement! ("p-awt" contains "awt" token)

    tok = replaceKeywordBlock(tok, "p-substance", "p-awt { substance();", "}");

    tok = replaceKeywordBlock(tok, "p-awt", "p { awt {", "}}");

    tok = replaceKeywordBlock(tok,

      "awt",

      "swingLater(r {",

      "});");

    // crazy stuff

    jreplace (tok, "for <id> over <id>:", "for (int $2 = 0; $2 < l($4); $2++)");

    jreplace (tok, "for <id> to <id>:", "for (int $2 = 0; $2 < $4; $2++)");

    // STANDARD CLASSES & INTERFACES

    for (S line : toLinesFullTrim(standardClasses)) {

      int idx = line.indexOf('/');

      addLibraryClasses(tok, line.substring(0, idx), line.substring(idx+1));

    }

    addLibraryClasses(tok, 

      "Quine", "#1003601",

      "Var", "#1003550",

      "TableFinder", "#1000850",

      "Snippet", "#1003519",

      "GermanDateAdapter", "#1003510",

      "BF", "#1003489",

      "BotChat", "#1003499",

      "StrictTreeMap", "#1003482",

      "E", "#1003474",

      "Dialog", "#1003356",

      "OccTree2", "#1003407",

      "OccTree", "#1003331",

      "EGDiff", "#1000882",

      "BlockDiffer", "#1000883",

      "Q", "#1000934",

      "F", "#1003033",

      "LoadEnv", "#1002955",

      "CIString", "#1002873",

      "PersistentCIMap", "#1002872",

      "EvalTransform", "#1002867",

      "Prolog", "#1002884", // Yay!

      "IEngine", "#1002809",

      "Native", "#1002774",

      "ProgramScan", "#1000804",

      "SNLMatches", "#1002748",

      "SNLInfo", "#1002744",

      "Lisp", "#1002696",

      "Explain", "#1002529",

      "NanoHTTPD", "#1001651",

      "Source", "#1002461",

      "ShowBigText", "#1001425",

      "DialogIO", "#1001065",

      "Matches", "#1001496",

      "MultiMap", "#1001296",

      "MultiSet", "#1000988",

      "MultiHashSet", "#1003597",

      "PersistentLog", "#1001972",

      "PersistentMap", "#1002063",

      "PersistentTreeMap", "#1003529",

      "Rat", "#1002052",

      "DiskTextMap", "#1002087",

      "ImageSurface", "#1002470",

      "RGB", "#1002470",

      "RGBImage", "#1002470");

    // the infamous missing functions (usually caused by class Matches)

    if (!hasCodeTokens(tok, "String", "unquote") && containsToken(tok, "unquote")) {

      print("Adding unquote");

      tok = includeInMain(tok, "#1001735");

    }

    if (!hasCodeTokens(tok, "String", "formatSnippetID") && containsToken(tok, "formatSnippetID")) {

      print("Adding formatSnippetID");

      tok = includeInMain(tok, "#1000709");

    }

    tok = expandShortTypes(tok);

    if (containsToken(tok, "cast")) {

      S s = join(tok);

      s = s.replaceAll("(\\w+<[\\w\\s,\\[\\]]+>|\\w+|\\w+\\[\\]|\\w+\\[\\]\\[\\])\\s+(\\w+)\\s*=\\s*cast(\\W[^;]*);", "$1 $2 = ($1) ($3);");

      tok = jtok(s);

    }

    for (S keyword : ll("runnable", "r"))

      tok = replaceKeywordBlock(tok,

        keyword,

        "new Runnable() { public void run() { try {",

        "} catch (Exception __e) { throw __e instanceof RuntimeException ? (RuntimeException) __e : new RuntimeException(__e); }}}");

    tok = replaceKeywordBlock(tok,

      "expectException",

      "{ bool __ok = false; try {",

      "} catch { __ok = true; } assertTrue(\"expected exception\", __ok); }");

    while ((i = tok.indexOf("tex")) >= 0) {

      tok.set(i, "throws Exception");

      tok = jtok(tok);

    }

    while ((i = findCodeTokens(tok, "while", "true")) >= 0) {

      tok.set(i+2, "(true)");

      tok = jtok(tok);

    }

    // "null;" instead of "return null;"

    jreplace(tok, "{ null;", "$1 return null;");

    jreplace(tok, "} null;", "$1 return null;");

    jreplace(tok, "; null;", "$1 return null;");

    jreplace(tok, ") null;", "$1 return null;");

    // "false;" instead of "return false;"

    jreplace(tok, "{ false;", "{ return false;");

    jreplace(tok, "} false;", "} return false;");

    jreplace(tok, "; false;", "; return false;");

    jreplace(tok, ") false;", ") return false;");

    // "true;" instead of "return true;"

    jreplace(tok, "} true;", "} return true;");

    jreplace(tok, "; true;", "; return true;");

    jreplace(tok, ") true;", ") return true;");

    // "myFunction;" instead of "myFunction();" - quite rough

    O cond = new O() {

      bool check(L<S> tok, int i) {

        S word = tok.get(i+3);

        //print("single word: " + word);

        ret !litlist("break", "continue", "return").contains(word);

      }

    };

    for (S pre : litlist("}", ";"))

      jreplace(tok, pre + " <id>;", "$1 $2();", cond);

    // shorter match syntax for answer methods

    jreplace(tok, "if <quoted>", "if (match($2, s, m))");

    jreplace(tok, "if match <quoted>", "if (match($3, s, m))");

    // extra commas ("litlist(1, 2,)")

    jreplace(tok, ",)", ")");

    // additional translations (if necessary)

    if (hasCodeTokens(tok, "pcall", "{"))

      tok = replaceKeywordBlock(tok,

        "pcall",

        "try {",

        "} catch (Throwable __e) { printStackTrace(__e); }");

    tok = dialogHandler(tok);

    tok = replaceKeywordBlock(tok, "exceptionToUser",

      "try {",

      "} catch (Throwable __e) { ret exceptionToUser(__e); }"); 

    if (hasCodeTokens(tok, "twice", "{"))

      tok = replaceKeywordBlock(tok, "twice",

        "for (int __twice = 0; __twice < 2; __twice++) {",

        "}"); 

    while ((i = findCodeTokens(tok, "repeat", "*", "{")) >= 0) {

      S v = makeVar("repeat");

      tok.set(i, "for (int " + v + " = 0; " + v + " < " + tok.get(i+2) + "; " + v + "++)");

      tok.set(i+2, "");

      tok = jtok(tok);

    }

    tok = replaceKeywordBlockDyn(tok,

      "time",

      new O() { S[] get() {

        S var = makeVar("startTime");

        ret new S[] {

          "{ long " + var + " = now(); try { ",

          "} finally { " + var + " = now()-" + var + "; saveTiming(" + var + "); } }"};

      }});

    if (hasCodeTokens(tok, "assertFail", "{")) {

      S var = makeVar("oops");

      tok = replaceKeywordBlock(tok,

        "assertFail",

        "boolean " + var + " = false; try {",

        "\n" + var + " = true; } catch (Exception e) { /* ok */ } assertFalse(" + var + ");"); 

    }

    tok = replaceKeywordBlock(tok,

      "yo",

      "try {",

      "} catch (Exception " + makeVar("e") + ") { ret false; }");

    tok = replaceKeywordBlock(tok,

      "awtIfNecessary",

      "swingNowOrLater(r {",

      "});");

    if (hasCodeTokens(tok, "ctex"))

      tok = ctex(tok);

    tok = replaceKeywordBlock(tok,

      "actionListener",

      "new java.awt.event.ActionListener() { " +

        "public void actionPerformed(java.awt.event.ActionEvent _evt) {",

      "}}");

    namedThreads(tok);

    threads(tok);

    // try answer

    while ((i = findCodeTokens(tok, "try", "answer")) >= 0) {

      int j = findEndOfStatement(tok, i);

      S v = makeVar("a");

      tok.set(i, "{ S " + v);

      tok.set(i+2, "=");

      tok.set(j-1, "; if (!empty(" + v + ")) ret " + v + "; }");

      tok = jtok(tok);

    }

    tok = jtok(moveImportsUp(join(tok)));

    ret tok;

  }

  static L<S> multilineStrings(L<S> tok) {

    for (int i = 1; i < tok.size(); i += 2) {

      S t = tok.get(i);

      if (t.startsWith("[") && isQuoted (t))

        tok.set(i, quote(unquote(t)));  

    }

    ret tok;

  }

  static L<S> quickmain(L<S> tok) {

    int i = findCodeTokens(tok, "main", "{");

    if (i < 0) i = findCodeTokens(tok, "m", "{");

    if (i >= 0 && !(i-2 > 0 && tok.get(i-2).equals("class")))

      tok.set(i, "public class main");

    i = findCodeTokens(tok, "psvm", "{");

    if (i < 0) i = findCodeTokens(tok, "p", "{");

    if (i >= 0)

      tok.set(i, "public static void main(String[] args) throws Exception");

    ret jtok(tok);

  }

  static S makeVar(S name) {

    ret "_" + name + "_" + varCount++;

  }

  static S makeVar() { ret makeVar(""); }

  /*static L<S> standardFunctions(L<S> tok) {

    ret rtq(tok, "#1002474");

  }*/

  static L<S> rtq(L<S> tok, S id) {

    ret runTranslatorQuick(tok, id);

  }

  static L<S> expandShortTypes(L<S> tok) {

    // replace <int> with <Integer>

    for (int i = 1; i+4 < tok.size(); i += 2)

      if (tok.get(i).equals("<")

        && litlist(">", ",").contains(tok.get(i+4))) {

        String type = tok.get(i+2);

        if (type.equals("int")) type = "Integer";

        else if (type.equals("long")) type = "Long";

        tok.set(i+2, type);

      }

    // O = Object, S = String, ret = return

    for (int i = 1; i < tok.size(); i += 2) {

      String t = tok.get(i);

      if (t.equals("O")) t = "Object";

      if (t.equals("S")) t = "String";

      else if (t.equals("L")) t = "List";

      else if (t.equals("F")) t = "Function";

      else if (t.equals("ret")) t = "return";

      else if (t.equals("bool") && i+2 < tok.size() && neq(tok.get(i+2), "(")) t = "boolean"; // bool -> boolean if it's not a function name

      tok.set(i, t);

    }

    ret tok;

  }

  static L<S> autoImports(L<S> tok) {

    S s = join(tok);

    List<String> imports = findImports(s);

    new StringBuilder buf;

    for (String c : standardImportClasses)

      if (!(imports.contains(c)))

        buf.append("import " + c + ";\n");

    if (buf.length() == 0) ret tok;

    ret jtok(buf+s);

  }

  static String[] standardImportClasses = {

    "java.util.*",

    "java.util.zip.*",

    "java.util.List",

    "java.util.regex.*",

    "java.util.concurrent.*",

    "java.util.concurrent.atomic.*",

    "javax.swing.*",

    "javax.swing.event.*",

    "javax.swing.text.*",

    "javax.swing.table.*",

    "java.io.*",

    "java.net.*",

    "java.lang.reflect.*",

    "java.lang.ref.*",

    "java.lang.management.*",

    "java.security.*",

    "java.security.spec.*",

    "java.awt.*",

    "java.awt.event.*",

    "java.awt.image.*",

    "javax.imageio.*",

    "java.math.*"

  };

  static L<S> expandStarConstructors(L<S> tok) {

    mainLoop: for (int i = 3; i+6 < tok.size(); i += 2) {

      String t = tok.get(i), l = tok.get(i-2);

      if (!t.equals("*"))

        continue;

      if (!tok.get(i+2).equals("("))

        continue;

      if (!eqOneOf(l, "}", "public", "private", "protected", ";", "{")) // is this correct...??

        continue;

      // ok, it seems like a constructor declaration.

      // Now find class name by going backwards.

      int j = i, level = 1;

      while (j > 0 && level > 0) {

        t = tok.get(j);

        if (t.equals("}")) ++level;

        if (t.equals("{")) --level;

        j -= 2;

      }

      while (j > 0) {

        t = tok.get(j);

        if (t.equals("class")) {

          String className = tok.get(j+2);

          tok.set(i, className); // exchange constructor name!

          // now for the parameters.

          // Syntax: *(Learner *learner) {

          // We will surely add type inference here in time... :)

          j = i+2;

          while (!tok.get(j).equals("{"))

            j += 2;

          int block = j+1;

          for (int k = i+2; k < block-1; k += 2)

            if (tok.get(k).equals("*")) {

              tok.remove(k);

              tok.remove(k);

              block -= 2;

              String name = tok.get(k);

              tok.addAll(block, Arrays.asList(new String[] {

                "\n  ", "this", "", ".", "", name, " ", "=", " ", name, "", ";" }));

            }

          continue mainLoop;

        }

        j -= 2;

      }

    }

    ret tok;

  }

  static L<S> processIncludes(L<S> tok) {

    int safety = 0;

    while (hasCodeTokens(tok, "!", "include") && ++safety < 100)

      tok = processIncludesSingle(tok);

    ret tok;

  }

  static L<S> processIncludesSingle(L<S> tok) {

    S s = join(tok);

    Matcher m = Pattern.compile("\n\\s*!include (#\\d+)").matcher(s);

    StringBuffer buf = new StringBuffer();

    while (m.find()) {

      String includedSrc = loadSnippet(m.group(1));

      m.appendReplacement(buf, m.quoteReplacement("\n" + includedSrc));

    }

    m.appendTail(buf);

    ret jtok(str(buf));

  }

  static L<S> ctex(L<S> tok) {

    S s = join(tok);

    Pattern regex = Pattern.compile("\\s+(no\\s+exceptions|ctex|null on exception)\\s*\\{");

    for (int i = 0; i < 100; i++) {

      Matcher matcher = regex.matcher(s);

      if (!matcher.find())

        break;

      String kind = matcher.group(1);

      System.out.println("Iteration " + (i+1));

      int start = matcher.start(), end = matcher.end();

      int endOfBlock = ctex_findEndOfBlock(s, end);

      String catchBlock, catchWhat;

      if (kind.startsWith("null")) {

        catchBlock = "return null;";

        catchWhat = "Throwable";

      } else {

        catchBlock = "throw __e instanceof RuntimeException ? (RuntimeException) __e : new RuntimeException(__e);";

        catchWhat = "Throwable";

      }

      String tryBlock = " { try {\n " +

        s.substring(end, endOfBlock) + "\n} catch (" + catchWhat + " __e) { " + catchBlock + " }";

      s = s.substring(0, start) + tryBlock + s.substring(endOfBlock);

    }

    ret jtok(s);

  }

  // start is the index AFTER the opening bracket

  // returns index OF closing bracket

  static int ctex_findEndOfBlock(String s, int start) {

    int level = 1;

    for (int i = start; i < s.length(); i++) {

      if (s.charAt(i) == '{') ++level;

      else if (s.charAt(i) == '}') --level;

      if (level == 0)

        return i;

    }

    return s.length();

  }

  static L<S> dialogHandler(L<S> tok) {

    ret replaceKeywordBlock(tok,

      "dialogHandler",

      "new DialogHandler() {\n" +

        "public void run(final DialogIO io) {",

      "}}");

  }

  /*static S defaultTrans(S s) {

    findTranslators...

  }*/

  static S quicknew(S s) {

    // TODO: switch to tokenizing!

    s = s.replaceAll("new\\s+(\\w+)\\s+(\\w+);", "$1 $2 = new $1();");

    s = s.replaceAll("new\\s+L<([\\w\\[\\]<>,\\s]+)>\\s+(\\w+);", "List<$1> $2 = new ArrayList<$1>();");

    s = s.replaceAll("new\\s+List<([\\w\\[\\]<>,\\s]+)>\\s+(\\w+);", "List<$1> $2 = new ArrayList<$1>();");

    s = s.replaceAll("new\\s+\\(Hash\\)Set<(\\w+)>\\s+(\\w+);", "Set<$1> $2 = new HashSet<$1>();");

    s = s.replaceAll("new\\s+\\(Tree\\)Set<(\\w+)>\\s+(\\w+);", "Set<$1> $2 = new TreeSet<$1>();");

    s = s.replaceAll("new\\s+Set<(\\w+)>\\s+(\\w+);", "Set<$1> $2 = new TreeSet<$1>();"); // TreeSet now default - pay attention to explicitly say HashSet if you need it.

    s = s.replaceAll("new\\s+\\(Hash\\)Map<([\\w\\s,]+)>\\s+(\\w+);", "Map<$1> $2 = new HashMap<$1>();");

    s = s.replaceAll("new\\s+\\(Tree\\)Map<([\\w\\s,]+)>\\s+(\\w+);", "Map<$1> $2 = new TreeMap<$1>();");

    // TreeMap when string as key

    s = s.replaceAll("new\\s+Map<(S,[\\w\\s,]+)>\\s+(\\w+);", "Map<$1> $2 = new TreeMap<$1>();");

    s = s.replaceAll("new\\s+Map<(String,[\\w\\s,]+)>\\s+(\\w+);", "Map<$1> $2 = new TreeMap<$1>();");

    // HashMap is default for everything else

    s = s.replaceAll("new\\s+Map<([\\w\\s,]+)>\\s+(\\w+);", "Map<$1> $2 = new HashMap<$1>();");

    s = s.replaceAll("new\\s+(\\w+<[\\w\\s,]+>)\\s+(\\w+);", "$1 $2 = new $1();");

    // Constructor calls without parantheses

    // So you can say something like: predictors.add(new P1);

    s = s.replaceAll("new\\s+(\\w+)\\s*([,\\);:])", "new $1()$2");

    // replace "new List" with "new ArrayList"

    s = s.replaceAll("new\\s+List\\s*\\(", "new ArrayList(");

    // for args

    s = s.replace("for args {", "for (int i = 0; i < args.length; i++) { final String arg = args[i];");

    ret s;

  }

  static L<S> extendClasses(L<S> tok) {

    int i;

    while ((i = jfind(tok, "extend <id> {")) >= 0) {

      S className = tok.get(i+2);

      int idx = findCodeTokens(tok, i, false, "{");

      int j = findEndOfBracketPart(tok, idx+2);

      S content = join(subList(tok, idx+1, j-1));

      L<S> c = findInnerClassOfMain(tok, className);

      clearTokens(tok.subList(i, j+1));

      if (c == null) {

        print("Warning: Can't extend class " + className + ", not found");

        continue;

      }

      int endOfClass = magicIndexOfSubList(tok, c) + l(c)-1;

      while (neq(tok.get(endOfClass), "}")) --endOfClass;

      tok.set(endOfClass, content + "\n" + tok.get(endOfClass));

      reTok(tok); // changed in 2 places, let's retok it all

    }

    ret tok;

  }

  static void listComprehensions(L<S> tok) {

    int i;

    while ((i = jfind(tok, "[<id> <id> in")) >= 0) {

      Map<Integer, Integer> bracketMap = getBracketMap(tok);

      S type = tok.get(i+2), id = tok.get(i+4);

      int j = scanOverExpression(tok, bracketMap, i+8, "|");

      S exp = join(tok.subList(i+8, j));

      j += 2;

      int k = scanOverExpression(tok, bracketMap, j, "]");

      S where = join(tok.subList(j, k));

      ++k;

      S code = "filter(" + exp + ", func(" + type + " " + id + ") { " + where + " })";

      replaceTokens(tok, i, k, code);

      reTok(tok, i, k);

    }

  }

  // lib 123 => !123

  static void libs(L<S> tok) {

    new Set<S> libs;

    int i;

    while ((i = jfind(tok, "lib <int>")) >= 0) {

      S id = tok.get(i+2);

      print("lib " + id);

      if (!libs.contains(id)) {

        libs.add(id);

        tok.set(i, "!");

        tok.set(i+1, "");

      } else {

        print("...ignoring (duplicate)");

        clearAllTokens(tok, i, i+3);

        reTok(tok, i, i+3);

      }

    }

  }

  // sourceCodeLine() => 1234

  static void sourceCodeLine(L<S> tok) {

    int i ;

    while ((i = jfind(tok, "sourceCodeLine()")) >= 0) {

      replaceTokens(tok, i, i+5, str(countChar(join(subList(tok, 0, i)), '\n')+1));

      reTok(tok, i, i+5);

    }

  }

  // done before any other processing

  static void earlyStuff(L<S> tok) {

    int i;

    // Note: this makes the word "quine" a special operator

    // (unusable as a function name)

    while ((i = jfind(tok, "quine(")) >= 0) {

      int idx = findCodeTokens(tok, i, false, "(");

      int j = findEndOfBracketPart(tok, idx+2);

      tok.set(i, "new Quine");

      tok.set(idx, "(" + quote(join(subList(tok, idx+1, j-1))) + ", ");

      reTok(tok, i, idx+1);

    }

  }

  static void throwFail(L<S> tok) {

    bool anyChange = false;

    for (int i = 1; i+2 < l(tok); i += 2)

      if (eq(get(tok, i+2), "fail") && !eqOneOf(get(tok, i), "throw", "RuntimeException", "return")) {

        tok.set(i+2, "throw fail");

        anyChange = true;

      }

    if (anyChange)

      reTok(tok);

  }

  static void namedThreads(L<S> tok) {

    for (int i = 0; i < 100; i++) {

      int idx = findCodeTokens(tok, "thread", "*", "{");

      if (idx < 0)

        break;

      int j = findEndOfBracketPart(tok, idx+4);

      S tName = tok.get(idx+2);

      S var = "_t_" + i;

      S pre = "{ Thread " + var + " = new Thread(" + tName + ") {\n" +

        "public void run() {\n" +

        "try";

      S post = "} catch (Exception _e) {\n" +

          "  throw _e instanceof RuntimeException ? (RuntimeException) _e : new RuntimeException(_e); } }\n};\n" +

        var + ".start(); }";

      tok.set(idx, pre);

      tok.set(idx+2, "");

      tok.set(j-1, post);

      reTok(tok, idx, j);

    }

  }

  static void threads(L<S> tok) {

    for (bool daemon : litlist(false, true))

      for (int i = 0; i < 100; i++) {

        int idx = findCodeTokens(tok, daemon ? "daemon" : "thread", "{");

        if (idx < 0)

          break;

        int j = findEndOfBracketPart(tok, idx+2);

        S var = "_t_" + i;

        S pre = "{ Thread " + var + " = new Thread() {\n" +

          "public void run() {\n" +

          "try";

        S post = "} catch (Exception _e) {\n" +

            "  throw _e instanceof RuntimeException ? (RuntimeException) _e : new RuntimeException(_e); } }\n};\n" +

          (daemon ? var + ".setDaemon(true);\n" : "") +

          var + ".start(); }";

        tok.set(idx, pre);

        tok.set(j-1, post);

        reTok(tok, idx, j);

      }

  }

  static Map<S, S> sf;

  static L<S> standardFunctions(L<S> tok) {

    if (sf == null) {

      L<S> standardFunctions = (L) loadVariableDefinition(

        loadSnippet("#761"), "standardFunctions");

      sf = new HashMap();

      for (String x : standardFunctions) {

        String[] f = x.split("/");

        sf.put(f[1], f[0]);

      }

    }

    for (int i = 0; ; i++) {

      Set<String> defd = new HashSet(findFunctions(tok));

      // changes tok

      Set<String> invocations = findFunctionInvocations(tok, sf);

      //print("Functions invoked: " + structure(invocations));

      List<String> needed = diff(invocations, defd);

      if (needed.isEmpty())

        break;

      print("Adding functions: " + join(" " , needed));

      new L<S> added;

      new StringBuilder buf;

      new L<S> preload;

      for (S x : needed)

        if (sf.containsKey(x))

          preload.add(sf.get(x));

      cachePreload(preload);

      for (String x : needed) {

        if (defd.contains(x)) continue;

        String id = sf.get(x);

        if (id == null) {

          print("Standard function " + x + " not found.");

          continue;

        }

        //print("Adding function: " + x + " (" + id + ")");

        S function = cacheGet(id);

        if (("\n" + function).contains("\n!")) print("Warning: " + id + " contains translators.");

        buf.append(function).append("\n");

        added.add(x);

        defd.addAll(findFunctionDefinitions(function));

      }

      tok = includeInMainLoaded(tok, str(buf));

      defd = new HashSet(findFunctions(tok));

      //print("Functions added: " + structure(added));

      for (String x : needed)

        if (!defd.contains(x)) {

          print(join(tok));

          fail("Function not defined properly: " + x);

        }

      //print("Iteration " + (i+2));

      if (i >= 1000) fail("Too many iterations");

    }

    ret tok;

  }

  static L<S> findFunctions(L<S> tok) {

    //ret findFunctionDefinitions(join(findMainClass(tok)));

    ret findFunctionDefs(findMainClass(tok));

  }

  static S cacheGet(S snippetID) {

    snippetID = formatSnippetID(snippetID);

    S text = snippetCache.get(snippetID);

    if (text == null)

      snippetCache.put(snippetID, text = loadSnippet(snippetID));

    ret text;

  }

  static void cachePreload(L<S> ids) {

    new L<S> needed;

    for (S id : ids)

      if (!snippetCache.containsKey(formatSnippetID(id)))

        needed.add(formatSnippetID(id));

    if (l(needed) > 1) {

      L<S> texts = loadSnippets(needed);

      for (int i = 0; i < l(needed); i++)

        if (texts.get(i) != null)

          snippetCache.put(needed.get(i), texts.get(i));

    }

  }

  static L<S> jtok(L<S> tok) {

    ret jtok(join(tok));

  }

  static L<S> jtok(S s) {

    L<S> l = javaTok(s);

    ret useIndexedList ? new IndexedList2(l) : l;

  }

  /*

  static L<S> loadClass(L<S> tok, S className, S snippetID) {

    if (containsToken(tok, className)

      && !hasCodeTokens(tok, "class", className) && !hasCodeTokens(tok, "interface", className)

      && !tok_importsClassNamed(tok, className)) {

      snippetID = formatSnippetID(snippetID);

      S text = cacheGet(snippetID);

      tok = includeInMainLoaded(tok, text);

    }

    ret tok;

  }

  */

  // works on Java level (no "sclass" etc)

  static void addLibraryClasses(L<S> tok, S... data) {

    new HashSet<S> have;

    for (L<S> c : innerClassesOfMain(tok))

      have.add(getClassDeclarationName(c));

    have.addAll(tok_importedClassNames(tok));

    L<S> idx = IndexedList2.ensureIndexed(tok);

    for (int i = 0; i+1 < l(data); i++) {

      S className = data[i], snippetID = data[i+1];

      if (idx.contains(className)

        && !have.contains(className)) {

        snippetID = formatSnippetID(snippetID);

        S text = cacheGet(snippetID);

        includeInMainLoaded(tok, text);

      }

    }

  }

}