!747

m {
  !include #1001496 // Matches
  !include #1000882 // EGDiff
  !include #1000883 // BlockDiffer
  !include #1001065 // DialoGIO
  !include #1004045 // IndexedList2
  //include #1001296 // MultiMap
  !include #1000988 // MultiSet

  !include #1002662 // isTrue
  
  !include #1003674 // Standard Classes List

  static int varCount;
  static new Map<S, S> snippetCache;
  static bool useIndexedList = false, opt_javaTok = true;

  p {
    if (useIndexedList) findCodeTokens_debug = true;
    javaTok_opt = opt_javaTok;
    findCodeTokens_indexed = findCodeTokens_unindexed = 0;
    javaTok_n = javaTok_elements = 0;
    
    print("759 STARTING " + identityHashCode(main.class));
    varCount = 0;
    L<S> tok = jtok(loadMainJava());
    
    L ts = findTranslators(toLines(join(tok)));
    print("Translators in source at start: " + structure(ts));
    
    // "duplicate" statement
    
    L<S> lines = toLines(join(tok));
    call(getJavaX(), "findTranslators", lines);
    new Matches m;
    if (match("duplicate *", fromLines(lines), m)) {
      // actual copying - unused
      // tok = jtok(loadSnippet(m.get(0)));
      
      // reference by include()
      tok = jtok("m { p { callMain(include(" + quote(m.get(0)) + ")); } }");
    }
    
    // add m { }
    
    if (!hasCodeTokens(tok, "m", "{") && !hasCodeTokens(tok, "main", "{") && !hasCodeTokens(tok, "class", "main"))
      tok = jtok(moveImportsUp("m {\n" + join(tok) + "\n}"));
    
    // standard translate
    
    ts = findTranslators(toLines(join(tok)));
    print("Translators in source: " + structure(ts));
    tok = jtok(defaultTranslate(join(tok)));
    
    print("end of default translate");
    //print(join(tok));
    
    tok = processIncludes(tok); // before standard functions
    earlyStuff(tok);
    
    int safety = 0;
    boolean same;
    do {
      S before = join(tok);
      tok = standardFunctions(tok);
      tok = stdstuff(tok); // standard functions and all the keywords
      S diff;
      long startTime = now();
      //diff = unidiff(before, join(tok));
      //print("unidiff: " + (now()-startTime) + " ms");
      //same = eq(diff, "");
      same = eq(before, join(tok)); // << could be sped up
      if (!same) {
        print("Not same " + safety + ".");
        //print(indent(2, diff));
      }
      if (safety++ >= 10) fail("safety 10 error!");
    } while (!same);
    
    // POST-PROCESSING after stdstuff loop
    
    tok = jtok(quicknew(join(tok)));
    tok = extendClasses(tok);
    libs(tok);
    sourceCodeLine(tok);
    throwFail(tok);
    tok = autoImports(tok); // faster to do it at the end

    if (useIndexedList)
      print("Indexed/unindexed lookups: " + findCodeTokens_indexed + "/" + findCodeTokens_unindexed + ", lists made: " + IndexedList2.instances);
    print("javaToks: " + javaTok_n + "/" + javaTok_elements);
    
    saveMainJava(tok);
  }

  static L<S> stdstuff(L<S> tok) {
    //if (++level >= 10) fail("woot? 10");
    
    print("stdstuff!");
    int i;
    
    if (jfind(tok, "!<int>") >= 0) {
      L<S> lines = toLines(join(tok));
      L ts = findTranslators(lines);
      tok = jtok(fromLines(lines));
      print("DROPPING TRANSLATORS: " + structure(ts));
    }

    tok = quickmain(tok);
    tok = processIncludes(tok);
    earlyStuff(tok);
    tok = multilineStrings(tok);
    listComprehensions(tok);
    
    tok = replaceKeywordBlock(tok, "answer",
      "static S answer(S s) {\nnew Matches m;\n",
      "\nret null;\n}");
      
    tok = replaceKeywordBlock(tok, "loading",
      "{ JWindow _loading_window = showLoadingAnimation(); try {",
      "} finally { disposeWindow(_loading_window); }}");

    tok = replaceKeywordBlock(tok, "html",
      "static O html(S uri, fMap<S, S> params) {\n", "}");
    
    // "static sync" => static synchronized
    jreplace(tok, "static sync", "static synchronized");

    // "sclass" => static class
    jreplace(tok, "sclass", "static class");

    // "asclass" => abstract static class
    jreplace(tok, "asclass", "abstract static class");

    // "sinterface" => static interface
    jreplace(tok, "sinterface", "static interface");

    // "ssynchronized" => static synchronized
    jreplace(tok, "ssynchronized", "static synchronized");

    jreplace(tok, "ssvoid", "static synchronized void");
    jreplace(tok, "svoid", "static void");
    jreplace(tok, "sbool", "static bool");
    jreplace(tok, "sint", "static int");
    jreplace(tok, "snew", "static new");
    jreplace(tok, "sv <id>", "static void $2");
    jreplace(tok, "pvoid", "public void");

    // "sS" => static S
    jreplace(tok, "sS", "static S");

    // "sO" => static O
    jreplace(tok, "sO", "static O");

    // "sL" => static L
    jreplace(tok, "sL", "static L");

    // "toString {" => "public S toString() {"
    jreplace(tok, "toString {", "public S toString() {");
    
    jreplace(tok, "Int", "Integer");
    jreplace(tok, "Bool", "Boolean");

    // "catch {" => "catch (Throwable _e) {"
    jreplace(tok, "catch {", "catch (Throwable _e) {");

    // "catch X e {" => "catch (X e) {"
    jreplace(tok, "catch <id> <id> {", "catch ($2 $3) {");

    // "catch e {" => "catch (Throwable e) {" (if e is lowercase)
    jreplace(tok, "catch <id> {", "catch (Throwable $2) {", new O() {
      bool check(L<S> tok, int i) {
        S word = tok.get(i+3);
        ret startsWithLowerCase(word);
      }
    });
    
    // some crazy fancy syntax
    jreplace(tok, "set <id>;", "$2 = true;");
    
    // more shortening

    jreplace(tok, "fS", "final S");
    jreplace(tok, "fO", "final O");
    jreplace(tok, "fL", "final L");
    jreplace(tok, "fMap", "final Map");
    jreplace(tok, "fRunnable", "final Runnable");
    jreplace(tok, "f int", "final int");
    
    jreplace(tok, "new <id> {", "new $2() {");
    
    // "continue unless"
    
    while ((i = jfind(tok, "continue unless")) >= 0) {
      int j = scanOverExpression(tok, getBracketMap(tok), i+4, ";");
      replaceTokens(tok, i, i+4, "{ if (!(");
      tok.set(j, ")) continue; }");
      reTok(tok, i, j+1);
    }
    
    // "continue if"
    
    while ((i = jfind(tok, "continue if")) >= 0) {
      int j = scanOverExpression(tok, getBracketMap(tok), i+4, ";");
      replaceTokens(tok, i, i+4, "{ if (");
      tok.set(j, ") continue; }");
      reTok(tok, i, j+1);
    }
    
    // shortened method declarations - hopefully that works
    
    jreplace(tok, "void <id> {", "$1 $2() {");
    jreplace(tok, "String <id> {", "$1 $2() {");
    jreplace(tok, "Object <id> {", "$1 $2() {");
    jreplace(tok, "List <id> {", "$1 $2() {");
    
    // "is a X" => instanceof X
    
    jreplace(tok, "is a <id>", "instanceof $3");
    
    // !a instanceof X => !(a instanceof X)
    
    jreplace(tok, "!<id> instanceof <id>", "!($2 instanceof $4)");
    
    // func keyword for lambdas - now automatically quines toString()
    
    while ((i = jfind(tok, "func(")) >= 0) {
      int argsFrom = i+4, argsTo = findCodeTokens(tok, i, false, ")");
      int idx = findCodeTokens(tok, argsTo, false, "{");
      int j = findEndOfBracketPart(tok, idx+2);
      L<S> contents = subList(tok, idx+1, j-1);
      replaceTokens(tok, i, j, "new O { O get(" + join(subList(tok, argsFrom, argsTo-1)) + ") { " + tok_addReturn(contents) + " }\n" +
      "  public S toString() { ret " + quote(trim(join(contents))) + "; }}");
      reTok(tok, i, j);
    }
    
    while ((i = jfind(tok, "voidfunc(")) >= 0) {
      int argsFrom = i+4, argsTo = findCodeTokens(tok, i, false, ")");
      int idx = findCodeTokens(tok, argsTo, false, "{");
      int j = findEndOfBracketPart(tok, idx+2);
      L<S> contents = subList(tok, idx+1, j-1);
      replaceTokens(tok, i, j, "new O { void get(" + join(subList(tok, argsFrom, argsTo-1)) + ") { " + join(contents) + " }\n" +
      "  public S toString() { ret " + quote(trim(join(contents))) + "; }}");
      reTok(tok, i, j);
    }
    
    while ((i = jfind(tok, "func {")) >= 0) {
      int idx = findCodeTokens(tok, i, false, "{");
      int j = findEndOfBracketPart(tok, idx+2);
      L<S> contents = subList(tok, idx+1, j-1);
      replaceTokens(tok, i, j, "new O { O get() { " + tok_addReturn(contents) + " }\n" +
      "  public S toString() { ret " + quote(trim(join(contents))) + "; }}");
      reTok(tok, i, j);
    }
    /*tok = replaceKeywordBlock(tok, "func",
      "new O { O get() { ret",
      ";}}");*/
    
    // * constructors
    if (hasCodeTokens(tok, "\\*", "("))
      tok = expandStarConstructors(tok);
      
    // Do this BEFORE awt replacement! ("p-awt" contains "awt" token)
    tok = replaceKeywordBlock(tok, "p-substance", "p-awt { substance();", "}");
    tok = replaceKeywordBlock(tok, "p-awt", "p { awt {", "}}");
    
    tok = replaceKeywordBlock(tok,
      "awt",
      "swingLater(r {",
      "});");

    // crazy stuff

    jreplace (tok, "for <id> over <id>:", "for (int $2 = 0; $2 < l($4); $2++)");
    jreplace (tok, "for <id> to <id>:", "for (int $2 = 0; $2 < $4; $2++)");
    
    // STANDARD CLASSES & INTERFACES
    
    for (S line : toLinesFullTrim(standardClasses)) {
      int idx = line.indexOf('/');
      addLibraryClasses(tok, line.substring(0, idx), line.substring(idx+1));
    }
    
    addLibraryClasses(tok, 
      "Quine", "#1003601",
      "Var", "#1003550",
      "TableFinder", "#1000850",
      "Snippet", "#1003519",
      "GermanDateAdapter", "#1003510",
      "BF", "#1003489",
      "BotChat", "#1003499",
      "StrictTreeMap", "#1003482",
      "E", "#1003474",
      "Dialog", "#1003356",
      "OccTree2", "#1003407",
      "OccTree", "#1003331",
      "EGDiff", "#1000882",
      "BlockDiffer", "#1000883",
      "Q", "#1000934",
      "F", "#1003033",
      "LoadEnv", "#1002955",
      "CIString", "#1002873",
      "PersistentCIMap", "#1002872",
      "EvalTransform", "#1002867",
      "Prolog", "#1002884", // Yay!
      "IEngine", "#1002809",
      "Native", "#1002774",
      "ProgramScan", "#1000804",
      "SNLMatches", "#1002748",
      "SNLInfo", "#1002744",
      "Lisp", "#1002696",
      "Explain", "#1002529",
      "NanoHTTPD", "#1001651",
      "Source", "#1002461",
      "ShowBigText", "#1001425",
      "DialogIO", "#1001065",
      "Matches", "#1001496",
      "MultiMap", "#1001296",
      "MultiSet", "#1000988",
      "MultiHashSet", "#1003597",
      "PersistentLog", "#1001972",
      "PersistentMap", "#1002063",
      "PersistentTreeMap", "#1003529",
      "Rat", "#1002052",
      "DiskTextMap", "#1002087",
      "ImageSurface", "#1002470",
      "RGB", "#1002470",
      "RGBImage", "#1002470");

    // the infamous missing functions (usually caused by class Matches)
    if (!hasCodeTokens(tok, "String", "unquote") && containsToken(tok, "unquote")) {
      print("Adding unquote");
      tok = includeInMain(tok, "#1001735");
    }
      
    if (!hasCodeTokens(tok, "String", "formatSnippetID") && containsToken(tok, "formatSnippetID")) {
      print("Adding formatSnippetID");
      tok = includeInMain(tok, "#1000709");
    }
    
    tok = expandShortTypes(tok);
      
    if (containsToken(tok, "cast")) {
      S s = join(tok);
      s = s.replaceAll("(\\w+<[\\w\\s,\\[\\]]+>|\\w+|\\w+\\[\\]|\\w+\\[\\]\\[\\])\\s+(\\w+)\\s*=\\s*cast(\\W[^;]*);", "$1 $2 = ($1) ($3);");
      tok = jtok(s);
    }
    
    for (S keyword : ll("runnable", "r"))
      tok = replaceKeywordBlock(tok,
        keyword,
        "new Runnable() { public void run() { try {",
        "} catch (Exception __e) { throw __e instanceof RuntimeException ? (RuntimeException) __e : new RuntimeException(__e); }}}");

    tok = replaceKeywordBlock(tok,
      "expectException",
      "{ bool __ok = false; try {",
      "} catch { __ok = true; } assertTrue(\"expected exception\", __ok); }");
      
    while ((i = tok.indexOf("tex")) >= 0) {
      tok.set(i, "throws Exception");
      tok = jtok(tok);
    }
    
    while ((i = findCodeTokens(tok, "while", "true")) >= 0) {
      tok.set(i+2, "(true)");
      tok = jtok(tok);
    }
    
    // "null;" instead of "return null;"
    jreplace(tok, "{ null;", "$1 return null;");
    jreplace(tok, "} null;", "$1 return null;");
    jreplace(tok, "; null;", "$1 return null;");
    jreplace(tok, ") null;", "$1 return null;");

    // "false;" instead of "return false;"
    jreplace(tok, "{ false;", "{ return false;");
    jreplace(tok, "} false;", "} return false;");
    jreplace(tok, "; false;", "; return false;");
    jreplace(tok, ") false;", ") return false;");

    // "true;" instead of "return true;"
    jreplace(tok, "} true;", "} return true;");
    jreplace(tok, "; true;", "; return true;");
    jreplace(tok, ") true;", ") return true;");

    // "myFunction;" instead of "myFunction();" - quite rough
    O cond = new O() {
      bool check(L<S> tok, int i) {
        S word = tok.get(i+3);
        //print("single word: " + word);
        ret !litlist("break", "continue", "return").contains(word);
      }
    };
    for (S pre : litlist("}", ";"))
      jreplace(tok, pre + " <id>;", "$1 $2();", cond);

    // shorter match syntax for answer methods
    
    jreplace(tok, "if <quoted>", "if (match($2, s, m))");
    jreplace(tok, "if match <quoted>", "if (match($3, s, m))");

    // extra commas ("litlist(1, 2,)")
    
    jreplace(tok, ",)", ")");
    
    // additional translations (if necessary)
    
    if (hasCodeTokens(tok, "pcall", "{"))
      tok = replaceKeywordBlock(tok,
        "pcall",
        "try {",
        "} catch (Throwable __e) { printStackTrace(__e); }");

    tok = dialogHandler(tok);
    
    tok = replaceKeywordBlock(tok, "exceptionToUser",
      "try {",
      "} catch (Throwable __e) { ret exceptionToUser(__e); }"); 

    if (hasCodeTokens(tok, "twice", "{"))
      tok = replaceKeywordBlock(tok, "twice",
        "for (int __twice = 0; __twice < 2; __twice++) {",
        "}"); 

    while ((i = findCodeTokens(tok, "repeat", "*", "{")) >= 0) {
      S v = makeVar("repeat");
      tok.set(i, "for (int " + v + " = 0; " + v + " < " + tok.get(i+2) + "; " + v + "++)");
      tok.set(i+2, "");
      tok = jtok(tok);
    }

    tok = replaceKeywordBlockDyn(tok,
      "time",
      new O() { S[] get() {
        S var = makeVar("startTime");
        ret new S[] {
          "{ long " + var + " = now(); try { ",
          "} finally { " + var + " = now()-" + var + "; saveTiming(" + var + "); } }"};
      }});
    
    if (hasCodeTokens(tok, "assertFail", "{")) {
      S var = makeVar("oops");
      
      tok = replaceKeywordBlock(tok,
        "assertFail",
        "boolean " + var + " = false; try {",
        "\n" + var + " = true; } catch (Exception e) { /* ok */ } assertFalse(" + var + ");"); 
    }
    
    tok = replaceKeywordBlock(tok,
      "yo",
      "try {",
      "} catch (Exception " + makeVar("e") + ") { ret false; }");

    tok = replaceKeywordBlock(tok,
      "awtIfNecessary",
      "swingNowOrLater(r {",
      "});");
      
    if (hasCodeTokens(tok, "ctex"))
      tok = ctex(tok);
      
    tok = replaceKeywordBlock(tok,
      "actionListener",
      "new java.awt.event.ActionListener() { " +
        "public void actionPerformed(java.awt.event.ActionEvent _evt) {",
      "}}");
      
    namedThreads(tok);
    threads(tok);

    // try answer
    while ((i = findCodeTokens(tok, "try", "answer")) >= 0) {
      int j = findEndOfStatement(tok, i);
      S v = makeVar("a");
      tok.set(i, "{ S " + v);
      tok.set(i+2, "=");
      tok.set(j-1, "; if (!empty(" + v + ")) ret " + v + "; }");
      tok = jtok(tok);
    }

    tok = jtok(moveImportsUp(join(tok)));
      
    ret tok;
  }
  
  static L<S> multilineStrings(L<S> tok) {
    for (int i = 1; i < tok.size(); i += 2) {
      S t = tok.get(i);
      if (t.startsWith("[") && isQuoted (t))
        tok.set(i, quote(unquote(t)));  
    }
    ret tok;
  }
  
  static L<S> quickmain(L<S> tok) {
    int i = findCodeTokens(tok, "main", "{");
    if (i < 0) i = findCodeTokens(tok, "m", "{");
    if (i >= 0 && !(i-2 > 0 && tok.get(i-2).equals("class")))
      tok.set(i, "public class main");
      
    i = findCodeTokens(tok, "psvm", "{");
    if (i < 0) i = findCodeTokens(tok, "p", "{");
    if (i >= 0)
      tok.set(i, "public static void main(String[] args) throws Exception");
      
    ret jtok(tok);
  }
  
  static S makeVar(S name) {
    ret "_" + name + "_" + varCount++;
  }
  
  static S makeVar() { ret makeVar(""); }
  
  /*static L<S> standardFunctions(L<S> tok) {
    ret rtq(tok, "#1002474");
  }*/
  
  static L<S> rtq(L<S> tok, S id) {
    ret runTranslatorQuick(tok, id);
  }
  
  static L<S> expandShortTypes(L<S> tok) {
    // replace <int> with <Integer>
    for (int i = 1; i+4 < tok.size(); i += 2)
      if (tok.get(i).equals("<")
        && litlist(">", ",").contains(tok.get(i+4))) {
        String type = tok.get(i+2);
        if (type.equals("int")) type = "Integer";
        else if (type.equals("long")) type = "Long";
        tok.set(i+2, type);
      }
      
    // O = Object, S = String, ret = return
    for (int i = 1; i < tok.size(); i += 2) {
      String t = tok.get(i);
      if (t.equals("O")) t = "Object";
      if (t.equals("S")) t = "String";
      else if (t.equals("L")) t = "List";
      else if (t.equals("F")) t = "Function";
      else if (t.equals("ret")) t = "return";
      else if (t.equals("bool") && i+2 < tok.size() && neq(tok.get(i+2), "(")) t = "boolean"; // bool -> boolean if it's not a function name
      tok.set(i, t);
    }
    
    ret tok;
  }
  
  static L<S> autoImports(L<S> tok) {
    S s = join(tok);
    List<String> imports = findImports(s);
    new StringBuilder buf;
    for (String c : standardImportClasses)
      if (!(imports.contains(c)))
        buf.append("import " + c + ";\n");
    if (buf.length() == 0) ret tok;
    ret jtok(buf+s);
  }
  
  static String[] standardImportClasses = {
    "java.util.*",
    "java.util.zip.*",
    "java.util.List",
    "java.util.regex.*",
    "java.util.concurrent.*",
    "java.util.concurrent.atomic.*",
    "javax.swing.*",
    "javax.swing.event.*",
    "javax.swing.text.*",
    "javax.swing.table.*",
    "java.io.*",
    "java.net.*",
    "java.lang.reflect.*",
    "java.lang.ref.*",
    "java.lang.management.*",
    "java.security.*",
    "java.security.spec.*",
    "java.awt.*",
    "java.awt.event.*",
    "java.awt.image.*",
    "javax.imageio.*",
    "java.math.*"
  };
  
  static L<S> expandStarConstructors(L<S> tok) {
    mainLoop: for (int i = 3; i+6 < tok.size(); i += 2) {
      String t = tok.get(i), l = tok.get(i-2);
      if (!t.equals("*"))
        continue;
      if (!tok.get(i+2).equals("("))
        continue;
      if (!eqOneOf(l, "}", "public", "private", "protected", ";", "{")) // is this correct...??
        continue;
        
      // ok, it seems like a constructor declaration.
      // Now find class name by going backwards.
      
      int j = i, level = 1;
      while (j > 0 && level > 0) {
        t = tok.get(j);
        if (t.equals("}")) ++level;
        if (t.equals("{")) --level;
        j -= 2;
      }
      
      while (j > 0) {
        t = tok.get(j);
        if (t.equals("class")) {
          String className = tok.get(j+2);
          tok.set(i, className); // exchange constructor name!
          
          // now for the parameters.
          // Syntax: *(Learner *learner) {
          // We will surely add type inference here in time... :)
          
          j = i+2;
          while (!tok.get(j).equals("{"))
            j += 2;
          int block = j+1;
          for (int k = i+2; k < block-1; k += 2)
            if (tok.get(k).equals("*")) {
              tok.remove(k);
              tok.remove(k);
              block -= 2;
              String name = tok.get(k);
              tok.addAll(block, Arrays.asList(new String[] {
                "\n  ", "this", "", ".", "", name, " ", "=", " ", name, "", ";" }));
            }
          
          continue mainLoop;
        }
        j -= 2;
      }
    }
    ret tok;
  }
  
  static L<S> processIncludes(L<S> tok) {
    int safety = 0;
    while (hasCodeTokens(tok, "!", "include") && ++safety < 100)
      tok = processIncludesSingle(tok);
    ret tok;
  }
  
  static L<S> processIncludesSingle(L<S> tok) {
    S s = join(tok);
    Matcher m = Pattern.compile("\n\\s*!include (#\\d+)").matcher(s);
    StringBuffer buf = new StringBuffer();
    while (m.find()) {
      String includedSrc = loadSnippet(m.group(1));
      m.appendReplacement(buf, m.quoteReplacement("\n" + includedSrc));
    }
    m.appendTail(buf);
    ret jtok(str(buf));
  }
  
  static L<S> ctex(L<S> tok) {
    S s = join(tok);
    Pattern regex = Pattern.compile("\\s+(no\\s+exceptions|ctex|null on exception)\\s*\\{");
    
    for (int i = 0; i < 100; i++) {
      Matcher matcher = regex.matcher(s);
      if (!matcher.find())
        break;
      String kind = matcher.group(1);

      System.out.println("Iteration " + (i+1));
      int start = matcher.start(), end = matcher.end();
      int endOfBlock = ctex_findEndOfBlock(s, end);
      
      String catchBlock, catchWhat;
      if (kind.startsWith("null")) {
        catchBlock = "return null;";
        catchWhat = "Throwable";
      } else {
        catchBlock = "throw __e instanceof RuntimeException ? (RuntimeException) __e : new RuntimeException(__e);";
        catchWhat = "Throwable";
      }
        
      String tryBlock = " { try {\n " +
        s.substring(end, endOfBlock) + "\n} catch (" + catchWhat + " __e) { " + catchBlock + " }";
      s = s.substring(0, start) + tryBlock + s.substring(endOfBlock);
    }
    ret jtok(s);
  }
  
  // start is the index AFTER the opening bracket
  // returns index OF closing bracket
  static int ctex_findEndOfBlock(String s, int start) {
    int level = 1;
    for (int i = start; i < s.length(); i++) {
      if (s.charAt(i) == '{') ++level;
      else if (s.charAt(i) == '}') --level;
      if (level == 0)
        return i;
    }
    return s.length();
  }
  
  static L<S> dialogHandler(L<S> tok) {
    ret replaceKeywordBlock(tok,
      "dialogHandler",
      "new DialogHandler() {\n" +
        "public void run(final DialogIO io) {",
      "}}");
  }
  
  /*static S defaultTrans(S s) {
    findTranslators...
  }*/
  
  static S quicknew(S s) {
    // TODO: switch to tokenizing!
    s = s.replaceAll("new\\s+(\\w+)\\s+(\\w+);", "$1 $2 = new $1();");
    s = s.replaceAll("new\\s+L<([\\w\\[\\]<>,\\s]+)>\\s+(\\w+);", "List<$1> $2 = new ArrayList<$1>();");
    s = s.replaceAll("new\\s+List<([\\w\\[\\]<>,\\s]+)>\\s+(\\w+);", "List<$1> $2 = new ArrayList<$1>();");
    s = s.replaceAll("new\\s+\\(Hash\\)Set<(\\w+)>\\s+(\\w+);", "Set<$1> $2 = new HashSet<$1>();");
    s = s.replaceAll("new\\s+\\(Tree\\)Set<(\\w+)>\\s+(\\w+);", "Set<$1> $2 = new TreeSet<$1>();");
    s = s.replaceAll("new\\s+Set<(\\w+)>\\s+(\\w+);", "Set<$1> $2 = new TreeSet<$1>();"); // TreeSet now default - pay attention to explicitly say HashSet if you need it.
    s = s.replaceAll("new\\s+\\(Hash\\)Map<([\\w\\s,]+)>\\s+(\\w+);", "Map<$1> $2 = new HashMap<$1>();");
    s = s.replaceAll("new\\s+\\(Tree\\)Map<([\\w\\s,]+)>\\s+(\\w+);", "Map<$1> $2 = new TreeMap<$1>();");
    
    // TreeMap when string as key
    s = s.replaceAll("new\\s+Map<(S,[\\w\\s,]+)>\\s+(\\w+);", "Map<$1> $2 = new TreeMap<$1>();");
    s = s.replaceAll("new\\s+Map<(String,[\\w\\s,]+)>\\s+(\\w+);", "Map<$1> $2 = new TreeMap<$1>();");
    
    // HashMap is default for everything else
    s = s.replaceAll("new\\s+Map<([\\w\\s,]+)>\\s+(\\w+);", "Map<$1> $2 = new HashMap<$1>();");
    
    s = s.replaceAll("new\\s+(\\w+<[\\w\\s,]+>)\\s+(\\w+);", "$1 $2 = new $1();");
    
    // Constructor calls without parantheses
    // So you can say something like: predictors.add(new P1);
    
    s = s.replaceAll("new\\s+(\\w+)\\s*([,\\);:])", "new $1()$2");
    
    // replace "new List" with "new ArrayList"
    s = s.replaceAll("new\\s+List\\s*\\(", "new ArrayList(");
    
    // for args
    
    s = s.replace("for args {", "for (int i = 0; i < args.length; i++) { final String arg = args[i];");
    ret s;
  }
  
  static L<S> extendClasses(L<S> tok) {
    int i;
    while ((i = jfind(tok, "extend <id> {")) >= 0) {
      S className = tok.get(i+2);
      int idx = findCodeTokens(tok, i, false, "{");
      int j = findEndOfBracketPart(tok, idx+2);
      S content = join(subList(tok, idx+1, j-1));
      L<S> c = findInnerClassOfMain(tok, className);
      clearTokens(tok.subList(i, j+1));
      if (c == null) {
        print("Warning: Can't extend class " + className + ", not found");
        continue;
      }
      int endOfClass = magicIndexOfSubList(tok, c) + l(c)-1;
      while (neq(tok.get(endOfClass), "}")) --endOfClass;
      tok.set(endOfClass, content + "\n" + tok.get(endOfClass));
      reTok(tok); // changed in 2 places, let's retok it all
    }
    ret tok;
  }
  
  static void listComprehensions(L<S> tok) {
    int i;
    while ((i = jfind(tok, "[<id> <id> in")) >= 0) {
      Map<Integer, Integer> bracketMap = getBracketMap(tok);
      S type = tok.get(i+2), id = tok.get(i+4);
      int j = scanOverExpression(tok, bracketMap, i+8, "|");
      S exp = join(tok.subList(i+8, j));
      j += 2;
      int k = scanOverExpression(tok, bracketMap, j, "]");
      S where = join(tok.subList(j, k));
      ++k;
      
      S code = "filter(" + exp + ", func(" + type + " " + id + ") { " + where + " })";
      replaceTokens(tok, i, k, code);
      reTok(tok, i, k);
    }
  }
  
  // lib 123 => !123
  static void libs(L<S> tok) {
    new Set<S> libs;
    int i;
    while ((i = jfind(tok, "lib <int>")) >= 0) {
      S id = tok.get(i+2);
      print("lib " + id);
      if (!libs.contains(id)) {
        libs.add(id);
        tok.set(i, "!");
        tok.set(i+1, "");
      } else {
        print("...ignoring (duplicate)");
        clearAllTokens(tok, i, i+3);
        reTok(tok, i, i+3);
      }
    }
  }
  
  // sourceCodeLine() => 1234
  static void sourceCodeLine(L<S> tok) {
    int i ;
    while ((i = jfind(tok, "sourceCodeLine()")) >= 0) {
      replaceTokens(tok, i, i+5, str(countChar(join(subList(tok, 0, i)), '\n')+1));
      reTok(tok, i, i+5);
    }
  }
  
  // done before any other processing
  static void earlyStuff(L<S> tok) {
    int i;
    
    // Note: this makes the word "quine" a special operator
    // (unusable as a function name)
    
    while ((i = jfind(tok, "quine(")) >= 0) {
      int idx = findCodeTokens(tok, i, false, "(");
      int j = findEndOfBracketPart(tok, idx+2);
      tok.set(i, "new Quine");
      tok.set(idx, "(" + quote(join(subList(tok, idx+1, j-1))) + ", ");
      reTok(tok, i, idx+1);
    }
  }
  
  static void throwFail(L<S> tok) {
    bool anyChange = false;
    for (int i = 1; i+2 < l(tok); i += 2)
      if (eq(get(tok, i+2), "fail") && !eqOneOf(get(tok, i), "throw", "RuntimeException", "return")) {
        tok.set(i+2, "throw fail");
        anyChange = true;
      }
    if (anyChange)
      reTok(tok);
  }
  
  static void namedThreads(L<S> tok) {
    for (int i = 0; i < 100; i++) {
      int idx = findCodeTokens(tok, "thread", "*", "{");
      if (idx < 0)
        break;
      int j = findEndOfBracketPart(tok, idx+4);
      S tName = tok.get(idx+2);
      
      S var = "_t_" + i;
      S pre = "{ Thread " + var + " = new Thread(" + tName + ") {\n" +
        "public void run() {\n" +
        "try";
      S post = "} catch (Exception _e) {\n" +
          "  throw _e instanceof RuntimeException ? (RuntimeException) _e : new RuntimeException(_e); } }\n};\n" +
        var + ".start(); }";

      tok.set(idx, pre);
      tok.set(idx+2, "");
      tok.set(j-1, post);
      reTok(tok, idx, j);
    }
  }
  
  static void threads(L<S> tok) {
    for (bool daemon : litlist(false, true))
      for (int i = 0; i < 100; i++) {
        int idx = findCodeTokens(tok, daemon ? "daemon" : "thread", "{");
        if (idx < 0)
          break;
        int j = findEndOfBracketPart(tok, idx+2);
  
        S var = "_t_" + i;
        S pre = "{ Thread " + var + " = new Thread() {\n" +
          "public void run() {\n" +
          "try";
        S post = "} catch (Exception _e) {\n" +
            "  throw _e instanceof RuntimeException ? (RuntimeException) _e : new RuntimeException(_e); } }\n};\n" +
          (daemon ? var + ".setDaemon(true);\n" : "") +
          var + ".start(); }";
  
        tok.set(idx, pre);
        tok.set(j-1, post);
        reTok(tok, idx, j);
      }
  }
  
  static Map<S, S> sf;
  
  static L<S> standardFunctions(L<S> tok) {
    if (sf == null) {
      L<S> standardFunctions = (L) loadVariableDefinition(
        loadSnippet("#761"), "standardFunctions");

      sf = new HashMap();
      for (String x : standardFunctions) {
        String[] f = x.split("/");
        sf.put(f[1], f[0]);
      }
    }
    
    for (int i = 0; ; i++) {
      Set<String> defd = new HashSet(findFunctions(tok));

      // changes tok
      Set<String> invocations = findFunctionInvocations(tok, sf);

      //print("Functions invoked: " + structure(invocations));
      List<String> needed = diff(invocations, defd);
      if (needed.isEmpty())
        break;
      print("Adding functions: " + join(" " , needed));
        
      new L<S> added;
      new StringBuilder buf;
      new L<S> preload;
      
      for (S x : needed)
        if (sf.containsKey(x))
          preload.add(sf.get(x));
      cachePreload(preload);
      
      for (String x : needed) {
        if (defd.contains(x)) continue;
        
        String id = sf.get(x);
        if (id == null) {
          print("Standard function " + x + " not found.");
          continue;
        }
        //print("Adding function: " + x + " (" + id + ")");
         
        S function = cacheGet(id);
        if (("\n" + function).contains("\n!")) print("Warning: " + id + " contains translators.");
        
        buf.append(function).append("\n");
        added.add(x);
        defd.addAll(findFunctionDefinitions(function));
      }
      tok = includeInMainLoaded(tok, str(buf));
      defd = new HashSet(findFunctions(tok));
      //print("Functions added: " + structure(added));
      
      for (String x : needed)
        if (!defd.contains(x)) {
          print(join(tok));
          fail("Function not defined properly: " + x);
        }
      //print("Iteration " + (i+2));
      if (i >= 1000) fail("Too many iterations");
    }
    
    ret tok;
  }
  
  static L<S> findFunctions(L<S> tok) {
    //ret findFunctionDefinitions(join(findMainClass(tok)));
    ret findFunctionDefs(findMainClass(tok));
  }
  
  static S cacheGet(S snippetID) {
    snippetID = formatSnippetID(snippetID);
    S text = snippetCache.get(snippetID);
    if (text == null)
      snippetCache.put(snippetID, text = loadSnippet(snippetID));
    ret text;
  }
  
  static void cachePreload(L<S> ids) {
    new L<S> needed;
    for (S id : ids)
      if (!snippetCache.containsKey(formatSnippetID(id)))
        needed.add(formatSnippetID(id));
    if (l(needed) > 1) {
      L<S> texts = loadSnippets(needed);
      for (int i = 0; i < l(needed); i++)
        if (texts.get(i) != null)
          snippetCache.put(needed.get(i), texts.get(i));
    }
  }
  
  static L<S> jtok(L<S> tok) {
    ret jtok(join(tok));
  }
  
  static L<S> jtok(S s) {
    L<S> l = javaTok(s);
    ret useIndexedList ? new IndexedList2(l) : l;
  }
  
  /*
  static L<S> loadClass(L<S> tok, S className, S snippetID) {
    if (containsToken(tok, className)
      && !hasCodeTokens(tok, "class", className) && !hasCodeTokens(tok, "interface", className)
      && !tok_importsClassNamed(tok, className)) {
      snippetID = formatSnippetID(snippetID);
      S text = cacheGet(snippetID);
      tok = includeInMainLoaded(tok, text);
    }
    ret tok;
  }
  */
  
  // works on Java level (no "sclass" etc)
  static void addLibraryClasses(L<S> tok, S... data) {
    new HashSet<S> have;
    for (L<S> c : innerClassesOfMain(tok))
      have.add(getClassDeclarationName(c));
    have.addAll(tok_importedClassNames(tok));
    
    L<S> idx = IndexedList2.ensureIndexed(tok);
    for (int i = 0; i+1 < l(data); i++) {
      S className = data[i], snippetID = data[i+1];
      if (idx.contains(className)
        && !have.contains(className)) {
        snippetID = formatSnippetID(snippetID);
        S text = cacheGet(snippetID);
        includeInMainLoaded(tok, text);
      }
    }
  }
}

Began life as a copy of #759