!1006722

// experimental

sbool stdstuffOnFunctions = false;

sbool verify_includes = true;

static bool autoQuine = true;

static int maxQuineLength = 80;

!include #1001496 // Matches

!include #1000882 // EGDiff

!include #1000883 // BlockDiffer

!include #1001065 // DialoGIO

!include #1004045 // IndexedList2

//include #1001296 // MultiMap

!include #1000988 // MultiSet

!include #1004543 // DynamicObject

!include #1005575 // CompilerBot

!include #1002662 // isTrue

static int varCount;

static new Map<S, S> snippetCache;

static bool useIndexedList = true, opt_javaTok = true;

p {

  setOpt(mc(), "includeInMainLoaded_safe", true);

  if (useIndexedList) findCodeTokens_debug = true;

  javaTok_opt = opt_javaTok;

  findCodeTokens_indexed = findCodeTokens_unindexed = 0;

  findCodeTokens_bails = findCodeTokens_nonbails = 0;

  javaTok_n = javaTok_elements = 0;

  S in = loadMainJava();

  print("759 STARTING " + identityHashCode(main.class));

  varCount = 0;

  //L ts = findTranslators(toLines(join(tok)));

  //print("Translators in source at start: " + structure(ts));

  // "duplicate" statement

  L<S> lines = toLines(in);

  call(getJavaX(), "findTranslators", lines);

  in = fromLines(lines);

  new Matches m;

  if (match("duplicate *", in, m)) {

    // actual copying - unused

    // tok = jtok(loadSnippet(m.get(0)));

    // reference by include()

    in = "m { p { callMain(include(" + quote(m.get(0)) + ")); } }";

  }

  L<S> tok = jtok(in);

  // add m { }

  if (!hasCodeTokens(tok, "m", "{") && !hasCodeTokens(tok, "main", "{") && !hasCodeTokens(tok, "class", "main"))

    tok = jtok(moveImportsUp("m {\n" + in + "\n}"));

  // standard translate

  //ts = findTranslators(toLines(join(tok)));

  //print("Translators in source: " + structure(ts));

  tok = jtok(defaultTranslate(join(tok)));

  //print("end of default translate");

  //print(join(tok));

  tok_autoCloseBrackets(tok);    

  tok = processIncludes(tok); // before standard functions

  processConceptsDot(tok);

  tok = processIncludes(tok);

  earlyStuff(tok);

  int safety = 0;

  boolean same;

  do {

    S before = join(tok);

    // shortened method declarations BEFORE standardFunctions

    jreplace(tok, "svoid", "static void");

    jreplace(tok, "void <id> {", "$1 $2() {");

    jreplace(tok, "String <id> {", "$1 $2() {");

    jreplace(tok, "Object <id> {", "$1 $2() {");

    jreplace(tok, "List <id> {", "$1 $2() {");

    standardFunctions(tok);

    stdstuff(tok, true); // JavaX keywords, special syntax etc.

    S diff;

    long startTime = now();

    //diff = unidiff(before, join(tok));

    //print("unidiff: " + (now()-startTime) + " ms");

    //same = eq(diff, "");

    same = eq(before, join(tok)); // << could be sped up

    if (!same) {

      print("Not same " + safety + ".");

      //print(indent(2, diff));

    }

    if (safety++ >= 10) {

      //print(unidiff(before, join(tok)));

      print("----");

      print(join(tok));

      print("----");

      fail("safety 10 error!");

    }

  } while (!same);

  // POST-PROCESSING after stdstuff loop

  quicknew2(tok);

  //tok = jtok(quicknew(join(tok)));

  tok = extendClasses(tok);

  libs(tok);

  sourceCodeLine(tok);

  throwFail(tok);

  innerClassesVar(tok);

  tok_ifclass(tok);

  tok = autoImports(tok); // faster to do it at the end

  if (useIndexedList)

    print("Indexed/unindexed lookups: " + findCodeTokens_indexed + "/" + findCodeTokens_unindexed + ", lists made: " + IndexedList2.instances);

  print("findCodeToken bails: " + findCodeTokens_bails + "/" + findCodeTokens_nonbails);

  print("javaToks: " + javaTok_n + "/" + javaTok_elements);

  if (tok.contains("package"))

    splitJavaFiles(tok);

  else

    saveMainJava(tok);

}

static S stdstuff(S s) { ret join(stdstuff(jtok(s), false)); }

static L<S> stdstuff(L<S> tok, bool globalStuff) {

  //if (++level >= 10) fail("woot? 10");

  print("stdstuff!");

  int i;

  if (jfind(tok, "!<int>") >= 0) {

    L<S> lines = toLines(join(tok));

    L ts = findTranslators(lines);

    tok = jtok(fromLines(lines));

    print("DROPPING TRANSLATORS: " + structure(ts));

  }

  tok = quickmain(tok);

  tok = processIncludes(tok);

  processConceptsDot(tok);

  tok = processIncludes(tok);

  earlyStuff(tok);

  tok = multilineStrings(tok);

  inStringEvals(tok);

  listComprehensions(tok);

  directSnippetRefs(tok);

  quicknu(tok);

  jreplace(tok, "synced <id>", "synchronized $2");

  replaceKeywordBlock(tok, "answer",

    "static S answer(S s) {\nfinal new Matches m;\n",

    "\nret null;\n}");

  replaceKeywordBlock(tok, "loading",

    "{ JWindow _loading_window = showLoadingAnimation(); try { /* loading try */ ",

    "/* loading try */ } finally { disposeWindow(_loading_window); }\n /* loading end */ } /* after loading */ \n");

  replaceKeywordBlock(tok, "html",

    "static O html(S uri, fMap<S, S> params) ctex " + "{\n", "}");

  // "static sync" => static synchronized

  jreplace(tok, "static sync", "static synchronized");

  // "sclass" => static class

  jreplace(tok, "sclass", "static class");

  // "asclass" => abstract static class

  jreplace(tok, "asclass", "abstract static class");

  // "sinterface" => static interface

  jreplace(tok, "sinterface", "static interface");

  // "ssynchronized" => static synchronized

  jreplace(tok, "ssynchronized", "static synchronized");

  jreplace(tok, "ssvoid", "static synchronized void");

  jreplace(tok, "sbool", "static bool");

  jreplace(tok, "sint", "static int");

  jreplace(tok, "snew", "static new");

  jreplace(tok, "sv <id>", "static void $2");

  jreplace(tok, "pvoid", "public void");

  // "sS" => static S

  jreplace(tok, "sS", "static S");

  // "sO" => static O

  jreplace(tok, "sO", "static O");

  // "sL" => static L

  jreplace(tok, "sL", "static L");

  // "toString {" => "public S toString() {"

  jreplace(tok, "toString {", "public S toString() {");

  jreplace(tok, "Int", "Integer");

  jreplace(tok, "Bool", "Boolean");

  jreplace(tok, "Char", "Character");

  // I REALLY wanted to avoid this, but eh...

  jreplace(tok, "SS", "Map<S, S>");

  // "on fail {" => "catch (Throwable _e) { ... rethrow(_e); }"

  replaceKeywordBlock(tok, "on fail",

    "catch (Throwable _e) {",

    "\nthrow rethrow(_e); }");

  // "catch {" => "catch (Throwable _e) {"

  jreplace(tok, "catch {", "catch (Throwable _e) {");

  // "catch X e {" => "catch (X e) {"

  jreplace(tok, "catch <id> <id> {", "catch ($2 $3) {");

  // "catch e {" => "catch (Throwable e) {" (if e is lowercase)

  jreplace(tok, "catch <id> {", "catch (Throwable $2) {", new O() {

    bool get(L<S> tok, int i) {

      S word = tok.get(i+3);

      ret startsWithLowerCaseOrUnderscore(word);

    }

  });

  jreplace(tok, "+ +", "+", new O() {

    bool get(L<S> tok, int i) {

      //printStructure("++: ", subList(tok, i-1, i+6));

      if (empty(_get(tok, i+2))) ret false; // no space between the pluses

      if (empty(_get(tok, i)) && eq("+", _get(tok, i-1))) ret false;  // an actual "++" at the left

      if (empty(_get(tok, i+4)) && eq("+", _get(tok, i+5))) ret false;  // an actual "++" at the right

      //print("doing it");

      ret true;

    }

  });

  // some crazy fancy syntax

  jreplace(tok, "set <id>;", "$2 = true;");

  // [stdEq] -> implementation of equals()

  jreplace(tok, "[stdEq]",

    "public bool equals(O o) { ret stdEq2(this, o); }\n" +

    "public int hashCode() { ret stdHash2(this); }");

  // [concepts] "concept.field -> bla" for dereferencing references

  jreplace(tok, "<id> ->", "$1.get().");

  // [concepts] "concept.field!" for dereferencing references

  jreplace(tok, "<id>!", "$1.get()", new O() { bool get(L<S> tok, int i) {

    if (tok.get(i+2).contains("\n")) ret false; // no line break between <id> and !

    S t = _get(tok, i+5);

    if (t == null) ret false;

    if (isIdentifier(t) || eqOneOf(t, "=", "(")) ret false;

    ret true;

  }});

  // [concepts] "field := value" for defining fields e.g. in "uniq"

  while ((i = jfind(tok, "<id> :=")) >= 0) {

    tok.set(i, quote(tok.get(i)));

    tok.set(i+2, ",");

    tok.set(i+4, "");

    reTok(tok, i, i+5);

  }

  // "quoted" := value

  while ((i = jfind(tok, "<quoted> :=")) >= 0) {

    tok.set(i, tok.get(i));

    tok.set(i+2, ",");

    tok.set(i+4, "");

    reTok(tok, i, i+5);

  }

  // more shortening

  jreplace(tok, "fS", "final S");

  jreplace(tok, "fO", "final O");

  jreplace(tok, "fL", "final L");

  jreplace(tok, "fMap", "final Map");

  jreplace(tok, "fRunnable", "final Runnable");

  jreplace(tok, "f int", "final int");

  // "continue unless"

  while ((i = jfind(tok, "continue unless")) >= 0) {

    int j = scanOverExpression(tok, getBracketMap(tok), i+4, ";");

    replaceTokens(tok, i, i+4, "{ if (!(");

    tok.set(j, ")) continue; }");

    reTok(tok, i, j+1);

  }

  // "continue if"

  while ((i = jfind(tok, "continue if")) >= 0) {

    int j = scanOverExpression(tok, getBracketMap(tok), i+4, ";");

    replaceTokens(tok, i, i+4, "{ if (");

    tok.set(j, ") continue; }");

    reTok(tok, i, j+1);

  }

  // "return if"

  while ((i = jfind(tok, "return if")) >= 0) {

    int j = scanOverExpression(tok, getBracketMap(tok), i+4, ";");

    replaceTokens(tok, i, i+4, "{ if (");

    tok.set(j, ") return; }");

    reTok(tok, i, j+1);

  }

  // while not null ()

  while ((i = jfind(tok, "while not null (")) >= 0) {

    int closingBracket = findEndOfBracketPart(tok, i+6)-1;

    replaceTokens(tok, i+2, i+6, "(");

    tok.set(closingBracket, ") != null)");

    reTok(tok, i, closingBracket+1);

  }

  // Replace $1 with m.unq(0) etc. - caveat: this blocks identifiers $1, $2, ...

  for (i = 1; i < l(tok); i += 2) {

    S s = tok.get(i);

    if (s.startsWith("$")) {

      s = substring(s, 1);

      if (isInteger(s)) {

        tok.set(i, "m.unq(" + (parseInt(s)-1) + ")");

        reTok(tok, i);

      }

    }

  }

  // instanceof trickery

  jreplace(tok, "is a <id>", "instanceof $3");

  jreplace(tok, "!<id> instanceof <id>.<id>", "!($2 instanceof $4.$6)");

  jreplace(tok, "!<id> instanceof <id>", "!($2 instanceof $4)");

  jreplace(tok, "<id> !instanceof <id>", "!($1 instanceof $4)");

  // func keyword for lambdas - now automatically quines toString() if enabled

  while ((i = jfind(tok, "func(")) >= 0) {

    int argsFrom = i+4, argsTo = findCodeTokens(tok, i, false, ")");

    int idx = findCodeTokens(tok, argsTo, false, "{");

    int j = findEndOfBracketPart(tok, idx);

    L<S> contents = subList(tok, idx+1, j-1);

    replaceTokens(tok, i, j, "new O { O get(" + join(subList(tok, argsFrom, argsTo-1)) + ") ct" + "ex { " + tok_addReturn(contents) + " }\n" +

    (autoQuine ? "  public S toString() { ret " + quote(shorten(maxQuineLength, trim(join(contents)))) + "; }" : "") + "}");

    reTok(tok, i, j);

  }

  while ((i = jfind(tok, "voidfunc(")) >= 0) {

    int argsFrom = i+4, argsTo = findCodeTokens(tok, i, false, ")");

    int idx = findCodeTokens(tok, argsTo, false, "{");

    int j = findEndOfBracketPart(tok, idx);

    L<S> contents = subList(tok, idx+1, j-1);

    replaceTokens(tok, i, j, "new O { void get(" + join(subList(tok, argsFrom, argsTo-1)) + ") ct" + "ex { " + tok_addSemicolon(contents) + " }\n" +

    (autoQuine ? "  public S toString() { ret " + quote(shorten(maxQuineLength, trim(join(contents)))) + "; }" : "") + "}");

    reTok(tok, i, j);

  }

  while ((i = jfind(tok, "func {")) >= 0) {

    int idx = findCodeTokens(tok, i, false, "{");

    int j = findEndOfBracketPart(tok, idx+2);

    L<S> contents = subList(tok, idx+1, j-1);

    replaceTokens(tok, i, j, "new O { O get() ct" + "ex { " + tok_addReturn(contents) + " }\n" +

    (autoQuine ? "  public S toString() { ret " + quote(shorten(maxQuineLength, trim(join(contents)))) + "; }" : "") + "}");

    reTok(tok, i, j);

  }

  /*replaceKeywordBlock(tok, "func",

    "new O { O get() { ret",

    ";}}");*/

  // shortened subconcept declaration (before star constructors!)

  jreplace(tok, "<id> > <id> {", "concept $3 extends $1 {", new O() { O get(L<S> tok, int i) {

    //print("subconcept: " + quote(tok.get(i)));

    ret tok.get(i).contains("\n"); // only at beginning of line

  }});

  // "case" as a variable name ( => _case)

  caseAsVariableName(tok);

  // "continue" as a function name ( => _continue)

  continueAsFunctionName(tok);

  // Do this BEFORE awt replacement! ("p-awt" contains "awt" token)

  if (hasCodeTokens(tok, "p", "-")) {

    jreplace(tok, "p-pretty {", "p-noconsole {");

    replaceKeywordBlock(tok, "p-awt-noconsole", "p-awt {", "\nhideConsole(); }");

    replaceKeywordBlock(tok, "p-substance-noconsole", "p-substance {", "\nhideConsole(); }");

    replaceKeywordBlock(tok, "p-noconsole", "p-subst {", "\nhideConsole(); }");

    replaceKeywordBlock(tok, "p-subst", "p-substance-thread {", "}");

    replaceKeywordBlock(tok, "p-substance-thread", "p { substance();", "}");

    replaceKeywordBlock(tok, "p-substance", "p-awt { substance();", "}");

    jreplace(tok, "p-type {", "p-typewriter {");

    jreplace(tok, "p-tt {", "p-typewriter {");

    replaceKeywordBlock(tok, "p-awt", "p { swing {", "}}");

    replaceKeywordBlock(tok, "p-typewriter", "p { typeWriterConsole();", "}");

    replaceKeywordBlock(tok, "p-lowprio", "p { lowPriorityThread(r " + "{", "}); }");

    tok_p_repeatWithSleep(tok);

  }

  replaceKeywordBlock(tok,

    "awt",

    "swingLater(r " + "{", // for #711

    "});");

  replaceKeywordBlock(tok,

    "swing",

    "{ swingAndWait(r " + "{", // for #711

    "}); }");

  // crazy stuff

  jreplace (tok, "for <id> over <id>:", "for (int $2 = 0; $2 < l($4); $2++)");

  jreplace (tok, "for <id> to <id>:", "for (int $2 = 0; $2 < $4; $2++)");

  jreplace (tok, "for <id> to <int>:", "for (int $2 = 0; $2 < $4; $2++)");

  conceptDeclarations(tok);

  // * constructors

  if (hasCodeTokens(tok, "\\*", "("))

    tok = expandStarConstructors(tok);

  // STANDARD CLASSES & INTERFACES

  S sc = cacheGet("#1003674");   

  new L<S> lclasses;

  for (S line : toLinesFullTrim(sc)) {

    line = javaDropComments(line).trim();

    int idx = line.indexOf('/');

    lclasses.addAll(ll(line.substring(0, idx), line.substring(idx+1)));

  }

  final Set<S> haveClasses = globalStuff

    ? addStandardClasses(tok, toStringArray(lclasses))

    : null;

  if (globalStuff) haveClasses.add("String");

  // Stuff that depends on the list of inner classes (haveClasses)

  if (globalStuff) {

    expandClassReferences(tok, haveClasses);

    slashCasts(tok, haveClasses);

    // "x << X" => "x instanceof X"

    jreplace(tok, "<id> << <id>", "$1 instanceof $4", new O() {

      O get(L<S> tok, int i) {

        ret haveClasses.contains(tok.get(i+7));

      }

    });

  }

  tok_expandVarCopies(tok);

  // concept-related stuff

  // auto-import concepts

  if (globalStuff) {

    bool _a = hasCodeTokens(tok, /*"extends",*/ "Concept") || hasCodeTokens(tok, "Concepts"), _b = !haveClasses.contains("Concept");

    print("auto-import: " + _a + ", " + _b);

    if (_a && _b) {

      printStruct(haveClasses);

      tok = includeInMainLoaded(tok, "concepts.");

      reTok(tok, l(tok)-1, l(tok));

      //processConceptsDot(tok);

    }

  }

  jreplace(tok, "for (<id> <id>)", "for ($3 $4 : list($3))");

  // the infamous missing functions (usually caused by class Matches)

  // maybe not needed anymore?

  /*if (!hasCodeTokens(tok, "String", "unquote") && containsToken(tok, "unquote")) {

    print("Adding unquote");

    tok = includeInMain(tok, "#1001735");

  }

  if (!hasCodeTokens(tok, "String", "formatSnippetID") && containsToken(tok, "formatSnippetID")) {

    print("Adding formatSnippetID");

    tok = includeInMain(tok, "#1000709");

  }*/

  tok = expandShortTypes(tok);

  if (containsToken(tok, "cast")) {

    S s = join(tok);

    s = s.replaceAll("(\\w+<[\\w\\s,\\[\\]]+>|\\w+|\\w+\\[\\]|\\w+\\[\\]\\[\\])\\s+(\\w+)\\s*=\\s*cast(\\W[^;]*);", "$1 $2 = ($1) ($3);");

    tok = jtok(s);

  }

  replaceKeywordBlock(tok, "r-thread", "runnableThread(r " + "{", "})");

  rNamedThread(tok);

  // runnable and r - now also with automatic toString if enabled

  for (S keyword : ll("runnable", "r"))

    while ((i = jfind(tok, keyword + " {")) >= 0) {

      int idx = findCodeTokens(tok, i, false, "{");

      int j = findEndOfBracketPart(tok, idx);

      L<S> contents = subList(tok, idx+1, j-1);

      //print("r contents: " + structure(contents));

      replaceTokens(tok, i, j+1, "new Runnable() { public void run() { try { " + tok_addSemicolon(contents) + 

        "\n} catch (Exception __e) { throw rethrow(__e); } }" +

    (autoQuine ? "  public S toString() { return " + quote(shorten(maxQuineLength, trim(join(contents)))) + "; }" : "") +

        "}");

      reTok(tok, i, j+1);

    }

  replaceKeywordBlock(tok,

    "expectException",

    "{ bool __ok = false; try {",

    "} catch { __ok = true; } assertTrue(\"expected exception\", __ok); }");

  while ((i = tok.indexOf("tex")) >= 0) {

    tok.set(i, "throws Exception");

    tok = jtok(tok);

  }

  // shorter & smarter whiles

  jreplace(tok, "while true", "while (true)");

  jreplace(tok, "while licensed", "while (licensed())");

  jreplace(tok, "repeat {", "while (licensed()) {");

  tok_repeatWithSleep(tok);

  // null; => return null; etc.

  O cond = new O() {

    bool get(L<S> tok, int i) {

      S t = _get(tok, i-1);

      ret l(t) == 1 && "{};)".contains(t) || eq(t, "else");

    }

  };

  jreplace(tok, "null;", "return null;", cond);

  jreplace(tok, "false;", "return false;", cond);

  jreplace(tok, "true;", "return true;", cond);

  jreplace(tok, "this;", "return this;", cond);

  // "myFunction;" instead of "myFunction();" - quite rough

  cond = new O() {

    bool get(L<S> tok, int i) {

      S word = tok.get(i+3);

      //print("single word: " + word);

      ret !litlist("break", "continue", "return").contains(word);

    }

  };

  for (S pre : litlist("}", ";"))

    jreplace(tok, pre + " <id>;", "$1 $2();", cond);

  // shorter match syntax for answer methods

  jreplace(tok, "if <quoted> || <quoted>",

    "if (matchOneOf(s, m, $2, $5))");

  jreplace(tok, "if <quoted>", "if (matchStartX($2, s, m))",

    new O() { bool get(L<S> tok, int i) {

      ret unquote(tok.get(i+3)).endsWith("...");

    }});

  jreplace(tok, "if <quoted>", "if (match($2, s))",

    new O() { bool get(L<S> tok, int i) {

      ret !javaTokC(unquote(tok.get(i+3))).contains("*");

    }});

  jreplace(tok, "if <quoted>", "if (match($2, s, m))");

  jreplace(tok, "if match <quoted>", "if (match($3, s, m))");

  // extra commas ("litlist(1, 2,)")

  jreplace(tok, ",)", ")");

  // additional translations (if necessary)

  replaceKeywordBlock(tok,

    "pcall",

    "try {",

    "} catch (Throwable __e) { printStackTrace(__e); }");

  replaceKeywordBlock(tok,

    "pcall-short",

    "try {",

    "} catch (Throwable __e) { print(exceptionToStringShort(__e)); }");

  replaceKeywordBlock(tok,

    "pcall-messagebox",

    "try {",

    "} catch __e { messageBox(__e); }");

  tok = dialogHandler(tok);

  replaceKeywordBlock(tok, "exceptionToUser",

    "try {",

    "} catch (Throwable __e) { ret exceptionToUser(__e); }"); 

  if (hasCodeTokens(tok, "twice", "{"))

    replaceKeywordBlock(tok, "twice",

      "for (int __twice = 0; __twice < 2; __twice++) {",

      "}"); 

  while ((i = findCodeTokens(tok, "repeat", "*", "{")) >= 0) {

    S v = makeVar("repeat");

    tok.set(i, "for (int " + v + " = 0; " + v + " < " + tok.get(i+2) + "; " + v + "++)");

    tok.set(i+2, "");

    tok = jtok(tok);

  }

  replaceKeywordBlockDyn(tok,

    "time",

    new O() { S[] get() {

      S var = makeVar("startTime");

      ret new S[] {

        "{ long " + var + " = sysNow(); try { ",

        "} finally { " + var + " = sysNow()-" + var + "; saveTiming(" + var + "); } }"};

    }});

  replaceKeywordBlockDyn(tok,

    "time2",

    new O() { S[] get() {

      S var = makeVar("startTime");

      ret new S[] {

        "long " + var + " = sysNow(); ",

        " " + var + " = sysNow()-" + var + "; saveTiming(" + var + "); "};

    }});

  if (hasCodeTokens(tok, "assertFail", "{")) {

    S var = makeVar("oops");

    replaceKeywordBlock(tok,

      "assertFail",

      "boolean " + var + " = false; try {",

      "\n" + var + " = true; } catch (Exception e) { /* ok */ } assertFalse(" + var + ");"); 

  }

  replaceKeywordBlock(tok,

    "yo",

    "try {",

    "} catch (Exception " + makeVar("e") + ") { ret false; }");

  replaceKeywordBlock(tok,

    "awtIfNecessary",

    "swingNowOrLater(r " + "{",

    "});");

  ctex(tok);

  replaceKeywordBlock(tok,

    "actionListener",

    "new java.awt.event.ActionListener() { " +

      "public void actionPerformed(java.awt.event.ActionEvent _evt) {",

    "}}");

  namedThreads(tok);

  threads(tok);

  // try answer

  while ((i = findCodeTokens(tok, "try", "answer")) >= 0) {

    int j = findEndOfStatement(tok, i);

    S v = makeVar("a");

    tok.set(i, "{ S " + v);

    tok.set(i+2, "=");

    tok.set(j-1, "; if (!empty(" + v + ")) ret " + v + "; }");

    tok = jtok(tok);

  }

  // return optional (return if not null)

  while ((i = jfind(tok, "return optional <id> =")) >= 0) {

    int j = findEndOfStatement(tok, i);

    S v = tok.get(i+4);

    clearTokens(tok, i+2, i+4);

    tok.set(i, "{");

    tok.set(j-1, "; if (" + v + " != null) ret " + v + "; }");

    tok = jtok(tok);

  }

  functionReferences(tok);

  // TODO: optimize

  S moved = moveImportsUp2(join(tok));

  if (moved != null)

    tok = jtok(moved);

  ret tok;

} // end of stdStuff!

static L<S> multilineStrings(L<S> tok) {

  for (int i = 1; i < tok.size(); i += 2) {

    S t = tok.get(i);

    if (isQuoted(t))

      if (t.startsWith("[") || t.contains("\r") || t.contains("\n"))

        tok.set(i, quote(unquote(t)));

  }

  ret tok;

}

static void inStringEvals(L<S> tok) {

  bool change = false;

  for (int i = 1; i < tok.size(); i += 2) {

    S t = tok.get(i);

    if (!isQuoted(t)) continue;

    if (t.contains("\\*") && !t.contains("\\\\")) { // << rough

      tok.set(i, inStringEval(t));

      change = true;

    }

  }

  if (change) reTok(tok);

}

static S inStringEval(S t) {

  t = dropPrefix("\"", dropSuffix("\"", t));

  new L<S> l;

  int idx;

  while ((idx = t.indexOf("\\*")) >= 0) {

    int j = indexOf(t, idx, "*/");

    if (j < 0) break;

    if (idx > 0)

      l.add("\"" + substring(t, 0, idx) + "\"");

    l.add("(" + trim(substring(t, idx+2, j)) + ")");

    t = substring(t, j+2);

  }

  if (nempty(t))

    l.add("\"" + t + "\"");

  ret "(" + join(" + ", l) + ")";

}

static L<S> quickmain(L<S> tok) {

  bool change = false;

  int i = findCodeTokens(tok, "main", "{");

  if (i < 0) i = findCodeTokens(tok, "m", "{");

  if (i >= 0 && !(i-2 > 0 && tok.get(i-2).equals("class"))) {

    tok.set(i, "public class main");

    change = true;

  }

  i = findCodeTokens(tok, "psvm", "{");

  if (i < 0) i = findCodeTokens(tok, "p", "{");

  if (i >= 0) {

    int idx = i+2;

    int j = findEndOfBracketPart(tok, idx);

    L<S> contents = subList(tok, idx+1, j-1);

    tok.set(i, "public static void main(final String[] args) throws Exception");

    replaceTokens(tok, idx+1, j-1, tok_addSemicolon(contents));

    change = true;

  }

  ret change ? jtok(tok) : tok;

}

static S makeVar(S name) {

  ret "_" + name + "_" + varCount++;

}

static S makeVar() { ret makeVar(""); }

/*static L<S> standardFunctions(L<S> tok) {

  ret rtq(tok, "#1002474");

}*/

static L<S> rtq(L<S> tok, S id) {

  ret runTranslatorQuick(tok, id);

}

static L<S> expandShortTypes(L<S> tok) {

  // replace <int> with <Integer>

  for (int i = 1; i+4 < tok.size(); i += 2)

    if (tok.get(i).equals("<")

      && litlist(">", ",").contains(tok.get(i+4))) {

      String type = tok.get(i+2);

      if (type.equals("int")) type = "Integer";

      else if (type.equals("long")) type = "Long";

      tok.set(i+2, type);

    }

  // O = Object, S = String, ret = return

  for (int i = 1; i < tok.size(); i += 2) {

    String t = tok.get(i);

    if (t.equals("O")) t = "Object";

    if (t.equals("S")) t = "String";

    else if (t.equals("L")) t = "List";

    //else if (t.equals("F")) t = "Function";

    else if (t.equals("ret")) t = "return";

    else if (t.equals("bool") && i+2 < tok.size() && neq(tok.get(i+2), "(")) t = "boolean"; // bool -> boolean if it's not a function name

    tok.set(i, t);

  }

  ret tok;

}

static L<S> autoImports(L<S> tok) {

  S s = join(tok);

  List<String> imports = findImports(s);

  new StringBuilder buf;

  for (String c : standardImports)

    if (!(imports.contains(c)))

      buf.append("import " + c + ";\n");

  if (buf.length() == 0) ret tok;

  ret jtok(buf+s);

}

static String[] standardImports = {

  "java.util.*",

  "java.util.zip.*",

  "java.util.List",

  "java.util.regex.*",

  "java.util.concurrent.*",

  "java.util.concurrent.atomic.*",

  "java.util.concurrent.locks.*",

  "javax.swing.*",

  "javax.swing.event.*",

  "javax.swing.text.*",

  "javax.swing.table.*",

  "java.io.*",

  "java.net.*",

  "java.lang.reflect.*",

  "java.lang.ref.*",

  "java.lang.management.*",

  "java.security.*",

  "java.security.spec.*",

  "java.awt.*",

  "java.awt.event.*",

  "java.awt.image.*",

  "javax.imageio.*",

  "java.math.*"

};

static L<S> expandStarConstructors(L<S> tok) {

  mainLoop: for (int i = 3; i+6 < tok.size(); i += 2) {

    String t = tok.get(i), l = tok.get(i-2);

    if (!t.equals("*"))

      continue;

    if (!tok.get(i+2).equals("("))

      continue;

    if (!eqOneOf(l, "}", "public", "private", "protected", ";", "{", "endif") && neq(get(tok, i-4), "ifclass")) // is this correct...??

      continue;

    // ok, it seems like a constructor declaration.

    // Now find class name by going backwards.

    int j = i, level = 1;

    while (j > 0 && level > 0) {

      t = tok.get(j);

      if (t.equals("}")) ++level;

      if (t.equals("{")) --level;

      j -= 2;

    }

    while (j > 0) {

      t = tok.get(j);

      if (t.equals("class")) {

        String className = tok.get(j+2);

        tok.set(i, className); // exchange constructor name!

        // now for the parameters.

        // Syntax: *(Learner *learner) {

        // We will surely add type inference here in time... :)

        j = i+2;

        while (!tok.get(j).equals("{"))

          j += 2;

        int block = j+1;

        for (int k = i+2; k < block-1; k += 2)

          if (tok.get(k).equals("*")) {

            tok.remove(k);

            tok.remove(k);

            block -= 2;

            String name = tok.get(k);

            tok.addAll(block, Arrays.asList(new String[] {

              "\n  ", "this", "", ".", "", name, " ", "=", " ", name, "", ";" }));

          }

        continue mainLoop;

      }

      j -= 2;

    }

  }

  ret tok;

}

static L<S> processIncludes(L<S> tok) {

  int safety = 0;

  while (hasCodeTokens(tok, "!", "include") && ++safety < 100)

    tok = processIncludesSingle_2(tok);

  tok_autoCloseBrackets(tok);

  ret tok;

}

static L<S> processIncludesSingle(L<S> tok) {

  S s = join(tok);

  Matcher m = Pattern.compile("\n\\s*!include (#\\d+)").matcher(s);

  if (!m.find()) ret tok;

  StringBuffer buf = new StringBuffer();

  do {

    String includedSrc = loadSnippet(m.group(1));

    m.appendReplacement(buf, m.quoteReplacement("\n" + includedSrc));

  } while (m.find());

  m.appendTail(buf);

  ret jtok(str(buf));

}

static L<S> processIncludesSingle_2(L<S> tok) {

  int i;

  while ((i = jfind(tok, "!include #<int>")) >= 0) {

    S id = tok.get(i+6);

    String includedSrc = loadSnippet(id);

    clearTokens(tok, i, i+8);

    tok.set(i, "\n" + includedSrc + "\n");

    reTok(tok, i, i+8);

  }

  ret tok;

}

static void ctex(L<S> tok) {

  replaceKeywordBlock(tok, "ctex",

    "{ try {",

    "} catch (Exception __e) { throw rethrow(__e); } }");

  replaceKeywordBlock(tok, "null on exception",

    "{ try {",

    "} catch (Throwable __e) { return null; } }");

  replaceKeywordBlock(tok, "false on exception",

    "{ try {",

    "} catch (Throwable __e) { return false; } }");

}

static L<S> dialogHandler(L<S> tok) {

  ret replaceKeywordBlock(tok,

    "dialogHandler",

    "new DialogHandler() {\n" +

      "public void run(final DialogIO io) {",

    "}}");

}

static void quicknew2(L<S> tok) {

  jreplace(tok, "new <id> <id>;", "$2 $3 = new $2;");

  jreplace(tok, "new <id><<id>> <id>;", "$2<$4> $6 = new $2;");

  jreplace(tok, "new <id><<id>,<id>> <id>;", "$2<$4,$6> $8 = new $2;");

  jreplace(tok, "new <id><<id><<id>>> <id>;", "$2 $3 $4 $5 $6 $7 $8 $9 = new $2;");

  jreplace(tok, "for args " + "{", "for (int i = 0; i < args.length; i++) { final String arg = args[i];");

  // Constructor calls without parentheses

  // So you can say something like: predictors.add(new P1);

  jreplace1(tok, "new <id>", "new $2()", new O() {

    bool get(L<S> tok, int i) {

      ret eqOneOf(_get(tok, i+5), "{", ",", ")", ";", ":");

    }

  });

  jreplace(tok, "new " + "List(", "new ArrayList(");

  jreplace(tok, "new " + "Map(", "new HashMap(");

  jreplace(tok, "\\*<id>[<id>] <id>;", "$2[] $6 = new $2[$4];");

}

// OLD

static S quicknew(S s) {

  s = s.replaceAll("new\\s+L<([\\w\\[\\]<>,\\s]+)>\\s+(\\w+);", "List<$1> $2 = new ArrayList<$1>();");

  s = s.replaceAll("new\\s+List<([\\w\\[\\]<>,\\s]+)>\\s+(\\w+);", "List<$1> $2 = new ArrayList<$1>();");

  s = s.replaceAll("new\\s+\\(Hash\\)Set<(\\w+)>\\s+(\\w+);", "Set<$1> $2 = new HashSet<$1>();");

  s = s.replaceAll("new\\s+\\(Tree\\)Set<(\\w+)>\\s+(\\w+);", "Set<$1> $2 = new TreeSet<$1>();");

  s = s.replaceAll("new\\s+Set<(\\w+)>\\s+(\\w+);", "Set<$1> $2 = new TreeSet<$1>();"); // TreeSet now default - pay attention to explicitly say HashSet if you need it.

  s = s.replaceAll("new\\s+\\(Hash\\)Map<([\\w\\s,]+)>\\s+(\\w+);", "Map<$1> $2 = new HashMap<$1>();");

  s = s.replaceAll("new\\s+\\(Tree\\)Map<([\\w\\s,]+)>\\s+(\\w+);", "Map<$1> $2 = new TreeMap<$1>();");

  // TreeMap when string as key

  s = s.replaceAll("new\\s+Map<(S,[\\w\\s,]+)>\\s+(\\w+);", "Map<$1> $2 = new TreeMap<$1>();");

  s = s.replaceAll("new\\s+Map<(String,[\\w\\s,]+)>\\s+(\\w+);", "Map<$1> $2 = new TreeMap<$1>();");

  // HashMap is default for everything else

  s = s.replaceAll("new\\s+Map<([\\w\\s,]+)>\\s+(\\w+);", "Map<$1> $2 = new HashMap<$1>();");

  s = s.replaceAll("new\\s+(\\w+<[\\w\\s,]+>)\\s+(\\w+);", "$1 $2 = new $1();");

  ret s;

}

static L<S> extendClasses(L<S> tok) {

  int i;

  while ((i = jfind(tok, "extend <id> {")) >= 0) {

    S className = tok.get(i+2);

    int idx = findCodeTokens(tok, i, false, "{");

    int j = findEndOfBracketPart(tok, idx+2);

    S content = join(subList(tok, idx+1, j-1));

    L<S> c = findInnerClassOfMain(tok, className);

    print("Extending class " + className + " ==> " + join(c));

    clearTokens(tok.subList(i, j+1));

    if (c == null) {

      print("Warning: Can't extend class " + className + ", not found");

      continue;

    }

    int startOfClass = indexOfSubList(tok, c); // magicIndexOfSubList is broken

    int endOfClass = startOfClass + l(c)-1;

    print("Extending class " + className + " ==> " + join(subList(tok, startOfClass, endOfClass)));

    while (neq(tok.get(endOfClass), "}")) --endOfClass;

    print("Extending class " + className + " ==> " + join(subList(tok, startOfClass, endOfClass)));

    tok.set(endOfClass, content + "\n" + tok.get(endOfClass));

    reTok(tok); // changed in 2 places, let's retok it all

  }

  ret tok;

}

static void listComprehensions(L<S> tok) {

  int i;

  for (S op : ll(":", "in"))

    while ((i = jfind(tok, "[<id> <id> " + op)) >= 0) {

      Map<Integer, Integer> bracketMap = getBracketMap(tok);

      S type = tok.get(i+2), id = tok.get(i+4);

      int j = scanOverExpression(tok, bracketMap, i+8, "|");

      S exp = join(tok.subList(i+8, j));

      j += 2;

      int k = scanOverExpression(tok, bracketMap, j, "]");

      S where = join(tok.subList(j, k));

      ++k;

      S code = "filter(" + exp + ", func(" + type + " " + id + ") { " + where + " })";

      replaceTokens(tok, i, k, code);

      reTok(tok, i, k);

    }

}

// lib 123 => !123

static void libs(L<S> tok) {

  new TreeSet<S> libs;

  int i;

  while ((i = jfind(tok, "lib <int>")) >= 0) {

    S id = tok.get(i+2);

    print("lib " + id);

    if (!libs.contains(id)) {

      libs.add(id);

      tok.set(i, "!");

      tok.set(i+1, "");

    } else {

      print("...ignoring (duplicate)");

      clearAllTokens(tok, i, i+3);

      reTok(tok, i, i+3);

    }

  }

}

// sourceCodeLine() => 1234

static void sourceCodeLine(L<S> tok) {

  int i ;

  while ((i = jfind(tok, "sourceCodeLine()")) >= 0) {

    replaceTokens(tok, i, i+5, str(countChar(join(subList(tok, 0, i)), '\n')+1));

    reTok(tok, i, i+5);

  }

}

// done before any other processing

static void earlyStuff(L<S> tok) {

  int i;

  // Note: this makes the word "quine" a special operator

  // (unusable as a function name)

  while ((i = jfind(tok, "quine(")) >= 0) {

    int idx = findCodeTokens(tok, i, false, "(");

    int j = findEndOfBracketPart(tok, idx+2);

    tok.set(i, "new Quine");

    tok.set(idx, "(" + quote(join(subList(tok, idx+1, j-1))) + ", ");

    reTok(tok, i, idx+1);

  }

}

static void throwFail(L<S> tok) {

  bool anyChange = false;

  for (int i = 1; i+2 < l(tok); i += 2)

    if (eq(get(tok, i+2), "fail") && !eqOneOf(get(tok, i), "throw", "RuntimeException", "return")) {

      tok.set(i+2, "throw fail");

      anyChange = true;

    }

  if (anyChange)

    reTok(tok);

}

static void namedThreads(L<S> tok) {

  for (int i = 0; i < 100; i++) {

    int idx = findCodeTokens(tok, "thread", "<quoted>", "{");

    if (idx < 0) idx = findCodeTokens(tok, "thread", "<id>", "{");

    if (idx < 0)

      break;

    int j = findEndOfBracketPart(tok, idx+4);

    S tName = tok.get(idx+2);

    S var = "_t_" + i;

    S pre = "{ /*nt*/ Thread " + var + " = new Thread(" + tName + ") {\n" +

      "public void run() { /* in run */ pcall { /* in thread */ \n";

    S post = "/* in thread */ } /* in run */ }\n};\n" +

      var + ".start(); }";

    tok.set(idx, pre);

    tok.set(idx+2, "");

    tok.set(idx+4, "");

    tok.set(j-1, post);

    //print(join(subList(tok, idx, j)));

    reTok(tok, idx, j);

  }

}

static void rNamedThread(L<S> tok) {

  for (int i = 0; i < 100; i++) {

    int idx = findCodeTokens(tok, "r", "-", "thread", "<quoted>", "{");

    if (idx < 0) idx = findCodeTokens(tok, "r", "-", "thread", "<id>", "{");

    if (idx < 0)

      break;

    int j = findEndOfBracketPart(tok, idx+8);

    S tName = tok.get(idx+6);

    S pre = "r { thread " + tName + " {";

    S post = "}}";

    replaceTokens(tok, idx, idx+9, pre);

    tok.set(j-1, post);

    reTok(tok, idx, j);

  }

}

static void threads(L<S> tok) {

  for (bool daemon : litlist(false, true))

    for (int i = 0; i < 100; i++) {

      int idx = findCodeTokens(tok, daemon ? "daemon" : "thread", "{");

      if (idx < 0)

        break;

      int j = findEndOfBracketPart(tok, idx+2);

      S var = "_t_" + i;

      S pre = "{ Thread " + var + " = new Thread() {\n" +

        "public void run() { pcall\n";

      S post = "} }\n};\n" +

        (daemon ? var + ".setDaemon(true);\n" : "") +

        var + ".start(); }";

      tok.set(idx, pre);

      tok.set(j-1, post);

      reTok(tok, idx, j);

    }

}

static Map<S, S> sf;

static L<S> standardFunctions(L<S> tok) {

  if (sf == null) {

    L<S> standardFunctions = concatLists(

      (L) loadVariableDefinition(cacheGet("#761"), "standardFunctions"),

      (L) loadVariableDefinition(cacheGet("#1006654"), "standardFunctions"));

    sf = new HashMap();

    for (String x : standardFunctions) {

      String[] f = x.split("/");

      sf.put(f[1], f[0]);

    }

  }

  for (int i = 0; ; i++) {

    Set<String> defd = new HashSet(findFunctions(tok));

    // changes tok

    Set<String> invocations = findFunctionInvocations(tok, sf);

    //print("Functions invoked: " + structure(invocations));

    List<String> needed = diff(invocations, defd);

    if (needed.isEmpty())

      break;

    print("Adding functions: " + join(" " , needed));

    new L<S> added;

    new StringBuilder buf;

    new L<S> preload;

    for (S x : needed)

      if (sf.containsKey(x))

        preload.add(sf.get(x));

    cachePreload(preload);

    for (String x : cloneList(needed)) {

      if (defd.contains(x)) continue;

      String id = sf.get(x);

      if (id == null) {

        print("Standard function " + x + " not found.");

        needed.remove(x);

        continue;

      }

      //print("Adding function: " + x + " (" + id + ")");

      S function = cacheGet(id);

      if (("\n" + function).contains("\n!")) print("Warning: " + id + " contains translators.");

      buf.append(function).append("\n");

      added.add(x);

      defd.addAll(findFunctionDefs(javaTok(function)));

    }

    S s = str(buf);

    if (stdstuffOnFunctions) s = stdstuff(s);

    tok = includeInMainLoaded(tok, s);

    if (verify_includes) assertTrue("main class", findMainClass(tok) != null);

    // defd = new HashSet(findFunctions(tok));

    //print("Functions added: " + structure(added));

    for (String x : needed)

      if (!defd.contains(x)) {

        print(join(tok));

        fail("Function not defined properly: " + x);

      }

    //print("Iteration " + (i+2));

    if (i >= 1000) fail("Too many iterations");

  }

  ret tok;

}

static L<S> findFunctions(L<S> tok) {

  //ret findFunctionDefinitions(join(findMainClass(tok)));

  ret findFunctionDefs(findMainClass(tok));

}

static S cacheGet(S snippetID) {

  snippetID = formatSnippetID(snippetID);

  S text = snippetCache.get(snippetID);

  if (text == null)

    snippetCache.put(snippetID, text = loadSnippet(snippetID));

  ret text;

}

static void cachePreload(L<S> ids) {

  new L<S> needed;

  for (S id : ids)

    if (!snippetCache.containsKey(formatSnippetID(id)))

      needed.add(formatSnippetID(id));

  if (l(needed) > 1) {

    L<S> texts = loadSnippets(needed);

    for (int i = 0; i < l(needed); i++)

      if (texts.get(i) != null)

        snippetCache.put(needed.get(i), texts.get(i));

  }

}

static L<S> jtok(L<S> tok) {

  ret jtok(join(tok));

}

static L<S> jtok(S s) {

  L<S> l = javaTok(s);

  ret useIndexedList ? new IndexedList2(l) : l;

}

// works on Java level (no "sclass" etc)

// returns list of classes we have (useful for other processing)

static Set<S> addStandardClasses(L<S> tok, S... data) {

  new HashSet<S> have;

  for (L<S> c : innerClassesOfMain(tok))

    have.add(getClassDeclarationName(c));

  have.addAll(tok_importedClassNames(tok));

  //L<S> idx = IndexedList2.ensureIndexed(tok);

  Set<S> idx = tokenIndexWithoutIfclass(tok);

  for (int i = 0; i+1 < l(data); i++) {

    S className = data[i], snippetID = data[i+1];

    if (idx.contains(className)

      && !have.contains(className)) {

      print("Adding class " + className + " / " + snippetID);

      snippetID = formatSnippetID(snippetID);

      S text = cacheGet(snippetID);

      includeInMainLoaded(tok, text);

      L<S> ct = javaTok(text);

      jreplace (ct, "sclass", "static class");

      jreplace (ct, "sinterface", "static interface");

      tok_autoCloseBrackets(ct);

      for (L<S> c : allClasses(ct))

        have.add(getClassDeclarationName(c));

      if (!have.contains(className))

        fail("Wrongly defined class: " + className + " / " + snippetID);

    }

  }

  ret have;

}

// magically append ".class" to class name references

static void expandClassReferences(L<S> tok, Set<S> classNames) {

  bool change = false;

  for (int i = 3; i+2 < l(tok); i += 2)

    if (classNames.contains(tok.get(i))) {

      S s = tok.get(i-2), t = tok.get(i+2);

      if (eqOneOf(s, "instanceof", "new", ".", "<", "implements", "throws", "extends", "/", "nu")) continue;

      if (isIdentifier(s)) continue; // XXX?

      if (eq(s, ",") && eqOneOf(_get(tok, i-6), "implements", "throws")) continue;

      // TODO: longer lists

      // check for cast

      if (eq(s, "(") && eq(t, ")") && i >= 5) {

        if (!eqOneOf(get(tok, i+4), "{", ";")) {

          S x = tok.get(i-4);

          if (!isIdentifier(x)) continue;

          if (eqOneOf(x, "ret", "return")) continue;

        }

      }

      if (eqOneOf(t, ",", ")", ";", ":")) {

        tok.set(i, tok.get(i) + ".class");

        change = true;

      }

    }

  if (change) reTok(tok);

}

// "<id>/<ClassName>" => "((ClassName) <id>)"

static void slashCasts(L<S> tok, final Set<S> classNames) {

  jreplace(tok, "<id>/<id>", "(($3) $1)", new O() {

    O get(L<S> tok, int i) {

      ret classNames.contains(tok.get(i+5));

    }

  });

}

static void processConceptsDot(L<S> tok) {

  bool change;

  do {

    change = false;

    for (int i : jfindAll(tok, "concepts."))

      if (contains(get(tok, i+3), "\n")) {

        replaceTokens(tok, i, i+3, "!" + "include #1004863 // Dynamic Concepts");