!1006722

static bool autoQuine = true;

static int maxQuineLength = 80;

static L<S> functionsToAlwaysInclude = ll("_registerThread");

!include #1001496 // Matches

!include #1000882 // EGDiff

!include #1000883 // BlockDiffer

!include #1001065 // DialoGIO

!include #1004045 // IndexedList2

//include #1001296 // MultiMap

!include #1000988 // MultiSet

!include #1004543 // DynamicObject

!include #1005575 // CompilerBot

!include #1002662 // isTrue

static int varCount;

static new Map<S, S> snippetCache;

static bool useIndexedList = true, opt_javaTok = true;

static new HashSet<Long> included;

static new HashSet<S> definitions;

static new HashSet<S> shouldNotIncludeFunction, shouldNotIncludeClass;

p {

  if (useIndexedList) findCodeTokens_debug = true;

  javaTok_opt = opt_javaTok;

  findCodeTokens_indexed = findCodeTokens_unindexed = 0;

  findCodeTokens_bails = findCodeTokens_nonbails = 0;

  javaTok_n = javaTok_elements = 0;

  S in = loadMainJava();

  print("759 STARTING " + identityHashCode(main.class));

  included.clear();

  shouldNotIncludeFunction.clear();

  shouldNotIncludeClass.clear();

  varCount = 0;

  //L ts = findTranslators(toLines(join(tok)));

  //print("Translators in source at start: " + structure(ts));

  // "duplicate" statement

  L<S> lines = toLines(in);

  call(getJavaX(), "findTranslators", lines);

  in = fromLines(lines);

  new Matches m;

  if (match("duplicate *", in, m)) {

    // actual copying - unused

    // tok = jtok(loadSnippet(m.get(0)));

    // reference by include()

    in = "m { p { callMain(include(" + quote(m.get(0)) + ")); } }";

  }

  L<S> tok = jtok(in);

  // add m { }

  if (!hasCodeTokens(tok, "m", "{") && !hasCodeTokens(tok, "main", "{") && !hasCodeTokens(tok, "class", "main")) {

    //tok = jtok(moveImportsUp("m {\n" + in + "\n}"));

    replaceTokens_reTok(tok, 1, 2, "m {\n" + tok.get(1));

    replaceTokens_reTok(tok, l(tok)-2, l(tok)-1, tok.get(l(tok)-2) + "}");

    tok_moveImportsUp(tok);

  }

  // standard translate

  //ts = findTranslators(toLines(join(tok)));

  //print("Translators in source: " + structure(ts));

  tok = jtok(defaultTranslate(join(tok)));

  //print("end of default translate");

  //print(join(tok));

  //tok_autoCloseBrackets(tok);    

  tok = processIncludes(tok); // before standard functions

  processConceptsDot(tok);

  tok = processIncludes(tok);

  earlyStuff(tok);

  int safety = 0;

  boolean same;

  do {

    S before = join(tok);

    // shortened method declarations BEFORE standardFunctions

    jreplace(tok, "svoid", "static void");

    jreplace(tok, "void <id> {", "$1 $2() {");

    jreplace(tok, "String <id> {", "$1 $2() {");

    jreplace(tok, "Object <id> {", "$1 $2() {");

    jreplace(tok, "List <id> {", "$1 $2() {");

    tok_definitions(tok);

    tok_ifndef(tok);

    tok_ifdef(tok);

    tok = standardFunctions(tok);

    tok = stdstuff(tok); // standard functions and all the keywords

    S diff;

    long startTime = now();

    //diff = unidiff(before, join(tok));

    //print("unidiff: " + (now()-startTime) + " ms");

    //same = eq(diff, "");

    same = eq(before, join(tok)); // << could be sped up

    if (!same) {

      print("Not same " + safety + ".");

      //print(indent(2, diff));

    }

    if (safety++ >= 10) {

      //print(unidiff(before, join(tok)));

      print("----");

      print(join(tok));

      print("----");

      fail("safety 10 error!");

    }

  } while (!same);

  // POST-PROCESSING after stdstuff loop

  for (S clazz : ll("Pair", "Either")) {

    jreplace(tok, clazz + "< <id> >", clazz + "<$3, $3>");

    jreplace(tok, clazz + "< <id><<id>> >", clazz + "<$3<$5>, $3<$5>>");

    jreplace(tok, clazz + "< <id><<id>,<id>> >", clazz + "<$3<$5,$7>, $3<$5,$7>>");

  }

  quicknew2(tok);

  //tok = jtok(quicknew(join(tok)));

  tok = extendClasses(tok);

  libs(tok);

  sourceCodeLine(tok);

  throwFail(tok);

  innerClassesVar(tok);

  tok_ifclass(tok);

  // Stuff that depends on the list of inner classes (haveClasses)

  HashSet<S> haveClasses = haveClasses(tok);

  expandClassReferences(tok, haveClasses);

  slashCasts(tok, haveClasses);

  newWithoutNew(tok, haveClasses);

  tok = autoImports(tok); // faster to do it at the end

  /*if (useIndexedList)

    print("Indexed/unindexed lookups: " + findCodeTokens_indexed + "/" + findCodeTokens_unindexed + ", lists made: " + IndexedList2.instances);

  print("findCodeToken bails: " + findCodeTokens_bails + "/" + findCodeTokens_nonbails);

  print("javaToks: " + javaTok_n + "/" + javaTok_elements);*/

  if (tok.contains("package"))

    splitJavaFiles(tok);

  else

    saveMainJava(tok);

}

static L<S> stdstuff(L<S> tok) {

  //if (++level >= 10) fail("woot? 10");

  print("stdstuff!");

  int i;

  if (jfind(tok, "!<int>") >= 0) {

    L<S> lines = toLines(join(tok));

    L ts = findTranslators(lines);

    tok = jtok(fromLines(lines));

    print("DROPPING TRANSLATORS: " + structure(ts));

  }

  tok = quickmain(tok);

  tok = processIncludes(tok);

  processConceptsDot(tok);

  tok = processIncludes(tok);

  earlyStuff(tok);

  tok = multilineStrings(tok);

  tok_singleQuoteIdentifiersToStringConstants(tok);

  inStringEvals(tok);

  listComprehensions(tok);

  doubleFor_simple(tok);

  forPing(tok);

  directSnippetRefs(tok);

  quicknu(tok);

  //tok_juxtaposeCalls(tok);

  jreplace(tok, "synced <id>", "synchronized $2");

  replaceKeywordBlock(tok, "answer",

    "static S answer(S s) {\nfinal new Matches m;\n",

    "\nret null;\n}");

  replaceKeywordBlock(tok, "static-pcall",

    "static { pcall {",

    "}}");

  replaceKeywordBlock(tok, "loading",

    "{ JWindow _loading_window = showLoadingAnimation(); try { /* loading try */ ",

    "/* loading try */ } finally { disposeWindow(_loading_window); }\n /* loading end */ } /* after loading */ \n");

  replaceKeywordBlock(tok, "html",

    "static O html(S uri, fMap<S, S> params) ctex " + "{\n", "}");

  // "static sync" => static synchronized

  jreplace(tok, "static sync", "static synchronized");

  // "sclass" => static class

  jreplace(tok, "sclass", "static class");

  // "asclass" => abstract static class

  jreplace(tok, "asclass", "abstract static class");

  // "sinterface" => static interface

  jreplace(tok, "sinterface", "static interface");

  // "ssynchronized" => static synchronized

  jreplace(tok, "ssynchronized", "static synchronized");

  jreplace(tok, "ssvoid", "static synchronized void");

  jreplace(tok, "sbool", "static bool");

  jreplace(tok, "sint", "static int");

  jreplace(tok, "snew", "static new");

  jreplace(tok, "sv <id>", "static void $2");

  jreplace(tok, "pvoid", "public void");

  // "sS" => static S

  jreplace(tok, "sS", "static S");

  // "sO" => static O

  jreplace(tok, "sO", "static O");

  // "sL" => static L

  jreplace(tok, "sL", "static L");

  // "toString {" => "public S toString() {"

  jreplace(tok, "toString {", "public S toString() {");

  jreplace(tok, "Int", "Integer");

  jreplace(tok, "Bool", "Boolean");

  jreplace(tok, "BigInt", "BigInteger");

  jreplace(tok, "Char", "Character");

  // I REALLY wanted to avoid this, but eh...

  jreplace(tok, "SS", "Map<S, S>");

  // "on fail {" => "catch (Throwable _e) { ... rethrow(_e); }"

  replaceKeywordBlock(tok, "on fail",

    "catch (Throwable _e) {",

    "\nthrow rethrow(_e); }");

  // "catch {" => "catch (Throwable _e) {"

  jreplace(tok, "catch {", "catch (Throwable _e) {");

  // "catch X e {" => "catch (X e) {"

  jreplace(tok, "catch <id> <id> {", "catch ($2 $3) {");

  // "catch e {" => "catch (Throwable e) {" (if e is lowercase)

  jreplace(tok, "catch <id> {", "catch (Throwable $2) {", new O() {

    bool get(L<S> tok, int i) {

      S word = tok.get(i+3);

      ret startsWithLowerCaseOrUnderscore(word);

    }

  });

  jreplace(tok, "+ +", "+", new O() {

    bool get(L<S> tok, int i) {

      //printStructure("++: ", subList(tok, i-1, i+6));

      if (empty(_get(tok, i+2))) ret false; // no space between the pluses

      if (empty(_get(tok, i)) && eq("+", _get(tok, i-1))) ret false;  // an actual "++" at the left

      if (empty(_get(tok, i+4)) && eq("+", _get(tok, i+5))) ret false;  // an actual "++" at the right

      //print("doing it");

      ret true;

    }

  });

  // some crazy fancy syntax

  jreplace(tok, "set <id>;", "$2 = true;");

  // [stdEq] -> implementation of equals() and hashCode()

  jreplace(tok, "[stdEq]",

    "public bool equals(O o) { ret stdEq2(this, o); }\n" +

    "public int hashCode() { ret stdHash2(this); }");

  // [concepts] "concept.field!" for dereferencing references

  jreplace(tok, "<id>!", "$1.get()", func(L<S> tok, int i) {

    if (tok.get(i+2).contains("\n")) false; // no line break between <id> and !

    if (nempty(tok.get(i+4))) true; // space after = ok

    S t = _get(tok, i+5);

    if (t == null) ret false;

    if (isIdentifier(t) || eqOneOf(t, "=", "(")) false;

    true;

  });

  // [concepts] "field := value" for defining fields e.g. in "uniq"

  while ((i = jfind(tok, "<id> :=")) >= 0) {

    tok.set(i, quote(tok.get(i)));

    tok.set(i+2, ",");

    tok.set(i+4, "");

    reTok(tok, i, i+5);

  }

  // "quoted" := value

  while ((i = jfind(tok, "<quoted> :=")) >= 0) {

    tok.set(i, tok.get(i));

    tok.set(i+2, ",");

    tok.set(i+4, "");

    reTok(tok, i, i+5);

  }

  // more shortening

  jreplace(tok, "fS", "final S");

  jreplace(tok, "fO", "final O");

  jreplace(tok, "fL", "final L");

  jreplace(tok, "fMap", "final Map");

  jreplace(tok, "fRunnable", "final Runnable");

  jreplace(tok, "f int", "final int");

  // "continue unless"

  while ((i = jfind(tok, "continue unless")) >= 0) {

    int j = scanOverExpression(tok, getBracketMap(tok), i+4, ";");

    replaceTokens(tok, i, i+4, "{ if (!(");

    tok.set(j, ")) continue; }");

    reTok(tok, i, j+1);

  }

  // "continue if"

  while ((i = jfind(tok, "continue if")) >= 0) {

    int j = scanOverExpression(tok, getBracketMap(tok), i+4, ";");

    replaceTokens(tok, i, i+4, "{ if (");

    tok.set(j, ") continue; }");

    reTok(tok, i, j+1);

  }

  // "return if"

  while ((i = jfind(tok, "return if")) >= 0) {

    int j = scanOverExpression(tok, getBracketMap(tok), i+4, ";");

    replaceTokens(tok, i, i+4, "{ if (");

    tok.set(j, ") return; }");

    reTok(tok, i, j+1);

  }

  // while not null ()

  while ((i = jfind(tok, "while not null (")) >= 0) {

    int closingBracket = findEndOfBracketPart(tok, i+6)-1;

    replaceTokens(tok, i+2, i+6, "(");

    tok.set(closingBracket, ") != null)");

    reTok(tok, i, closingBracket+1);

  }

  // Replace $1 with m.unq(0) etc. - caveat: this blocks identifiers $1, $2, ...

  for (i = 1; i < l(tok); i += 2) {

    S s = tok.get(i);

    if (s.startsWith("$")) {

      s = substring(s, 1);

      if (isInteger(s)) {

        tok.set(i, "m.unq(" + (parseInt(s)-1) + ")");

        reTok(tok, i);

      }

    }

  }

  // instanceof trickery

  jreplace(tok, "is a <id>", "instanceof $3");

  jreplace(tok, "!<id> instanceof <id>.<id>", "!($2 instanceof $4.$6)");

  jreplace(tok, "!<id> instanceof <id>", "!($2 instanceof $4)");

  jreplace(tok, "<id> !instanceof <id>", "!($1 instanceof $4)");

  // map funcname(...) => map(f funcname, ...)

  jreplace(tok, "map <id>(", "map(f $2,");

  // func keyword for lambdas - now automatically quines toString() if enabled

  while ((i = jfind(tok, "func(")) >= 0) {

    int argsFrom = i+4, argsTo = findCodeTokens(tok, i, false, ")");

    int idx = findCodeTokens(tok, argsTo, false, "{");

    int j = findEndOfBracketPart(tok, idx);

    L<S> contents = subList(tok, idx+1, j-1);

    S returnType = null;

    if (eq(tok.get(argsTo+2), "-") && eq(tok.get(argsTo+4), ">"))

      returnType = join(subList(tok, argsTo+6, idx-1));

    S toString = autoQuine ? "  public S toString() { ret " + quote(shorten(maxQuineLength, trim(join(contents)))) + "; }" : "";

    // TODO: infer argument types

    /* if (nempty(returnType))

      replaceTokens(tok, i, j, "new F1<?, " + returnType + ">{ " + returnType + " get(" + join(subList(tok, argsFrom, argsTo-1)) + ") ctex { " + tok_addReturn(contents) + " }\n" +*/

    replaceTokens(tok, i, j, "new O { O get(" + join(subList(tok, argsFrom, argsTo-1)) + ") ctex { " + tok_addReturn(contents) + " }\n" +

     + toString + "}");

    reTok(tok, i, j);

  }

  while ((i = jfind(tok, "voidfunc(")) >= 0) {

    int argsFrom = i+4, argsTo = findCodeTokens(tok, i, false, ")");

    int idx = findCodeTokens(tok, argsTo, false, "{");

    int j = findEndOfBracketPart(tok, idx);

    L<S> contents = subList(tok, idx+1, j-1);

    replaceTokens(tok, i, j, "new O { void get(" + join(subList(tok, argsFrom, argsTo-1)) + ") ct" + "ex { " + tok_addSemicolon(contents) + " }\n" +

    (autoQuine ? "  public S toString() { ret " + quote(shorten(maxQuineLength, trim(join(contents)))) + "; }" : "") + "}");

    reTok(tok, i, j);

  }

  for (S keyword : ll("f", "func")) {

    while ((i = jfind(tok, keyword + " {")) >= 0) {

      int idx = findCodeTokens(tok, i, false, "{");

      int j = findEndOfBracketPart(tok, idx+2); // XXX - not idx???

      L<S> contents = subList(tok, idx+1, j-1);

      replaceTokens(tok, i, j, "new O { O get() ct" + "ex { " + tok_addReturn(contents) + " }\n" +

      (autoQuine ? "  public S toString() { ret " + quote(shorten(maxQuineLength, trim(join(contents)))) + "; }" : "") + "}");

      reTok(tok, i, j);

    }

    while ((i = jfind(tok, keyword + " ->")) >= 0) {

      int idx = findCodeTokens(tok, i, false, "{");

      int j = findEndOfBracketPart(tok, idx);

      S returnType = join(subList(tok, i+6, idx-1));

      L<S> contents = subList(tok, idx+1, j-1);

      replaceTokens(tok, i, j, "new F0<" + returnType + ">() { " + returnType + " get() ctex { " + tok_addReturn(contents) + " }\n" +

      (autoQuine ? "  public S toString() { ret " + quote(shorten(maxQuineLength, trim(join(contents)))) + "; }" : "") + "}");

      reTok(tok, i, j);

    }

  }

  // "ref -> bla" for dereferencing Concept.Ref or ThreadLocal or other

  //jreplace(tok, "<id> ->", "$1.get().");

  jreplace(tok, "->", ".get().", func(L<S> tok, int i) {

    empty(tok.get(i+2))

      && !(empty(_get(tok, i)) && eq(_get(tok, i-1), "-")) // i-->0

  });

  // shortened subconcept declaration (before star constructors!)

  shortenedSubconcepts(tok);

  // "case" as a variable name ( => _case)

  caseAsVariableName(tok);

  // "do" as a function name ( => dO)

  tok_doAsMethodName(tok);

  // "continue" as a function name ( => _continue)

  continueAsFunctionName(tok);

  // Do this BEFORE awt replacement! ("p-awt" contains "awt" token)

  if (hasCodeTokens(tok, "p", "-")) {

    jreplace(tok, "p-pretty {", "p-noconsole {");

    replaceKeywordBlock(tok, "p-awt-noconsole", "p-awt {", "\nhideConsole(); }");

    replaceKeywordBlock(tok, "p-substance-noconsole", "p-substance {", "\nhideConsole(); }");

    replaceKeywordBlock(tok, "p-nimbus-noconsole", "p-nimbus {", "\nhideConsole(); }");

    replaceKeywordBlock(tok, "p-subst-noconsole", "p-subst {", "\nhideConsole(); }");

    replaceKeywordBlock(tok, "p-noconsole", "p-subst {", "\nhideConsole(); }");

    replaceKeywordBlock(tok, "p-subst", "p-substance-thread {", "}");

    replaceKeywordBlock(tok, "p-substance-thread", "p { substance();", "}");

    replaceKeywordBlock(tok, "p-magellan-thread", "p { magellan();", "}");

    replaceKeywordBlock(tok, "p-substance", "p-awt { substance();", "}");

    replaceKeywordBlock(tok, "p-nimbus", "p-awt { nimbus();", "}");

    jreplace(tok, "p-type {", "p-typewriter {");

    jreplace(tok, "p-tt {", "p-typewriter {");

    replaceKeywordBlock(tok, "p-awt", "p { swing {", "}}");

    replaceKeywordBlock(tok, "p-typewriter", "p { typeWriterConsole();", "}");

    replaceKeywordBlock(tok, "p-lowprio", "p { lowPriorityThread(r " + "{", "}); }");

    tok_p_repeatWithSleep(tok);

  }

  replaceKeywordBlock(tok,

    "awt",

    "swingLater(r " + "{", // for #711

    "});");

  replaceKeywordBlock(tok,

    "swing",

    "{ swingAndWait(r " + "{", // for #711

    "}); }");

  unswing(tok);

  lockBlocks(tok);

  // crazy stuff

  jreplace (tok, "for <id> over <id>:", "for (int $2 = 0; $2 < l($4); $2++)");

  jreplace (tok, "for <id>, <id> <id> over <id>: {", "for (int $2 = 0; $2 < l($7); $2++) { $4 $5 = $7.get($2);");

  jreplace (tok, "for <id> to <id>:", "for (int $2 = 0; $2 < $4; $2++)");

  jreplace (tok, "for <id> to <int>:", "for (int $2 = 0; $2 < $4; $2++)");

  conceptDeclarations(tok);

  // * constructors

  if (hasCodeTokens(tok, "\\*", "("))

    tok = expandStarConstructors(tok);

  // STANDARD CLASSES & INTERFACES

  S sc = cacheGet("#1003674");   

  new L<S> lclasses;

  for (S line : toLinesFullTrim(sc)) {

    line = javaDropComments(line).trim();

    int idx = line.indexOf('/');

    lclasses.addAll(ll(line.substring(0, idx), line.substring(idx+1)));

  }

  final Set<S> haveClasses = addStandardClasses(tok, toStringArray(lclasses));

  if (tok.contains("Triple") && !haveClasses.contains("Triple"))

    jreplace(tok, "Triple", "T3");

  // "x << X" or "x >> X" => "x instanceof X"

  for (S op : ll("<<", ">>"))

    jreplace(tok, "<id> " + op + " <id>", "$1 instanceof $4", func(L<S> tok, int i) {

      haveClasses.contains(tok.get(i+7))

    });

  expandVarCopies(tok);

  // concept-related stuff

  // auto-import concepts

  bool _a = tok_hasClassRef2(tok, /*"extends",*/ "Concept") || tok_hasClassRef2(tok, "Concepts"), _b = !haveClasses.contains("Concept");

  //print("auto-import: " + _a + ", " + _b);

  if (_a && _b) {

    print("Auto-including concepts.");

    if (shouldNotIncludeClass.contains("Concepts")) {

      print(join(tok));

      fail("Unwanted concepts import");

    }

    printStruct(haveClasses);

    tok = includeInMainLoaded(tok, "concepts.");

    reTok(tok, l(tok)-1, l(tok));

    //processConceptsDot(tok);

  }

  jreplace(tok, "for (<id> <id>)", "for ($3 $4 : list($3))");

  jreplace(tok, "for (final <id> <id>)", "for (final $4 $5 : list($4))");

  // the infamous missing functions (usually caused by class Matches)

  // maybe not needed anymore?

  if (!hasCodeTokens(tok, "String", "unquote") && containsToken(tok, "unquote")) {

    print("Adding unquote");

    tok = includeInMain(tok, "#1001735");

  }

  if (!hasCodeTokens(tok, "String", "formatSnippetID") && containsToken(tok, "formatSnippetID")) {

    print("Adding formatSnippetID");

    tok = includeInMain(tok, "#1000709");

  }

  tok = expandShortTypes(tok);

  if (containsToken(tok, "cast")) {

    S s = join(tok);

    s = s.replaceAll("(\\w+<[\\w\\s,\\[\\]]+>|\\w+|\\w+\\[\\]|\\w+\\[\\]\\[\\])\\s+(\\w+)\\s*=\\s*cast(\\W[^;]*);", "$1 $2 = ($1) ($3);");

    tok = jtok(s);

  }

  replaceKeywordBlock(tok, "r-thread", "runnableThread(r " + "{", "})");

  rNamedThread(tok);

  // runnable and r - now also with automatic toString if enabled

  for (S keyword : ll("runnable", "r"))

    while ((i = jfind(tok, keyword + " {")) >= 0) {

      int idx = findCodeTokens(tok, i, false, "{");

      int j = findEndOfBracketPart(tok, idx);

      L<S> contents = subList(tok, idx+1, j-1);

      //print("r contents: " + structure(contents));

      replaceTokens(tok, i, j+1, "new Runnable() { public void run() { try { " + tok_addSemicolon(contents) + 

        "\n} catch (Exception __e) { throw rethrow(__e); } }" +

    (autoQuine ? "  public S toString() { return " + quote(shorten(maxQuineLength, trim(join(contents)))) + "; }" : "") +

        "}");

      reTok(tok, i, j+1);

    }

  replaceKeywordBlock(tok,

    "expectException",

    "{ bool __ok = false; try {",

    "} catch { __ok = true; } assertTrue(\"expected exception\", __ok); }");

  while ((i = tok.indexOf("tex")) >= 0) {

    tok.set(i, "throws Exception");

    tok = jtok(tok);

  }

  // shorter & smarter whiles

  jreplace(tok, "while true", "while (true)");

  jreplace(tok, "while licensed", "while (licensed())");

  jreplace(tok, "repeat {", "while (licensed()) {");

  tok_repeatWithSleep(tok);

  // null; => return null; etc.

  O cond = func(L<S> tok, int i) {

    tok_tokenBeforeLonelyReturnValue(_get(tok, i-1))

  };

  jreplace(tok, "null;", "return null;", cond);

  jreplace(tok, "false;", "return false;", cond);

  jreplace(tok, "true;", "return true;", cond);

  jreplace(tok, "this;", "return this;", cond);

  // "myFunction;" instead of "myFunction();" - quite rough

  cond = new O() {

    bool get(L<S> tok, int i) {

      S word = tok.get(i+3);

      //print("single word: " + word);

      ret !litlist("break", "continue", "return", "else").contains(word);

    }

  };

  for (S pre : litlist("}", ";"))

    jreplace(tok, pre + " <id>;", "$1 $2();", cond);

  // shorter match syntax for answer methods

  jreplace(tok, "if <quoted> || <quoted>",

    "if (matchOneOf(s, m, $2, $5))");

  // "bla..."

  jreplace(tok, "if <quoted>", "if (matchStartX($2, s, m))",

    new O() { bool get(L<S> tok, int i) {

      ret unquote(tok.get(i+3)).endsWith("...");

    }});

  // "bla * bla | blubb * blubb"

  jreplace_dyn(tok, "if <quoted>", func(L<S> tok, int cIdx) {

    S s = unquote(tok.get(cIdx+2));

    //print("multimatch: " + quote(s));

    new L<S> l;

    for (S pat : splitAtJavaToken(s, "|")) {

      //print("multimatch part: " + quote(pat));

      if (javaTok(pat).contains("*"))

        l.add("match(" + quote(trim(pat)) + ", s, m)");

      else

        l.add("match(" + quote(trim(pat)) + ", s)");

    }

    ret "if (" + join(" || ", l) + ")";

  }, func(L<S> tok, int i) {

    javaTokC(unquote(tok.get(i+3))).contains("|")

  });

  // "bla * bla"

  jreplace(tok, "if <quoted>", "if (match($2, s))",

    new O() { bool get(L<S> tok, int i) {

      ret !javaTokC(unquote(tok.get(i+3))).contains("*");

    }});

  // "bla"

  jreplace(tok, "if <quoted>", "if (match($2, s, m))");

  jreplace(tok, "if match <quoted>", "if (match($3, s, m))");

  // extra commas ("litlist(1, 2,)")

  jreplace(tok, ",)", ")");

  // additional translations (if necessary)

  replaceKeywordBlock(tok,

    "pcall",

    "try {",

    "} catch (Throwable __e) { printStackTrace2(__e); }");

  replaceKeywordBlock(tok,

    "pcall-short",

    "try {",

    "} catch (Throwable __e) { print(exceptionToStringShort(__e)); }");

  replaceKeywordBlock(tok,

    "pcall-silent",

    "try {",

    "} catch (Throwable __e) { silentException(__e); }");

  replaceKeywordBlock(tok,

    "pcall-messagebox",

    "try {",

    "} catch __e { messageBox(__e); }");

  replaceKeywordBlock(tok,

    "pcall-infobox",

    "try {",

    "} catch __e { infoBox(__e); }");

  tok = dialogHandler(tok);

  replaceKeywordBlock(tok, "exceptionToUser",

    "try {",

    "} catch (Throwable __e) { ret exceptionToUser(__e); }"); 

  if (hasCodeTokens(tok, "twice", "{"))

    replaceKeywordBlock(tok, "twice",

      "for (int __twice = 0; __twice < 2; __twice++) {",

      "}"); 

  while ((i = findCodeTokens(tok, "repeat", "*", "{")) >= 0) {

    S v = makeVar("repeat");

    tok.set(i, "for (int " + v + " = 0; " + v + " < " + tok.get(i+2) + "; " + v + "++)");

    tok.set(i+2, "");

    tok = jtok(tok);

  }

  replaceKeywordBlockDyn(tok,

    "time",

    new O() { S[] get() {

      S var = makeVar("startTime");

      ret new S[] {

        "{ long " + var + " = sysNow(); try { ",

        "} finally { " + var + " = sysNow()-" + var + "; saveTiming(" + var + "); } }"};

    }});

  // version without { }

  replaceKeywordBlockDyn(tok,

    "time2",

    new O() { S[] get() {

      S var = makeVar("startTime");

      ret new S[] {

        "long " + var + " = sysNow(); ",

        " " + var + " = sysNow()-" + var + "; saveTiming(" + var + "); "};

    }});

  // time "bla" {

  replaceKeywordPlusQuotedBlock(tok,

    "time",

    func(L<S> tok, int i) {

      S var = makeVar("startTime");

      ret new S[] {

        "long " + var + " = sysNow(); ",

        " done2(" + tok.get(i+2) + ", " + var + "); "};

    });

  if (hasCodeTokens(tok, "assertFail", "{")) {

    S var = makeVar("oops");

    replaceKeywordBlock(tok,

      "assertFail",

      "boolean " + var + " = false; try {",

      "\n" + var + " = true; } catch (Exception e) { /* ok */ } assertFalse(" + var + ");"); 

  }

  replaceKeywordBlock(tok,

    "yo",

    "try {",

    "} catch (Exception " + makeVar("e") + ") { ret false; }");

  replaceKeywordBlock(tok,

    "awtIfNecessary",

    "swingNowOrLater(r " + "{",

    "});");

  ctex(tok);

  replaceKeywordBlock(tok,

    "actionListener",

    "new java.awt.event.ActionListener() { " +

      "public void actionPerformed(java.awt.event.ActionEvent _evt) {",

    "}}");

  namedThreads(tok);

  threads(tok);

  // try answer

  while ((i = findCodeTokens(tok, "try", "answer")) >= 0) {

    int j = findEndOfStatement(tok, i);

    S v = makeVar("a");

    tok.set(i, "{ S " + v);

    tok.set(i+2, "=");

    tok.set(j-1, "; if (!empty(" + v + ")) ret " + v + "; }");

    tok = jtok(tok);

  }

  // return optional (return if not null)

  while ((i = jfind(tok, "return optional <id> =")) >= 0) {

    int j = findEndOfStatement(tok, i);

    S v = tok.get(i+4);

    clearTokens(tok, i+2, i+4);

    tok.set(i, "{");

    tok.set(j-1, "; if (" + v + " != null) ret " + v + "; }");

    tok = jtok(tok);

  }

  functionReferences(tok);

  tok_moveImportsUp(tok);

  /*S moved = moveImportsUp2(join(tok));

  if (moved != null) tok = jtok(moved);*/

  ret tok;

} // end of stdStuff!

static L<S> multilineStrings(L<S> tok) {

  for (int i = 1; i < tok.size(); i += 2) {

    S t = tok.get(i);

    if (isQuoted(t))

      if (t.startsWith("[") || t.contains("\r") || t.contains("\n"))

        tok.set(i, quote(unquote(t)));

  }

  ret tok;

}

static void inStringEvals(L<S> tok) {

  bool change = false;

  for (int i = 1; i < tok.size(); i += 2) {

    S t = tok.get(i);

    if (!isQuoted(t)) continue;

    if (t.contains("\\*") && !t.contains("\\\\")) { // << rough

      tok.set(i, inStringEval(t));

      change = true;

    }

  }

  if (change) reTok(tok);

}

static S inStringEval(S t) {

  t = dropPrefix("\"", dropSuffix("\"", t));

  new L<S> l;

  int idx;

  while ((idx = t.indexOf("\\*")) >= 0) {

    int j = indexOf(t, idx, "*/");

    if (j < 0) break;

    if (idx > 0)

      l.add("\"" + substring(t, 0, idx) + "\"");

    l.add("(" + trim(substring(t, idx+2, j)) + ")");

    t = substring(t, j+2);

  }

  if (nempty(t))

    l.add("\"" + t + "\"");

  ret "(" + join(" + ", l) + ")";

}

static L<S> quickmain(L<S> tok) {

  bool change = false;

  int i = findCodeTokens(tok, "main", "{");

  if (i < 0) i = findCodeTokens(tok, "m", "{");

  if (i >= 0 && !(i-2 > 0 && tok.get(i-2).equals("class"))) {

    tok.set(i, "class main");

    change = true;

  }

  i = findCodeTokens(tok, "psvm", "{");

  if (i < 0) i = findCodeTokens(tok, "p", "{");

  if (i >= 0) {

    int idx = i+2;

    int j = findEndOfBracketPart(tok, idx);

    L<S> contents = subList(tok, idx+1, j-1);

    tok.set(i, "public static void main(final String[] args) throws Exception");

    replaceTokens(tok, idx+1, j-1, tok_addSemicolon(contents));

    change = true;

  }

  ret change ? jtok(tok) : tok;

}

static S makeVar(S name) {

  ret "_" + name + "_" + varCount++;

}

static S makeVar() { ret makeVar(""); }

/*static L<S> standardFunctions(L<S> tok) {

  ret rtq(tok, "#1002474");

}*/

static L<S> rtq(L<S> tok, S id) {

  ret runTranslatorQuick(tok, id);

}

static L<S> expandShortTypes(L<S> tok) {

  // replace <int> with <Integer>

  for (int i = 1; i+4 < tok.size(); i += 2)

    if (tok.get(i).equals("<")

      && litlist(">", ",").contains(tok.get(i+4))) {

      String type = tok.get(i+2);

      if (type.equals("int")) type = "Integer";

      else if (type.equals("long")) type = "Long";

      tok.set(i+2, type);

    }

  // O = Object, S = String, ret = return

  for (int i = 1; i < tok.size(); i += 2) {

    String t = tok.get(i);

    if (t.equals("O")) t = "Object";

    if (t.equals("S")) t = "String";

    else if (t.equals("L")) t = "List";

    //else if (t.equals("F")) t = "Function";

    else if (t.equals("ret")) t = "return";

    else if (t.equals("bool") && i+2 < tok.size() && neq(tok.get(i+2), "(")) t = "boolean"; // bool -> boolean if it's not a function name

    tok.set(i, t);

  }

  jreplace(tok, "LL< <id> >", "L<L<$3>>");

  ret tok;

}

static L<S> autoImports(L<S> tok) {

  S s = join(tok);

  List<String> imports = findImports(s);

  new StringBuilder buf;

  for (String c : standardImports)

    if (!(imports.contains(c)))

      buf.append("import " + c + ";\n");

  if (buf.length() == 0) ret tok;

  ret jtok(buf+s);

}

static String[] standardImports = {

  "java.util.*",

  "java.util.zip.*",

  "java.util.List",

  "java.util.regex.*",

  "java.util.concurrent.*",

  "java.util.concurrent.atomic.*",

  "java.util.concurrent.locks.*",

  "javax.swing.*",

  "javax.swing.event.*",

  "javax.swing.text.*",

  "javax.swing.table.*",

  "java.io.*",

  "java.net.*",

  "java.lang.reflect.*",

  "java.lang.ref.*",

  "java.lang.management.*",

  "java.security.*",

  "java.security.spec.*",

  "java.awt.*",

  "java.awt.event.*",

  "java.awt.image.*",

  "javax.imageio.*",

  "java.math.*"

};

static L<S> expandStarConstructors(L<S> tok) {

  mainLoop: for (int i = 3; i+6 < tok.size(); i += 2) {

    String t = tok.get(i), l = tok.get(i-2);

    if (!t.equals("*"))

      continue;

    if (!tok.get(i+2).equals("("))

      continue;

    if (!eqOneOf(l, "}", "public", "private", "protected", ";", "{", "endif") && neq(get(tok, i-4), "ifclass")) // is this correct...??

      continue;

    // ok, it seems like a constructor declaration.

    // Now find class name by going backwards.

    int j = i, level = 1;

    while (j > 0 && level > 0) {

      t = tok.get(j);

      if (t.equals("}")) ++level;

      if (t.equals("{")) --level;

      j -= 2;

    }

    while (j > 0) {

      t = tok.get(j);

      if (t.equals("class")) {

        String className = tok.get(j+2);

        tok.set(i, className); // exchange constructor name!

        // now for the parameters.

        // Syntax: *(Learner *learner) {

        // We will surely add type inference here in time... :)

        j = i+2;

        while (!tok.get(j).equals("{"))

          j += 2;

        int block = j+1;

        for (int k = i+2; k < block-1; k += 2)

          if (tok.get(k).equals("*")) {

            tok.remove(k);

            tok.remove(k);

            block -= 2;

            String name = tok.get(k);

            tok.addAll(block, Arrays.asList(new String[] {

              "\n  ", "this", "", ".", "", name, " ", "=", " ", name, "", ";" }));

          }

        continue mainLoop;

      }

      j -= 2;

    }

  }

  ret tok;

}

static L<S> processIncludes(L<S> tok) {

  int safety = 0;

  while (hasCodeTokens(tok, "!", "include") && ++safety < 100)

    tok = processIncludesSingle(tok);

  //tok_autoCloseBrackets(tok);

  ret tok;

}

static L<S> processIncludesSingle(L<S> tok) {

  int i;

  while ((i = jfind(tok, "!include #<int>")) >= 0) {

    S id = tok.get(i+6);

    included.add(parseLong(id));

    replaceTokens(tok, i, i+8, "\n" + loadSnippet(id) + "\n");

    reTok(tok, i, i+8);

  }

  while ((i = jfind(tok, "!include once #<int>")) >= 0) {

    S id = tok.get(i+8);

    bool isNew = included.add(parseLong(id));

    replaceTokens(tok, i, i+10, 

      isNew ? "\n" + loadSnippet(id) + "\n" : "");

    reTok(tok, i, i+10);

  }

  ret tok;

}

static void ctex(L<S> tok) {

  replaceKeywordBlock(tok, "ctex",

    "{ try {",

    "} catch (Exception __e) { throw rethrow(__e); } }");

  replaceKeywordBlock(tok, "null on exception",

    "{ try {",

    "} catch (Throwable __e) { return null; } }");

  replaceKeywordBlock(tok, "false on exception",

    "{ try {",

    "} catch (Throwable __e) { return false; } }");

}

static L<S> dialogHandler(L<S> tok) {

  ret replaceKeywordBlock(tok,

    "dialogHandler",

    "new DialogHandler() {\n" +

      "public void run(final DialogIO io) {",

    "}}");

}

static void quicknew2(L<S> tok) {

  jreplace(tok, "new <id> <id>;", "$2 $3 = new $2;");

  jreplace(tok, "new <id><<id>> <id>;", "$2<$4> $6 = new $2;");

  jreplace(tok, "new <id><<id>> <id>, <id>;", "$2<$4> $6 = new $2, $8 = new $2;");

  jreplace(tok, "new <id><<id>,<id>> <id>;", "$2<$4,$6> $8 = new $2;");

  jreplace(tok, "new <id><<id><<id>>> <id>;", "$2 $3 $4 $5 $6 $7 $8 $9 = new $2;");

  jreplace(tok, "new <id>< <id><<id>,<id>> > <id>;", "$2 $3 $4 $5 $6 $7 $8 $9 $10 $11 = new $2;");

  // [abandoned, confusing, looks like a function definition] with arguments - new A a(...); => A a = new A(...);

  //jreplace(tok, "new <id> <id>(", "$2 $3 = new $2(");

  jreplace(tok, "for args " + "{", "for (int i = 0; i < args.length; i++) { final String arg = args[i];");

  // Constructor calls without parentheses

  // So you can say something like: predictors.add(new P1);

  jreplace1(tok, "new <id>", "new $2()", func(L<S> tok, int i) {

    eqOneOf(_get(tok, i+5), "{", ",", ")", ";", ":")

  });

  jreplace(tok, "new List(", "new ArrayList(");

  jreplace(tok, "new Map(", "new HashMap(");

  jreplace(tok, "new Set(", "new HashSet(");

  jreplace(tok, "\\*<id>[<id>] <id>;", "$2[] $6 = new $2[$4];");

  // X x = new(...)  =>  X x = new X(...)

  // X x = new       =>  X x = new

  jreplace(tok, "<id> <id> = new", "$1 $2 = new $1", func(L<S> tok, int i) {

    eqOneOf(_get(tok, i+9), "(", ";", ",", ")")

  });

  jreplace(tok, "<id> <id> = new \\*", "$1 $2 = new $1");

  jreplace(tok, "\\* <id> = new <id>", "$5 $2 = new $5");

  jreplace(tok, "<id><<id>> <id> = new", "$1 $2 $3 $4 $5 = new $1", func(L<S> tok, int i) {

    eqOneOf(_get(tok, i+9+3*2), "(", ";", ",", ")")

  });

  jreplace(tok, "<id><<id>,<id>> <id> = new", "$1 $2 $3 $4 $5 $6 $7 = new $1", func(L<S> tok, int i) {

    eqOneOf(_get(tok, i+9+5*2), "(", ";", ",", ")")

  });

}

static L<S> extendClasses(L<S> tok) {

  int i;

  while ((i = jfind(tok, "extend <id> {")) >= 0) {

    S className = tok.get(i+2);

    int idx = findCodeTokens(tok, i, false, "{");

    int j = findEndOfBracketPart(tok, idx+2);

    S content = join(subList(tok, idx+1, j-1));

    L<S> c = findInnerClassOfMain(tok, className);

    //print("Extending class " + className + " ==> " + join(c));

    clearTokens(tok.subList(i, j+1));

    if (c == null) {

      print("Warning: Can't extend class " + className + ", not found");

      continue;

    }

    int startOfClass = indexOfSubList(tok, c); // magicIndexOfSubList is broken

    int endOfClass = startOfClass + l(c)-1;

    //print("Extending class " + className + " ==> " + join(subList(tok, startOfClass, endOfClass)));

    while (neq(tok.get(endOfClass), "}")) --endOfClass;

    //print("Extending class " + className + " ==> " + join(subList(tok, startOfClass, endOfClass)));

    tok.set(endOfClass, content + "\n" + tok.get(endOfClass));

    reTok(tok); // changed in 2 places, let's retok it all

  }

  ret tok;

}

static void listComprehensions(L<S> tok) {

  int i;

  for (S op : ll(":", "in"))

    while ((i = jfind(tok, "[<id> <id> " + op)) >= 0) {

      Map<Integer, Integer> bracketMap = getBracketMap(tok); // XXX - optimize

      S type = tok.get(i+2), id = tok.get(i+4);

      int j = scanOverExpression(tok, bracketMap, i+8, "|");

      S exp = join(tok.subList(i+8, j));

      j += 2;

      int k = scanOverExpression(tok, bracketMap, j, "]");

      S where = join(tok.subList(j, k));

      ++k;

      S code = "filter(" + exp + ", func(" + type + " " + id + ") { " + where + " })";

      replaceTokens(tok, i, k, code);

      reTok(tok, i, k);

    }

}

/* complicated version - TODO */

// for (S key, S value : map)

/*static void doubleFor(L<S> tok) {

  int i;

  while ((i = jfind(tok, "for (<id> <id>, <id> <id> :")) >= 0) {

    int iComma = indexOf(tok, ",", i);

    S valType = tok.get(iComma+2);

    S valName = tok.get(iComma+4);

    int expStart = indexOf(tok, ":", i)+2;

    int expEnd = findEndOfBracketPart(expStart-2);

    assertEquals(")", tok.get(expEnd));

    assertEquals("{", tok.get(expEnd+2));

    replaceTokens(tok, iComma, expStart, ": keys(");

    tok.set(expEnd, "))");

    tok.set(expEnd+2, "{ " + valType + " " + valName + " = " + 

    S type = tok.get(i+2), id = tok.get(i+4);

      int j = scanOverExpression(tok, bracketMap, i+8, "|");

      S exp = join(tok.subList(i+8, j));

      j += 2;

      int k = scanOverExpression(tok, bracketMap, j, "]");

      S where = join(tok.subList(j, k));

      ++k;

      S code = "filter(" + exp + ", func(" + type + " " + id + ") { " + where + " })";

      replaceTokens(tok, i, k, code);

      reTok(tok, i, k);

    }

}*/

static void doubleFor_simple(L<S> tok) {

  jreplace(tok, "for (<id> <id>, <id> <id> : <id>) {",

    "for ($3 $4 : keys($9)) { $6 $7 = $9.get($4);");

  jreplace(tok, "for (<id> <id>, <id> : <id>) {",

    "for ($3 $4 : keys($8)) { $3 $6 = $8.get($4);");

}

static void forPing(L<S> tok) {

  int i, safety = 0;

  while ((i = jfind(tok, "for ping (")) >= 0) {

    if (++safety >= 10) fail("safety");

    int bracketEnd = findEndOfBracketPart(tok, i+4)-1;

    if (neq("{", tok.get(bracketEnd+2)))

      fail("Please use { with 'for ping' : " + joinSubList(tok, i+4, bracketEnd+3));

    tokSet(tok, bracketEnd+2, "{ ping();");

    tokSet(tok, i+2, "");

  }

}

// lib 123 => !123

static void libs(L<S> tok) {

  new TreeSet<S> libs;

  int i;

  while ((i = jfind(tok, "lib <int>")) >= 0) {

    S id = tok.get(i+2);

    print("lib " + id);

    if (!libs.contains(id)) {

      libs.add(id);

      tok.set(i, "!");

      tok.set(i+1, "");

    } else {

      print("...ignoring (duplicate)");

      clearAllTokens(tok, i, i+3);

      reTok(tok, i, i+3);

    }

  }

}

// sourceCodeLine() => 1234

static void sourceCodeLine(L<S> tok) {

  int i ;

  while ((i = jfind(tok, "sourceCodeLine()")) >= 0) {

    replaceTokens(tok, i, i+5, str(countChar(join(subList(tok, 0, i)), '\n')+1));

    reTok(tok, i, i+5);

  }

}

// done before any other processing

static void earlyStuff(L<S> tok) {

  int i;

  tok_autosemi(tok);

  tok_autoCloseBrackets(tok);

  jreplace(tok, "°", "()");

  // Note: this makes the word "quine" a special operator

  // (unusable as a function name)

  while ((i = jfind(tok, "quine(")) >= 0) {

    int idx = findCodeTokens(tok, i, false, "(");

    int j = findEndOfBracketPart(tok, idx+2);

    tok.set(i, "new Quine");

    tok.set(idx, "(" + quote(join(subList(tok, idx+1, j-1))) + ", ");

    reTok(tok, i, idx+1);

  }

}

static void throwFail(L<S> tok) {

  bool anyChange = false;

  for (int i = 1; i+2 < l(tok); i += 2)

    if (eq(get(tok, i+2), "fail") && !eqOneOf(get(tok, i), "throw", "RuntimeException", "return")) {

      tok.set(i+2, "throw fail");

      anyChange = true;

    }

  if (anyChange)

    reTok(tok);

}

static void namedThreads(L<S> tok) {

  for (int i = 0; i < 100; i++) {

    int idx = findCodeTokens(tok, "thread", "<quoted>", "{");

    if (idx < 0) idx = findCodeTokens(tok, "thread", "<id>", "{");

    if (idx < 0)

      break;

    int j = findEndOfBracketPart(tok, idx+4);

    S tName = tok.get(idx+2);

    S var = "_t_" + i;

    S pre = "{ Thread " + var + " = new Thread(" + tName + ") {\n" +

      "public void run() { pcall {\n";

    S post = "} }\n};\n" +

      "startThread(" + var + "); }";

    tok.set(idx, pre);

    tok.set(idx+2, "");

    tok.set(idx+4, "");

    tok.set(j-1, post);

    //print(join(subList(tok, idx, j)));

    reTok(tok, idx, j);

  }

}

static void rNamedThread(L<S> tok) {

  for (int i = 0; i < 100; i++) {

    int idx = findCodeTokens(tok, "r", "-", "thread", "<quoted>", "{");

    if (idx < 0) idx = findCodeTokens(tok, "r", "-", "thread", "<id>", "{");

    if (idx < 0)

      break;

    int j = findEndOfBracketPart(tok, idx+8);

    S tName = tok.get(idx+6);

    S pre = "r { thread " + tName + " {";

    S post = "}}";

    replaceTokens(tok, idx, idx+9, pre);

    tok.set(j-1, post);

    reTok(tok, idx, j);

  }

}

static void threads(L<S> tok) {

  for (bool daemon : litlist(false, true))

    for (int i = 0; i < 100; i++) {

      int idx = findCodeTokens(tok, daemon ? "daemon" : "thread", "{");

      if (idx < 0)

        break;

      int j = findEndOfBracketPart(tok, idx+2);

      S var = "_t_" + i;

      S pre = "{ Thread " + var + " = new Thread() {\n" +

        "public void run() { pcall\n";

      S post = "} }\n};\n" +

        (daemon ? var + ".setDaemon(true);\n" : "") +

        "startThread(" + var + "); }";

      tok.set(idx, pre);

      tok.set(j-1, post);

      reTok(tok, idx, j);

    }

}

static Map<S, S> sf;

static Bool _761ok;

static L<S> standardFunctions(L<S> tok) {

  if (sf == null) {

    S _761 = cacheGet("#761");

    if (_761ok == null)

      _761ok = isBracketHygienic(_761);

    assertTrue("Whoa! #761 truncated?", _761ok);

    L<S> standardFunctions = concatLists(

      (L) loadVariableDefinition(_761, "standardFunctions"),

      (L) loadVariableDefinition(cacheGet("#1006654"), "standardFunctions"));

    sf = new HashMap();

    for (String x : standardFunctions) {

      String[] f = x.split("/");

      sf.put(f[1], f[0]);

    }

  }

  for (int i = 0; ; i++) {

    Set<String> defd = new HashSet(findFunctions(tok));

    int j;

    while ((j = jfind(tok, "should not include function *.")) >= 0) {

      S fname = tok.get(j+8);

      shouldNotIncludeFunction.add(fname);

      clearAllTokens(tok.subList(j, j+12));

    }

    // changes tok

    Set<String> invocations = findFunctionInvocations(tok, sf);

    invocations.addAll(functionsToAlwaysInclude);

    //print("Functions invoked: " + structure(invocations));

    List<String> needed = diff(invocations, defd);

    if (needed.isEmpty())

      break;

    print("Adding functions: " + join(" " , needed));