!1006722

!include once #1025802 // optimized findCodeTokens

set flag AllowMetaCode.

//set flag tok_forUnnull_debug.

// ifdef in cached includes? => i think cached includes still contain the ifdefs, so it's fine. tok_ifdef is not called within localStuff1

set flag noAI.

set flag isQuoted_dirty. // should be OK, we're dealing with tokens

static bool autoQuine = true;

static int maxQuineLength = 80;

sbool assumeTriple = true;

sbool quickInstanceOfEnabled; // interferes with things

// _registerThread usually costs nothing because we need

// the registerWeakHashMap mechanism anyway for ping().

// Anyway - no forced functions for now :)

static L<S> functionsToAlwaysInclude = ll(

  //"_registerThread",

  //"asRuntimeException"

);

// classes with two type parameters that can written with just one

// e.g. Pair<S> => Pair<S, S>

static Set<S> pairClasses = lithashset("Pair", "Either", "Map", "AbstractMap", "HashMap", "TreeMap", "LinkedHashMap", "MultiMap", "CompactHashMap", "WrappedMap", "F1", "IF1", "AllOnAll", "AllOnAllWithUpdates");

sS transpilingSnippetID;

//static new AtomicInteger varCount;

static new ThreadLocal<AtomicInteger> varCountByThread;

static new Map<S, S> snippetCache;

sbool useIndexedList2 = false, useTokenIndexedList = true;

sbool opt_javaTok = true;

static bool cacheStdFunctions = true, cacheStdClasses = true;

static new HashMap<Long, CachedInclude> cachedIncludes;

static ExecutorService executor;

static L lclasses;

static long startTime, lastPrint;

// These variables have to be cleared manually for each transpilation

static new HashSet<Long> included;

static Set<S> definitions = ciSet();

static new HashMap<S> rewrites;

static new HashSet<S> shouldNotIncludeFunction;

static new HashSet<S> shouldNotIncludeClass;

static new HashSet<S> doNotIncludeFunction;

static new HashSet<S> addedFunctions;

static new HashSet<S> addedClasses;

static new HashSet<S> hardFunctionReferences;

static new HashSet<S> mapLikeFunctions;

static new HashSet<S> mapMethodLikeFunctions;

static new HashSet<S> nuLikeFunctions;

static SS extraStandardFunctions;

sbool quickmainDone1, quickmainDone2;

static new TreeSet<S> libs;

sS mainBaseClass, mainPackage, mainClassName;

sbool localStuffOnly; // for transpiling a fragment

sbool asInclude; // for transpiling an include (auto-close scopes)

sbool allowMetaCode = false; // run any embedded meta code

static LS metaPostBlocks, metaTransformers;

sbool dontPrintSource;

sbool dontLoadCachedIncludesFromVM; // for benchmarking

sclass CachedInclude {

  S javax;

  Future<S> java;

  S realJava;

  S java() {

    ret realJava != null ? realJava : getFuture(java);

  }

  Future<S> javaFuture() {

    ret realJava != null ? nowFuture(realJava) : java;

  }

  void clean {

    if (java != null) {

      realJava = getFuture(java);

      java = null;

    }

  }

}

p {

  startTime = lastPrint = sysNow();

  pcall {

    if (!dontLoadCachedIncludesFromVM)

      vmKeepWithProgramMD5_get('cachedIncludes);

  }

  executor = Executors.newFixedThreadPool(numberOfCores());

  transpilingSnippetID = or(getThreadLocal((ThreadLocal<S>) getOpt(javax(), 'transpilingSnippetID)), transpilingSnippetID);

  print(+transpilingSnippetID);

  set transpileRaw_dontCopyFromCreator;

  fO oldPrint = or(print_byThread()!, f print_raw);

  temp tempInterceptPrint(new F1<S, Bool>() {

    Bool get(S s) {

      long now = sysNow();

      long time = now-lastPrint; // -startTime;

      lastPrint = now;

      callF(oldPrint, "[" + formatInt(time/1000, 2) + ":" + formatInt(time % 1000, 3) + "] " + s);

      false;

    }

  });

  try {

    _main();

  } finally {

    interceptPrintInThisThread(oldPrint);

    if (executor != null) executor.shutdown();

    executor = null;

    localStuffOnly = false;

    asInclude = false;

  }

}

svoid _main() ctex {

  if (sameSnippetID(programID(), defaultJavaXTranslatorID())) setDefaultJavaXTranslatorID(#7);

  //reTok_modify_check = true;

  //if (useIndexedList) findCodeTokens_debug = true;

  javaTok_opt = opt_javaTok;

  //findCodeTokens_indexed = findCodeTokens_unindexed = 0;

  findCodeTokens_bails = findCodeTokens_nonbails = 0;

  javaTok_n = javaTok_elements = 0;

  S in = loadMainJava();

  print("759 STARTING " + identityHashCode(main.class));

  included.clear();

  definitions.clear();

  rewrites.clear();

  definitions.add("SymbolAsString");

  shouldNotIncludeFunction.clear();

  shouldNotIncludeClass.clear();

  doNotIncludeFunction.clear();

  addedFunctions.clear();

  addedClasses.clear();

  hardFunctionReferences.clear();

  mapLikeFunctions = cloneHashSet(tok_mapLikeFunctions());

  mapMethodLikeFunctions = cloneHashSet(tok_mapMethodLikeFunctions());

  nuLikeFunctions.clear();

  extraStandardFunctions = new HashMap;

  libs.clear();

  mainBaseClass = mainPackage = mainClassName = null;

  varCountByThread.set(null);

  quickmainDone1 = quickmainDone2 = false;

  metaPostBlocks = new L;

  metaTransformers = new L;

  dontPrintSource = false;

  //L ts = findTranslators(toLines(join(tok)));

  //print("Translators in source at start: " + structure(ts));

  L<S> tok = jtok(in);

  try {

    tok_definitions(tok);

    // add m { }

    if (!localStuffOnly && !hasCodeTokens(tok, "m", "{") && !hasCodeTokens(tok, "main", "{") && !hasCodeTokens(tok, "class", "main")) {

      //tok = jtok(moveImportsUp("m {\n" + in + "\n}"));

      if (l(tok) == 1) tok = singlePlusList(first(tok), dropFirst(javaTok("m {}")));

      else {

        replaceTokens_reTok(tok, 1, 2, "m {\n\n" + unnull(get(tok, 1)));

        replaceTokens_reTok(tok, l(tok)-2, l(tok)-1, unnull(get(tok, l(tok)-2)) + "}");

      }

      tok_moveImportsUp(tok);

    }

    // standard translate

    //ts = findTranslators(toLines(join(tok)));

    //print("Translators in source: " + structure(ts));

    if (tok_hasTranslators(tok))

      tok = jtok(defaultTranslate(join(tok)));

    //print("end of default translate");

    //print(join(tok));

    //tok_autoCloseBrackets(tok);    

    tok_metaTransformNow(tok);

    tok_processEarlyIncludes(tok);

    tok_earlyGeneralStuff(tok);

    tok = tok_processIncludes(tok); // before standard functions

    if (processConceptsDot(tok))

      tok = tok_processIncludes(tok);

    tok = localStuff1(tok);

   if (!localStuffOnly) {

    int safety = 0;

    boolean same;

    do { // BIG LOOP

      S before = join(tok);

      // do the non-local stuff (flags and rewrites, things that correlate across includes like tok_selfType)

      tok_selfType(tok);

      tok_mainClassNameAndPackage(tok);

      tok_definitions(tok);

      tok_ifndef(tok);

      tok_ifdef(tok);

      defineMapLikes(tok);

      if (tok_applyMapLikeFunctions(tok, mapLikeFunctions))

        functionReferences(tok);

      defineMapMethodLikes(tok);

      tok_applyMapMethodLikeFunctions(tok, mapMethodLikeFunctions);

      defineNuLikes(tok);

      tok_applyNuLikeFunctions(tok, nuLikeFunctions);

      tok_dropExtraCommas(tok); // from e.g. tok_applyMapMethodLikeFunctions

      tok_delegateTo(tok);

      tok_replaceWith(tok);

      tok_findRewrites(tok);

      tok_processRewrites(tok);

      // main bla(...) => mainClassName.bla(...)

      jreplace(tok, "main <id>(", or(mainClassName, "main") + ".$2(");

      try {

        if (safety == 0) tok = quickmain(tok);

      } catch e {

        printSources(tok);

        rethrow(e);

      }

      tok_collectMetaPostBlocks(tok, metaPostBlocks);

      tok_collectTransformers(tok, metaTransformers);

      tok_metaTransformNow(tok);

      // Hack to allow DynModule to reimplement _registerThread

      /*if (tok.contains("DynModule") && !addedClasses.contains("DynModule"))

        addStandardClasses_v2(tok);*/

      defineExtraSF(tok);

      tok = standardFunctions(tok);

      tok = stdstuff(tok); // all the keywords, standard

      S diff;

      long startTime = now();

      //diff = unidiff(before, join(tok));

      //print("unidiff: " + (now()-startTime) + " ms");

      //same = eq(diff, "");

      same = tok_sameTest(tok, before);

      if (!same) {

        print("Not same " + safety + ".");

        //print(indent(2, diff));

      }

      if (safety++ >= 10) {

        //print(unidiff(before, join(tok)));

        printSources(tok);

        fail("safety 10 error!");

      }

    } while (!same); // END OF BIG LOOP

    print("Post.");

    if (mainBaseClass != null) {

      jreplace1(tok, "class main", "class main extends " + mainBaseClass);

      mainBaseClass = null;

    }

    print('moveImportsUp); tok_moveImportsUp(tok);

    print('Indexing); tok = indexTokenList(tok);

    // POST-PROCESSING after stdstuff loop

    if (transpilingSnippetID != null)

      jreplace_dyn(tok, "class whatever", 

        func(L<S> tok, int cIndex) -> S {

          pcall { ret "class " + stringToLegalIdentifier(getSnippetTitle(transpilingSnippetID)); }

          ret "class Whatever";

        });

    //print("Type<A, A>"); // Type<A> to Type<A, A>

    print('extendClasses); tok = extendClasses(tok);

    print('libs); libs(tok);

    print('sourceCodeLine); sourceCodeLine(tok);

    tok_overridableFunctionDefs(tok, null);

    // escaping for lambdas

    //jreplace(tok, "-=>", "->");

    // Stuff that depends on the list of inner classes (haveClasses)

    HashSet<S> haveClasses = haveClasses_actual(tok);

    print('innerClassesVar); innerClassesVar(tok, haveClasses);

    fillVar_transpilationDate(tok);

    haveClasses_addImported(tok, haveClasses);

    print('ifclass); tok_ifclass(tok, haveClasses);

    print('slashCasts); slashCasts(tok, haveClasses);

    print('newWithoutNew); newWithoutNew(tok, haveClasses);

    if (!assumeTriple) {

      print('Triple);

      if (tok.contains("Triple") && !haveClasses.contains("Triple")) {

        jreplace(tok, "Triple", "T3");

        haveClasses.remove("Triple");

        haveClasses.add("T3");

        slashCasts(tok, lithashset("T3"));

        tok_quickInstanceOf(tok, lithashset("T3"));

      }

      expandTriple(tok);

    }

    if (hasDef("SymbolAsString"))

      jreplace(tok, "Symbol", "String");

    print('classReferences); expandClassReferences_lazy(tok, haveClasses);

    if (metaCodeAllowed()) runMetaPostBlocks(tok);

    // Error-checking

    print("Error-checking"); 

    Set<S> functions = new HashSet(findFunctions(tok));

    for (S f : hardFunctionReferences)

      if (!functions.contains(f))

        fail("Function " + f + " requested, but not supplied");

    print('autoImports); tok = autoImports(tok); // faster to do it at the end

    print("definitions=" + sfu(definitions));

    if (containsOneOfIC(definitions, "allpublic", "reparse", "PublicExceptTopClass")) {

      // Fire up the Java parser & pretty printer!

      print(containsIC(definitions, "allpublic")? "Making all public." : "Reparsing.");

      //try {

        S src = join(tok);

        try {

          if (containsIC(definitions, "PublicExceptTopClass"))

            src = javaParser_makeAllPublic(src, notTopLevelClassDecl := true);

          else if (containsIC(definitions, "keepComments"))

            src = javaParser_makeAllPublic_keepComments(src);

          else if (containsIC(definitions, "allpublic"))

            src = javaParser_makeAllPublic(src);

          else

            src = javaParser_reparse_keepComments(src);

        } catch e {

          extractAndPrintJavaParseError(src, e);

          File f = javaxCachesDir("error-source.java");

          saveTextFileVerbose(f, src);

          dontPrintSource = true;

          throw e;

        }

        tok = jtok(src);

      /*} catch e {

        S src = join(tok);

        if (!dontPrintSource)

          print(src);

        print(f2s(saveProgramTextFile("error.java", src)));

        throw rethrow(e);

      }*/

    }

    // Do this after JavaParser (because it doesn't like package after class)

    if (mainPackage != null) {

      print('mainPackage);

      tokPrepend(tok, 1, "package " + mainPackage + ";\n");

      reTok(tok, 1, 2);

    }

    if (mainClassName != null) {

      print('mainClassName);

      jreplace(tok, "class main", "class " + mainClassName);

      jreplace(tok, "main.class", mainClassName + ".class");

      //tokPrepend(tok, 1, "class main {}\n"); // so main.class is generated and compiler sanity checks succeed. we can later skip it in the JavaXClassLoader

    }

    if (nempty(libs)) {

      print("Adding libs: " + libs);

      tok.add(concatMap_strings(func(S s) -> S { "\n!" + s }, libs));

    }

   } // if (!localStuffOnly)

  } catch e {

    S src = join(tok);

    if (!dontPrintSource)

      print(src);

    print(f2s(saveProgramTextFile("error.java", src)));

    throw rethrow(e);

  }

  /*if (useIndexedList)

    print("Indexed/unindexed lookups: " + findCodeTokens_indexed + "/" + findCodeTokens_unindexed + ", lists made: " + IndexedList2.instances);

  print("findCodeToken bails: " + findCodeTokens_bails + "/" + findCodeTokens_nonbails);

  print("javaToks: " + javaTok_n + "/" + javaTok_elements);*/

  print("Saving.");

  // for dexcompile.php

  if (mainClassName != null)

    tokPrepend(tok, 0, "//FILENAME: " 

      + (mainPackage != null ? mainPackage.replace(".", "/") + "/" : "") + mainClassName + ".java\n");

  if (mainJava != null)

    mainJava = join(tok);

  else if (tok.contains("package"))

    splitJavaFiles(tok);

  else

    saveMainJava(tok);

}

static L<S> localStuff1(L<S> tok) {

  int safety = 0, i;

  boolean same;

  tok = indexTokenList(tok);

  tok_scopes(tok, autoCloseScopes := true);

  do {

    S before = join(tok);

    //print("localStuff loop " + safety);

    earlyStuff(tok);

    // EARLY local stuff goes here

    tok_earlyGeneralStuff(tok);

    conceptDeclarations(tok);

    tok_recordDecls(tok);

    tok = multilineStrings(tok);

    tok_singleQuoteIdentifiersToStringConstants(tok);

    inStringEvals(tok);

    tok_listComprehensions(tok);

    tok_for_single(tok);

    tok_for_unpair(tok); // Do this...

    tok_doubleFor_v2(tok); // ...before this

    tok_forUnnull(tok);

    tok_ifCast(tok);

    forPing(tok);

    tok_directSnippetRefs(tok);

    quicknu(tok);

    //tok_juxtaposeCalls(tok);

    jreplace(tok, "LLS", "L<LS>");

    jreplace(tok, "LS", "L<S>");

    jreplace(tok, "ES", "Ext<S>");

    jreplace(tok, "ExtS", "Ext<S>");

    jreplace(tok, "dispose <id>;", "{ cleanUp($2); $2 = null; }");

    jreplace(tok, "do ping {", "do { ping();");

    replaceKeywordBlock(tok,

      "swing",

      "{ swing(r {",

      "}); }");

    replaceKeywordBlock(tok,

      "androidUI",

      "{ androidUI(r {",

      "}); }");

    replaceKeywordBlock(tok,

      "withDBLock",

      "{ withDBLock(r {",

      "}); }");

    replaceKeywordBlock(tok, "afterwards", "temp tempAfterwards(r {", "});");

    for (S keyword : ll("tokcondition", "tokCondition"))

      replaceKeywordBlock(tok,

        keyword,

        "new TokCondition { bool get(final L<S> tok, final int i) {",

        "}}");

    jreplace(tok, "synced <id>", "synchronized $2");

    jreplace(tok, "sync <id>", "synchronized $2");

    replaceKeywordBlock(tok, "answer",

      "static S answer(S s) {\nfinal new Matches m;\n",

      "\nret null;\n}");

    replaceKeywordBlock(tok, "static-pcall",

      "static { pcall {",

      "}}");

    replaceKeywordBlock(tok, "loading",

      "{ temp tempShowLoadingAnimation(); ",

      "}");

    replaceKeywordPlusQuotedBlock(tok, "loading",

      new O { S[] get(L<S> tok, int i) {

        S text = tok.get(i+2);

        ret new S[] {

          "{ temp tempShowLoadingAnimation(" + text + "); ",

          "}" };

      }});

    while ((i = jfind(tok, "visualize as")) >= 0) {

      int j = tok_findEndOfStatement(tok, i); // expression, rather

      tok.set(i+2, "{ ret");

      tok.set(j-1, "; }");

      reTok(tok, i, j);

    }

    jreplace(tok, "visualize {", "public JComponent visualize() {", tokCondition_beginningOfMethodDeclaration());

    jreplace(tok, "visualize2 {", "JComponent visualize2() {", tokCondition_beginningOfMethodDeclaration());

    replaceKeywordBlock(tok, "start-thread-printDone", "start-thread {", "printDone(); }");

    replaceKeywordBlock(tok, "start-thread", [[start { thread "Start" { temp enter(); pcall {]], "}}}");

    jreplace(tok, "start {", "void start() { super.start();", tokCondition_beginningOfMethodDeclaration());

    // run { ... } => public void run() { ... }

    jreplace(tok, "run {", "public void run() {", tokcondition {

      ret neqGet(tok, i-1, "void");

    });

    replaceKeywordBlock(tok, "html",

      "static O html(S uri, fMap<S, S> params) ctex " + "{\n", "}");

    replaceKeywordBlock(tok, "afterVisualize",

      "visualize { JComponent _c = super.visualize();",

      "ret _c; }");

    replaceKeywordBlock(tok, "enhanceFrame",

      "void enhanceFrame(Container f) { super.enhanceFrame(f);",

      "}");

    if (assumeTriple) {

      jreplace(tok, "Triple", "T3");

      expandTriple(tok);

    }

    tok_shortFinals(tok);

    tok_moduleClassDecls(tok);

    jreplace(tok, "static sync", "static synchronized");

    jreplace(tok, "sclass", "static class");

    jreplace(tok, "srecord", "static record");

    jreplace(tok, "record noeq", "noeq record");

    jreplace(tok, "asclass", "abstract static class");

    jreplace(tok, "sinterface", "static interface");

    jreplace(tok, "ssynchronized", "static synchronized");

    jreplace(tok, "ssvoid", "static synchronized void");

    jreplace(tok, "sbool", "static bool");

    jreplace(tok, "fbool", "final bool");

    jreplace(tok, "sint", "static int");

    jreplace(tok, "snew", "static new");

    jreplace(tok, "sv <id>", "static void $2");

    jreplace(tok, "pvoid", "public void");

    // "sS" => static S

    jreplace(tok, "sS", "static S");

    // "sO" => static O

    jreplace(tok, "sO", "static O");

    // "sL" => static L

    jreplace(tok, "sL", "static L");

    // "toString {" => "public S toString() {"

    jreplace(tok, "toString {", "public S toString() {");

    jreplace(tok, "Int", "Integer");

    jreplace(tok, "Bool", "Boolean");

    jreplace(tok, "BigInt", "BigInteger");

    jreplace(tok, "Char", "Character");

    jreplace(tok, "Sym", "Symbol");

    jreplace(tok, "SymSym", "SymbolSymbol");

    jreplace(tok, "SS", "Map<S>");

    jreplace(tok, "SymbolSymbol", "Map<Symbol>");

    jreplace(tok, "PairS", "Pair<S>");

    jreplace(tok, "LPairS", "L<Pair<S>>");

    jreplace(tok, "T3S", "T3<S>");

    jreplace(tok, "F1S", "F1<S>");

    jreplace(tok, "ItIt", "IterableIterator");

    jreplace(tok, "CloseableItIt", "CloseableIterableIterator");

    jreplace(tok, "class <id> > <id> {", "class $2 extends $4 {");

    jreplace(tok, "class <id> > <id><<id>> {", "class $2 extends $4  $5 $6 $7 {");

    jreplace(tok, "ISegmenter", "IF1<BufferedImage, L<Rect>>");

    // IPred<A> => IF1<A, Bool>

    jreplace(tok, "IPred<<id>>", "IF1<$3, Bool>");

    // "on fail {" => "catch (Throwable _e) { ... rethrow(_e); }"

    replaceKeywordBlock(tok, "on fail",

      "catch (Throwable _e) {",

      "\nthrow rethrow(_e); }");

    // "catch {" => "catch (Throwable _e) {"

    jreplace(tok, "catch {", "catch (Throwable _e) {");

    // "catch print e {" => "catch e { _handleException(e); "

    jreplace(tok, "catch print <id> {", "catch $3 { _handleException($3);");

    // "catch print short e {" => "catch e { printExceptionShort(e); "

    jreplace(tok, "catch print short <id> {", "catch $4 { printExceptionShort($4);");

    // "catch X e {" => "catch (X e) {"

    jreplace(tok, "catch <id> <id> {", "catch ($2 $3) {");

    // "catch e {" => "catch (Throwable e) {" (if e is lowercase)

    jreplace(tok, "catch <id> {", "catch (Throwable $2) {", tokcondition {

      S word = tok.get(i+3);

      ret startsWithLowerCaseOrUnderscore(word);

    });

    jreplace(tok, "+ +", "+", tokcondition {

      //printStructure("++: ", subList(tok, i-1, i+6));

      if (empty(_get(tok, i+2))) ret false; // no space between the pluses

      if (empty(_get(tok, i)) && eq("+", _get(tok, i-1))) ret false;  // an actual "++" at the left

      if (empty(_get(tok, i+4)) && eq("+", _get(tok, i+5))) ret false;  // an actual "++" at the right

      //print("doing it");

      ret true;

    });

    // some crazy fancy syntax

    jreplace(tok, "set <id>;", "$2 = true;");

    // single underscore (not allowed in Java anymore) to double underscore

    jreplace(tok, "_", "__");

    // [stdEq] -> implementation of equals() and hashCode()

    jreplace(tok, "[stdEq]",

      "public bool equals(O o) { ret stdEq2(this, o); }\n" +

      "public int hashCode() { ret stdHash2(this); }");

    // [concepts] "concept.field!" for dereferencing references

    jreplace(tok, "*!", "$1.get()", tokcondition {

      S l = tok.get(i+1);

      if (!(isIdentifier(l) || eq(l, ")"))) false;

      if (tok.get(i+2).contains("\n")) false; // no line break between <id> and !

      if (nempty(tok.get(i+4))) true; // space after = ok

      S t = _get(tok, i+5);

      if (t == null) ret false;

      if (isIdentifier(t) || eqOneOf(t, "=", "(")) false;

      true;

    });

    jreplace(tok, "for (<id> <id>)", "for ($3 $4 : list($3))");

    jreplace(tok, "for (final <id> <id>)", "for (final $4 $5 : list($4))");

    // "continue unless", "break unless"

    for (S phrase : ll("continue unless", "break unless"))

      while ((i = jfind(tok, phrase)) >= 0) {

        S keyword = tok.get(i);

        int j = scanOverExpression(tok, getBracketMap(tok), i+4, ";");

        replaceTokens(tok, i, i+4, "{ if (!(");

        tok.set(j, ")) " + keyword + "; }");

        reTok(tok, i, j+1);

      }

    // "continue if", "break if"

    for (S phrase : ll("continue if", "break if"))

      while ((i = jfind(tok, phrase)) >= 0) {

        S keyword = tok.get(i);

        int j = scanOverExpression(tok, getBracketMap(tok), i+4, ";");

        replaceTokens(tok, i, i+4, "{ if (");

        tok.set(j, ") " + keyword + "; }");

        reTok(tok, i, j+1);

      }

    // "return if"

    while ((i = jfind(tok, "return if")) >= 0) {

      int j = scanOverExpression(tok, getBracketMap(tok), i+4, ";");

      replaceTokens(tok, i, i+4, "{ if (");

      tok.set(j, ") return; }");

      reTok(tok, i, j+1);

    }

    // "return unless"

    while ((i = jfind(tok, "return unless")) >= 0) {

      int j = scanOverExpression(tok, getBracketMap(tok), i+4, ";");

      replaceTokens(tok, i, i+4, "{ if (!(");

      tok.set(j, ")) return; }");

      reTok(tok, i, j+1);

    }

    // "return <id> if"

    while ((i = jfind(tok, "return <id> if")) >= 0) {

      int j = scanOverExpression(tok, getBracketMap(tok), i+4, ";");

      tok.set(j, ") return " + tok.get(i+2) + "; }");

      replaceTokens(tok, i, i+6, "{ if (");

      reTok(tok, i, j+1);

    }

    // "return with <statement>" / "continue with <statement>" / "break with <statement>"

    while ((i = jfind(tok, "<id> with", tokcondition { ret eqOneOf(tok.get(i+1), "return", "continue", "break"); })) >= 0) {

      int j = scanOverExpression(tok, getBracketMap(tok), i+4, ";");

      tok.set(j, "; " + tok.get(i) + "; }");

      replaceTokens(tok, i, i+3, "{");

      reTok(tok, i, j+1);

    }

    // return "bla" with <statement>

    while ((i = jfindOneOf(tok,

      "return <quoted> with", "return <id> with")) >= 0) {

      S result = tok.get(i+2);

      int j = scanOverExpression(tok, getBracketMap(tok), i+6, ";");

      replaceTokens(tok, i, i+5, "{");

      tok.set(j, "; return " + result + "; }");

      reTok(tok, i, j+1);

    }

    tok_debugStatements(tok);

    // while not null (...) / if not null (...)

    while ((i = jfind_check not(tok, "<id> not null (", tokcondition {

      ret eqOneOf(_get(tok, i+1), "if", "while");

    })) >= 0) {

      int closingBracket = findEndOfBracketPart(tok, i+6)-1;

      replaceTokens(tok, i+2, i+6, "(");

      tok.set(closingBracket, ") != null)");

      reTok(tok, i, closingBracket+1);

    }

    // while null (...) / if null (...)

    while ((i = jfind_check null(tok, "<id> null (", tokcondition {

      ret eqOneOf(_get(tok, i+1), "if", "while");

    })) >= 0) {

      int closingBracket = findEndOfBracketPart(tok, i+4)-1;

      replaceTokens(tok, i+2, i+4, "(");

      tok.set(closingBracket, ") == null)");

      reTok(tok, i, closingBracket+1);

    }

    // Replace $1 with m.unq(0) etc. - caveat: this blocks identifiers $1, $2, ...

    for (i = 1; i < l(tok); i += 2) {

      S s = tok.get(i);

      if (s.startsWith("$")) {

        s = substring(s, 1);

        if (isInteger(s)) {

          tok.set(i, "m.unq(" + (parseInt(s)-1) + ")");

          reTok(tok, i);

        }

      }

    }

    // instanceof trickery

    jreplace(tok, "is a <id>", "instanceof $3");

    jreplace(tok, "!<id> instanceof <id>.<id>", "!($2 instanceof $4.$6)");

    jreplace(tok, "!<id> instanceof <id>", "!($2 instanceof $4)");

    jreplace(tok, "<id> !instanceof <id>", "!($1 instanceof $4)");

    // map func1 func2 func3(...) => mapFGH(f func1, f func2, f func3, ...)

    jreplace(tok, "map <id> <id> <id>(", "mapFGH(f $2, f $3, f $4,");

    // map func1 func2(...) => mapFG(f func1, f func2, ...)

    jreplace(tok, "map <id> <id>(", "mapFG(f $2, f $3,");

    // "ref->bla" for dereferencing Concept.Ref or ThreadLocal or other

    // For lambdas, use SPACES on the left or right of the arrow!

    //jreplace(tok, "<id> ->", "$1.get().");

    jreplace(tok, "->", ".get().", tokcondition {

      ret empty(tok.get(i)) // no space on left of arrow

        && empty(tok.get(i+2)) // no space inside of arrow

        && empty(tok.get(i+4)) // no space on right of arrow

        && !eq(_get(tok, i-1), "-"); // i-->0;

    });

    // shortened subconcept declaration (before star constructors!)

    shortenedSubconcepts(tok);

    // "case" as a variable name ( => _case)

    caseAsVariableName(tok);

    // "do" as a function name ( => dO)

    tok_doAsMethodName(tok);

    // "continue" as a function name ( => _continue)

    continueAsFunctionName(tok);

    tok_extend(tok);

    jreplace(tok, "pn {", "p-noconsole {");

    // Do these BEFORE awt replacement! ("p-awt" contains "awt" token)

    replaceKeywordBlock(tok, "r-awt", "r { awt {", "}}");

    if (hasCodeTokens(tok, "p", "-")) tok_p_old(tok);

    replaceKeywordBlock(tok, "awt-messagebox", "awt { pcall-messagebox {", "}}");

    replaceKeywordBlock(tok, "awt", "swingLater(r {", "});");

    jreplace(tok, "p-android {", "set flag Android. p {");

    unswing(tok);

    lockBlocks(tok);

    tempBlocks(tok);

    tok_switchTo(tok);

    // trim x;

    jreplace(tok, "trim <id>;", "$2 = trim($2);");

    // iterate with index

    jreplace (tok, "for <id> over <id>:", "for (int $2 = 0; $2 < l($4); $2++)");

    jreplace (tok, "for <id> backwards over <id>:", "for (int $2 = l($5)-1; $2 >= 0; $2--)");

    jreplace (tok, "for <id>, <id> <id> over <id>: {", "for (int $2 = 0; $2 < l($7); $2++) { $4 $5 = $7.get($2);");

    jreplace (tok, "for <id>, <id> <id> backwards over <id>: {", "for (int $2 = l($8)-1; $2 >= 0; $2--) { $4 $5 = $8.get($2);");

    jreplace (tok, "for <id> to <id>:", "for (int $2 = 0; $2 < $4; $2++)");

    jreplace (tok, "for <id> to <int>:", "for (int $2 = 0; $2 < $4; $2++)");

    tok = expandShortTypes(tok);

    tok_equalsCast(tok);

    tok_equalsOptCast(tok);

    replaceKeywordBlock(tok, "r-thread-messagebox", "r-thread { pcall-messagebox {", "}}");

    replaceKeywordBlock(tok, "thread-messagebox", "thread { pcall-messagebox {", "}}");

    jreplace(tok, "rThread {", "r-thread {");

    jreplace(tok, "rThreadEnter {", "r-thread { temp enter(); ");

    replaceKeywordBlock(tok, "r-thread", "runnableThread(r {", "})");

    rNamedThread(tok);

    // only works in the scope of a DynModule

    jreplace(tok, "rEnter {", "r { temp enter(); ");

    replaceKeywordBlock(tok, "r-messagebox", "r { pcall-messagebox {", "}}");

    jreplace(tok, "r <id> + r <id>", "r { $2(); $5(); }");

    // runnable and r - now also with automatic toString if enabled

    for (S keyword : ll("runnable", "r")) {

      while ((i = jfind(tok, keyword + " {")) >= 0) {

        int idx = findCodeTokens(tok, i, false, "{");

        int j = findEndOfBracketPart(tok, idx);

        L<S> contents = subList(tok, idx+1, j-1);

        replaceTokens(tok, i, j+1, "new Runnable {"

          + "  public void run() ctex { " + tok_addSemicolon(contents) + "\n}"

          + (autoQuine ? tok_autoQuineFunc(contents) : "")

          + "}");

        reTok(tok, i, j+1);

      }

      while ((i = jfind(tok, keyword + " <quoted> {")) >= 0) {

        int idx = findCodeTokens(tok, i, false, "{");

        int j = findEndOfBracketPart(tok, idx);

        L<S> contents = subList(tok, idx+1, j-1);

        replaceTokens(tok, i, j+1, "new Runnable {"

          + "  public void run() ctex { " + tok_addSemicolon(contents) + "\n}"

          + "  toString { ret " + tok.get(i+2) + "; }"

          + (autoQuine ? tok_autoQuineFunc(contents, '_shortenedSourceCode) : "")

          + "}");

        reTok(tok, i, j+1);

      }

    }

    replaceKeywordBlock(tok,

      "expectException",

      "{ bool __ok = false; try {",

      "} catch { __ok = true; } assertTrue(\"expected exception\", __ok); }");

    while ((i = tok.indexOf("tex")) >= 0) {

      tok.set(i, "throws Exception");

      tok = jtok(tok);

    }

    // shorter & smarter whiles

    jreplace(tok, "while true", "while (true)");

    jreplace(tok, "while licensed", "while (licensed())");

    jreplace(tok, "repeat {", "while (licensed()) {");

    tok_repeatWithSleep(tok);

    // null; => return null; etc.

    O cond = tokcondition {

      ret tok_tokenBeforeLonelyReturnValue(tok, i-1);

    };

    jreplace(tok, "null;", "return null;", cond);

    jreplace(tok, "false;", "return false;", cond);

    jreplace(tok, "true;", "return true;", cond);

    jreplace(tok, "this;", "return this;", cond);

    // ok <cmd> => ret "OK" with <cmd>

    jreplace(tok, "ok <id>", "return \"OK\" with $2");

    replaceKeywordBlock(tok, "ok", "{", " return \"OK\"; }");

    // "myFunction;" instead of "myFunction();" - quite rough

    // (isolated identifier as function call)

    cond = new TokCondition {

      bool get(L<S> tok, int i) {

        S word = tok.get(i+3);

        //print("single word: " + word);

        ret !eqOneOf(word, "break", "continue", "return", "else", "endifdef", "endif");

      }

    };

    for (S pre : litlist("}", ";"))

      jreplace(tok, pre + " <id>;", "$1 $2();", cond);

    // shorter match syntax for answer methods

    jreplace(tok, "if <quoted> || <quoted>",

      "if (matchOneOf(s, m, $2, $5))");

    // "bla * bla | blubb * blubb"

    jreplace_dyn(tok, "if <quoted>", func(L<S> tok, int cIdx) -> S {

      S s = unquote(tok.get(cIdx+2));

      //print("multimatch: " + quote(s));

      new L<S> l;

      for (S pat : splitAtJavaToken(s, "|")) {

        //print("multimatch part: " + quote(pat));

        if (pat.contains("..."))

          l.add("matchX(" + quote(trim(pat)) + ", s, m)");

        else if (javaTok(pat).contains("*"))

          l.add("match(" + quote(trim(pat)) + ", s, m)");

        else

          l.add("match(" + quote(trim(pat)) + ", s)");

      }

      ret "if (" + join(" || ", l) + ")";

    }, tokcondition {

      ret javaTokC(unquote(tok.get(i+3))).contains("|");

    });

    // "...bla..."

    jreplace(tok, "if <quoted>", "if (find3plusRestsX($2, s, m))",

      tokcondition {

        ret startsAndEndsWith(unquote(tok.get(i+3)), "...");

      });

    // "bla..."

    jreplace(tok, "if <quoted>", "if (matchStartX($2, s, m))",

      tokcondition {

        ret unquote(tok.get(i+3)).endsWith("...");

      });

    // "bla"

    jreplace(tok, "if <quoted>", "if (match($2, s))",

      tokcondition {

        ret !javaTokC(unquote(tok.get(i+3))).contains("*");

      });

    // "bla * bla"

    jreplace(tok, "if <quoted>", "if (match($2, s, m))");

    jreplace(tok, "if match <quoted>", "if (match($3, s, m))");

    jreplace(tok, "if <id> eq <quoted>", "if (eq($2, $4))");

    tok_dropExtraCommas(tok);

    // additional translations (if necessary)

    jreplace(tok, "pcall ping {", "pcall { ping();");

    replaceKeywordBlock(tok, ") pcall",

      ") { pcall {",

      "}}");

    replaceKeywordBlock(tok,

      "pcall",

      "try {",

      "} catch (Throwable __e) { _handleException(__e); }");

    replaceKeywordBlock(tok,

      "pcall-short",

      "try {",

      "} catch (Throwable __e) { print(exceptionToStringShort(__e)); }");

    replaceKeywordBlock(tok,

      "pcall-silent",

      "try {",

      "} catch (Throwable __e) { silentException(__e); }");

    replaceKeywordBlock(tok,

      "pcall-messagebox",

      "try {",

      "} catch __e { messageBox(__e); }");

    replaceKeywordBlock(tok,

      "pcall-infobox",

      "try {",

      "} catch __e { infoBox(__e); }");

    tok = dialogHandler(tok);

    replaceKeywordBlock(tok, "exceptionToUser",

      "try {",

      "} catch (Throwable __e) { ret exceptionToUser(__e); }"); 

    if (hasCodeTokens(tok, "twice", "{"))

      replaceKeywordBlock(tok, "twice",

        "for (int __twice = 0; __twice < 2; __twice++) {",

        "}"); 

    while ((i = findCodeTokens(tok, "bench", "*", "{")) >= 0) {

      int j = findEndOfBracketPart(tok, i+4)-1;

      S time = makeVar("time");

      S v = makeVar("bench");

      S n = tok.get(i+2);

      tok.set(i, "{ long " + time + " = sysNow(); for (int " + v + " = 0; " + v + " < " + n + "; " + v + "++)");

      tok.set(i+2, "");

      tok.set(j, "} printBenchResult(sysNow()-" + time + ", " + n + "); }");

      reTok(tok, i, j+1);

    }

    replaceKeywordBlockDyn(tok,

      "time",

      new O { S[] get() {

        S var = makeVar("startTime");

        ret new S[] {

          "{ long " + var + " = sysNow(); try { ",

          "} finally { " + var + " = sysNow()-" + var + "; saveTiming(" + var + "); } }"};

      }});

    // version without { }

    replaceKeywordBlockDyn(tok,

      "time2",

      new O { S[] get() {

        S var = makeVar("startTime");

        ret new S[] {

          "long " + var + " = sysNow(); ",

          " " + var + " = sysNow()-" + var + "; saveTiming(" + var + "); "};

      }});

    // time "bla" {

    // time msg {

    replaceKeywordPlusQuotedOrIDBlock(tok,

      "time",

      new O { S[] get(L<S> tok, int i) {

        S var = makeVar("startTime");

        ret new S[] {

          "long " + var + " = sysNow(); ",

          " done2_always(" + tok.get(i+2) + ", " + var + "); "};

      }});

    if (hasCodeTokens(tok, "assertFail", "{")) {

      S var = makeVar("oops");

      replaceKeywordBlock(tok,

        "assertFail",

        "boolean " + var + " = false; try {",

        "\n" + var + " = true; } catch (Exception e) { /* ok */ } assertFalse(" + var + ");"); 

    }

    replaceKeywordBlock(tok,

      "yo",

      "try {",

      "} catch (Exception " + makeVar("e") + ") { ret false; }",

      tokcondition {

        ret neqOneOf(_get(tok, i-1), "svoid", "void");

      });

    replaceKeywordBlock(tok,

      "awtIfNecessary",

      "swingNowOrLater(r " + "{",

      "});");

    ctex(tok);

    replaceKeywordBlock(tok,

      "actionListener",

      "new java.awt.event.ActionListener() { " +

        "public void actionPerformed(java.awt.event.ActionEvent _evt) { pcall-messagebox {",

      "}}}");

    for (S keyword : ll("autocloseable", "autoCloseable"))

      /*replaceKeywordBlock(tok,

        keyword,

        "new AutoCloseable() { public void close() throws Exception {",

        "}}");*/

        replaceKeywordBlock_dyn2_legacy(tok, keyword, new O {

          S[] get(LS tok, int iOpening, int iClosing) {

            LS contents = subList(tok, iOpening+1, iClosing);

            ret new S[] {

              "new AutoCloseable() { toString { ret " + quote(shorten(maxQuineLength, trimJoin(contents))) + "; } public void close() throws Exception {",

              "}}"

            };

          }

        });

    // try answer (string, test with nempty)

    while ((i = findCodeTokens(tok, "try", "answer")) >= 0) {

      int j = findEndOfStatement(tok, i);

      S v = makeVar();

      bool needCurly = !eqGet(tok, i-2, "{");

      tok.set(i, (needCurly ? "{" : "") + " S " + v);

      tok.set(i+2, "=");

      tok.set(j-1, "; if (!empty(" + v + ")) ret " + v + "; " + (needCurly ? "}" : ""));

      reTok(tok, i, j);

    }

    // try bool[ean] (try answer with Bool type)

    while ((i = findCodeTokens(tok, "try", "boolean")) >= 0) {

      int j = findEndOfStatement(tok, i);

      S v = makeVar();

      tok.set(i, "{ Bool " + v);

      tok.set(i+2, "=");

      tok.set(j-1, "; if (" + v + " != null) ret " + v + "; }");

      reTok(tok, i, j);

    }

    // S s = bla(), return if null; => S s = bla(); if (s == null) return;

    while ((i = jfind(tok, ", return if null;")) >= 0) {

      int j = tok_findBeginningOfStatement(tok, i);

      print("Found statement " + j + "/" + i + " - " + joinSubList(tok, j-1, i+5*2-1));

      S var = getVarDeclarationName(subList(tok, j-1, i));

      replaceTokens_reTok(tok, i, i+5*2-1, "; if (" + var + " == null) ret;");

    }

    // <statement>, print "..."; => { <statement>; print("..."); }

    while ((i = jfind(tok, ", print <quoted>;")) >= 0) {

      int j = tok_findBeginningOfStatement(tok, i);

      replaceTokens_reTok(tok, i, i+4*2-1, "; print(" + tok.get(i+4) + "); }");

      tokPrepend_reTok(tok, j, "{ ");

    }

    // return if null <expression> => if (<expression> == null) return;

    while ((i = jfind(tok, "return if null")) >= 0) {

      int j = findEndOfStatement(tok, i);

      clearTokens(tok, i, i+2);

      tok.set(i+4, "((");

      tok.set(j-1, ") == null) ret;");

      reTok(tok, i, j);

    }

    // return optional (return if not null)

    while ((i = jfind_check optional(tok, "return optional <id> =")) >= 0) {

      int j = findEndOfStatement(tok, i);

      S v = tok.get(i+4);

      clearTokens(tok, i+2, i+4);

      tok.set(i, "{");

      tok.set(j-1, "; if (" + v + " != null) ret " + v + "; }");

      reTok(tok, i, j);

    }

    // try object (return if not null)

    while ((i = jfind_check object(tok, "try object")) >= 0) {

      int j = findEndOfStatement(tok, i);

      clearTokens(tok, i, i+3);

      bool isDecl = isIdentifier(get(tok, i+4)) && isIdentifier(get(tok, i+6)) && eqGet(tok, i+8, "=");

      if (isDecl) {

        S v = get(tok, i+6);

        tok.set(i, "{");

        tok.set(j-1, "; if (" + v + " != null) ret " + v + "; }");

      } else {

        S v = makeVar();

        tok.set(i, "{ O " + v + "=");

        tok.set(j-1, "; if (" + v + " != null) ret " + v + "; }");

      }

      reTok(tok, i, j);

    }

    // try Int i = ...; (return if not null, shorter version of "try object")

    while ((i = jfind(tok, "try <id> <id> =")) >= 0) {

      int j = findEndOfStatement(tok, i);

      S type = tok.get(i+2), v = tok.get(i+4);

      tok.set(i, "{");

      tok.set(j-1, "; if (" + v + " != null) ret " + v + "; }");

      reTok(tok, i, j);

    }

    // debug print ...; => if (debug) print(...);

    while ((i = jfind(tok, "debug print")) >= 0) {

      int j = findEndOfStatement(tok, i);

      replaceTokens(tok, i, i+4, "if (debug) print(");

      tok.set(j-1, ");");

      reTok(tok, i, j);

    }

    functionReferences(tok);

    tok_expandLPair(tok);

    tok_expandPairL(tok);

    tok_expandLT3(tok);

    quicknew2(tok);

    // X x = nu(+...)  =>  X x = nu X(+...)

    jreplace(tok, "<id> <id> = nu(+", "$1 $2 = nu $1(+");

    // X x = nu(a := ...)  =>  X x = nu X(a := ...)

    jreplace(tok, "<id> <id> = nu(<id> :=", "$1 $2 = nu $1($6 :=");

    tok_expandVarCopies(tok); // AFTER the lines just above

    tok_replaceColonEqualsSyntax(tok); // ditto

    tok_unpair(tok);

    tok_cachedFunctions(tok);

    tok_simplyCachedFunctions(tok);

    tok_optPar(tok);

    throwFailEtc(tok);

    tok_typeAA(tok, pairClasses);

    // do this after expanding sclass etc.

    tok = tok_expandStarConstructors(tok);

    tok_kiloConstants(tok);

    //tok_colonMessages(tok);

    while ((i = jfind(tok, "shit:")) >= 0) {

      int j = tok_findEndOfStatement(tok, i);

      tok.set(i+2, "(");

      tok.set(j-1, ");");

      reTok(tok, i, j);

    }

    jreplace("shit(", "ret with print(");

    tok_virtualTypes(tok);

    tok_autoLongConstants(tok);

    // common misordering of type arguments

    jreplace("boolean <A>", "<A> boolean");

    tok_unimplementedMethods(tok);

    tok_switchableFields(tok);

    tok_autoDisposeFields(tok);

    tok_shortVisualize(tok);

    tok_whileGreaterThan(tok);

    tok_ifThenEnd(tok);

    tok_autoInitVars(tok);

    tok_fixBadTypeParameterOrder(tok);

    // shortened method declarations BEFORE standardFunctions

    jreplace(tok, "svoid", "static void");

    jreplace(tok, "void <id> {", "$1 $2() {");

    jreplace(tok, "void <id> thread {", "$1 $2() thread {");

    jreplace(tok, "String <id> {", "$1 $2() {");

    jreplace(tok, "Object <id> {", "$1 $2() {");

    jreplace(tok, "List <id> {", "$1 $2() {");

    namedThreads(tok);

    threads(tok);

    //tok_maxEquals(tok);

    tok_questionDot(tok);

    tok_embeddedFunctions(tok);

    jreplace("<id>[] <id> = new[", "$1[] $4 = new $1[");

    jreplace("<id> ifNull =", "if ($1 == null) $1 =");

    jreplace("class <id> extends <id><.<id>>", "class $2 extends $4<$2.$7>");

    tok_once(tok); // must come before next line so you can combine them

    tok_ifRecordMatch(tok);

    jreplace(tok, "<id> ||=", "$1 = $1 ||");

    // magicValue followed by a numerical constant

    jreplace("magicValue", "", (_tok, _i) -> {

      S t = _get(_tok, _i+3);

      ret eqOneOf(t, ".", "-") || isInteger(t);

    });

    // end of local stuff

    tok_processMetaBlocks(tok, metaCodeAllowed());

    if (metaCodeAllowed()) runMetaTransformers(tok);

    same = tok_sameTest(tok, before);

    /*if (!same)

      print("local not same " + safety + " (" + l(tok) + " tokens)");*/

    if (safety++ >= 10) {

      printSources(tok);

      fail("safety 10 error!");

    }

  } while (!same);

  ret tok;

}

static L<S> reTok_include(L<S> tok, int i, int j) {

  ret reTok_modify(tok, i, j, f localStuff1);

}

static L<S> includeInMainLoaded_reTok(L<S> tok, int i, int j) {

  ret reTok_include(tok, i, j);

}

static L<S> stdstuff(L<S> tok) {

  //if (++level >= 10) fail("woot? 10");

  print("stdstuff!");

  int i;

  new L<S> ts;

  tok_findTranslators(tok, ts);

  if (nempty(ts))

    print("DROPPING TRANSLATORS: " + structure(ts));

  print('quickmain); tok = quickmain(tok);

  print('includes); tok = tok_processIncludes(tok);

  print('conceptsDot); if (processConceptsDot(tok))

    tok = tok_processIncludes(tok);

  //print('starConstructors); tok = tok_expandStarConstructors(tok);

  // drop Java 8 annotations since we're compiling for Java 7

  jreplace(tok, "@Nullable", "");