!1006722

// TODO: deprecate (remove) the "f <id>" syntax

// TODO: skip ifclass blocks in findFunctionInvocations.

//       after main loop completes (including standard functions+

//       classes), run tok_ifclass and if any change, run main loop

//       again.

//       This will allow clean handling of functions like "unnull"

//       which right now _always_ include Symbol (because unnull

//       references emptySymbol inside an ifclass Symbol block).

// Note: Before editing this, transpile #7

// Use 'Very fresh' (R-transpile doesn't seem to work) or Q/M if you haven't screwed yourself currently

// by making your transpiler unable to transpile itself... or

// something

// Note: Don't try hijackPrint, it doesn't seem to work in here

// (because of the way it is called by e.g. transpileForServer_returnPair?)

/* functions to possibly edit when creating/changing JavaX constructs:

  tok_pingify

*/

// new JavaParser includes

!include once #1025814

!include once #1025813

!include once #1025802 // optimized findCodeTokens

set flag OurSubLists.

set flag NoIllegalAccesses.

set flag javaTokForJFind_array_simpleCache.

set flag tok_moveImportsUp_skipMostReToks.

set flag AllowMetaCode.

set flag jreplace_performing. // record replacements

//set flag tok_forUnnull_debug.

// ifdef in cached includes? => i think cached includes still contain the ifdefs, so it's fine. tok_ifdef is not called within localStuff1

set flag noAI.

set flag isQuoted_dirty. // should be OK, we're dealing with tokens

static bool autoQuine = true;

static int maxQuineLength = 80;

sbool assumeTriple = true;

sbool quickInstanceOfEnabled; // interferes with things

sbool debug_jreplace;

// _registerThread usually costs nothing because we need

// the registerWeakHashMap mechanism anyway for ping().

// Anyway - no forced functions for now :)

static L<S> functionsToAlwaysInclude = ll(

  //"_registerThread",

  //"asRuntimeException"

);

// classes with two type parameters that can written with just one

// e.g. Pair<S> => Pair<S, S>

static Set<S> pairClasses = lithashset("Pair", "Either", "Map", "AbstractMap", "HashMap", "TreeMap", "LinkedHashMap", "MultiMap", "CompactHashMap", "WrappedMap", "F1", "IF1", "AllOnAll", "AllOnAllWithUpdates", "MultiSetMap", "AutoMap", "ValueOnDemandMap", "NavigableMap", "SortedMap");

// classes with 3 type parameters that can written with just one

// e.g. T3<S> => T3<S, S, S>

static Set<S> tripleClasses = lithashset("T3", "IF2", "F2", "IVF3", "VF3");

sS transpilingSnippetID;

//static new AtomicInteger varCount;

static new ThreadLocal<AtomicInteger> varCountByThread;

static Map<S, S> snippetCache = syncMap();

sbool useIndexedList2 = false, useTokenIndexedList = true;

sbool opt_javaTok = true;

static bool cacheStdFunctions = true, cacheStdClasses = true;

static new HashMap<Long, CachedInclude> cachedIncludes;

static ExecutorService executor;

static L lclasses;

static long startTime, lastPrint;

// These variables have to be cleared manually for each transpilation

static new HashSet<Long> included;

static NavigableSet<S> definitions = ciSet();

static new HashMap<S> rewrites;

static new HashSet<S> shouldNotIncludeFunction; // this verifies after the fact that the function was not included

static new HashSet<S> shouldNotIncludeClass;

static new HashSet<S> doNotIncludeFunction; // this actually prevents the function from being included

static new SS functionToPackage;

static new HashSet<S> doNotIncludeClass;

static new HashSet<S> needLatest; // this forces a function to be included

static new HashSet<S> addedFunctions;

static new HashSet<S> addedClasses;

static new HashSet<S> hardFunctionReferences;

static new HashSet<S> mapLikeFunctions;

static new HashSet<S> lambdaMapLikeFunctions;

static new HashSet<S> curry1LikeFunctions;

static new HashSet<S> mapMethodLikeFunctions;

static new HashSet<S> nuLikeFunctions;

static new HashSet<S> getLikeFunctions; // name is slightly misleading. this turns a pre-argument into a fieldLambda

static new HashSet<S> lambdaMethod0LikeFunctions; // we're getting better again with the names (maybe)

static new HashSet<S> lambda0LikeFunctions;

static SS extraStandardFunctions;

sbool quickmainDone1, quickmainDone2;

static new TreeSet<S> libs;

sS mainBaseClass, mainPackage, mainClassName;

sbool localStuffOnly; // for transpiling a fragment

sbool asInclude; // for transpiling an include (auto-close scopes, enable all standard class ifclass [todo])

sbool allowMetaCode = false; // run any embedded meta code

static LS metaPostBlocks, metaTransformers, localTransformers;

sbool dontPrintSource;

sbool dontLoadCachedIncludesFromVM; // for benchmarking

static new HashSet<S> haveClasses;

sclass CachedInclude {

  S javax;

  Future<S> java;

  S realJava;

  long snippetID;

  *() {}

  *(long *snippetID) {}

  S java() {

    ret realJava != null ? realJava : getFuture(java);

  }

  Future<S> javaFuture() {

    ret realJava != null ? nowFuture(realJava) : java;

  }

  void clean {

    if (java != null) {

      realJava = getFuture(java);

      java = null;

    }

  }

}

p {

  startTime = lastPrint = sysNow();

  pcall {

    if (!dontLoadCachedIncludesFromVM)

      vmKeepWithProgramMD5_get('cachedIncludes);

  }

  executor = Executors.newFixedThreadPool(numberOfCores());

  transpilingSnippetID = or(getThreadLocal((ThreadLocal<S>) getOpt(javax(), 'transpilingSnippetID)), transpilingSnippetID);

  //print(+transpilingSnippetID);

  set transpileRaw_dontCopyFromCreator;

  fO oldPrint = or(print_byThread()!, f print_raw);

  temp tempInterceptPrint(new F1<S, Bool>() {

    Bool get(S s) {

      long now = sysNow();

      long time = now-lastPrint; // -startTime;

      lastPrint = now;

      callF(oldPrint, "[" + formatInt(time/1000, 2) + ":" + formatInt(time % 1000, 3) + "] " + s);

      false;

    }

  });

  try {

    _main();

  } finally {

    interceptPrintInThisThread(oldPrint);

    if (executor != null) executor.shutdown();

    executor = null;

    localStuffOnly = false;

    asInclude = false;

  }

}

svoid _main() ctex {

  if (sameSnippetID(programID(), defaultJavaXTranslatorID())) setDefaultJavaXTranslatorID(#7);

  //reTok_modify_check = true;

  //if (useIndexedList) findCodeTokens_debug = true;

  javaTok_opt = opt_javaTok;

  //findCodeTokens_indexed = findCodeTokens_unindexed = 0;

  findCodeTokens_bails = findCodeTokens_nonbails = 0;

  javaTok_n = javaTok_elements = 0;

  S in = loadMainJava();

  print("759 STARTING " + identityHashCode(main.class));

  // clear things

  includeInMainLoaded_magicComment = null;

  included.clear();

  definitions.clear();

  rewrites.clear();

  // XXX definitions.add("SymbolAsString");

  shouldNotIncludeFunction.clear();

  shouldNotIncludeClass.clear();

  doNotIncludeFunction.clear();

  functionToPackage.clear();

  needLatest.clear();

  doNotIncludeClass.clear();

  addedFunctions.clear();

  addedClasses.clear();

  hardFunctionReferences.clear();

  mapLikeFunctions = cloneHashSet(tok_mapLikeFunctions());

  lambdaMapLikeFunctions = new HashSet;

  curry1LikeFunctions = new HashSet;

  mapMethodLikeFunctions = cloneHashSet(tok_mapMethodLikeFunctions());

  nuLikeFunctions.clear();

  getLikeFunctions.clear();

  lambdaMethod0LikeFunctions.clear();

  lambda0LikeFunctions.clear();

  extraStandardFunctions = new HashMap;

  libs.clear();

  mainBaseClass = mainPackage = mainClassName = null;

  varCountByThread.set(null);

  quickmainDone1 = quickmainDone2 = false;

  metaPostBlocks = new L;

  metaTransformers = new L;

  localTransformers = new L;

  dontPrintSource = false;

  defaultMaxQuineLength_value = defaultMaxQuineLength_defaultValue;

  debug_jreplace = false;

  haveClasses.clear();

  //L ts = findTranslators(toLines(join(tok)));

  //print("Translators in source at start: " + structure(ts));

  L<S> tok = jtok(in);

  try {

    tok_definitions(tok);

    // add m { }

    if (!localStuffOnly && !hasCodeTokens(tok, "m", "{") && !hasCodeTokens(tok, "main", "{") && !hasCodeTokens(tok, "class", "main")) {

      if (l(tok) == 1) tok = singlePlusList(first(tok), dropFirst(javaTok("m {}")));

      else {

        replaceTokens_reTok(tok, 1, 2, "m {\n\n" + unnull(get(tok, 1)));

        replaceTokens_reTok(tok, l(tok)-2, l(tok)-1, unnull(get(tok, l(tok)-2)) + "}");

      }

      tok_moveImportsUp(tok);

    }

    // standard translate

    //ts = findTranslators(toLines(join(tok)));

    //print("Translators in source: " + structure(ts));

    if (tok_hasTranslators(tok))

      tok = jtok(defaultTranslate(join(tok)));

    //print("end of default translate");

    //print(join(tok));

    //tok_autoCloseBrackets(tok);    

    tok_metaTransformNow(tok);

    tok_processEarlyIncludes(tok);

    tok_earlyGeneralStuff(tok);

    tok = tok_processIncludes(tok); // before standard functions

    if (processConceptsDot(tok))

      tok = tok_processIncludes(tok);

    tok = localStuff1(tok);

   if (!localStuffOnly) {

    int safety = 0;

    boolean same;

    do { // BIG LOOP

      ping();

      LS before = cloneList(tok);

      // do the non-local stuff (flags and rewrites, things that correlate across includes like tok_selfType)

      // to allow crazy stuff like "nonStatic !include #someStaticClass"

      tok_processEarlyIncludes(tok);

      jreplace(tok, "nonStatic static", "");

      tok_selfType(tok);

      tok_mainClassNameAndPackage(tok);

      tok_definitions(tok);

      tok_ifndef(tok);

      tok_ifdef(tok);

      defineMapLikesEtc(tok);

      if (tok_applyAllXLikeFunctions(tok)) {

        functionReferences(tok);

        lambdaReferences(tok);

      }

      tok_dropExtraCommas(tok); // from e.g. tok_applyMapMethodLikeFunctions

      tok_delegateTo(tok);

      tok_replaceWith(tok);

      tok_findAndClearRewrites(tok);

      tok_processRewrites(tok);

      tok_multiTypeArguments_v2(tok);

      // main bla(...) => mainClassName.bla(...)

      // or main ClassName => main.ClassName

      //jreplace(tok, "main <id>(", mainClassName() + ".$2(");

      grabImportedStaticFunctions(tok);

      jreplace_dyn_allowNull(tok, "main <id>", (_tok, cIdx, end) -> {

        S fname = _tok.get(cIdx+2);

        bool isClass = haveClasses.contains(fname);

        printVars expandMainRef(+fname, +isClass);

        if (isClass || eqGet(_tok, cIdx+4, "(")) {

          S pkg = functionToPackage.get(fname);

          printVars expandMainRef(+pkg);

          S call = "." + fname;

          ret or(pkg, mainClassName()) + call;

        }

        null;

      });

      try {

        if (safety == 0) tok = quickmain(tok);

      } catch e {

        printSources(tok);

        rethrow(e);

      }

      tok_collectMetaPostBlocks(tok, metaPostBlocks);

      tok_collectTransformers(tok, metaTransformers);

      tok_collectTransformers(tok, localTransformers, "meta-transformLocal");

      tok_metaTransformNow(tok);

      // Hack to allow DynModule to reimplement _registerThread

      /*if (tok.contains("DynModule") && !addedClasses.contains("DynModule"))

        addStandardClasses_v2(tok);*/

      defineExtraSF(tok);

      tok = standardFunctions(tok);

      tok = stdstuff(tok); // all the keywords, standard

      S diff;

      long startTime = now();

      //diff = unidiff(before, join(tok));

      //print("unidiff: " + (now()-startTime) + " ms");

      //same = eq(diff, "");

      same = eq(tok, before);

      if (!same) {

        print("Not same " + safety + ".");

        //print(indent(2, diff));

      }

      if (safety++ >= 10) {

        //print(unidiff(before, join(tok)));

        printSources(tok);

        fail("safety 10 error!");

      }

    } while (!same); // END OF BIG LOOP

    print("Post.");

    if (mainBaseClass != null) {

      jreplace1(tok, "class main", "class main extends " + mainBaseClass);

      mainBaseClass = null;

    }

    print('moveImportsUp); tok_moveImportsUp(tok);

    print('Indexing); tok = indexTokenList(tok);

    // POST-PROCESSING after stdstuff loop

    if (transpilingSnippetID != null)

      jreplace_dyn(tok, "class whatever", 

        func(L<S> tok, int cIndex) -> S {

          pcall { ret "class " + stringToLegalIdentifier(getSnippetTitle(transpilingSnippetID)); }

          ret "class Whatever";

        });

    //print("Type<A, A>"); // Type<A> to Type<A, A>

    print('extendClasses); tok = extendClasses(tok);

    print('libs); libs(tok);

    print('sourceCodeLine); sourceCodeLine(tok);

    tok_overridableFunctionDefs(tok, null);

    // escaping for lambdas

    //jreplace(tok, "-=>", "->");

    // Stuff that depends on the list of inner classes (haveClasses)

    haveClasses = haveClasses_actual(tok);

    print('innerClassesVar); innerClassesVar(tok, haveClasses);

    fillVar_transpilationDate(tok);

    haveClasses_addImported(tok, haveClasses);

    print('ifclass); tok_ifclass(tok, haveClasses);

    print('slashCasts); slashCasts(tok, haveClasses);

    print('newWithoutNew); newWithoutNew(tok, haveClasses);

    if (!assumeTriple) {

      print('Triple);

      if (tok.contains("Triple") && !haveClasses.contains("Triple")) {

        jreplace(tok, "Triple", "T3");

        haveClasses.remove("Triple");

        haveClasses.add("T3");

        slashCasts(tok, lithashset("T3"));

        tok_quickInstanceOf(tok, lithashset("T3"));

      }

    }

    if (hasDef("SymbolAsString"))

      jreplace(tok, "Symbol", "String");

    // using non-lazy version now for cleanImports

    print('classReferences); expandClassReferences/*_lazy*/(tok, haveClasses);

    if (metaCodeAllowed()) runMetaPostBlocks(tok);

    // Error-checking

    print("Error-checking"); 

    Set<S> functions = new HashSet(findFunctions(tok));

    for (S f : hardFunctionReferences)

      if (!functions.contains(f))

        fail("Function " + f + " requested, but not supplied");

    print('autoImports); tok = autoImports(tok); // faster to do it at the end

    if (hasDef("cleanImports"))

      tok_cleanImports(tok);

    if (includeInMainLoaded_magicComment != null) {

      int i = lastIndexOfStartingWith(tok, includeInMainLoaded_magicComment);

      if (i >= 0) tok.set(i, dropPrefix(includeInMainLoaded_magicComment, tok.get(i)));

    }

    print("definitions=" + sfu(definitions));

    if (containsOneOf(definitions, "allpublic", "reparse", "PublicExceptTopClass")) {

      // Fire up the Java parser & pretty printer!

      print(containsIC(definitions, "allpublic")? "Making all public." : "Reparsing.");

      //try {

        S src = join(tok);

        try {

          if (containsIC(definitions, "allpublic"))

            src = javaParser_makeAllPublic(src);

          else if (containsIC(definitions, "PublicExceptTopClass"))

            src = javaParser_makeAllPublic(src, notTopLevelClassDecl := true);

          else if (containsIC(definitions, "keepComments"))

            src = javaParser_makeAllPublic_keepComments(src);

          else {

            print("javaParser_reparse_keepComments. size=" + l(src));

            src = javaParser_reparse_keepComments(src);

            print("size=" + l(src));

          }

        } catch e {

          S _src = src;

          S errorContext = extractAndPrintJavaParseError(_src, e);

          File f = transpilerErrorSourceFile();

          saveTextFileVerbose(f, src);

          dontPrintSource = true;

          fail("Java parsing error:" + errorContext + innerMessage(e)); // drop the long nested parser stacktraces

        }

        tok = jtok(src);

      /*} catch e {

        S src = join(tok);

        if (!dontPrintSource)

          print(src);

        print(f2s(saveProgramTextFile("error.java", src)));

        throw rethrow(e);

      }*/

    }

    // Do this after JavaParser (because it doesn't like package after class)

    if (mainPackage != null) {

      print('mainPackage);

      tokPrepend(tok, 1, "package " + mainPackage + ";\n");

      reTok(tok, 1, 2);

    }

    if (mainClassName != null) {

      print('mainClassName);

      jreplace(tok, "class main", "class " + mainClassName);

      jreplace(tok, "main.class", mainClassName + ".class");

      //tokPrepend(tok, 1, "class main {}\n"); // so main.class is generated and compiler sanity checks succeed. we can later skip it in the JavaXClassLoader

    }

    if (nempty(libs)) {

      print("Adding libs: " + libs);

      tok.add(concatMap_strings(func(S s) -> S { "\n!" + s }, libs));

    }

   } // if (!localStuffOnly)

  } catch e {

    if (!dontPrintSource)

      printSources(tok);

    S src = join(tok);

    print(f2s(saveProgramTextFile("error.java", src)));

    throw rethrow(e);

  }

  /*if (useIndexedList)

    print("Indexed/unindexed lookups: " + findCodeTokens_indexed + "/" + findCodeTokens_unindexed + ", lists made: " + IndexedList2.instances);

  print("findCodeToken bails: " + findCodeTokens_bails + "/" + findCodeTokens_nonbails);

  print("javaToks: " + javaTok_n + "/" + javaTok_elements);*/

  print("Saving.");

  // for dexcompile.php

  if (mainClassName != null)

    tokPrepend(tok, 0, "//FILENAME: " 

      + (mainPackage != null ? mainPackage.replace(".", "/") + "/" : "") + mainClassName + ".java\n");

  if (mainJava != null)

    mainJava = join(tok);

  else if (tok.contains("package"))

    splitJavaFiles(tok);

  else

    saveMainJava(tok);

}

static LS localStuff1(LS tok) {

  int safety = 0, i;

  boolean same;

  tok = indexTokenList(tok);

  tok_scopes(tok, autoCloseScopes := true);

  do {

    ping();

    LS before = cloneList(tok);

    //print("localStuff loop " + safety);

    earlyStuff(tok);

    // EARLY local stuff goes here

    tok_dropMetaComments(tok);

    tok_earlyGeneralStuff(tok);

    conceptDeclarations(tok);

    tok_recordDecls(tok);

    tok = multilineStrings(tok);

    tok_singleQuoteIdentifiersToStringConstants(tok);

    tok_inStringEvals(tok);

    tok_listComprehensions(tok);

    tok_eventFunctions(tok);

    tok_for_single(tok);

    tok_for_unpair(tok); // Do this...

    //tok_doubleFor_v2(tok);

    tok_doubleFor_v3(tok); // ...before this

    tok_forUnnull(tok);

    tok_ifCast(tok);

    forPing(tok);

    tok_directSnippetRefs(tok);

    quicknu(tok);

    //tok_juxtaposeCalls(tok);

    jreplace(tok, "LLS", "L<LS>");

    jreplace(tok, "LS", "L<S>");

    jreplace(tok, "ES", "Ext<S>");

    jreplace(tok, "ExtS", "Ext<S>");

    jreplace(tok, "WeakRef", "WeakReference");

    jreplace(tok, "dispose <id>;", "{ cleanUp($2); $2 = null; }");

    tok_doPing(tok);

    replaceKeywordBlock(tok,

      "swing",

      "{ swing(r {",

      "}); }");

    replaceKeywordBlock(tok,

      "androidUI",

      "{ androidUI(r {",

      "}); }");

    replaceKeywordBlock(tok,

      "withDBLock",

      "{ withDBLock(r {",

      "}); }");

    replaceKeywordBlock(tok, "afterwards", "temp tempAfterwards(r {", "});");

    for (S keyword : ll("tokcondition", "tokCondition"))

      replaceKeywordBlock(tok,

        keyword,

        "new TokCondition { public bool get(final L<S> tok, final int i) {",

        "}}");

    jreplace(tok, "synced <id>", "synchronized $2");

    jreplace(tok, "sync <id>", "synchronized $2");

    jreplace(tok, "synchronized {", "synchronized(this) {");

    replaceKeywordBlock(tok, "answer",

      "static S answer(S s) {\nfinal new Matches m;\n",

      "\nret null;\n}");

    replaceKeywordBlock(tok, "static-pcall",

      "static { pcall {",

      "}}");

    replaceKeywordBlock(tok, "loading",

      "{ temp tempShowLoadingAnimation(); ",

      "}");

    replaceKeywordPlusQuotedBlock(tok, "loading",

      new O { S[] get(L<S> tok, int i) {

        S text = tok.get(i+2);

        ret new S[] {

          "{ temp tempShowLoadingAnimation(" + text + "); ",

          "}" };

      }});

    while ((i = jfind(tok, "visualize as")) >= 0) {

      int j = tok_findEndOfStatement(tok, i); // expression, rather

      tok.set(i+2, "{ ret");

      tok.set(j-1, "; }");

      reTok(tok, i, j);

    }

    for (S pat : ll("visual {", "visualize {"))

      jreplace(tok, pat, "public JComponent visualize() {", tokCondition_beginningOfMethodDeclaration());

    jreplace(tok, "visualize2 {", "JComponent visualize2() {", tokCondition_beginningOfMethodDeclaration());

    replaceKeywordBlock(tok, "start-thread-printDone", "start-thread {", "printDone(); }");

    replaceKeywordBlock(tok, "start-thread", [[start { thread "Start" { temp enter(); pcall {]], "}}}");

    jreplace(tok, "start {", "void start() ctex { super.start();", tokCondition_beginningOfMethodDeclaration());

    var notVoidMethod = tokcondition {

      ret neqGet(tok, i-1, "void");

    };

    // run { ... } => public void run() { ... }

    // same with close

    jreplace(tok, "run {", "public void run() tex {", notVoidMethod);

    jreplace(tok, "close {", "public void close() ctex {", notVoidMethod);

    replaceKeywordBlock(tok, "html",

      "static O html(S uri, fMap<S, S> params) ctex " + "{\n", "}");

    replaceKeywordBlock(tok, "afterVisualize",

      "visualize { JComponent _c = super.visualize();",

      "ret _c; }");

    replaceKeywordBlock(tok, "enhanceFrame",

      "void enhanceFrame(Container f) { super.enhanceFrame(f);",

      "}");

    if (assumeTriple)

      jreplace(tok, "Triple", "T3");

    tok_shortFinals(tok);

    tok_moduleClassDecls(tok);

    jreplace(tok, "static sync", "static synchronized");

    jreplace(tok, "sclass", "static class");

    jreplace(tok, "fclass", "final class");

    jreplace(tok, "fsclass", "final static class");

    jreplace(tok, "srecord", "static record");

    jreplace(tok, "strecord", "static transformable record");

    jreplace(tok, "record noeq", "noeq record");

    jreplace(tok, "asclass", "abstract static class");

    jreplace(tok, "sinterface", "static interface");

    jreplace(tok, "ssynchronized", "static synchronized");

    jreplace(tok, "ssvoid", "static synchronized void");

    jreplace(tok, "sbool", "static bool");

    jreplace(tok, "fbool", "final bool");

    jreplace(tok, "sint", "static int");

    jreplace(tok, "snew", "static new");

    jreplace(tok, "sv <id>", "static void $2");

    jreplace(tok, "pvoid", "public void");

    // "sS" => static S

    jreplace(tok, "sS", "static S");

    // "sO" => static O

    jreplace(tok, "sO", "static O");

    // "sL" => static L

    jreplace(tok, "sL", "static L");

    // "toString {" => "public S toString() {"

    jreplace(tok, "toString {", "public S toString() {");

    jreplace(tok, "Int", "Integer");

    jreplace(tok, "Bool", "Boolean");

    jreplace(tok, "BigInt", "BigInteger");

    jreplace(tok, "Char", "Character");

    jreplace(tok, "Sym", "Symbol");

    jreplace(tok, "SymSym", "SymbolSymbol");

    jreplace(tok, "SS", "Map<S>");

    jreplace(tok, "SymbolSymbol", "Map<Symbol>");

    jreplace(tok, "CISet", "Set<S>");

    jreplace(tok, "MapSO", "Map<S, O>");

    jreplace(tok, "ByName<", "IF1<S, ");

    jreplace(tok, "ITransform<<id>>", "IF1<$3>");

    jreplace(tok, "ITransform", "IF1");

    jreplace(tok, "PairS", "Pair<S>");

    jreplace(tok, "LPairS", "L<Pair<S>>");

    jreplace(tok, "T3S", "T3<S>");

    jreplace(tok, "F1S", "F1<S>");

    jreplace(tok, "ItIt", "IterableIterator");

    jreplace(tok, "CloseableItIt", "CloseableIterableIterator");

    jreplace(tok, "class <id> > <id>", "class $2 extends $4", tokCondition {

      S t = _get(tok, i+1+4*2);

      ret eq(t, "{") || isIdentifier(t);

    });

    jreplace(tok, "class <id> > <id><<id>> {", "class $2 extends $4  $5 $6 $7 {");

    jreplace(tok, "ISegmenter", "IF1<BufferedImage, L<Rect>>");

    // IPred<A, B> => IF2<A, B, Bool>

    jreplace(tok, "IPred<<id>, <id>>", "IF2<$3, $5, Bool>");

    // IPred<A> => IF1<A, Bool>

    jreplace(tok, "IPred<<id>>", "IF1<$3, Bool>");

    jreplace(tok, "IPred<<id>[]>", "IF1<$3[], Bool>");

    // Proposition => WithReasoning<S> (should we really define this?)

    jreplace("Proposition", "WithReasoning<S>");

    replaceKeywordBlock(tok, "print exceptions",

      "{ try {",

      "} on fail __e { printStackTrace(__e); } }");

    // "on fail {" => "catch (Throwable _e) { ... rethrow(_e); }"

    replaceKeywordBlock(tok, "on fail",

      "catch (Throwable _e) {",

      "\nthrow rethrow(_e); }");

    // "on fail e {" => "catch (Throwable e) { ... rethrow(e); }"

    replaceKeywordBlock_dyn2_legacy(tok, "on fail <id>", new O {

      S[] get(LS tok, int iOpening, int iClosing) {

        S var = tok.get(iOpening-2);

        ret new S[] {

          "catch (Throwable " + var + ") {",

          "\nthrow rethrow(" + var + "); }"

        };

      }

    });

    // "on fail Type e {" => "catch (Type e) { ... rethrow(e); }"

    replaceKeywordBlock_dyn2_legacy(tok, "on fail <id> <id>", new O {

      S[] get(LS tok, int iOpening, int iClosing) {

        S type = tok.get(iOpening-4);

        S var = tok.get(iOpening-2);

        ret new S[] {

          "catch (" + type + " " + var + ") {",

          "\nthrow " + var + "; }"

        };

      }

    });

    // "catch {" => "catch (Throwable _e) {"

    jreplace(tok, "catch {", "catch (Throwable _e) {");

    jreplace_dyn(tok, "catch print {", -> {

      S var = makeVar();

      ret "catch \*var*/ { printStackTrace(\*var*/);";

    });

    // "catch print e {" => "catch e { printStackTrace(e); "

    jreplace(tok, "catch print <id> {", "catch $3 { printStackTrace($3);");

    // "catch print short e {" => "catch e { printExceptionShort(e); "

    jreplace(tok, "catch print short <id> {", "catch $4 { printExceptionShort($4);");

    // "catch X e {" => "catch (X e) {"

    jreplace(tok, "catch <id> <id> {", "catch ($2 $3) {");

    // "catch e {" => "catch (Throwable e) {" (if e is lowercase)

    jreplace(tok, "catch <id> {", "catch (Throwable $2) {", tokcondition {

      S word = tok.get(i+3);

      ret startsWithLowerCaseOrUnderscore(word);

    });

    jreplace(tok, "+ +", "+", tokcondition {

      //printStructure("++: ", subList(tok, i-1, i+6));

      if (empty(_get(tok, i+2))) ret false; // no space between the pluses

      if (empty(_get(tok, i)) && eq("+", _get(tok, i-1))) ret false;  // an actual "++" at the left

      if (empty(_get(tok, i+4)) && eq("+", _get(tok, i+5))) ret false;  // an actual "++" at the right

      //print("doing it");

      ret true;

    });

    // single underscore (not allowed in Java anymore) to double underscore

    jreplace(tok, "_", "__");

    // [stdEq] -> implementation of equals() and hashCode()

    jreplace(tok, "[stdEq]",

      "public bool equals(O o) { ret stdEq2(this, o); }\n" +

      "public int hashCode() { ret stdHash2(this); }");

    // [stdToString] -> toString { ret stdToString(this); }

    jreplace(tok, "[stdToString]",

      "toString { ret stdToString(this); }");

    // [concepts] "concept.field!" for dereferencing references

    // "<id>!", "?!" etc.

    jreplace(tok, "*!", "$1.get()", tokcondition {

      S l = tok.get(i+1);

      if (!(isIdentifier(l) || eqOneOf(l, ")", "?"))) false;

      if (tok.get(i+2).contains("\n")) false; // no line break between <id> and !

      if (nempty(tok.get(i+4))) true; // space after = ok

      S t = _get(tok, i+5);

      if (t == null) ret false;

      if (isIdentifier(t) || eqOneOf(t, "=", "(")) false;

      true;

    });

    jreplace(tok, "for (<id> :", "for (var $3 :");

    jreplace(tok, "for (<id> <id>)", "for ($3 $4 : list($3))");

    jreplace(tok, "for (final <id> <id>)", "for (final $4 $5 : list($4))");

    jreplace(tok, "ret", "return", tokCondition {

      if (eqGetOneOf(tok, i-1, "int", "return")) false;

      S next = _get(tok, i+3);

      if (eq(next, ".")) ret isInteger(_get(tok, i+5)); // e.g. "ret .1", but not "ret.bla"

      ret !eqOneOf(next, "=", ")");

    });

    // "continue unless", "break unless"

    for (S phrase : ll("continue unless", "break unless"))

      while ((i = jfind(tok, phrase)) >= 0) {

        S keyword = tok.get(i);

        int j = scanOverExpression(tok, getBracketMap(tok), i+4, ";");

        replaceTokens(tok, i, i+4, "{ if (!(");

        tok.set(j, ")) " + keyword + "; }");

        reTok(tok, i, j+1);

      }

    // S s = bla(), return if null; => S s = bla(); if (s == null) return;

    // same with continue, break

    while ((i = jfind(tok, ", <id> if null;", tokCondition {

      ret eqOneOf(tok.get(i+3), "return", "ret", "continue", "break");

    })) >= 0) {

      S cmd = tok.get(i+2);

      int j = tok_findBeginningOfStatement(tok, i);

      print("Found statement " + j + "/" + i + " - " + joinSubList(tok, j-1, i+5*2-1));

      S var = getVarDeclarationName(subList(tok, j-1, i));

      replaceTokens_reTok(tok, i, i+5*2-1, "; if (" + var + " == null) " + cmd + ";");

    }

    // S s = bla(), return false if null; => S s = bla(); if (s == null) return false;

    // (with false being any identifier)

    while ((i = jfind(tok, ", return <id> if null;")) >= 0) {

      S returnValue = tok.get(i+4);

      int j = tok_findBeginningOfStatement(tok, i);

      S var = getVarDeclarationName(subList(tok, j-1, i));

      replaceTokens_reTok(tok, i, i+6*2-1, "; if (" + var + " == null) return " + returnValue + ";");

    }

    // "continue if", "break if"

    for (S phrase : ll("continue if", "break if"))

      while ((i = jfind(tok, phrase)) >= 0) {

        S keyword = tok.get(i);

        int j = scanOverExpression(tok, getBracketMap(tok), i+4, ";");

        replaceTokens(tok, i, i+4, "{ if (");

        tok.set(j, ") " + keyword + "; }");

        reTok(tok, i, j+1);

      }

    // return unless set <var>; => { if (var) return; set var; }

    jreplace(tok, "return unless set <id>;", "{ if ($4) return; set $4; }");

    // set x; => x = true;

    // yeah it doesn't save many characters, but I like it

    jreplace(tok, "set <id>;", "$2 = true;", tokCondition {

      ret !eqGet(tok, i-1, "unless");

    });

    // set !x; => x = false; (did this one day and thought yes it makes sense in a way)

    jreplace(tok, "set !<id>;", "$3 = false;");

    // unset x; => x = false;

    jreplace(tok, "unset <id>;", "$2 = false;");

    // "return if"

    while ((i = jfind(tok, "return if")) >= 0) {

      int j = scanOverExpression(tok, getBracketMap(tok), i+4, ";");

      replaceTokens(tok, i, i+4, "{ if (");

      tok.set(j, ") return; }");

      reTok(tok, i, j+1);

    }

    // "return unless"

    while ((i = jfind(tok, "return unless")) >= 0) {

      int j = scanOverExpression(tok, getBracketMap(tok), i+4, ";");

      replaceTokens(tok, i, i+4, "{ if (!(");

      tok.set(j, ")) return; }");

      reTok(tok, i, j+1);

    }

    tok_ifNullAssign(tok);

    // "return <id> if"

    // "return with <statement>" / "continue with <statement>" / "break with <statement>"

    while ((i = jfind(tok, "<id> with", tokcondition { ret eqOneOf(tok.get(i+1), "return", "continue", "break"); })) >= 0) {

      // XXX int j = scanOverExpression(tok, getBracketMap(tok), i+4, ";");

      int j = tok_findEndOfStatement(tok, i+4)-1;

      //print("Found statement: " + joinSubList(tok, i+4, j+1));

      tok.set(j, "; " + tok.get(i) + "; }");

      replaceTokens(tok, i, i+3, "{");

      reTok(tok, i, j+1);

    }

    while ((i = jfind(tok, "return <id> if")) >= 0) {

      int j = scanOverExpression(tok, getBracketMap(tok), i+4, ";");

      tok.set(j, ") return " + tok.get(i+2) + "; }");

      replaceTokens(tok, i, i+6, "{ if (");

      reTok(tok, i, j+1);

    }

    // return "bla" with <statement>

    while ((i = jfindOneOf(tok,

      "return <quoted> with", "return <id> with")) >= 0) {

      S result = tok.get(i+2);

      int j = scanOverExpression(tok, getBracketMap(tok), i+6, ";");

      replaceTokens(tok, i, i+5, "{");

      tok.set(j, "; return " + result + "; }");

      reTok(tok, i, j+1);

    }

    tok_debugStatements(tok);

    // while not null (...) / if not null (...)

    while ((i = jfind_check not(tok, "<id> not null (", tokcondition {

      ret eqOneOf(_get(tok, i+1), "if", "while");

    })) >= 0) {

      int closingBracket = findEndOfBracketPart(tok, i+6)-1;

      replaceTokens(tok, i+2, i+6, "(");

      tok.set(closingBracket, ") != null)");

      reTok(tok, i, closingBracket+1);

    }

    // while null (...) / if null (...)

    while ((i = jfind_check null(tok, "<id> null (", tokcondition {

      ret eqOneOf(_get(tok, i+1), "if", "while");

    })) >= 0) {

      int closingBracket = findEndOfBracketPart(tok, i+4)-1;

      replaceTokens(tok, i+2, i+4, "(");

      tok.set(closingBracket, ") == null)");

      reTok(tok, i, closingBracket+1);

    }

    // Replace $1 with m.unq(0) etc. - caveat: this blocks identifiers $1, $2, ...

    for (i = 1; i < l(tok); i += 2) {

      S s = tok.get(i);

      if (s.startsWith("$")) {

        s = substring(s, 1);

        if (isInteger(s)) {

          tok.set(i, "m.unq(" + (parseInt(s)-1) + ")");

          reTok(tok, i);

        }

      }

    }

    // instanceof trickery

    jreplace(tok, "is a <id>", "instanceof $3");

    jreplace(tok, "!<id> instanceof <id>.<id>", "!($2 instanceof $4.$6)");

    jreplace(tok, "!<id> instanceof <id>", "!($2 instanceof $4)");

    jreplace(tok, "<id> !instanceof <id>", "!($1 instanceof $4)");

    // map func1 func2 func3(...) => mapFGH(f func1, f func2, f func3, ...)

    jreplace(tok, "map <id> <id> <id>(", "mapFGH(f $2, f $3, f $4,");

    // map func1 func2(...) => mapFG(f func1, f func2, ...)

    jreplace(tok, "map <id> <id>(", "mapFG(f $2, f $3,");

    // "ref->bla" for dereferencing Concept.Ref or ThreadLocal or other

    // For lambdas, use SPACES on the left or right of the arrow!

    //jreplace(tok, "<id> ->", "$1.get().");

    jreplace(tok, "->", ".get().", tokcondition {

      ret empty(tok.get(i)) // no space on left of arrow

        && empty(tok.get(i+2)) // no space inside of arrow

        && empty(tok.get(i+4)) // no space on right of arrow

        && !eq(_get(tok, i-1), "-"); // i-->0;

    });

    // shortened subconcept declaration (before star constructors!)

    shortenedSubconcepts(tok);

    tok_beaConceptDecls(tok);

    // "case" as a variable name ( => _case)

    caseAsVariableName(tok);

    // "do" as a function name ( => dO)

    tok_doAsMethodName(tok);

    // "continue" as a function name ( => _continue)

    continueAsFunctionName(tok);

    tok_extend(tok);

    jreplace(tok, "pn {", "p-noconsole {");

    // Do these BEFORE awt replacement! ("p-awt" contains "awt" token)

    replaceKeywordBlock(tok, "r-awt", "r { awt {", "}}");

    if (hasCodeTokens(tok, "p", "-")) tok_p_old(tok);

    replaceKeywordBlock(tok, "awt-messagebox", "awt { pcall-messagebox {", "}}");

    replaceKeywordBlock(tok, "awt", "swingLater(r {", "});");

    jreplace(tok, "p-android {", "set flag Android. p {");

    unswing(tok);

    lockBlocks(tok);

    tok_switchTo(tok);

    // trim x;

    jreplace(tok, "trim <id>;", "$2 = trim($2);");

    // iterate with index

    jreplace (tok, "for <id> over <id>:", "for (int $2 = 0; $2 < l($4); $2++)");

    jreplace (tok, "for <id> backwards over <id>:", "for (int $2 = l($5)-1; $2 >= 0; $2--)");

    jreplace (tok, "for <id>, <id> <id> over <id>: {", "for (int $2 = 0; $2 < l($7); $2++) { $4 $5 = $7.get($2);");

    jreplace (tok, "for <id>, <id> <id> backwards over <id>: {", "for (int $2 = l($8)-1; $2 >= 0; $2--) { $4 $5 = $8.get($2);");

    jreplace (tok, "for <id> to <id>:", "for (int $2 = 0; $2 < $4; $2++)");

    jreplace (tok, "for <id> to <int>:", "for (int $2 = 0; $2 < $4; $2++)");

    tok = expandShortTypes(tok);

    tok_equalsCast(tok);

    tok_equalsOptCast(tok);

    replaceKeywordBlock(tok, "r-thread-messagebox", "r-thread { pcall-messagebox {", "}}");

    replaceKeywordBlock(tok, "thread-messagebox", "thread { pcall-messagebox {", "}}");

    jreplace(tok, "rThread {", "r-thread {");

    jreplace(tok, "rEnterThread {", "rThreadEnter {");

    jreplace(tok, "rThreadEnter {", "r-thread { temp enter(); ");

    replaceKeywordBlock(tok, "r-thread", "runnableThread(r {", "})");

    rNamedThread(tok);

    // only works in the scope of a DynModule

    jreplace(tok, "rEnter {", "r { temp enter(); ");

    replaceKeywordBlock(tok, "r-pcall", "r { pcall {", "}}");

    replaceKeywordBlock(tok, "r-messagebox", "r { pcall-messagebox {", "}}");

    jreplace(tok, "r <id> + r <id>", "r { $2(); $5(); }");

    // runnable and r - now also with automatic toString if enabled

    for (S keyword : ll("runnable", "r")) {

      while ((i = jfind(tok, keyword + " {")) >= 0) {

        int idx = findCodeTokens(tok, i, false, "{");

        int j = findEndOfBracketPart(tok, idx);

        L<S> contents = subList(tok, idx+1, j-1);

        replaceTokens(tok, i, j+1, "new Runnable {"

          + "  public void run() ctex { " + tok_addSemicolon(contents) + "\n}"

          + (autoQuine ? tok_autoQuineFunc(contents) : "")

          + "}");

        reTok(tok, i, j+1);

      }

      while ((i = jfind(tok, keyword + " <quoted> {")) >= 0) {

        int idx = findCodeTokens(tok, i, false, "{");

        int j = findEndOfBracketPart(tok, idx);

        L<S> contents = subList(tok, idx+1, j-1);

        replaceTokens(tok, i, j+1, "new Runnable {"

          + "  public void run() ctex { " + tok_addSemicolon(contents) + "\n}"

          + "  toString { ret " + tok.get(i+2) + "; }"

          + (autoQuine ? tok_autoQuineFunc(contents, '_shortenedSourceCode) : "")

          + "}");

        reTok(tok, i, j+1);

      }

    }

    replaceKeywordBlock(tok,

      "expectException",

      "{ bool __ok = false; try {",

      "} catch { __ok = true; } assertTrue(\"expected exception\", __ok); }");

    while ((i = tok.indexOf("tex")) >= 0) {

      tok.set(i, "throws Exception");

      tok = jtok(tok);

    }

    // shorter & smarter whiles

    jreplace(tok, "while true", "while (true)");

    jreplace(tok, "while licensed", "while (licensed())");

    jreplace(tok, "repeat {", "while (licensed()) {");

    tok_repeatWithSleep(tok);

    // null; => return null; etc.

    O cond = tokcondition {

      ret tok_tokenBeforeLonelyReturnValue(tok, i-1);

    };

    jreplace(tok, "null;", "return null;", cond);

    jreplace(tok, "false;", "return false;", cond);

    jreplace(tok, "true;", "return true;", cond);

    jreplace(tok, "this;", "return this;", cond);

    // ok <cmd> => ret "OK" with <cmd>

    jreplace(tok, "ok <id>", "return \"OK\" with $2");

    replaceKeywordBlock(tok, "ok", "{", " return \"OK\"; }",

      func(LS _tok, int nIdx) -> bool { !eqGetOneOf(_tok, nIdx-1, "void", "svoid") });

    // "myFunction;" instead of "myFunction();" - quite rough

    // (isolated identifier as function call)

    cond = new TokCondition {

      public bool get(L<S> tok, int i) {

        S word = tok.get(i+3);

        //print("single word: " + word);

        ret !eqOneOf(word, "break", "continue", "return", "else", "endifdef", "endif");

      }

    };

    for (S pre : litlist("}", ";"))

      jreplace(tok, pre + " <id>;", "$1 $2();", cond);

    // shorter match syntax for answer methods

    tok_expandIfQuoted(tok);

    jreplace(tok, "if <id> eq <quoted>", "if (eq($2, $4))");

    tok_dropExtraCommas(tok);

    // additional translations (if necessary)

    jreplace(tok, "pcall ping {", "pcall { ping();");

    replaceKeywordBlock(tok, ") pcall",

      ") { pcall {",

      "}}");

    //jreplace(tok, "void <id> pcall {", "$1 $2() pcall {");

    replaceKeywordBlock(tok,

      "pcall",

      "try {",

      "} catch (Throwable __e) { printStackTrace(__e); }");

    replaceKeywordBlock(tok,

      "pcall-print",

      "try {",

      "} catch (Throwable __e) { printStackTrace(__e); }");

    replaceKeywordBlock(tok,

      "pcall-short",

      "try {",

      "} catch (Throwable __e) { print(exceptionToStringShort(__e)); }");

    replaceKeywordBlock(tok,

      "pcall-silent",

      "try {",

      "} catch (Throwable __e) { silentException(__e); }");

    replaceKeywordBlock(tok,

      "pcall-messagebox",

      "try {",

      "} catch __e { messageBox(__e); }");

    replaceKeywordBlock(tok,

      "pcall-infobox",

      "try {",

      "} catch __e { infoBox(__e); }");

    tok = dialogHandler(tok);

    replaceKeywordBlock(tok, "exceptionToUser",

      "try {",

      "} catch (Throwable __e) { ret exceptionToUser(__e); }"); 

    if (hasCodeTokens(tok, "twice", "{"))

      replaceKeywordBlock(tok, "twice",

        "for (int __twice = 0; __twice < 2; __twice++) {",

        "}"); 

    while ((i = findCodeTokens(tok, "bench", "*", "{")) >= 0) {

      int j = findEndOfBracketPart(tok, i+4)-1;

      S time = makeVar("time");

      S v = makeVar("bench");

      S n = tok.get(i+2);

      tok.set(i, "{ long " + time + " = sysNow(); for (int " + v + " = 0; " + v + " < " + n + "; " + v + "++)");

      tok.set(i+2, "");

      tok.set(j, "} printBenchResult(sysNow()-" + time + ", " + n + "); }");

      reTok(tok, i, j+1);

    }

    replaceKeywordBlockDyn(tok,

      "time",

      new O { S[] get() {

        S var = makeVar("startTime");

        ret new S[] {

          "{ long " + var + " = sysNow(); try { ",

          "} finally { " + var + " = sysNow()-" + var + "; saveTiming(" + var + "); } }"};

      }});

    // version without { }

    replaceKeywordBlockDyn(tok,

      "time2",

      new O { S[] get() {

        S var = makeVar("startTime");

        ret new S[] {

          "long " + var + " = sysNow(); ",

          " " + var + " = sysNow()-" + var + "; saveTiming(" + var + "); "};

      }});

    // time "bla" {

    // time msg {

    replaceKeywordPlusQuotedOrIDBlock(tok,

      "time",

      new O { S[] get(L<S> tok, int i) {

        S var = makeVar("startTime");

        ret new S[] {

          "long " + var + " = sysNow(); ",

          " done2_always(" + tok.get(i+2) + ", " + var + "); "};

      }});

    if (hasCodeTokens(tok, "assertFail", "{")) {

      S var = makeVar("oops");

      replaceKeywordBlock(tok,

        "assertFail",

        "boolean " + var + " = false; try {",

        "\n" + var + " = true; } catch (Exception e) { /* ok */ } assertFalse(" + var + ");"); 

    }

    replaceKeywordBlock(tok,

      "yo",

      "try {",

      "} catch (Exception " + makeVar("e") + ") { ret false; }",

      tokcondition {

        ret neqOneOf(_get(tok, i-1), "svoid", "void");

      });

    replaceKeywordBlock(tok,

      "awtIfNecessary",

      "swingNowOrLater(r " + "{",

      "});");

    ctex(tok);

    replaceKeywordBlock(tok,

      "actionListener",

      "new java.awt.event.ActionListener() { " +

        "public void actionPerformed(java.awt.event.ActionEvent _evt) { pcall-messagebox {",

      "}}}");

    for (S keyword : ll("autocloseable", "autoCloseable"))

      /*replaceKeywordBlock(tok,

        keyword,

        "new AutoCloseable() { public void close() throws Exception {",

        "}}");*/

        replaceKeywordBlock_dyn2_legacy(tok, keyword, new O {

          S[] get(LS tok, int iOpening, int iClosing) {

            LS contents = subList(tok, iOpening+1, iClosing);

            ret new S[] {

              "new AutoCloseable() { toString { ret " + quote(shorten(defaultMaxQuineLength(), trimJoin(contents))) + "; } public void close() throws Exception {",

              "}}"

            };

          }

        });

    // try answer (string, test with nempty)

    while ((i = findCodeTokens(tok, "try", "answer")) >= 0) {

      int j = findEndOfStatement(tok, i);

      S v = makeVar();

      bool needCurly = !eqGet(tok, i-2, "{");

      tok.set(i, (needCurly ? "{" : "") + " S " + v);

      tok.set(i+2, "=");

      tok.set(j-1, "; if (!empty(" + v + ")) ret " + v + "; " + (needCurly ? "}" : ""));

      reTok(tok, i, j);

    }

    // try bool[ean] (try answer with Bool type)

    while ((i = findCodeTokens(tok, "try", "boolean")) >= 0) {

      int j = findEndOfStatement(tok, i);

      S v = makeVar();

      tok.set(i, "{ Bool " + v);

      tok.set(i+2, "=");

      tok.set(j-1, "; if (" + v + " != null) ret " + v + "; }");

      reTok(tok, i, j);

    }

    // <statement>, print "..."; => { <statement>; print("..."); }

    while ((i = jfind(tok, ", print <quoted>;")) >= 0) {

      int j = tok_findBeginningOfStatement(tok, i);

      replaceTokens_reTok(tok, i, i+4*2-1, "; print(" + tok.get(i+4) + "); }");

      tokPrepend_reTok(tok, j, "{ ");

    }

    // return if null <expression> => if (<expression> == null) return;

    while ((i = jfind(tok, "return if null")) >= 0) {

      int j = findEndOfStatement(tok, i);

      clearTokens(tok, i, i+2);

      tok.set(i+4, "((");

      tok.set(j-1, ") == null) ret;");