!1006722

// TODO: deprecate (remove) the "f <id>" syntax

// TODO: skip ifclass blocks in findFunctionInvocations.
//       after main loop completes (including standard functions+
//       classes), run tok_ifclass and if any change, run main loop
//       again.
//       This will allow clean handling of functions like "unnull"
//       which right now _always_ include Symbol (because unnull
//       references emptySymbol inside an ifclass Symbol block).

// Note: Before editing this, transpile #7
// Use 'Very fresh' (R-transpile doesn't seem to work) or Q/M if you haven't screwed yourself currently
// by making your transpiler unable to transpile itself... or
// something

// Note: Don't try hijackPrint, it doesn't seem to work in here
// (because of the way it is called by e.g. transpileForServer_returnPair?)

/* functions to possibly edit when creating/changing JavaX constructs:
  tok_pingify
*/

// new JavaParser includes
!include once #1025814
!include once #1025813

!include once #1025802 // optimized findCodeTokens

set flag OurSubLists.
set flag NoIllegalAccesses.

set flag javaTokForJFind_array_simpleCache.
set flag tok_moveImportsUp_skipMostReToks.

set flag AllowMetaCode.
set flag jreplace_performing. // record replacements

//set flag tok_forUnnull_debug.

// ifdef in cached includes? => i think cached includes still contain the ifdefs, so it's fine. tok_ifdef is not called within localStuff1

set flag noAI.
set flag isQuoted_dirty. // should be OK, we're dealing with tokens

static bool autoQuine = true;
static int maxQuineLength = 80;
sbool assumeTriple = true;
sbool quickInstanceOfEnabled; // interferes with things
sbool debug_jreplace;

// _registerThread usually costs nothing because we need
// the registerWeakHashMap mechanism anyway for ping().
// Anyway - no forced functions for now :)
static L<S> functionsToAlwaysInclude = ll(
  //"_registerThread",
  //"asRuntimeException"
);

// classes with two type parameters that can written with just one
// e.g. Pair<S> => Pair<S, S>
static Set<S> pairClasses = lithashset("Pair", "Either", "Map", "AbstractMap", "HashMap", "TreeMap", "LinkedHashMap", "MultiMap", "CompactHashMap", "WrappedMap", "F1", "IF1", "AllOnAll", "AllOnAllWithUpdates", "MultiSetMap", "AutoMap", "ValueOnDemandMap", "NavigableMap", "SortedMap");

// classes with 3 type parameters that can written with just one
// e.g. T3<S> => T3<S, S, S>
static Set<S> tripleClasses = lithashset("T3", "IF2", "F2", "IVF3", "VF3");

sS transpilingSnippetID;
//static new AtomicInteger varCount;
static new ThreadLocal<AtomicInteger> varCountByThread;
static Map<S, S> snippetCache = syncMap();
sbool useIndexedList2 = false, useTokenIndexedList = true;
sbool opt_javaTok = true;
static bool cacheStdFunctions = true, cacheStdClasses = true;
static new HashMap<Long, CachedInclude> cachedIncludes;
static ExecutorService executor;
static L lclasses;
static long startTime, lastPrint;

// These variables have to be cleared manually for each transpilation

static new HashSet<Long> included;
static NavigableSet<S> definitions = ciSet();
static new HashMap<S> rewrites;
static new HashSet<S> shouldNotIncludeFunction; // this verifies after the fact that the function was not included
static new HashSet<S> shouldNotIncludeClass;
static new HashSet<S> doNotIncludeFunction; // this actually prevents the function from being included
static new SS functionToPackage;
static new HashSet<S> doNotIncludeClass;
static new HashSet<S> needLatest; // this forces a function to be included
static new HashSet<S> addedFunctions;
static new HashSet<S> addedClasses;
static new HashSet<S> hardFunctionReferences;
static new HashSet<S> mapLikeFunctions;
static new HashSet<S> lambdaMapLikeFunctions;
static new HashSet<S> curry1LikeFunctions;
static new HashSet<S> mapMethodLikeFunctions;
static new HashSet<S> nuLikeFunctions;
static new HashSet<S> getLikeFunctions; // name is slightly misleading. this turns a pre-argument into a fieldLambda
static new HashSet<S> lambdaMethod0LikeFunctions; // we're getting better again with the names (maybe)
static new HashSet<S> lambda0LikeFunctions;
static SS extraStandardFunctions;
sbool quickmainDone1, quickmainDone2;
static new TreeSet<S> libs;
sS mainBaseClass, mainPackage, mainClassName;
sbool localStuffOnly; // for transpiling a fragment
sbool asInclude; // for transpiling an include (auto-close scopes, enable all standard class ifclass [todo])
sbool allowMetaCode = false; // run any embedded meta code
static LS metaPostBlocks, metaTransformers, localTransformers;
sbool dontPrintSource;
sbool dontLoadCachedIncludesFromVM; // for benchmarking
static new HashSet<S> haveClasses;

sclass CachedInclude {
  S javax;
  Future<S> java;
  S realJava;
  long snippetID;
  
  *() {}
  *(long *snippetID) {}
  
  S java() {
    ret realJava != null ? realJava : getFuture(java);
  }
  
  Future<S> javaFuture() {
    ret realJava != null ? nowFuture(realJava) : java;
  }
  
  void clean {
    if (java != null) {
      realJava = getFuture(java);
      java = null;
    }
  }
}

p {
  startTime = lastPrint = sysNow();
  pcall {
    if (!dontLoadCachedIncludesFromVM)
      vmKeepWithProgramMD5_get('cachedIncludes);
  }
  executor = Executors.newFixedThreadPool(numberOfCores());
  transpilingSnippetID = or(getThreadLocal((ThreadLocal<S>) getOpt(javax(), 'transpilingSnippetID)), transpilingSnippetID);
  //print(+transpilingSnippetID);
  set transpileRaw_dontCopyFromCreator;
  
  fO oldPrint = or(print_byThread()!, f print_raw);
  temp tempInterceptPrint(new F1<S, Bool>() {
    Bool get(S s) {
      long now = sysNow();
      long time = now-lastPrint; // -startTime;
      lastPrint = now;
      callF(oldPrint, "[" + formatInt(time/1000, 2) + ":" + formatInt(time % 1000, 3) + "] " + s);
      false;
    }
  });
  
  try {
    _main();
  } finally {
    interceptPrintInThisThread(oldPrint);
    if (executor != null) executor.shutdown();
    executor = null;
    localStuffOnly = false;
    asInclude = false;
  }
}

svoid _main() ctex {
  if (sameSnippetID(programID(), defaultJavaXTranslatorID())) setDefaultJavaXTranslatorID(#7);
  
  //reTok_modify_check = true;
  //if (useIndexedList) findCodeTokens_debug = true;
  javaTok_opt = opt_javaTok;
  //findCodeTokens_indexed = findCodeTokens_unindexed = 0;
  findCodeTokens_bails = findCodeTokens_nonbails = 0;
  javaTok_n = javaTok_elements = 0;
  S in = loadMainJava();
  
  print("759 STARTING " + identityHashCode(main.class));
  // clear things
  includeInMainLoaded_magicComment = null;
  included.clear();
  definitions.clear();
  rewrites.clear();
  // XXX definitions.add("SymbolAsString");
  shouldNotIncludeFunction.clear();
  shouldNotIncludeClass.clear();
  doNotIncludeFunction.clear();
  functionToPackage.clear();
  needLatest.clear();
  doNotIncludeClass.clear();
  addedFunctions.clear();
  addedClasses.clear();
  hardFunctionReferences.clear();
  mapLikeFunctions = cloneHashSet(tok_mapLikeFunctions());
  lambdaMapLikeFunctions = new HashSet;
  curry1LikeFunctions = new HashSet;
  mapMethodLikeFunctions = cloneHashSet(tok_mapMethodLikeFunctions());
  nuLikeFunctions.clear();
  getLikeFunctions.clear();
  lambdaMethod0LikeFunctions.clear();
  lambda0LikeFunctions.clear();
  extraStandardFunctions = new HashMap;
  libs.clear();
  mainBaseClass = mainPackage = mainClassName = null;
  varCountByThread.set(null);
  quickmainDone1 = quickmainDone2 = false;
  metaPostBlocks = new L;
  metaTransformers = new L;
  localTransformers = new L;
  dontPrintSource = false;
  defaultMaxQuineLength_value = defaultMaxQuineLength_defaultValue;
  debug_jreplace = false;
  haveClasses.clear();
  
  //L ts = findTranslators(toLines(join(tok)));
  //print("Translators in source at start: " + structure(ts));
  
  L<S> tok = jtok(in);
  
  try {
    tok_definitions(tok);
    
    // add m { }
    
    if (!localStuffOnly && !hasCodeTokens(tok, "m", "{") && !hasCodeTokens(tok, "main", "{") && !hasCodeTokens(tok, "class", "main")) {
      if (l(tok) == 1) tok = singlePlusList(first(tok), dropFirst(javaTok("m {}")));
      else {
        replaceTokens_reTok(tok, 1, 2, "m {\n\n" + unnull(get(tok, 1)));
        replaceTokens_reTok(tok, l(tok)-2, l(tok)-1, unnull(get(tok, l(tok)-2)) + "}");
      }
      tok_moveImportsUp(tok);
    }
    
    // standard translate
    
    //ts = findTranslators(toLines(join(tok)));
    //print("Translators in source: " + structure(ts));
    
    if (tok_hasTranslators(tok))
      tok = jtok(defaultTranslate(join(tok)));
    
    //print("end of default translate");
    //print(join(tok));
  
    //tok_autoCloseBrackets(tok);    
    
    tok_metaTransformNow(tok);
    
    tok_processEarlyIncludes(tok);
      
    tok_earlyGeneralStuff(tok);
    
    tok = tok_processIncludes(tok); // before standard functions
    if (processConceptsDot(tok))
      tok = tok_processIncludes(tok);
    tok = localStuff1(tok);
    
   if (!localStuffOnly) {
    int safety = 0;
    boolean same;
    do { // BIG LOOP
      ping();
      LS before = cloneList(tok);
      
      // do the non-local stuff (flags and rewrites, things that correlate across includes like tok_selfType)
      
      // to allow crazy stuff like "nonStatic !include #someStaticClass"
      tok_processEarlyIncludes(tok);
      
      jreplace(tok, "nonStatic static", "");
      
      tok_selfType(tok);
      tok_mainClassNameAndPackage(tok);
      tok_definitions(tok);
      tok_ifndef(tok);
      tok_ifdef(tok);
      defineMapLikesEtc(tok);
      if (tok_applyAllXLikeFunctions(tok)) {
        functionReferences(tok);
        lambdaReferences(tok);
      }
      
      tok_dropExtraCommas(tok); // from e.g. tok_applyMapMethodLikeFunctions
      tok_delegateTo(tok);
      tok_replaceWith(tok);
      tok_findAndClearRewrites(tok);
      tok_processRewrites(tok);
      
      tok_multiTypeArguments_v2(tok);
      
      // main bla(...) => mainClassName.bla(...)
      // or main ClassName => main.ClassName
      //jreplace(tok, "main <id>(", mainClassName() + ".$2(");
      grabImportedStaticFunctions(tok);
      jreplace_dyn_allowNull(tok, "main <id>", (_tok, cIdx, end) -> {
        S fname = _tok.get(cIdx+2);
        bool isClass = haveClasses.contains(fname);
        printVars expandMainRef(+fname, +isClass);
        if (isClass || eqGet(_tok, cIdx+4, "(")) {
          S pkg = functionToPackage.get(fname);
          printVars expandMainRef(+pkg);
          S call = "." + fname;
          ret or(pkg, mainClassName()) + call;
        }
        null;
      });
      
      try {
        if (safety == 0) tok = quickmain(tok);
      } catch e {
        printSources(tok);
        rethrow(e);
      }
      tok_collectMetaPostBlocks(tok, metaPostBlocks);
      tok_collectTransformers(tok, metaTransformers);
      tok_collectTransformers(tok, localTransformers, "meta-transformLocal");

      tok_metaTransformNow(tok);
      
      // Hack to allow DynModule to reimplement _registerThread
      /*if (tok.contains("DynModule") && !addedClasses.contains("DynModule"))
        addStandardClasses_v2(tok);*/
      
      defineExtraSF(tok);
      tok = standardFunctions(tok);
      tok = stdstuff(tok); // all the keywords, standard
      S diff;
      long startTime = now();
      //diff = unidiff(before, join(tok));
      //print("unidiff: " + (now()-startTime) + " ms");
      //same = eq(diff, "");
      same = eq(tok, before);
      if (!same) {
        print("Not same " + safety + ".");
        //print(indent(2, diff));
      }
      if (safety++ >= 10) {
        //print(unidiff(before, join(tok)));
        printSources(tok);
        fail("safety 10 error!");
      }
    } while (!same); // END OF BIG LOOP
    
    print("Post.");
    
    if (mainBaseClass != null) {
      jreplace1(tok, "class main", "class main extends " + mainBaseClass);
      mainBaseClass = null;
    }
    
    print('moveImportsUp); tok_moveImportsUp(tok);
    
    print('Indexing); tok = indexTokenList(tok);
    
    // POST-PROCESSING after stdstuff loop
    
    if (transpilingSnippetID != null)
      jreplace_dyn(tok, "class whatever", 
        func(L<S> tok, int cIndex) -> S {
          pcall { ret "class " + stringToLegalIdentifier(getSnippetTitle(transpilingSnippetID)); }
          ret "class Whatever";
        });
    
    //print("Type<A, A>"); // Type<A> to Type<A, A>
    
    print('extendClasses); tok = extendClasses(tok);
    print('libs); libs(tok);
    print('sourceCodeLine); sourceCodeLine(tok);
    
    tok_overridableFunctionDefs(tok, null);
    
    // escaping for lambdas
    //jreplace(tok, "-=>", "->");
  
    // Stuff that depends on the list of inner classes (haveClasses)
    haveClasses = haveClasses_actual(tok);
    print('innerClassesVar); innerClassesVar(tok, haveClasses);
    fillVar_transpilationDate(tok);
    haveClasses_addImported(tok, haveClasses);
    print('ifclass); tok_ifclass(tok, haveClasses);
    
    print('slashCasts); slashCasts(tok, haveClasses);
    print('newWithoutNew); newWithoutNew(tok, haveClasses);
    
    if (!assumeTriple) {
      print('Triple);
      if (tok.contains("Triple") && !haveClasses.contains("Triple")) {
        jreplace(tok, "Triple", "T3");
        haveClasses.remove("Triple");
        haveClasses.add("T3");
        slashCasts(tok, lithashset("T3"));
        tok_quickInstanceOf(tok, lithashset("T3"));
      }
    }
    
    if (hasDef("SymbolAsString"))
      jreplace(tok, "Symbol", "String");
  
    // using non-lazy version now for cleanImports
    print('classReferences); expandClassReferences/*_lazy*/(tok, haveClasses);
    
    if (metaCodeAllowed()) runMetaPostBlocks(tok);

    // Error-checking
    print("Error-checking"); 
    Set<S> functions = new HashSet(findFunctions(tok));
    for (S f : hardFunctionReferences)
      if (!functions.contains(f))
        fail("Function " + f + " requested, but not supplied");
  
    print('autoImports); tok = autoImports(tok); // faster to do it at the end
    
    if (hasDef("cleanImports"))
      tok_cleanImports(tok);
    
    if (includeInMainLoaded_magicComment != null) {
      int i = lastIndexOfStartingWith(tok, includeInMainLoaded_magicComment);
      if (i >= 0) tok.set(i, dropPrefix(includeInMainLoaded_magicComment, tok.get(i)));
    }

    print("definitions=" + sfu(definitions));
    if (containsOneOf(definitions, "allpublic", "reparse", "PublicExceptTopClass")) {
      // Fire up the Java parser & pretty printer!
      print(containsIC(definitions, "allpublic")? "Making all public." : "Reparsing.");
      //try {
        S src = join(tok);
        try {
          if (containsIC(definitions, "allpublic"))
            src = javaParser_makeAllPublic(src);
          else if (containsIC(definitions, "PublicExceptTopClass"))
            src = javaParser_makeAllPublic(src, notTopLevelClassDecl := true);
          else if (containsIC(definitions, "keepComments"))
            src = javaParser_makeAllPublic_keepComments(src);
          else {
            print("javaParser_reparse_keepComments. size=" + l(src));
            src = javaParser_reparse_keepComments(src);
            print("size=" + l(src));
          }
        } catch e {
          S _src = src;
          S errorContext = extractAndPrintJavaParseError(_src, e);
          File f = transpilerErrorSourceFile();
          saveTextFileVerbose(f, src);
          dontPrintSource = true;
          fail("Java parsing error:" + errorContext + innerMessage(e)); // drop the long nested parser stacktraces
        }
        tok = jtok(src);
      /*} catch e {
        S src = join(tok);
        if (!dontPrintSource)
          print(src);
        print(f2s(saveProgramTextFile("error.java", src)));
        throw rethrow(e);
      }*/
    }
    
    // Do this after JavaParser (because it doesn't like package after class)
    
    if (mainPackage != null) {
      print('mainPackage);
      tokPrepend(tok, 1, "package " + mainPackage + ";\n");
      reTok(tok, 1, 2);
    }
    
    if (mainClassName != null) {
      print('mainClassName);
      jreplace(tok, "class main", "class " + mainClassName);
      jreplace(tok, "main.class", mainClassName + ".class");
      //tokPrepend(tok, 1, "class main {}\n"); // so main.class is generated and compiler sanity checks succeed. we can later skip it in the JavaXClassLoader
    }
    
    if (nempty(libs)) {
      print("Adding libs: " + libs);
      tok.add(concatMap_strings(func(S s) -> S { "\n!" + s }, libs));
    }
   } // if (!localStuffOnly)
  } catch e {
    if (!dontPrintSource)
      printSources(tok);
    S src = join(tok);
    print(f2s(saveProgramTextFile("error.java", src)));
    throw rethrow(e);
  }
  
  /*if (useIndexedList)
    print("Indexed/unindexed lookups: " + findCodeTokens_indexed + "/" + findCodeTokens_unindexed + ", lists made: " + IndexedList2.instances);
  print("findCodeToken bails: " + findCodeTokens_bails + "/" + findCodeTokens_nonbails);
  print("javaToks: " + javaTok_n + "/" + javaTok_elements);*/
  
  print("Saving.");
  
  // for dexcompile.php
  if (mainClassName != null)
    tokPrepend(tok, 0, "//FILENAME: " 
      + (mainPackage != null ? mainPackage.replace(".", "/") + "/" : "") + mainClassName + ".java\n");
    
  if (mainJava != null)
    mainJava = join(tok);
  else if (tok.contains("package"))
    splitJavaFiles(tok);
  else
    saveMainJava(tok);
}

static LS localStuff1(LS tok) {
  int safety = 0, i;
  boolean same;
  tok = indexTokenList(tok);
  
  tok_scopes(tok, autoCloseScopes := true);
    
  do {
    ping();
    LS before = cloneList(tok);
    
    //print("localStuff loop " + safety);
    
    earlyStuff(tok);
    
    // EARLY local stuff goes here
    
    tok_dropMetaComments(tok);
    
    tok_earlyGeneralStuff(tok);
    
    conceptDeclarations(tok);
    tok_recordDecls(tok);
    
    tok = multilineStrings(tok);
    tok_singleQuoteIdentifiersToStringConstants(tok);
    tok_inStringEvals(tok);
    tok_listComprehensions(tok);
    
    tok_eventFunctions(tok);
    
    tok_for_single(tok);
    
    tok_for_unpair(tok); // Do this...
    //tok_doubleFor_v2(tok);
    tok_doubleFor_v3(tok); // ...before this
    
    tok_forUnnull(tok);
    tok_ifCast(tok);
    forPing(tok);
    tok_directSnippetRefs(tok);
    quicknu(tok);
    //tok_juxtaposeCalls(tok);
    
    jreplace(tok, "LLS", "L<LS>");
    jreplace(tok, "LS", "L<S>");
    jreplace(tok, "ES", "Ext<S>");
    jreplace(tok, "ExtS", "Ext<S>");
    
    jreplace(tok, "WeakRef", "WeakReference");
    
    jreplace(tok, "dispose <id>;", "{ cleanUp($2); $2 = null; }");
  
    tok_doPing(tok);
      
    replaceKeywordBlock(tok,
      "swing",
      "{ swing(r {",
      "}); }");
      
    replaceKeywordBlock(tok,
      "androidUI",
      "{ androidUI(r {",
      "}); }");
      
    replaceKeywordBlock(tok,
      "withDBLock",
      "{ withDBLock(r {",
      "}); }");
      
    replaceKeywordBlock(tok, "afterwards", "temp tempAfterwards(r {", "});");
      
    for (S keyword : ll("tokcondition", "tokCondition"))
      replaceKeywordBlock(tok,
        keyword,
        "new TokCondition { public bool get(final L<S> tok, final int i) {",
        "}}");
      
    jreplace(tok, "synced <id>", "synchronized $2");
    jreplace(tok, "sync <id>", "synchronized $2");
    
    jreplace(tok, "synchronized {", "synchronized(this) {");
    
    replaceKeywordBlock(tok, "answer",
      "static S answer(S s) {\nfinal new Matches m;\n",
      "\nret null;\n}");
      
    replaceKeywordBlock(tok, "static-pcall",
      "static { pcall {",
      "}}");
      
    replaceKeywordBlock(tok, "loading",
      "{ temp tempShowLoadingAnimation(); ",
      "}");
  
    replaceKeywordPlusQuotedBlock(tok, "loading",
      new O { S[] get(L<S> tok, int i) {
        S text = tok.get(i+2);
        ret new S[] {
          "{ temp tempShowLoadingAnimation(" + text + "); ",
          "}" };
      }});
      
    while ((i = jfind(tok, "visualize as")) >= 0) {
      int j = tok_findEndOfStatement(tok, i); // expression, rather
      tok.set(i+2, "{ ret");
      tok.set(j-1, "; }");
      reTok(tok, i, j);
    }
      
    for (S pat : ll("visual {", "visualize {"))
      jreplace(tok, pat, "public JComponent visualize() {", tokCondition_beginningOfMethodDeclaration());
  
    jreplace(tok, "visualize2 {", "JComponent visualize2() {", tokCondition_beginningOfMethodDeclaration());
    
    replaceKeywordBlock(tok, "start-thread-printDone", "start-thread {", "printDone(); }");

    replaceKeywordBlock(tok, "start-thread", [[start { thread "Start" { temp enter(); pcall {]], "}}}");
    
    jreplace(tok, "start {", "void start() ctex { super.start();", tokCondition_beginningOfMethodDeclaration());
    
    var notVoidMethod = tokcondition {
      ret neqGet(tok, i-1, "void");
    };
    
    // run { ... } => public void run() { ... }
    // same with close
    jreplace(tok, "run {", "public void run() tex {", notVoidMethod);
    jreplace(tok, "close {", "public void close() ctex {", notVoidMethod);
    
    replaceKeywordBlock(tok, "html",
      "static O html(S uri, fMap<S, S> params) ctex " + "{\n", "}");
    
    replaceKeywordBlock(tok, "afterVisualize",
      "visualize { JComponent _c = super.visualize();",
      "ret _c; }");
    
    replaceKeywordBlock(tok, "enhanceFrame",
      "void enhanceFrame(Container f) { super.enhanceFrame(f);",
      "}");
    
    if (assumeTriple)
      jreplace(tok, "Triple", "T3");

    tok_shortFinals(tok);
    
    tok_moduleClassDecls(tok);

    jreplace(tok, "static sync", "static synchronized");
    jreplace(tok, "sclass", "static class");
    jreplace(tok, "fclass", "final class");
    jreplace(tok, "fsclass", "final static class");
    jreplace(tok, "srecord", "static record");
    jreplace(tok, "strecord", "static transformable record");
    jreplace(tok, "record noeq", "noeq record");
    jreplace(tok, "asclass", "abstract static class");
    jreplace(tok, "sinterface", "static interface");
    jreplace(tok, "ssynchronized", "static synchronized");
  
    jreplace(tok, "ssvoid", "static synchronized void");
    jreplace(tok, "sbool", "static bool");
    jreplace(tok, "fbool", "final bool");
    jreplace(tok, "sint", "static int");
    jreplace(tok, "snew", "static new");
    jreplace(tok, "sv <id>", "static void $2");
    jreplace(tok, "pvoid", "public void");
  
    // "sS" => static S
    jreplace(tok, "sS", "static S");
  
    // "sO" => static O
    jreplace(tok, "sO", "static O");
  
    // "sL" => static L
    jreplace(tok, "sL", "static L");
  
    // "toString {" => "public S toString() {"
    jreplace(tok, "toString {", "public S toString() {");
    
    jreplace(tok, "Int", "Integer");
    jreplace(tok, "Bool", "Boolean");
    jreplace(tok, "BigInt", "BigInteger");
    jreplace(tok, "Char", "Character");
    
    jreplace(tok, "Sym", "Symbol");
    jreplace(tok, "SymSym", "SymbolSymbol");
    
    jreplace(tok, "SS", "Map<S>");
    jreplace(tok, "SymbolSymbol", "Map<Symbol>");
    
    jreplace(tok, "CISet", "Set<S>");
    
    jreplace(tok, "MapSO", "Map<S, O>");
    
    jreplace(tok, "ByName<", "IF1<S, ");
    
    jreplace(tok, "ITransform<<id>>", "IF1<$3>");
    jreplace(tok, "ITransform", "IF1");
    
    jreplace(tok, "PairS", "Pair<S>");
    jreplace(tok, "LPairS", "L<Pair<S>>");
    
    jreplace(tok, "T3S", "T3<S>");
    jreplace(tok, "F1S", "F1<S>");
    
    jreplace(tok, "ItIt", "IterableIterator");
    jreplace(tok, "CloseableItIt", "CloseableIterableIterator");

    jreplace(tok, "class <id> > <id>", "class $2 extends $4", tokCondition {
      S t = _get(tok, i+1+4*2);
      ret eq(t, "{") || isIdentifier(t);
    });
    jreplace(tok, "class <id> > <id><<id>> {", "class $2 extends $4  $5 $6 $7 {");
    
    jreplace(tok, "ISegmenter", "IF1<BufferedImage, L<Rect>>");
    
    // IPred<A, B> => IF2<A, B, Bool>
    jreplace(tok, "IPred<<id>, <id>>", "IF2<$3, $5, Bool>");
    
    // IPred<A> => IF1<A, Bool>
    jreplace(tok, "IPred<<id>>", "IF1<$3, Bool>");
    jreplace(tok, "IPred<<id>[]>", "IF1<$3[], Bool>");
    
    // Proposition => WithReasoning<S> (should we really define this?)
    jreplace("Proposition", "WithReasoning<S>");
    
    replaceKeywordBlock(tok, "print exceptions",
      "{ try {",
      "} on fail __e { printStackTrace(__e); } }");
    
    // "on fail {" => "catch (Throwable _e) { ... rethrow(_e); }"
    replaceKeywordBlock(tok, "on fail",
      "catch (Throwable _e) {",
      "\nthrow rethrow(_e); }");
  
    // "on fail e {" => "catch (Throwable e) { ... rethrow(e); }"
    replaceKeywordBlock_dyn2_legacy(tok, "on fail <id>", new O {
      S[] get(LS tok, int iOpening, int iClosing) {
        S var = tok.get(iOpening-2);
        ret new S[] {
          "catch (Throwable " + var + ") {",
          "\nthrow rethrow(" + var + "); }"
        };
      }
    });

    // "on fail Type e {" => "catch (Type e) { ... rethrow(e); }"
    replaceKeywordBlock_dyn2_legacy(tok, "on fail <id> <id>", new O {
      S[] get(LS tok, int iOpening, int iClosing) {
        S type = tok.get(iOpening-4);
        S var = tok.get(iOpening-2);
        ret new S[] {
          "catch (" + type + " " + var + ") {",
          "\nthrow " + var + "; }"
        };
      }
    });

    // "catch {" => "catch (Throwable _e) {"
    jreplace(tok, "catch {", "catch (Throwable _e) {");
  
    jreplace_dyn(tok, "catch print {", -> {
      S var = makeVar();
      ret "catch \*var*/ { printStackTrace(\*var*/);";
    });
  
    // "catch print e {" => "catch e { printStackTrace(e); "
    jreplace(tok, "catch print <id> {", "catch $3 { printStackTrace($3);");
  
    // "catch print short e {" => "catch e { printExceptionShort(e); "
    jreplace(tok, "catch print short <id> {", "catch $4 { printExceptionShort($4);");
    
    // "catch X e {" => "catch (X e) {"
    jreplace(tok, "catch <id> <id> {", "catch ($2 $3) {");
  
    // "catch e {" => "catch (Throwable e) {" (if e is lowercase)
    jreplace(tok, "catch <id> {", "catch (Throwable $2) {", tokcondition {
      S word = tok.get(i+3);
      ret startsWithLowerCaseOrUnderscore(word);
    });
    
    jreplace(tok, "+ +", "+", tokcondition {
      //printStructure("++: ", subList(tok, i-1, i+6));
      if (empty(_get(tok, i+2))) ret false; // no space between the pluses
      if (empty(_get(tok, i)) && eq("+", _get(tok, i-1))) ret false;  // an actual "++" at the left
      if (empty(_get(tok, i+4)) && eq("+", _get(tok, i+5))) ret false;  // an actual "++" at the right
      //print("doing it");
      ret true;
    });
  
    // single underscore (not allowed in Java anymore) to double underscore
    jreplace(tok, "_", "__");
    
    // [stdEq] -> implementation of equals() and hashCode()
    jreplace(tok, "[stdEq]",
      "public bool equals(O o) { ret stdEq2(this, o); }\n" +
      "public int hashCode() { ret stdHash2(this); }");
    
    // [stdToString] -> toString { ret stdToString(this); }
    jreplace(tok, "[stdToString]",
      "toString { ret stdToString(this); }");
    
    // [concepts] "concept.field!" for dereferencing references
    
    // "<id>!", "?!" etc.
    jreplace(tok, "*!", "$1.get()", tokcondition {
      S l = tok.get(i+1);
      if (!(isIdentifier(l) || eqOneOf(l, ")", "?"))) false;
      if (tok.get(i+2).contains("\n")) false; // no line break between <id> and !
      if (nempty(tok.get(i+4))) true; // space after = ok
      S t = _get(tok, i+5);
      if (t == null) ret false;
      if (isIdentifier(t) || eqOneOf(t, "=", "(")) false;
      true;
    });
    
    jreplace(tok, "for (<id> :", "for (var $3 :");
    
    jreplace(tok, "for (<id> <id>)", "for ($3 $4 : list($3))");
    jreplace(tok, "for (final <id> <id>)", "for (final $4 $5 : list($4))");

    jreplace(tok, "ret", "return", tokCondition {
      if (eqGetOneOf(tok, i-1, "int", "return")) false;
      S next = _get(tok, i+3);
      if (eq(next, ".")) ret isInteger(_get(tok, i+5)); // e.g. "ret .1", but not "ret.bla"
      ret !eqOneOf(next, "=", ")");
    });
    
    // "continue unless", "break unless"
    
    for (S phrase : ll("continue unless", "break unless"))
      while ((i = jfind(tok, phrase)) >= 0) {
        S keyword = tok.get(i);
        int j = scanOverExpression(tok, getBracketMap(tok), i+4, ";");
        replaceTokens(tok, i, i+4, "{ if (!(");
        tok.set(j, ")) " + keyword + "; }");
        reTok(tok, i, j+1);
      }
    
    // S s = bla(), return if null; => S s = bla(); if (s == null) return;
    // same with continue, break
    while ((i = jfind(tok, ", <id> if null;", tokCondition {
      ret eqOneOf(tok.get(i+3), "return", "ret", "continue", "break");
    })) >= 0) {
      S cmd = tok.get(i+2);
      int j = tok_findBeginningOfStatement(tok, i);
      print("Found statement " + j + "/" + i + " - " + joinSubList(tok, j-1, i+5*2-1));
      S var = getVarDeclarationName(subList(tok, j-1, i));
      replaceTokens_reTok(tok, i, i+5*2-1, "; if (" + var + " == null) " + cmd + ";");
    }
    
    // S s = bla(), return false if null; => S s = bla(); if (s == null) return false;
    // (with false being any identifier)
    while ((i = jfind(tok, ", return <id> if null;")) >= 0) {
      S returnValue = tok.get(i+4);
      int j = tok_findBeginningOfStatement(tok, i);
      S var = getVarDeclarationName(subList(tok, j-1, i));
      replaceTokens_reTok(tok, i, i+6*2-1, "; if (" + var + " == null) return " + returnValue + ";");
    }
    
    // "continue if", "break if"
    
    for (S phrase : ll("continue if", "break if"))
      while ((i = jfind(tok, phrase)) >= 0) {
        S keyword = tok.get(i);
        int j = scanOverExpression(tok, getBracketMap(tok), i+4, ";");
        replaceTokens(tok, i, i+4, "{ if (");
        tok.set(j, ") " + keyword + "; }");
        reTok(tok, i, j+1);
      }
    
    // return unless set <var>; => { if (var) return; set var; }
    jreplace(tok, "return unless set <id>;", "{ if ($4) return; set $4; }");

    // set x; => x = true;
    // yeah it doesn't save many characters, but I like it
    jreplace(tok, "set <id>;", "$2 = true;", tokCondition {
      ret !eqGet(tok, i-1, "unless");
    });
    
    // set !x; => x = false; (did this one day and thought yes it makes sense in a way)
    jreplace(tok, "set !<id>;", "$3 = false;");
    
    // unset x; => x = false;
    jreplace(tok, "unset <id>;", "$2 = false;");
    
    // "return if"
    
    while ((i = jfind(tok, "return if")) >= 0) {
      int j = scanOverExpression(tok, getBracketMap(tok), i+4, ";");
      replaceTokens(tok, i, i+4, "{ if (");
      tok.set(j, ") return; }");
      reTok(tok, i, j+1);
    }
    
    // "return unless"
    
    while ((i = jfind(tok, "return unless")) >= 0) {
      int j = scanOverExpression(tok, getBracketMap(tok), i+4, ";");
      replaceTokens(tok, i, i+4, "{ if (!(");
      tok.set(j, ")) return; }");
      reTok(tok, i, j+1);
    }
    
    tok_ifNullAssign(tok);

    // "return <id> if"
    
    // "return with <statement>" / "continue with <statement>" / "break with <statement>"
    
    while ((i = jfind(tok, "<id> with", tokcondition { ret eqOneOf(tok.get(i+1), "return", "continue", "break"); })) >= 0) {
      // XXX int j = scanOverExpression(tok, getBracketMap(tok), i+4, ";");
      int j = tok_findEndOfStatement(tok, i+4)-1;
      //print("Found statement: " + joinSubList(tok, i+4, j+1));
      tok.set(j, "; " + tok.get(i) + "; }");
      replaceTokens(tok, i, i+3, "{");
      reTok(tok, i, j+1);
    }
    
    while ((i = jfind(tok, "return <id> if")) >= 0) {
      int j = scanOverExpression(tok, getBracketMap(tok), i+4, ";");
      tok.set(j, ") return " + tok.get(i+2) + "; }");
      replaceTokens(tok, i, i+6, "{ if (");
      reTok(tok, i, j+1);
    }
    
    // return "bla" with <statement>
    
    while ((i = jfindOneOf(tok,
      "return <quoted> with", "return <id> with")) >= 0) {
      S result = tok.get(i+2);
      int j = scanOverExpression(tok, getBracketMap(tok), i+6, ";");
      replaceTokens(tok, i, i+5, "{");
      tok.set(j, "; return " + result + "; }");
      reTok(tok, i, j+1);
    }
    
    tok_debugStatements(tok);

    // while not null (...) / if not null (...)
    
    while ((i = jfind_check not(tok, "<id> not null (", tokcondition {
      ret eqOneOf(_get(tok, i+1), "if", "while");
    })) >= 0) {
      int closingBracket = findEndOfBracketPart(tok, i+6)-1;
      replaceTokens(tok, i+2, i+6, "(");
      tok.set(closingBracket, ") != null)");
      reTok(tok, i, closingBracket+1);
    }
    
    // while null (...) / if null (...)
    
    while ((i = jfind_check null(tok, "<id> null (", tokcondition {
      ret eqOneOf(_get(tok, i+1), "if", "while");
    })) >= 0) {
      int closingBracket = findEndOfBracketPart(tok, i+4)-1;
      replaceTokens(tok, i+2, i+4, "(");
      tok.set(closingBracket, ") == null)");
      reTok(tok, i, closingBracket+1);
    }
    
    // Replace $1 with m.unq(0) etc. - caveat: this blocks identifiers $1, $2, ...
    for (i = 1; i < l(tok); i += 2) {
      S s = tok.get(i);
      if (s.startsWith("$")) {
        s = substring(s, 1);
        if (isInteger(s)) {
          tok.set(i, "m.unq(" + (parseInt(s)-1) + ")");
          reTok(tok, i);
        }
      }
    }
  
    // instanceof trickery
    
    jreplace(tok, "is a <id>", "instanceof $3");
    jreplace(tok, "!<id> instanceof <id>.<id>", "!($2 instanceof $4.$6)");
    jreplace(tok, "!<id> instanceof <id>", "!($2 instanceof $4)");
    jreplace(tok, "<id> !instanceof <id>", "!($1 instanceof $4)");
    
    // map func1 func2 func3(...) => mapFGH(f func1, f func2, f func3, ...)
    jreplace(tok, "map <id> <id> <id>(", "mapFGH(f $2, f $3, f $4,");

    // map func1 func2(...) => mapFG(f func1, f func2, ...)
    jreplace(tok, "map <id> <id>(", "mapFG(f $2, f $3,");

    // "ref->bla" for dereferencing Concept.Ref or ThreadLocal or other
    // For lambdas, use SPACES on the left or right of the arrow!
    //jreplace(tok, "<id> ->", "$1.get().");
    jreplace(tok, "->", ".get().", tokcondition {
      ret empty(tok.get(i)) // no space on left of arrow
        && empty(tok.get(i+2)) // no space inside of arrow
        && empty(tok.get(i+4)) // no space on right of arrow
        && !eq(_get(tok, i-1), "-"); // i-->0;
    });
    
    // shortened subconcept declaration (before star constructors!)
    shortenedSubconcepts(tok);
    
    tok_beaConceptDecls(tok);
    
    // "case" as a variable name ( => _case)
    
    caseAsVariableName(tok);
    
    // "do" as a function name ( => dO)
    
    tok_doAsMethodName(tok);
    
    // "continue" as a function name ( => _continue)
    continueAsFunctionName(tok);
    
    tok_extend(tok);
    
    jreplace(tok, "pn {", "p-noconsole {");
      
    // Do these BEFORE awt replacement! ("p-awt" contains "awt" token)
    replaceKeywordBlock(tok, "r-awt", "r { awt {", "}}");
    if (hasCodeTokens(tok, "p", "-")) tok_p_old(tok);

    replaceKeywordBlock(tok, "awt-messagebox", "awt { pcall-messagebox {", "}}");
    replaceKeywordBlock(tok, "awt", "swingLater(r {", "});");
  
    jreplace(tok, "p-android {", "set flag Android. p {");
      
    unswing(tok);
    lockBlocks(tok);
    tok_switchTo(tok);
    
    // trim x;
    
    jreplace(tok, "trim <id>;", "$2 = trim($2);");
  
    // iterate with index
  
    jreplace (tok, "for <id> over <id>:", "for (int $2 = 0; $2 < l($4); $2++)");
    jreplace (tok, "for <id> backwards over <id>:", "for (int $2 = l($5)-1; $2 >= 0; $2--)");
    jreplace (tok, "for <id>, <id> <id> over <id>: {", "for (int $2 = 0; $2 < l($7); $2++) { $4 $5 = $7.get($2);");
    jreplace (tok, "for <id>, <id> <id> backwards over <id>: {", "for (int $2 = l($8)-1; $2 >= 0; $2--) { $4 $5 = $8.get($2);");
    jreplace (tok, "for <id> to <id>:", "for (int $2 = 0; $2 < $4; $2++)");
    jreplace (tok, "for <id> to <int>:", "for (int $2 = 0; $2 < $4; $2++)");
    
    tok = expandShortTypes(tok);
      
    tok_equalsCast(tok);
    tok_equalsOptCast(tok);

    replaceKeywordBlock(tok, "r-thread-messagebox", "r-thread { pcall-messagebox {", "}}");
    
    replaceKeywordBlock(tok, "thread-messagebox", "thread { pcall-messagebox {", "}}");
    
    jreplace(tok, "rThread {", "r-thread {");
    jreplace(tok, "rEnterThread {", "rThreadEnter {");
    jreplace(tok, "rThreadEnter {", "r-thread { temp enter(); ");
  
    replaceKeywordBlock(tok, "r-thread", "runnableThread(r {", "})");
    rNamedThread(tok);
    
    // only works in the scope of a DynModule
    jreplace(tok, "rEnter {", "r { temp enter(); ");
  
    replaceKeywordBlock(tok, "r-pcall", "r { pcall {", "}}");
  
    replaceKeywordBlock(tok, "r-messagebox", "r { pcall-messagebox {", "}}");
    
    jreplace(tok, "r <id> + r <id>", "r { $2(); $5(); }");
    
    // runnable and r - now also with automatic toString if enabled
    for (S keyword : ll("runnable", "r")) {
      while ((i = jfind(tok, keyword + " {")) >= 0) {
        int idx = findCodeTokens(tok, i, false, "{");
        int j = findEndOfBracketPart(tok, idx);
        L<S> contents = subList(tok, idx+1, j-1);
        replaceTokens(tok, i, j+1, "new Runnable {"
          + "  public void run() ctex { " + tok_addSemicolon(contents) + "\n}"
          + (autoQuine ? tok_autoQuineFunc(contents) : "")
          + "}");
        reTok(tok, i, j+1);
      }
      
      while ((i = jfind(tok, keyword + " <quoted> {")) >= 0) {
        int idx = findCodeTokens(tok, i, false, "{");
        int j = findEndOfBracketPart(tok, idx);
        L<S> contents = subList(tok, idx+1, j-1);
        replaceTokens(tok, i, j+1, "new Runnable {"
          + "  public void run() ctex { " + tok_addSemicolon(contents) + "\n}"
          + "  toString { ret " + tok.get(i+2) + "; }"
          + (autoQuine ? tok_autoQuineFunc(contents, '_shortenedSourceCode) : "")
          + "}");
        reTok(tok, i, j+1);
      }
    }
    
    replaceKeywordBlock(tok,
      "expectException",
      "{ bool __ok = false; try {",
      "} catch { __ok = true; } assertTrue(\"expected exception\", __ok); }");
      
    while ((i = tok.indexOf("tex")) >= 0) {
      tok.set(i, "throws Exception");
      tok = jtok(tok);
    }
    
    // shorter & smarter whiles
    
    jreplace(tok, "while true", "while (true)");
    jreplace(tok, "while licensed", "while (licensed())");
    jreplace(tok, "repeat {", "while (licensed()) {");
    tok_repeatWithSleep(tok);
    
    // null; => return null; etc.
  
    O cond = tokcondition {
      ret tok_tokenBeforeLonelyReturnValue(tok, i-1);
    };
    jreplace(tok, "null;", "return null;", cond);
    jreplace(tok, "false;", "return false;", cond);
    jreplace(tok, "true;", "return true;", cond);
    jreplace(tok, "this;", "return this;", cond);
    
    // ok <cmd> => ret "OK" with <cmd>
    jreplace(tok, "ok <id>", "return \"OK\" with $2");
    replaceKeywordBlock(tok, "ok", "{", " return \"OK\"; }",
      func(LS _tok, int nIdx) -> bool { !eqGetOneOf(_tok, nIdx-1, "void", "svoid") });
  
    // "myFunction;" instead of "myFunction();" - quite rough
    // (isolated identifier as function call)
    cond = new TokCondition {
      public bool get(L<S> tok, int i) {
        S word = tok.get(i+3);
        //print("single word: " + word);
        ret !eqOneOf(word, "break", "continue", "return", "else", "endifdef", "endif");
      }
    };
    for (S pre : litlist("}", ";"))
      jreplace(tok, pre + " <id>;", "$1 $2();", cond);
  
    // shorter match syntax for answer methods
    
    tok_expandIfQuoted(tok);
    
    jreplace(tok, "if <id> eq <quoted>", "if (eq($2, $4))");
  
    tok_dropExtraCommas(tok);
    
    // additional translations (if necessary)
    
    jreplace(tok, "pcall ping {", "pcall { ping();");
    
    replaceKeywordBlock(tok, ") pcall",
      ") { pcall {",
      "}}");
    
    //jreplace(tok, "void <id> pcall {", "$1 $2() pcall {");
    
    replaceKeywordBlock(tok,
      "pcall",
      "try {",
      "} catch (Throwable __e) { printStackTrace(__e); }");
  
    replaceKeywordBlock(tok,
      "pcall-print",
      "try {",
      "} catch (Throwable __e) { printStackTrace(__e); }");
  
    replaceKeywordBlock(tok,
      "pcall-short",
      "try {",
      "} catch (Throwable __e) { print(exceptionToStringShort(__e)); }");
  
    replaceKeywordBlock(tok,
      "pcall-silent",
      "try {",
      "} catch (Throwable __e) { silentException(__e); }");
  
    replaceKeywordBlock(tok,
      "pcall-messagebox",
      "try {",
      "} catch __e { messageBox(__e); }");
  
    replaceKeywordBlock(tok,
      "pcall-infobox",
      "try {",
      "} catch __e { infoBox(__e); }");
  
    tok = dialogHandler(tok);
    
    replaceKeywordBlock(tok, "exceptionToUser",
      "try {",
      "} catch (Throwable __e) { ret exceptionToUser(__e); }"); 
  
    if (hasCodeTokens(tok, "twice", "{"))
      replaceKeywordBlock(tok, "twice",
        "for (int __twice = 0; __twice < 2; __twice++) {",
        "}"); 
  
    while ((i = findCodeTokens(tok, "bench", "*", "{")) >= 0) {
      int j = findEndOfBracketPart(tok, i+4)-1;
      S time = makeVar("time");
      S v = makeVar("bench");
      S n = tok.get(i+2);
      tok.set(i, "{ long " + time + " = sysNow(); for (int " + v + " = 0; " + v + " < " + n + "; " + v + "++)");
      tok.set(i+2, "");
      tok.set(j, "} printBenchResult(sysNow()-" + time + ", " + n + "); }");
      reTok(tok, i, j+1);
    }
  
    replaceKeywordBlockDyn(tok,
      "time",
      new O { S[] get() {
        S var = makeVar("startTime");
        ret new S[] {
          "{ long " + var + " = sysNow(); try { ",
          "} finally { " + var + " = sysNow()-" + var + "; saveTiming(" + var + "); } }"};
      }});
    
    // version without { }
    replaceKeywordBlockDyn(tok,
      "time2",
      new O { S[] get() {
        S var = makeVar("startTime");
        ret new S[] {
          "long " + var + " = sysNow(); ",
          " " + var + " = sysNow()-" + var + "; saveTiming(" + var + "); "};
      }});
    
    // time "bla" {
    // time msg {
    replaceKeywordPlusQuotedOrIDBlock(tok,
      "time",
      new O { S[] get(L<S> tok, int i) {
        S var = makeVar("startTime");
        ret new S[] {
          "long " + var + " = sysNow(); ",
          " done2_always(" + tok.get(i+2) + ", " + var + "); "};
      }});
    
    if (hasCodeTokens(tok, "assertFail", "{")) {
      S var = makeVar("oops");
      
      replaceKeywordBlock(tok,
        "assertFail",
        "boolean " + var + " = false; try {",
        "\n" + var + " = true; } catch (Exception e) { /* ok */ } assertFalse(" + var + ");"); 
    }
    
    replaceKeywordBlock(tok,
      "yo",
      "try {",
      "} catch (Exception " + makeVar("e") + ") { ret false; }",
      tokcondition {
        ret neqOneOf(_get(tok, i-1), "svoid", "void");
      });
  
    replaceKeywordBlock(tok,
      "awtIfNecessary",
      "swingNowOrLater(r " + "{",
      "});");
      
    ctex(tok);
      
    replaceKeywordBlock(tok,
      "actionListener",
      "new java.awt.event.ActionListener() { " +
        "public void actionPerformed(java.awt.event.ActionEvent _evt) { pcall-messagebox {",
      "}}}");
      
    for (S keyword : ll("autocloseable", "autoCloseable"))
      /*replaceKeywordBlock(tok,
        keyword,
        "new AutoCloseable() { public void close() throws Exception {",
        "}}");*/
        replaceKeywordBlock_dyn2_legacy(tok, keyword, new O {
          S[] get(LS tok, int iOpening, int iClosing) {
            LS contents = subList(tok, iOpening+1, iClosing);
            ret new S[] {
              "new AutoCloseable() { toString { ret " + quote(shorten(defaultMaxQuineLength(), trimJoin(contents))) + "; } public void close() throws Exception {",
              "}}"
            };
          }
        });
      
    // try answer (string, test with nempty)
    while ((i = findCodeTokens(tok, "try", "answer")) >= 0) {
      int j = findEndOfStatement(tok, i);
      S v = makeVar();
      bool needCurly = !eqGet(tok, i-2, "{");
      tok.set(i, (needCurly ? "{" : "") + " S " + v);
      tok.set(i+2, "=");
      tok.set(j-1, "; if (!empty(" + v + ")) ret " + v + "; " + (needCurly ? "}" : ""));
      reTok(tok, i, j);
    }
  
    // try bool[ean] (try answer with Bool type)
    while ((i = findCodeTokens(tok, "try", "boolean")) >= 0) {
      int j = findEndOfStatement(tok, i);
      S v = makeVar();
      tok.set(i, "{ Bool " + v);
      tok.set(i+2, "=");
      tok.set(j-1, "; if (" + v + " != null) ret " + v + "; }");
      reTok(tok, i, j);
    }
    
    // <statement>, print "..."; => { <statement>; print("..."); }
    while ((i = jfind(tok, ", print <quoted>;")) >= 0) {
      int j = tok_findBeginningOfStatement(tok, i);
      replaceTokens_reTok(tok, i, i+4*2-1, "; print(" + tok.get(i+4) + "); }");
      tokPrepend_reTok(tok, j, "{ ");
    }
    
    // return if null <expression> => if (<expression> == null) return;
    while ((i = jfind(tok, "return if null")) >= 0) {
      int j = findEndOfStatement(tok, i);
      clearTokens(tok, i, i+2);
      tok.set(i+4, "((");
      tok.set(j-1, ") == null) ret;");
      reTok(tok, i, j);
    }
    
    // return optional (return if not null)
    while ((i = jfind_check optional(tok, "return optional <id> =")) >= 0) {
      int j = findEndOfStatement(tok, i);
      S v = tok.get(i+4);
      clearTokens(tok, i+2, i+4);
      tok.set(i, "{");
      tok.set(j-1, "; if (" + v + " != null) ret " + v + "; }");
      reTok(tok, i, j);
    }
    
    // try object (return if not null)
    
    jreplace_dyn(tok, "try object (<id>)", (_tok, cIdx) -> {
      S type = _tok.get(cIdx+6);
      ret "try object " + type + " " + makeVar() + " = (" + type + ")";
    });
    
    while ((i = jfind_check object(tok, "try object")) >= 0) {
      int j = findEndOfStatement(tok, i);
      clearTokens(tok, i, i+3);
      bool isDecl = isIdentifier(get(tok, i+4)) && isIdentifier(get(tok, i+6)) && eqGet(tok, i+8, "=");
      if (isDecl) {
        S v = get(tok, i+6);
        tok.set(i, "{");
        tok.set(j-1, "; if (" + v + " != null) ret " + v + "; }");
      } else {
        S v = makeVar();
        tok.set(i, "{ O " + v + "=");
        tok.set(j-1, "; if (" + v + " != null) ret " + v + "; }");
      }
      reTok(tok, i, j);
    }
    
    // try Int i = ...; (return if not null, shorter version of "try object")
    /*while ((i = jfind(tok, "try <id> <id> =")) >= 0) {
      int j = findEndOfStatement(tok, i);
      S type = tok.get(i+2), v = tok.get(i+4);
      tok.set(i, "{");
      tok.set(j-1, "; if (" + v + " != null) ret " + v + "; }");
      reTok(tok, i, j);
    }*/
    while ((i = jfind(tok, "try <id>")) >= 0) {
      int iType = i+2, iVar = tok_findEndOfType(tok, iType);
      S v = tok.get(iVar);
      assertEquals("try object", "=", get(tok, iVar+2));
      int j = findEndOfStatement(tok, iVar);
      tok.set(i, "{");
      tok.set(j-1, "; if (" + v + " != null) ret " + v + "; }");
      reTok(tok, i, j);
    }
    
    // debug print ...; => if (debug) print(...);
    while ((i = jfind(tok, "debug print")) >= 0) {
      int j = findEndOfStatement(tok, i);
      replaceTokens(tok, i, i+4, "if (debug) print(");
      tok.set(j-1, ");");
      reTok(tok, i, j);
    }
    
    functionReferences(tok);
    tok_expandLPair(tok);
    tok_expandPairL(tok);
    tok_expandLT3(tok);
    tok_quicknew2(tok);
    
    // before temp blocks
    tok_varNamedLikeFunction(tok);
    
    tempBlocks(tok); // after quicknew2 for stuff like "temp new X x;"

    // X x = nu(+...)  =>  X x = nu X(+...)
    jreplace(tok, "<id> <id> = nu(+", "$1 $2 = nu $1(+");
    // X x = nu(a := ...)  =>  X x = nu X(a := ...)
    jreplace(tok, "<id> <id> = nu(<id> :=", "$1 $2 = nu $1($6 :=");
    
    tok_expandVarCopies(tok); // AFTER the lines just above
    tok_replaceColonEqualsSyntax(tok); // ditto

    tok_unpair(tok);
    tok_cachedFunctions(tok);
    tok_simplyCachedFunctions(tok);
    tok_timedCachedFunctions(tok);
    
    tok_optPar(tok);
    
    throwFailEtc(tok);
    tok_typeAA(tok, pairClasses);
    tok_typeAAA(tok, tripleClasses);
    tok_typeAO(tok, litset("WithTrail"));

    // before star constructors so we can define star constructors in a macro
    tok_localMacro(tok);

    // do this after expanding sclass etc.
    tok = tok_expandStarConstructors(tok);
    
    tok_kiloConstants(tok);
    //tok_colonMessages(tok);
    
    while ((i = jfind(tok, "shit:")) >= 0) {
      int j = tok_findEndOfStatement(tok, i);
      tok.set(i+2, "(");
      tok.set(j-1, ");");
      reTok(tok, i, j);
    }
      
    jreplace("shit(", "ret with print(");
    
    tok_virtualTypes(tok);
    tok_autoLongConstants(tok);
    
    // common misordering of type arguments
    jreplace("boolean <A>", "<A> boolean");
    
    tok_unimplementedMethods(tok);
    tok_switchableFields(tok);
    tok_autoDisposeFields(tok);
    tok_shortVisualize(tok);
    tok_whileGreaterThan(tok);
    tok_ifThenEnd(tok);
    
    tok_autoInitVars(tok);
    
    tok_fixBadTypeParameterOrder(tok);
    
    // shortened method declarations BEFORE standardFunctions
    tok_svoidEtc(tok);
    jreplace(tok, "void <id> {", "$1 $2() {");
    jreplace(tok, "void <id> thread {", "$1 $2() thread {");
    jreplace(tok, "String <id> {", "$1 $2() {");
    jreplace(tok, "Object <id> {", "$1 $2() {");
    jreplace(tok, "List <id> {", "$1 $2() {");

    namedThreads(tok);
    threads(tok);
    
    //tok_maxEquals(tok);
    
    tok_questionDot(tok);
    
    jreplace(tok, "if (<id>?!)", "if ($3 != null && $3!)");
    
    tok_embeddedFunctions(tok);
    
    // quicknew for arrays
    
    jreplace("<id>[] <id> = new[", "$1[] $4 = new $1[");
    jreplace("<id>[] <id> = new {", "$1[] $4 = {");
    jreplace("<id><<id>>[] <id> = new[", "$1 $2 $3 $4[] $7 = new $1[");
    
    jreplace("<id> ifNull =", "if ($1 == null) $1 =");
    
    jreplace("class <id> extends <id><.<id>>", "class $2 extends $4<$2.$7>");
    
    tok_once(tok); // must come before next line so you can combine them
    tok_ifRecordMatch(tok);
    
    jreplace(tok, "<id> ||=", "$1 = $1 ||");
    
    // magicValue followed by a numerical constant
    jreplace("magicValue", "", (_tok, _i) -> {
      S t = _get(_tok, _i+3);
      ret eqOneOf(t, ".", "-") || isInteger(t);
    });
    
    lambdaReferences(tok);
    
    tok_returnSelf(tok);
    
    // Lua-like print statement
    jreplace(tok, "print <quoted>", "print($2);");
    
    tok_tildeCalls(tok);
    
    tok_svoidEtc(tok);
    tok_swappableFunctions(tok);
    
    tok_optParLambda(tok);
    
    tok_runnableClasses(tok);
    
    tok_swapStatement(tok);
    
    //defineMapLikesEtc(tok);
    
    tok_defaultArguments_debug = isDef("tok_defaultArguments_debug"); 
    tok_defaultArguments(tok);
    
    tok_akaFunctionNames(tok);
    
    tok_persistableClasses(tok);
    
    tok_transientClasses(tok);
    
    tok_shortMethodReferences(tok);
    
    jreplace(tok, "== null ?:", "== null ? null :");
    jreplace(tok, "?:", "== null ? null :");
    
    tok_orCase(tok);
    
    tok_beforeMethods(tok);
    tok_afterMethods(tok);
    
    tok_returnAsFunctionName(tok);
    
    tok_transpileGetSet(tok);
    
    tok_pcallPrefix(tok);
    
    tok_numberFunctionNames(tok);
    
    tok_castToStatements(tok);
    
    tok_optionalFields(tok);
    
    tok_getFromMap(tok);
    
    jreplace(tok, "for <id> : <id> {", "for (var $2 : $4) {");
    
    tok_shortLambdas(tok);
    
    if (metaCodeAllowed()) runTransformers(tok, localTransformers);
    
    tok_andVarMixIn(tok);
    
    tok_stages(tok);
    
    tok_beginPeriod(tok);
    
    // end of local stuff
    
    tok_processMetaBlocks(tok, metaCodeAllowed());
    if (metaCodeAllowed()) runTransformers(tok, metaTransformers);

    same = eq(tok, before);
    /*if (!same)
      print("local not same " + safety + " (" + l(tok) + " tokens)");*/
    if (safety++ >= 10) {
      printSources(tok);
      fail("safety 10 error!");
    }
  } while (!same);
  
  ret tok;
}

static L<S> reTok_include(L<S> tok, int i, int j) {
  ret reTok_modify(tok, i, j, f localStuff1);
}

static L<S> includeInMainLoaded_reTok(L<S> tok, int i, int j) {
  ret reTok_include(tok, i, j);
}

static L<S> stdstuff(L<S> tok) {
  //if (++level >= 10) fail("woot? 10");
  
  print("stdstuff!");
  int i;
  
  new L<S> ts;
  tok_findTranslators(tok, ts);
  if (nempty(ts))
    print("DROPPING TRANSLATORS: " + structure(ts));

  print('quickmain); tok = quickmain(tok);
  print('includes); tok = tok_processIncludes(tok);
  print('conceptsDot); if (processConceptsDot(tok))
  tok = tok_processIncludes(tok);
  
  LS dontImports = tok_processDontImports(tok, definitions);
  for (S s : dontImports)
    doNotIncludeFunction.remove(afterLastDot(s));
  
  //print('starConstructors); tok = tok_expandStarConstructors(tok);
    
  // drop Java 8 annotations since we're compiling for Java 7
  jreplace(tok, "@Nullable", "");

  // STANDARD CLASSES & INTERFACES
  
  print("standard classes");
  haveClasses = addStandardClasses_v2(tok);
  
  tok_quickInstanceOf(tok, haveClasses);

  // concept-related stuff
  
  // auto-import concepts
  /*bool _a = tok_hasClassRef2(tok, "Concept") || tok_hasClassRef2(tok, "Concepts"), _b = !haveClasses.contains("Concept");
  //print("auto-import: " + _a + ", " + _b);
  if (_a && _b) {
    print("Auto-including concepts.");
    if (shouldNotIncludeClass.contains("Concepts")) {
      print(join(tok));
      fail("Unwanted concepts import");
    }
    printStruct(haveClasses);
    tok = includeInMainLoaded(tok, "concepts.");
    reTok(tok, l(tok)-1, l(tok));
    //processConceptsDot(tok);
  }*/
  
  ret tok;
} // end of stdStuff!

static L<S> multilineStrings(L<S> tok) {
  for (int i = 1; i < tok.size(); i += 2) {
    S t = tok.get(i);
    if (isQuoted(t))
      if (t.startsWith("[") || t.contains("\r") || t.contains("\n"))
        tok.set(i, quote(unquote(t)));
  }
  ret tok;
}

static L<S> quickmain(L<S> tok) {
  if (quickmainDone1 && quickmainDone2) ret tok;

  int i = findCodeTokens(tok, "main", "{");
  if (i < 0) i = findCodeTokens(tok, "m", "{");
  if (i >= 0 && !(i-2 > 0 && tok.get(i-2).equals("class"))) {
    tokSet_reTok(tok, i, (definitions.contains("mainClassPublic") ? "public " : "") + "class main");
    quickmainDone1 = true;
  }
    
  i = findCodeTokens(tok, "psvm", "{");
  if (i < 0) i = findCodeTokens(tok, "p", "{");
  if (i >= 0) {
    int idx = i+2;
    int j = findEndOfBracketPart(tok, idx)-1;
    L<S> contents = subList(tok, idx+1, j);
    //print("contents: " + sfu(contents));
    tok.set(i, "public static void main(final String[] args) throws Exception");
    replaceTokens(tok, idx+1, j, tok_addSemicolon(contents));
    reTok(tok, i, j);
    quickmainDone2 = true;
  }
    
  ret tok;
}

sS makeVar(S name) {
  AtomicInteger counter = varCountByThread!;
  if (counter == null)
    varCountByThread.set(counter = new AtomicInteger);
  ret "_" + name + "_" + getAndInc(counter);
}

static S makeVar() { ret makeVar(""); }

static L<S> rtq(L<S> tok, S id) {
  ret runTranslatorQuick(tok, id);
}

static L<S> expandShortTypes(L<S> tok) {
  // replace <int> with <Integer>
  for (int i = 1; i+4 < tok.size(); i += 2)
    if (tok.get(i).equals("<")
      && litlist(">", ",").contains(tok.get(i+4))) {
      String type = tok.get(i+2);
      if (type.equals("int")) type = "Integer";
      else if (type.equals("long")) type = "Long";
      tok.set(i+2, type);
    }
    
  jreplace(tok, "O", "Object");
  jreplace(tok, "S", "String");
  jreplace(tok, "L", "List");
  jreplace(tok, "Cl", "Collection");
  
  // bool -> boolean if it's not a function name
  jreplace(tok, "bool", "boolean", tokCondition {
    ret neqGetOneOf(tok, i+3, "(", null);
  });
  
  jreplace(tok, "AtomicBool", "AtomicBoolean");
  jreplace(tok, "AtomicInt", "AtomicInteger");

  jreplace(tok, "LL< <id> >", "L<L<$3>>");
  jreplace(tok, "LL< <id><<id>> >", "L<L<$3<$5>>>");
  jreplace(tok, "LL<?>", "L<L<?>>");
  jreplace(tok, "LL", "L<L>");
  jreplace(tok, "LPt", "L<Pt>");
  
  jreplace(tok, "Clusters< <id> >", "Map<$3, Collection<$3>>");
  
  ret tok;
}

static L<S> autoImports(L<S> tok) {
  HashSet<S> imports = new HashSet(tok_findImports(tok));
  new StringBuilder buf;
  for (S c : standardImports())
    if (!(imports.contains(c)))
      buf.append("import " + c + ";\n");
  if (buf.length() == 0) ret tok;
  tok.set(0, buf+tok.get(0));
  ret reTok(tok, 0, 1);
}

static L<S> tok_processIncludes(L<S> tok) {
  int safety = 0;
  while (hasCodeTokens(tok, "!", "include") && ++safety < 100)
    tok = tok_processIncludesSingle(tok);
  
  //tok_autoCloseBrackets(tok);
  ret tok;
}

svoid tok_processEarlyIncludes(L<S> tok) {
  int i;
  while ((i = jfind_check include(tok, "!include early #<int>")) >= 0) {
    S id = tok.get(i+8);
    included.add(parseLong(id));
    replaceTokens_reTok(tok, i, i+10, "\n" + cacheGet(id) + "\n");
  }
}

static L<S> tok_processIncludesSingle(L<S> tok) {
  int i;
  while ((i = jfind_check include(tok, "!include #<int>")) >= 0) {
    S id = tok.get(i+6);
    included.add(parseLong(id));
    replaceTokens(tok, i, i+8, "\n" + cacheGet(id) + "\n");
    reTok_include(tok, i, i+8);
  }
  while ((i = jfind_check include(tok, "!include once #<int>")) >= 0) {
    S id = tok.get(i+8);
    bool isNew = included.add(parseLong(id));
    replaceTokens(tok, i, i+10, 
      isNew ? "\n" + cacheGet(id) + "\n" : "");
    reTok_include(tok, i, i+10);
  }
  ret tok;
}

// ctex and various other error-related keyword blocks
static void ctex(L<S> tok) {
  replaceKeywordBlock(tok, "ctex",
    "{ try {",
    "} catch (Exception __e) { throw rethrow(__e); } }");
  for (S keyword : ll("null on exception", "null on error"))
    replaceKeywordBlock(tok, keyword,
      "{ try {",
      "} catch (Throwable __e) { return null; } }");
  replaceKeywordBlock(tok, "false on exception",
    "{ try {",
    "} catch (Throwable __e) { return false; } }");
  replaceKeywordBlock(tok, "try-OrError",
    "{ try {",
    "} catch (Throwable __e) { return OrError.error(__e); } }");
}
  
static L<S> dialogHandler(L<S> tok) {
  ret replaceKeywordBlock(tok,
    "dialogHandler",
    "new DialogHandler() {\n" +
      "public void run(final DialogIO io) {",
    "}}");
}

static L<S> extendClasses(L<S> tok) {
  int i;
  while ((i = jfind(tok, "extend <id> {")) >= 0) {
    S className = tok.get(i+2);
    int idx = findCodeTokens(tok, i, false, "{");
    int j = findEndOfBracketPart(tok, idx+2);
    S content = joinSubList(tok, idx+1, j-1);
    L<S> c = findInnerClassOfMain(tok, className);
    print("Extending class " + className);
    clearTokens(subList(tok, i, j+1));
    if (c == null) {
      print("Warning: Can't extend class " + className + ", not found");
      continue;
    }
    int startOfClass = indexOfSubList(tok, c);
    int endOfClass = startOfClass + l(c)-1;
    //print("Extending class " + className + " ==> " + join(subList(tok, startOfClass, endOfClass)));
    while (neq(tok.get(endOfClass), "}")) --endOfClass;
    //print("Extending class " + className + " ==> " + join(subList(tok, startOfClass, endOfClass)));
    tok.set(endOfClass, content + "\n" + tok.get(endOfClass));
    
    doubleReTok(tok, i, j+1, endOfClass, endOfClass+1);
  }
  ret tok;
}

// for ping / while ping
static void forPing(L<S> tok) {
  int i;
  for (S keyword : ll("for", "fOr", "while"))
    while ((i = jfind(tok, keyword + " ping (")) >= 0) {
      int bracketEnd = findEndOfBracketPart(tok, i+4)-1;
      int iStatement = bracketEnd+2;
      int iEnd = findEndOfStatement(tok, iStatement);
      tok.set(i+2, "");
      
      // turn into block
      if (!eq(get(tok, iStatement), "{")) {
        tok.set(iStatement, "{ " + tok.get(iStatement));
        tok.set(iEnd-1, tok.get(iEnd-1) + " }");
      }
        
      // add ping
      tok.set(iStatement, "{ ping(); " + dropPrefixTrim("{", tok.get(iStatement)));
      reTok(tok, i+2, iEnd);
    }
}

// lib 123 => !123
static void libs(L<S> tok) {
  int i;
  while ((i = jfind(tok, "lib <int>")) >= 0) {
    S id = tok.get(i+2);
    print("lib " + id);
    if (!libs.contains(id)) {
      libs.add(id);
      clearAllTokens(tok, i, i+3);
      /*tok.set(i, "!");
      tok.set(i+1, "");*/
    } else {
      print("...ignoring (duplicate)");
      clearAllTokens(tok, i, i+3);
      reTok(tok, i, i+3);
    }
  }
  print("libs found: " + libs);
}

// sourceCodeLine() => 1234
static void sourceCodeLine(L<S> tok) {
  int i ;
  while ((i = jfind(tok, "sourceCodeLine()")) >= 0) {
    replaceTokens(tok, i, i+5, str(countChar(joinSubList(tok, 0, i), '\n')+1));
    reTok(tok, i, i+5);
  }
}

// done before any other processing
svoid earlyStuff(LS tok) {
  int i;
  
  tok_processEarlyIncludes(tok);
      
  tok_scopes(tok, autoCloseScopes := asInclude);
  tok_autosemi(tok);
  tok_autoCloseBrackets(tok);
  jreplace(tok, "°", "()");
  
  // Note: this makes the word "quine" a special operator
  // (unusable as a function name)
  
  while ((i = jfind(tok, "quine(")) >= 0) {
    int idx = findCodeTokens(tok, i, false, "(");
    int j = findEndOfBracketPart(tok, idx+2);
    tok.set(i, "new Quine");
    tok.set(idx, "(" + quote(joinSubList(tok, idx+1, j-1)) + ", ");
    reTok(tok, i, idx+1);
  }
  
  jreplace_check after(tok, "void <id> after super {", "void $2 { super.$2();");
  
  replaceKeywordBlock(tok, "r-enter", "r { enter {", "}}");
    
  // do this before func & voidfunc because otherwise they swallow it
  jreplace(tok, "enter {", "{ temp enter();");
  
  tok_mutatorMethods(tok);
  
  // func keyword for lambdas - now automatically quines toString() if enabled
  
  replaceKeywordBlock(tok,
    "null",
    "{",
    "null; }");
    
  // do func & voidfunc early to preserve original code as toString
  
  while ((i = jfind(tok, "func(")) >= 0) {
    int argsFrom = i+4, argsTo = findCodeTokens(tok, i, false, ")");
    int idx = findCodeTokens(tok, argsTo, false, "{");
    int j = findEndOfBracketPart(tok, idx);
    L<S> contents = subList(tok, idx+1, j-1);
    
    S returnType = "O";
    if (eq(tok.get(argsTo+2), "-") && eq(tok.get(argsTo+4), ">"))
      returnType = tok_toNonPrimitiveTypes(joinSubList(tok, argsTo+6, idx-1));
      
    S toString = autoQuine ? "  public S toString() { ret " + quote(shorten(defaultMaxQuineLength(), trimJoin(contents))) + "; }" : "";
    
    L<S> args = cloneSubList(tok, argsFrom-1, argsTo);
    tok_shortFinals(args);
    tok_toNonPrimitiveTypes(args);
    
    L<S> types = tok_typesOfParams(args);
    O type = "O";
    if (l(types) <= 3)
      type = "F" + l(types) + "<" + joinWithComma(types) + ", " + returnType + ">";
    S body = tok_addReturn(contents);
    
    replaceTokens_reTok(tok, i, j,
      "new " + type + "() { public "
        + returnType + " get(" + trimJoin(args) + ") ctex { "
          + body
        + " }\n" +
       + toString + "}");
  }
  
  while ((i = jfind(tok, "voidfunc(")) >= 0) {
    int argsFrom = i+4, argsTo = findCodeTokens(tok, i, false, ")");
    int idx = findCodeTokens(tok, argsTo, false, "{");
    int j = findEndOfBracketPart(tok, idx);
    L<S> contents = subList(tok, idx+1, j-1);
    
    if (jcontains(subList(tok, argsTo+1, idx), "->"))
      fail("voidfunc with return type: " + joinSubList(tok, i, j+1));
    
    L<S> args = cloneSubList(tok, argsFrom-1, argsTo);
    tok_shortFinals(args);
    tok_toNonPrimitiveTypes(args);
    
    L<S> types = tok_typesOfParams(args);
    O type = "O";
    if (l(types) <= 4)
      type = "VF" + l(types) + "<" + joinWithComma(types) + ">";

    replaceTokens(tok, i, j, "new " + type + "() { public void get(" + trimJoin(args) + ") ctex { " + tok_addSemicolon(contents) + " }\n" +
    (autoQuine ? "  public S toString() { ret " + quote(shorten(defaultMaxQuineLength(), trim(join(contents)))) + "; }" : "") + "}");
    reTok(tok, i, j);
  }
  
  jreplace(tok, "func {", "func -> O {");
  jreplace(tok, "f {", "f -> O {");
  
  // swing -> S { ... } => swing(func -> S { ... })
  while ((i = jfind(tok, "swing ->")) >= 0) {
    int bracket = findCodeTokens(tok, i, false, "{");
    int j = findEndOfBracketPart(tok, bracket);
    tok.set(i, "swing(func");
    tok.set(j-1, "})");
    reTok(tok, i, j);
  }
  
  tok_qFunctions(tok);
  
  jreplace(tok, "func fullToString {", "func -> O fullToString {");
  
  for (S keyword : ll(/*"f",*/ "func")) {
    while ((i = jfind(tok, keyword + " ->", tokcondition {
      ret isIdentifier(_get(tok, i+7)); // avoid lambda declaration like: f -> { ... }
    })) >= 0) {
      // I think there is a bug here for something like func -> x { new x { } }
      int idx = findCodeTokens(tok, i, false, "{");
      int j = findEndOfBracketPart(tok, idx);
      S returnType = tok_toNonPrimitiveTypes(joinSubList(tok, i+6, idx-1));
      int quineLength = defaultMaxQuineLength();
      if (eq(lastJavaToken(returnType), "fullToString")) {
        quineLength = Int.MAX_VALUE;
        returnType = dropLastJavaTokenAndSpacing(returnType);
      }
      L<S> contents = subList(tok, idx+1, j-1);
      replaceTokens(tok, i, j, "new F0<" + returnType + ">() { public " + returnType + " get() ctex { " + tok_addReturn(contents) + " }\n" +
        (autoQuine ? "  public S toString() { ret " + quote(shorten(quineLength, trimJoin(contents))) + "; }" : "") + "}");
      reTok(tok, i, j);
    }
  }
    
  while ((i = jfind(tok, "time repeat * {")) >= 0) {
    int j = findEndOfBlock(tok, i+6)-1;
    tok.set(i, "time {");
    tok.set(j, "}}");
    reTok(tok, i, j+1);
  }

  // do this before "m {" stuff because "repeat m" [doesn't seem to work yet though)
  while ((i = findCodeTokens(tok, "repeat", "*", "{")) >= 0) {
    S v = makeVar("repeat");
    tok.set(i, "for (int " + v + " = 0; " + v + " < " + tok.get(i+2) + "; " + v + "++)");
    tok.set(i+2, "");
    reTok(tok, i, i+3);
  }
}

static void throwFailEtc(L<S> tok) {
  bool anyChange = false;
  for (int i = 1; i+4 < l(tok); i += 2)
    if (eqOneOf(get(tok, i+2), "fail", "todo")
      && eq(get(tok, i+4), "(")
      && !eqOneOf(get(tok, i), "throw", "RuntimeException", "return", "f", "function", ".", "void", "main", "Throwable")) {
      tok.set(i+2, "throw " + tok.get(i+2));
      anyChange = true;
    }
  if (anyChange)
    reTok(tok);
}

static void namedThreads(L<S> tok) {
  for (int i = 0; i < 100; i++) {
    int idx = findCodeTokens(tok, "thread", "<quoted>", "{");
    if (idx < 0) idx = findCodeTokens(tok, "thread", "<id>", "{");
    if (idx < 0)
      break;
    int j = findEndOfBracketPart(tok, idx+4);
    S tName = tok.get(idx+2);
    
    S pre = "startThread(" + tName + ", r { ";
    S post = "})";
    if (!tok_tokenLeftOfExpression(tok, idx-2)) post += ";";

    tok.set(idx, pre);
    tok.set(idx+2, "");
    tok.set(idx+4, "");
    tok.set(j-1, post);
    //print(join(subList(tok, idx, j)));
    reTok(tok, idx, j);
  }
}

static void rNamedThread(L<S> tok) {
  for (int i = 0; i < 100; i++) {
    int idx = findCodeTokens(tok, "r", "-", "thread", "<quoted>", "{");
    if (idx < 0) idx = findCodeTokens(tok, "r", "-", "thread", "<id>", "{");
    if (idx < 0)
      break;
    int j = findEndOfBracketPart(tok, idx+8);
    S tName = tok.get(idx+6);
    
    S pre = "r { thread " + tName + " {";
    S post = "}}";

    replaceTokens(tok, idx, idx+9, pre);
    tok.set(j-1, post);
    reTok(tok, idx, j);
  }
}

static void threads(L<S> tok) {
  replaceKeywordBlock(tok, "daemon", "startDaemonThread(r {", "});");
  //replaceKeywordBlock(tok, "thread", "startThread(r {", "});");
  
  // now enclose in { } to allow using like "void bla() thread { ... }" - if it's not used as an expression
  
  replaceKeywordBlock_dyn2_legacy(tok, "thread", new O { // don't use func here, it can't be transpiled
    S[] get (LS tok, int iOpening, int iClosing) {
      // is the thread clause used as an expression?
      bool isExpression = tok_tokenLeftOfExpression(tok, iOpening-4);
      ret new S[] {
        (isExpression ? "" : "{ ") + "startThread(r {",
        "})" +
        (isExpression ? "" : "; }")
      };
    }
  });
}

static SS sf; // standard standard functions (haha)
static Bool _761ok;

static LS standardFunctions(LS tok) {
  if (sf == null) {
    S _761 = cacheGet("#761");
    if (_761ok == null)
      _761ok = isBracketHygienic(_761);
    assertTrue("Whoa! #761 truncated?", _761ok);
    L<S> standardFunctions = concatLists(
      (L) loadVariableDefinition(_761, "standardFunctions"),
      (L) loadVariableDefinition(cacheGet("#1006654"), "standardFunctions"));

    sf = new HashMap;
    for (S x : standardFunctions) {
      S[] f = x.split("/");
      sf.put(f[1], f[0]);
    }
  }
  
  SS sfMap = combinedMap(extraStandardFunctions, sf);
    
  for (int i = 0; ; i++) {
    print("Looking for required functions");
    Set<S> defd = new HashSet(findFunctions(tok));
    
    int j;
    while ((j = jfind(tok, "should not include function *.")) >= 0) {
      S fname = tok.get(j+8);
      shouldNotIncludeFunction.add(fname);
      clearAllTokens(subList(tok, j, j+12));
    }

    while ((j = jfind(tok, "do not include function *.")) >= 0) {
      S fname = tok.get(j+8);
      doNotIncludeFunction.add(fname);
      clearAllTokens(subList(tok, j, j+12));
    }
    
    while ((j = jfind(tok, "need latest <id>.")) >= 0) {
      S t = tok.get(j+4);
      needLatest.add(t);
      doNotIncludeClass.remove(t);
      clearAllTokens(subList(tok, j, j+8));
    }
    
    grabImportedStaticFunctions(tok);
    doNotIncludeFunction.removeAll(needLatest);
    
    // changes tok
    Set<String> invocations = findFunctionInvocations(tok, sfMap, hardFunctionReferences, defd, true, mainClassName());
    /*if (invocations.contains("str"))
      print("==STR==" + join(tok) + "==STR==");*/
    if (!cic(definitions, "leanMode"))
      invocations.addAll(functionsToAlwaysInclude);

    //print("Functions invoked: " + structure(invocations));
    List<String> needed = diff(invocations, defd);
    for (S f : doNotIncludeFunction) needed.remove(f);
    if (needed.isEmpty())
      break;
    print("Adding functions: " + join(" " , needed));
    
    HashSet neededSet = new HashSet(needed);
    Collection<S> bad = setIntersection(neededSet, shouldNotIncludeFunction);
    if (nempty(bad)) {
      S msg = "INCLUDING BAD FUNCTIONS: " + sfu(bad);
      print(msg);
      print(join(tok));
      fail(msg);
    }
      
    new L<S> added;
    new L<Future<S>> parts;
    new L<S> preload;
    
    for (S x : needed)
      if (sfMap.containsKey(x))
        preload.add(sfMap.get(x));
    print("Preloading: " + preload);
    cachePreload(preload);
    
    for (String x : cloneList(needed)) {
      if (defd.contains(x)) continue;
      
      String id = sfMap.get(x);
      if (id == null) {
        print("Standard function " + x + " not found.");
        needed.remove(x);
        continue;
      }
      //print("Adding function: " + x + " (" + id + ")");
       
      S function = cacheGet(id) + "\n";
      //if (("\n" + function).contains("\n!")) print("Warning: " + id + " contains translators.");
      
      if (cacheStdFunctions) {
        long _id = psI(id);
        CachedInclude ci = cachedIncludes.get(_id);
        if (ci == null) {
          cachedIncludes.put(_id, ci = new CachedInclude(_id));
          //println("Caching include " + _id + ", l=" + l(cachedIncludes));
        }
        function += "\n";
        fS _function = function;
        if (neq(ci.javax, function)) {
          print("Compiling function: " + x);
          ci.javax = function;
          ci.java = executor.submit(new Callable<S>() {
            public S call() {
              // We assume that variables made with makeVar() will always be local to some block
              varCountByThread.set(null);
              ret join(localStuff1(jtok(_function)));
            }
          });
          ci.realJava = null;
        }
        parts.add(ci.javaFuture());
      } else
        parts.add(nowFuture(function));
        
      added.add(x);
      Collection<S> newFunctions = new HashSet(findFunctionDefs(javaTok(function)));
      defd.addAll(newFunctions);
      for (S f : newFunctions)
        if (!addedFunctions.add(f)) {
          printSources(tok);
          fail("Trying to add function " + f + " again - main class syntax broken!");
        }
    }
    
    print("Getting " + nParts(parts));
    S text = lines(getAllFutures(parts));
    print("Including " + nParts(parts));
    tok = includeInMainLoaded(tok, text);
      
    // defd = new HashSet(findFunctions(tok));
    //print("Functions added: " + joinWithSpace(added));
    
    for (String x : needed)
      if (!defd.contains(x)) {
        print(join(tok));
        fail("Function not defined properly: " + x);
      }
    //print("Iteration " + (i+2));
    
    print("Processing definitions");
    
    dontPrint('definitions); tok_definitions(tok);
    dontPrint('ifndef); tok_ifndef(tok);
    dontPrint('ifdef); tok_ifdef(tok);
    
    if (i >= 1000) fail("Too many iterations");
  }
  
  ret tok;
}

// TODO: skip functions defined in inner classes!
static LS findFunctions(L<S> tok) {
  //ret findFunctionDefinitions(join(findMainClass(tok)));
  ret findFunctionDefs(findMainClass(tok));
}

static S cacheGet(S snippetID) {
  snippetID = formatSnippetID(snippetID);
  S text = snippetCache.get(snippetID);
  if (text == null) {
    text = loadSnippet(snippetID);
    // very early processing/checks for includes
    
    if (hasUnclosedStringLiterals(text))
      fail("Unclosed string literals in " + snippetID);
      
    if (regexpContains("\\bscope\\b", text)) {
      LS tok = javaTok(text);
      tok_scopes(tok, autoCloseScopes := true);
      text = join(tok);
    }
    
    snippetCache.put(snippetID, text);
  }
  ret text;
}

static void cachePreload(Collection<S> ids) {
  new L<S> needed;
  for (S id : ids)
    if (!snippetCache.containsKey(formatSnippetID(id)))
      needed.add(formatSnippetID(id));
  if (l(needed) > 1) {
    L<S> texts = loadSnippets(needed);
    for (int i = 0; i < l(needed); i++)
      if (texts.get(i) != null)
        snippetCache.put(needed.get(i), texts.get(i));
  }
}

static L<S> jtok(L<S> tok) {
  ret jtok(join(tok));
}

static L<S> jtok(S s) {
  ret indexTokenList(javaTok(s));
}

static HashSet<S> haveClasses_actual(L<S> tok) {
  new HashSet<S> have;
  for (L<S> c : innerClassesOfMain(tok))
    have.add(getClassDeclarationName(c));
  ret have;
}

static HashSet<S> haveClasses_addImported(LS tok, HashSet<S> have, bool mainLevel default true) {
  have.addAll(tok_importedClassNames(tok, mainLevel ? lambda2 onImportFound : null));
  have.addAll(usualJavaClassNames()); // for S => S.class
  ret have;
}

// works on Java level (no "sclass" etc)
// returns list of classes we have (useful for other processing)
static HashSet<S> addStandardClasses_v2(L<S> tok) {
  if (lclasses == null) {
    S sc = cacheGet(#1003674);
    lclasses = new L;
    for (S line : tlft_j(sc)) {
      int idx = line.indexOf('/');
      lclasses.addAll(ll(line.substring(0, idx), line.substring(idx+1)));
    }
  }
  L<S> definitions = lclasses;

  for (int safety = 0; safety < 10; safety++) {
    HashSet<S> have = haveClasses_actual(tok);
    haveClasses_addImported(tok, have);
    have.addAll(keys(rewrites));

    int j;
    while ((j = jfind(tok, "should not include class *.")) >= 0) {
      S cname = tok.get(j+8);
      shouldNotIncludeClass.add(cname);
      clearAllTokens(subList(tok, j, j+12));
    }
    
    while ((j = jfind(tok, "do not include class *.")) >= 0) {
      S name = tok.get(j+8);
      if (!needLatest.contains(name)) doNotIncludeClass.add(name);
      clearAllTokens(subList(tok, j, j+12));
    }
    
    new SS need;
    new Set<S> snippets;
    Set<S> idx = tokenIndexWithoutIfclass_forStdClasses(tok);
    
    while ((j = jfind(tok, "please include class *.")) >= 0) {
      S cname = tok.get(j+6);
      idx.add(cname);
      clearAllTokens(subList(tok, j, j+10));
    }

    for (int i = 0; i+1 < l(definitions); i += 2) {
      S className = definitions.get(i);
      S snippetID = fsI(definitions.get(i+1));
      if (idx.contains(className) && !have.contains(className) && snippets.add(snippetID))
        need.put(className, snippetID);
    }
    if (hasDef("SymbolAsString")) {
      print("Have SymbolAsString.");
      if (need.containsKey("Symbol")) {
        print("Have Symbol.");
        need.remove("Symbol");
      }
    } else
      print("No SymbolAsString.");
      
    removeAll(need, doNotIncludeClass);
    if (empty(need)) ret have;
  
    for (S className : keys(need))
      if (shouldNotIncludeClass.contains(className)) {
        S msg = "INCLUDING BAD CLASS: " + className;
        print(msg);
        print(join(tok));
        fail(msg);
      }
      
    cachePreload(values(need));
    
    new LPair<S, CachedInclude> parts; // class name, Java code
    
    print("Adding classes: " + joinWithSpace(keys(need)));
    for (S className : keys(need)) {
      if (have.contains(className)) continue; // intermittent add
      S snippetID = need.get(className);
      //print("Adding class " + className + " / " + snippetID);
      snippetID = fsI(snippetID);
      S text = cacheGet(snippetID) + "\n";
      
      assertTrue(cacheStdClasses);
      long _id = psI(snippetID);
      CachedInclude ci = cachedIncludes.get(_id);
      if (ci == null) cachedIncludes.put(_id, ci = new CachedInclude(_id));
      if (neq(ci.javax, text)) {
        print("Compiling class: " + className);
        ci.javax = text;
        ci.java = executor.submit(new Callable<S>() {
          public S call() {
            // We assume that variables made with makeVar() will always be local to some block
            varCountByThread.set(null);
            ret join(localStuff1(jtok(text)));
          }
        });
        ci.realJava = null;
      }
      parts.add(pair(className, ci));
    }
    
    new StringBuilder buf;
    for (Pair<S, CachedInclude> p : parts) {
      S className = p.a;
      LS ct = javaTok(p.b.java());
      new SS rewritten;
      tok_findAndClearRewrites(ct, rewritten);
      if (rewritten.containsKey(className))
        have.add(className);
      for (LS c : allClasses(ct)) {
        S name = getClassDeclarationName(c);
        have.add(name);
      }
      
      haveClasses_addImported(ct, have, false);
      
      if (!have.contains(className))
        fail("Wrongly defined class: " + className + " / " + p.b.snippetID + "\n\n" + p.b.java());
      if (!addedClasses.add(className)) {
        printSources(tok);
        fail("Trying to add class " + className + " again - main class syntax broken!");
      }
      buf.append(p.b.java());
    } // for part
    
    tok = includeInMainLoaded(tok, str(buf));
  }
  fail("safety 10");
}

static Set<S> expandableClassNames = lithashset("BigInteger");

// no reTok - leaves tok dirty
// magically append ".class" to class name references
svoid expandClassReferences_lazy(L<S> tok, Set<S> classNames, L<IntRange> reToks default null) {
  for (int i = 3; i+2 < l(tok); i += 2) {
    S t = tok.get(i);
    
    // skip implements/extends/throws lists
    if (eqOneOf(t, "implements", "extends", "throws")) {
      i = tok_endOfImplementsList(tok, i);
      continue;
    }
    
    if (classNames.contains(t) || expandableClassNames.contains(t)) {
      S s = tok.get(i-2); t = tok.get(i+2);
      // Now s is token before class name, t is token after class name
      // TODO: This whole logic ain't very good
      // (Hard to distinguish between "Int.class" as an argument
      // and "Int" as a type parameter.)
      if (eqOneOf(s, "instanceof", "new", ".", "<", ">", "/", "nu")) continue;
      if (eq(t, ":")) continue; // e.g. String::concat
      if (isInteger(s)) continue;
      if (isIdentifier(s) && !eqOneOf(s, "ret", "return")) continue;
        
      if (eq(t, ",") && isIdentifier(get(tok, i+4)) && eqGet(tok, i+6, ">")) continue; // e.g. T3<L<S>, S, S>
      if (eq(s, ",") && isIdentifier(get(tok, i-4)) &&
        (eqGet(tok, i-6, "<") || eqGet(tok, i-6, ",")
        && isIdentifier(get(tok, i-8)) && eqGet(tok, i-10, "<"))) continue; // e.g. T3<S, S, S> or IF3<S, S, S, S>
      if (eq(s, ",") && eqOneOf(_get(tok, i-6), "implements", "throws")) continue;
      
      // check for cast
      if (eq(s, "(") && eq(t, ")") && i >= 5) {
        if (!eqOneOf(get(tok, i+4), "{", ";")) {
          S x = tok.get(i-4);
          if (!isIdentifier(x)) continue;
          if (eqOneOf(x, "ret", "return")) continue;
        }
      }
      
      // check following token
      if (eqOneOf(t, ",", ")", ";", ":", "|")) {
        tok.set(i, tok.get(i) + ".class");
        reToks?.add(intRange(i, i+1));
      }
    }
  }
}

svoid expandClassReferences(L<S> tok, Set<S> classNames) {
  new L<IntRange> reToks;
  expandClassReferences_lazy(tok, classNames, reToks);
  reTok_multi(tok, reToks);
}

// "<id>/<ClassName>" => "((ClassName) <id>)"
static void slashCasts(L<S> tok, final Set<S> classNames) {
  /*jreplace(tok, "<id>/<id>", "(($3) $1)", tokcondition {
    ret classNames.contains(tok.get(i+5));
  });*/
  int n = l(tok)-4;
  for (int i = 1; i < n; i += 2)
    if (tok.get(i+2).equals("/") && isIdentifier(tok.get(i))
      && classNames.contains(tok.get(i+4)))
      replaceTokens_reTok(tok, i, i+5, "((" + tok.get(i+4) + ") " + tok.get(i) + ")");
}

// "<ClassName>(...)" => "new <ClassName>(...)"
// doesn't work at beginning of statement as we can't easily
// distinguish it from a constructor declaration.
static void newWithoutNew(L<S> tok, final Set<S> classNames) {
  TokCondition cond = newWithoutNew_condition(classNames);
  jreplace(tok, "<id>(", "new $1(", cond);
  // just two cases with type args for now
  jreplace(tok, "<id><<id>>(", "new $1<$3>(", cond);
  jreplace(tok, "<id><>(", "new $1<>(", cond);
}

static TokCondition newWithoutNew_condition(final Set<S> classNames) {
  ret tokcondition {
    if (!classNames.contains(tok.get(i+1))) false;
    S prev = _get(tok, i-1);
    bool ok = //!(eqGet(tok, i-3, "ifclass") && isIdentifier(prev))
      nempty(prev) // discarded ifclass
      && (eq(prev, ">") ? eqGet(tok, i-3, "-")
      : neqOneOf(prev, "new", ";", "}", "{", "public", "protected", "private", "."));
    //print("newWithoutNew: checking " + struct(subList(tok, i-3, i+2)) + " => " + ok);
    ret ok;
  };
}

static bool processConceptsDot(L<S> tok) {
  bool anyChange = false, change;
  do {
    change = false;
    for (int i : jfindAll(tok, "concepts."))
      if (contains(get(tok, i+3), "\n")) {
        replaceTokens(tok, i, i+3, "!" + "include once #1004863 // Dynamic Concepts");
        reTok(tok, i, i+3);
        change = anyChange = true;
        break;
      }
  } while (change);
  ret anyChange;
}

svoid caseAsVariableName(L<S> tok) {
  if (!tok.contains("case")) ret;
  for (int i = 1; i+2 < l(tok); i += 2) {
    S t = tok.get(i+2);
    if (tok.get(i).equals("case")
      && !(t.startsWith("'") || isInteger(t) || isIdentifier(t) || isQuoted(t)
        || eq(t, "-") && eqGet(tok, i+3, "") && isInteger(get(tok, i+4))))
      tok.set(i, "_case");
  }
}

static void continueAsFunctionName(L<S> tok) {
  jreplace(tok, "continue(", "_continue(");
}

// f bla => "bla" - and "please include function bla."
static void functionReferences(L<S> tok) {
  int i;
  
  jreplace_dyn(tok, "f-thread <id>", func(L<S> tok, int cIdx) {
    "dynamicCallableMC_thread(" + quote(tok.get(cIdx+6)) + ")"
  });
  
  S keyword = "f";
  while ((i = jfind(tok, keyword + " <id>", tokcondition {
    ret !eqOneOf(tok.get(i+3), "instanceof", "default");
  })) >= 0) {
    S f = tok.get(i+2);
    clearTokens(tok, i, i+2);
    tok.set(i+2, quote(f));
    reTok(tok, i, i+2);
    tok.set(l(tok)-1, last(tok) + "\nplease include function " + f + ".");
    reTok(tok, l(tok)-1, l(tok));
  }
  
  // r|rThread|rEnter|rThreadEnter|rEnterThread fname => r|rThread|... { fname() }
  while ((i = jfindOneOf_cond(tok, tokcondition {
    ret !eq(tok.get(i+3), "instanceof")
      && !eq(_get(tok, i-1), "cast");
  }, "r <id>", "rThread <id>", "rEnter <id>", "rThreadEnter <id>",
    "rEnterThread <id>")) >= 0) {
    S f = tok.get(i+2);
    replaceTokens(tok, i, i+3, tok.get(i) + " { " + f + "(); }");
    reTok(tok, i, i+3);
  }
  
  // dm_q fname => r_dm_q(r fname)
  jreplace(tok, "dm_q <id>", "r_dm_q(r $2)");
  
  // vf<S> fname => voidfunc(S s) { fname(s) }
  jreplace(tok, "vf<<id>> <id>", "voidfunc($3 a) { $5(a) }");
  
  // vf<L<S>> fname => voidfunc(L<S> a) { fname(a) }
  jreplace(tok, "vf<<id><<id>>> <id>", "voidfunc($3<$5> a) { $8(a) }");
  
  // construct<S> Entry => func(S s) -> Entry { new Entry(s) }
  jreplace(tok, "construct<<id>> <id>", "func($3 a) -> $5 { new $5(a) }");
  
  // f<S> fname => func -> S { fname() }
  jreplace(tok, "f<<id>> <id>", "func -> $3 { $5() }");
  
  // f<S, S> fname => func(S x) -> S { fname(x) }
  jreplace(tok, "f<<id>, <id>> <id>", "func($3 x) -> $5 { $7(x) }");
  
  // f<S, L<S>> fname => func(S x) -> L<S> { fname(x) }
  jreplace(tok, "f<<id>, <id><<id>>> <id>", "func($3 x) -> $5 $6 $7 $8 { $10(x) }");
  
  // f<L<S>, S> fname => func(L<S> x) -> S { fname(x) }
  jreplace(tok, "f<<id><<id>>, <id>> <id>", "func($3 $4 $5 $6 x) -> $8 { $10(x) }");
  
  // f<S, L<S>, S> fname => func(S x, L<S> y) -> S { fname(x, y) }
  jreplace(tok, "f<<id>, <id><<id>>, <id>> <id>", "func($3 x, $5 $6 $7 $8 y) -> $10 { $12(x, y) }");
  
  // if1 fname => a -> fname(a)
  // ivf1 fname => a -> fname(a)
  for (S _keyword : ll("if1", "ivf1"))
    jreplace_dyn(tok, _keyword + " <id>", func(LS tok, int i) -> S {
      S var = makeVar();
      ret var + " -> " + tok.get(i+2) + "(" + var + ")";
    });
}

static void quicknu(L<S> tok) {
  jreplace(tok, "nu <id>(", "nu($2.class, "); // not needed anymore
  jreplace(tok, "nu <id>", "new $2");
}

// fill variable innerClasses_list
static void innerClassesVar(L<S> tok, Set<S> have) {
  int i = jfind_check myInnerClasses_list(tok, ">myInnerClasses_list;");
  if (i < 0) ret;
  tok.set(i+4, "=litlist(\n" + joinQuoted(", ", have) + ");");
  reTok(tok, i+4, i+5);
}

// fill variable innerClasses_list
static void fillVar_transpilationDate(L<S> tok) {
  int i = jfind_check myTranspilationDate_value(tok, "long myTranspilationDate_value;");
  if (i < 0) ret;
  tok.set(i+4, " = " + now() + "L;");
  reTok(tok, i+4, i+5);
}

sbool ifclass_reTokImmediately = false;
sbool ifclass_noReTok = true;

// process ifclass x ... endif blocks
static void tok_ifclass(LS tok, Set<S> have) {
  tok_conditionals(tok, "ifclass", "endif", id -> contains(have, id), ifclass_reTokImmediately, ifclass_noReTok);
}

// returns number of changes
static int tok_conditionals(LS tok, S keyword, S keywordEnd, IPred<S> pred, bool reTokImmediately, bool noReTok) {
  int changes = 0;
  if (tok instanceof IContentsIndexedList) {
    int[] l = tok/IContentsIndexedList.indicesOf(keyword);
    for (int j = l(l)-1; j >= 0; j--) {
      int i = l[j];
      if (isIdentifier(get(tok, i+2))) {
        processConditional(tok, i, keyword, keywordEnd, pred, reTokImmediately && !noReTok);
        ++changes;
      }
    }
  } else {
    if (!tok.contains(keyword)) ret changes;
    int i = l(tok);
    while ((i = rjfind(tok, 1, i-1, keyword + " <id>")) >= 0) {
      ++changes;
      processConditional(tok, i, keyword, keywordEnd, pred, reTokImmediately && !noReTok);
    }
  }
  if (changes != 0 && !reTokImmediately && !noReTok) reTok(tok);
  //print(keyword + ": " + nChanges(changes));
  ret changes;
}

svoid processConditional(LS tok, int i, S keyword, S keywordEnd, IPred<S> pred, bool reTokImmediately) {
  int j = jfind(tok, i+4, keywordEnd);
  if (j < 0) j = l(tok)-1;
  S name = tok.get(i+2);
  bool has = pred.get(name);
  //print(keyword + " " + name + " => " + has);
  if (has) {
    clearTokens_maybeReTok(tok, j, j+1, reTokImmediately);
    clearTokens_maybeReTok(tok, i, i+3, reTokImmediately);
  } else
    clearTokens_maybeReTok(tok, i, j+1, reTokImmediately); // safer than before
}

// set flag *.
static void tok_definitions(L<S> tok) {
  int i;
  while ((i = jfind_check flag(tok, "set flag <id>.")) >= 0) {
    S fname = tok.get(i+4);
    print("Setting flag " + fname);
    definitions.add(fname);
    if (eqic(fname, "debug_jreplace")) set debug_jreplace;
    clearAllTokens(subList(tok, i, i+8));
  }
  
  while ((i = jfind_check flag(tok, "unset flag <id>.")) >= 0) {
    S fname = tok.get(i+4);
    print("Unsetting flag " + fname);
    definitions.remove(fname);
    clearAllTokens(subList(tok, i, i+8));
  }
}

// rewrite <id> [=|with|to] <definition>
// - a global version of "replace <id> with"
// new version - may not work yet
/*svoid tok_findAndClearRewrites(LS tok, SS newlyDefined default null) {
  tok_findRewrites(tok, newlyDefined, f -> {
    print("Have rewrite: " + f.token + " => " + f.replacement());
    clearTokens(f.tok, f.startCIdx(), f.endNIdx());
  });
}*/


// rewrite <id> [=|with|to] <definition>
// - a global version of "replace <id> with"
// old version (works)
svoid tok_findAndClearRewrites(LS tok, SS newlyDefined default null) {
  int i;
  while ((i = jfind(tok, "rewrite <id>", (_tok, nIdx) ->
    eqGetOneOf(_tok, nIdx+5, "with", "=", "to"))) >= 0) {
    S token = tok.get(i+2);
    int repStart = i+6;
    int repEnd = smartIndexOf(tok, repStart, ".");
    S replacement = joinSubList(tok, repStart, repEnd-1);
    clearTokens(tok, i, repEnd+1);
    mapPut(newlyDefined, token, replacement);
    rewrites.put(token, replacement);
    print("Have rewrite: " + token + " => " + replacement);
  }
}

static void tok_processRewrites(L<S> tok) {
  for (S token : keys(rewrites))
    jreplace(tok, token, rewrites.get(token));
}

// extend *. (set base class of main class)
static void tok_extend(L<S> tok) {
  int i;
  while ((i = jfind(tok, "extend <id>.")) >= 0) {
    mainBaseClass = tok.get(i+2);
    clearAllTokens(tok, i, i+7);
  }
}

// process ifndef x ... endifndef blocks
static void tok_ifndef(LS tok) {
  tok_conditionals(tok, "ifndef", "endifndef", id -> !definitions.contains(id), true, false);
}
  
// process ifdef x ... endifdef blocks
static void tok_ifdef(LS tok) {
  tok_conditionals(tok, "ifdef", "endifdef", id -> definitions.contains(id), true, false);
}
  
svoid conceptDeclarations(L<S> tok) {
  for (S kw : ll("concept", "sconcept")) {
    O cond = tokcondition { tok_addFieldOrder(tok, i+1); true; };
    bool re = false;
    if (jreplace(tok, kw + " <id> {", "static class $2 extends Concept {", cond)) re = true;
    if (jreplace(tok, kw + " <id> implements", "static class $2 extends Concept implements", cond)) re = true;
    if (jreplace(tok, kw + " <id>", "static class $2", cond)) re = true;
    if (re) reTok(tok);
  }
}

svoid shortenedSubconcepts(L<S> tok) {
  jreplace(tok, "<id> > <id> {", "concept $3 extends $1 {", tokcondition {
    bool b = (i == 0 || tok.get(i).contains("\n")) || eq(_get(tok, i-1), "abstract"); // only at beginning of line or after "abstract"
    //print("subconcept " + b + ": " + structure(subList(tok, i-1, i+5)));
    ret b;
  });
}

// -slightly experimental
// -do calculation in another thread, then return to AWT thread
// -must be placed in a block
// -transforms rest of block 
svoid unswing(L<S> tok) {
  int i;
  while ((i = jfind(tok, "unswing {")) >= 0) {
    int idx = i+2;
    int closingBracket = findEndOfBracketPart(tok, idx)-1;
    int endOfOuterBlock = findEndOfBracketPart(tok, closingBracket)-1;
    tok.set(i, "thread");
    tok.set(closingBracket, " awt {");
    tok.set(endOfOuterBlock, "}}}");
    reTok(tok, closingBracket-1, endOfOuterBlock+1);
  }
}

// -Syntax: lock theLock;
// -lock a lock, unlock at end of current block with finally
svoid lockBlocks(L<S> tok) {
  int i;
  while ((i = jfind(tok, "lock <id>", tokcondition { ret neq(tok.get(i+3), "instanceof"); })) >= 0) {
    int semicolon = findEndOfStatement(tok, i)-1;
    S var = makeVar();
    int endOfOuterBlock = findEndOfBracketPart(tok, semicolon)-1;
    replaceTokens(tok, i, semicolon+1,
      "Lock " + var + " = " + joinSubList(tok, i+2, semicolon-1) + "; lock(" + var + "); try {");
    tok.set(endOfOuterBlock, "} finally { unlock(" + var + "); } }");
    reTok(tok, i, endOfOuterBlock+1);
  }
}

// -Syntax: temp Bla bla = bla();
// -expands to try(Bla bla = bla()) { ... } with rest of block inside
svoid tempBlocks(L<S> tok) {
  int i;
  jreplace_dyn(tok, "temp (<id>) <id>", (_tok, cIdx) -> {
    S var = makeVar(), type = tok.get(cIdx+4), firstTokenOfExpr = tok.get(cIdx+8);
    ret "temp \*type*/ \*var*/ = cast \*firstTokenOfExpr*/";
  });
  
  jreplace(tok, "temp <id> =", "temp var $2 =");
  
  while ((i = jfind(tok, "temp <id>")) >= 0) {
    int semicolon = findEndOfStatement(tok, i)-1;
    int endOfOuterBlock = findEndOfBracketPart(tok, semicolon)-1;
    L<S> sub = subList(tok, i-1, semicolon);
    int eq = subList(sub, 0, smartIndexOfOneOf(sub, "{", "(")).indexOf("=");
    S var;
    if (eq >= 0)
      var = sub.get(eq-2);
    else {
      // no var name, e.g. temp newThoughtSpace();
      var = makeVar();
      tok.set(i+2, "AutoCloseable " + var + " = " + tok.get(i+2));
    }
      
    //tok.set(i, "try (");
    //tok.set(semicolon, ") {";
    //tok.set(endOfOuterBlock, "}}");
    
    tok.set(i, "");
    tok.set(semicolon, "; try {");
    tok.set(endOfOuterBlock, "} finally { _close(" + var + "); }}");
    
    reTok(tok, i, endOfOuterBlock+1);
  }
}

svoid forgetCachedIncludes {
  cachedIncludes.clear();
}

// TODO: when to do this / merge contents if there are multiple transpilers?
svoid cleanMeUp {
  for (CachedInclude ci : values(cachedIncludes))
    ci.clean();
  vmKeepWithProgramMD5_save('cachedIncludes);
}

svoid printSources(L<S> tok) {
  S src = join(tok);
  print("----");
  print(src);
  print("----");
  print("Bracket hygiene: " + testBracketHygiene2(src));
}

svoid tok_quickInstanceOf(L<S> tok, final Set<S> haveClasses) {
  if (!quickInstanceOfEnabled) ret;
  // "x << X" or "x >> X" => "x instanceof X"
  for (S op : ll("<<", ">>"))
    jreplace(tok, "<id> " + op + " <id>", "$1 instanceof $4", tokcondition {
      ret haveClasses.contains(tok.get(i+7))
        && !eqOneOf(tok.get(i-1), "<", "extends", "implements");
    });
}

sbool hasDef aka isDef(S s) {
  ret definitions.contains(s);
}

svoid tok_shortFinals(L<S> tok) {
  jreplace(tok, "fS", "final S");
  jreplace(tok, "fO", "final O");
  jreplace(tok, "fL", "final L");
  jreplace(tok, "fMap", "final Map");
  jreplace(tok, "fRunnable", "final Runnable");
  jreplace(tok, "f int", "final int");
}

svoid tok_mainClassNameAndPackage(LS tok) {
  int i;
  if ((i = jfind(tok, "mainClassName <id>")) >= 0) {
    mainClassName = tok.get(i+2);
    if (eqGet(tok, i+4, ".") && isIdentifier(get(tok, i+6))) {
      mainPackage = mainClassName;
      mainClassName = tok.get(i+6);
      clearTokensAndReTok(tok, i, i+7);
    } else
      clearTokensAndReTok(tok, i, i+3);
  }
  
  if ((i = jfind(tok, "mainPackage <id>")) >= 0) {
    int j = i+2;
    while (subListEquals(tok, j+1, "", ".", "") && isIdentifier(get(tok, j+4)))
      j += 4;
    mainPackage = joinSubList(tok, i+2, j+1);
    clearTokens_reTok(tok, i, j+1);
  }
}

svoid defineMapLikesEtc(LS tok) {
  defineMapLikes(tok);
  defineLambdaMapLikes(tok);
  defineCurry1Likes(tok);
  defineMapMethodLikes(tok);
  defineNuLikes(tok);
  defineXLikes(tok, "getLike", getLikeFunctions);
  defineXLikes(tok, "lambdaMethod0Like", lambdaMethod0LikeFunctions);
  defineXLikes(tok, "lambda0Like", lambda0LikeFunctions);
}

svoid defineMapLikes(LS tok) {
  int i;
  while ((i = jfind(tok, "mapLike <id>")) >= 0) {
    mapLikeFunctions.add(printIf(printMapLikes(), "mapLike", tok.get(i+2)));
    clearTokens_reTok(tok, i, i+2);
  }
}

svoid defineLambdaMapLikes(LS tok) {
  int i;
  while ((i = jfind(tok, "lambdaMapLike <id>")) >= 0) {
    lambdaMapLikeFunctions.add(printIf(printMapLikes(), "lambdaMapLike", tok.get(i+2)));
    clearTokens_reTok(tok, i, i+2);
  }
}

svoid defineCurry1Likes(LS tok) {
  int i;
  while ((i = jfind(tok, "curry1Like <id>")) >= 0) {
    curry1LikeFunctions.add(printIf(printMapLikes(), "curry1Like", tok.get(i+2)));
    clearTokens_reTok(tok, i, i+2);
  }
}

svoid defineMapMethodLikes(LS tok) {
  int i;
  while ((i = jfind(tok, "mapMethodLike <id>")) >= 0) {
    mapMethodLikeFunctions.add(printIf(printMapLikes(), "mapMethodLike", tok.get(i+2)));
    clearTokens_reTok(tok, i, i+2);
  }
}

// functions that work like "nu" syntactically (accept a class as "super-first" parameter [left of the bracket])
svoid defineNuLikes(LS tok) {
  int i;
  while ((i = jfind(tok, "nuLike <id>")) >= 0) {
    nuLikeFunctions.add(printIf(printMapLikes(), "nuLike", tok.get(i+2)));
    clearTokens_reTok(tok, i, i+2);
  }
}

svoid defineXLikes(LS tok, S keyword, Set<S> xLikeFunctions) {
  int i;
  while ((i = jfind(tok, keyword + " <id>")) >= 0) {
    xLikeFunctions.add(printIf(printMapLikes(), keyword, tok.get(i+2)));
    clearTokens_reTok(tok, i, i+2);
  }
}

svoid defineExtraSF(LS tok) {
  int i;
  IntRange reTok = null;
  while ((i = jfind(tok, "function <id> is in *.")) >= 0) {
    extraStandardFunctions.put(tok.get(i+2), fsI(unquote(tok.get(i+8))));
    clearTokens(tok, i, i+12);
    reTok = combineIntRanges(reTok, intRange(i, i+12));
  }
  reTok(tok, reTok);
}

sbool tok_applyAllXLikeFunctions(LS tok) {
  new L<IntRange> reToks;
  for (int i : jfindAll(tok, "<id> <id> (")) {
    S f = tok.get(i), arg = tok.get(i+2), replacement = null;
    if (contains(mapLikeFunctions, f))
      replacement = "\*f*/(f \*arg*/,";
    else if (contains(lambdaMapLikeFunctions, f))
      replacement = "\*f*/(lambda1 \*arg*/,";
    else if (contains(curry1LikeFunctions, f))
      replacement = "\*f*/(lambda2 \*arg*/,";
    else if (contains(mapMethodLikeFunctions, f))
      replacement = "\*f*/(\*quote(arg)*/,";
    else if (contains(nuLikeFunctions, f))
      replacement = "\*f*/(\*arg*/.class,";
    else if (contains(getLikeFunctions, f))
      replacement = "\*f*/(lambdaField \*arg*/,";
    else if (contains(lambdaMethod0LikeFunctions, f))
      replacement = "\*f*/(methodLambda0 \*arg*/,";
    else if (contains(lambda0LikeFunctions, f))
      replacement = "\*f*/(lambda0 \*arg*/,";
    
    if (replacement != null)
      replaceTokens_reTokLater(tok, reToks, i, i+5, replacement + " ");
  }
  reTok_multi(tok, reToks);
  ret nempty(reToks);
}

/*sbool tok_applyMapLikeFunctions(LS tok, final Set<S> mapLikeFunctions) {
  // map funcname(...) => map(f funcname, ...)
  // filter funcname(...) => filter(f funcname, ...)
  // ...
  ret jreplace(tok, "<id> <id>(", "$1(f $2,", func(L<S> tok, int i) -> bool {
    contains(mapLikeFunctions, tok.get(i+1))
  });
}*/

/*sbool tok_applyLambdaMapLikeFunctions(LS tok, final Set<S> lambdaMapLikeFunctions) {
  // mapNonNulls funcname(...) => mapNonNulls(lambda1 funcname, ...)
  // mapKeysAndValues funcname(...) => mapKeysAndValues(lambda1 funcname, ...)
  // ...
  ret jreplace(tok, "<id> <id>(", "$1(lambda1 $2,", func(L<S> tok, int i) -> bool {
    contains(lambdaMapLikeFunctions, tok.get(i+1))
  });
}*/

/*sbool tok_applyCurry1LikeFunctions(LS tok, final Set<S> curry1LikeFunctions) {
  // curry1 funcname(...) => curry1(lambda2 funcname, ...)
  ret jreplace(tok, "<id> <id>(", "$1(lambda2 $2,", func(L<S> tok, int i) -> bool {
    contains(curry1LikeFunctions, tok.get(i+1))
  });
}*/

/*sbool tok_applyMapMethodLikeFunctions(LS tok, final Set<S> mapMethodLikeFunctions) {
  // mapMethod funcname(...) => mapMethod('funcname, ...)
  // collect funcname(...) => collect('funcname, ...)
  // ...
  ret jreplace_dyn(tok, "<id> <id>(",
    func(L<S> tok, int cIdx) -> S { tok.get(cIdx) + "(" + quote(tok.get(cIdx+2)) + "," },
    func(L<S> tok, int i) -> bool {
      contains(mapMethodLikeFunctions, tok.get(i+1))
    });
}*/

svoid runMetaPostBlocks(LS tok) {
  for ping (S code : unnull(metaPostBlocks)) {
    S snippetID = standardFunctionSnippet(assertIdentifier(code));
    if (empty(snippetID))
      fail("meta-post function not found: " + code);
    call(hotwireCached(snippetID), code, tok);
    //callF(codeToFunctionOnArbitraryType(code, "LS", L, "tok"), tok);
  }
}

svoid runTransformers(LS tok, LS transformers) {
  fOr ping (S code : transformers)
    tok_runMetaTransformer(tok, code);
}

/*sbool tok_applyNuLikeFunctions(LS tok, final Set<S> nuLikeFunctions) {
  // nu ClassName(...) => nu(ClassName, ...)
  // ...
  ret jreplace_dyn(tok, "<id> <id>(",
    func(L<S> tok, int cIdx) -> S { tok.get(cIdx) + "(" + tok.get(cIdx+2) + ".class," },
    func(L<S> tok, int i) -> bool {
      contains(nuLikeFunctions, tok.get(i+1))
    });
}*/

/*sbool tok_applyGetLikeFunctions(LS tok, Set<S> getLikeFunctions) {
  // get fieldName(...) => get(fieldLambda fieldName, ...)
  // ...
  ret jreplace_dyn(tok, "<id> <id>(",
    func(L<S> tok, int cIdx) -> S { tok.get(cIdx) + "(lambdaField " + tok.get(cIdx+2) + "," },
    func(L<S> tok, int i) -> bool {
      contains(getLikeFunctions, tok.get(i+1))
    });
}*/

sbool metaCodeAllowed() {
  ret allowMetaCode || containsIC(definitions, "allowMetaCode");
}

static LS indexTokenList(LS tok) {
  if (useIndexedList2) ret indexedList2(tok);
  if (useTokenIndexedList) ret tokenIndexedList3(tok);
  ret tok;
}

svoid tok_earlyGeneralStuff(LS tok) {
  // self-compile construct (TODO)
  /*jreplace(tok, "self-compile", (tok, iOpening, iClosing) -> S[] {
  
    selfCompiling
  });*/
  
  tok_standardBot1(tok);
  tok_processSimplified(tok);
  tok_compactModules(tok);
  
  tok_metaFor(tok);
  
  // is, short for "implements"
  jreplace(tok, "is <id>", "implements $2", tokCondition {
    ret !eqGetOneOf(tok, i+3, "a", "in"); // "is a", "is in" are defined as something else
  });
}

svoid lambdaReferences(LS tok) {
  // lambda0 myFunction => () -> myFunction()
  jreplace(tok, "lambda0 <id>", "() -> $2()");
  
  // lambda1 myFunction => var123 -> myFunction(var123)
  jreplace_dyn(tok, "lambda1 <id>", func(L<S> tok, int cIdx) {
    S var = makeVar();
    S s = var + " -> " + tok.get(cIdx+2) + "(" + var + ")";
    ret eqGet(tok, cIdx-2, ")") ? roundBracket(s) : s;
  });
  
  // lambda2 myFunction => (a, b) -> myFunction(a, b)
  jreplace_dyn(tok, "lambda2 <id>", func(LS tok, int cIdx) {
    S a = makeVar();
    S b = makeVar();
    S s = "(\*a*/, \*b*/) -> \*tok.get(cIdx+2)*/(\*a*/, \*b*/)";
    ret eqGet(tok, cIdx-2, ")") ? roundBracket(s) : s;
  });
  
  // methodLambda0 methodName => var123 -> var123.methodName()
  jreplace_dyn(tok, "methodLambda0 <id>", func(LS tok, int cIdx) {
    S var = makeVar();
    ret var + " -> " + var + "." + tok.get(cIdx+2) + "()";
  });
  
  // fieldLambda fieldName => var123 -> var123.fieldName
  jreplace_dyn(tok, "fieldLambda <id>", func(LS tok, int cIdx) {
    S var = makeVar();
    ret var + " -> " + var + "." + tok.get(cIdx+2);
  });
}

svoid clearSnippetCache {
  snippetCache.clear();
  sf = null;
  lclasses = null;
}

svoid mediumRefresh {
  clearSnippetCache();
  print("Medium-refreshed transpiler " + mc());
}

svoid jreplace_performing(LS tok, int i, int end, S expansion) {
  if (debug_jreplace)
    print("jreplace: " + quote(joinSubList(tok, i, end)) + " => " + quote(expansion));
}

sS mainClassName() {
  ret or(mainClassName, "main");
}

svoid grabImportedStaticFunctions(LS tok) {
  for (S name : tok_importedStaticFunctionNamesWithPackages(tok)) {
    int idx = lastIndexOf(name, '.');
    S pkg = takeFirst(idx, name);
    S functionName = dropFirst(idx+1, name);
    //print(functionName + " => " + pkg);
    doNotIncludeFunction.add(functionName);
    functionToPackage.put(functionName, pkg);
  }
}

sbool printMapLikes() {
  ret contains(definitions, "printMapLikesEtc");
}

// delete import if marked as "need latest"
svoid onImportFound(LS tok, IntRange r) {
  S id = get(tok, r.end-3);
  if (needLatest.contains(id)) {
    print("Purging import: " + joinSubList(tok, r));
    clearTokens(tok, r);
  }
}

Began life as a copy of #759