Transpiler with TokenIndexedList3 [dev.] [1025804]

!1006722
//!7

!include once #1025802 // optimized findCodeTokens

set flag AllowMetaCode.

//set flag tok_forUnnull_debug.

// ifdef in cached includes? => i think cached includes still contain the ifdefs, so it's fine. tok_ifdef is not called within localStuff1

set flag noAI.
set flag isQuoted_dirty. // should be OK, we're dealing with tokens

static bool autoQuine = true;
static int maxQuineLength = 80;
sbool assumeTriple = true;
sbool quickInstanceOfEnabled; // interferes with things

// _registerThread usually costs nothing because we need
// the registerWeakHashMap mechanism anyway for ping().
// Anyway - no forced functions for now :)
static L<S> functionsToAlwaysInclude = ll(
  //"_registerThread",
  //"asRuntimeException"
);

// classes with two type parameters that can written with just one
// e.g. Pair<S> => Pair<S, S>
static Set<S> pairClasses = lithashset("Pair", "Either", "Map", "AbstractMap", "HashMap", "TreeMap", "LinkedHashMap", "MultiMap", "CompactHashMap", "WrappedMap", "F1", "IF1", "AllOnAll", "AllOnAllWithUpdates");

sS transpilingSnippetID;
//static new AtomicInteger varCount;
static new ThreadLocal<AtomicInteger> varCountByThread;
static new Map<S, S> snippetCache;
sbool useIndexedList2 = false, useTokenIndexedList = true;
sbool opt_javaTok = true;
static bool cacheStdFunctions = true, cacheStdClasses = true;
static new HashMap<Long, CachedInclude> cachedIncludes;
static ExecutorService executor;
static L lclasses;
static long startTime, lastPrint;

// These variables have to be cleared manually for each transpilation

static new HashSet<Long> included;
static Set<S> definitions = ciSet();
static new HashMap<S> rewrites;
static new HashSet<S> shouldNotIncludeFunction;
static new HashSet<S> shouldNotIncludeClass;
static new HashSet<S> doNotIncludeFunction;
static new HashSet<S> addedFunctions;
static new HashSet<S> addedClasses;
static new HashSet<S> hardFunctionReferences;
static new HashSet<S> mapLikeFunctions;
static new HashSet<S> mapMethodLikeFunctions;
static new HashSet<S> nuLikeFunctions;
static SS extraStandardFunctions;
sbool quickmainDone1, quickmainDone2;
static new TreeSet<S> libs;
sS mainBaseClass, mainPackage, mainClassName;
sbool localStuffOnly; // for transpiling a fragment
sbool asInclude; // for transpiling an include (auto-close scopes)
sbool allowMetaCode = false; // run any embedded meta code
static LS metaPostBlocks, metaTransformers;
sbool dontPrintSource;
sbool dontLoadCachedIncludesFromVM; // for benchmarking

sclass CachedInclude {
  S javax;
  Future<S> java;
  S realJava;
  
  S java() {
    ret realJava != null ? realJava : getFuture(java);
  }
  
  Future<S> javaFuture() {
    ret realJava != null ? nowFuture(realJava) : java;
  }
  
  void clean {
    if (java != null) {
      realJava = getFuture(java);
      java = null;
    }
  }
}

p {
  startTime = lastPrint = sysNow();
  pcall {
    if (!dontLoadCachedIncludesFromVM)
      vmKeepWithProgramMD5_get('cachedIncludes);
  }
  executor = Executors.newFixedThreadPool(numberOfCores());
  transpilingSnippetID = or(getThreadLocal((ThreadLocal<S>) getOpt(javax(), 'transpilingSnippetID)), transpilingSnippetID);
  print(+transpilingSnippetID);
  set transpileRaw_dontCopyFromCreator;
  
  fO oldPrint = or(print_byThread()!, f print_raw);
  temp tempInterceptPrint(new F1<S, Bool>() {
    Bool get(S s) {
      long now = sysNow();
      long time = now-lastPrint; // -startTime;
      lastPrint = now;
      callF(oldPrint, "[" + formatInt(time/1000, 2) + ":" + formatInt(time % 1000, 3) + "] " + s);
      false;
    }
  });
  
  try {
    _main();
  } finally {
    interceptPrintInThisThread(oldPrint);
    if (executor != null) executor.shutdown();
    executor = null;
    localStuffOnly = false;
    asInclude = false;
  }
}

svoid _main() ctex {
  if (sameSnippetID(programID(), defaultJavaXTranslatorID())) setDefaultJavaXTranslatorID(#7);
  
  //reTok_modify_check = true;
  //if (useIndexedList) findCodeTokens_debug = true;
  javaTok_opt = opt_javaTok;
  //findCodeTokens_indexed = findCodeTokens_unindexed = 0;
  findCodeTokens_bails = findCodeTokens_nonbails = 0;
  javaTok_n = javaTok_elements = 0;
  S in = loadMainJava();
  
  print("759 STARTING " + identityHashCode(main.class));
  included.clear();
  definitions.clear();
  rewrites.clear();
  definitions.add("SymbolAsString");
  shouldNotIncludeFunction.clear();
  shouldNotIncludeClass.clear();
  doNotIncludeFunction.clear();
  addedFunctions.clear();
  addedClasses.clear();
  hardFunctionReferences.clear();
  mapLikeFunctions = cloneHashSet(tok_mapLikeFunctions());
  mapMethodLikeFunctions = cloneHashSet(tok_mapMethodLikeFunctions());
  nuLikeFunctions.clear();
  extraStandardFunctions = new HashMap;
  libs.clear();
  mainBaseClass = mainPackage = mainClassName = null;
  varCountByThread.set(null);
  quickmainDone1 = quickmainDone2 = false;
  metaPostBlocks = new L;
  metaTransformers = new L;
  dontPrintSource = false;
  
  //L ts = findTranslators(toLines(join(tok)));
  //print("Translators in source at start: " + structure(ts));
  
  L<S> tok = jtok(in);
  
  try {
    tok_definitions(tok);
    
    // add m { }
    
    if (!localStuffOnly && !hasCodeTokens(tok, "m", "{") && !hasCodeTokens(tok, "main", "{") && !hasCodeTokens(tok, "class", "main")) {
      //tok = jtok(moveImportsUp("m {\n" + in + "\n}"));
      if (l(tok) == 1) tok = singlePlusList(first(tok), dropFirst(javaTok("m {}")));
      else {
        replaceTokens_reTok(tok, 1, 2, "m {\n\n" + unnull(get(tok, 1)));
        replaceTokens_reTok(tok, l(tok)-2, l(tok)-1, unnull(get(tok, l(tok)-2)) + "}");
      }
      tok_moveImportsUp(tok);
    }
    
    // standard translate
    
    //ts = findTranslators(toLines(join(tok)));
    //print("Translators in source: " + structure(ts));
    
    if (tok_hasTranslators(tok))
      tok = jtok(defaultTranslate(join(tok)));
    
    //print("end of default translate");
    //print(join(tok));
  
    //tok_autoCloseBrackets(tok);    
    
    tok_metaTransformNow(tok);
    
    tok_processEarlyIncludes(tok);
      
    tok_earlyGeneralStuff(tok);
    
    tok = tok_processIncludes(tok); // before standard functions
    if (processConceptsDot(tok))
      tok = tok_processIncludes(tok);
    tok = localStuff1(tok);
    
   if (!localStuffOnly) {
    int safety = 0;
    boolean same;
    do { // BIG LOOP
      S before = join(tok);
      
      // do the non-local stuff (flags and rewrites, things that correlate across includes like tok_selfType)
      tok_selfType(tok);
      tok_mainClassNameAndPackage(tok);
      tok_definitions(tok);
      tok_ifndef(tok);
      tok_ifdef(tok);
      defineMapLikes(tok);
      if (tok_applyMapLikeFunctions(tok, mapLikeFunctions))
        functionReferences(tok);
      defineMapMethodLikes(tok);
      tok_applyMapMethodLikeFunctions(tok, mapMethodLikeFunctions);
      defineNuLikes(tok);
      tok_applyNuLikeFunctions(tok, nuLikeFunctions);
      tok_dropExtraCommas(tok); // from e.g. tok_applyMapMethodLikeFunctions
      tok_delegateTo(tok);
      tok_replaceWith(tok);
      tok_findRewrites(tok);
      tok_processRewrites(tok);
      
      // main bla(...) => mainClassName.bla(...)
      jreplace(tok, "main <id>(", or(mainClassName, "main") + ".$2(");
      
      try {
        if (safety == 0) tok = quickmain(tok);
      } catch e {
        printSources(tok);
        rethrow(e);
      }
      tok_collectMetaPostBlocks(tok, metaPostBlocks);
      tok_collectTransformers(tok, metaTransformers);
      tok_metaTransformNow(tok);
      
      // Hack to allow DynModule to reimplement _registerThread
      /*if (tok.contains("DynModule") && !addedClasses.contains("DynModule"))
        addStandardClasses_v2(tok);*/
      
      defineExtraSF(tok);
      tok = standardFunctions(tok);
      tok = stdstuff(tok); // all the keywords, standard
      S diff;
      long startTime = now();
      //diff = unidiff(before, join(tok));
      //print("unidiff: " + (now()-startTime) + " ms");
      //same = eq(diff, "");
      same = tok_sameTest(tok, before);
      if (!same) {
        print("Not same " + safety + ".");
        //print(indent(2, diff));
      }
      if (safety++ >= 10) {
        //print(unidiff(before, join(tok)));
        printSources(tok);
        fail("safety 10 error!");
      }
    } while (!same); // END OF BIG LOOP
    
    print("Post.");
    
    if (mainBaseClass != null) {
      jreplace1(tok, "class main", "class main extends " + mainBaseClass);
      mainBaseClass = null;
    }
    
    print('moveImportsUp); tok_moveImportsUp(tok);
    
    print('Indexing); tok = indexTokenList(tok);
    
    // POST-PROCESSING after stdstuff loop
    
    if (transpilingSnippetID != null)
      jreplace_dyn(tok, "class whatever", 
        func(L<S> tok, int cIndex) -> S {
          pcall { ret "class " + stringToLegalIdentifier(getSnippetTitle(transpilingSnippetID)); }
          ret "class Whatever";
        });
    
    //print("Type<A, A>"); // Type<A> to Type<A, A>
    
    print('extendClasses); tok = extendClasses(tok);
    print('libs); libs(tok);
    print('sourceCodeLine); sourceCodeLine(tok);
    
    tok_overridableFunctionDefs(tok, null);
    
    // escaping for lambdas
    //jreplace(tok, "-=>", "->");
  
    // Stuff that depends on the list of inner classes (haveClasses)
    HashSet<S> haveClasses = haveClasses_actual(tok);
    print('innerClassesVar); innerClassesVar(tok, haveClasses);
    fillVar_transpilationDate(tok);
    haveClasses_addImported(tok, haveClasses);
    print('ifclass); tok_ifclass(tok, haveClasses);
    print('slashCasts); slashCasts(tok, haveClasses);
    print('newWithoutNew); newWithoutNew(tok, haveClasses);
    
    if (!assumeTriple) {
      print('Triple);
      if (tok.contains("Triple") && !haveClasses.contains("Triple")) {
        jreplace(tok, "Triple", "T3");
        haveClasses.remove("Triple");
        haveClasses.add("T3");
        slashCasts(tok, lithashset("T3"));
        tok_quickInstanceOf(tok, lithashset("T3"));
      }
      expandTriple(tok);
    }
    
    if (hasDef("SymbolAsString"))
      jreplace(tok, "Symbol", "String");
  
    print('classReferences); expandClassReferences_lazy(tok, haveClasses);
    
    if (metaCodeAllowed()) runMetaPostBlocks(tok);

    // Error-checking
    print("Error-checking"); 
    Set<S> functions = new HashSet(findFunctions(tok));
    for (S f : hardFunctionReferences)
      if (!functions.contains(f))
        fail("Function " + f + " requested, but not supplied");
  
    print('autoImports); tok = autoImports(tok); // faster to do it at the end
    
    print("definitions=" + sfu(definitions));
    if (containsOneOfIC(definitions, "allpublic", "reparse", "PublicExceptTopClass")) {
      // Fire up the Java parser & pretty printer!
      print(containsIC(definitions, "allpublic")? "Making all public." : "Reparsing.");
      //try {
        S src = join(tok);
        try {
          if (containsIC(definitions, "PublicExceptTopClass"))
            src = javaParser_makeAllPublic(src, notTopLevelClassDecl := true);
          else if (containsIC(definitions, "keepComments"))
            src = javaParser_makeAllPublic_keepComments(src);
          else if (containsIC(definitions, "allpublic"))
            src = javaParser_makeAllPublic(src);
          else
            src = javaParser_reparse_keepComments(src);
        } catch e {
          extractAndPrintJavaParseError(src, e);
          File f = javaxCachesDir("error-source.java");
          saveTextFileVerbose(f, src);
          dontPrintSource = true;
          throw e;
        }
        tok = jtok(src);
      /*} catch e {
        S src = join(tok);
        if (!dontPrintSource)
          print(src);
        print(f2s(saveProgramTextFile("error.java", src)));
        throw rethrow(e);
      }*/
    }
    
    // Do this after JavaParser (because it doesn't like package after class)
    
    if (mainPackage != null) {
      print('mainPackage);
      tokPrepend(tok, 1, "package " + mainPackage + ";\n");
      reTok(tok, 1, 2);
    }
    
    if (mainClassName != null) {
      print('mainClassName);
      jreplace(tok, "class main", "class " + mainClassName);
      jreplace(tok, "main.class", mainClassName + ".class");
      //tokPrepend(tok, 1, "class main {}\n"); // so main.class is generated and compiler sanity checks succeed. we can later skip it in the JavaXClassLoader
    }
    
    if (nempty(libs)) {
      print("Adding libs: " + libs);
      tok.add(concatMap_strings(func(S s) -> S { "\n!" + s }, libs));
    }
   } // if (!localStuffOnly)
  } catch e {
    S src = join(tok);
    if (!dontPrintSource)
      print(src);
    print(f2s(saveProgramTextFile("error.java", src)));
    throw rethrow(e);
  }
  
  /*if (useIndexedList)
    print("Indexed/unindexed lookups: " + findCodeTokens_indexed + "/" + findCodeTokens_unindexed + ", lists made: " + IndexedList2.instances);
  print("findCodeToken bails: " + findCodeTokens_bails + "/" + findCodeTokens_nonbails);
  print("javaToks: " + javaTok_n + "/" + javaTok_elements);*/
  
  print("Saving.");
  
  // for dexcompile.php
  if (mainClassName != null)
    tokPrepend(tok, 0, "//FILENAME: " 
      + (mainPackage != null ? mainPackage.replace(".", "/") + "/" : "") + mainClassName + ".java\n");
    
  if (mainJava != null)
    mainJava = join(tok);
  else if (tok.contains("package"))
    splitJavaFiles(tok);
  else
    saveMainJava(tok);
}

static L<S> localStuff1(L<S> tok) {
  int safety = 0, i;
  boolean same;
  tok = indexTokenList(tok);
  
  tok_scopes(tok, autoCloseScopes := true);
    
  do {
    S before = join(tok);
    
    //print("localStuff loop " + safety);
    
    earlyStuff(tok);
    
    // EARLY local stuff goes here
    
    tok_earlyGeneralStuff(tok);
    
    conceptDeclarations(tok);
    tok_recordDecls(tok);
  
    tok = multilineStrings(tok);
    tok_singleQuoteIdentifiersToStringConstants(tok);
    inStringEvals(tok);
    tok_listComprehensions(tok);
    
    tok_for_single(tok);
    
    tok_for_unpair(tok); // Do this...
    tok_doubleFor_v2(tok); // ...before this
    
    tok_forUnnull(tok);
    tok_ifCast(tok);
    forPing(tok);
    tok_directSnippetRefs(tok);
    quicknu(tok);
    //tok_juxtaposeCalls(tok);
    
    jreplace(tok, "LLS", "L<LS>");
    jreplace(tok, "LS", "L<S>");
    jreplace(tok, "ES", "Ext<S>");
    jreplace(tok, "ExtS", "Ext<S>");
    
    jreplace(tok, "dispose <id>;", "{ cleanUp($2); $2 = null; }");
  
    jreplace(tok, "do ping {", "do { ping();");
    
    replaceKeywordBlock(tok,
      "swing",
      "{ swing(r {",
      "}); }");
      
    replaceKeywordBlock(tok,
      "androidUI",
      "{ androidUI(r {",
      "}); }");
      
    replaceKeywordBlock(tok,
      "withDBLock",
      "{ withDBLock(r {",
      "}); }");
      
    replaceKeywordBlock(tok, "afterwards", "temp tempAfterwards(r {", "});");
      
    for (S keyword : ll("tokcondition", "tokCondition"))
      replaceKeywordBlock(tok,
        keyword,
        "new TokCondition { bool get(final L<S> tok, final int i) {",
        "}}");
      
    jreplace(tok, "synced <id>", "synchronized $2");
    jreplace(tok, "sync <id>", "synchronized $2");
    
    replaceKeywordBlock(tok, "answer",
      "static S answer(S s) {\nfinal new Matches m;\n",
      "\nret null;\n}");
      
    replaceKeywordBlock(tok, "static-pcall",
      "static { pcall {",
      "}}");
      
    replaceKeywordBlock(tok, "loading",
      "{ temp tempShowLoadingAnimation(); ",
      "}");
  
    replaceKeywordPlusQuotedBlock(tok, "loading",
      new O { S[] get(L<S> tok, int i) {
        S text = tok.get(i+2);
        ret new S[] {
          "{ temp tempShowLoadingAnimation(" + text + "); ",
          "}" };
      }});
      
    while ((i = jfind(tok, "visualize as")) >= 0) {
      int j = tok_findEndOfStatement(tok, i); // expression, rather
      tok.set(i+2, "{ ret");
      tok.set(j-1, "; }");
      reTok(tok, i, j);
    }
      
    jreplace(tok, "visualize {", "public JComponent visualize() {", tokCondition_beginningOfMethodDeclaration());
  
    jreplace(tok, "visualize2 {", "JComponent visualize2() {", tokCondition_beginningOfMethodDeclaration());
    
    replaceKeywordBlock(tok, "start-thread-printDone", "start-thread {", "printDone(); }");

    replaceKeywordBlock(tok, "start-thread", [[start { thread "Start" { temp enter(); pcall {]], "}}}");
    
    jreplace(tok, "start {", "void start() { super.start();", tokCondition_beginningOfMethodDeclaration());
    
    // run { ... } => public void run() { ... }
    jreplace(tok, "run {", "public void run() {", tokcondition {
      ret neqGet(tok, i-1, "void");
    });

    replaceKeywordBlock(tok, "html",
      "static O html(S uri, fMap<S, S> params) ctex " + "{\n", "}");
    
    replaceKeywordBlock(tok, "afterVisualize",
      "visualize { JComponent _c = super.visualize();",
      "ret _c; }");
    
    replaceKeywordBlock(tok, "enhanceFrame",
      "void enhanceFrame(Container f) { super.enhanceFrame(f);",
      "}");
    
    if (assumeTriple) {
      jreplace(tok, "Triple", "T3");
      expandTriple(tok);
    }
    
    tok_shortFinals(tok);
    
    tok_moduleClassDecls(tok);

    jreplace(tok, "static sync", "static synchronized");
    jreplace(tok, "sclass", "static class");
    jreplace(tok, "srecord", "static record");
    jreplace(tok, "record noeq", "noeq record");
    jreplace(tok, "asclass", "abstract static class");
    jreplace(tok, "sinterface", "static interface");
    jreplace(tok, "ssynchronized", "static synchronized");
  
    jreplace(tok, "ssvoid", "static synchronized void");
    jreplace(tok, "sbool", "static bool");
    jreplace(tok, "fbool", "final bool");
    jreplace(tok, "sint", "static int");
    jreplace(tok, "snew", "static new");
    jreplace(tok, "sv <id>", "static void $2");
    jreplace(tok, "pvoid", "public void");
  
    // "sS" => static S
    jreplace(tok, "sS", "static S");
  
    // "sO" => static O
    jreplace(tok, "sO", "static O");
  
    // "sL" => static L
    jreplace(tok, "sL", "static L");
  
    // "toString {" => "public S toString() {"
    jreplace(tok, "toString {", "public S toString() {");
    
    jreplace(tok, "Int", "Integer");
    jreplace(tok, "Bool", "Boolean");
    jreplace(tok, "BigInt", "BigInteger");
    jreplace(tok, "Char", "Character");
    
    jreplace(tok, "Sym", "Symbol");
    jreplace(tok, "SymSym", "SymbolSymbol");
    
    jreplace(tok, "SS", "Map<S>");
    jreplace(tok, "SymbolSymbol", "Map<Symbol>");
    
    jreplace(tok, "PairS", "Pair<S>");
    jreplace(tok, "LPairS", "L<Pair<S>>");
    
    jreplace(tok, "T3S", "T3<S>");
    jreplace(tok, "F1S", "F1<S>");
    
    jreplace(tok, "ItIt", "IterableIterator");
    jreplace(tok, "CloseableItIt", "CloseableIterableIterator");

    jreplace(tok, "class <id> > <id> {", "class $2 extends $4 {");
    jreplace(tok, "class <id> > <id><<id>> {", "class $2 extends $4  $5 $6 $7 {");
    
    jreplace(tok, "ISegmenter", "IF1<BufferedImage, L<Rect>>");
    
    // IPred<A> => IF1<A, Bool>
    jreplace(tok, "IPred<<id>>", "IF1<$3, Bool>");
    
    // "on fail {" => "catch (Throwable _e) { ... rethrow(_e); }"
    replaceKeywordBlock(tok, "on fail",
      "catch (Throwable _e) {",
      "\nthrow rethrow(_e); }");
  
    // "catch {" => "catch (Throwable _e) {"
    jreplace(tok, "catch {", "catch (Throwable _e) {");
  
    // "catch print e {" => "catch e { _handleException(e); "
    jreplace(tok, "catch print <id> {", "catch $3 { _handleException($3);");
  
    // "catch print short e {" => "catch e { printExceptionShort(e); "
    jreplace(tok, "catch print short <id> {", "catch $4 { printExceptionShort($4);");
    
    // "catch X e {" => "catch (X e) {"
    jreplace(tok, "catch <id> <id> {", "catch ($2 $3) {");
  
    // "catch e {" => "catch (Throwable e) {" (if e is lowercase)
    jreplace(tok, "catch <id> {", "catch (Throwable $2) {", tokcondition {
      S word = tok.get(i+3);
      ret startsWithLowerCaseOrUnderscore(word);
    });
    
    jreplace(tok, "+ +", "+", tokcondition {
      //printStructure("++: ", subList(tok, i-1, i+6));
      if (empty(_get(tok, i+2))) ret false; // no space between the pluses
      if (empty(_get(tok, i)) && eq("+", _get(tok, i-1))) ret false;  // an actual "++" at the left
      if (empty(_get(tok, i+4)) && eq("+", _get(tok, i+5))) ret false;  // an actual "++" at the right
      //print("doing it");
      ret true;
    });
  
    // some crazy fancy syntax
    jreplace(tok, "set <id>;", "$2 = true;");
    
    // single underscore (not allowed in Java anymore) to double underscore
    jreplace(tok, "_", "__");
    
    // [stdEq] -> implementation of equals() and hashCode()
    jreplace(tok, "[stdEq]",
      "public bool equals(O o) { ret stdEq2(this, o); }\n" +
      "public int hashCode() { ret stdHash2(this); }");
    
    // [concepts] "concept.field!" for dereferencing references
    
    jreplace(tok, "*!", "$1.get()", tokcondition {
      S l = tok.get(i+1);
      if (!(isIdentifier(l) || eq(l, ")"))) false;
      if (tok.get(i+2).contains("\n")) false; // no line break between <id> and !
      if (nempty(tok.get(i+4))) true; // space after = ok
      S t = _get(tok, i+5);
      if (t == null) ret false;
      if (isIdentifier(t) || eqOneOf(t, "=", "(")) false;
      true;
    });
    
    jreplace(tok, "for (<id> <id>)", "for ($3 $4 : list($3))");
    jreplace(tok, "for (final <id> <id>)", "for (final $4 $5 : list($4))");

    // "continue unless", "break unless"
    
    for (S phrase : ll("continue unless", "break unless"))
      while ((i = jfind(tok, phrase)) >= 0) {
        S keyword = tok.get(i);
        int j = scanOverExpression(tok, getBracketMap(tok), i+4, ";");
        replaceTokens(tok, i, i+4, "{ if (!(");
        tok.set(j, ")) " + keyword + "; }");
        reTok(tok, i, j+1);
      }
    
    // "continue if", "break if"
    
    for (S phrase : ll("continue if", "break if"))
      while ((i = jfind(tok, phrase)) >= 0) {
        S keyword = tok.get(i);
        int j = scanOverExpression(tok, getBracketMap(tok), i+4, ";");
        replaceTokens(tok, i, i+4, "{ if (");
        tok.set(j, ") " + keyword + "; }");
        reTok(tok, i, j+1);
      }
    
    // "return if"
    
    while ((i = jfind(tok, "return if")) >= 0) {
      int j = scanOverExpression(tok, getBracketMap(tok), i+4, ";");
      replaceTokens(tok, i, i+4, "{ if (");
      tok.set(j, ") return; }");
      reTok(tok, i, j+1);
    }
    
    // "return unless"
    
    while ((i = jfind(tok, "return unless")) >= 0) {
      int j = scanOverExpression(tok, getBracketMap(tok), i+4, ";");
      replaceTokens(tok, i, i+4, "{ if (!(");
      tok.set(j, ")) return; }");
      reTok(tok, i, j+1);
    }
    
    // "return <id> if"
    
    while ((i = jfind(tok, "return <id> if")) >= 0) {
      int j = scanOverExpression(tok, getBracketMap(tok), i+4, ";");
      tok.set(j, ") return " + tok.get(i+2) + "; }");
      replaceTokens(tok, i, i+6, "{ if (");
      reTok(tok, i, j+1);
    }
    
    // "return with <statement>" / "continue with <statement>" / "break with <statement>"
    
    while ((i = jfind(tok, "<id> with", tokcondition { ret eqOneOf(tok.get(i+1), "return", "continue", "break"); })) >= 0) {
      int j = scanOverExpression(tok, getBracketMap(tok), i+4, ";");
      tok.set(j, "; " + tok.get(i) + "; }");
      replaceTokens(tok, i, i+3, "{");
      reTok(tok, i, j+1);
    }
    
    // return "bla" with <statement>
    
    while ((i = jfindOneOf(tok,
      "return <quoted> with", "return <id> with")) >= 0) {
      S result = tok.get(i+2);
      int j = scanOverExpression(tok, getBracketMap(tok), i+6, ";");
      replaceTokens(tok, i, i+5, "{");
      tok.set(j, "; return " + result + "; }");
      reTok(tok, i, j+1);
    }
    
    tok_debugStatements(tok);

    // while not null (...) / if not null (...)
    
    while ((i = jfind_check not(tok, "<id> not null (", tokcondition {
      ret eqOneOf(_get(tok, i+1), "if", "while");
    })) >= 0) {
      int closingBracket = findEndOfBracketPart(tok, i+6)-1;
      replaceTokens(tok, i+2, i+6, "(");
      tok.set(closingBracket, ") != null)");
      reTok(tok, i, closingBracket+1);
    }
    
    // while null (...) / if null (...)
    
    while ((i = jfind_check null(tok, "<id> null (", tokcondition {
      ret eqOneOf(_get(tok, i+1), "if", "while");
    })) >= 0) {
      int closingBracket = findEndOfBracketPart(tok, i+4)-1;
      replaceTokens(tok, i+2, i+4, "(");
      tok.set(closingBracket, ") == null)");
      reTok(tok, i, closingBracket+1);
    }
    
    // Replace $1 with m.unq(0) etc. - caveat: this blocks identifiers $1, $2, ...
    for (i = 1; i < l(tok); i += 2) {
      S s = tok.get(i);
      if (s.startsWith("$")) {
        s = substring(s, 1);
        if (isInteger(s)) {
          tok.set(i, "m.unq(" + (parseInt(s)-1) + ")");
          reTok(tok, i);
        }
      }
    }
  
    // instanceof trickery
    
    jreplace(tok, "is a <id>", "instanceof $3");
    jreplace(tok, "!<id> instanceof <id>.<id>", "!($2 instanceof $4.$6)");
    jreplace(tok, "!<id> instanceof <id>", "!($2 instanceof $4)");
    jreplace(tok, "<id> !instanceof <id>", "!($1 instanceof $4)");
    
    // map func1 func2 func3(...) => mapFGH(f func1, f func2, f func3, ...)
    jreplace(tok, "map <id> <id> <id>(", "mapFGH(f $2, f $3, f $4,");

    // map func1 func2(...) => mapFG(f func1, f func2, ...)
    jreplace(tok, "map <id> <id>(", "mapFG(f $2, f $3,");

    // "ref->bla" for dereferencing Concept.Ref or ThreadLocal or other
    // For lambdas, use SPACES on the left or right of the arrow!
    //jreplace(tok, "<id> ->", "$1.get().");
    jreplace(tok, "->", ".get().", tokcondition {
      ret empty(tok.get(i)) // no space on left of arrow
        && empty(tok.get(i+2)) // no space inside of arrow
        && empty(tok.get(i+4)) // no space on right of arrow
        && !eq(_get(tok, i-1), "-"); // i-->0;
    });
    
    // shortened subconcept declaration (before star constructors!)
    shortenedSubconcepts(tok);
    
    // "case" as a variable name ( => _case)
    
    caseAsVariableName(tok);
    
    // "do" as a function name ( => dO)
    
    tok_doAsMethodName(tok);
    
    // "continue" as a function name ( => _continue)
    continueAsFunctionName(tok);
    
    tok_extend(tok);
    
    jreplace(tok, "pn {", "p-noconsole {");
      
    // Do these BEFORE awt replacement! ("p-awt" contains "awt" token)
    replaceKeywordBlock(tok, "r-awt", "r { awt {", "}}");
    if (hasCodeTokens(tok, "p", "-")) tok_p_old(tok);

    replaceKeywordBlock(tok, "awt-messagebox", "awt { pcall-messagebox {", "}}");
    replaceKeywordBlock(tok, "awt", "swingLater(r {", "});");
  
    jreplace(tok, "p-android {", "set flag Android. p {");
      
    unswing(tok);
    lockBlocks(tok);
    tempBlocks(tok);
    tok_switchTo(tok);
    
    // trim x;
    
    jreplace(tok, "trim <id>;", "$2 = trim($2);");
  
    // iterate with index
  
    jreplace (tok, "for <id> over <id>:", "for (int $2 = 0; $2 < l($4); $2++)");
    jreplace (tok, "for <id> backwards over <id>:", "for (int $2 = l($5)-1; $2 >= 0; $2--)");
    jreplace (tok, "for <id>, <id> <id> over <id>: {", "for (int $2 = 0; $2 < l($7); $2++) { $4 $5 = $7.get($2);");
    jreplace (tok, "for <id>, <id> <id> backwards over <id>: {", "for (int $2 = l($8)-1; $2 >= 0; $2--) { $4 $5 = $8.get($2);");
    jreplace (tok, "for <id> to <id>:", "for (int $2 = 0; $2 < $4; $2++)");
    jreplace (tok, "for <id> to <int>:", "for (int $2 = 0; $2 < $4; $2++)");
    
    tok = expandShortTypes(tok);
      
    tok_equalsCast(tok);
    tok_equalsOptCast(tok);

    replaceKeywordBlock(tok, "r-thread-messagebox", "r-thread { pcall-messagebox {", "}}");
    
    replaceKeywordBlock(tok, "thread-messagebox", "thread { pcall-messagebox {", "}}");
    
    jreplace(tok, "rThread {", "r-thread {");
    jreplace(tok, "rThreadEnter {", "r-thread { temp enter(); ");
  
    replaceKeywordBlock(tok, "r-thread", "runnableThread(r {", "})");
    rNamedThread(tok);
    
    // only works in the scope of a DynModule
    jreplace(tok, "rEnter {", "r { temp enter(); ");
  
    replaceKeywordBlock(tok, "r-messagebox", "r { pcall-messagebox {", "}}");
    
    jreplace(tok, "r <id> + r <id>", "r { $2(); $5(); }");
    
    // runnable and r - now also with automatic toString if enabled
    for (S keyword : ll("runnable", "r")) {
      while ((i = jfind(tok, keyword + " {")) >= 0) {
        int idx = findCodeTokens(tok, i, false, "{");
        int j = findEndOfBracketPart(tok, idx);
        L<S> contents = subList(tok, idx+1, j-1);
        replaceTokens(tok, i, j+1, "new Runnable {"
          + "  public void run() ctex { " + tok_addSemicolon(contents) + "\n}"
          + (autoQuine ? tok_autoQuineFunc(contents) : "")
          + "}");
        reTok(tok, i, j+1);
      }
      
      while ((i = jfind(tok, keyword + " <quoted> {")) >= 0) {
        int idx = findCodeTokens(tok, i, false, "{");
        int j = findEndOfBracketPart(tok, idx);
        L<S> contents = subList(tok, idx+1, j-1);
        replaceTokens(tok, i, j+1, "new Runnable {"
          + "  public void run() ctex { " + tok_addSemicolon(contents) + "\n}"
          + "  toString { ret " + tok.get(i+2) + "; }"
          + (autoQuine ? tok_autoQuineFunc(contents, '_shortenedSourceCode) : "")
          + "}");
        reTok(tok, i, j+1);
      }
    }
    
    replaceKeywordBlock(tok,
      "expectException",
      "{ bool __ok = false; try {",
      "} catch { __ok = true; } assertTrue(\"expected exception\", __ok); }");
      
    while ((i = tok.indexOf("tex")) >= 0) {
      tok.set(i, "throws Exception");
      tok = jtok(tok);
    }
    
    // shorter & smarter whiles
    
    jreplace(tok, "while true", "while (true)");
    jreplace(tok, "while licensed", "while (licensed())");
    jreplace(tok, "repeat {", "while (licensed()) {");
    tok_repeatWithSleep(tok);
    
    // null; => return null; etc.
  
    O cond = tokcondition {
      ret tok_tokenBeforeLonelyReturnValue(tok, i-1);
    };
    jreplace(tok, "null;", "return null;", cond);
    jreplace(tok, "false;", "return false;", cond);
    jreplace(tok, "true;", "return true;", cond);
    jreplace(tok, "this;", "return this;", cond);
    
    // ok <cmd> => ret "OK" with <cmd>
    jreplace(tok, "ok <id>", "return \"OK\" with $2");
    replaceKeywordBlock(tok, "ok", "{", " return \"OK\"; }");
  
    // "myFunction;" instead of "myFunction();" - quite rough
    // (isolated identifier as function call)
    cond = new TokCondition {
      bool get(L<S> tok, int i) {
        S word = tok.get(i+3);
        //print("single word: " + word);
        ret !eqOneOf(word, "break", "continue", "return", "else", "endifdef", "endif");
      }
    };
    for (S pre : litlist("}", ";"))
      jreplace(tok, pre + " <id>;", "$1 $2();", cond);
  
    // shorter match syntax for answer methods
    
    jreplace(tok, "if <quoted> || <quoted>",
      "if (matchOneOf(s, m, $2, $5))");
      
    // "bla * bla | blubb * blubb"
    jreplace_dyn(tok, "if <quoted>", func(L<S> tok, int cIdx) -> S {
      S s = unquote(tok.get(cIdx+2));
      //print("multimatch: " + quote(s));
      new L<S> l;
      for (S pat : splitAtJavaToken(s, "|")) {
        //print("multimatch part: " + quote(pat));
        if (pat.contains("..."))
          l.add("matchX(" + quote(trim(pat)) + ", s, m)");
        else if (javaTok(pat).contains("*"))
          l.add("match(" + quote(trim(pat)) + ", s, m)");
        else
          l.add("match(" + quote(trim(pat)) + ", s)");
      }
      ret "if (" + join(" || ", l) + ")";
    }, tokcondition {
      ret javaTokC(unquote(tok.get(i+3))).contains("|");
    });
    
    // "...bla..."
    jreplace(tok, "if <quoted>", "if (find3plusRestsX($2, s, m))",
      tokcondition {
        ret startsAndEndsWith(unquote(tok.get(i+3)), "...");
      });
      
    // "bla..."
    jreplace(tok, "if <quoted>", "if (matchStartX($2, s, m))",
      tokcondition {
        ret unquote(tok.get(i+3)).endsWith("...");
      });
      
    // "bla"
    jreplace(tok, "if <quoted>", "if (match($2, s))",
      tokcondition {
        ret !javaTokC(unquote(tok.get(i+3))).contains("*");
      });
    // "bla * bla"
    jreplace(tok, "if <quoted>", "if (match($2, s, m))");
    jreplace(tok, "if match <quoted>", "if (match($3, s, m))");
    
    jreplace(tok, "if <id> eq <quoted>", "if (eq($2, $4))");
  
    tok_dropExtraCommas(tok);
    
    // additional translations (if necessary)
    
    jreplace(tok, "pcall ping {", "pcall { ping();");
    
    replaceKeywordBlock(tok, ") pcall",
      ") { pcall {",
      "}}");
    
    replaceKeywordBlock(tok,
      "pcall",
      "try {",
      "} catch (Throwable __e) { _handleException(__e); }");
  
    replaceKeywordBlock(tok,
      "pcall-short",
      "try {",
      "} catch (Throwable __e) { print(exceptionToStringShort(__e)); }");
  
    replaceKeywordBlock(tok,
      "pcall-silent",
      "try {",
      "} catch (Throwable __e) { silentException(__e); }");
  
    replaceKeywordBlock(tok,
      "pcall-messagebox",
      "try {",
      "} catch __e { messageBox(__e); }");
  
    replaceKeywordBlock(tok,
      "pcall-infobox",
      "try {",
      "} catch __e { infoBox(__e); }");
  
    tok = dialogHandler(tok);
    
    replaceKeywordBlock(tok, "exceptionToUser",
      "try {",
      "} catch (Throwable __e) { ret exceptionToUser(__e); }"); 
  
    if (hasCodeTokens(tok, "twice", "{"))
      replaceKeywordBlock(tok, "twice",
        "for (int __twice = 0; __twice < 2; __twice++) {",
        "}"); 
  
    while ((i = findCodeTokens(tok, "bench", "*", "{")) >= 0) {
      int j = findEndOfBracketPart(tok, i+4)-1;
      S time = makeVar("time");
      S v = makeVar("bench");
      S n = tok.get(i+2);
      tok.set(i, "{ long " + time + " = sysNow(); for (int " + v + " = 0; " + v + " < " + n + "; " + v + "++)");
      tok.set(i+2, "");
      tok.set(j, "} printBenchResult(sysNow()-" + time + ", " + n + "); }");
      reTok(tok, i, j+1);
    }
  
    replaceKeywordBlockDyn(tok,
      "time",
      new O { S[] get() {
        S var = makeVar("startTime");
        ret new S[] {
          "{ long " + var + " = sysNow(); try { ",
          "} finally { " + var + " = sysNow()-" + var + "; saveTiming(" + var + "); } }"};
      }});
    
    // version without { }
    replaceKeywordBlockDyn(tok,
      "time2",
      new O { S[] get() {
        S var = makeVar("startTime");
        ret new S[] {
          "long " + var + " = sysNow(); ",
          " " + var + " = sysNow()-" + var + "; saveTiming(" + var + "); "};
      }});
    
    // time "bla" {
    // time msg {
    replaceKeywordPlusQuotedOrIDBlock(tok,
      "time",
      new O { S[] get(L<S> tok, int i) {
        S var = makeVar("startTime");
        ret new S[] {
          "long " + var + " = sysNow(); ",
          " done2_always(" + tok.get(i+2) + ", " + var + "); "};
      }});
    
    if (hasCodeTokens(tok, "assertFail", "{")) {
      S var = makeVar("oops");
      
      replaceKeywordBlock(tok,
        "assertFail",
        "boolean " + var + " = false; try {",
        "\n" + var + " = true; } catch (Exception e) { /* ok */ } assertFalse(" + var + ");"); 
    }
    
    replaceKeywordBlock(tok,
      "yo",
      "try {",
      "} catch (Exception " + makeVar("e") + ") { ret false; }",
      tokcondition {
        ret neqOneOf(_get(tok, i-1), "svoid", "void");
      });
  
    replaceKeywordBlock(tok,
      "awtIfNecessary",
      "swingNowOrLater(r " + "{",
      "});");
      
    ctex(tok);
      
    replaceKeywordBlock(tok,
      "actionListener",
      "new java.awt.event.ActionListener() { " +
        "public void actionPerformed(java.awt.event.ActionEvent _evt) { pcall-messagebox {",
      "}}}");
      
    for (S keyword : ll("autocloseable", "autoCloseable"))
      /*replaceKeywordBlock(tok,
        keyword,
        "new AutoCloseable() { public void close() throws Exception {",
        "}}");*/
        replaceKeywordBlock_dyn2_legacy(tok, keyword, new O {
          S[] get(LS tok, int iOpening, int iClosing) {
            LS contents = subList(tok, iOpening+1, iClosing);
            ret new S[] {
              "new AutoCloseable() { toString { ret " + quote(shorten(maxQuineLength, trimJoin(contents))) + "; } public void close() throws Exception {",
              "}}"
            };
          }
        });
      
    // try answer (string, test with nempty)
    while ((i = findCodeTokens(tok, "try", "answer")) >= 0) {
      int j = findEndOfStatement(tok, i);
      S v = makeVar();
      bool needCurly = !eqGet(tok, i-2, "{");
      tok.set(i, (needCurly ? "{" : "") + " S " + v);
      tok.set(i+2, "=");
      tok.set(j-1, "; if (!empty(" + v + ")) ret " + v + "; " + (needCurly ? "}" : ""));
      reTok(tok, i, j);
    }
  
    // try bool[ean] (try answer with Bool type)
    while ((i = findCodeTokens(tok, "try", "boolean")) >= 0) {
      int j = findEndOfStatement(tok, i);
      S v = makeVar();
      tok.set(i, "{ Bool " + v);
      tok.set(i+2, "=");
      tok.set(j-1, "; if (" + v + " != null) ret " + v + "; }");
      reTok(tok, i, j);
    }
    
    // S s = bla(), return if null; => S s = bla(); if (s == null) return;
    while ((i = jfind(tok, ", return if null;")) >= 0) {
      int j = tok_findBeginningOfStatement(tok, i);
      print("Found statement " + j + "/" + i + " - " + joinSubList(tok, j-1, i+5*2-1));
      S var = getVarDeclarationName(subList(tok, j-1, i));
      replaceTokens_reTok(tok, i, i+5*2-1, "; if (" + var + " == null) ret;");
    }
    
    // <statement>, print "..."; => { <statement>; print("..."); }
    while ((i = jfind(tok, ", print <quoted>;")) >= 0) {
      int j = tok_findBeginningOfStatement(tok, i);
      replaceTokens_reTok(tok, i, i+4*2-1, "; print(" + tok.get(i+4) + "); }");
      tokPrepend_reTok(tok, j, "{ ");
    }
    
    // return if null <expression> => if (<expression> == null) return;
    while ((i = jfind(tok, "return if null")) >= 0) {
      int j = findEndOfStatement(tok, i);
      clearTokens(tok, i, i+2);
      tok.set(i+4, "((");
      tok.set(j-1, ") == null) ret;");
      reTok(tok, i, j);
    }
    
    // return optional (return if not null)
    while ((i = jfind_check optional(tok, "return optional <id> =")) >= 0) {
      int j = findEndOfStatement(tok, i);
      S v = tok.get(i+4);
      clearTokens(tok, i+2, i+4);
      tok.set(i, "{");
      tok.set(j-1, "; if (" + v + " != null) ret " + v + "; }");
      reTok(tok, i, j);
    }
    
    // try object (return if not null)
    while ((i = jfind_check object(tok, "try object")) >= 0) {
      int j = findEndOfStatement(tok, i);
      clearTokens(tok, i, i+3);
      bool isDecl = isIdentifier(get(tok, i+4)) && isIdentifier(get(tok, i+6)) && eqGet(tok, i+8, "=");
      if (isDecl) {
        S v = get(tok, i+6);
        tok.set(i, "{");
        tok.set(j-1, "; if (" + v + " != null) ret " + v + "; }");
      } else {
        S v = makeVar();
        tok.set(i, "{ O " + v + "=");
        tok.set(j-1, "; if (" + v + " != null) ret " + v + "; }");
      }
      reTok(tok, i, j);
    }
    
    // try Int i = ...; (return if not null, shorter version of "try object")
    while ((i = jfind(tok, "try <id> <id> =")) >= 0) {
      int j = findEndOfStatement(tok, i);
      S type = tok.get(i+2), v = tok.get(i+4);
      tok.set(i, "{");
      tok.set(j-1, "; if (" + v + " != null) ret " + v + "; }");
      reTok(tok, i, j);
    }
    
    // debug print ...; => if (debug) print(...);
    while ((i = jfind(tok, "debug print")) >= 0) {
      int j = findEndOfStatement(tok, i);
      replaceTokens(tok, i, i+4, "if (debug) print(");
      tok.set(j-1, ");");
      reTok(tok, i, j);
    }
    
    functionReferences(tok);
    tok_expandLPair(tok);
    tok_expandPairL(tok);
    tok_expandLT3(tok);
    quicknew2(tok);
    
    // X x = nu(+...)  =>  X x = nu X(+...)
    jreplace(tok, "<id> <id> = nu(+", "$1 $2 = nu $1(+");
    // X x = nu(a := ...)  =>  X x = nu X(a := ...)
    jreplace(tok, "<id> <id> = nu(<id> :=", "$1 $2 = nu $1($6 :=");
    
    tok_expandVarCopies(tok); // AFTER the lines just above
    tok_replaceColonEqualsSyntax(tok); // ditto

    tok_unpair(tok);
    tok_cachedFunctions(tok);
    tok_simplyCachedFunctions(tok);
    
    tok_optPar(tok);
    
    throwFailEtc(tok);
    tok_typeAA(tok, pairClasses);

    // do this after expanding sclass etc.
    tok = tok_expandStarConstructors(tok);
    
    tok_kiloConstants(tok);
    //tok_colonMessages(tok);
    
    while ((i = jfind(tok, "shit:")) >= 0) {
      int j = tok_findEndOfStatement(tok, i);
      tok.set(i+2, "(");
      tok.set(j-1, ");");
      reTok(tok, i, j);
    }
      
    jreplace("shit(", "ret with print(");
    
    tok_virtualTypes(tok);
    tok_autoLongConstants(tok);
    
    // common misordering of type arguments
    jreplace("boolean <A>", "<A> boolean");
    
    tok_unimplementedMethods(tok);
    tok_switchableFields(tok);
    tok_autoDisposeFields(tok);
    tok_shortVisualize(tok);
    tok_whileGreaterThan(tok);
    tok_ifThenEnd(tok);
    
    tok_autoInitVars(tok);
    
    tok_fixBadTypeParameterOrder(tok);
    
    // shortened method declarations BEFORE standardFunctions
    jreplace(tok, "svoid", "static void");
    jreplace(tok, "void <id> {", "$1 $2() {");
    jreplace(tok, "void <id> thread {", "$1 $2() thread {");
    jreplace(tok, "String <id> {", "$1 $2() {");
    jreplace(tok, "Object <id> {", "$1 $2() {");
    jreplace(tok, "List <id> {", "$1 $2() {");

    namedThreads(tok);
    threads(tok);
    
    //tok_maxEquals(tok);
    
    tok_questionDot(tok);
    
    tok_embeddedFunctions(tok);
    
    jreplace("<id>[] <id> = new[", "$1[] $4 = new $1[");
    
    jreplace("<id> ifNull =", "if ($1 == null) $1 =");
    
    jreplace("class <id> extends <id><.<id>>", "class $2 extends $4<$2.$7>");
    
    tok_once(tok); // must come before next line so you can combine them
    tok_ifRecordMatch(tok);
    
    jreplace(tok, "<id> ||=", "$1 = $1 ||");
    
    // magicValue followed by a numerical constant
    jreplace("magicValue", "", (_tok, _i) -> {
      S t = _get(_tok, _i+3);
      ret eqOneOf(t, ".", "-") || isInteger(t);
    });
    
    // end of local stuff
    
    tok_processMetaBlocks(tok, metaCodeAllowed());
    if (metaCodeAllowed()) runMetaTransformers(tok);

    same = tok_sameTest(tok, before);
    /*if (!same)
      print("local not same " + safety + " (" + l(tok) + " tokens)");*/
    if (safety++ >= 10) {
      printSources(tok);
      fail("safety 10 error!");
    }
  } while (!same);
  
  ret tok;
}

static L<S> reTok_include(L<S> tok, int i, int j) {
  ret reTok_modify(tok, i, j, f localStuff1);
}

static L<S> includeInMainLoaded_reTok(L<S> tok, int i, int j) {
  ret reTok_include(tok, i, j);
}

static L<S> stdstuff(L<S> tok) {
  //if (++level >= 10) fail("woot? 10");
  
  print("stdstuff!");
  int i;
  
  new L<S> ts;
  tok_findTranslators(tok, ts);
  if (nempty(ts))
    print("DROPPING TRANSLATORS: " + structure(ts));

  print('quickmain); tok = quickmain(tok);
  print('includes); tok = tok_processIncludes(tok);
  print('conceptsDot); if (processConceptsDot(tok))
    tok = tok_processIncludes(tok);
  
  //print('starConstructors); tok = tok_expandStarConstructors(tok);
    
  // drop Java 8 annotations since we're compiling for Java 7
  jreplace(tok, "@Nullable", "");

  // STANDARD CLASSES & INTERFACES
  
  print("standard classes");
  final Set<S> haveClasses = addStandardClasses_v2(tok);
  
  tok_quickInstanceOf(tok, haveClasses);

  // concept-related stuff
  
  // auto-import concepts
  bool _a = tok_hasClassRef2(tok, /*"extends",*/ "Concept") || tok_hasClassRef2(tok, "Concepts"), _b = !haveClasses.contains("Concept");
  //print("auto-import: " + _a + ", " + _b);
  if (_a && _b) {
    print("Auto-including concepts.");
    if (shouldNotIncludeClass.contains("Concepts")) {
      print(join(tok));
      fail("Unwanted concepts import");
    }
    printStruct(haveClasses);
    tok = includeInMainLoaded(tok, "concepts.");
    reTok(tok, l(tok)-1, l(tok));
    //processConceptsDot(tok);
  }
  
  ret tok;
} // end of stdStuff!

static L<S> multilineStrings(L<S> tok) {
  for (int i = 1; i < tok.size(); i += 2) {
    S t = tok.get(i);
    if (isQuoted(t))
      if (t.startsWith("[") || t.contains("\r") || t.contains("\n"))
        tok.set(i, quote(unquote(t)));
  }
  ret tok;
}

static void inStringEvals(L<S> tok) {
  bool change = false;
  for (int i = 1; i < tok.size(); i += 2) {
    S t = tok.get(i);
    if (!isQuoted(t)) continue;
    if (t.contains("\\*") && !t.contains("\\\\")) { // << rough
      tok.set(i, inStringEval(t));
      change = true;
    }
  }
  if (change) reTok(tok);
}

static S inStringEval(S t) {
  t = dropPrefix("\"", dropSuffix("\"", t));
  new L<S> l;
  int idx;
  while ((idx = t.indexOf("\\*")) >= 0) {
    int j = indexOf(t, idx, "*/");
    if (j < 0) break;
    if (idx > 0)
      l.add("\"" + substring(t, 0, idx) + "\"");
    l.add("(" + trim(substring(t, idx+2, j)) + ")");
    t = substring(t, j+2);
  }
  if (nempty(t))
    l.add("\"" + t + "\"");
  ret "(" + join(" + ", l) + ")";
}

static L<S> quickmain(L<S> tok) {
  if (quickmainDone1 && quickmainDone2) ret tok;

  int i = findCodeTokens(tok, "main", "{");
  if (i < 0) i = findCodeTokens(tok, "m", "{");
  if (i >= 0 && !(i-2 > 0 && tok.get(i-2).equals("class"))) {
    tokSet_reTok(tok, i, "class main");
    quickmainDone1 = true;
  }
    
  i = findCodeTokens(tok, "psvm", "{");
  if (i < 0) i = findCodeTokens(tok, "p", "{");
  if (i >= 0) {
    int idx = i+2;
    int j = findEndOfBracketPart(tok, idx)-1;
    L<S> contents = subList(tok, idx+1, j);
    //print("contents: " + sfu(contents));
    tok.set(i, "public static void main(final String[] args) throws Exception");
    replaceTokens(tok, idx+1, j, tok_addSemicolon(contents));
    reTok(tok, i, j);
    quickmainDone2 = true;
  }
    
  ret tok;
}

sS makeVar(S name) {
  AtomicInteger counter = varCountByThread!;
  if (counter == null)
    varCountByThread.set(counter = new AtomicInteger);
  ret "_" + name + "_" + getAndInc(counter);
}

static S makeVar() { ret makeVar(""); }

static L<S> rtq(L<S> tok, S id) {
  ret runTranslatorQuick(tok, id);
}

static L<S> expandShortTypes(L<S> tok) {
  // replace <int> with <Integer>
  for (int i = 1; i+4 < tok.size(); i += 2)
    if (tok.get(i).equals("<")
      && litlist(">", ",").contains(tok.get(i+4))) {
      String type = tok.get(i+2);
      if (type.equals("int")) type = "Integer";
      else if (type.equals("long")) type = "Long";
      tok.set(i+2, type);
    }
    
  // O = Object, S = String, ret = return
  for (int i = 1; i < tok.size(); i += 2) {
    String t = tok.get(i);
    if (t.equals("O")) t = "Object";
    if (t.equals("S")) t = "String";
    else if (t.equals("L")) t = "List";
    else if (t.equals("Cl")) t = "Collection";
    //else if (t.equals("F")) t = "Function";
    else if (t.equals("ret") && neqOneOf(get(tok, i+2), "=", ")", ".")) t = "return";
    else if (t.equals("bool") && i+2 < tok.size() && neq(tok.get(i+2), "(")) t = "boolean"; // bool -> boolean if it's not a function name
    else if (t.equals("AtomicBool")) t = "AtomicBoolean";
    tok.set(i, t);
  }
  
  jreplace(tok, "LL< <id> >", "L<L<$3>>");
  jreplace(tok, "LL<?>", "L<L<?>>");
  jreplace(tok, "LL", "L<L>");
  
  jreplace(tok, "Clusters< <id> >", "Map<$3, Collection<$3>>");
  
  ret tok;
}

static L<S> autoImports(L<S> tok) {
  HashSet<S> imports = new HashSet(tok_findImports(tok));
  new StringBuilder buf;
  for (S c : standardImports)
    if (!(imports.contains(c)))
      buf.append("import " + c + ";\n");
  if (buf.length() == 0) ret tok;
  tok.set(0, buf+tok.get(0));
  ret reTok(tok, 0, 1);
}

static String[] standardImports = {
  "java.util.*",
  "java.util.zip.*",
  "java.util.List",
  "java.util.regex.*",
  "java.util.concurrent.*",
  "java.util.concurrent.atomic.*",
  "java.util.concurrent.locks.*",
  "javax.swing.*",
  "javax.swing.event.*",
  "javax.swing.text.*",
  "javax.swing.table.*",
  "java.io.*",
  "java.net.*",
  "java.lang.reflect.*",
  "java.lang.ref.*",
  "java.lang.management.*",
  "java.security.*",
  "java.security.spec.*",
  "java.awt.*",
  "java.awt.event.*",
  "java.awt.image.*",
  "javax.imageio.*",
  "java.math.*"
};

static L<S> tok_processIncludes(L<S> tok) {
  int safety = 0;
  while (hasCodeTokens(tok, "!", "include") && ++safety < 100)
    tok = tok_processIncludesSingle(tok);
  
  //tok_autoCloseBrackets(tok);
  ret tok;
}

svoid tok_processEarlyIncludes(L<S> tok) {
  int i;
  while ((i = jfind_check include(tok, "!include early #<int>")) >= 0) {
    S id = tok.get(i+8);
    included.add(parseLong(id));
    replaceTokens_reTok(tok, i, i+10, "\n" + cacheGet(id) + "\n");
  }
}

static L<S> tok_processIncludesSingle(L<S> tok) {
  int i;
  while ((i = jfind_check include(tok, "!include #<int>")) >= 0) {
    S id = tok.get(i+6);
    included.add(parseLong(id));
    replaceTokens(tok, i, i+8, "\n" + cacheGet(id) + "\n");
    reTok_include(tok, i, i+8);
  }
  while ((i = jfind_check include(tok, "!include once #<int>")) >= 0) {
    S id = tok.get(i+8);
    bool isNew = included.add(parseLong(id));
    replaceTokens(tok, i, i+10, 
      isNew ? "\n" + cacheGet(id) + "\n" : "");
    reTok_include(tok, i, i+10);
  }
  ret tok;
}

static void ctex(L<S> tok) {
  replaceKeywordBlock(tok, "ctex",
    "{ try {",
    "} catch (Exception __e) { throw rethrow(__e); } }");
  for (S keyword : ll("null on exception", "null on error"))
    replaceKeywordBlock(tok, keyword,
      "{ try {",
      "} catch (Throwable __e) { return null; } }");
  replaceKeywordBlock(tok, "false on exception",
    "{ try {",
    "} catch (Throwable __e) { return false; } }");
}
  
static L<S> dialogHandler(L<S> tok) {
  ret replaceKeywordBlock(tok,
    "dialogHandler",
    "new DialogHandler() {\n" +
      "public void run(final DialogIO io) {",
    "}}");
}

static void quicknew2(L<S> tok) {
  tok_quicknew(tok);

  /*jreplace(tok, "new <id> <id>;", "$2 $3 = new $2;");
  jreplace(tok, "new <id><<id>> <id>;", "$2<$4> $6 = new $2;");
  jreplace(tok, "new <id>.<id> <id>;", "$2.$4 $5 = new $2.$4();");
  jreplace(tok, "new <id><<id>> <id>, <id>;", "$2<$4> $6 = new $2, $8 = new $2;");
  jreplace(tok, "new <id><<id>,<id>> <id>;", "$2<$4,$6> $8 = new $2;");
  jreplace(tok, "new <id><<id><<id>>> <id>;", "$2<$4<$6>> $9 = new $2;");
  jreplace(tok, "new <id><<id>[]> <id>;", "$2 $3 $4 $5 $6 $7 $8 = new $2;");
  jreplace(tok, "new <id>< <id><<id>>, <id>> <id>;", "$2 $3 $4 $5 $6 $7 $8 $9 $10 $11 = new $2;");
  jreplace(tok, "new <id>< <id><<id>,<id>> > <id>;", "$2 $3 $4 $5 $6 $7 $8 $9 $10 $11 = new $2;");
  jreplace(tok, "new <id>< <id>, <id><<id>> > <id>;", "$2 $3 $4 $5 $6 $7 $8 $9 $10 $11 = new $2;");
  jreplace(tok, "new <id>< <id><<id>,<id>,<id>> > <id>;", "$2 $3 $4 $5 $6 $7 $8 $9 $10 $11 $12 $13 = new $2;");
  jreplace(tok, "new <id>< <id><<id>,<id>>, <id>> <id>;", "$2 $3 $4 $5 $6 $7 $8 $9 $10 $11 $12 $13 = new $2;");
  jreplace(tok, "new <id>< <id>, <id><<id>,<id>>> <id>;", "$2 $3 $4 $5 $6 $7 $8 $9 $10 $11 $12 $13 = new $2;");
  jreplace(tok, "new <id>< <id>< <id>, <id><<id>,<id>>>> <id>;", "$2 $3 $4 $5 $6 $7 $8 $9 $10 $11 $12 $13 $14 $15 $16 = new $2;");
  jreplace(tok, "new <id>< <id>< <id>.<id>, <id><<id>,<id>>>> <id>;", "$2 $3 $4 $5 $6 $7 $8 $9 $10 $11 $12 $13 $14 $15 $16 $17 $18 = new $2;");
  jreplace(tok, "new <id>< <id>< <id><<id>>, <id><<id>>>> <id>;", "$2 $3 $4 $5 $6 $7 $8 $9 $10 $11 $12 $13 $14 $15 $16 $17 = new $2;");*/
  
  // [abandoned, confusing, looks like a function definition] with arguments - new A a(...); => A a = new A(...);
  //jreplace(tok, "new <id> <id>(", "$2 $3 = new $2(");

  jreplace(tok, "for args " + "{", "for (int i = 0; i < args.length; i++) { final String arg = args[i];");
  
  // Constructor calls without parentheses
  // So you can say something like: predictors.add(new P1);
  
  jreplace1(tok, "new <id>", "new $2()", tokcondition {
    ret eqOneOf(_get(tok, i+5), "{", ",", ")", ";", ":");
  });
  
  jreplace1(tok, "new <id> < <id> >", "new $2<$4>()", tokcondition {
    ret eqOneOf(_get(tok, i+11), "{", ",", ")", ";", ":");
  });
  
  jreplace(tok, "new List(", "new ArrayList(");
  jreplace(tok, "new Map(", "new HashMap(");
  jreplace(tok, "new Set(", "new HashSet(");
  jreplace(tok, "new (Hash)Set", "new HashSet"); // rough
  jreplace(tok, "new (Tree)Set", "new TreeSet");
  jreplace(tok, "new (Hash)Map", "new HashMap");
  jreplace(tok, "new (Tree)Map", "new TreeMap");
  jreplace(tok, "\\*<id>[<id>] <id>;", "$2[] $6 = new $2[$4];");
  
  // X x = new(...)  =>  X x = new X(...)
  // X x = new {...} =>  X x = new X {...}
  // X x = new       =>  X x = new
  jreplace(tok, "<id> <id> = new", "$1 $2 = new $1", tokcondition {
    ret eqOneOf(_get(tok, i+9), "(", ";", ",", ")", "{");
  });
  jreplace(tok, "<id> <id> = new \\*", "$1 $2 = new $1");
  jreplace(tok, "\\* <id> = new <id>", "$5 $2 = new $5");
  
  // TODO: use backward type scanning
  
  jreplace(tok, "<id><<id>> <id> = new", "$1 $2 $3 $4 $5 = new $1", tokcondition {
    ret eqOneOf(_get(tok, i+9+3*2), "(", ";", ",", ")");
  });
  jreplace(tok, "<id><<id>,<id>> <id> = new", "$1 $2 $3 $4 $5 $6 $7 = new $1", tokcondition {
    ret eqOneOf(_get(tok, i+9+5*2), "(", ";", ",", ")");
  });
  jreplace(tok, "<id><<id><<id>>> <id> = new", "$1 $2 $3 $4 $5 $6 $7 $8 = new $1", tokcondition {
    ret eqOneOf(_get(tok, i+9+6*2), "(", ";", ",", ")");
  });
}

static L<S> extendClasses(L<S> tok) {
  int i;
  while ((i = jfind(tok, "extend <id> {")) >= 0) {
    S className = tok.get(i+2);
    int idx = findCodeTokens(tok, i, false, "{");
    int j = findEndOfBracketPart(tok, idx+2);
    S content = join(subList(tok, idx+1, j-1));
    L<S> c = findInnerClassOfMain(tok, className);
    print("Extending class " + className);
    clearTokens(tok.subList(i, j+1));
    if (c == null) {
      print("Warning: Can't extend class " + className + ", not found");
      continue;
    }
    int startOfClass = indexOfSubList(tok, c); // magicIndexOfSubList is broken
    int endOfClass = startOfClass + l(c)-1;
    //print("Extending class " + className + " ==> " + join(subList(tok, startOfClass, endOfClass)));
    while (neq(tok.get(endOfClass), "}")) --endOfClass;
    //print("Extending class " + className + " ==> " + join(subList(tok, startOfClass, endOfClass)));
    tok.set(endOfClass, content + "\n" + tok.get(endOfClass));
    
    doubleReTok(tok, i, j+1, endOfClass, endOfClass+1);
  }
  ret tok;
}

// for ping / while ping
static void forPing(L<S> tok) {
  int i;
  for (S keyword : ll("for", "fOr", "while"))
    while ((i = jfind(tok, keyword + " ping (")) >= 0) {
      int bracketEnd = findEndOfBracketPart(tok, i+4)-1;
      int iStatement = bracketEnd+2;
      int iEnd = findEndOfStatement(tok, iStatement);
      tok.set(i+2, "");
      
      // turn into block
      if (!eq(get(tok, iStatement), "{")) {
        tok.set(iStatement, "{ " + tok.get(iStatement));
        tok.set(iEnd-1, tok.get(iEnd-1) + " }");
      }
        
      // add ping
      tok.set(iStatement, "{ ping(); " + dropPrefixTrim("{", tok.get(iStatement)));
      reTok(tok, i+2, iEnd);
    }
}

// lib 123 => !123
static void libs(L<S> tok) {
  int i;
  while ((i = jfind(tok, "lib <int>")) >= 0) {
    S id = tok.get(i+2);
    print("lib " + id);
    if (!libs.contains(id)) {
      libs.add(id);
      clearAllTokens(tok, i, i+3);
      /*tok.set(i, "!");
      tok.set(i+1, "");*/
    } else {
      print("...ignoring (duplicate)");
      clearAllTokens(tok, i, i+3);
      reTok(tok, i, i+3);
    }
  }
  print("libs found: " + libs);
}

// sourceCodeLine() => 1234
static void sourceCodeLine(L<S> tok) {
  int i ;
  while ((i = jfind(tok, "sourceCodeLine()")) >= 0) {
    replaceTokens(tok, i, i+5, str(countChar(join(subList(tok, 0, i)), '\n')+1));
    reTok(tok, i, i+5);
  }
}

// done before any other processing
static void earlyStuff(L<S> tok) {
  int i;
  
  tok_scopes(tok, autoCloseScopes := asInclude);
  tok_autosemi(tok);
  tok_autoCloseBrackets(tok);
  jreplace(tok, "°", "()");
  
  // Note: this makes the word "quine" a special operator
  // (unusable as a function name)
  
  while ((i = jfind(tok, "quine(")) >= 0) {
    int idx = findCodeTokens(tok, i, false, "(");
    int j = findEndOfBracketPart(tok, idx+2);
    tok.set(i, "new Quine");
    tok.set(idx, "(" + quote(join(subList(tok, idx+1, j-1))) + ", ");
    reTok(tok, i, idx+1);
  }
  
  jreplace_check after(tok, "void <id> after super {", "void $2 { super.$2();");
  
  // do this before func & voidfunc because otherwise they swallow it
  jreplace(tok, "enter {", "{ temp enter();");
  
  // func keyword for lambdas - now automatically quines toString() if enabled
  
  // do func & voidfunc early to preserve original code as toString
  
  while ((i = jfind(tok, "func(")) >= 0) {
    int argsFrom = i+4, argsTo = findCodeTokens(tok, i, false, ")");
    int idx = findCodeTokens(tok, argsTo, false, "{");
    int j = findEndOfBracketPart(tok, idx);
    L<S> contents = subList(tok, idx+1, j-1);
    
    S returnType = "O";
    if (eq(tok.get(argsTo+2), "-") && eq(tok.get(argsTo+4), ">"))
      returnType = tok_toNonPrimitiveTypes(join(subList(tok, argsTo+6, idx-1)));
      
    S toString = autoQuine ? "  public S toString() { ret " + quote(shorten(maxQuineLength, trimJoin(contents))) + "; }" : "";
    
    L<S> args = cloneSubList(tok, argsFrom-1, argsTo);
    tok_shortFinals(args);
    tok_toNonPrimitiveTypes(args);
    
    L<S> types = tok_typesOfParams(args);
    O type = "O";
    if (l(types) <= 3)
      type = "F" + l(types) + "<" + joinWithComma(types) + ", " + returnType + ">";
    S body = tok_addReturn(contents);
    
    replaceTokens_reTok(tok, i, j,
      "new " + type + "() { "
        + returnType + " get(" + trimJoin(args) + ") ctex { "
          + body
        + " }\n" +
       + toString + "}");
  }
  
  while ((i = jfind(tok, "voidfunc(")) >= 0) {
    int argsFrom = i+4, argsTo = findCodeTokens(tok, i, false, ")");
    int idx = findCodeTokens(tok, argsTo, false, "{");
    int j = findEndOfBracketPart(tok, idx);
    L<S> contents = subList(tok, idx+1, j-1);
    
    L<S> args = cloneSubList(tok, argsFrom-1, argsTo);
    tok_shortFinals(args);
    tok_toNonPrimitiveTypes(args);
    
    L<S> types = tok_typesOfParams(args);
    O type = "O";
    if (l(types) <= 4)
      type = "VF" + l(types) + "<" + joinWithComma(types) + ">";

    replaceTokens(tok, i, j, "new " + type + "() { public void get(" + trimJoin(args) + ") ctex { " + tok_addSemicolon(contents) + " }\n" +
    (autoQuine ? "  public S toString() { ret " + quote(shorten(maxQuineLength, trim(join(contents)))) + "; }" : "") + "}");
    reTok(tok, i, j);
  }
  
  jreplace(tok, "func {", "func -> O {");
  jreplace(tok, "f {", "f -> O {");
  
  // swing -> S { ... } => swing(func -> S { ... })
  while ((i = jfind(tok, "swing ->")) >= 0) {
    int bracket = findCodeTokens(tok, i, false, "{");
    int j = findEndOfBracketPart(tok, bracket);
    tok.set(i, "swing(func");
    tok.set(j-1, "})");
    reTok(tok, i, j);
  }
  
  tok_qFunctions(tok);
  
  for (S keyword : ll(/*"f",*/ "func")) {
    while ((i = jfind(tok, keyword + " ->", tokcondition {
      ret isIdentifier(_get(tok, i+7)); // avoid lambda declaration like: f -> { ... }
    })) >= 0) {
      // I think there is a bug here for something like func -> x { new x { } }
      int idx = findCodeTokens(tok, i, false, "{");
      int j = findEndOfBracketPart(tok, idx);
      S returnType = tok_toNonPrimitiveTypes(join(subList(tok, i+6, idx-1)));
      L<S> contents = subList(tok, idx+1, j-1);
      replaceTokens(tok, i, j, "new F0<" + returnType + ">() { " + returnType + " get() ctex { " + tok_addReturn(contents) + " }\n" +
      (autoQuine ? "  public S toString() { ret " + quote(shorten(maxQuineLength, trim(join(contents)))) + "; }" : "") + "}");
      reTok(tok, i, j);
    }
  }
    
  while ((i = jfind(tok, "time repeat * {")) >= 0) {
    int j = findEndOfBlock(tok, i+6)-1;
    tok.set(i, "time {");
    tok.set(j, "}}");
    reTok(tok, i, j+1);
  }

  // do this before "m {" stuff because "repeat m" [doesn't seem to work yet though)
  while ((i = findCodeTokens(tok, "repeat", "*", "{")) >= 0) {
    S v = makeVar("repeat");
    tok.set(i, "for (int " + v + " = 0; " + v + " < " + tok.get(i+2) + "; " + v + "++)");
    tok.set(i+2, "");
    reTok(tok, i, i+3);
  }
}

static void throwFailEtc(L<S> tok) {
  bool anyChange = false;
  for (int i = 1; i+4 < l(tok); i += 2)
    if (eqOneOf(get(tok, i+2), "fail", "todo")
      && eq(get(tok, i+4), "(")
      && !eqOneOf(get(tok, i), "throw", "RuntimeException", "return", "f", "function")) {
      tok.set(i+2, "throw " + tok.get(i+2));
      anyChange = true;
    }
  if (anyChange)
    reTok(tok);
}

static void namedThreads(L<S> tok) {
  for (int i = 0; i < 100; i++) {
    int idx = findCodeTokens(tok, "thread", "<quoted>", "{");
    if (idx < 0) idx = findCodeTokens(tok, "thread", "<id>", "{");
    if (idx < 0)
      break;
    int j = findEndOfBracketPart(tok, idx+4);
    S tName = tok.get(idx+2);
    
    S pre = "startThread(" + tName + ", r { ";
    S post = "})";
    if (!tok_tokenLeftOfExpression(tok, idx-2)) post += ";";

    tok.set(idx, pre);
    tok.set(idx+2, "");
    tok.set(idx+4, "");
    tok.set(j-1, post);
    //print(join(subList(tok, idx, j)));
    reTok(tok, idx, j);
  }
}

static void rNamedThread(L<S> tok) {
  for (int i = 0; i < 100; i++) {
    int idx = findCodeTokens(tok, "r", "-", "thread", "<quoted>", "{");
    if (idx < 0) idx = findCodeTokens(tok, "r", "-", "thread", "<id>", "{");
    if (idx < 0)
      break;
    int j = findEndOfBracketPart(tok, idx+8);
    S tName = tok.get(idx+6);
    
    S pre = "r { thread " + tName + " {";
    S post = "}}";

    replaceTokens(tok, idx, idx+9, pre);
    tok.set(j-1, post);
    reTok(tok, idx, j);
  }
}

static void threads(L<S> tok) {
  replaceKeywordBlock(tok, "daemon", "startDaemonThread(r {", "});");
  //replaceKeywordBlock(tok, "thread", "startThread(r {", "});");
  
  // now enclose in { } to allow using like "void bla() thread { ... }" - if it's not used as an expression
  
  replaceKeywordBlock_dyn2_legacy(tok, "thread", new O { // don't use func here, it can't be transpiled
    S[] get (LS tok, int iOpening, int iClosing) {
      // is the thread clause used as an expression?
      bool isExpression = tok_tokenLeftOfExpression(tok, iOpening-4);
      ret new S[] {
        (isExpression ? "" : "{ ") + "startThread(r {",
        "})" +
        (isExpression ? "" : "; }")
      };
    }
  });
}

static SS sf; // standard standard functions (haha)
static Bool _761ok;

static L<S> standardFunctions(L<S> tok) {
  if (sf == null) {
    S _761 = cacheGet("#761");
    if (_761ok == null)
      _761ok = isBracketHygienic(_761);
    assertTrue("Whoa! #761 truncated?", _761ok);
    L<S> standardFunctions = concatLists(
      (L) loadVariableDefinition(_761, "standardFunctions"),
      (L) loadVariableDefinition(cacheGet("#1006654"), "standardFunctions"));

    sf = new HashMap;
    for (S x : standardFunctions) {
      S[] f = x.split("/");
      sf.put(f[1], f[0]);
    }
  }
  
  SS sfMap = combinedMap(extraStandardFunctions, sf);
    
  for (int i = 0; ; i++) {
    Set<S> defd = new HashSet(findFunctions(tok));
    
    int j;
    while ((j = jfind(tok, "should not include function *.")) >= 0) {
      S fname = tok.get(j+8);
      shouldNotIncludeFunction.add(fname);
      clearAllTokens(tok.subList(j, j+12));
    }

    while ((j = jfind(tok, "do not include function *.")) >= 0) {
      S fname = tok.get(j+8);
      doNotIncludeFunction.add(fname);
      clearAllTokens(tok.subList(j, j+12));
    }
    
    doNotIncludeFunction.addAll(tok_importedStaticFunctionNames(tok));
    
    // changes tok
    Set<String> invocations = findFunctionInvocations(tok, sfMap, hardFunctionReferences, defd, true);
    /*if (invocations.contains("str"))
      print("==STR==" + join(tok) + "==STR==");*/
    if (!cic(definitions, "leanMode"))
      invocations.addAll(functionsToAlwaysInclude);

    //print("Functions invoked: " + structure(invocations));
    List<String> needed = diff(invocations, defd);
    for (S f : doNotIncludeFunction) needed.remove(f);
    if (needed.isEmpty())
      break;
    print("Adding functions: " + join(" " , needed));
    
    HashSet neededSet = new HashSet(needed);
    Collection<S> bad = setIntersection(neededSet, shouldNotIncludeFunction);
    if (nempty(bad)) {
      S msg = "INCLUDING BAD FUNCTIONS: " + sfu(bad);
      print(msg);
      print(join(tok));
      fail(msg);
    }
      
    new L<S> added;
    new L<Future<S>> parts;
    new L<S> preload;
    
    for (S x : needed)
      if (sfMap.containsKey(x))
        preload.add(sfMap.get(x));
    print("Preloading: " + preload);
    cachePreload(preload);
    
    for (String x : cloneList(needed)) {
      if (defd.contains(x)) continue;
      
      String id = sfMap.get(x);
      if (id == null) {
        print("Standard function " + x + " not found.");
        needed.remove(x);
        continue;
      }
      //print("Adding function: " + x + " (" + id + ")");
       
      S function = cacheGet(id);
      //if (("\n" + function).contains("\n!")) print("Warning: " + id + " contains translators.");
      
      if (cacheStdFunctions) {
        long _id = psI(id);
        CachedInclude ci = cachedIncludes.get(_id);
        if (ci == null) {
          cachedIncludes.put(_id, ci = new CachedInclude);
          //println("Caching include " + _id + ", l=" + l(cachedIncludes));
        }
        function += "\n";
        fS _function = function;
        if (neq(ci.javax, function)) {
          print("Compiling function: " + x);
          ci.javax = function;
          ci.java = executor.submit(new Callable<S>() {
            public S call() {
              // We assume that variables made with makeVar() will always be local to some block
              varCountByThread.set(null);
              ret join(localStuff1(jtok(_function)));
            }
          });
          ci.realJava = null;
        }
        parts.add(ci.javaFuture());
      } else
        parts.add(nowFuture(function + "\n"));
        
      added.add(x);
      Collection<S> newFunctions = new HashSet(findFunctionDefs(javaTok(function)));
      defd.addAll(newFunctions);
      for (S f : newFunctions)
        if (!addedFunctions.add(f)) {
          printSources(tok);
          fail("Trying to add function " + f + " again - main class syntax broken!");
        }
    }
    
    S text = join(getAllFutures(parts));
    tok = includeInMainLoaded(tok, text);
      
    // defd = new HashSet(findFunctions(tok));
    //print("Functions added: " + joinWithSpace(added));
    
    for (String x : needed)
      if (!defd.contains(x)) {
        print(join(tok));
        fail("Function not defined properly: " + x);
      }
    //print("Iteration " + (i+2));
    
    tok_definitions(tok);
    tok_ifndef(tok);
    tok_ifdef(tok);
    
    if (i >= 1000) fail("Too many iterations");
  }
  
  ret tok;
}

static L<S> findFunctions(L<S> tok) {
  //ret findFunctionDefinitions(join(findMainClass(tok)));
  ret findFunctionDefs(findMainClass(tok));
}

static S cacheGet(S snippetID) {
  snippetID = formatSnippetID(snippetID);
  S text = snippetCache.get(snippetID);
  if (text == null) {
    text = loadSnippet(snippetID);
    // very early processing/checks for includes
    
    if (hasUnclosedStringLiterals(text))
      fail("Unclosed string literals in " + snippetID);
      
    if (regexpContains("\\bscope\\b", text)) {
      LS tok = javaTok(text);
      tok_scopes(tok, autoCloseScopes := true);
      text = join(tok);
    }
    
    snippetCache.put(snippetID, text);
  }
  ret text;
}

static void cachePreload(Collection<S> ids) {
  new L<S> needed;
  for (S id : ids)
    if (!snippetCache.containsKey(formatSnippetID(id)))
      needed.add(formatSnippetID(id));
  if (l(needed) > 1) {
    L<S> texts = loadSnippets(needed);
    for (int i = 0; i < l(needed); i++)
      if (texts.get(i) != null)
        snippetCache.put(needed.get(i), texts.get(i));
  }
}

static L<S> jtok(L<S> tok) {
  ret jtok(join(tok));
}

static L<S> jtok(S s) {
  ret indexTokenList(javaTok(s));
}

static HashSet<S> haveClasses_actual(L<S> tok) {
  new HashSet<S> have;
  for (L<S> c : innerClassesOfMain(tok))
    have.add(getClassDeclarationName(c));
  ret have;
}

static HashSet<S> haveClasses_addImported(L<S> tok, HashSet<S> have) {
  have.addAll(tok_importedClassNames(tok));
  have.addAll(usualJavaClassNames()); // for S => S.class
  ret have;
}

// works on Java level (no "sclass" etc)
// returns list of classes we have (useful for other processing)
static Set<S> addStandardClasses_v2(L<S> tok) {
  if (lclasses == null) {
    S sc = cacheGet("#1003674");   
    lclasses = new L;
    for (S line : tlft_j(sc)) {
      int idx = line.indexOf('/');
      lclasses.addAll(ll(line.substring(0, idx), line.substring(idx+1)));
    }
  }
  L<S> definitions = lclasses;

  for (int safety = 0; safety < 10; safety++) {
    HashSet<S> have = haveClasses_actual(tok);
    have.addAll(tok_importedClassNames(tok));

    int j;
    while ((j = jfind(tok, "should not include class *.")) >= 0) {
      S cname = tok.get(j+8);
      shouldNotIncludeClass.add(cname);
      clearAllTokens(tok.subList(j, j+12));
    }
    
    new SS need;
    Set<S> idx = tokenIndexWithoutIfclass_forStdClasses(tok);
    
    while ((j = jfind(tok, "please include class *.")) >= 0) {
      S cname = tok.get(j+6);
      idx.add(cname);
      clearAllTokens(tok.subList(j, j+10));
    }

    for (int i = 0; i+1 < l(definitions); i += 2) {
      S className = definitions.get(i);
      if (idx.contains(className) && !have.contains(className))
        need.put(className, definitions.get(i+1));
    }
    if (hasDef("SymbolAsString")) {
      print("Have SymbolAsString.");
      if (need.containsKey("Symbol")) {
        print("Have Symbol.");
        need.remove("Symbol");
      }
    } else
      print("No SymbolAsString.");
      
    if (empty(need)) ret have;
  
    for (S className : keys(need))
      if (shouldNotIncludeClass.contains(className)) {
        S msg = "INCLUDING BAD CLASS: " + className;
        print(msg);
        print(join(tok));
        fail(msg);
      }
      
    cachePreload(values(need));
    
    new StringBuilder buf;
    print("Adding classes: " + joinWithSpace(keys(need)));
    for (S className : keys(need)) {
      if (have.contains(className)) continue; // intermittent add
      S snippetID = need.get(className);
      //print("Adding class " + className + " / " + snippetID);
      snippetID = fsI(snippetID);
      S text = cacheGet(snippetID);
      
      assertTrue(cacheStdClasses);
      long _id = psI(snippetID);
      CachedInclude ci = cachedIncludes.get(_id);
      if (ci == null) cachedIncludes.put(_id, ci = new CachedInclude);
      if (neq(ci.javax, text)) {
        // TODO: do it in multiple threads
        print("Compiling class: " + className);
        ci.javax = text;
        ci.java = null;
        ci.realJava = join(localStuff1(jtok(text)));
      }
      buf.append(ci.java());

      L<S> ct = javaTok(ci.java());
      for (L<S> c : allClasses(ct)) {
        S name = getClassDeclarationName(c);
        have.add(name);
      }
      if (!have.contains(className))
        fail("Wrongly defined class: " + className + " / " + snippetID);
      if (!addedClasses.add(className)) {
        printSources(tok);
        fail("Trying to add class " + className + " again - main class syntax broken!");
      }
    } // for className
    
    tok = includeInMainLoaded(tok, str(buf));
  }
  fail("safety 10");
}

static Set<S> expandableClassNames = lithashset("BigInteger");

// no reTok - leaves tok dirty
// magically append ".class" to class name references
static bool expandClassReferences_lazy(L<S> tok, Set<S> classNames) {
  bool change = false;
  for (int i = 3; i+2 < l(tok); i += 2) {
    S t = tok.get(i);
    
    // skip implements/extends/throws lists
    if (eqOneOf(t, "implements", "extends", "throws")) {
      i = tok_endOfImplementsList(tok, i);
      continue;
    }
    
    if (classNames.contains(t) || expandableClassNames.contains(t)) {
      S s = tok.get(i-2); t = tok.get(i+2);
      // TODO: This whole logic ain't very good
      // (Hard to distinguish between "Int.class" as an argument
      // and "Int" as a type parameter.)
      if (eqOneOf(s, "instanceof", "new", ".", "<", ">", "/", "nu")) continue;
      if (isIdentifier(s) || isInteger(s)) continue;
      if (eq(t, ",") && isIdentifier(get(tok, i+4)) && eqGet(tok, i+6, ">")) continue; // e.g. T3<L<S>, S, S>
      if (eq(s, ",") && isIdentifier(get(tok, i-4)) && eqGet(tok, i-6, "<")) continue; // e.g. T3<S, S, S>
      if (eq(s, ",") && eqOneOf(_get(tok, i-6), "implements", "throws")) continue;
      
      // check for cast
      if (eq(s, "(") && eq(t, ")") && i >= 5) {
        if (!eqOneOf(get(tok, i+4), "{", ";")) {
          S x = tok.get(i-4);
          if (!isIdentifier(x)) continue;
          if (eqOneOf(x, "ret", "return")) continue;
        }
      }
      if (eqOneOf(t, ",", ")", ";", ":")) {
        tok.set(i, tok.get(i) + ".class");
        change = true;
      }
    }
  }
  ret change;
}

static void expandClassReferences(L<S> tok, Set<S> classNames) {
  if (expandClassReferences_lazy(tok, classNames))
    reTok(tok);
}

// "<id>/<ClassName>" => "((ClassName) <id>)"
static void slashCasts(L<S> tok, final Set<S> classNames) {
  /*jreplace(tok, "<id>/<id>", "(($3) $1)", tokcondition {
    ret classNames.contains(tok.get(i+5));
  });*/
  int n = l(tok)-4;
  for (int i = 1; i < n; i += 2)
    if (tok.get(i+2).equals("/") && isIdentifier(tok.get(i))
      && classNames.contains(tok.get(i+4)))
      replaceTokens_reTok(tok, i, i+5, "((" + tok.get(i+4) + ") " + tok.get(i) + ")");
}

// "<ClassName>(...)" => "new <ClassName>(...)"
// doesn't work at beginning of statement as we can't easily
// distinguish it from a constructor declaration.
static void newWithoutNew(L<S> tok, final Set<S> classNames) {
  TokCondition cond = newWithoutNew_condition(classNames);
  jreplace(tok, "<id>(", "new $1(", cond);
  // just one case with type arg for now
  jreplace(tok, "<id><<id>>(", "new $1<$3>(", cond);
}

static TokCondition newWithoutNew_condition(final Set<S> classNames) {
  ret tokcondition {
    if (!classNames.contains(tok.get(i+1))) false;
    S prev = _get(tok, i-1);
    bool ok = eq(prev, ">") ? eqGet(tok, i-3, "-")
      : neqOneOf(prev, "new", ";", "}", "{", "public", "protected", "private", ".");
    //print("newWithoutNew: checking " + struct(subList(tok, i-1, i+2)) + " => " + ok);
    ret ok;
  };
}

static bool processConceptsDot(L<S> tok) {
  bool anyChange = false, change;
  do {
    change = false;
    for (int i : jfindAll(tok, "concepts."))
      if (contains(get(tok, i+3), "\n")) {
        replaceTokens(tok, i, i+3, "!" + "include once #1004863 // Dynamic Concepts");
        reTok(tok, i, i+3);
        change = anyChange = true;
        break;
      }
  } while (change);
  ret anyChange;
}

static void caseAsVariableName(L<S> tok) {
  if (!tok.contains("case")) ret;
  for (int i = 1; i+2 < l(tok); i += 2) {
    S t = tok.get(i+2);
    if (tok.get(i).equals("case")
      && !(t.startsWith("'") || isInteger(t) || isIdentifier(t)))
      tok.set(i, "_case");
  }
}

static void continueAsFunctionName(L<S> tok) {
  jreplace(tok, "continue(", "_continue(");
}

// f bla => "bla" - and "please include function bla."
static void functionReferences(L<S> tok) {
  int i;
  
  jreplace_dyn(tok, "f-thread <id>", func(L<S> tok, int cIdx) {
    "dynamicCallableMC_thread(" + quote(tok.get(cIdx+6)) + ")"
  });
  
  S keyword = "f";
  while ((i = jfind(tok, keyword + " <id>", tokcondition {
    ret !eq(tok.get(i+3), "instanceof");
  })) >= 0) {
    S f = tok.get(i+2);
    clearTokens(tok, i, i+2);
    tok.set(i+2, quote(f));
    reTok(tok, i, i+2);
    tok.set(l(tok)-1, last(tok) + "\nplease include function " + f + ".");
    reTok(tok, l(tok)-1, l(tok));
  }
  
  // r|rThread|rEnter|rThreadEnter fname => r|rThread|... { fname() }
  while ((i = jfindOneOf_cond(tok, tokcondition {
    ret !eq(tok.get(i+3), "instanceof");
  }, "r <id>", "rThread <id>", "rEnter <id>", "rThreadEnter <id>")) >= 0) {
    S f = tok.get(i+2);
    replaceTokens(tok, i, i+3, tok.get(i) + " { " + f + "(); }");
    reTok(tok, i, i+3);
  }
  
  // dm_q fname => r_dm_q(r fname)
  jreplace(tok, "dm_q <id>", "r_dm_q(r $2)");
  
  // vf<S> fname => voidfunc(S s) { fname(s) }
  jreplace(tok, "vf<<id>> <id>", "voidfunc($3 a) { $5(a) }");
  
  // vf<L<S>> fname => voidfunc(L<S> a) { fname(a) }
  jreplace(tok, "vf<<id><<id>>> <id>", "voidfunc($3<$5> a) { $8(a) }");
  
  // construct<S> Entry => func(S s) -> Entry { new Entry(s) }
  jreplace(tok, "construct<<id>> <id>", "func($3 a) -> $5 { new $5(a) }");
  
  // f<S> fname => func -> S { fname() }
  jreplace(tok, "f<<id>> <id>", "func -> $3 { $5() }");
  
  // f<S, S> fname => func(S x) -> S { fname(x) }
  jreplace(tok, "f<<id>, <id>> <id>", "func($3 x) -> $5 { $7(x) }");
  
  // f<S, L<S>> fname => func(S x) -> L<S> { fname(x) }
  jreplace(tok, "f<<id>, <id><<id>>> <id>", "func($3 x) -> $5 $6 $7 $8 { $10(x) }");
  
  // f<S, L<S>, S> fname => func(S x, L<S> y) -> S { fname(x, y) }
  jreplace(tok, "f<<id>, <id><<id>>, <id>> <id>", "func($3 x, $5 $6 $7 $8 y) -> $10 { $12(x, y) }");
  
  // if1 fname => a -> fname(a)
  jreplace_dyn(tok, "if1 <id>", func(LS tok, int i) -> S {
    S var = makeVar();
    ret var + " -> " + tok.get(i+2) + "(" + var + ")";
  });
}

static void quicknu(L<S> tok) {
  jreplace(tok, "nu <id>(", "nu($2.class, "); // not needed anymore
  jreplace(tok, "nu <id>", "new $2");
}

// fill variable innerClasses_list
static void innerClassesVar(L<S> tok, Set<S> have) {
  int i = jfind_check myInnerClasses_list(tok, ">myInnerClasses_list;");
  if (i < 0) ret;
  tok.set(i+4, "=litlist(\n" + joinQuoted(", ", have) + ");");
  reTok(tok, i+4, i+5);
}

// fill variable innerClasses_list
static void fillVar_transpilationDate(L<S> tok) {
  int i = jfind_check myTranspilationDate_value(tok, "long myTranspilationDate_value;");
  if (i < 0) ret;
  tok.set(i+4, " = " + now() + "L;");
  reTok(tok, i+4, i+5);
}

// process ifclass x ... endif blocks
static void tok_ifclass(L<S> tok, Set<S> have) {
  if (!tok.contains("ifclass")) ret;
  
  int i = l(tok);
  while ((i = rjfind(tok, 1, i-1, "ifclass <id>")) >= 0) {
    int j = jfind(tok, i+4, "endif");
    if (j < 0) j = l(tok)-1;
    S name = tok.get(i+2);
    bool has = have.contains(name);
    //print("ifclass " + name + " => " + has);
    if (has) {
      clearTokens_reTok(tok, j, j+1);
      clearTokens_reTok(tok, i, i+3);
    } else
      clearTokens(tok, i, j+1);
  }
}

// set flag *.
static void tok_definitions(L<S> tok) {
  int i;
  while ((i = jfind_check flag(tok, "set flag <id>.")) >= 0) {
    S fname = tok.get(i+4);
    print("Setting flag " + fname);
    definitions.add(fname);
    clearAllTokens(tok.subList(i, i+8));
  }
  
  while ((i = jfind_check flag(tok, "unset flag <id>.")) >= 0) {
    S fname = tok.get(i+4);
    print("Unsetting flag " + fname);
    definitions.remove(fname);
    clearAllTokens(tok.subList(i, i+8));
  }
}

static void tok_findRewrites(L<S> tok) {
  int i;
  while ((i = jfind(tok, "rewrite <id> =")) >= 0) {
    S token = tok.get(i+2);
    int repStart = i+6;
    int repEnd = smartIndexOf(tok, repStart, ".");
    S replacement = joinSubList(tok, repStart, repEnd-1);
    clearTokens(tok, i, repEnd+1);
    rewrites.put(token, replacement);
    print("Have rewrite: " + token + " => " + replacement);
  }
}

static void tok_processRewrites(L<S> tok) {
  for (S token : keys(rewrites))
    jreplace(tok, token, rewrites.get(token));
}

// extend *. (set base class of main class)
static void tok_extend(L<S> tok) {
  int i;
  while ((i = jfind(tok, "extend <id>.")) >= 0) {
    mainBaseClass = tok.get(i+2);
    clearAllTokens(tok, i, i+7);
  }
}

// process ifndef x ... endifndef blocks
static void tok_ifndef(L<S> tok) {
  if (!tok.contains("ifndef")) ret;
  
  int i;
  while ((i = rjfind(tok, "ifndef <id>")) >= 0) {
    int j = jfind(tok, i+4, "endifndef");
    if (j < 0) j = l(tok)-1;
    S fname = tok.get(i+2);
    bool has = !definitions.contains(fname);
    //print("ifndef " + fname + " => " + has);
    clearTokens(tok, i, i+3);
    clearTokens(tok, j, j+1);
    if (!has) clearTokens(tok, i+3, j);
    reTok(tok, i, j+1);
  }
}

// process ifdef x ... endifdef blocks
static void tok_ifdef(L<S> tok) {
  if (!tok.contains("ifdef")) ret;
  
  int i;
  while ((i = rjfind(tok, "ifdef <id>")) >= 0) {
    int j = jfind(tok, i+4, "endifdef");
    if (j < 0) j = l(tok)-1;
    S fname = tok.get(i+2);
    bool has = definitions.contains(fname);
    //print("ifdef " + fname + " => " + has);
    clearTokens(tok, i, i+3);
    clearTokens(tok, j, j+1);
    if (!has) clearTokens(tok, i+3, j);
    reTok(tok, i, j+1);
  }
}

svoid conceptDeclarations(L<S> tok) {
  for (S kw : ll("concept", "sconcept")) {
    O cond = tokcondition { tok_addFieldOrder(tok, i+1); true; };
    bool re = false;
    if (jreplace(tok, kw + " <id> {", "static class $2 extends Concept {", cond)) re = true;
    if (jreplace(tok, kw + " <id> implements", "static class $2 extends Concept implements", cond)) re = true;
    if (jreplace(tok, kw + " <id>", "static class $2", cond)) re = true;
    if (re) reTok(tok);
  }
}

svoid shortenedSubconcepts(L<S> tok) {
  jreplace(tok, "<id> > <id> {", "concept $3 extends $1 {", tokcondition {
    bool b = (i == 0 || tok.get(i).contains("\n")) || eq(_get(tok, i-1), "abstract"); // only at beginning of line or after "abstract"
    //print("subconcept " + b + ": " + structure(subList(tok, i-1, i+5)));
    ret b;
  });
}

// -slightly experimental
// -do calculation in another thread, then return to AWT thread
// -must be placed in a block
// -transforms rest of block 
svoid unswing(L<S> tok) {
  int i;
  while ((i = jfind(tok, "unswing {")) >= 0) {
    int idx = i+2;
    int closingBracket = findEndOfBracketPart(tok, idx)-1;
    int endOfOuterBlock = findEndOfBracketPart(tok, closingBracket)-1;
    tok.set(i, "thread");
    tok.set(closingBracket, " awt {");
    tok.set(endOfOuterBlock, "}}}");
    reTok(tok, closingBracket-1, endOfOuterBlock+1);
  }
}

// -Syntax: lock theLock;
// -lock a lock, unlock at end of current block with finally
svoid lockBlocks(L<S> tok) {
  int i;
  while ((i = jfind(tok, "lock <id>", tokcondition { ret neq(tok.get(i+3), "instanceof"); })) >= 0) {
    int semicolon = findEndOfStatement(tok, i)-1;
    S var = makeVar();
    int endOfOuterBlock = findEndOfBracketPart(tok, semicolon)-1;
    replaceTokens(tok, i, semicolon+1,
      "Lock " + var + " = " + joinSubList(tok, i+2, semicolon-1) + "; lock(" + var + "); try {");
    tok.set(endOfOuterBlock, "} finally { unlock(" + var + "); } }");
    reTok(tok, i, endOfOuterBlock+1);
  }
}

// -Syntax: temp Bla bla = bla();
// -expands to try(Bla bla = bla()) { ... } with rest of block inside
svoid tempBlocks(L<S> tok) {
  int i;
  while ((i = jfind(tok, "temp <id>")) >= 0) {
    int semicolon = findEndOfStatement(tok, i)-1;
    int endOfOuterBlock = findEndOfBracketPart(tok, semicolon)-1;
    L<S> sub = subList(tok, i-1, semicolon);
    int eq = subList(sub, 0, smartIndexOfOneOf(sub, "{", "(")).indexOf("=");
    S var;
    if (eq >= 0)
      var = sub.get(eq-2);
    else {
      // no var name, e.g. temp newThoughtSpace();
      var = makeVar();
      tok.set(i+2, "AutoCloseable " + var + " = " + tok.get(i+2));
    }
      
    //tok.set(i, "try (");
    //tok.set(semicolon, ") {";
    //tok.set(endOfOuterBlock, "}}");
    
    tok.set(i, "");
    tok.set(semicolon, "; try {");
    tok.set(endOfOuterBlock, "} finally { _close(" + var + "); }}");
    
    reTok(tok, i, endOfOuterBlock+1);
  }
}

svoid forgetCachedIncludes {
  cachedIncludes.clear();
}

// TODO: when to do this / merge contents if there are multiple transpilers?
svoid cleanMeUp {
  for (CachedInclude ci : values(cachedIncludes))
    ci.clean();
  vmKeepWithProgramMD5_save('cachedIncludes);
}

svoid printSources(L<S> tok) {
  print("----");
  print(join(tok));
  print("----");
}

svoid tok_quickInstanceOf(L<S> tok, final Set<S> haveClasses) {
  if (!quickInstanceOfEnabled) ret;
  // "x << X" or "x >> X" => "x instanceof X"
  for (S op : ll("<<", ">>"))
    jreplace(tok, "<id> " + op + " <id>", "$1 instanceof $4", tokcondition {
      ret haveClasses.contains(tok.get(i+7))
        && !eqOneOf(tok.get(i-1), "<", "extends", "implements");
    });
}

sbool hasDef(S s) {
  ret definitions.contains(s);
}

svoid expandTriple(L<S> tok) {
  jreplace(tok, "T3< <id> >", "T3<$3, $3, $3>");
  jreplace(tok, "T3< <id><<id>> >", "T3<$3<$5>, $3<$5>, $3<$5>>");
}

svoid tok_shortFinals(L<S> tok) {
  jreplace(tok, "fS", "final S");
  jreplace(tok, "fO", "final O");
  jreplace(tok, "fL", "final L");
  jreplace(tok, "fMap", "final Map");
  jreplace(tok, "fRunnable", "final Runnable");
  jreplace(tok, "f int", "final int");
}

svoid tok_mainClassNameAndPackage(LS tok) {
  int i;
  if ((i = jfind(tok, "mainClassName <id>")) >= 0) {
    mainClassName = tok.get(i+2);
    if (eqGet(tok, i+4, ".") && isIdentifier(get(tok, i+6))) {
      mainPackage = mainClassName;
      mainClassName = tok.get(i+6);
      clearTokensAndReTok(tok, i, i+7);
    } else
      clearTokensAndReTok(tok, i, i+3);
  }
  
  if ((i = jfind(tok, "mainPackage <id>")) >= 0) {
    mainPackage = tok.get(i+2);
    clearTokens(tok, i, i+3);
    reTok(tok, i, i+3);
  }
}

svoid defineMapLikes(LS tok) {
  int i;
  while ((i = jfind(tok, "mapLike <id>")) >= 0) {
    mapLikeFunctions.add(tok.get(i+2));
    clearTokens_reTok(tok, i, i+2);
  }
}

svoid defineMapMethodLikes(LS tok) {
  int i;
  while ((i = jfind(tok, "mapMethodLike <id>")) >= 0) {
    mapMethodLikeFunctions.add(tok.get(i+2));
    clearTokens_reTok(tok, i, i+2);
  }
}

// functions that work like "nu" syntactically (accept a class as "super-first" parameter [left of the bracket])
svoid defineNuLikes(LS tok) {
  int i;
  while ((i = jfind(tok, "nuLike <id>")) >= 0) {
    nuLikeFunctions.add(tok.get(i+2));
    clearTokens_reTok(tok, i, i+2);
  }
}

svoid defineExtraSF(LS tok) {
  int i;
  IntRange reTok = null;
  while ((i = jfind(tok, "function <id> is in *.")) >= 0) {
    extraStandardFunctions.put(tok.get(i+2), fsI(unquote(tok.get(i+8))));
    clearTokens(tok, i, i+12);
    reTok = combineIntRanges(reTok, intRange(i, i+12));
  }
  reTok(tok, reTok);
}

sbool tok_applyMapLikeFunctions(LS tok, final Set<S> mapLikeFunctions) {
  // map funcname(...) => map(f funcname, ...)
  // filter funcname(...) => filter(f funcname, ...)
  // ...
  ret jreplace(tok, "<id> <id>(", "$1(f $2,", func(L<S> tok, int i) -> bool {
    contains(mapLikeFunctions, tok.get(i+1))
  });
}

sbool tok_applyMapMethodLikeFunctions(LS tok, final Set<S> mapMethodLikeFunctions) {
  // mapMethod funcname(...) => mapMethod('funcname, ...)
  // collect funcname(...) => collect('funcname, ...)
  // ...
  ret jreplace_dyn(tok, "<id> <id>(",
    func(L<S> tok, int cIdx) -> S { tok.get(cIdx) + "(" + quote(tok.get(cIdx+2)) + "," },
    func(L<S> tok, int i) -> bool {
      contains(mapMethodLikeFunctions, tok.get(i+1))
    });
}

svoid runMetaPostBlocks(LS tok) {
  for ping (S code : unnull(metaPostBlocks))
    call(hotwireCached(standardFunctionSnippet(assertIdentifier(code))), code, tok);
    //callF(codeToFunctionOnArbitraryType(code, "LS", L, "tok"), tok);
}

svoid runMetaTransformers(LS tok) {
  for ping (S code : unnull(metaTransformers))
    tok_runMetaTransformer(tok, code);
}

sbool tok_applyNuLikeFunctions(LS tok, final Set<S> nuLikeFunctions) {
  // nu ClassName(...) => nu(ClassName, ...)
  // ...
  ret jreplace_dyn(tok, "<id> <id>(",
    func(L<S> tok, int cIdx) -> S { tok.get(cIdx) + "(" + tok.get(cIdx+2) + ".class," },
    func(L<S> tok, int i) -> bool {
      contains(nuLikeFunctions, tok.get(i+1))
    });
}

sbool metaCodeAllowed() {
  ret allowMetaCode || containsIC(definitions, "allowMetaCode");
}

static LS indexTokenList(LS tok) {
  if (useIndexedList2) ret indexedList2(tok);
  if (useTokenIndexedList) ret tokenIndexedList3(tok);
  ret tok;
}

svoid tok_earlyGeneralStuff(LS tok) {
  tok_standardBot1(tok);
  tok_processSimplified(tok);
  tok_compactModules(tok);
}
Travelled to 7 computer(s): bhatertpkbcr, mqqgnosmbjvj, onxytkatvevr, pyentgdyhuwx, pzhvpgtvlbxg, tvejysmllsmz, vouqrxazstgt
Snippet ID:	#1025804
Snippet name:	Transpiler with TokenIndexedList3 [dev.]
Eternal ID of this version:	#1025804/5
Text MD5:	17173d60b4fb926477587ddc5e6b9377
Transpilation MD5:	c93fde82cf662445e3975e9d9c741df4
Author:	stefan
Category:	javax
Type:	JavaX source code (desktop)
Public (visible to everyone):	Yes
Archived (hidden from active list):	No
Created/modified:	2019-10-21 14:54:00
Source code size:	91420 bytes / 2733 lines
Pitched / IR pitched:	No / No
Views / Downloads:	278 / 907
Version history:	4 change(s)
Referenced in:	[show references]
< > BotCompany Repo | #1025804 // Transpiler with TokenIndexedList3 [dev.]

JavaX source code (desktop) [tags: use-pretranspiled] - run with: x30.jar

Author comment
