!1006722

// ifdef in cached includes? => i think cached includes still contain the ifdefs, so it's fine. tok_ifdef is not called within localStuff1

set flag noAI.
set flag isQuoted_dirty. // should be OK, we're dealing with tokens

static bool autoQuine = true;
static int maxQuineLength = 80;
sbool assumeTriple = true;

// _registerThread usually costs nothing because we need
// the registerWeakHashMap mechanism anyway for ping().
static L<S> functionsToAlwaysInclude = ll(
  "_registerThread",
  //"asRuntimeException"
);

// classes with two type parameters that can written with just one
// e.g. Pair<S> => Pair<S, S>
static Set<S> pairClasses = lithashset("Pair", "Either", "Map", "AbstractMap", "HashMap", "TreeMap", "LinkedHashMap", "MultiMap", "CompactHashMap"); 

!include #1001496 // Matches
!include #1000882 // EGDiff
!include #1000883 // BlockDiffer
!include #1001065 // DialoGIO
!include #1004045 // IndexedList2
//include #1001296 // MultiMap
!include #1000988 // MultiSet
!include #1004543 // DynamicObject
!include #1005575 // CompilerBot

!include #1002662 // isTrue
!include #1010581 // unquote

static int varCount;
static new Map<S, S> snippetCache;
static bool useIndexedList = true, opt_javaTok = true;
static bool cacheStdFunctions = true, cacheStdClasses = true;
static new HashMap<Long, CachedInclude> cachedIncludes;
static ExecutorService executor;
static L lclasses;
static long startTime, lastPrint;

// These variables have to be cleared manually for each transpilation

static new HashSet<Long> included;
static Set<S> definitions = ciSet();
static new HashMap<S> rewrites;
static new HashSet<S> shouldNotIncludeFunction, shouldNotIncludeClass;
static new HashSet<S> doNotIncludeFunction;
static new HashSet<S> addedFunctions;
static new HashSet<S> addedClasses;
static new HashSet<S> hardFunctionReferences;
sbool quickmainDone1, quickmainDone2;
static new TreeSet<S> libs;
sS mainBaseClass;
sbool localStuffOnly; // for transpiling a fragment

sclass CachedInclude {
  S javax;
  Future<S> java;
  S realJava;
  
  S java() {
    ret realJava != null ? realJava : getFuture(java);
  }
  
  Future<S> javaFuture() {
    ret realJava != null ? nowFuture(realJava) : java;
  }
  
  void clean {
    if (java != null) {
      realJava = getFuture(java);
      java = null;
    }
  }
}

p {
  startTime = lastPrint = sysNow();
  pcall { vmKeepWithProgramMD5_get('cachedIncludes); }
  executor = Executors.newFixedThreadPool(numberOfCores());
  
  O oldPrint = interceptPrintInThisThread(new F1<S, Bool>() {
    Bool get(S s) {
      long now = sysNow();
      long time = now-lastPrint; // -startTime;
      lastPrint = now;
      print_raw("[" + formatInt(time/1000, 2) + ":" + formatInt(time % 1000, 3) + "] " + s);
      false;
    }
  });
  
  try {
    _main();
  } finally {
    interceptPrintInThisThread(oldPrint);
    executor.shutdown();
    executor = null;
    localStuffOnly = false;
  }
}

svoid _main() ctex {
  //reTok_modify_check = true;
  if (useIndexedList) findCodeTokens_debug = true;
  javaTok_opt = opt_javaTok;
  findCodeTokens_indexed = findCodeTokens_unindexed = 0;
  findCodeTokens_bails = findCodeTokens_nonbails = 0;
  javaTok_n = javaTok_elements = 0;
  S in = loadMainJava();
  
  print("759 STARTING " + identityHashCode(main.class));
  included.clear();
  definitions.clear();
  rewrites.clear();
  definitions.add("SymbolAsString");
  shouldNotIncludeFunction.clear();
  shouldNotIncludeClass.clear();
  doNotIncludeFunction.clear();
  addedFunctions.clear();
  addedClasses.clear();
  hardFunctionReferences.clear();
  libs.clear();
  mainBaseClass = null;
  varCount = 0;
  quickmainDone1 = quickmainDone2 = false;
  
  //L ts = findTranslators(toLines(join(tok)));
  //print("Translators in source at start: " + structure(ts));
  
  // "duplicate" statement - unused
  
  /*L<S> lines = toLines(in);
  call(getJavaX(), "findTranslators", lines);
  in = fromLines(lines);
  new Matches m;
  if (match("duplicate *", in, m)) {
    // actual copying - unused
    // tok = jtok(loadSnippet(m.get(0)));
    
    // reference by include()
    in = "m { p { callMain(include(" + quote(m.get(0)) + ")); } }";
  }*/
  
  L<S> tok = jtok(in);

  tok_definitions(tok);
  
  // add m { }
  
  if (!localStuffOnly && !hasCodeTokens(tok, "m", "{") && !hasCodeTokens(tok, "main", "{") && !hasCodeTokens(tok, "class", "main")) {
    //tok = jtok(moveImportsUp("m {\n" + in + "\n}"));
    replaceTokens_reTok(tok, 1, 2, "m {\n" + tok.get(1));
    replaceTokens_reTok(tok, l(tok)-2, l(tok)-1, tok.get(l(tok)-2) + "}");
    tok_moveImportsUp(tok);
  }
  
  // standard translate
  
  //ts = findTranslators(toLines(join(tok)));
  //print("Translators in source: " + structure(ts));
  
  if (tok_hasTranslators(tok))
    tok = jtok(defaultTranslate(join(tok)));
  
  //print("end of default translate");
  //print(join(tok));

  //tok_autoCloseBrackets(tok);    
  tok = tok_processIncludes(tok); // before standard functions
  if (processConceptsDot(tok))
    tok = tok_processIncludes(tok);
  tok = localStuff1(tok);
  
 if (!localStuffOnly) {
  int safety = 0;
  boolean same;
  do {
    S before = join(tok);
    
    tok = localStuff1(tok);
    
    // shortened method declarations BEFORE standardFunctions
    jreplace(tok, "svoid", "static void");
    jreplace(tok, "void <id> {", "$1 $2() {");
    jreplace(tok, "String <id> {", "$1 $2() {");
    jreplace(tok, "Object <id> {", "$1 $2() {");
    jreplace(tok, "List <id> {", "$1 $2() {");
    
    // do the non-local stuff (flags and rewrites)
    tok_definitions(tok);
    tok_ifndef(tok);
    tok_ifdef(tok);
    tok_replaceWith(tok);
    tok_findRewrites(tok);
    tok_processRewrites(tok);
    try {
      if (safety == 0) tok = quickmain(tok);
    } catch e {
      printSources(tok);
      rethrow(e);
    }
    tok = standardFunctions(tok);
    tok = stdstuff(tok); // standard functions and all the keywords
    S diff;
    long startTime = now();
    //diff = unidiff(before, join(tok));
    //print("unidiff: " + (now()-startTime) + " ms");
    //same = eq(diff, "");
    same = tok_sameTest(tok, before);
    if (!same) {
      print("Not same " + safety + ".");
      //print(indent(2, diff));
    }
    if (safety++ >= 10) {
      //print(unidiff(before, join(tok)));
      printSources(tok);
      fail("safety 10 error!");
    }
  } while (!same);
  
  print("Post.");
  
  if (mainBaseClass != null) {
    jreplace1(tok, "class main", "class main extends " + mainBaseClass);
    mainBaseClass = null;
  }
  
  print('moveImportsUp); tok_moveImportsUp(tok);
  
  print('Indexing); tok = indexedList2(tok);
  
  // POST-PROCESSING after stdstuff loop
  
  //print("Type<A, A>"); // Type<A> to Type<A, A>
  
  print('extendClasses); tok = extendClasses(tok);
  print('libs); libs(tok);
  print('sourceCodeLine); sourceCodeLine(tok);
  
  // escaping for lambdas
  jreplace(tok, "-=>", "->");

  // Stuff that depends on the list of inner classes (haveClasses)
  HashSet<S> haveClasses = haveClasses_actual(tok);
  print('innerClassesVar); innerClassesVar(tok, haveClasses);
  haveClasses_addImported(haveClasses);
  print('ifclass); tok_ifclass(tok, haveClasses);
  print('slashCasts); slashCasts(tok, haveClasses);
  print('newWithoutNew); newWithoutNew(tok, haveClasses);
  
  if (!assumeTriple) {
    print('Triple);
    if (tok.contains("Triple") && !haveClasses.contains("Triple")) {
      jreplace(tok, "Triple", "T3");
      haveClasses.remove("Triple");
      haveClasses.add("T3");
      slashCasts(tok, lithashset("T3"));
      tok_quickInstanceOf(tok, lithashset("T3"));
    }
    expandTriple(tok);
  }
  
  if (hasDef("SymbolAsString"))
    jreplace(tok, "Symbol", "String");

  print('classReferences); expandClassReferences_lazy(tok, haveClasses);
  
  // Error-checking
  print("Error-checking"); 
  Set<S> functions = new HashSet(findFunctions(tok));
  for (S f : hardFunctionReferences)
    if (!functions.contains(f))
      fail("Function " + f + " requested, but not supplied");

  print('autoImports); tok = autoImports(tok); // faster to do it at the end
  
  print("definitions=" + sfu(definitions));
  if (containsOneOfIC(definitions, "allpublic", "reparse")) {
    bool dontPrintSource = false;
    // Fire up the Java parser & pretty printer!
    print(containsIC(definitions, "allpublic")? "Making all public." : "Reparsing.");
    try {
      S src = join(tok);
      try {
        if (containsIC(definitions, "keepComments"))
          src = javaParser_makeAllPublic_keepComments(src);
        else if (containsIC(definitions, "allpublic"))
          src = javaParser_makeAllPublic(src);
        else
          src = javaParser_reparse_keepComments(src);
      } catch e {
        extractAndPrintJavaParseError(src, e);
        dontPrintSource = true;
        throw e;
      }
      tok = jtok(src);
    } catch e {
      S src = join(tok);
      if (!dontPrintSource)
        print(src);
      print(f2s(saveProgramTextFile("error.java", src)));
      throw rethrow(e);
    }
  }
  
  if (nempty(libs))
    for (S lib : libs)
      tok.add(concatMap_strings(func(S s) -> S { "\n!" + s }, libs));
 } // if (!localStuffOnly)
  
  /*if (useIndexedList)
    print("Indexed/unindexed lookups: " + findCodeTokens_indexed + "/" + findCodeTokens_unindexed + ", lists made: " + IndexedList2.instances);
  print("findCodeToken bails: " + findCodeTokens_bails + "/" + findCodeTokens_nonbails);
  print("javaToks: " + javaTok_n + "/" + javaTok_elements);*/
  
  print("Saving.");
  if (tok.contains("package"))
    splitJavaFiles(tok);
  else
    saveMainJava(tok);
}

static L<S> localStuff1(L<S> tok) {
  int safety = 0, i;
  boolean same;
  tok = indexedList2(tok);
  do {
    S before = join(tok);
    
    earlyStuff(tok);
  
    conceptDeclarations(tok);
    tok_recordDecls(tok);
  
    tok = multilineStrings(tok);
    tok_singleQuoteIdentifiersToStringConstants(tok);
    inStringEvals(tok);
    listComprehensions(tok);
    tok_doubleFor_v2(tok);
    forPing(tok);
    directSnippetRefs(tok);
    quicknu(tok);
    //tok_juxtaposeCalls(tok);
    expandVarCopies(tok);
    
    jreplace(tok, "do ping {", "do { ping();");

    replaceKeywordBlock(tok,
      "swing",
      "{ swing(r {",
      "}); }");
      
    replaceKeywordBlock(tok,
      "tokcondition",
      "new TokCondition { bool get(final L<S> tok, final int i) {",
      "}}");
      
    jreplace(tok, "synced <id>", "synchronized $2");
    
    replaceKeywordBlock(tok, "answer",
      "static S answer(S s) {\nfinal new Matches m;\n",
      "\nret null;\n}");
      
    replaceKeywordBlock(tok, "static-pcall",
      "static { pcall {",
      "}}");
      
    replaceKeywordBlock(tok, "loading",
      "{ JWindow _loading_window = showLoadingAnimation(); try {",
      "} finally { disposeWindow(_loading_window); }}");
  
    replaceKeywordPlusQuotedBlock(tok, "loading",
      new O { S[] get(L<S> tok, int i) {
        S text = tok.get(i+2);
        ret new S[] {
          "{ JWindow _loading_window = showLoadingAnimation(" + text + "); try {",
          "} finally { disposeWindow(_loading_window); }}" };
      }});
  
    replaceKeywordBlock(tok, "html",
      "static O html(S uri, fMap<S, S> params) ctex " + "{\n", "}");
    
    if (assumeTriple) {
      jreplace(tok, "Triple", "T3");
      expandTriple(tok);
    }
    
    tok_shortFinals(tok);
    
    jreplace(tok, "static sync", "static synchronized");
    jreplace(tok, "sclass", "static class");
    jreplace(tok, "srecord", "static record");
    jreplace(tok, "record noeq", "noeq record");
    jreplace(tok, "asclass", "abstract static class");
    jreplace(tok, "sinterface", "static interface");
    jreplace(tok, "ssynchronized", "static synchronized");
  
    jreplace(tok, "ssvoid", "static synchronized void");
    jreplace(tok, "sbool", "static bool");
    jreplace(tok, "sint", "static int");
    jreplace(tok, "snew", "static new");
    jreplace(tok, "sv <id>", "static void $2");
    jreplace(tok, "pvoid", "public void");
  
    // "sS" => static S
    jreplace(tok, "sS", "static S");
  
    // "sO" => static O
    jreplace(tok, "sO", "static O");
  
    // "sL" => static L
    jreplace(tok, "sL", "static L");
  
    // "toString {" => "public S toString() {"
    jreplace(tok, "toString {", "public S toString() {");
    
    jreplace(tok, "Int", "Integer");
    jreplace(tok, "Bool", "Boolean");
    jreplace(tok, "BigInt", "BigInteger");
    jreplace(tok, "Char", "Character");
    
    jreplace(tok, "Sym", "Symbol");
    jreplace(tok, "SymSym", "SymbolSymbol");
    
    jreplace(tok, "SS", "Map<S>");
    jreplace(tok, "SymbolSymbol", "Map<Symbol>");
  
    // "on fail {" => "catch (Throwable _e) { ... rethrow(_e); }"
    replaceKeywordBlock(tok, "on fail",
      "catch (Throwable _e) {",
      "\nthrow rethrow(_e); }");
  
    // "catch {" => "catch (Throwable _e) {"
    jreplace(tok, "catch {", "catch (Throwable _e) {");
  
    // "catch print e {" => "catch e { printException(e); "
    jreplace(tok, "catch print <id> {", "catch $3 { printException($3);");
  
    // "catch print short e {" => "catch e { printExceptionShort(e); "
    jreplace(tok, "catch print short <id> {", "catch $4 { printExceptionShort($4);");
    
    // "catch X e {" => "catch (X e) {"
    jreplace(tok, "catch <id> <id> {", "catch ($2 $3) {");
  
    // "catch e {" => "catch (Throwable e) {" (if e is lowercase)
    jreplace(tok, "catch <id> {", "catch (Throwable $2) {", tokcondition {
      S word = tok.get(i+3);
      ret startsWithLowerCaseOrUnderscore(word);
    });
    
    jreplace(tok, "+ +", "+", tokcondition {
      //printStructure("++: ", subList(tok, i-1, i+6));
      if (empty(_get(tok, i+2))) ret false; // no space between the pluses
      if (empty(_get(tok, i)) && eq("+", _get(tok, i-1))) ret false;  // an actual "++" at the left
      if (empty(_get(tok, i+4)) && eq("+", _get(tok, i+5))) ret false;  // an actual "++" at the right
      //print("doing it");
      ret true;
    });
  
    // some crazy fancy syntax
    jreplace(tok, "set <id>;", "$2 = true;");
    
    // [stdEq] -> implementation of equals() and hashCode()
    jreplace(tok, "[stdEq]",
      "public bool equals(O o) { ret stdEq2(this, o); }\n" +
      "public int hashCode() { ret stdHash2(this); }");
    
    // [concepts] "concept.field!" for dereferencing references
    
    jreplace(tok, "*!", "$1.get()", tokcondition {
      S l = tok.get(i+1);
      if (!(isIdentifier(l) || eq(l, ")"))) false;
      if (tok.get(i+2).contains("\n")) false; // no line break between <id> and !
      if (nempty(tok.get(i+4))) true; // space after = ok
      S t = _get(tok, i+5);
      if (t == null) ret false;
      if (isIdentifier(t) || eqOneOf(t, "=", "(")) false;
      true;
    });
    
    // [concepts] "field := value" for defining fields e.g. in "uniq"
    while ((i = jfind(tok, "<id> :=")) >= 0) {
      tok.set(i, quote(tok.get(i)));
      tok.set(i+2, ",");
      tok.set(i+4, "");
      reTok(tok, i, i+5);
    }
    
    // "quoted" := value
    while ((i = jfind(tok, "<quoted> :=")) >= 0) {
      tok.set(i, tok.get(i));
      tok.set(i+2, ",");
      tok.set(i+4, "");
      reTok(tok, i, i+5);
    }
    
    jreplace(tok, "for (<id> <id>)", "for ($3 $4 : list($3))");
    jreplace(tok, "for (final <id> <id>)", "for (final $4 $5 : list($4))");

    // "continue unless"
    
    while ((i = jfind(tok, "continue unless")) >= 0) {
      int j = scanOverExpression(tok, getBracketMap(tok), i+4, ";");
      replaceTokens(tok, i, i+4, "{ if (!(");
      tok.set(j, ")) continue; }");
      reTok(tok, i, j+1);
    }
    
    // "continue if"
    
    while ((i = jfind(tok, "continue if")) >= 0) {
      int j = scanOverExpression(tok, getBracketMap(tok), i+4, ";");
      replaceTokens(tok, i, i+4, "{ if (");
      tok.set(j, ") continue; }");
      reTok(tok, i, j+1);
    }
    
    // "return if"
    
    while ((i = jfind(tok, "return if")) >= 0) {
      int j = scanOverExpression(tok, getBracketMap(tok), i+4, ";");
      replaceTokens(tok, i, i+4, "{ if (");
      tok.set(j, ") return; }");
      reTok(tok, i, j+1);
    }
    
    // "return unless"
    
    while ((i = jfind(tok, "return unless")) >= 0) {
      int j = scanOverExpression(tok, getBracketMap(tok), i+4, ";");
      replaceTokens(tok, i, i+4, "{ if (!(");
      tok.set(j, ")) return; }");
      reTok(tok, i, j+1);
    }
    
    // "return with <statement>" / "continue with <statement>"
    
    while ((i = jfind(tok, "<id> with", tokcondition { ret eqOneOf(tok.get(i+1), "return", "continue"); })) >= 0) {
      int j = scanOverExpression(tok, getBracketMap(tok), i+4, ";");
      tok.set(j, "; " + tok.get(i) + "; }");
      replaceTokens(tok, i, i+3, "{");
      reTok(tok, i, j+1);
    }
    
    // return "bla" with <statement>
    
    while ((i = jfindOneOf(tok,
      "return <quoted> with", "return <id> with")) >= 0) {
      S result = tok.get(i+2);
      int j = scanOverExpression(tok, getBracketMap(tok), i+6, ";");
      replaceTokens(tok, i, i+5, "{");
      tok.set(j, "; return " + result + "; }");
      reTok(tok, i, j+1);
    }
    
    // while not null ()
    
    while ((i = jfind(tok, "while not null (")) >= 0) {
      int closingBracket = findEndOfBracketPart(tok, i+6)-1;
      replaceTokens(tok, i+2, i+6, "(");
      tok.set(closingBracket, ") != null)");
      reTok(tok, i, closingBracket+1);
    }
    
    // Replace $1 with m.unq(0) etc. - caveat: this blocks identifiers $1, $2, ...
    for (i = 1; i < l(tok); i += 2) {
      S s = tok.get(i);
      if (s.startsWith("$")) {
        s = substring(s, 1);
        if (isInteger(s)) {
          tok.set(i, "m.unq(" + (parseInt(s)-1) + ")");
          reTok(tok, i);
        }
      }
    }
  
    // instanceof trickery
    
    jreplace(tok, "is a <id>", "instanceof $3");
    jreplace(tok, "!<id> instanceof <id>.<id>", "!($2 instanceof $4.$6)");
    jreplace(tok, "!<id> instanceof <id>", "!($2 instanceof $4)");
    jreplace(tok, "<id> !instanceof <id>", "!($1 instanceof $4)");
    
    // map funcname(...) => map(f funcname, ...)
    // filter funcname(...) => filter(f funcname, ...)
    // ...
    jreplace(tok, "<id> <id>(", "$1(f $2,", func(L<S> tok, int i) -> bool {
      eqOneOf(tok.get(i+1), "map", "mapLL", "filter", "concatMap", "pairMap")
    });

    // "ref -> bla" for dereferencing Concept.Ref or ThreadLocal or other
    //jreplace(tok, "<id> ->", "$1.get().");
    jreplace(tok, "->", ".get().", tokcondition {
      ret empty(tok.get(i+2))
        && !(empty(_get(tok, i)) && eq(_get(tok, i-1), "-")); // i-->0;
    });
    
    // shortened subconcept declaration (before star constructors!)
    shortenedSubconcepts(tok);
    
    // "case" as a variable name ( => _case)
    
    caseAsVariableName(tok);
    
    // "do" as a function name ( => dO)
    
    tok_doAsMethodName(tok);
    
    // "continue" as a function name ( => _continue)
    continueAsFunctionName(tok);
    
    tok_extend(tok);
    
    jreplace(tok, "pn {", "p-noconsole {");
      
    // Do these BEFORE awt replacement! ("p-awt" contains "awt" token)
    replaceKeywordBlock(tok, "r-awt", "r { awt {", "}}");
    if (hasCodeTokens(tok, "p", "-")) tok_p_old(tok);

    replaceKeywordBlock(tok, "awt-messagebox", "awt { pcall-messagebox {", "}}");
    replaceKeywordBlock(tok, "awt", "swingLater(r {", "});");
  
    unswing(tok);
    lockBlocks(tok);
    tempBlocks(tok);
    
    // trim x;
    
    jreplace(tok, "trim <id>;", "$2 = trim($2);");
  
    // crazy stuff
  
    jreplace (tok, "for <id> over <id>:", "for (int $2 = 0; $2 < l($4); $2++)");
    jreplace (tok, "for <id>, <id> <id> over <id>: {", "for (int $2 = 0; $2 < l($7); $2++) { $4 $5 = $7.get($2);");
    jreplace (tok, "for <id> to <id>:", "for (int $2 = 0; $2 < $4; $2++)");
    jreplace (tok, "for <id> to <int>:", "for (int $2 = 0; $2 < $4; $2++)");
    
    tok = expandShortTypes(tok);
      
    if (containsToken(tok, "cast")) {
      // TODO: use tokens
      S s = join(tok);
      s = s.replaceAll("(\\w+<[\\w\\s,\\[\\]]+>|\\w+|\\w+\\[\\]|\\w+\\[\\]\\[\\])\\s+(\\w+)\\s*=\\s*cast(\\W[^;]*);", "$1 $2 = ($1) ($3);");
      tok = jtok(s);
    }
    
    replaceKeywordBlock(tok, "r-thread-messagebox", "r-thread { pcall-messagebox {", "}}");
    
    replaceKeywordBlock(tok, "thread-messagebox", "thread { pcall-messagebox {", "}}");
  
    replaceKeywordBlock(tok, "r-thread", "runnableThread(r " + "{", "})");
    rNamedThread(tok);
    
    replaceKeywordBlock(tok, "r-messagebox", "r { pcall-messagebox {", "}}");
    
    // runnable and r - now also with automatic toString if enabled
    for (S keyword : ll("runnable", "r"))
      while ((i = jfind(tok, keyword + " {")) >= 0) {
        int idx = findCodeTokens(tok, i, false, "{");
        int j = findEndOfBracketPart(tok, idx);
        L<S> contents = subList(tok, idx+1, j-1);
        //print("r contents: " + structure(contents));
        replaceTokens(tok, i, j+1, "new Runnable() { public void run() { try { " + tok_addSemicolon(contents) + 
          "\n} catch (Exception __e) { throw rethrow(__e); } }" +
      (autoQuine ? "  public S toString() { return " + quote(shorten(maxQuineLength, trim(join(contents)))) + "; }" : "") +
          "}");
        reTok(tok, i, j+1);
      }
    
    replaceKeywordBlock(tok,
      "expectException",
      "{ bool __ok = false; try {",
      "} catch { __ok = true; } assertTrue(\"expected exception\", __ok); }");
      
    while ((i = tok.indexOf("tex")) >= 0) {
      tok.set(i, "throws Exception");
      tok = jtok(tok);
    }
    
    // shorter & smarter whiles
    
    jreplace(tok, "while true", "while (true)");
    jreplace(tok, "while licensed", "while (licensed())");
    jreplace(tok, "repeat {", "while (licensed()) {");
    tok_repeatWithSleep(tok);
    
    // null; => return null; etc.
  
    O cond = tokcondition {
      ret tok_tokenBeforeLonelyReturnValue(_get(tok, i-1));
    };
    jreplace(tok, "null;", "return null;", cond);
    jreplace(tok, "false;", "return false;", cond);
    jreplace(tok, "true;", "return true;", cond);
    jreplace(tok, "this;", "return this;", cond);
    
    // ok <cmd> => ret "OK" with <cmd>
    jreplace(tok, "ok <id>", "return \"OK\" with $2");
    replaceKeywordBlock(tok, "ok", "{", " return \"OK\"; }");
  
    // "myFunction;" instead of "myFunction();" - quite rough
    cond = new TokCondition {
      bool get(L<S> tok, int i) {
        S word = tok.get(i+3);
        //print("single word: " + word);
        ret !litlist("break", "continue", "return", "else").contains(word);
      }
    };
    for (S pre : litlist("}", ";"))
      jreplace(tok, pre + " <id>;", "$1 $2();", cond);
  
    // shorter match syntax for answer methods
    
    jreplace(tok, "if <quoted> || <quoted>",
      "if (matchOneOf(s, m, $2, $5))");
      
    // "...bla..."
    jreplace(tok, "if <quoted>", "if (find3plusRestsX($2, s, m))",
      tokcondition {
        ret startsAndEndsWith(unquote(tok.get(i+3)), "...");
      });
      
    // "bla..."
    jreplace(tok, "if <quoted>", "if (matchStartX($2, s, m))",
      tokcondition {
        ret unquote(tok.get(i+3)).endsWith("...");
      });
      
    // "bla * bla | blubb * blubb"
    jreplace_dyn(tok, "if <quoted>", func(L<S> tok, int cIdx) {
      S s = unquote(tok.get(cIdx+2));
      //print("multimatch: " + quote(s));
      new L<S> l;
      for (S pat : splitAtJavaToken(s, "|")) {
        //print("multimatch part: " + quote(pat));
        if (javaTok(pat).contains("*"))
          l.add("match(" + quote(trim(pat)) + ", s, m)");
        else
          l.add("match(" + quote(trim(pat)) + ", s)");
      }
      ret "if (" + join(" || ", l) + ")";
    }, tokcondition {
      ret javaTokC(unquote(tok.get(i+3))).contains("|");
    });
    
    // "bla"
    jreplace(tok, "if <quoted>", "if (match($2, s))",
      tokcondition {
        ret !javaTokC(unquote(tok.get(i+3))).contains("*");
      });
    // "bla * bla"
    jreplace(tok, "if <quoted>", "if (match($2, s, m))");
    jreplace(tok, "if match <quoted>", "if (match($3, s, m))");
  
    // extra commas ("litlist(1, 2,)")
    
    jreplace(tok, ",)", ")");
    
    // additional translations (if necessary)
    
    jreplace(tok, "pcall ping {", "pcall { ping();");
    
    replaceKeywordBlock(tok,
      "pcall",
      "try {",
      "} catch (Throwable __e) { printStackTrace2(__e); }");
  
    replaceKeywordBlock(tok,
      "pcall-short",
      "try {",
      "} catch (Throwable __e) { print(exceptionToStringShort(__e)); }");
  
    replaceKeywordBlock(tok,
      "pcall-silent",
      "try {",
      "} catch (Throwable __e) { silentException(__e); }");
  
    replaceKeywordBlock(tok,
      "pcall-messagebox",
      "try {",
      "} catch __e { messageBox(__e); }");
  
    replaceKeywordBlock(tok,
      "pcall-infobox",
      "try {",
      "} catch __e { infoBox(__e); }");
  
    tok = dialogHandler(tok);
    
    replaceKeywordBlock(tok, "exceptionToUser",
      "try {",
      "} catch (Throwable __e) { ret exceptionToUser(__e); }"); 
  
    if (hasCodeTokens(tok, "twice", "{"))
      replaceKeywordBlock(tok, "twice",
        "for (int __twice = 0; __twice < 2; __twice++) {",
        "}"); 
  
    while ((i = findCodeTokens(tok, "repeat", "*", "{")) >= 0) {
      S v = makeVar("repeat");
      tok.set(i, "for (int " + v + " = 0; " + v + " < " + tok.get(i+2) + "; " + v + "++)");
      tok.set(i+2, "");
      reTok(tok, i, i+3);
    }
  
    while ((i = findCodeTokens(tok, "bench", "*", "{")) >= 0) {
      int j = findEndOfBracketPart(tok, i+4)-1;
      S time = makeVar("time");
      S v = makeVar("bench");
      S n = tok.get(i+2);
      tok.set(i, "{ long " + time + " = sysNow(); for (int " + v + " = 0; " + v + " < " + n + "; " + v + "++)");
      tok.set(i+2, "");
      tok.set(j, "} printBenchResult(sysNow()-" + time + ", " + n + "); }");
      reTok(tok, i, j+1);
    }
  
    replaceKeywordBlockDyn(tok,
      "time",
      new O { S[] get() {
        S var = makeVar("startTime");
        ret new S[] {
          "{ long " + var + " = sysNow(); try { ",
          "} finally { " + var + " = sysNow()-" + var + "; saveTiming(" + var + "); } }"};
      }});
    
    // version without { }
    replaceKeywordBlockDyn(tok,
      "time2",
      new O { S[] get() {
        S var = makeVar("startTime");
        ret new S[] {
          "long " + var + " = sysNow(); ",
          " " + var + " = sysNow()-" + var + "; saveTiming(" + var + "); "};
      }});
    
    // time "bla" {
    replaceKeywordPlusQuotedBlock(tok,
      "time",
      new O { S[] get(L<S> tok, int i) {
        S var = makeVar("startTime");
        ret new S[] {
          "long " + var + " = sysNow(); ",
          " done2_always(" + tok.get(i+2) + ", " + var + "); "};
      }});
    
    if (hasCodeTokens(tok, "assertFail", "{")) {
      S var = makeVar("oops");
      
      replaceKeywordBlock(tok,
        "assertFail",
        "boolean " + var + " = false; try {",
        "\n" + var + " = true; } catch (Exception e) { /* ok */ } assertFalse(" + var + ");"); 
    }
    
    replaceKeywordBlock(tok,
      "yo",
      "try {",
      "} catch (Exception " + makeVar("e") + ") { ret false; }",
      tokcondition {
        ret neqOneOf(_get(tok, i-1), "svoid", "void");
      });
  
    replaceKeywordBlock(tok,
      "awtIfNecessary",
      "swingNowOrLater(r " + "{",
      "});");
      
    ctex(tok);
      
    replaceKeywordBlock(tok,
      "actionListener",
      "new java.awt.event.ActionListener() { " +
        "public void actionPerformed(java.awt.event.ActionEvent _evt) { pcall-messagebox {",
      "}}}");
      
    namedThreads(tok);
    threads(tok);
  
    // try answer (string, test with nempty)
    while ((i = findCodeTokens(tok, "try", "answer")) >= 0) {
      int j = findEndOfStatement(tok, i);
      S v = makeVar("a");
      tok.set(i, "{ S " + v);
      tok.set(i+2, "=");
      tok.set(j-1, "; if (!empty(" + v + ")) ret " + v + "; }");
      reTok(tok, i, j);
    }
  
    // try bool[ean] (try answer with Bool type)
    while ((i = findCodeTokens(tok, "try", "boolean")) >= 0) {
      int j = findEndOfStatement(tok, i);
      S v = makeVar("a");
      tok.set(i, "{ Bool " + v);
      tok.set(i+2, "=");
      tok.set(j-1, "; if (" + v + " != null) ret " + v + "; }");
      reTok(tok, i, j);
    }
  
    // return optional (return if not null)
    while ((i = jfind(tok, "return optional <id> =")) >= 0) {
      int j = findEndOfStatement(tok, i);
      S v = tok.get(i+4);
      clearTokens(tok, i+2, i+4);
      tok.set(i, "{");
      tok.set(j-1, "; if (" + v + " != null) ret " + v + "; }");
      reTok(tok, i, j);
    }
    
    // try object (return if not null)
    while ((i = jfind(tok, "try object")) >= 0) {
      int j = findEndOfStatement(tok, i);
      S v = makeVar();
      clearTokens(tok, i, i+3);
      tok.set(i, "{ O " + v + "=");
      tok.set(j-1, "; if (" + v + " != null) ret " + v + "; }");
      reTok(tok, i, j);
    }
    
    // debug print ...; => if (debug) print(...);
    while ((i = jfind(tok, "debug print")) >= 0) {
      int j = findEndOfStatement(tok, i);
      replaceTokens(tok, i, i+4, "if (debug) print(");
      tok.set(j-1, ");");
      reTok(tok, i, j);
    }
    
    functionReferences(tok);
    quicknew2(tok);
    
    tok_unpair(tok);
    tok_cachedFunctions(tok);
    
    throwFailEtc(tok);
    tok_typeAA(tok);

    // do this after expanding sclass etc.
    tok = expandStarConstructors(tok);
    
    tok_kiloConstants(tok);
    
    // end of localStuff1

    same = tok_sameTest(tok, before);
    /*if (!same)
      print("local not same " + safety + " (" + l(tok) + " tokens)");*/
    if (safety++ >= 10) {
      printSources(tok);
      fail("safety 10 error!");
    }
  } while (!same);
  
  ret tok;
}

static L<S> reTok_include(L<S> tok, int i, int j) {
  ret reTok_modify(tok, i, j, f localStuff1);
}

static L<S> includeInMainLoaded_reTok(L<S> tok, int i, int j) {
  ret reTok_include(tok, i, j);
}

static L<S> stdstuff(L<S> tok) {
  //if (++level >= 10) fail("woot? 10");
  
  print("stdstuff!");
  int i;
  
  new L<S> ts;
  tok_findTranslators(tok, ts);
  if (nempty(ts))
    print("DROPPING TRANSLATORS: " + structure(ts));

  print('quickmain); tok = quickmain(tok);
  print('includes); tok = tok_processIncludes(tok);
  print('conceptsDot); if (processConceptsDot(tok))
    tok = tok_processIncludes(tok);
  
  //print('starConstructors); tok = expandStarConstructors(tok);
    
  // drop Java 8 annotations since we're compiling for Java 7
  jreplace(tok, "@Nullable", "");

  // STANDARD CLASSES & INTERFACES
  
  if (lclasses == null) {
    S sc = cacheGet("#1003674");   
    lclasses = new L;
    for (S line : tlft_j(sc)) {
      int idx = line.indexOf('/');
      lclasses.addAll(ll(line.substring(0, idx), line.substring(idx+1)));
    }
  }
  
  //final Set<S> haveClasses = addStandardClasses(tok, toStringArray(lclasses));
  print("standard classes");
  final Set<S> haveClasses = addStandardClasses_v2(tok, lclasses);
  
  tok_quickInstanceOf(tok, haveClasses);

  // concept-related stuff
  
  // auto-import concepts
  bool _a = tok_hasClassRef2(tok, /*"extends",*/ "Concept") || tok_hasClassRef2(tok, "Concepts"), _b = !haveClasses.contains("Concept");
  //print("auto-import: " + _a + ", " + _b);
  if (_a && _b) {
    print("Auto-including concepts.");
    if (shouldNotIncludeClass.contains("Concepts")) {
      print(join(tok));
      fail("Unwanted concepts import");
    }
    printStruct(haveClasses);
    tok = includeInMainLoaded(tok, "concepts.");
    reTok(tok, l(tok)-1, l(tok));
    //processConceptsDot(tok);
  }
  
  ret tok;
} // end of stdStuff!

static L<S> multilineStrings(L<S> tok) {
  for (int i = 1; i < tok.size(); i += 2) {
    S t = tok.get(i);
    if (isQuoted(t))
      if (t.startsWith("[") || t.contains("\r") || t.contains("\n"))
        tok.set(i, quote(unquote(t)));
  }
  ret tok;
}

static void inStringEvals(L<S> tok) {
  bool change = false;
  for (int i = 1; i < tok.size(); i += 2) {
    S t = tok.get(i);
    if (!isQuoted(t)) continue;
    if (t.contains("\\*") && !t.contains("\\\\")) { // << rough
      tok.set(i, inStringEval(t));
      change = true;
    }
  }
  if (change) reTok(tok);
}

static S inStringEval(S t) {
  t = dropPrefix("\"", dropSuffix("\"", t));
  new L<S> l;
  int idx;
  while ((idx = t.indexOf("\\*")) >= 0) {
    int j = indexOf(t, idx, "*/");
    if (j < 0) break;
    if (idx > 0)
      l.add("\"" + substring(t, 0, idx) + "\"");
    l.add("(" + trim(substring(t, idx+2, j)) + ")");
    t = substring(t, j+2);
  }
  if (nempty(t))
    l.add("\"" + t + "\"");
  ret "(" + join(" + ", l) + ")";
}

static L<S> quickmain(L<S> tok) {
  if (quickmainDone1 && quickmainDone2) ret tok;

  int i = findCodeTokens(tok, "main", "{");
  if (i < 0) i = findCodeTokens(tok, "m", "{");
  if (i >= 0 && !(i-2 > 0 && tok.get(i-2).equals("class"))) {
    tokSet(tok, i, "class main");
    quickmainDone1 = true;
  }
    
  i = findCodeTokens(tok, "psvm", "{");
  if (i < 0) i = findCodeTokens(tok, "p", "{");
  if (i >= 0) {
    int idx = i+2;
    int j = findEndOfBracketPart(tok, idx)-1;
    L<S> contents = subList(tok, idx+1, j);
    //print("contents: " + sfu(contents));
    tok.set(i, "public static void main(final String[] args) throws Exception");
    replaceTokens(tok, idx+1, j, tok_addSemicolon(contents));
    reTok(tok, i, j);
    quickmainDone2 = true;
  }
    
  ret tok;
}

static S makeVar(S name) {
  ret "_" + name + "_" + varCount++;
}

static S makeVar() { ret makeVar(""); }

/*static L<S> standardFunctions(L<S> tok) {
  ret rtq(tok, "#1002474");
}*/

static L<S> rtq(L<S> tok, S id) {
  ret runTranslatorQuick(tok, id);
}

static L<S> expandShortTypes(L<S> tok) {
  // replace <int> with <Integer>
  for (int i = 1; i+4 < tok.size(); i += 2)
    if (tok.get(i).equals("<")
      && litlist(">", ",").contains(tok.get(i+4))) {
      String type = tok.get(i+2);
      if (type.equals("int")) type = "Integer";
      else if (type.equals("long")) type = "Long";
      tok.set(i+2, type);
    }
    
  // O = Object, S = String, ret = return
  for (int i = 1; i < tok.size(); i += 2) {
    String t = tok.get(i);
    if (t.equals("O")) t = "Object";
    if (t.equals("S")) t = "String";
    else if (t.equals("L")) t = "List";
    //else if (t.equals("F")) t = "Function";
    else if (t.equals("ret") && neqOneOf(get(tok, i+2), "=", ")", ".")) t = "return";
    else if (t.equals("bool") && i+2 < tok.size() && neq(tok.get(i+2), "(")) t = "boolean"; // bool -> boolean if it's not a function name
    tok.set(i, t);
  }
  
  jreplace(tok, "LL< <id> >", "L<L<$3>>");
  
  jreplace(tok, "Clusters< <id> >", "Map<$3, Collection<$3>>");
  
  ret tok;
}

static L<S> autoImports(L<S> tok) {
  HashSet<S> imports = new HashSet(tok_findImports(tok));
  new StringBuilder buf;
  for (S c : standardImports)
    if (!(imports.contains(c)))
      buf.append("import " + c + ";\n");
  if (buf.length() == 0) ret tok;
  tok.set(0, buf+tok.get(0));
  ret reTok(tok, 0, 1);
}

static String[] standardImports = {
  "java.util.*",
  "java.util.zip.*",
  "java.util.List",
  "java.util.regex.*",
  "java.util.concurrent.*",
  "java.util.concurrent.atomic.*",
  "java.util.concurrent.locks.*",
  "javax.swing.*",
  "javax.swing.event.*",
  "javax.swing.text.*",
  "javax.swing.table.*",
  "java.io.*",
  "java.net.*",
  "java.lang.reflect.*",
  "java.lang.ref.*",
  "java.lang.management.*",
  "java.security.*",
  "java.security.spec.*",
  "java.awt.*",
  "java.awt.event.*",
  "java.awt.image.*",
  "javax.imageio.*",
  "java.math.*"
};

static L<S> expandStarConstructors(L<S> tok) {
  mainLoop: for (int i = 3; i+6 < tok.size(); i += 2) {
    String t = tok.get(i), l = tok.get(i-2);
    if (!t.equals("*"))
      continue;
    if (!tok.get(i+2).equals("("))
      continue;
    if (!eqOneOf(l, "}", "public", "private", "protected", ";", "{", "endif") && neq(get(tok, i-4), "ifclass")) // is this correct...??
      continue;
      
    // ok, it seems like a constructor declaration.
    // Now find class name by going backwards.
    
    int j = i, level = 1;
    while (j > 0 && level > 0) {
      t = tok.get(j);
      if (t.equals("}")) ++level;
      if (t.equals("{")) --level;
      j -= 2;
    }
    
    while (j > 0) {
      t = tok.get(j);
      if (t.equals("class")) {
        String className = tok.get(j+2);
        tok.set(i, className); // exchange constructor name!
        
        // now for the parameters.
        // Syntax: *(Learner *learner) {
        // We will surely add type inference here in time... :)
        
        j = i+2;
        while (!tok.get(j).equals("{"))
          j += 2;
        int block = j+1;
        for (int k = i+2; k < block-1; k += 2)
          if (tok.get(k).equals("*")) {
            tok.remove(k);
            tok.remove(k);
            block -= 2;
            String name = tok.get(k);
            tok.addAll(block, Arrays.asList(new String[] {
              "\n  ", "this", "", ".", "", name, " ", "=", " ", name, "", ";" }));
          }
        
        continue mainLoop;
      }
      j -= 2;
    }
  }
  ret tok;
}

static L<S> tok_processIncludes(L<S> tok) {
  int safety = 0;
  while (hasCodeTokens(tok, "!", "include") && ++safety < 100)
    tok = tok_processIncludesSingle(tok);
  
  //tok_autoCloseBrackets(tok);
  ret tok;
}

static L<S> tok_processIncludesSingle(L<S> tok) {
  int i;
  while ((i = jfind(tok, "!include #<int>")) >= 0) {
    S id = tok.get(i+6);
    included.add(parseLong(id));
    replaceTokens(tok, i, i+8, "\n" + cacheGet(id) + "\n");
    reTok_include(tok, i, i+8);
  }
  while ((i = jfind(tok, "!include once #<int>")) >= 0) {
    S id = tok.get(i+8);
    bool isNew = included.add(parseLong(id));
    replaceTokens(tok, i, i+10, 
      isNew ? "\n" + cacheGet(id) + "\n" : "");
    reTok_include(tok, i, i+10);
  }
  ret tok;
}

static void ctex(L<S> tok) {
  replaceKeywordBlock(tok, "ctex",
    "{ try {",
    "} catch (Exception __e) { throw rethrow(__e); } }");
  replaceKeywordBlock(tok, "null on exception",
    "{ try {",
    "} catch (Throwable __e) { return null; } }");
  replaceKeywordBlock(tok, "false on exception",
    "{ try {",
    "} catch (Throwable __e) { return false; } }");
}
  
static L<S> dialogHandler(L<S> tok) {
  ret replaceKeywordBlock(tok,
    "dialogHandler",
    "new DialogHandler() {\n" +
      "public void run(final DialogIO io) {",
    "}}");
}

static void quicknew2(L<S> tok) {
  jreplace(tok, "new <id> <id>;", "$2 $3 = new $2;");
  jreplace(tok, "new <id><<id>> <id>;", "$2<$4> $6 = new $2;");
  jreplace(tok, "new <id><<id>> <id>, <id>;", "$2<$4> $6 = new $2, $8 = new $2;");
  jreplace(tok, "new <id><<id>,<id>> <id>;", "$2<$4,$6> $8 = new $2;");
  jreplace(tok, "new <id><<id><<id>>> <id>;", "$2 $3 $4 $5 $6 $7 $8 $9 = new $2;");
  jreplace(tok, "new <id><<id>[]> <id>;", "$2 $3 $4 $5 $6 $7 $8 = new $2;");
  jreplace(tok, "new <id>< <id><<id>>, <id>> <id>;", "$2 $3 $4 $5 $6 $7 $8 $9 $10 $11 = new $2;");
  jreplace(tok, "new <id>< <id><<id>,<id>> > <id>;", "$2 $3 $4 $5 $6 $7 $8 $9 $10 $11 = new $2;");
  jreplace(tok, "new <id>< <id>, <id><<id>> > <id>;", "$2 $3 $4 $5 $6 $7 $8 $9 $10 $11 = new $2;");
  jreplace(tok, "new <id>< <id><<id>,<id>,<id>> > <id>;", "$2 $3 $4 $5 $6 $7 $8 $9 $10 $11 $12 $13 = new $2;");
  jreplace(tok, "new <id>< <id><<id>,<id>>, <id>> <id>;", "$2 $3 $4 $5 $6 $7 $8 $9 $10 $11 $12 $13 = new $2;");
  
  // [abandoned, confusing, looks like a function definition] with arguments - new A a(...); => A a = new A(...);
  //jreplace(tok, "new <id> <id>(", "$2 $3 = new $2(");

  jreplace(tok, "for args " + "{", "for (int i = 0; i < args.length; i++) { final String arg = args[i];");
  
  // Constructor calls without parentheses
  // So you can say something like: predictors.add(new P1);
  
  jreplace1(tok, "new <id>", "new $2()", tokcondition {
    ret eqOneOf(_get(tok, i+5), "{", ",", ")", ";", ":");
  });
  
  jreplace1(tok, "new <id> < <id> >", "new $2<$4>()", tokcondition {
    ret eqOneOf(_get(tok, i+11), "{", ",", ")", ";", ":");
  });
  
  jreplace(tok, "new List(", "new ArrayList(");
  jreplace(tok, "new Map(", "new HashMap(");
  jreplace(tok, "new Set(", "new HashSet(");
  jreplace(tok, "new (Hash)Set", "new HashSet"); // rough
  jreplace(tok, "new (Tree)Set", "new TreeSet");
  jreplace(tok, "new (Hash)Map", "new HashMap");
  jreplace(tok, "new (Tree)Map", "new TreeMap");
  jreplace(tok, "\\*<id>[<id>] <id>;", "$2[] $6 = new $2[$4];");
  
  // X x = new(...)  =>  X x = new X(...)
  // X x = new       =>  X x = new
  jreplace(tok, "<id> <id> = new", "$1 $2 = new $1", tokcondition {
    ret eqOneOf(_get(tok, i+9), "(", ";", ",", ")");
  });
  jreplace(tok, "<id> <id> = new \\*", "$1 $2 = new $1");
  jreplace(tok, "\\* <id> = new <id>", "$5 $2 = new $5");
  jreplace(tok, "<id><<id>> <id> = new", "$1 $2 $3 $4 $5 = new $1", tokcondition {
    ret eqOneOf(_get(tok, i+9+3*2), "(", ";", ",", ")");
  });
  jreplace(tok, "<id><<id>,<id>> <id> = new", "$1 $2 $3 $4 $5 $6 $7 = new $1", tokcondition {
    ret eqOneOf(_get(tok, i+9+5*2), "(", ";", ",", ")");
  });
}

static L<S> extendClasses(L<S> tok) {
  int i;
  while ((i = jfind(tok, "extend <id> {")) >= 0) {
    S className = tok.get(i+2);
    int idx = findCodeTokens(tok, i, false, "{");
    int j = findEndOfBracketPart(tok, idx+2);
    S content = join(subList(tok, idx+1, j-1));
    L<S> c = findInnerClassOfMain(tok, className);
    print("Extending class " + className);
    clearTokens(tok.subList(i, j+1));
    if (c == null) {
      print("Warning: Can't extend class " + className + ", not found");
      continue;
    }
    int startOfClass = indexOfSubList(tok, c); // magicIndexOfSubList is broken
    int endOfClass = startOfClass + l(c)-1;
    //print("Extending class " + className + " ==> " + join(subList(tok, startOfClass, endOfClass)));
    while (neq(tok.get(endOfClass), "}")) --endOfClass;
    //print("Extending class " + className + " ==> " + join(subList(tok, startOfClass, endOfClass)));
    tok.set(endOfClass, content + "\n" + tok.get(endOfClass));
    
    doubleReTok(tok, i, j+1, endOfClass, endOfClass+1);
  }
  ret tok;
}

static void listComprehensions(L<S> tok) {
  int i;
  for (S op : ll(":", "in"))
    while ((i = jfind(tok, "[<id> <id> " + op)) >= 0) {
      Map<Integer, Integer> bracketMap = getBracketMap(tok); // XXX - optimize
      S type = tok.get(i+2), id = tok.get(i+4);
      int j = scanOverExpression(tok, bracketMap, i+8, "|");
      S exp = join(tok.subList(i+8, j));
      j += 2;
      int k = scanOverExpression(tok, bracketMap, j, "]");
      S where = join(tok.subList(j, k));
      ++k;
      
      S code = "filter(" + exp + ", func(" + type + " " + id + ") -> Bool { " + where + " })";
      replaceTokens(tok, i, k, code);
      reTok(tok, i, k);
    }
}

// for ping / while ping
static void forPing(L<S> tok) {
  int i;
  for (S keyword : ll("for", "while"))
    while ((i = jfind(tok, keyword + " ping (")) >= 0) {
      int bracketEnd = findEndOfBracketPart(tok, i+4)-1;
      int iStatement = bracketEnd+2;
      int iEnd = findEndOfStatement(tok, iStatement);
      tok.set(i+2, "");
      
      // turn into block
      if (!eq(get(tok, iStatement), "{")) {
        tok.set(iStatement, "{ " + tok.get(iStatement));
        tok.set(iEnd-1, tok.get(iEnd-1) + " }");
      }
        
      // add ping
      tok.set(iStatement, "{ ping(); " + dropPrefixTrim("{", tok.get(iStatement)));
      reTok(tok, i+2, iEnd);
    }
}

// lib 123 => !123
static void libs(L<S> tok) {
  int i;
  while ((i = jfind(tok, "lib <int>")) >= 0) {
    S id = tok.get(i+2);
    print("lib " + id);
    if (!libs.contains(id)) {
      libs.add(id);
      clearAllTokens(tok, i, i+3);
      /*tok.set(i, "!");
      tok.set(i+1, "");*/
    } else {
      print("...ignoring (duplicate)");
      clearAllTokens(tok, i, i+3);
      reTok(tok, i, i+3);
    }
  }
  print("libs found: " + libs);
}

// sourceCodeLine() => 1234
static void sourceCodeLine(L<S> tok) {
  int i ;
  while ((i = jfind(tok, "sourceCodeLine()")) >= 0) {
    replaceTokens(tok, i, i+5, str(countChar(join(subList(tok, 0, i)), '\n')+1));
    reTok(tok, i, i+5);
  }
}

// done before any other processing
static void earlyStuff(L<S> tok) {
  int i;
  
  tok_scopes(tok);
  tok_autosemi(tok);
  tok_autoCloseBrackets(tok);
  jreplace(tok, "°", "()");
  
  // Note: this makes the word "quine" a special operator
  // (unusable as a function name)
  
  while ((i = jfind(tok, "quine(")) >= 0) {
    int idx = findCodeTokens(tok, i, false, "(");
    int j = findEndOfBracketPart(tok, idx+2);
    tok.set(i, "new Quine");
    tok.set(idx, "(" + quote(join(subList(tok, idx+1, j-1))) + ", ");
    reTok(tok, i, idx+1);
  }
  
    // func keyword for lambdas - now automatically quines toString() if enabled
    
    // do func & voidfunc early to preserve original code as toString
    
    while ((i = jfind(tok, "func(")) >= 0) {
      int argsFrom = i+4, argsTo = findCodeTokens(tok, i, false, ")");
      int idx = findCodeTokens(tok, argsTo, false, "{");
      int j = findEndOfBracketPart(tok, idx);
      L<S> contents = subList(tok, idx+1, j-1);
      
      S returnType = "O";
      if (eq(tok.get(argsTo+2), "-") && eq(tok.get(argsTo+4), ">"))
        returnType = tok_toNonPrimitiveTypes(join(subList(tok, argsTo+6, idx-1)));
        
      S toString = autoQuine ? "  public S toString() { ret " + quote(shorten(maxQuineLength, trim(join(contents)))) + "; }" : "";
      
      L<S> args = cloneSubList(tok, argsFrom-1, argsTo);
      tok_shortFinals(args);
      tok_toNonPrimitiveTypes(args);
      
      L<S> types = tok_typesOfParams(args);
      O type = "O";
      if (l(types) == 1)
        type = "F1<" + first(types) + ", " + returnType + ">";
      S body = tok_addReturn(contents);
      
      replaceTokens_reTok(tok, i, j,
        "new " + type + "() { "
          + returnType + " get(" + trimJoin(args) + ") ctex { "
            + body
          + " }\n" +
         + toString + "}");
    }
    
    while ((i = jfind(tok, "voidfunc(")) >= 0) {
      int argsFrom = i+4, argsTo = findCodeTokens(tok, i, false, ")");
      int idx = findCodeTokens(tok, argsTo, false, "{");
      int j = findEndOfBracketPart(tok, idx);
      L<S> contents = subList(tok, idx+1, j-1);
      
      L<S> args = cloneSubList(tok, argsFrom-1, argsTo);
      tok_shortFinals(args);
      tok_toNonPrimitiveTypes(args);
      
      L<S> types = tok_typesOfParams(args);
      O type = "O";
      if (l(types) <= 2)
        type = "VF" + l(types) + "<" + joinWithComma(types) + ">";

      replaceTokens(tok, i, j, "new " + type + "() { void get(" + trimJoin(args) + ") ctex { " + tok_addSemicolon(contents) + " }\n" +
      (autoQuine ? "  public S toString() { ret " + quote(shorten(maxQuineLength, trim(join(contents)))) + "; }" : "") + "}");
      reTok(tok, i, j);
    }
    
    jreplace(tok, "func {", "func -> O {");
    jreplace(tok, "f {", "f -> O {");
    
    for (S keyword : ll("f", "func")) {
      while ((i = jfind(tok, keyword + " ->")) >= 0) {
        // I think there is a bug here for something like func -> x { new x { } }
        int idx = findCodeTokens(tok, i, false, "{");
        int j = findEndOfBracketPart(tok, idx);
        S returnType = tok_toNonPrimitiveTypes(join(subList(tok, i+6, idx-1)));
        L<S> contents = subList(tok, idx+1, j-1);
        replaceTokens(tok, i, j, "new F0<" + returnType + ">() { " + returnType + " get() ctex { " + tok_addReturn(contents) + " }\n" +
        (autoQuine ? "  public S toString() { ret " + quote(shorten(maxQuineLength, trim(join(contents)))) + "; }" : "") + "}");
        reTok(tok, i, j);
      }
    }
}

static void throwFailEtc(L<S> tok) {
  bool anyChange = false;
  for (int i = 1; i+4 < l(tok); i += 2)
    if (eqOneOf(get(tok, i+2), "fail", "todo")
      && eq(get(tok, i+4), "(")
      && !eqOneOf(get(tok, i), "throw", "RuntimeException", "return", "f", "function")) {
      tok.set(i+2, "throw " + tok.get(i+2));
      anyChange = true;
    }
  if (anyChange)
    reTok(tok);
}

static void namedThreads(L<S> tok) {
  for (int i = 0; i < 100; i++) {
    int idx = findCodeTokens(tok, "thread", "<quoted>", "{");
    if (idx < 0) idx = findCodeTokens(tok, "thread", "<id>", "{");
    if (idx < 0)
      break;
    int j = findEndOfBracketPart(tok, idx+4);
    S tName = tok.get(idx+2);
    
    S var = "_t_" + i;
    S pre = "{ Thread " + var + " = new Thread(" + tName + ") {\n" +
      "public void run() { pcall {\n";
    S post = "} }\n};\n" +
      "startThread(" + var + "); }";

    tok.set(idx, pre);
    tok.set(idx+2, "");
    tok.set(idx+4, "");
    tok.set(j-1, post);
    //print(join(subList(tok, idx, j)));
    reTok(tok, idx, j);
  }
}

static void rNamedThread(L<S> tok) {
  for (int i = 0; i < 100; i++) {
    int idx = findCodeTokens(tok, "r", "-", "thread", "<quoted>", "{");
    if (idx < 0) idx = findCodeTokens(tok, "r", "-", "thread", "<id>", "{");
    if (idx < 0)
      break;
    int j = findEndOfBracketPart(tok, idx+8);
    S tName = tok.get(idx+6);
    
    S pre = "r { thread " + tName + " {";
    S post = "}}";

    replaceTokens(tok, idx, idx+9, pre);
    tok.set(j-1, post);
    reTok(tok, idx, j);
  }
}

static void threads(L<S> tok) {
  for (bool daemon : litlist(false, true))
    for (int i = 0; i < 100; i++) {
      int idx = findCodeTokens(tok, daemon ? "daemon" : "thread", "{");
      if (idx < 0)
        break;
      int j = findEndOfBracketPart(tok, idx+2);

      S var = "_t_" + i;
      S pre = "{ Thread " + var + " = new Thread() {\n" +
        "public void run() { pcall\n";
      S post = "} }\n};\n" +
        (daemon ? var + ".setDaemon(true);\n" : "") +
        "startThread(" + var + "); }";

      tok.set(idx, pre);
      tok.set(j-1, post);
      reTok(tok, idx, j);
    }
}

static Map<S, S> sf;
static Bool _761ok;

static L<S> standardFunctions(L<S> tok) {
  if (sf == null) {
    S _761 = cacheGet("#761");
    if (_761ok == null)
      _761ok = isBracketHygienic(_761);
    assertTrue("Whoa! #761 truncated?", _761ok);
    L<S> standardFunctions = concatLists(
      (L) loadVariableDefinition(_761, "standardFunctions"),
      (L) loadVariableDefinition(cacheGet("#1006654"), "standardFunctions"));

    sf = new HashMap;
    for (S x : standardFunctions) {
      S[] f = x.split("/");
      sf.put(f[1], f[0]);
    }
  }
  
  for (int i = 0; ; i++) {
    Set<S> defd = new HashSet(findFunctions(tok));
    
    int j;
    while ((j = jfind(tok, "should not include function *.")) >= 0) {
      S fname = tok.get(j+8);
      shouldNotIncludeFunction.add(fname);
      clearAllTokens(tok.subList(j, j+12));
    }

    while ((j = jfind(tok, "do not include function *.")) >= 0) {
      S fname = tok.get(j+8);
      doNotIncludeFunction.add(fname);
      clearAllTokens(tok.subList(j, j+12));
    }
    
    // changes tok
    Set<String> invocations = findFunctionInvocations(tok, sf, hardFunctionReferences);
    /*if (invocations.contains("str"))
      print("==STR==" + join(tok) + "==STR==");*/
    if (!cic(definitions, "leanMode"))
      invocations.addAll(functionsToAlwaysInclude);

    //print("Functions invoked: " + structure(invocations));
    List<String> needed = diff(invocations, defd);
    for (S f : doNotIncludeFunction) needed.remove(f);
    if (needed.isEmpty())
      break;
    print("Adding functions: " + join(" " , needed));
    
    HashSet neededSet = new HashSet(needed);
    Collection<S> bad = setIntersection(neededSet, shouldNotIncludeFunction);
    if (nempty(bad)) {
      S msg = "INCLUDING BAD FUNCTIONS: " + sfu(bad);
      print(msg);
      print(join(tok));
      fail(msg);
    }
      
    new L<S> added;
    new L<Future<S>> parts;
    new L<S> preload;
    
    for (S x : needed)
      if (sf.containsKey(x))
        preload.add(sf.get(x));
    cachePreload(preload);
    
    for (String x : cloneList(needed)) {
      if (defd.contains(x)) continue;
      
      String id = sf.get(x);
      if (id == null) {
        print("Standard function " + x + " not found.");
        needed.remove(x);
        continue;
      }
      //print("Adding function: " + x + " (" + id + ")");
       
      S function = cacheGet(id);
      //if (("\n" + function).contains("\n!")) print("Warning: " + id + " contains translators.");
      
      if (cacheStdFunctions) {
        long _id = psI(id);
        CachedInclude ci = cachedIncludes.get(_id);
        if (ci == null) {
          cachedIncludes.put(_id, ci = new CachedInclude);
          //println("Caching include " + _id + ", l=" + l(cachedIncludes));
        }
        function += "\n";
        fS _function = function;
        if (neq(ci.javax, function)) {
          print("Compiling function: " + x);
          ci.javax = function;
          ci.java = executor.submit(new Callable<S>() {
            public S call() {
              ret join(localStuff1(jtok(_function)));
            }
          });
          ci.realJava = null;
        }
        parts.add(ci.javaFuture());
      } else
        parts.add(nowFuture(function + "\n"));
        
      added.add(x);
      Collection<S> newFunctions = new HashSet(findFunctionDefs(javaTok(function)));
      defd.addAll(newFunctions);
      for (S f : newFunctions)
        if (!addedFunctions.add(f)) {
          printSources(tok);
          fail("Trying to add function " + f + " again - main class syntax broken!");
        }
    }
    
    S text = join(getAllFutures(parts));
    if (cacheStdFunctions)
      tok = includeInMainLoaded_stdReTok(tok, text);
    else
      tok = includeInMainLoaded(tok, text);
      
    // defd = new HashSet(findFunctions(tok));
    //print("Functions added: " + joinWithSpace(added));
    
    for (String x : needed)
      if (!defd.contains(x)) {
        print(join(tok));
        fail("Function not defined properly: " + x);
      }
    //print("Iteration " + (i+2));
    
    tok_definitions(tok);
    tok_ifndef(tok);
    tok_ifdef(tok);
    
    if (i >= 1000) fail("Too many iterations");
  }
  
  ret tok;
}

static L<S> findFunctions(L<S> tok) {
  //ret findFunctionDefinitions(join(findMainClass(tok)));
  ret findFunctionDefs(findMainClass(tok));
}

static S cacheGet(S snippetID) {
  snippetID = formatSnippetID(snippetID);
  S text = snippetCache.get(snippetID);
  if (text == null) {
    snippetCache.put(snippetID, text = loadSnippet(snippetID));
    if (hasUnclosedStringLiterals(text))
      fail("Unclosed string literals in " + snippetID);
  }
  ret text;
}

static void cachePreload(Collection<S> ids) {
  new L<S> needed;
  for (S id : ids)
    if (!snippetCache.containsKey(formatSnippetID(id)))
      needed.add(formatSnippetID(id));
  if (l(needed) > 1) {
    L<S> texts = loadSnippets(needed);
    for (int i = 0; i < l(needed); i++)
      if (texts.get(i) != null)
        snippetCache.put(needed.get(i), texts.get(i));
  }
}

static L<S> jtok(L<S> tok) {
  ret jtok(join(tok));
}

static L<S> jtok(S s) {
  L<S> l = javaTok(s);
  ret useIndexedList ? new IndexedList2(l) : l;
}

static HashSet<S> haveClasses_actual(L<S> tok) {
  new HashSet<S> have;
  for (L<S> c : innerClassesOfMain(tok))
    have.add(getClassDeclarationName(c));
  ret have;
}

static HashSet<S> haveClasses_addImported(HashSet<S> have) {
  have.addAll(tok_importedClassNames(tok));
  have.addAll(usualJavaClassNames()); // for S => S.class
  ret have;
}

// works on Java level (no "sclass" etc)
// returns list of classes we have (useful for other processing)
static Set<S> addStandardClasses_v2(L<S> tok, L<S> definitions) {
  for (int safety = 0; safety < 10; safety++) {
    HashSet<S> have = haveClasses_actual(tok);

    int j;
    while ((j = jfind(tok, "should not include class *.")) >= 0) {
      S cname = tok.get(j+8);
      shouldNotIncludeClass.add(cname);
      clearAllTokens(tok.subList(j, j+12));
    }
    
    new SS need;
    Set<S> idx = tokenIndexWithoutIfclass(tok);
    for (int i = 0; i+1 < l(definitions); i += 2) {
      S className = definitions.get(i);
      if (idx.contains(className) && !have.contains(className))
        need.put(className, definitions.get(i+1));
    }
    if (hasDef("SymbolAsString")) {
      print("Have SymbolAsString.");
      if (need.containsKey("Symbol")) {
        print("Have Symbol.");
        need.remove("Symbol");
      }
    } else
      print("No SymbolAsString.");
      
    if (empty(need)) ret have;
  
    for (S className : keys(need))
      if (shouldNotIncludeClass.contains(className)) {
        S msg = "INCLUDING BAD CLASS: " + className;
        print(msg);
        print(join(tok));
        fail(msg);
      }
      
    cachePreload(values(need));
    
    new StringBuilder buf;
    print("Adding classes: " + joinWithSpace(keys(need)));
    for (S className : keys(need)) {
      if (have.contains(className)) continue; // intermittent add
      S snippetID = need.get(className);
      //print("Adding class " + className + " / " + snippetID);
      snippetID = fsI(snippetID);
      S text = cacheGet(snippetID);
      
      assertTrue(cacheStdClasses);
      long _id = psI(snippetID);
      CachedInclude ci = cachedIncludes.get(_id);
      if (ci == null) cachedIncludes.put(_id, ci = new CachedInclude);
      if (neq(ci.javax, text)) {
        print("Compiling class: " + className);
        ci.javax = text;
        ci.java = null;
        ci.realJava = join(localStuff1(jtok(text)));
      }
      buf.append(ci.java());

      L<S> ct = javaTok(ci.java());
      for (L<S> c : allClasses(ct)) {
        S name = getClassDeclarationName(c);
        have.add(name);
      }
      if (!have.contains(className))
        fail("Wrongly defined class: " + className + " / " + snippetID);
      if (!addedClasses.add(className)) {
        printSources(tok);
        fail("Trying to add class " + className + " again - main class syntax broken!");
      }
    } // for className
    
    tok = includeInMainLoaded_stdReTok(tok, str(buf));
  }
  fail("safety 10");
}

static Set<S> expandableClassNames = lithashset("BigInteger");

// no reTok - leaves tok dirty
// magically append ".class" to class name references
static bool expandClassReferences_lazy(L<S> tok, Set<S> classNames) {
  bool change = false;
  for (int i = 3; i+2 < l(tok); i += 2) {
    S t = tok.get(i);
    
    // skip implements/extends/throws lists
    if (eqOneOf(t, "implements", "extends", "throws")) {
      i = tok_endOfImplementsList(tok, i);
      continue;
    }
    
    if (classNames.contains(t) || expandableClassNames.contains(t)) {
      S s = tok.get(i-2); t = tok.get(i+2);
      if (eqOneOf(s, "instanceof", "new", ".", "<", ">", "/", "nu")) continue;
      if (isIdentifier(s) || isInteger(s)) continue;
      if (eq(s, ",") && isIdentifier(get(tok, i-4)) && eqGet(tok, i-6, "<")) continue; // e.g. T3<S, S, S>
      if (eq(s, ",") && eqOneOf(_get(tok, i-6), "implements", "throws")) continue;
      // TODO: longer lists
      
      // check for cast
      if (eq(s, "(") && eq(t, ")") && i >= 5) {
        if (!eqOneOf(get(tok, i+4), "{", ";")) {
          S x = tok.get(i-4);
          if (!isIdentifier(x)) continue;
          if (eqOneOf(x, "ret", "return")) continue;
        }
      }
      if (eqOneOf(t, ",", ")", ";", ":")) {
        tok.set(i, tok.get(i) + ".class");
        change = true;
      }
    }
  }
  ret change;
}

static void expandClassReferences(L<S> tok, Set<S> classNames) {
  if (expandClassReferences_lazy(tok, classNames))
    reTok(tok);
}

// "<id>/<ClassName>" => "((ClassName) <id>)"
static void slashCasts(L<S> tok, final Set<S> classNames) {
  /*jreplace(tok, "<id>/<id>", "(($3) $1)", tokcondition {
    ret classNames.contains(tok.get(i+5));
  });*/
  int n = l(tok)-4;
  for (int i = 1; i < n; i += 2)
    if (tok.get(i+2).equals("/") && isIdentifier(tok.get(i))
      && classNames.contains(tok.get(i+4)))
      replaceTokens_reTok(tok, i, i+5, "((" + tok.get(i+4) + ") " + tok.get(i) + ")");
}

// experimental - "<ClassName>(...)" => "new <ClassName>(...)"
// doesn't work at beginning of statement as we can't easily
// distinguish it from a constructor declaration.
static void newWithoutNew(L<S> tok, final Set<S> classNames) {
  jreplace(tok, "<id>(", "new $1(", tokcondition {
    if (!classNames.contains(tok.get(i+1))) false;
    bool ok = neqOneOf(_get(tok, i-1), "new", ";", "}", "{", "public", "protected", "private", ".");
    //print("newWithoutNew: checking " + struct(subList(tok, i-1, i+2)) + " => " + ok);
    ret ok;
  });
}

// +var => "var", +var
static void expandVarCopies(L<S> tok) {
  bool change = false;
  for (int i = 3; i+2 < l(tok); i += 2) {
    if (!eq(tok.get(i), "+")) continue;
    if (!eqOneOf(tok.get(i-2), "(", ",", "{")) continue;
    S s = tok.get(i+2);
    if (!isIdentifier(s)) continue;
    tok.set(i, quote(s) + ", ");
    change = true;
  }
  if (change) reTok(tok);
}

static bool processConceptsDot(L<S> tok) {
  bool anyChange = false, change;
  do {
    change = false;
    for (int i : jfindAll(tok, "concepts."))
      if (contains(get(tok, i+3), "\n")) {
        replaceTokens(tok, i, i+3, "!" + "include once #1004863 // Dynamic Concepts");
        reTok(tok, i, i+3);
        change = anyChange = true;
        break;
      }
  } while (change);
  ret anyChange;
}

static void addFieldOrder(L<S> tok, int i) {
  int idx = findCodeTokens(tok, i, false, "{");
  if (idx < 0) ret;
  int j = findEndOfBracketPart(tok, idx);
  L<S> vars = allVarNames(subList(tok, idx+1, j-1));
  print("addFieldOrder " + struct(vars));
  if (!vars.contains("_fieldOrder")
    && !isSortedList(vars)) {
    print("Adding field order");
    tok.set(idx+2, "static String _fieldOrder = " + quote(join(" ", vars)) + ";\n  " + tok.get(idx+2));
    // reTok has to be done by caller
  }
}

static void caseAsVariableName(L<S> tok) {
  if (!tok.contains("case")) ret;
  for (int i = 1; i+2 < l(tok); i += 2) {
    S t = tok.get(i+2);
    if (tok.get(i).equals("case")
      && !(t.startsWith("'") || isInteger(t) || isIdentifier(t)))
      tok.set(i, "_case");
  }
}

static void continueAsFunctionName(L<S> tok) {
  jreplace(tok, "continue(", "_continue(");
}

// f bla => "bla" - and "please include function bla."
static void functionReferences(L<S> tok) {
  int i;
  
  jreplace_dyn(tok, "f-thread <id>", func(L<S> tok, int cIdx) {
    "dynamicCallableMC_thread(" + quote(tok.get(cIdx+6)) + ")"
  });
  
  S keyword = "f";
  while ((i = jfind(tok, keyword + " <id>", tokcondition {
    ret !eq(tok.get(i+3), "instanceof");
  })) >= 0) {
    S f = tok.get(i+2);
    clearTokens(tok, i, i+2);
    tok.set(i+2, quote(f));
    reTok(tok, i, i+2);
    tok.set(l(tok)-1, last(tok) + "\nplease include function " + f + ".");
    reTok(tok, l(tok)-1, l(tok));
  }
  
  // r fname => r { fname() }
  while ((i = jfind(tok, "r <id>", tokcondition {
    ret !eq(tok.get(i+3), "instanceof");
  })) >= 0) {
    S f = tok.get(i+2);
    replaceTokens(tok, i, i+3, "r { " + f + "(); }");
    reTok(tok, i, i+3);
  }
}

// # 123 => "#123"
static void directSnippetRefs(L<S> tok) {
  int i;
  while ((i = jfind(tok, "#<int>", tokcondition {
    ret !eqOneOf(_get(tok, i-1), "include", "once");
  })) >= 0) {
    S id = tok.get(i+2);
    clearTokens(tok, i+1, i+3);
    tok.set(i, quote("#" + id));
    reTok(tok, i, i+3);
  }
}

static void quicknu(L<S> tok) {
  jreplace(tok, "nu <id>(", "nu($2.class, ");
  jreplace(tok, "nu <id>", "new $2");
}

// fill variable innerClasses_list
static void innerClassesVar(L<S> tok, Set<S> have) {
  if (!tok.contains("myInnerClasses_list")) ret;
  
  int i = jfind(tok, ">myInnerClasses_list;");
  if (i < 0) ret;
  tok.set(i+4, "=litlist(\n" + joinQuoted(", ", have) + ");");
  reTok(tok, i+4, i+5);
}

// process ifclass x ... endif blocks
static void tok_ifclass(L<S> tok, Set<S> have) {
  if (!tok.contains("ifclass")) ret;
  
  int i;
  while ((i = rjfind(tok, "ifclass <id>")) >= 0) {
    int j = jfind(tok, i+4, "endif");
    if (j < 0) j = l(tok)-1;
    S name = tok.get(i+2);
    bool has = have.contains(name);
    //print("ifclass " + name + " => " + has);
    clearTokens(tok, i, i+3);
    clearTokens(tok, j, j+1);
    if (!has) clearTokens(tok, i+3, j);
    reTok(tok, i, j+1);
  }
}

// set flag *.
static void tok_definitions(L<S> tok) {
  int i;
  while ((i = jfind(tok, "set flag <id>.")) >= 0) {
    S fname = tok.get(i+4);
    print("Setting flag " + fname);
    definitions.add(fname);
    clearAllTokens(tok.subList(i, i+8));
  }
  
  while ((i = jfind(tok, "unset flag <id>.")) >= 0) {
    S fname = tok.get(i+4);
    print("Unsetting flag " + fname);
    definitions.remove(fname);
    clearAllTokens(tok.subList(i, i+8));
  }
}

static void tok_findRewrites(L<S> tok) {
  int i;
  while ((i = jfind(tok, "rewrite <id> =")) >= 0) {
    S token = tok.get(i+2);
    int repStart = i+6;
    int repEnd = smartIndexOf(tok, repStart, ".");
    S replacement = joinSubList(tok, repStart, repEnd-1);
    clearTokens(tok, i, repEnd+1);
    rewrites.put(token, replacement);
    print("Have rewrite: " + token + " => " + replacement);
  }
}

static void tok_processRewrites(L<S> tok) {
  for (S token : keys(rewrites))
    jreplace(tok, token, rewrites.get(token));
}

// extend *. (set base class of main class)
static void tok_extend(L<S> tok) {
  int i;
  while ((i = jfind(tok, "extend <id>.")) >= 0) {
    mainBaseClass = tok.get(i+2);
    clearAllTokens(tok, i, i+7);
  }
}

// process ifndef x ... endifndef blocks
static void tok_ifndef(L<S> tok) {
  if (!tok.contains("ifndef")) ret;
  
  int i;
  while ((i = rjfind(tok, "ifndef <id>")) >= 0) {
    int j = jfind(tok, i+4, "endifndef");
    if (j < 0) j = l(tok)-1;
    S fname = tok.get(i+2);
    bool has = !definitions.contains(fname);
    //print("ifndef " + fname + " => " + has);
    clearTokens(tok, i, i+3);
    clearTokens(tok, j, j+1);
    if (!has) clearTokens(tok, i+3, j);
    reTok(tok, i, j+1);
  }
}

// process ifdef x ... endifdef blocks
static void tok_ifdef(L<S> tok) {
  if (!tok.contains("ifdef")) ret;
  
  int i;
  while ((i = rjfind(tok, "ifdef <id>")) >= 0) {
    int j = jfind(tok, i+4, "endifdef");
    if (j < 0) j = l(tok)-1;
    S fname = tok.get(i+2);
    bool has = definitions.contains(fname);
    //print("ifdef " + fname + " => " + has);
    clearTokens(tok, i, i+3);
    clearTokens(tok, j, j+1);
    if (!has) clearTokens(tok, i+3, j);
    reTok(tok, i, j+1);
  }
}

svoid conceptDeclarations(L<S> tok) {
  for (S kw : ll("concept", "sconcept")) {
    O cond = tokcondition {
      addFieldOrder(tok, i+1);
      ret true;
    };
    bool re = false;
    if (jreplace(tok, kw + " <id> {", "static class $2 extends Concept {", cond)) re = true;
    if (jreplace(tok, kw + " <id> implements", "static class $2 extends Concept implements", cond)) re = true;
    if (jreplace(tok, kw + " <id>", "static class $2", cond)) re = true;
    if (re) reTok(tok);
  }
}

svoid shortenedSubconcepts(L<S> tok) {
  jreplace(tok, "<id> > <id> {", "concept $3 extends $1 {", tokcondition {
    bool b = (i == 0 || tok.get(i).contains("\n")) || eq(_get(tok, i-1), "abstract"); // only at beginning of line or after "abstract"
    //print("subconcept " + b + ": " + structure(subList(tok, i-1, i+5)));
    ret b;
  });
}

// -slightly experimental
// -do calculation in another thread, then return to AWT thread
// -must be placed in a block
// -transforms rest of block 
svoid unswing(L<S> tok) {
  int i;
  while ((i = jfind(tok, "unswing {")) >= 0) {
    int idx = i+2;
    int closingBracket = findEndOfBracketPart(tok, idx)-1;
    int endOfOuterBlock = findEndOfBracketPart(tok, closingBracket)-1;
    tok.set(i, "thread");
    tok.set(closingBracket, " awt {");
    tok.set(endOfOuterBlock, "}}}");
    reTok(tok, closingBracket-1, endOfOuterBlock+1);
  }
}

// -Syntax: lock theLock;
// -lock a lock, unlock at end of current block with finally
svoid lockBlocks(L<S> tok) {
  int i;
  while ((i = jfind(tok, "lock <id>", tokcondition { ret neq(tok.get(i+3), "instanceof"); })) >= 0) {
    int semicolon = findEndOfStatement(tok, i)-1;
    S var = makeVar();
    int endOfOuterBlock = findEndOfBracketPart(tok, semicolon)-1;
    replaceTokens(tok, i, semicolon+1,
      "Lock " + var + " = " + joinSubList(tok, i+2, semicolon-1) + "; lock(" + var + "); try {");
    tok.set(endOfOuterBlock, "} finally { unlock(" + var + "); } }");
    reTok(tok, i, endOfOuterBlock+1);
  }
}

// -Syntax: temp Bla bla = bla();
// -expands to try(Bla bla = bla()) { ... } with rest of block inside
svoid tempBlocks(L<S> tok) {
  int i;
  while ((i = jfind(tok, "temp <id>")) >= 0) {
    int semicolon = findEndOfStatement(tok, i)-1;
    int endOfOuterBlock = findEndOfBracketPart(tok, semicolon)-1;
    L<S> sub = subList(tok, i-1, semicolon);
    int eq = sub.indexOf("=");
    S var;
    if (eq >= 0)
      var = sub.get(eq-2);
    else {
      // no var name, e.g. temp newThoughtSpace();
      var = makeVar();
      tok.set(i+2, "AutoCloseable " + var + " = " + tok.get(i+2));
    }
      
    //tok.set(i, "try (");
    //tok.set(semicolon, ") {";
    //tok.set(endOfOuterBlock, "}}");
    
    tok.set(i, "");
    tok.set(semicolon, "; try {";
    tok.set(endOfOuterBlock, "} finally { _close(" + var + "); }}");
    
    reTok(tok, i, endOfOuterBlock+1);
  }
}

svoid cleanMeUp {
  for (CachedInclude ci : values(cachedIncludes))
    ci.clean();
  vmKeepWithProgramMD5_save('cachedIncludes);
}

svoid printSources(L<S> tok) {
  print("----");
  print(join(tok));
  print("----");
}

svoid tok_typeAA(L<S> tok) {
  int n = l(tok)-6;
  bool change = false;
  for (int i = 1; i < l(tok); i += 2) {
    if (!(eq(get(tok, i+2), "<") && contains(pairClasses, tok.get(i)))) continue;
    if (tok_isSingleTypeArg(tok, i+2)) {
      int j = findEndOfTypeArgs(tok, i+2)-1;
      S type = joinSubList(tok, i+4, j);
      replaceTokens_reTok(tok, i+4, j, type + ", " + type);
      change = true;
    }
  }
  //if (change) reTok(tok);
  
  /*  
  O cond = tokcondition { pairClasses.contains(tok.get(i+1)) };
  jreplace(tok, "<id> < <id> >", "$1<$3, $3>", cond);
  jreplace(tok, "<id> < <id><<id>> >", "$1<$3<$5>, $3<$5>>", cond);
  jreplace(tok, "<id> < <id><<id>,<id>> >", "$1<$3<$5,$7>, $3<$5,$7>>", cond);
  */
}

svoid tok_quickInstanceOf(L<S> tok, final Set<S> haveClasses) {
  // "x << X" or "x >> X" => "x instanceof X"
  for (S op : ll("<<", ">>"))
    jreplace(tok, "<id> " + op + " <id>", "$1 instanceof $4", tokcondition {
      ret haveClasses.contains(tok.get(i+7))
        && !eqOneOf(tok.get(i-1), "<", "extends", "implements");
    });
}

sbool hasDef(S s) {
  ret definitions.contains(s);
}

svoid expandTriple(L<S> tok) {
  jreplace(tok, "T3< <id> >", "T3<$3, $3, $3>");
  jreplace(tok, "T3< <id><<id>> >", "T3<$3<$5>, $3<$5>, $3<$5>>");
}

svoid tok_shortFinals(L<S> tok) {
  jreplace(tok, "fS", "final S");
  jreplace(tok, "fO", "final O");
  jreplace(tok, "fL", "final L");
  jreplace(tok, "fMap", "final Map");
  jreplace(tok, "fRunnable", "final Runnable");
  jreplace(tok, "f int", "final int");
}