/* e.g.

  overlay <- ScreenOverlay
  bounds <- rightScreenBounds
  overlay bounds bounds
  overlay show

Can separate commands with ";" also.
For how to define functions in a script see #1033988.

Note: LeftArrowScriptAutoCompleter uses a lot of this class's internals

*/

// TODO: "then" is used twice which is probably not good
// (for if-then and for chaining calls)

sclass GazelleV_LeftArrowScriptParser > SimpleLeftToRightParser {
  // most important "experimental" setting (will set this to true)
  settable bool magicSwitch = true;
  
  // New object scopes (class members without "this")
  // - MAY potentially break old scripts (although probably not)
  settable bool objectScopesEnabled = true;

  replace Toolbox with GazelleV_LeftArrowScript.
  delegate Script, Evaluable, FunctionDef, FixedVarBase,
    LambdaMethodOnArgument, CallOnTarget to Toolbox.
  replace const with _const.
  
  ClassNameResolver classNameResolver;
  new L functionContainers;
  
  settable ILASClassLoader lasClassLoader;
  settable S classDefPrefix;
  
  // will be noted in source references
  settable O sourceInfo;
  
  settable bool optimize = true;
  settable bool useFixedVarContexts = false; // doesn't completely work yet
  
  Scope scope;
  new LinkedHashMap<S, LASValueDescriptor> knownVars;
  
  //new L<FixedVarBase> varAccessesToFix;
  
  Set<S> closerTokens = litset(";", "}", ")");
  BuildingScript currentReturnableScript;
  BuildingScript currentLoop;
  bool inParens;
  int idCounter;
  
  new Map<S, LASClassDef> classDefs;
  
  gettable Map<S, FunctionDef> functionDefs = new Map;
  
  Set<S> flags = ciSet();
  
  // events for auto-completer
  // 1. record which vars are known at current parse location
  event knownVarsSnapshot(Map<S, LASValueDescriptor> knownVars);
  
  // 2. indicates we just parsed an expression of type type
  //    (useful for listing methods+fields for auto-completion)
  event typeHook(LASValueDescriptor type);
  
  replace print with if (GazelleV_LeftArrowScriptParser.this.scaffoldingEnabled()) print.
  replace printVars with if (GazelleV_LeftArrowScriptParser.this.scaffoldingEnabled()) printVars.
  
  srecord EvaluableWrapper(Evaluable expr) {}

  class BuildingScript {
    int id = ++idCounter;
    settable bool returnable;
    settable bool isLoopBody;
    BuildingScript returnableParent, loopParent;
    //settable LASScope scope;
    
    new Script script;
    new L<Evaluable> steps;
    Map<S, FunctionDef> functionDefs = new Map;
    
    *(bool *returnable) { this(); }
    *(bool *returnable, bool *isLoopBody) { this(); }
    *() { /*scope = currentScope();*/ }
    
    void add(Evaluable step) { if (step != null) steps.add(step); }

    Evaluable get() {
      //script.scope = scope;
      
      // if the last command is a return from THIS script,
      // convert into a simple expression
      
      var lastStep = last(steps);
      if (lastStep cast Toolbox.ReturnFromScript)
        if (lastStep.script == script)
          replaceLast(steps, lastStep.value);
        
      // if the script is only step, is not returnable
      // and defines no functions, replace it with its only step
      
      if (!returnable && l(steps) == 1 && empty(functionDefs))
        ret first(steps);
        
      // make and return actual script
      
      if (nempty(functionDefs)) script.functionDefs = functionDefs;
      script.steps = toTypedArray(Evaluable.class, steps);
      ret script;
    }
    
    S toStringLong() { ret pnlToLines(steps); }
    toString {
      ret formatRecordVars BuildingScript(+id, +returnable , +returnableParent, +script);
    }
  } // end of BuildingScript
  
  // methods begin here
  
  {
    tokenize = text -> {
      LS tok = tokenize_base(text);
      tok = preprocess(tok);
      print("=== PREPROCESSED\n" + indentx(join(tok)) + "\n===");
      ret tok;
    };
  }
  
  LS preprocess(LS tok) {
    tok = tokenIndexedList3(tok);
    tok_ifdef(tok, l1 getFlag);
    tok_pcall_script(tok);
    tok_script_settable(tok);
    jreplace(tok, "LS", "L<S>");
    jreplace(tok, ", +<id>", ", $3 $3");
    ret cloneList(tok);
  }
  
  Script parse(S text) {
    setText(text);
    init();
    ret parse();
  }
  
  Script parse() {
    Script script = parseReturnableScript();
    /*for (varAccess : varAccessesToFix)
      varAccess.resolve();*/
    if (optimize) script = script.optimizeScript();
    ret script;
  }
  
  Script parseReturnableScript() {
    ret (Script) parseScript(new BuildingScript().returnable(true));
  }
  
  // if returnable is true, it's always a Script
  Evaluable parseScript(BuildingScript script) {
    ret linkToSrc(-> {
      script.returnableParent = currentReturnableScript;
      script.loopParent = currentLoop;
      if (script.returnable)
        currentReturnableScript = script;
      if (script.isLoopBody)
        currentLoop = script;
      ret parseBuildingScript(script);
    });
  }
    
  Evaluable parseBuildingScript(BuildingScript script) {
    try {
      parseScript_2(script);
      var builtScript = script!;
      currentReturnableScript = script.returnableParent;
      currentLoop = script.loopParent;
      ret builtScript;
    } catch e {
      print("Parsed so far:\n" + script);
      
      // TODO: could use an exception class [like ScriptError]
      throw rethrowAndAppendToMessage(e,
        squareBracketed(spaceCombine(sourceInfo, lineAndColumn(-1)));
    }
  }
  
  void parseScript_2(BuildingScript script) {
    temp tempRestoreMap(knownVars);
    
    new AssureAdvance assure;
    while (assure!) {
      // for auto-completer
      knownVarsSnapshot(knownVars);
      
      print("parseScript_2: Next token is " + quote(token()));
      
      if (is(";")) continue with next();
      if (isOneOf("}", ")")) break;
      
      Evaluable instruction = linkToSrc(-> parseInstruction(script));
      if (instruction != null)
        script.add(instruction);
    }
    
    print("parseScript_2 done");
      
    knownVarsSnapshot(knownVars);
  }
  
  Evaluable parseParamDecl(BuildingScript script) {
    S var = assertIdentifier(tpp());
    new LASValueDescriptor valueDescriptor;
    if (is(":")) {
      var type = parseColonType();
      valueDescriptor = LASValueDescriptor.nonExactCanBeNull(type);
    }
    knownVars.put(var, valueDescriptor);
    script.script.params = putOrCreateLinkedHashMap(script.script.params, var, valueDescriptor);
    null;
  }

  Evaluable parseInstruction(BuildingScript script) {
    /*if (is("var") && isIdentifier(token(1))) {
      consume();
      ret parseAssignment();
    }TODO*/
      
    // Script parameter (old)
    if (consumeOpt("param"))
      ret parseParamDecl(script);

    if (is("throw")) {
      consume();
      ret new Toolbox.Throw(parseExpr());
    }
    
    /* Hmm. Too hard. Need to use jreplace.  
    if (is("pcall")) {
      consume();
      Evaluable body = parseCurlyBlock();
      
      S var = "_pcallError";
      temp tempAddKnownVars(var);
      Evaluable catchBlock = parseCurlyBlock("{ pcallFail " + var + " });
      ret new Toolbox.TryCatch(body, var, catchBlock);
    }*/
      
    if (isOneOf("return", "ret")) {
      consume();
      Evaluable expr;
      if (atCmdEnd())
        expr = const(null);
      else
        expr = parseAssignmentOrExpr();
      ret new Toolbox.ReturnFromScript(currentReturnableScript.script, expr);
    }
    
    if (consumeOpt("continue")) {
      assertCmdEnd();
      if (currentLoop == null)
        fail("continue outside of loop");
      ret new Toolbox.Continue(currentLoop.script);
    }
    
    /* TODO
    if (consumeOpt("break")) {
      assertCmdEnd();
      if (currentLoop == null)
        fail("break outside of loop");
      ret new Toolbox.Break(currentLoop.script);
    }*/
    
    if (is("temp")) {
      consume();
      Evaluable tempExpr = parseAssignmentOrExpr();
      print(+tempExpr);
      Evaluable body = parseScript(new BuildingScript);
      print(+body);
      ret new Toolbox.TempBlock(tempExpr, body);
    }
    
    if (is("will") && is(1, "return")) {
      consume(2);
      Evaluable exp = parseAssignmentOrExpr();
      Evaluable body = parseScript(new BuildingScript);
      ret new Toolbox.WillReturn(exp, body);
    }
    
    if (is("var") && isIdentifier(token(1))) {
      consume();
      S varName = consume();
      var type = parseColonTypeOpt();

      if (consumeLeftArrowOpt()) {
        print("Found assignment");
        Evaluable rhs = parseExpr();
        assertNotNull("Expression expected", rhs);
        
        knownVars.put(varName, type == null
          ? or(rhs.returnType(), new LASValueDescriptor)
          : LASValueDescriptor.nonExactCanBeNull(type));
        ret new Toolbox.VarDeclaration(varName, typeToClass(type), rhs);
      }
    }
    
    ret parseAssignmentOrExpr();
  }
  
  Evaluable parseAssignmentOrExpr() {
    try object parseAssignmentOpt();
    ret parseExpr();
  }
  
  // Parse assignment starting with a simple identifier (e.g. "i <- 5")
  Evaluable parseAssignmentOpt() {
    S t = token();
    if (isIdentifier(t)) {
      Type type = null;
      var ptr = ptr(); // save parsing position
      
      // optional type
      if (is(1, ":")) {
        next();
        type = parseColonType();
        unconsume();
      }
        
      if (is(1, "<") && is(2, "-")) {
        print("Found assignment");
        next(3);
        Evaluable rhs = parseExpr();
        assertNotNull("Expression expected", rhs);
        
        bool newVar = !knownVars.containsKey(t);
        printVars(+newVar, +t, +knownVars);
        
        if (newVar && scope != null)
          try object scope.completeAssignmentOpt(t, rhs);
        
        knownVars.put(t, type == null
          ? or(rhs.returnType(), new LASValueDescriptor)
          : LASValueDescriptor.nonExactCanBeNull(type));
        ret newVar
          ? new Toolbox.VarDeclaration(t, null, rhs)
          : new Toolbox.Assignment(t, rhs);
      }
      
      // revert parser, it was just a typed expression
      // (TODO: this technically violates the O(n) parsing principle)
      ptr(ptr);
    }
    null;
  }

  Evaluable parseOptionalInnerExpression() {
    printVars parseOptionalInnerExpression(+token());
    if (atCmdEnd() || isOneOf("{", ",", "then")) null;
    ret parseInnerExpr();
  }
  
  Evaluable const(O o) { ret new Toolbox.Const(o); }
  
  Evaluable parseInnerExpr() { ret parseExpr(true); }
  
  scaffolded Evaluable parseExpr(bool inner default false) {
    Evaluable e = linkToSrc(-> inner ? parseExpr_impl(true) : parseFullExpr());
    print("parseExpr done:\n" + Toolbox.indentedScriptStruct(e));
    ret e;
  }
  
  Evaluable parseFullExpr() {
    ret linkToSrc(-> parseExprPlusOptionalComma());
    //ret parseExprPlusOptionalCommaOrThen();
  }
  
  /*Evaluable parseExprPlusOptionalCommaOrThen() {
    Evaluable expr = parseExpr_impl(false);
    
    //printVars("parseExprPlusOptionalCommaOrThen", t := t());
    while true {
      if (is("then"))
        expr = parseThenCall(expr);
      else if (consumeOpt(","))
        expr = parseCall_noCmdEndCheck(expr);
      else
        break;
      //printVars("parseExprPlusOptionalCommaOrThen", t2 := t());
    }
    ret expr;
  }*/
  
  // TODO: How to store the object between the two calls?
  // We need to put it in the VarContext somehow.
  // Or allow passing in a target to CallOnTarget
  CallOnTarget parseThenCall(CallOnTarget expr) {
    consume("then");
    CallOnTarget secondCall = cast parseCall(null);
    ret new Toolbox.Then(expr, secondCall);
  }
  
  Evaluable parseExprPlusOptionalComma() {
    Evaluable expr = parseExpr_impl(false);
    
    //printVars("parseExprPlusOptionalComma", t := t());
    while (consumeOpt(",")) {
      expr = parseCall_noCmdEndCheck(expr);
      //printVars("parseExprPlusOptionalComma", t2 := t());
    }
    ret expr;
  }
  
  Evaluable parseExpr_impl(bool inner) {
    if (atEnd()) null;
    
    S t = token();
    printVars parseExpr(token := t);
    
    if (is(";")) null; // empty command
    
    if (consumeOpt("try")) {
      Evaluable body = parseCurlyBlock();
      
      while (consumeOpt("catch")) {
        S var = consumeIdentifierOpt();
        temp tempAddKnownVars(var);
        Evaluable catchBlock = parseCurlyBlock();
        body = new Toolbox.TryCatch(body, var, catchBlock);
      }
      
      if (consumeOpt("finally")) {
        Evaluable finallyBlock = parseCurlyBlock();
        ret new Toolbox.TryFinally(body, finallyBlock);
      }
      
      ret body;
    }
    
    if (is("def")) {
      var fd = parseFunctionDefinition(currentReturnableScript.functionDefs);
      ret const(fd);
    }
  
    if (is("{"))
      ret parseCurlyBlock();
      
    // int or double literal
    if (is("-") && empty(nextSpace()) && startsWithDigit(token(1))
      || startsWithDigit(t)) {
      var e = parseNumberLiteral();
      ret parseCall(inner, e);
    }

    if (isQuoted(t)) {
      var e = parseStringLiteral();
      ret parseCall(inner, e);
    }
    
    if (startsWith(t, '\'')) {
      consume();
      var e = const(first(unquote(t)));
      ret parseCall(inner, e);
    }
      
    if (isIdentifier(t)) {
      if (consumeOpt("synchronized")) {
        var target = parseExpr();
        var body = parseCurlyBlock();
        ret new Toolbox.Synchronized(target, body);
      }
  
      if (is("while"))
        ret parseWhileLoop();
  
      if (is("do"))
        ret parseDoWhileLoop();
  
      if (is("for"))
        ret parseForEach();
  
      if (is("if"))
        ret parseIfStatement();
        
      if (is("repeat"))
        ret parseRepeatStatement();
        
      if (is("outer")) {
        consume();
        var a = parseAssignmentOpt();
        if (!a instanceof Toolbox.Assignment)
          fail("Assignment expected");
        cast a to Toolbox.Assignment;
        ret new Toolbox.AssignmentToOuterVar(a.var, a.expression);
      }
        
      consume();
      print("Consumed identifier " + t + ", next token: " + token() + ", inner: " + inner);
      var e = parseExprStartingWithIdentifier(t, inner);
      ret inner ? parseCall(inner, e) : e;
    }
    
    // nested expression
    
    if (eq(t, "(")) {
      consume();
      //print("Consumed opening parens (level " + parenLevels + ")");
      Evaluable e = parseExpr_inParens();
      //print("Consuming closing parens for expr: " + e + " (closing paren level " + parenLevels + ")");
      consume(")");
      ret parseCall(inner, e);
    }
    
    if (isOneOf("&", "|") && empty(nextSpace()) && is(1, token())) {
      ret parseBinaryOperator();
    }
      
    fail("Identifier, literal, operator or opening parentheses expected (got: " + quote(t));
  }
  
  Evaluable parseExpr_inParens() {
    bool inParensOld = inParens;
    inParens = true;
    try {
      ret parseExpr();
    } finally {
      inParens = inParensOld;
    }
  }
  
  Evaluable parseNumberLiteral() {
    S t = consumeMultiTokenLiteral();
      
    if (swic(t, "0x")) ret const(parseHexInt(dropFirst(t, 2)));
    if (swic(t, "0b")) ret const(intFromBinary(dropFirst(t, 2)));
    
    if (isInteger(t)) ret const(parseIntOrLong(t));
    if (ewic(t, "f")) ret const(parseFloat(t));
    
    if (endsWith(t, "L")) ret const(parseLong(t));
    
    ret const(parseDouble(t));
  }
  
  Evaluable parseBinaryOperator() {
    bool and = is("&");
    next(2);
    Evaluable a = parseInnerExpr();
    Evaluable b = parseInnerExpr();
    ret and
      ? new Toolbox.BoolAnd(a, b)
      : new Toolbox.BoolOr(a, b);
  }
  
  bool qualifiedNameContinues() {
    ret empty(prevSpace()) && eq(token(), ".") && empty(nextSpace())
      && isIdentifier(token(1));
  }

  // assumes we have consumed the "new"  
  Evaluable parseNewExpr() {
    S className = assertIdentifier(tpp());
    
    parseTypeArguments(null);
    
    // script-defined class?
    
    LASClassDef cd = classDefs.get(className);
    if (cd != null)
      ret new Toolbox.NewObject_LASClass(cd.resolvable(lasClassLoader));
      
    // class in variable?
    
    var type = knownVars.get(className);
    if (type != null)
      ret new Toolbox.NewObject_UnknownClass(new Toolbox.GetVar(className), parseArguments());

    // external class
      
    O o = findExternalObject(className);
    if (o cast Class) {
      Class c = o;
      
      if (c == L.class) c = ArrayList.class;
      else if (c == Map.class) c = HashMap.class;
      else if (c == Set.class) c = HashSet.class;
      
      ret new Toolbox.NewObject(c, parseArguments());
    }
    
    throw new ClassNotFound(className);
  }
  
  // t is last consumed token (the identifier the expression starts with)
  // TODO: do parseCall in only one place!
  Evaluable parseExprStartingWithIdentifier(S t, bool inner) {
    if (eq(t, "true")) ret const(true);
    if (eq(t, "false")) ret const(false);
    if (eq(t, "null")) ret const(null);
    
    if (eq(t, "new"))
      ret parseNewExpr();
      
    if (eq(t, "class")) {
      unconsume();
      ret new Toolbox.ClassDef(parseClassDef().resolvable(lasClassLoader));
    }
    
    if (eq(t, "list") && is("{")) {
      Script block = parseReturnableCurlyBlock();
      Evaluable e = new Toolbox.ListFromScript(block);
      ret parseCall(inner, e);
    }
    
    // Search in locally known variables

    var type = knownVars.get(t);
    if (type != null) {
      var e = new Toolbox.GetVar(t).returnType(type);
      print("Found var acccess: " + e + ", " + (!inner ? "Checking for call" : "Returning expression"));
      ret parseCall(inner, e);
    }
    
    // Search in scope
    if (scope != null) {
      try object scope.parseExprStartingWithIdentifierOpt(t, inner);
    }
    
    // script-defined class?
    
    LASClassDef cd = classDefs.get(t);
    if (cd != null)
      ret new Toolbox.ClassDef(cd.resolvable(lasClassLoader));

    if (!inner) {
      var fdef = lookupFunction(t);
      if (fdef != null) {
        if (is("~"))
          ret parseTildeCall(new Toolbox.CallFunction(fdef, new Evaluable[0]));
          
        ret new Toolbox.CallFunction(fdef, parseArguments());
      }
    }
    
    if (eq(t, "_context"))
      ret new Toolbox.GetVarContext;
      
    O o = findExternalObject(t);
    
    var start = ptr().minus(2);
    
    if (o == null) {
      //unconsume();
      throw new UnknownObject(t);
    } else if (o cast EvaluableWrapper) {
      ret parseCall(inner, o.expr);
    } else if (inner)
      ret linkToSrc(start, const(o));
    else if (o cast Class) {
      ret parseExprStartingWithClass(start, o);
    } else if (o cast MethodOnObject) {
      // it's a function from a functionContainer
      
      if (inner) fail("Can't call methods in arguments"); // Does this ever happen?!
      
      ret new Toolbox.CallMethod(const(o.object), o.method, parseArguments());
    } else
      ret parseCall(const(o));
  }
  
  Evaluable parseExprStartingWithClass(TokPtr start, Class c) {
    printVars("parseExprStartingWithClass", +c);
    
    if (atCmdEnd())
      ret linkToSrc(start, const(c));
      
    // new without new
    if (is("("))
      ret new Toolbox.NewObject(c, parseArguments());
      
    /* old object creation syntax (e.g. Pair new a b)
    if (is("new")) {
      next();
      ret new Toolbox.NewObject(c, parseArguments());
    } else*/
    
    // check if it's a lambda definition first (with ->)
    try object parseLambdaOpt(c);
    
    if (isIdentifier()) {
      S name = tpp();
      // We have now parsed a class (c) plus an as yet unknown identifier (name) right next to it
      
      // look for a static method call
      if (hasStaticMethodNamed(c, name))
        ret new Toolbox.CallMethod(const(c), name, parseArguments());

      // if the class is an interface, it's a lambda method reference or a lambda
      if (isInterface(c))
        ret parseLambdaMethodRef(c, name);
      
      // look for field second
      
      var field = getField(c, name);
      if (field == null)
        field = findFieldInInterfaces(c, name);
        
      if (field != null) {
        if (!isStaticField(field))
          fail(field + " is not a static field");
        
        // check for field assignment
        if (consumeLeftArrowOpt()) {
          Evaluable rhs = parseExpr();
          ret new Toolbox.SetStaticField(field, rhs);
        }
      
        assertCmdEnd();
        
        ret new Toolbox.GetStaticField(field);
      }
      
      fail(name + " not found in " + c + " (looked for method or field)");
    } else
      fail("Method name expected: " + token());
  }
  
  // parse lambda, e.g. IF1 a -> plus a a
  // returns null if it's not a lambda
  // assumes that c was just parsed
  Evaluable parseLambdaOpt(Class c) {
    // must be an interface
    if (!isInterface(c)) null;
    
    // speculatively parse optional type args first
    var ptr = ptr();
    var parameterized = parseTypeArgumentsOpt(c);
    
    // e.g. IF0 { 5 }
    if (is("{"))
      ret finishLambdaDef(c, null);
      
    // e.g. IF1 _ size
    if (consumeOpt("_")) {
      S methodName = consumeIdentifier();
      ret new LambdaMethodOnArgument(c, methodName, parseArguments());
    }
    
    new LinkedHashMap<S, LASValueDescriptor> args;
    
    while (isIdentifier()) {
      S name = consume();
      args.put(name, typeToValueDescriptor(parseColonTypeOpt()));
    }
    
    // Check for lambda arrow
    if (!(is("-") && is(1, ">"))) {
      ptr(ptr); // revert pointer
      null; // not a lambda
    }
    
    skip(2); // skip the arrow
    ret finishLambdaDef(c, args);
  }
  
  Evaluable finishLambdaDef(Class c, LinkedHashMap<S, LASValueDescriptor> args) {
    temp tempAddKnownVars(args);
    knownVarsSnapshot(knownVars);

    Evaluable body;
    
    // if curly brackets: parse lambda body as returnable script
    // TODO: avoid script overhead if there are no return calls inside
    if (is("{"))
      body = parseReturnableCurlyBlock();
    else
      body = parseExpr();
    
    // return lambda
    var lambda = new Toolbox.LambdaDef(c, toStringArray(keys(args)), body);
    print("finishLambdaDef done:\n" + Toolbox.indentedScriptStruct(lambda));
    ret lambda;
  }
  
  // method ref, e.g. "IF1 l" meaning "l1 l"
  // also, constructor ref, e.g. "IF1 Pair" meaning (x -> new Pair(x))
  //
  // c must be a single method interface
  Evaluable parseLambdaMethodRef(Class c, S name) {
    var fdef = lookupFunction(name);
    if (fdef != null) {
      Evaluable[] curriedArguments = parseArguments();
      ret new Toolbox.CurriedScriptFunctionLambda(c, fdef, curriedArguments);
    }

    O function = findExternalObject(name);
    
    if (function == null) 
      throw new UnknownObject(name);
      
    if (function cast MethodOnObject) {
      // it's a function from a functionContainer
      
      O target = function.object;
      S targetMethod = function.method;
        
      Evaluable[] curriedArguments = parseArguments();
      ret new Toolbox.CurriedMethodLambda(c, target, targetMethod, curriedArguments);
    } else if (function cast Class) {
      Class c2 = function;
      
      // No currying here yet
      assertCmdEnd();
      
      var ctors = constructorsWithNumberOfArguments(c2, 1);
      if (empty(ctors))
        fail("No single argument constructor found in " + c2);
        
      ret new Toolbox.CurriedConstructorLambda(c, toArray Constructor(ctors), null);
    } else
      fail(function + " is not an instantiable class or callable method");

  }
  
  FunctionDef lookupFunction(S name) {
    var script = currentReturnableScript;
    while (script != null) {
      var f = script.functionDefs.get(name);
      printVars("lookupFunction", +script, +name, +f);
      if (f != null) ret f;
      script = script.returnableParent;
    }
    
    // Try parser-wide function defs
    ret functionDefs.get(name);
  }
  
  Evaluable[] parseArguments() {
    new L<Evaluable> l;
    try {
      while (true) {
        print("parseArgumentsAsList: token is " + quote(t()));
        
        // +id
        if (is("+") && empty(nextSpace()) && isIdentifier(token(1))) {
          consume();
          l.add(const(token()));
          
          // just leave identifier unconsumed
          continue;
        }

        if (consumeOpt("<")) {
          Evaluable a = parseFullExpr();
          print("parseArgumentsAsList: token after expr is " + quote(t()));
          if (a == null) fail("expression expected");
          l.add(a);
          break;
        }
        
        Evaluable a = parseOptionalInnerExpression();
        if (a == null) break;
        l.add(a);
      }
      
      ret toArrayOrNull(Evaluable.class, l);
    } on fail {
      print("Arguments parsed so far: " + l);
    }
  }
  
  S consumeMultiTokenLiteral() {
    ret consumeUntilSpaceOr(-> atCmdEnd() || is(","));
  }
  
  bool atCmdEnd() {
    ret
      !inParens && atEndOrLineBreak()
      || closerTokens.contains(token());
  }
  
  void assertCmdEnd() {
    if (!atCmdEnd())
      fail("Expected end of command, token is: " + quote(token()));
  }
  
  // After we just parsed an expression, see if there is something
  // to the right of it.
  // Currently that can be
  // -a method call
  // -a field access
  // -a field assignment (e.g.: myPair a <- 1)
  Evaluable parseCall(bool inner default false, Evaluable target) {
    returnTypeHook(target);
    
    // normally, do a check for end of line
    if (atCmdEnd()) ret target;
    
    if (inner) {
      try object parseTildeCall(target);
      ret target;
    }
    
    ret parseCall_noCmdEndCheck(target);
  }
  
  // For auto-completer
  void returnTypeHook(Evaluable e) {
    if (e != null && e.returnType() != null)
      typeHook(e.returnType());
  }
  
  Evaluable parseCall_noCmdEndCheck(Evaluable target) {
    var start = ptr();
    
    returnTypeHook(target);
    
    // We usually expect an identifier
    
    if (isIdentifier()) {
      S name = tpp(); // the identifier
      
      // check for field assignment
      if (consumeLeftArrowOpt()) {
        Evaluable rhs = parseExpr();
        ret new Toolbox.SetField(target, name, rhs);
      }
      
      bool allowNullReference = consumeOpt("?");

      var args = parseArguments();
      var call = finalizeCall(start, target, name, args);
      set(call, +allowNullReference);
      ret call;
    }
    
    // ~ also does stuff
    // (map ~a == map getOpt "a")
    // (list ~0 == list listGet 0)
    
    try object parseTildeCall(target);
    
    // special syntax to get a field, bypassing a method of the same name
    if (is("!") && is(1, "_getField_")) {
      next(2);
      S field = consumeIdentifier();
      ret new Toolbox.GetField(target, field);
    }
    
    // no call found
    ret target;
  }
  
  bool consumeLeftArrowOpt() {
    if (is("<") && eq(token(1), "-"))
      ret true with next(2);
    false;
  }
  
  Evaluable parseTildeCall(Evaluable target) {
    var start = ptr();
    
    if (consumeOpt("~")) {
      if (isIdentifier()) {
        S key = consumeIdentifier();
        please include function getOpt.
        CallOnTarget call = finalizeCall1(start, target, "getOpt", const(key));
        call.allowNullReference(true);
        ret parseCall(finalizeCall2(call));
      } else {
        int idx = consumeInteger();
        please include function listGet.
        ret parseCall(finalizeCall(start, target, "listGet", const(idx)));
      }
    }
    
    null;
  }
  
  Evaluable finalizeCall(TokPtr start, Evaluable target, S name, Evaluable... args) {
    ret finalizeCall2(finalizeCall1(start, target, name, args));
  }
  
  Evaluable finalizeCall2(CallOnTarget call) {
    while (is("then"))
      call = parseThenCall(call);
    ret call;
  }
  
  CallOnTarget finalizeCall1(TokPtr start, Evaluable target, S name, Evaluable... args) {
    // Is the method name also the name of a global function?
    if (magicSwitch) {
      O ext = findExternalObject(name);
      if (ext cast MethodOnObject) {
        if (nempty(args))
          ret new Toolbox.CallMethodOrGlobalFunction(target, name, ext, args);
        else
          ret new Toolbox.CallMethodOrGetFieldOrGlobalFunction(target, name, ext);
      }
    }
    
    if (nempty(args))
      ret new Toolbox.CallMethod(target, name, args);
    else
      ret linkToSrc(start, new Toolbox.CallMethodOrGetField(target, name));
  } // end of parseCall_noCmdEndCheck
  
  // add link to source code to parsed object if it supports it
  // and doesn't have a source code link already
  <A> A linkToSrc(TokPtr start, A a) {
    if (a cast IHasTokenRangeWithSrc)
      if (a.tokenRangeWithSrc() == null)
        a.setTokenRangeWithSrc(TokenRangeWithSrc(start, ptr().plus(-1)).sourceInfo(sourceInfo()));
    ret a;
  }
  
  // lambda version of linkToSrc
  <A> A linkToSrc(IF0<A> a) {
    var start = ptr();
    ret linkToSrc(start, a!);
  }
  
  // can return MethodOnObject
  // Note: Assumes we just parsed the name
  // TODO: rename this part of the method
  swappable O findExternalObject(S name) {
    //try object findClassThroughDefaultClassFinder(name);
    //try object findClassInStandardImports(name);
    
    if (eq(name, "void")) ret void.class;
    try object parsePrimitiveType(name);
      
    if (qualifiedNameContinues()) {
      int idx = idx()-2;
      do 
        next(2);
      while (qualifiedNameContinues());

      S fqn = joinSubList(tok, idx, idx()-1);
      ret classForName(fqn);
    }
    
    ret findExternalObject2(name);
  }
  
  swappable O findExternalObject2(S name) {
    S fullName = globalClassNames().get(name);
    if (fullName != null)
      ret classForName(fullName);
      
    try object classForNameOpt_noCache(name);
    
    fOr (container : functionContainers) {
      if (hasMethodNamed(container, name)
        && !isBlockedFunctionContainerMethod(container, name)) {
        var moo = new MethodOnObject(container, name);
        ifdef debugMethodFinding _print(+moo); endifdef
        ret moo;
      }
      
      var field = getField(container, name);
      if (field != null && isStaticField(field))
        ret new EvaluableWrapper(new Toolbox.GetStaticField(field));
    }
    
    null;
  }
  
  swappable bool isBlockedFunctionContainerMethod(O container, S name) {
    false;
  }
  
  selfType allowTheWorld() { ret allowTheWorld(mc()); }
  
  // first containers within argument list have priority
  // last calls to allowTheWorld/first arguments passed have priority
  selfType allowTheWorld(O... functionContainers) {
    fOr (O o : reversed(functionContainers))
      if (!contains(this.functionContainers, o)) {
        this.functionContainers.add(0, o);
        globalClassNames_cache = null; // recalculate
      }
    this;
  }
  
  void printFunctionDefs(Script script) {
    print(values(script.functionDefs));
  }
  
  AutoCloseable tempAddKnownVars(S... vars) {
    ret tempAddKnownVars(nonNulls(vars));
  }
  
  AutoCloseable tempAddKnownTypedVars(Iterable<TypedVar> vars) {
    ret tempAddKnownVars(mapToMap(vars, v -> pair(v.name, v.type)));
  }
    
  AutoCloseable tempAddKnownVars(Iterable<S> vars) {
    var newVars = mapWithSingleValue(vars, new LASValueDescriptor);
    ret tempAddKnownVars(newVars);
  }
    
  AutoCloseable tempAddKnownVar(S var, LASValueDescriptor type) {
    ret tempAddKnownVars(litmap(var, type));
  }
  
  AutoCloseable tempAddKnownVars(Map<S, LASValueDescriptor> newVars) {
    /*if (scope != null)
      for (name, type : newVars)
        scope.addDeclaredVar(name, type);*/
    ret tempMapPutAll(knownVars, newVars);
  }
  
  S parseFunctionDefArg(Map<S, LASValueDescriptor> argsOut) {
    if (consumeOpt("(")) {
      S result = parseFunctionDefArg(argsOut);
      consume(")");
      ret result;
    }
    
    if (isIdentifier()) {
      S arg = tpp();
      var type = typeToValueDescriptor(parseColonTypeOpt());
      argsOut.put(arg, type);
      ret arg;
    }
    
    null;
  }

  
  Toolbox.FunctionDef parseFunctionDefinition(Map<S, FunctionDef> functionDefsToAddTo) {
    var start = ptr();
    consume("def");
    bool synthetic = consumeOpt("!");
    if (synthetic) consume("synthetic");
    S functionName = assertIdentifier(tpp());
    print("parseFunctionDefinition " + functionName);
    ret parseFunctionDefinition_step2(functionDefsToAddTo, start, functionName)
      .synthetic(synthetic);
  }
    
  Toolbox.FunctionDef parseFunctionDefinition_step2(Map<S, FunctionDef> functionDefsToAddTo, TokPtr start, S functionName) {
    // argument names and types
    new LinkedHashMap<S, LASValueDescriptor> args;
    
    while (parseFunctionDefArg(args) != null) {}
    
    var returnType = parseColonTypeOpt();
    
    var fd = new Toolbox.FunctionDef(functionName, keysList(args), null); // no body yet
    fd.argTypes(valuesArray(LASValueDescriptor, args));
    functionDefsToAddTo?.put(functionName, fd);
    
    //var scope = newScope();
    //scope.useFixedVars(useFixedVarContexts);
    //temp tempScope(scope);
    temp tempAddKnownVars(args);
    print("Parsing function body");
    fd.body = parseReturnableCurlyBlock();
    
    print("Defined function " + functionName + ", added to " + functionDefsToAddTo);
    
    //fd.scope(scope);
    if (returnType != null) fd.returnType(returnType);
    ret linkToSrc(start, fd);
  }
  
  /*LASScope newScope() {
    ret new LASScope(scope);
  }*/
  
  Scope currentScope() { ret scope; }
  
  // enter the scope temporarily
  AutoCloseable tempScope(Scope scope) {
    var oldScope = currentScope();
    this.scope = scope;
    ret -> { this.scope = oldScope; };
  }
  
  Script parseReturnableCurlyBlock() {
    ret (Script) parseCurlyBlock(new BuildingScript().returnable(true));
  }
  
  Evaluable parseCurlyBlock(BuildingScript script default new) {
    //print(+knownVars);
    consume("{");
    bool inParensOld = inParens;
    inParens = false;
    var body = parseScript(script);
    consume("}");
    inParens = inParensOld;
    ret body;
  }
  
  Evaluable parseWhileLoop() {
    consume("while");
    var condition = parseExpr();
    var body = parseCurlyBlock(new BuildingScript().isLoopBody(true));
    ret new Toolbox.While(condition, body);
  }
  
  Evaluable parseDoWhileLoop() {
    consume("do");
    var body = parseCurlyBlock(new BuildingScript().isLoopBody(true));
    consume("while");
    var condition = parseExpr();
    ret new Toolbox.DoWhile(condition, body);
  }
  
  Evaluable parseForEach() {
    ret new ParseForEach()!;
  }
  
  class ParseForEach {
    Evaluable collection, body;
    IF0<Evaluable> finish;
    new L<TypedVar> vars;
    
    TypedVar addVar(TypedVar var) { ret addAndReturn(vars, var); }
    TypedVar consumeVar() {
      S name = consumeIdentifier();
      var type = parseColonTypeOpt();
      ret addVar(new TypedVar(name, typeToValueDescriptor(type)));
    }
    
    void parseBody {
      temp tempAddKnownTypedVars(vars);
      body = parseCurlyBlock(new BuildingScript().isLoopBody(true));
    }
    
    Evaluable get() {
      consume("for");
      
      // for i to n { ... }
      if (is(1, "to")) {
        TypedVar var = consumeVar();
        consume("to");
        Evaluable endValue = parseExpr();
        parseBody();
        ret new Toolbox.ForIntTo(endValue, var.name, body);
      }
    
      int iIn = relativeIndexOf("in");
      if (iIn < 0) fail("for without in");
      print(+iIn);
      
      if (iIn == 1 || is(1, ":")) { // just a variable (with optional type)
        TypedVar var = consumeVar();
        finish = -> new Toolbox.ForEach(collection, var.name, body).varType(var.type);
      } else if (iIn == 2) { // for iterator (mapI)
        if (consumeOpt("iterator")) {
          TypedVar var = consumeVar();
          finish = -> new Toolbox.ForIterator(collection, var.name, body);
        } else if (consumeOpt("nested")) {
          TypedVar var = consumeVar();
          finish = -> new Toolbox.ForNested(collection, var.name, body);
        } else
          fail("Unknown pattern for 'for' loop");
      } else if (iIn == 3) {
        if (isOneOf("pair", "Pair")) {
          // "pair a b" or "Pair a b"
          consume();
          TypedVar varA = consumeVar();
          TypedVar varB = consumeVar();
          printVars(+varA, +varB);
          
          finish = -> new Toolbox.ForPairs(collection, body, varA.name, varB.name);
        } else {
          // "a, b"
          TypedVar varA = consumeVar();
          consume(",");
          TypedVar varB = consumeVar();

          finish = -> new Toolbox.ForKeyValue(collection, body, varA.name, varB.name);
        }
      } else if (iIn == 4) {
        consume("index");
        S varIndex = consumeVar().name;
        consume(",");
        S varElement = consumeVar().name;
        
        finish = -> new Toolbox.ForIndex(collection, body, varIndex, varElement);
      } else
        fail("Unknown pattern for 'for' loop");
      
      consume("in");
      collection = parseExpr_inParens();
      print(+collection);
      parseBody();
      ret finish!;
    }
  }
  
  Evaluable parseIfStatement() {
    consume("if");
    Evaluable condition, body, elseBranch = null;
    {
      temp tempAdd(closerTokens, "then");
      condition = parseExpr();
    }
    
    if (consumeOpt("then")) {
      temp tempAdd(closerTokens, "else");
      body = parseExpr();
      if (consumeOpt("else"))
        elseBranch = parseExpr();
    } else {
      body = parseCurlyBlock();
      if (consumeOpt("else")) {
        if (is("if"))
          elseBranch = parseIfStatement();
        else
          elseBranch = parseCurlyBlock();
      }
    }
    
    ret new Toolbox.IfThen(condition, body, elseBranch);
  }
  
  Evaluable parseRepeatStatement() {
    consume("repeat");
    var n = parseExpr();
    var body = parseCurlyBlock();
    ret new Toolbox.RepeatN(n, body);
  }
  
  // declare an external variable with optional type info
  void addVar(S var, LASValueDescriptor type default new) { knownVars.put(var, type); }
  void addVar(S var, Class type, bool canBeNull) {
    addVar(var, typeToValueDescriptor(type, canBeNull));
  }
  
  LASValueDescriptor typeToValueDescriptor(Type type, bool canBeNull default true) {
    ret type == null ? new LASValueDescriptor
      : new LASValueDescriptor.NonExact(typeToClass(type), canBeNull);
  }
  
  // short name to full name
  // TODO: sort by priority if classes appear twice
  simplyCached SS globalClassNames() {
    var packages = mapToTreeSet(importedPackages(), pkg -> pkg + ".");
    
    new SS out;
    
    fOr (name : importedClasses())
      out.put(shortenClassName(name), name);
    
    // add function containers themselves (if they're classes)
    for (fc : functionContainers)
      if (fc cast Class) {
        if (isAnonymousClass(fc)) continue;
        out.put(shortClassName(fc), className(fc));
      }
    
    // add inner classes of function containers
    var classContainers = classContainerPrefixes();
    TreeSet<S> classContainerSet = asTreeSet(classContainers);

    for (className : classNameResolver().allFullyQualifiedClassNames()) {
      if (isAnonymousClassName(className)) continue;
      if (!contains(className, '$')) {
        S pkg = longestPrefixInTreeSet(className, packages);
        if (pkg != null) {
          S shortName = dropPrefix(pkg, className);
          if (!shortName.contains(".") && !out.containsKey(shortName))
            out.put(shortName, className);
        }
      }
        
      S container = longestPrefixInTreeSet(className, classContainerSet);
      if (container != null) {
        S shortName = dropPrefix(container, className);
        S existing = out.get(shortName);
        if (existing != null) {
          int priority = indexOf(classContainers, container);
          S oldContainer = longestPrefixInTreeSet(existing, classContainerSet);
          //int oldPriority = indexOf(classContainers, oldContainer);
          int oldPriority = smartIndexOf(classContainers, oldContainer);
          printVars(+className, +shortName, +container, +priority, +existing, +oldPriority);
          if (priority > oldPriority)
            continue;
        }
        out.put(shortName, className);
      }
    }
        
    fOr (key, val : classShortcuts()) {
      S fullName = out.get(val);
      if (fullName != null)
        out.put(key, fullName);
    }

    ret out;
  }
  
  swappable SS classShortcuts() {
    ret javaxClassShortcuts();
  }
  
  swappable Cl<S> importedPackages() {
    ret itemPlus("java.lang", standardImports_fullyImportedPackages());
  }
  
  swappable Cl<S> importedClasses() {
    ret standardImports_singleClasses();
  }
  
  void addClassAlias(S alias, S longName) {
    S fullName = globalClassNames().get(longName);
    if (fullName != null)
      globalClassNames().put(alias, fullName);
  }
  
  LS classContainerPrefixes() {
    ret map(functionContainers, fc -> className(fc) + "$");
  }
  
  // typed exceptions (unusual, I know!)
  
  srecord noeq UnknownObject(S name) > RuntimeException {
    public S getMessage() { ret "Unknown object: " + name; }
  }
  
  sclass ClassNotFound > UnknownObject {
    *(S className) { super(className); }
    
    public S getMessage() { ret "Class not found: " + name; }
  }
  
  asclass Scope {
    // Parse expression due to special rules in this scope
    // (return null if no parse)
    Evaluable parseExprStartingWithIdentifierOpt(S t, bool inner) { null; }
    
    Evaluable completeAssignmentOpt(S lhs, Evaluable rhs) { null; }
  }
  
  record ClassDefScope(LASClassDef classDef) extends Scope {
    Evaluable getThis() { ret new Toolbox.GetVar("this"); }
    
    Evaluable parseExprStartingWithIdentifierOpt(S t, bool inner) {
      printVars("ClassDefScope.parseExprStartingWithIdentifierOpt", +t);
      
      ret linkToSrc(-> {
        var field = classDef.fieldsByName.get(t);
        if (field != null) {
          print("Found field " + field);
          var e = inner
            ? new Toolbox.GetField(getThis(), field.name)
            : new Toolbox.CallMethodOrGetField(getThis(), field.name);
          ret parseCall(inner, e);
        }
        
        Class superClass = typeToClass(classDef.superClass);
        var field2 = findFieldOfClass(superClass, t);
        printVars(+field2, +superClass, +t);
        if (field2 != null) {
          print("Found field " + field2);
          var e = inner
            ? new Toolbox.GetField(getThis(), field2.getName())
            : new Toolbox.CallMethodOrGetField(getThis(), field2.getName());
          ret parseCall(inner, e);
        }
        
        if (!inner
          && (classDef.methodsByName.containsKey(t)
          || classHasMethodNamed(typeToClass(classDef.superClass), t))) {
          ret new Toolbox.CallMethod(getThis(), t, parseArguments());
        }
        
        null;
      });
    }
    
    Evaluable completeAssignmentOpt(S lhs, Evaluable rhs) {         var field = classDef.fieldsByName.get(lhs);
      if (field != null)
        ret new Toolbox.SetField(getThis(), lhs, rhs);
        
      Class superClass = typeToClass(classDef.superClass);
      var field2 = findFieldOfClass(superClass, lhs);
      if (field2 != null)
        ret new Toolbox.SetField(getThis(), lhs, rhs);
          
      null;
    }
  }
  
  LASClassDef parseClassDef() {
    ret linkToSrc(-> {
      consume("class");
      new LASClassDef classDef;
      classDef.verbose(scaffoldingEnabled());
      if (nempty(classDefPrefix))
        classDef.classDefPrefix(classDefPrefix);
      S name = consumeIdentifier();
      classDef.userGivenName(name);
      classDefs.put(name, classDef);
      temp tempMapPut(classDefs, "selfType", classDef);
      
      if (consumeOpt("extends")) {
        classDef.superClass(parseType());
      }
        
      if (consumeOpt("is"))
        classDef.interfaces.add(parseType());
  
      consume("{");
      
      // parse class body
      
      temp tempAddKnownVar("this", new LASValueDescriptor.NonExactType(classDef.resolvable(lasClassLoader), false));
      temp !objectScopesEnabled ? null : tempScope(new ClassDefScope(classDef));
      
      while (!is("}")) {
        if (is(";")) continue with next();
        
        // parse method declaration
        
        if (is("def")) {
          Toolbox.FunctionDef fd = parseFunctionDefinition(null);
          printVars(knownVarsAfterFunctionDef := knownVars);
          classDef.addMethod(fd);
          continue;
        }
        
        // parse constructor declaration
        
        var start = ptr();
        if (consumeOpt("ctor")) {
          var fd = parseFunctionDefinition_step2(null, start, "<init>");
          fd.returnType(void.class);
          classDef.addMethod(fd);
          continue;
        }
        
        // parse initializer block
        
        if (consumeOpt("initializer")) {
          classDef.initializers.add(parseCurlyBlock());
          continue;
        }
          
        // parse field declaration
        
        new LASClassDef.FieldDef fd;
        while (isOneOf("transient", "static")) { fd.addModifier(tpp()); }
        
        fd.name(consumeIdentifier());
        var type = parseColonType();
        fd.type(type);
        
        // parse field initializer
        if (consumeLeftArrowOpt())
          fd.initializer(parseExpr());

        classDef.addField(fd);
      }
      
      consume("}");
      ret classDef;
    });
  }
  
  Type parseColonTypeOpt() {
    if (is(":"))
      ret parseColonType();
    null;
  }
  
  Type parseColonType() {
    consume(":");
    ret parseType();
  }
  
  Type findType(S typeName) {
    // script-defined class?
    LASClassDef cd = classDefs.get(typeName);
    if (cd != null)
      ret cd.resolvable(lasClassLoader);
      
    O o = findExternalObject(typeName);
    if (o cast Class)
      ret o;
      
    throw new ClassNotFound(typeName);
  }
  
  Type parseType() {
    S typeName = consumeIdentifier();
    Type type = findType(typeName);
    ret parseArrayType(parseTypeArguments(type));
  }
  
  Type parseArrayType(Type type) {
    while (is("[") && is(1, "]")) {
      consume(2);
      type = new GenericArrayTypeImpl(type);
    }
    ret type;
  }
  
  Type parseTypeArguments aka parseTypeArgumentsOpt(Type type) {
    // type arguments
    if (is("<") && empty(nextSpace()) && isIdentifier(token(1))) {
      next();
      new L<Type> args;
      while true {
        args.add(parseType());
        if (is(">")) break;
        consume(",");
      }
      consume(">");
      
      if (type != null)
        type = new ParameterizedTypeImpl(null, type, toTypedArray(Type, args));
    }
    ret type;
  }
  
  swappable Class classForName(S name) ctex {
    ret Class.forName(name);
  }
  
  // also copies the globalClassNames_cache which otherwise takes
  // a few ms to calculate
  void copyFunctionContainersFrom(GazelleV_LeftArrowScriptParser parser) {
    functionContainers = cloneList(parser.functionContainers);
    globalClassNames_cache = parser.globalClassNames();
    isBlockedFunctionContainerMethod = parser.isBlockedFunctionContainerMethod;
  }
  
  ClassNameResolver classNameResolver() {
    if (classNameResolver == null)
      classNameResolver = new ClassNameResolver().byteCodePath(assertNotNull(getBytecodePathForClass(this))).init();
    ret classNameResolver;
  }
  
  selfType classNameResolver(ClassNameResolver classNameResolver) {
    this.classNameResolver = classNameResolver;
    this;
  }

  Evaluable parseStringLiteral() {  
    ret const(unquote_relaxedMLS(consume()));
  }
  
  void addFunctionDefs(Map<S, FunctionDef> map) {
    putAll(functionDefs, map);
  }
  
  selfType setFlag(S flag) {
    flags.add(flag);
    this;
  }
  
  bool getFlag(S flag) {
    ret flags.contains(flag);
  }

  // forget normally imported class names (e.g. for testing)
  void noImports() {  
    importedPackages = -> null;
    importedClasses = -> null;
    classShortcuts = -> null;
  }
  
  srecord TypedVar(S name, LASValueDescriptor type) {}
} // end of GazelleV_LeftArrowScriptParser