/* e.g.

  overlay <- ScreenOverlay

  bounds <- rightScreenBounds

  overlay bounds bounds

  overlay show

Can separate commands with ";" also.

For how to define functions in a script see #1033988.

Note: LeftArrowScriptAutoCompleter uses a lot of this class's internals

*/

// TODO: "then" is used twice which is probably not good

// (for if-then and for chaining calls)

sclass GazelleV_LeftArrowScriptParser > SimpleLeftToRightParser {

  // most important "experimental" setting (will set this to true)

  settable bool magicSwitch = true;

  // New object scopes (class members without "this")

  // - MAY potentially break old scripts (although probably not)

  settable bool objectScopesEnabled = true;

  replace Toolbox with GazelleV_LeftArrowScript.

  delegate Script, Evaluable, FunctionDef, FixedVarBase,

    LambdaMethodOnArgument, CallOnTarget to Toolbox.

  replace const with _const.

  ClassNameResolver classNameResolver;

  new L functionContainers;

  settable ILASClassLoader lasClassLoader;

  settable S classDefPrefix;

  // will be noted in source references

  settable O sourceInfo;

  settable bool optimize = true;

  settable bool useFixedVarContexts = false; // doesn't completely work yet

  Scope scope;

  new LinkedHashMap<S, LASValueDescriptor> knownVars;

  //new L<FixedVarBase> varAccessesToFix;

  Set<S> closerTokens = litset(";", "}", ")");

  BuildingScript currentReturnableScript;

  BuildingScript currentLoop;

  bool inParens;

  int idCounter;

  new Map<S, LASClassDef> classDefs;

  gettable Map<S, FunctionDef> functionDefs = new Map;

  Set<S> flags = ciSet();

  settable bool internStringLiterals = true;

  // events for auto-completer

  // 1. record which vars are known at current parse location

  event knownVarsSnapshot(Map<S, LASValueDescriptor> knownVars);

  // 2. indicates we just parsed an expression of type type

  //    (useful for listing methods+fields for auto-completion)

  event typeHook(LASValueDescriptor type);

  replace print with if (GazelleV_LeftArrowScriptParser.this.scaffoldingEnabled()) print.

  replace printVars with if (GazelleV_LeftArrowScriptParser.this.scaffoldingEnabled()) printVars.

  srecord EvaluableWrapper(Evaluable expr) {}

  class BuildingScript {

    int id = ++idCounter;

    settable bool returnable;

    settable bool isLoopBody;

    BuildingScript returnableParent, loopParent;

    //settable LASScope scope;

    new Script script;

    new L<Evaluable> steps;

    Map<S, FunctionDef> functionDefs = new Map;

    *(bool *returnable) { this(); }

    *(bool *returnable, bool *isLoopBody) { this(); }

    *() { /*scope = currentScope();*/ }

    void add(Evaluable step) { if (step != null) steps.add(step); }

    Evaluable get() {

      //script.scope = scope;

      // if the last command is a return from THIS script,

      // convert into a simple expression

      var lastStep = last(steps);

      if (lastStep cast Toolbox.ReturnFromScript)

        if (lastStep.script == script)

          replaceLast(steps, lastStep.value);

      // if the script is only step, is not returnable

      // and defines no functions, replace it with its only step

      if (!returnable && l(steps) == 1 && empty(functionDefs))

        ret first(steps);

      // make and return actual script

      if (nempty(functionDefs)) script.functionDefs = functionDefs;

      script.steps = toTypedArray(Evaluable.class, steps);

      ret script;

    }

    S toStringLong() { ret pnlToLines(steps); }

    toString {

      ret formatRecordVars BuildingScript(+id, +returnable , +returnableParent, +script);

    }

  } // end of BuildingScript

  // methods begin here

  {

    tokenize = text -> {

      LS tok = tokenize_base(text);

      tok = preprocess(tok);

      print("=== PREPROCESSED\n" + indentx(join(tok)) + "\n===");

      ret tok;

    };

  }

  LS preprocess(LS tok) {

    tok = tokenIndexedList3(tok);

    tok_ifdef(tok, l1 getFlag);

    tok_pcall_script(tok);

    tok_script_settable(tok);

    jreplace(tok, "LS", "L<S>");

    jreplace(tok, ", +<id>", ", $3 $3");

    ret cloneList(tok);

  }

  Script parse(S text) {

    setText(text);

    init();

    Script script = parse();

    if (!atEnd())

      fail("End of script expected");

    ret script;

  }

  Script parse() {

    Script script = parseReturnableScript();

    /*for (varAccess : varAccessesToFix)

      varAccess.resolve();*/

    if (optimize) script = script.optimizeScript();

    ret script;

  }

  Script parseReturnableScript() {

    ret (Script) parseScript(new BuildingScript().returnable(true));

  }

  // if returnable is true, it's always a Script

  Evaluable parseScript(BuildingScript script) {

    ret linkToSrc(-> {

      script.returnableParent = currentReturnableScript;

      script.loopParent = currentLoop;

      if (script.returnable)

        currentReturnableScript = script;

      if (script.isLoopBody)

        currentLoop = script;

      ret parseBuildingScript(script);

    });

  }

  Evaluable parseBuildingScript(BuildingScript script) {

    try {

      parseScript_2(script);

      var builtScript = script!;

      currentReturnableScript = script.returnableParent;

      currentLoop = script.loopParent;

      ret builtScript;

    } catch e {

      print("Parsed so far:\n" + script);

      // TODO: could use an exception class [like ScriptError]

      throw rethrowAndAppendToMessage(e,

        squareBracketed(spaceCombine(sourceInfo, lineAndColumn(-1)));

    }

  }

  void parseScript_2(BuildingScript script) {

    temp tempRestoreMap(knownVars);

    new AssureAdvance assure;

    while (assure!) {

      // for auto-completer

      knownVarsSnapshot(knownVars);

      print("parseScript_2: Next token is " + quote(token()));

      if (is(";")) continue with next();

      if (isOneOf("}", ")")) break;

      Evaluable instruction = linkToSrc(-> parseInstruction(script));

      if (instruction != null)

        script.add(instruction);

    }

    print("parseScript_2 done");

    knownVarsSnapshot(knownVars);

  }

  Evaluable parseParamDecl(BuildingScript script) {

    S var = assertIdentifier(tpp());

    new LASValueDescriptor valueDescriptor;

    if (is(":")) {

      var type = parseColonType();

      valueDescriptor = LASValueDescriptor.nonExactCanBeNull(type);

    }

    knownVars.put(var, valueDescriptor);

    script.script.params = putOrCreateLinkedHashMap(script.script.params, var, valueDescriptor);

    null;

  }

  Evaluable parseInstruction(BuildingScript script) {

    /*if (is("var") && isIdentifier(token(1))) {

      consume();

      ret parseAssignment();

    }TODO*/

    // Script parameter (old)

    if (consumeOpt("param"))

      ret parseParamDecl(script);

    if (is("throw")) {

      consume();

      ret new Toolbox.Throw(parseExpr());

    }

    /* Hmm. Too hard. Need to use jreplace.  

    if (is("pcall")) {

      consume();

      Evaluable body = parseCurlyBlock();

      S var = "_pcallError";

      temp tempAddKnownVars(var);

      Evaluable catchBlock = parseCurlyBlock("{ pcallFail " + var + " });

      ret new Toolbox.TryCatch(body, var, catchBlock);

    }*/

    if (isOneOf("return", "ret")) {

      consume();

      Evaluable expr;

      if (atCmdEnd())

        expr = const(null);

      else

        expr = parseAssignmentOrExpr();

      ret new Toolbox.ReturnFromScript(currentReturnableScript.script, expr);

    }

    if (consumeOpt("continue")) {

      assertCmdEnd();

      if (currentLoop == null)

        fail("continue outside of loop");

      ret new Toolbox.Continue(currentLoop.script);

    }

    /* TODO

    if (consumeOpt("break")) {

      assertCmdEnd();

      if (currentLoop == null)

        fail("break outside of loop");

      ret new Toolbox.Break(currentLoop.script);

    }*/

    if (is("temp")) {

      consume();

      Evaluable tempExpr = parseAssignmentOrExpr();

      print(+tempExpr);

      Evaluable body = parseScript(new BuildingScript);

      print(+body);

      ret new Toolbox.TempBlock(tempExpr, body);

    }

    if (is("will") && is(1, "return")) {

      consume(2);

      Evaluable exp = parseAssignmentOrExpr();

      Evaluable body = parseScript(new BuildingScript);

      ret new Toolbox.WillReturn(exp, body);

    }

    if (is("var") && isIdentifier(token(1))) {

      consume();

      S varName = consumeIdentifier();

      var type = parseColonTypeOpt();

      if (consumeLeftArrowOpt()) {

        print("Found assignment");

        Evaluable rhs = parseExpr();

        assertNotNull("Expression expected", rhs);

        knownVars.put(varName, type == null

          ? or(rhs.returnType(), new LASValueDescriptor)

          : LASValueDescriptor.nonExactCanBeNull(type));

        ret new Toolbox.VarDeclaration(varName, typeToClass(type), rhs);

      }

    }

    ret parseAssignmentOrExpr();

  }

  Evaluable parseAssignmentOrExpr() {

    try object parseAssignmentOpt();

    ret parseExpr();

  }

  // Parse assignment starting with a simple identifier (e.g. "i <- 5")

  Evaluable parseAssignmentOpt() {

    S t = token();

    if (isIdentifier(t)) {

      Type type = null;

      var ptr = ptr(); // save parsing position

      // optional type

      if (is(1, ":")) {

        next();

        type = parseColonType();

        unconsume();

      }

      if (is(1, "<") && is(2, "-")) {

        print("Found assignment");

        next(3);

        Evaluable rhs = parseExpr();

        assertNotNull("Expression expected", rhs);

        bool newVar = !knownVars.containsKey(t);

        printVars(+newVar, +t, +knownVars);

        if (newVar && scope != null)

          try object scope.completeAssignmentOpt(t, rhs);

        knownVars.put(t, type == null

          ? or(rhs.returnType(), new LASValueDescriptor)

          : LASValueDescriptor.nonExactCanBeNull(type));

        ret newVar

          ? new Toolbox.VarDeclaration(t, null, rhs)

          : new Toolbox.Assignment(t, rhs);

      }

      // revert parser, it was just a typed expression

      // (TODO: this technically violates the O(n) parsing principle)

      ptr(ptr);

    }

    null;

  }

  Evaluable parseOptionalInnerExpression() {

    printVars parseOptionalInnerExpression(+token());

    if (atCmdEnd() || isOneOf("{", ",", "then")) null;

    ret parseInnerExpr();

  }

  Evaluable const(O o) { ret new Toolbox.Const(o); }

  Evaluable parseInnerExpr() { ret parseExpr(true); }

  scaffolded Evaluable parseExpr(bool inner default false) {

    Evaluable e = linkToSrc(-> inner ? parseExpr_impl(true) : parseFullExpr());

    print("parseExpr done:\n" + Toolbox.indentedScriptStruct(e));

    ret e;

  }

  Evaluable parseFullExpr() {

    ret linkToSrc(-> parseExprPlusOptionalComma());

    //ret parseExprPlusOptionalCommaOrThen();

  }

  /*Evaluable parseExprPlusOptionalCommaOrThen() {

    Evaluable expr = parseExpr_impl(false);

    //printVars("parseExprPlusOptionalCommaOrThen", t := t());

    while true {

      if (is("then"))

        expr = parseThenCall(expr);

      else if (consumeOpt(","))

        expr = parseCall_noCmdEndCheck(expr);

      else

        break;

      //printVars("parseExprPlusOptionalCommaOrThen", t2 := t());

    }

    ret expr;

  }*/

  // TODO: How to store the object between the two calls?

  // We need to put it in the VarContext somehow.

  // Or allow passing in a target to CallOnTarget

  CallOnTarget parseThenCall(CallOnTarget expr) {

    consume("then");

    CallOnTarget secondCall = cast parseCall(null);

    ret new Toolbox.Then(expr, secondCall);

  }

  Evaluable parseExprPlusOptionalComma() {

    Evaluable expr = parseExpr_impl(false);

    //printVars("parseExprPlusOptionalComma", t := t());

    while (consumeOpt(",")) {

      expr = parseCall_noCmdEndCheck(expr);

      //printVars("parseExprPlusOptionalComma", t2 := t());

    }

    ret expr;

  }

  Evaluable parseExpr_impl(bool inner) {

    if (atEnd()) null;

    S t = token();

    printVars parseExpr(token := t);

    if (is(";")) null; // empty command

    if (consumeOpt("try")) {

      Evaluable body = parseCurlyBlock();

      while (consumeOpt("catch")) {

        S var = consumeIdentifierOpt();

        temp tempAddKnownVars(var);

        Evaluable catchBlock = parseCurlyBlock();

        body = new Toolbox.TryCatch(body, var, catchBlock);

      }

      if (consumeOpt("finally")) {

        Evaluable finallyBlock = parseCurlyBlock();

        ret new Toolbox.TryFinally(body, finallyBlock);

      }

      ret body;

    }

    if (is("def")) {

      var fd = parseFunctionDefinition(currentReturnableScript.functionDefs);

      ret const(fd);

    }

    if (is("{"))

      ret parseCurlyBlock();

    // int or double literal

    if (is("-") && empty(nextSpace()) && startsWithDigit(token(1))

      || startsWithDigit(t)) {

      var e = parseNumberLiteral();

      ret parseCall(inner, e);

    }

    if (isQuoted(t)) {

      var e = parseStringLiteral();

      ret parseCall(inner, e);

    }

    if (startsWith(t, '\'')) {

      consume();

      var e = const(first(unquote(t)));

      ret parseCall(inner, e);

    }

    if (isIdentifier(t)) {

      if (consumeOpt("synchronized")) {

        var target = parseExpr();

        var body = parseCurlyBlock();

        ret new Toolbox.Synchronized(target, body);

      }

      if (is("while"))

        ret parseWhileLoop();

      if (is("do"))

        ret parseDoWhileLoop();

      if (is("for"))

        ret parseForEach();

      if (is("if"))

        ret parseIfStatement();

      if (is("repeat"))

        ret parseRepeatStatement();

      if (is("outer")) {

        consume();

        var a = parseAssignmentOpt();

        if (!a instanceof Toolbox.Assignment)

          fail("Assignment expected");

        cast a to Toolbox.Assignment;

        ret new Toolbox.AssignmentToOuterVar(a.var, a.expression);

      }

      consume();

      print("Consumed identifier " + t + ", next token: " + token() + ", inner: " + inner);

      var e = parseExprStartingWithIdentifier(t, inner);

      ret inner ? parseCall(inner, e) : e;

    }

    // nested expression

    if (eq(t, "(")) {

      consume();

      //print("Consumed opening parens (level " + parenLevels + ")");

      Evaluable e = parseExpr_inParens();

      //print("Consuming closing parens for expr: " + e + " (closing paren level " + parenLevels + ")");

      consume(")");

      ret parseCall(inner, e);

    }

    if (isOneOf("&", "|") && empty(nextSpace()) && is(1, token())) {

      ret parseBinaryOperator();

    }

    fail("Identifier, literal, operator or opening parentheses expected (got: " + quote(t));

  }

  Evaluable parseExpr_inParens() {

    bool inParensOld = inParens;

    inParens = true;

    try {

      ret parseExpr();

    } finally {

      inParens = inParensOld;

    }

  }

  Evaluable parseNumberLiteral() {

    S t = consumeMultiTokenLiteral();

    if (swic(t, "0x"))

      if (ewic(t, "L")) {

        t = dropLast(t);

        ret const(parseHexLong(dropFirst(t, 2)));

      } else

        ret const(parseHexInt(dropFirst(t, 2)));

    if (swic(t, "0b")) ret const(intFromBinary(dropFirst(t, 2)));

    if (isInteger(t)) ret const(parseIntOrLong(t));

    if (ewic(t, "f")) ret const(parseFloat(t));

    if (endsWith(t, "L")) ret const(parseLong(t));

    ret const(parseDouble(t));

  }

  Evaluable parseBinaryOperator() {

    bool and = is("&");

    next(2);

    Evaluable a = parseInnerExpr();

    Evaluable b = parseInnerExpr();

    ret and

      ? new Toolbox.BoolAnd(a, b)

      : new Toolbox.BoolOr(a, b);

  }

  bool qualifiedNameContinues() {

    ret empty(prevSpace()) && eq(token(), ".") && empty(nextSpace())

      && isIdentifier(token(1));

  }

  // assumes we have consumed the "new"  

  Evaluable parseNewExpr() {

    S className = assertIdentifier(tpp());

    parseTypeArguments(null);

    // script-defined class?

    LASClassDef cd = classDefs.get(className);

    if (cd != null)

      ret new Toolbox.NewObject_LASClass(cd.resolvable(lasClassLoader));

    // class in variable?

    var type = knownVars.get(className);

    if (type != null)

      ret new Toolbox.NewObject_UnknownClass(new Toolbox.GetVar(className), parseArguments());

    // external class

    O o = findExternalObject(className);

    if (o cast Class) {

      Class c = o;

      if (c == L.class) c = ArrayList.class;

      else if (c == Map.class) c = HashMap.class;

      else if (c == Set.class) c = HashSet.class;

      ret new Toolbox.NewObject(c, parseArguments());

    }

    throw new ClassNotFound(className);

  }

  // t is last consumed token (the identifier the expression starts with)

  // TODO: do parseCall in only one place!

  Evaluable parseExprStartingWithIdentifier(S t, bool inner) {

    if (eq(t, "true")) ret const(true);

    if (eq(t, "false")) ret const(false);

    if (eq(t, "null")) ret const(null);

    if (eq(t, "new"))

      ret parseNewExpr();

    if (eq(t, "class")) {

      unconsume();

      ret new Toolbox.ClassDef(parseClassDef().resolvable(lasClassLoader));

    }

    if (eq(t, "list") && is("{")) {

      Script block = parseReturnableCurlyBlock();

      Evaluable e = new Toolbox.ListFromScript(block);

      ret parseCall(inner, e);

    }

    // Search in locally known variables

    var type = knownVars.get(t);

    if (type != null) {

      var e = new Toolbox.GetVar(t).returnType(type);

      print("Found var acccess: " + e + ", " + (!inner ? "Checking for call" : "Returning expression"));

      ret parseCall(inner, e);

    }

    // Search in scope

    if (scope != null) {

      try object scope.parseExprStartingWithIdentifierOpt(t, inner);

    }

    // script-defined class?

    LASClassDef cd = classDefs.get(t);

    if (cd != null)

      ret new Toolbox.ClassDef(cd.resolvable(lasClassLoader));

    if (!inner) {

      var fdef = lookupFunction(t);

      if (fdef != null) {

        if (is("~"))

          ret parseTildeCall(new Toolbox.CallFunction(fdef, new Evaluable[0]));

        ret new Toolbox.CallFunction(fdef, parseArguments());

      }

    }

    if (eq(t, "_context"))

      ret new Toolbox.GetVarContext;

    O o = findExternalObject(t);

    var start = ptr().minus(2);

    if (o == null) {

      //unconsume();

      throw new UnknownObject(t);

    } else if (o cast EvaluableWrapper) {

      ret parseCall(inner, o.expr);

    } else if (inner)

      ret linkToSrc(start, const(o));

    else if (o cast Class) {

      ret parseExprStartingWithClass(start, o);

    } else if (o cast MethodOnObject) {

      // it's a function from a functionContainer

      if (inner) fail("Can't call methods in arguments"); // Does this ever happen?!

      ret new Toolbox.CallMethod(const(o.object), o.method, parseArguments());

    } else

      ret parseCall(const(o));

  }

  Evaluable parseExprStartingWithClass(TokPtr start, Class c) {

    printVars("parseExprStartingWithClass", +c);

    if (atCmdEnd())

      ret linkToSrc(start, const(c));

    // new without new

    if (is("("))

      ret new Toolbox.NewObject(c, parseArguments());

    /* old object creation syntax (e.g. Pair new a b)

    if (is("new")) {

      next();

      ret new Toolbox.NewObject(c, parseArguments());

    } else*/

    // check if it's a lambda definition first (with ->)

    try object parseLambdaOpt(c);

    if (isIdentifier()) {

      S name = tpp();

      // We have now parsed a class (c) plus an as yet unknown identifier (name) right next to it

      // look for a static method call

      if (hasStaticMethodNamed(c, name))

        ret new Toolbox.CallMethod(const(c), name, parseArguments());

      // if the class is an interface, it's a lambda method reference or a lambda

      if (isInterface(c))

        ret parseLambdaMethodRef(c, name);

      // look for field second

      var field = getField(c, name);

      if (field == null)

        field = findFieldInInterfaces(c, name);

      if (field != null) {

        if (!isStaticField(field))

          fail(field + " is not a static field");

        // check for field assignment

        if (consumeLeftArrowOpt()) {

          Evaluable rhs = parseExpr();

          ret new Toolbox.SetStaticField(field, rhs);

        }

        assertCmdEnd();

        ret new Toolbox.GetStaticField(field);

      }

      fail(name + " not found in " + c + " (looked for method or field)");

    } else

      fail("Method name expected: " + token());

  }

  // parse lambda, e.g. IF1 a -> plus a a

  // returns null if it's not a lambda

  // assumes that c was just parsed

  Evaluable parseLambdaOpt(Class c) {

    // must be an interface

    if (!isInterface(c)) null;

    // speculatively parse optional type args first

    var ptr = ptr();

    var parameterized = parseTypeArgumentsOpt(c);

    // e.g. IF0 { 5 }

    if (is("{"))

      ret finishLambdaDef(c, null);

    // e.g. IF1 _ size

    if (consumeOpt("_")) {

      S methodName = consumeIdentifier();

      ret new LambdaMethodOnArgument(c, methodName, parseArguments());

    }

    new LinkedHashMap<S, LASValueDescriptor> args;

    while (isIdentifier()) {

      S name = consumeIdentifier();

      args.put(name, typeToValueDescriptor(parseColonTypeOpt()));

    }

    // Check for lambda arrow

    if (!(is("-") && is(1, ">"))) {

      ptr(ptr); // revert pointer

      null; // not a lambda

    }

    skip(2); // skip the arrow

    ret finishLambdaDef(c, args);

  }

  Evaluable finishLambdaDef(Class c, LinkedHashMap<S, LASValueDescriptor> args) {

    temp tempAddKnownVars(args);

    knownVarsSnapshot(knownVars);

    Evaluable body;

    // if curly brackets: parse lambda body as returnable script

    // TODO: avoid script overhead if there are no return calls inside

    if (is("{"))

      body = parseReturnableCurlyBlock();

    else

      body = parseExpr();

    // return lambda

    var lambda = new Toolbox.LambdaDef(c, toStringArray(keys(args)), body);

    print("finishLambdaDef done:\n" + Toolbox.indentedScriptStruct(lambda));

    ret lambda;

  }

  // method ref, e.g. "IF1 l" meaning "l1 l"

  // also, constructor ref, e.g. "IF1 Pair" meaning (x -> new Pair(x))

  //

  // c must be a single method interface

  Evaluable parseLambdaMethodRef(Class c, S name) {

    var fdef = lookupFunction(name);

    if (fdef != null) {

      Evaluable[] curriedArguments = parseArguments();

      ret new Toolbox.CurriedScriptFunctionLambda(c, fdef, curriedArguments);

    }

    O function = findExternalObject(name);

    if (function == null) 

      throw new UnknownObject(name);

    if (function cast MethodOnObject) {

      // it's a function from a functionContainer

      O target = function.object;

      S targetMethod = function.method;

      Evaluable[] curriedArguments = parseArguments();

      ret new Toolbox.CurriedMethodLambda(c, target, targetMethod, curriedArguments);

    } else if (function cast Class) {

      Class c2 = function;

      // No currying here yet

      assertCmdEnd();

      var ctors = constructorsWithNumberOfArguments(c2, 1);

      if (empty(ctors))

        fail("No single argument constructor found in " + c2);

      ret new Toolbox.CurriedConstructorLambda(c, toArray Constructor(ctors), null);

    } else

      fail(function + " is not an instantiable class or callable method");

  }

  FunctionDef lookupFunction(S name) {

    var script = currentReturnableScript;

    while (script != null) {

      var f = script.functionDefs.get(name);

      printVars("lookupFunction", +script, +name, +f);

      if (f != null) ret f;

      script = script.returnableParent;

    }

    // Try parser-wide function defs

    ret functionDefs.get(name);

  }

  Evaluable[] parseArguments() {

    new L<Evaluable> l;

    try {

      while (true) {

        print("parseArgumentsAsList: token is " + quote(t()));

        // +id

        if (is("+") && empty(nextSpace()) && isIdentifier(token(1))) {

          consume();

          l.add(const(token()));

          // just leave identifier unconsumed

          continue;

        }

        if (consumeOpt("<")) {

          Evaluable a = parseFullExpr();

          print("parseArgumentsAsList: token after expr is " + quote(t()));

          if (a == null) fail("expression expected");

          l.add(a);

          break;

        }

        Evaluable a = parseOptionalInnerExpression();

        if (a == null) break;

        l.add(a);

      }

      ret toArrayOrNull(Evaluable.class, l);

    } on fail {

      print("Arguments parsed so far: " + l);

    }

  }

  S consumeMultiTokenLiteral() {

    ret consumeUntilSpaceOr(-> atCmdEnd() || is(","));

  }

  bool atCmdEnd() {

    ret

      !inParens && atEndOrLineBreak()

      || closerTokens.contains(token());

  }

  void assertCmdEnd() {

    if (!atCmdEnd())

      fail("Expected end of command, token is: " + quote(token()));

  }

  // After we just parsed an expression, see if there is something

  // to the right of it.

  // Currently that can be

  // -a method call

  // -a field access

  // -a field assignment (e.g.: myPair a <- 1)

  Evaluable parseCall(bool inner default false, Evaluable target) {

    returnTypeHook(target);

    // normally, do a check for end of line

    if (atCmdEnd()) ret target;

    if (inner) {

      try object parseTildeCall(target);

      ret target;

    }

    ret parseCall_noCmdEndCheck(target);

  }

  // For auto-completer

  void returnTypeHook(Evaluable e) {

    if (e != null && e.returnType() != null)

      typeHook(e.returnType());

  }

  Evaluable parseCall_noCmdEndCheck(Evaluable target) {

    var start = ptr();

    returnTypeHook(target);

    // We usually expect an identifier

    if (isIdentifier()) {

      S name = tpp(); // the identifier

      // check for field assignment

      if (consumeLeftArrowOpt()) {

        Evaluable rhs = parseExpr();

        ret new Toolbox.SetField(target, name, rhs);

      }

      bool allowNullReference = consumeOpt("?");

      var args = parseArguments();

      var call = finalizeCall(start, target, name, args);

      set(call, +allowNullReference);

      ret call;

    }

    // ~ also does stuff

    // (map ~a == map getOpt "a")

    // (list ~0 == list listGet 0)

    try object parseTildeCall(target);

    // special syntax to get a field, bypassing a method of the same name

    if (is("!") && is(1, "_getField_")) {

      next(2);

      S field = consumeIdentifier();

      ret new Toolbox.GetField(target, field);

    }

    // no call found

    ret target;

  }

  bool consumeLeftArrowOpt() {

    if (is("<") && eq(token(1), "-"))

      ret true with next(2);

    false;

  }

  Evaluable parseTildeCall(Evaluable target) {

    var start = ptr();

    if (consumeOpt("~")) {

      if (isIdentifier()) {

        S key = consumeIdentifier();

        please include function getOpt.

        CallOnTarget call = finalizeCall1(start, target, "getOpt", const(key));

        call.allowNullReference(true);

        ret parseCall(finalizeCall2(call));

      } else {

        int idx = consumeInteger();

        please include function listGet.

        ret parseCall(finalizeCall(start, target, "listGet", const(idx)));

      }

    }

    null;

  }

  Evaluable finalizeCall(TokPtr start, Evaluable target, S name, Evaluable... args) {

    ret finalizeCall2(finalizeCall1(start, target, name, args));

  }

  Evaluable finalizeCall2(CallOnTarget call) {

    while (is("then"))

      call = parseThenCall(call);

    ret call;

  }

  CallOnTarget finalizeCall1(TokPtr start, Evaluable target, S name, Evaluable... args) {

    // Is the method name also the name of a global function?

    if (magicSwitch) {

      O ext = findExternalObject(name);

      if (ext cast MethodOnObject) {

        if (nempty(args))

          ret new Toolbox.CallMethodOrGlobalFunction(target, name, ext, args);

        else

          ret new Toolbox.CallMethodOrGetFieldOrGlobalFunction(target, name, ext);

      }

    }

    if (nempty(args))

      ret new Toolbox.CallMethod(target, name, args);

    else

      ret linkToSrc(start, new Toolbox.CallMethodOrGetField(target, name));

  } // end of parseCall_noCmdEndCheck

  // add link to source code to parsed object if it supports it

  // and doesn't have a source code link already

  <A> A linkToSrc(TokPtr start, A a) {

    if (a cast IHasTokenRangeWithSrc)

      if (a.tokenRangeWithSrc() == null)

        a.setTokenRangeWithSrc(TokenRangeWithSrc(start, ptr().plus(-1)).sourceInfo(sourceInfo()));

    ret a;

  }

  // lambda version of linkToSrc

  <A> A linkToSrc(IF0<A> a) {

    var start = ptr();

    ret linkToSrc(start, a!);

  }

  // can return MethodOnObject

  // Note: Assumes we just parsed the name

  // TODO: rename this part of the method

  swappable O findExternalObject(S name) {

    //try object findClassThroughDefaultClassFinder(name);

    //try object findClassInStandardImports(name);

    if (eq(name, "void")) ret void.class;

    try object parsePrimitiveType(name);

    if (qualifiedNameContinues()) {

      int idx = idx()-2;

      do 

        next(2);

      while (qualifiedNameContinues());

      S fqn = joinSubList(tok, idx, idx()-1);

      ret classForName(fqn);

    }

    ret findExternalObject2(name);

  }

  swappable O findExternalObject2(S name) {

    S fullName = globalClassNames().get(name);

    if (fullName != null)

      ret classForName(fullName);

    try object classForNameOpt_noCache(name);

    fOr (container : functionContainers) {

      if (hasMethodNamed(container, name)

        && !isBlockedFunctionContainerMethod(container, name)) {

        var moo = new MethodOnObject(container, name);

        ifdef debugMethodFinding _print(+moo); endifdef

        ret moo;

      }

      var field = getField(container, name);

      if (field != null && isStaticField(field))

        ret new EvaluableWrapper(new Toolbox.GetStaticField(field));

    }

    null;

  }

  swappable bool isBlockedFunctionContainerMethod(O container, S name) {

    false;

  }

  selfType allowTheWorld() { ret allowTheWorld(mc()); }

  // first containers within argument list have priority

  // last calls to allowTheWorld/first arguments passed have priority

  selfType allowTheWorld(O... functionContainers) {

    fOr (O o : reversed(functionContainers))

      if (!contains(this.functionContainers, o)) {

        this.functionContainers.add(0, o);

        globalClassNames_cache = null; // recalculate

      }

    this;

  }

  void printFunctionDefs(Script script) {

    print(values(script.functionDefs));

  }

  AutoCloseable tempAddKnownVars(S... vars) {

    ret tempAddKnownVars(nonNulls(vars));

  }

  AutoCloseable tempAddKnownTypedVars(Iterable<TypedVar> vars) {

    ret tempAddKnownVars(mapToMap(vars, v -> pair(v.name, v.type)));

  }

  AutoCloseable tempAddKnownVars(Iterable<S> vars) {

    var newVars = mapWithSingleValue(vars, new LASValueDescriptor);

    ret tempAddKnownVars(newVars);

  }

  AutoCloseable tempAddKnownVar(S var, LASValueDescriptor type) {

    ret tempAddKnownVars(litmap(var, type));

  }

  AutoCloseable tempAddKnownVars(Map<S, LASValueDescriptor> newVars) {

    /*if (scope != null)

      for (name, type : newVars)

        scope.addDeclaredVar(name, type);*/

    ret tempMapPutAll(knownVars, newVars);

  }

  S parseFunctionDefArg(Map<S, LASValueDescriptor> argsOut) {

    if (consumeOpt("(")) {

      S result = parseFunctionDefArg(argsOut);

      consume(")");

      ret result;

    }

    if (isIdentifier()) {

      S arg = tpp();

      var type = typeToValueDescriptor(parseColonTypeOpt());

      argsOut.put(arg, type);

      ret arg;

    }

    null;

  }

  Toolbox.FunctionDef parseFunctionDefinition(Map<S, FunctionDef> functionDefsToAddTo) {

    var start = ptr();

    consume("def");

    bool synthetic = consumeOpt("!");

    if (synthetic) consume("synthetic");

    S functionName = assertIdentifier(tpp());

    print("parseFunctionDefinition " + functionName);

    ret parseFunctionDefinition_step2(functionDefsToAddTo, start, functionName)

      .synthetic(synthetic);

  }

  Toolbox.FunctionDef parseFunctionDefinition_step2(Map<S, FunctionDef> functionDefsToAddTo, TokPtr start, S functionName) {

    // argument names and types

    new LinkedHashMap<S, LASValueDescriptor> args;

    while (parseFunctionDefArg(args) != null) {}

    var fd = new Toolbox.FunctionDef(functionName, keysList(args), null); // no body yet

    if (is(".", ".", ".")) {

      consume(3);

      fd.varargs(true);

    }

    var returnType = parseColonTypeOpt();

    fd.argTypes(valuesArray(LASValueDescriptor, args));

    functionDefsToAddTo?.put(functionName, fd);

    //var scope = newScope();

    //scope.useFixedVars(useFixedVarContexts);

    //temp tempScope(scope);

    temp tempAddKnownVars(args);

    print("Parsing function body");

    fd.body = parseReturnableCurlyBlock();

    print("Defined function " + functionName + ", added to " + functionDefsToAddTo);

    //fd.scope(scope);

    if (returnType != null) fd.returnType(returnType);

    ret linkToSrc(start, fd);

  }

  /*LASScope newScope() {

    ret new LASScope(scope);

  }*/

  Scope currentScope() { ret scope; }

  // enter the scope temporarily

  AutoCloseable tempScope(Scope scope) {

    var oldScope = currentScope();

    this.scope = scope;

    ret -> { this.scope = oldScope; };

  }

  Script parseReturnableCurlyBlock() {

    ret (Script) parseCurlyBlock(new BuildingScript().returnable(true));

  }

  Evaluable parseCurlyBlock(BuildingScript script default new) {

    //print(+knownVars);

    consume("{");

    bool inParensOld = inParens;

    inParens = false;

    var body = parseScript(script);

    consume("}");

    inParens = inParensOld;

    ret body;

  }

  Evaluable parseWhileLoop() {

    consume("while");

    var condition = parseExpr();

    var body = parseCurlyBlock(new BuildingScript().isLoopBody(true));

    ret new Toolbox.While(condition, body);

  }

  Evaluable parseDoWhileLoop() {

    consume("do");

    var body = parseCurlyBlock(new BuildingScript().isLoopBody(true));

    consume("while");

    var condition = parseExpr();

    ret new Toolbox.DoWhile(condition, body);

  }

  Evaluable parseForEach() {

    ret new ParseForEach()!;

  }

  class ParseForEach {

    Evaluable collection, body;

    IF0<Evaluable> finish;

    new L<TypedVar> vars;

    TypedVar addVar(TypedVar var) { ret addAndReturn(vars, var); }

    TypedVar consumeVar() {

      S name = consumeIdentifier();

      var type = parseColonTypeOpt();

      ret addVar(new TypedVar(name, typeToValueDescriptor(type)));

    }

    void parseBody {

      temp tempAddKnownTypedVars(vars);

      body = parseCurlyBlock(new BuildingScript().isLoopBody(true));

    }

    Evaluable get() {

      consume("for");

      // for i to n { ... }

      if (is(1, "to")) {

        TypedVar var = consumeVar();

        consume("to");

        Evaluable endValue = parseExpr();

        parseBody();

        ret new Toolbox.ForIntTo(endValue, var.name, body);

      }

      int iIn = relativeIndexOf("in");

      if (iIn < 0) fail("for without in");

      print(+iIn);

      if (iIn == 1 || is(1, ":")) { // just a variable (with optional type)

        TypedVar var = consumeVar();

        finish = -> new Toolbox.ForEach(collection, var.name, body).varType(var.type);

      } else if (iIn == 2) { // for iterator (mapI)

        if (consumeOpt("iterator")) {

          TypedVar var = consumeVar();

          finish = -> new Toolbox.ForIterator(collection, var.name, body);

        } else if (consumeOpt("nested")) {

          TypedVar var = consumeVar();

          finish = -> new Toolbox.ForNested(collection, var.name, body);

        } else

          fail("Unknown pattern for 'for' loop");

      } else if (iIn == 3) {

        if (isOneOf("pair", "Pair")) {

          // "pair a b" or "Pair a b"

          consume();

          TypedVar varA = consumeVar();

          TypedVar varB = consumeVar();

          printVars(+varA, +varB);

          finish = -> new Toolbox.ForPairs(collection, body, varA.name, varB.name);

        } else {

          // "a, b"

          TypedVar varA = consumeVar();

          consume(",");

          TypedVar varB = consumeVar();

          finish = -> new Toolbox.ForKeyValue(collection, body, varA.name, varB.name);

        }

      } else if (iIn == 4) {

        consume("index");

        S varIndex = consumeVar().name;

        consume(",");

        S varElement = consumeVar().name;

        finish = -> new Toolbox.ForIndex(collection, body, varIndex, varElement);

      } else

        fail("Unknown pattern for 'for' loop");

      consume("in");

      collection = parseExpr_inParens();

      print(+collection);

      parseBody();

      ret finish!;

    }

  }

  Evaluable parseIfStatement() {

    consume("if");

    Evaluable condition, body, elseBranch = null;

    {

      temp tempAdd(closerTokens, "then");

      condition = parseExpr();

    }

    if (consumeOpt("then")) {

      temp tempAdd(closerTokens, "else");

      body = parseExpr();

      if (consumeOpt("else"))

        elseBranch = parseExpr();

    } else {

      body = parseCurlyBlock();

      if (consumeOpt("else")) {

        if (is("if"))