unstructure (v19, LIVE) [1035484]

scope unstructure

// TODO: cyclic structures involving certain lists & sets

ifdef UseEclipseCollections
!include once #1027304 // Eclipse Collections
endifdef

sO unstructure(String text) {
  ret unstructure(text, false);
}

sO unstructure(S text, bool allDynamic) {
  ret unstructure(text, allDynamic, null);
}

sO unstructure(S text, IF1<S, Class> classFinder) {
  ret unstructure(text, false, classFinder);
}

static int structure_internStringsLongerThan = 50;
static int unstructure_unquoteBufSize = 100;

static int unstructure_tokrefs; // stats

abstract sclass unstructure_Receiver {
  abstract void set(O o);
}

// classFinder: func(name) -> class (optional)
static Object unstructure(String text, boolean allDynamic,
  O classFinder) {
  if (text == null) ret null;
  ret unstructure_tok(javaTokC_noMLS_iterator(text), allDynamic, classFinder);
}

static O unstructure_reader(BufferedReader reader) {
  ret unstructure_tok(javaTokC_noMLS_onReader(reader), false, null);
}

interface #Handler {
  void parse(int refID, int tokIndex, unstructure_Receiver out);
}
  
static O unstructure_tok(final Producer<S> tok, final boolean allDynamic, final O _classFinder) {
  final boolean debug = unstructure_debug;
  
  final class X {
    int i = -1;
    final O classFinder = _classFinder != null ? _classFinder : _defaultClassFinder();
    S mcDollar = actualMCDollar();

    // use Eclipse primitive collection if possible (smaller & hopefully faster?)
    ifdef UseEclipseCollections
    new IntObjectHashMap<O> refs;
    new IntObjectHashMap<O> tokrefs;
    endifdef
    ifndef UseEclipseCollections
    new HashMap<Integer, O> refs;
    new HashMap<Integer, O> tokrefs;
    endifndef
    
    new HashSet<S> concepts;
    new L<Runnable> stack;
    new SS baseClassMap;
    new HashMap<Class, Constructor> innerClassConstructors;
    S curT;
    char[] unquoteBuf = new char[unstructure_unquoteBufSize];
    
    // value is a class or a Handler
    final new HashMap<S, O> handlers;
    
    *() {
      pcall {
        Class mc = cast callF(_classFinder, "<main>");
        if (mc != null) mcDollar = mc.getName() + "$";
      }
      
      makeHandlers();
    }
    
    replace HandlerArgs with (Handler) (refID, tokIndex, out).
    
    void makeHandlers {
      Handler h;
      
      handlers.put("bigint", HandlerArgs
        -> out.set(parseBigInt()));
        
      handlers.put("d", HandlerArgs
        -> out.set(parseDouble()));
        
      handlers.put("fl", HandlerArgs
        -> out.set(parseFloat()));
        
      handlers.put("sh", HandlerArgs -> {
        consume();
        S t = tpp();
        if (t.equals("-")) {
          t = tpp();
          out.set((short) (-parseInt(t)); ret;
        }
        out.set((short) parseInt(t));
      });
      
      handlers.put("enum", HandlerArgs -> {
        consume();
        S t = tpp();
        assertTrue(isJavaIdentifier(t));
        S fullClassName = mcDollar + t;
        Class _c = findAClass(fullClassName);
        if (_c == null) fail("Enum class not found: " + fullClassName);
        int ordinal = parseInt(tpp());
        out.set(_c.getEnumConstants()[ordinal]);
      });

      handlers.put("false", h = HandlerArgs -> {
        consume(); out.set(false);
      });
      handlers.put("f", h);
      
      handlers.put("true", h = HandlerArgs -> {
        consume(); out.set(true);
      });
      handlers.put("t", h);
      
      handlers.put("{", HandlerArgs -> parseMap(out));
      
      handlers.put("[", HandlerArgs -> {
        new ArrayList l;
        if (refID >= 0) refs.put(refID, l);
        this.parseList(l, out);
      });
      
      handlers.put("bitset", HandlerArgs -> parseBitSet(out));
      
      handlers.put("array", h = HandlerArgs -> parseArray(out));
      handlers.put("intarray", h);
      handlers.put("dblarray", h);
      
    } // end of makeHandlers - add more handlers here

    Class findAClass(S fullClassName) null on exception {
      ret classFinder != null ? (Class) callF(classFinder, fullClassName) : findClass_fullName(fullClassName);
    }
    
    S unquote(S s) {
      ret unquoteUsingCharArray(s, unquoteBuf); 
    }

    // look at current token
    S t() {
      ret curT;
    }
    
    // get current token, move to next
    S tpp() {
      S t = curT;
      consume();
      ret t;
    }
    
    void parse(final unstructure_Receiver out) {
      S t = t();
      
      int refID;
      if (structure_isMarker(t, 0, l(t))) {
        refID = parseInt(t.substring(1));
        consume();
      } else refID = -1;
      
      // if (debug) print("parse: " + quote(t));
      
      final int tokIndex = i;  
      parse_inner(refID, tokIndex, new unstructure_Receiver {
        void set(O o) {
          if (refID >= 0)
            refs.put(refID, o);
          if (o != null)
            tokrefs.put(tokIndex, o);
          out.set(o);
        }
      });
    }
    
    void parse_inner(int refID, int tokIndex, unstructure_Receiver out) {
      S t = t();
      
      // if (debug) print("parse_inner: " + quote(t));
      
      O handler = handlers.get(t);
      if (handler cast Handler)
        ret with handler.parse(refID, tokIndex, out);
        
      Class c = cast handler;
      if (c == null) {
        if (t.startsWith("\"")) {
          S s = internIfLongerThan(unquote(tpp()), structure_internStringsLongerThan);
          out.set(s); ret;
        }
        
        if (t.startsWith("'")) {
          out.set(unquoteCharacter(tpp())); ret;
        }
        if (t.equals("-")) {
          consume();
          t = tpp();
          out.set(isLongConstant(t) ? (O) (-parseLong(t)) : (O) (-parseInt(t))); ret;
        }
        if (isInteger(t) || isLongConstant(t)) {
          consume();
          //if (debug) print("isLongConstant " + quote(t) + " => " + isLongConstant(t));
          if (isLongConstant(t)) {
            out.set(parseLong(t)); ret;
          }
          long l = parseLong(t);
          bool isInt = l == (int) l;
          ifdef unstructure_debug
            print("l=" + l + ", isInt: " + isInt);
          endifdef
          out.set(isInt ? (O) Integer.valueOf((int) l) : (O) Long.valueOf(l)); ret;
        }
        if (t.equals("-")) {
          consume();
          t = tpp();
          out.set(isLongConstant(t) ? (O) (-parseLong(t)) : (O) (-parseInt(t))); ret;
        }
        if (isInteger(t) || isLongConstant(t)) {
          consume();
          //if (debug) print("isLongConstant " + quote(t) + " => " + isLongConstant(t));
          if (isLongConstant(t)) {
            out.set(parseLong(t)); ret;
          }
          long l = parseLong(t);
          bool isInt = l == (int) l;
          ifdef unstructure_debug
            print("l=" + l + ", isInt: " + isInt);
          endifdef
          out.set(isInt ? (O) Integer.valueOf((int) l) : (O) Long.valueOf(l)); ret;
        }
        
        if (t.equals("File")) {
          consume();
          File f = new File(unquote(tpp()));
          out.set(f); ret;
        }
        
        if (t.startsWith("r") && isInteger(t.substring(1))) {
          consume();
          int ref = Integer.parseInt(t.substring(1));
          O o = refs.get(ref);
          if (o == null)
            warn("unsatisfied back reference " + ref);
          out.set(o); ret;
        }
      
        if (t.startsWith("t") && isInteger(t.substring(1))) {
          consume();
          int ref = Integer.parseInt(t.substring(1));
          O o = tokrefs.get(ref);
          if (o == null)
            warn("unsatisfied token reference " + ref + " at " + tokIndex);
          out.set(o); ret;
        }
        
        if (t.equals("hashset")) ret with parseHashSet(out);
        if (t.equals("lhs")) ret with parseLinkedHashSet(out);
        if (t.equals("treeset")) ret with parseTreeSet(out);
        if (t.equals("ciset")) ret with parseCISet(out);
        
        if (eqOneOf(t, "hashmap", "hm")) {
          consume();
          parseMap(new HashMap, out);
          ret;
        }
        if (t.equals("lhm")) {
          consume();
          parseMap(new LinkedHashMap, out);
          ret;
        }
        if (t.equals("tm")) {
          consume();
          parseMap(new TreeMap, out);
          ret;
        }
        if (t.equals("cimap")) {
          consume();
          parseMap(ciMap(), out);
          ret;
        }
        
        if (t.equals("ll")) {
          consume();
          new LinkedList l;
          if (refID >= 0) refs.put(refID, l);
          ret with parseList(l, out);
        }

        if (t.equals("syncLL")) { // legacy
          consume();
          ret with parseList(synchroLinkedList(), out);
        }

        if (t.equals("sync")) {
          consume();
          ret with parse(new unstructure_Receiver {
            void set(O value) {
              if (value instanceof Map) {
                ifndef Android // Java 7
                if (value cast NavigableMap)
                  ret with out.set(synchroNavigableMap(value);
                endifndef
                if (value cast SortedMap)
                  ret with out.set(synchroSortedMap(value);
                ret with out.set(synchroMap((Map) value));
              } else
                ret with out.set(synchroList((L) value);
            }
          });
        }
        
        if (t.equals("ba")) {
          consume();
          S hex = unquote(tpp());
          out.set(hexToBytes(hex)); ret;
        }
        if (t.equals("boolarray")) {
          consume();
          int n = parseInt(tpp());
          S hex = unquote(tpp());
          out.set(boolArrayFromBytes(hexToBytes(hex), n)); ret;
        }
        if (t.equals("class")) {
          out.set(parseClass()); ret;
        }
        if (t.equals("l")) {
          parseLisp(out); ret;
        }
        if (t.equals("null")) {
          consume(); out.set(null); ret;
        }
        
        if (eq(t, "c")) {
          consume();
          t = t();
          assertTrue(isJavaIdentifier(t));
          concepts.add(t);
        }
        
        // custom deserialization (new static method method)
        if (eq(t, "cu")) {
          consume();
          t = tpp();
          assertTrue(isJavaIdentifier(t));
          S fullClassName = mcDollar + t;
          Class _c = findAClass(fullClassName);
          if (_c == null) fail("Class not found: " + fullClassName);
          parse(new unstructure_Receiver {
            void set(O value) {
              ifdef unstructure_debug
                print("Consumed custom object, next token: " + t());
              endifdef
              out.set(call(_c, "_deserialize", value);
            }
          });
          ret;
        }
      }
      
      if (eq(t, "j")) {
        consume();
        out.set(parseJava()); ret;
      }
      
      if (eq(t, "bc")) {
        consume();
        S c1 = tpp();
        S c2 = tpp();
        baseClassMap.put(c1, c2);
        ret with parse_inner(refID, i, out);
      }
      
      // add more tokens here

      // Now we want to find our target class c
      // Have we failed to look up the class before?
      //bool seenBefore = handlers.containsKey(cname);

      // If we have seen the class before, we skip all of this
      // and simply leave c as null
      // TODO - how do we fill className?
      //if (!seenBefore) {
        if (c == null && !isJavaIdentifier(t))
          throw new RuntimeException("Unknown token " + (i+1) + ": " + quote(t));
          
        // any other class name (or package name)
        consume();
        S className, fullClassName;
        
        // Is it a package name?
        if (eq(t(), ".")) {
          className = t;
          do {
            consume();
            className += "." + assertIdentifier(tpp());
          } while (eq(t(), "."));
          fullClassName = className;
        } else {
          className = t;
          fullClassName = mcDollar + t;
        }
        
        if (c == null && !allDynamic) {
          // First, find class
          c = findAClass(fullClassName);
          handlers.put(className, c);
        }
        
        // check for existing base class
        if (c == null && !allDynamic) {
          new Set<S> seen;
          S parent = className;
          while true {
            S baseName = baseClassMap.get(parent);
            if (baseName == null)
              break;
            if (!seen.add(baseName))
              fail("Cyclic superclass info: " + baseName);
            c = findAClass(mcDollar + baseName);
            if (c == null)
              print("Base class " + baseName + " of " + parent +  " doesn't exist either");
            else if (isAbstract(c))
              print("Can't instantiate abstract base class: " + c);
            else {
              printVars_str("Reverting to base class", +className, +baseName, +c);
              handlers.put(className, c);
              break;
            }
            parent = baseName;
          }
        }
      //}
          
      // Check if it has an outer reference
      bool hasBracket = eq(t(), "(");
      if (hasBracket) consume();
      bool hasOuter = hasBracket && startsWith(t(), "this$");
      
      DynamicObject dO = null;
      O o = null;
      fS thingName = t;
      pcall {
        if (c != null) {
          if (hasOuter) try {
            Constructor ctor = innerClassConstructors.get(c);
            if (ctor == null)
              innerClassConstructors.put(c, ctor = nuStubInnerObject_findConstructor(c, classFinder));
            o = ctor.newInstance(new O[] {null});
          } catch Exception e {
            print("Error deserializing " + c + ": " + e);
            o = nuEmptyObject(c);
          } else
            o = nuEmptyObject(c);
          if (o instanceof DynamicObject) dO = (DynamicObject) o;
        } else {
          if (concepts.contains(t) && (c = findAClass(mcDollar + "Concept")) != null)
            o = dO = (DynamicObject) nuEmptyObject(c);
          else
            dO = new DynamicObject;
          dO.className = className;
          ifdef unstructure_debug
            print("Made dynamic object " + t + " " + shortClassName(dO));
          endifdef
        }
      } // end of pcall
      
      // Creating instance failed? Use DynamicObject
      if (o == null && dO == null)
        dO = new DynamicObject;
      
      // Save in references list early because contents of object
      // might link back to main object
      
      if (refID >= 0)
        refs.put(refID, o != null ? o : dO);
      tokrefs.put(tokIndex, o != null ? o : dO);
      
      // NOW parse the fields!
      
      new /*Linked*/HashMap<S, O> fields; // no longer preserving order (why did we do this?)
      O _o = o;
      DynamicObject _dO = dO;
      if (hasBracket) {
        stack.add(r {
          ifdef unstructure_debug
            print("in object values, token: " + t());
          endifdef
          if (eq(t(), ",")) consume();
          if (eq(t(), ")")) {
            consume(")");
            objRead(_o, _dO, fields, hasOuter);
            out.set(_o != null ? _o : _dO);
          } else {
            final S key = unquote(tpp());
            S t = tpp();
            if (!eq(t, "="))
              fail("= expected, got " + t + " after " + quote(key) + " in object " + thingName /*+ " " + sfu(fields)*/);
            stack.add(this);
            parse(new unstructure_Receiver {
              void set(O value) {
                fields.put(key, value);
                /*ifdef unstructure_debug
                  print("Got field value " + value + ", next token: " + t());
                endifdef*/
                //if (eq(t(), ",")) consume();
              }
            });
          }
        });
      } else {
        objRead(o, dO, fields, hasOuter);
        out.set(o != null ? o : dO);
      }
    }
    
    void objRead(O o, DynamicObject dO, MapSO fields, bool hasOuter) {
      ifdef unstructure_debug
      print("objRead " + className(o) + " " + className(dO) + " " + struct(fields));
      endifdef
      
      // translate between diferent compilers (this$0 vs this$1)
      O outer = fields.get("this$0");
      if (outer != null) fields.put("this$1", outer);
      else {
        outer = fields.get("this$1");
        if (outer != null) fields.put("this$0", outer);
      }
      
      if (o != null) {
        if (dO != null) {
          ifdef unstructure_debug
            printStructure("setOptAllDyn", fields);
          endifdef
          setOptAllDyn_pcall(dO, fields);
        } else {
          setOptAll_pcall(o, fields);
          ifdef unstructure_debug
            print("objRead now: " + struct(o));
          endifdef
        }
        if (hasOuter)
          fixOuterRefs(o);
      } else for (Map.Entry<S, O> e : fields.entrySet())
        setDynObjectValue(dO, intern(e.getKey()), e.getValue());

      if (o != null)
        pcallOpt_noArgs(o, "_doneLoading");
    }
    
    void parseSet(final Set set, final unstructure_Receiver out) {
      this.parseList(new ArrayList, new unstructure_Receiver {
        void set(O o) {
          set.addAll((L) o);
          out.set(set);
        }
      });
    }
    
    void parseLisp(final unstructure_Receiver out) {
      ifclass Lisp
        consume("l");
        consume("(");
        final new ArrayList list;
        stack.add(r {
          if (eq(t(), ")")) {
            consume(")");
            out.set(Lisp((S) list.get(0), subList(list, 1)));
          } else {
            stack.add(this);
            parse(new unstructure_Receiver {
              void set(O o) {
                list.add(o);
                if (eq(t(), ",")) consume();
              }
            });
          }
        });
        if (false) // skip fail line
      endif
      
      fail("class Lisp not included");
    }
    
    void parseBitSet(final unstructure_Receiver out) {
      consume("bitset");
      consume("{");
      final new BitSet bs;
      stack.add(r {
        if (eq(t(), "}")) {
          consume("}");
          out.set(bs);
        } else {
          stack.add(this);
          parse(new unstructure_Receiver {
            void set(O o) {
              bs.set((Integer) o);
              if (eq(t(), ",")) consume();
            }
          });
        }
      });
    }
    
    void parseList(final L list, final unstructure_Receiver out) {
      tokrefs.put(i, list);
      consume("[");
      stack.add(r {
        if (eq(t(), "]")) {
          consume();
          ifdef unstructure_debug
            print("Consumed list, next token: " + t());
          endifdef
          out.set(list);
        } else {
          stack.add(this);
          parse(new unstructure_Receiver {
            void set(O o) {
              //if (debug) print("List element type: " + getClassName(o));
              list.add(o);
              if (eq(t(), ",")) consume();
            }
          });
        }
      });
    }
    
    void parseArray(unstructure_Receiver out) {
      S _type = tpp();
      int dims;

      if (eq(t(), "S")) { // string array
        _type = "S";
        consume();
      }
      
      if (eq(t(), "/")) { // multi-dimensional array
        consume();
        dims = parseInt(tpp());
      } else
        dims = 1;
      
      consume("{");
      List list = new ArrayList;
      S type = _type;
      
      stack.add(r {
        if (eq(t(), "}")) {
          consume("}");
          if (dims > 1) {
            Class atype;
            if (type.equals("intarray")) atype = int.class;
            else if (type.equals("S")) atype = S.class;
            else todo("multi-dimensional arrays of other types");
            
            out.set(list.toArray((O[]) newMultiDimensionalOuterArray(atype, dims, l(list))));
          } else
            out.set(
              type.equals("intarray") ? toIntArray(list)
              : type.equals("dblarray") ? toDoubleArray(list)
              : type.equals("S") ? toStringArray(list)
              : list.toArray());
        } else {
          stack.add(this);
          parse(new unstructure_Receiver {
            void set(O o) {
              list.add(o);
              if (eq(t(), ",")) consume();
            }
          });
        }
      });
    }
    
    Object parseClass() {
      consume("class");
      consume("(");
      S name = unquote(tpp());
      consume(")");
      Class c = allDynamic ? null : findAClass(name);
      if (c != null) ret c;
      new DynamicObject dO;
      dO.className = "java.lang.Class";
      name = dropPrefix(mcDollar, name);
      dO.fieldValues.put("name", name);
      ret dO;
    }
    
    Object parseBigInt() {
      consume("bigint");
      consume("(");
      S val = tpp();
      if (eq(val, "-"))
        val = "-" + tpp();
      consume(")");
      ret new BigInteger(val);
    }
    
    Object parseDouble() {
      consume("d");
      consume("(");
      S val = unquote(tpp());
      consume(")");
      ret Double.parseDouble(val);
    }
    
    Object parseFloat() {
      consume("fl");
      S val;
      if (eq(t(), "(")) {
        consume("(");
        val = unquote(tpp());
        consume(")");
      } else {
        val = unquote(tpp());
      }
      ret Float.parseFloat(val);
    }
    
    void parseHashSet(unstructure_Receiver out) {
      consume("hashset");
      parseSet(new HashSet, out);
    }
    
    void parseLinkedHashSet(unstructure_Receiver out) {
      consume("lhs");
      parseSet(new LinkedHashSet, out);
    }
    
    void parseTreeSet(unstructure_Receiver out) {
      consume("treeset");
      parseSet(new TreeSet, out);
    }
    
    void parseCISet(unstructure_Receiver out) {
      consume("ciset");
      parseSet(ciSet(), out);
    }
    
    void parseMap(unstructure_Receiver out) {
      parseMap(new TreeMap, out);
    }
    
    O parseJava() {
      S j = unquote(tpp());
      new Matches m;
      if (jmatch("java.awt.Color[r=*,g=*,b=*]", j, m))
        ret nuObject("java.awt.Color", parseInt($1), parseInt($2), parseInt($3));
      else {
        warn("Unknown Java object: " + j);
        null;
      }
    }
    
    void parseMap(final Map map, final unstructure_Receiver out) {
      consume("{");
      stack.add(new Runnable {
        bool v;
        O key;
        
        public void run() { 
          if (v) {
            v = false;
            stack.add(this);
            if (!eq(tpp(), "="))
              fail("= expected, got " + t() + " in map of size " + l(map));

            parse(new unstructure_Receiver {
              void set(O value) {
                map.put(key, value);
                ifdef unstructure_debug
                  print("parseMap: Got value " + getClassName(value) + ", next token: " + quote(t()));
                endifdef
                if (eq(t(), ",")) consume();
              }
            });
          } else {
            if (eq(t(), "}")) {
              consume("}");
              out.set(map);
            } else {
              v = true;
              stack.add(this);
              parse(new unstructure_Receiver {
                void set(O o) {
                  key = o;
                }
              });
            }
          } // if v else
        } // run()
      });
    }
    
    /*void parseSub(unstructure_Receiver out) {
      int n = l(stack);
      parse(out);
      while (l(stack) > n)
        stack
    }*/
    
    void consume() { curT = tok.next(); ++i; }
    
    void consume(S s) {
      if (!eq(t(), s)) {
        /*S prevToken = i-1 >= 0 ? tok.get(i-1) : "";
        S nextTokens = join(tok.subList(i, Math.min(i+2, tok.size())));
        fail(quote(s) + " expected: " + prevToken + " " + nextTokens + " (" + i + "/" + tok.size() + ")");*/
        fail(quote(s) + " expected, got " + quote(t()));
      }
      consume();
    }
    
    // outer wrapper function getting first token and unwinding the stack
    void parse_initial(unstructure_Receiver out) {
      consume(); // get first token
      parse(out);
      while (nempty(stack))
        popLast(stack).run();
    }
  }
  
  ThreadLocal<Bool> tlLoading = dynamicObjectIsLoading_threadLocal();
  Bool b = tlLoading!;
  tlLoading.set(true);
  try {
    final new Var v;
    new X x;
    x.parse_initial(new unstructure_Receiver {
      void set(O o) { v.set(o); }
    });
    unstructure_tokrefs = x.tokrefs.size();
    ret v.get();
  } finally {
    tlLoading.set(b);
  }
}

static boolean unstructure_debug;

Snippet ID:	#1035484
Snippet name:	unstructure (v19, LIVE)
Eternal ID of this version:	#1035484/4
Text MD5:	a48da68b32e358f26d4c119f022b0ab2
Transpilation MD5:	b2cf123a4dde8763bc74e600bb02a5cb
Author:	stefan
Category:	javax
Type:	JavaX fragment (include)
Public (visible to everyone):	Yes
Archived (hidden from active list):	No
Created/modified:	2022-07-29 20:02:33
Source code size:	25342 bytes / 848 lines
Pitched / IR pitched:	No / No
Views / Downloads:	338 / 418
Version history:	3 change(s)
Referenced in:	[show references]

< > BotCompany Repo | #1035484 // unstructure (v19, LIVE)

JavaX fragment (include) [tags: use-pretranspiled]

Author comment

1	scope unstructure
2
3	// TODO: cyclic structures involving certain lists & sets
4
5	ifdef UseEclipseCollections
6	!include once #1027304 // Eclipse Collections
7	endifdef
8
9	sO unstructure(String text) {
10	ret unstructure(text, false);
11	}
12
13	sO unstructure(S text, bool allDynamic) {
14	ret unstructure(text, allDynamic, null);
15	}
16
17	sO unstructure(S text, IF1<S, Class> classFinder) {
18	ret unstructure(text, false, classFinder);
19	}
20
21	static int structure_internStringsLongerThan = 50;
22	static int unstructure_unquoteBufSize = 100;
23
24	static int unstructure_tokrefs; // stats
25
26	abstract sclass unstructure_Receiver {
27	abstract void set(O o);
28	}
29
30	// classFinder: func(name) -> class (optional)
31	static Object unstructure(String text, boolean allDynamic,
32	O classFinder) {
33	if (text == null) ret null;
34	ret unstructure_tok(javaTokC_noMLS_iterator(text), allDynamic, classFinder);
35	}
36
37	static O unstructure_reader(BufferedReader reader) {
38	ret unstructure_tok(javaTokC_noMLS_onReader(reader), false, null);
39	}
40
41	interface #Handler {
42	void parse(int refID, int tokIndex, unstructure_Receiver out);
43	}
44
45	static O unstructure_tok(final Producer<S> tok, final boolean allDynamic, final O _classFinder) {
46	final boolean debug = unstructure_debug;
47
48	final class X {
49	int i = -1;
50	final O classFinder = _classFinder != null ? _classFinder : _defaultClassFinder();
51	S mcDollar = actualMCDollar();
52
53	// use Eclipse primitive collection if possible (smaller & hopefully faster?)
54	ifdef UseEclipseCollections
55	new IntObjectHashMap<O> refs;
56	new IntObjectHashMap<O> tokrefs;
57	endifdef
58	ifndef UseEclipseCollections
59	new HashMap<Integer, O> refs;
60	new HashMap<Integer, O> tokrefs;
61	endifndef
62
63	new HashSet<S> concepts;
64	new L<Runnable> stack;
65	new SS baseClassMap;
66	new HashMap<Class, Constructor> innerClassConstructors;
67	S curT;
68	char[] unquoteBuf = new char[unstructure_unquoteBufSize];
69
70	// value is a class or a Handler
71	final new HashMap<S, O> handlers;
72
73	*() {
74	pcall {
75	Class mc = cast callF(_classFinder, "<main>");
76	if (mc != null) mcDollar = mc.getName() + "$";
77	}
78
79	makeHandlers();
80	}
81
82	replace HandlerArgs with (Handler) (refID, tokIndex, out).
83
84	void makeHandlers {
85	Handler h;
86
87	handlers.put("bigint", HandlerArgs
88	-> out.set(parseBigInt()));
89
90	handlers.put("d", HandlerArgs
91	-> out.set(parseDouble()));
92
93	handlers.put("fl", HandlerArgs
94	-> out.set(parseFloat()));
95
96	handlers.put("sh", HandlerArgs -> {
97	consume();
98	S t = tpp();
99	if (t.equals("-")) {
100	t = tpp();
101	out.set((short) (-parseInt(t)); ret;
102	}
103	out.set((short) parseInt(t));
104	});
105
106	handlers.put("enum", HandlerArgs -> {
107	consume();
108	S t = tpp();
109	assertTrue(isJavaIdentifier(t));
110	S fullClassName = mcDollar + t;
111	Class _c = findAClass(fullClassName);
112	if (_c == null) fail("Enum class not found: " + fullClassName);
113	int ordinal = parseInt(tpp());
114	out.set(_c.getEnumConstants()[ordinal]);
115	});
116
117	handlers.put("false", h = HandlerArgs -> {
118	consume(); out.set(false);
119	});
120	handlers.put("f", h);
121
122	handlers.put("true", h = HandlerArgs -> {
123	consume(); out.set(true);
124	});
125	handlers.put("t", h);
126
127	handlers.put("{", HandlerArgs -> parseMap(out));
128
129	handlers.put("[", HandlerArgs -> {
130	new ArrayList l;
131	if (refID >= 0) refs.put(refID, l);
132	this.parseList(l, out);
133	});
134
135	handlers.put("bitset", HandlerArgs -> parseBitSet(out));
136
137	handlers.put("array", h = HandlerArgs -> parseArray(out));
138	handlers.put("intarray", h);
139	handlers.put("dblarray", h);
140
141	} // end of makeHandlers - add more handlers here
142
143	Class findAClass(S fullClassName) null on exception {
144	ret classFinder != null ? (Class) callF(classFinder, fullClassName) : findClass_fullName(fullClassName);
145	}
146
147	S unquote(S s) {
148	ret unquoteUsingCharArray(s, unquoteBuf);
149	}
150
151	// look at current token
152	S t() {
153	ret curT;
154	}
155
156	// get current token, move to next
157	S tpp() {
158	S t = curT;
159	consume();
160	ret t;
161	}
162
163	void parse(final unstructure_Receiver out) {
164	S t = t();
165
166	int refID;
167	if (structure_isMarker(t, 0, l(t))) {
168	refID = parseInt(t.substring(1));
169	consume();
170	} else refID = -1;
171
172	// if (debug) print("parse: " + quote(t));
173
174	final int tokIndex = i;
175	parse_inner(refID, tokIndex, new unstructure_Receiver {
176	void set(O o) {
177	if (refID >= 0)
178	refs.put(refID, o);
179	if (o != null)
180	tokrefs.put(tokIndex, o);
181	out.set(o);
182	}
183	});
184	}
185
186	void parse_inner(int refID, int tokIndex, unstructure_Receiver out) {
187	S t = t();
188
189	// if (debug) print("parse_inner: " + quote(t));
190
191	O handler = handlers.get(t);
192	if (handler cast Handler)
193	ret with handler.parse(refID, tokIndex, out);
194
195	Class c = cast handler;
196	if (c == null) {
197	if (t.startsWith("\"")) {
198	S s = internIfLongerThan(unquote(tpp()), structure_internStringsLongerThan);
199	out.set(s); ret;
200	}
201
202	if (t.startsWith("'")) {
203	out.set(unquoteCharacter(tpp())); ret;
204	}
205	if (t.equals("-")) {
206	consume();
207	t = tpp();
208	out.set(isLongConstant(t) ? (O) (-parseLong(t)) : (O) (-parseInt(t))); ret;
209	}
210	if (isInteger(t) \|\| isLongConstant(t)) {
211	consume();
212	//if (debug) print("isLongConstant " + quote(t) + " => " + isLongConstant(t));
213	if (isLongConstant(t)) {
214	out.set(parseLong(t)); ret;
215	}
216	long l = parseLong(t);
217	bool isInt = l == (int) l;
218	ifdef unstructure_debug
219	print("l=" + l + ", isInt: " + isInt);
220	endifdef
221	out.set(isInt ? (O) Integer.valueOf((int) l) : (O) Long.valueOf(l)); ret;
222	}
223	if (t.equals("-")) {
224	consume();
225	t = tpp();
226	out.set(isLongConstant(t) ? (O) (-parseLong(t)) : (O) (-parseInt(t))); ret;
227	}
228	if (isInteger(t) \|\| isLongConstant(t)) {
229	consume();
230	//if (debug) print("isLongConstant " + quote(t) + " => " + isLongConstant(t));
231	if (isLongConstant(t)) {
232	out.set(parseLong(t)); ret;
233	}
234	long l = parseLong(t);
235	bool isInt = l == (int) l;
236	ifdef unstructure_debug
237	print("l=" + l + ", isInt: " + isInt);
238	endifdef
239	out.set(isInt ? (O) Integer.valueOf((int) l) : (O) Long.valueOf(l)); ret;
240	}
241
242	if (t.equals("File")) {
243	consume();
244	File f = new File(unquote(tpp()));
245	out.set(f); ret;
246	}
247
248	if (t.startsWith("r") && isInteger(t.substring(1))) {
249	consume();
250	int ref = Integer.parseInt(t.substring(1));
251	O o = refs.get(ref);
252	if (o == null)
253	warn("unsatisfied back reference " + ref);
254	out.set(o); ret;
255	}
256
257	if (t.startsWith("t") && isInteger(t.substring(1))) {
258	consume();
259	int ref = Integer.parseInt(t.substring(1));
260	O o = tokrefs.get(ref);
261	if (o == null)
262	warn("unsatisfied token reference " + ref + " at " + tokIndex);
263	out.set(o); ret;
264	}
265
266	if (t.equals("hashset")) ret with parseHashSet(out);
267	if (t.equals("lhs")) ret with parseLinkedHashSet(out);
268	if (t.equals("treeset")) ret with parseTreeSet(out);
269	if (t.equals("ciset")) ret with parseCISet(out);
270
271	if (eqOneOf(t, "hashmap", "hm")) {
272	consume();
273	parseMap(new HashMap, out);
274	ret;
275	}
276	if (t.equals("lhm")) {
277	consume();
278	parseMap(new LinkedHashMap, out);
279	ret;
280	}
281	if (t.equals("tm")) {
282	consume();
283	parseMap(new TreeMap, out);
284	ret;
285	}
286	if (t.equals("cimap")) {
287	consume();
288	parseMap(ciMap(), out);
289	ret;
290	}
291
292	if (t.equals("ll")) {
293	consume();
294	new LinkedList l;
295	if (refID >= 0) refs.put(refID, l);
296	ret with parseList(l, out);
297	}
298
299	if (t.equals("syncLL")) { // legacy
300	consume();
301	ret with parseList(synchroLinkedList(), out);
302	}
303
304	if (t.equals("sync")) {
305	consume();
306	ret with parse(new unstructure_Receiver {
307	void set(O value) {
308	if (value instanceof Map) {
309	ifndef Android // Java 7
310	if (value cast NavigableMap)
311	ret with out.set(synchroNavigableMap(value);
312	endifndef
313	if (value cast SortedMap)
314	ret with out.set(synchroSortedMap(value);
315	ret with out.set(synchroMap((Map) value));
316	} else
317	ret with out.set(synchroList((L) value);
318	}
319	});
320	}
321
322	if (t.equals("ba")) {
323	consume();
324	S hex = unquote(tpp());
325	out.set(hexToBytes(hex)); ret;
326	}
327	if (t.equals("boolarray")) {
328	consume();
329	int n = parseInt(tpp());
330	S hex = unquote(tpp());
331	out.set(boolArrayFromBytes(hexToBytes(hex), n)); ret;
332	}
333	if (t.equals("class")) {
334	out.set(parseClass()); ret;
335	}
336	if (t.equals("l")) {
337	parseLisp(out); ret;
338	}
339	if (t.equals("null")) {
340	consume(); out.set(null); ret;
341	}
342
343	if (eq(t, "c")) {
344	consume();
345	t = t();
346	assertTrue(isJavaIdentifier(t));
347	concepts.add(t);
348	}
349
350	// custom deserialization (new static method method)
351	if (eq(t, "cu")) {
352	consume();
353	t = tpp();
354	assertTrue(isJavaIdentifier(t));
355	S fullClassName = mcDollar + t;
356	Class _c = findAClass(fullClassName);
357	if (_c == null) fail("Class not found: " + fullClassName);
358	parse(new unstructure_Receiver {
359	void set(O value) {
360	ifdef unstructure_debug
361	print("Consumed custom object, next token: " + t());
362	endifdef
363	out.set(call(_c, "_deserialize", value);
364	}
365	});
366	ret;
367	}
368	}
369
370	if (eq(t, "j")) {
371	consume();
372	out.set(parseJava()); ret;
373	}
374
375	if (eq(t, "bc")) {
376	consume();
377	S c1 = tpp();
378	S c2 = tpp();
379	baseClassMap.put(c1, c2);
380	ret with parse_inner(refID, i, out);
381	}
382
383	// add more tokens here
384
385	// Now we want to find our target class c
386	// Have we failed to look up the class before?
387	//bool seenBefore = handlers.containsKey(cname);
388
389	// If we have seen the class before, we skip all of this
390	// and simply leave c as null
391	// TODO - how do we fill className?
392	//if (!seenBefore) {
393	if (c == null && !isJavaIdentifier(t))
394	throw new RuntimeException("Unknown token " + (i+1) + ": " + quote(t));
395
396	// any other class name (or package name)
397	consume();
398	S className, fullClassName;
399
400	// Is it a package name?
401	if (eq(t(), ".")) {
402	className = t;
403	do {
404	consume();
405	className += "." + assertIdentifier(tpp());
406	} while (eq(t(), "."));
407	fullClassName = className;
408	} else {
409	className = t;
410	fullClassName = mcDollar + t;
411	}
412
413	if (c == null && !allDynamic) {
414	// First, find class
415	c = findAClass(fullClassName);
416	handlers.put(className, c);
417	}
418
419	// check for existing base class
420	if (c == null && !allDynamic) {
421	new Set<S> seen;
422	S parent = className;
423	while true {
424	S baseName = baseClassMap.get(parent);
425	if (baseName == null)
426	break;
427	if (!seen.add(baseName))
428	fail("Cyclic superclass info: " + baseName);
429	c = findAClass(mcDollar + baseName);
430	if (c == null)
431	print("Base class " + baseName + " of " + parent + " doesn't exist either");
432	else if (isAbstract(c))
433	print("Can't instantiate abstract base class: " + c);
434	else {
435	printVars_str("Reverting to base class", +className, +baseName, +c);
436	handlers.put(className, c);
437	break;
438	}
439	parent = baseName;
440	}
441	}
442	//}
443
444	// Check if it has an outer reference
445	bool hasBracket = eq(t(), "(");
446	if (hasBracket) consume();
447	bool hasOuter = hasBracket && startsWith(t(), "this$");
448
449	DynamicObject dO = null;
450	O o = null;
451	fS thingName = t;
452	pcall {
453	if (c != null) {
454	if (hasOuter) try {
455	Constructor ctor = innerClassConstructors.get(c);
456	if (ctor == null)
457	innerClassConstructors.put(c, ctor = nuStubInnerObject_findConstructor(c, classFinder));
458	o = ctor.newInstance(new O[] {null});
459	} catch Exception e {
460	print("Error deserializing " + c + ": " + e);
461	o = nuEmptyObject(c);
462	} else
463	o = nuEmptyObject(c);
464	if (o instanceof DynamicObject) dO = (DynamicObject) o;
465	} else {
466	if (concepts.contains(t) && (c = findAClass(mcDollar + "Concept")) != null)
467	o = dO = (DynamicObject) nuEmptyObject(c);
468	else
469	dO = new DynamicObject;
470	dO.className = className;
471	ifdef unstructure_debug
472	print("Made dynamic object " + t + " " + shortClassName(dO));
473	endifdef
474	}
475	} // end of pcall
476
477	// Creating instance failed? Use DynamicObject
478	if (o == null && dO == null)
479	dO = new DynamicObject;
480
481	// Save in references list early because contents of object
482	// might link back to main object
483
484	if (refID >= 0)
485	refs.put(refID, o != null ? o : dO);
486	tokrefs.put(tokIndex, o != null ? o : dO);
487
488	// NOW parse the fields!
489
490	new /Linked/HashMap<S, O> fields; // no longer preserving order (why did we do this?)
491	O _o = o;
492	DynamicObject _dO = dO;
493	if (hasBracket) {
494	stack.add(r {
495	ifdef unstructure_debug
496	print("in object values, token: " + t());
497	endifdef
498	if (eq(t(), ",")) consume();
499	if (eq(t(), ")")) {
500	consume(")");
501	objRead(_o, _dO, fields, hasOuter);
502	out.set(_o != null ? _o : _dO);
503	} else {
504	final S key = unquote(tpp());
505	S t = tpp();
506	if (!eq(t, "="))
507	fail("= expected, got " + t + " after " + quote(key) + " in object " + thingName /+ " " + sfu(fields)/);
508	stack.add(this);
509	parse(new unstructure_Receiver {
510	void set(O value) {
511	fields.put(key, value);
512	/*ifdef unstructure_debug
513	print("Got field value " + value + ", next token: " + t());
514	endifdef*/
515	//if (eq(t(), ",")) consume();
516	}
517	});
518	}
519	});
520	} else {
521	objRead(o, dO, fields, hasOuter);
522	out.set(o != null ? o : dO);
523	}
524	}
525
526	void objRead(O o, DynamicObject dO, MapSO fields, bool hasOuter) {
527	ifdef unstructure_debug
528	print("objRead " + className(o) + " " + className(dO) + " " + struct(fields));
529	endifdef
530
531	// translate between diferent compilers (this$0 vs this$1)
532	O outer = fields.get("this$0");
533	if (outer != null) fields.put("this$1", outer);
534	else {
535	outer = fields.get("this$1");
536	if (outer != null) fields.put("this$0", outer);
537	}
538
539	if (o != null) {
540	if (dO != null) {
541	ifdef unstructure_debug
542	printStructure("setOptAllDyn", fields);
543	endifdef
544	setOptAllDyn_pcall(dO, fields);
545	} else {
546	setOptAll_pcall(o, fields);
547	ifdef unstructure_debug
548	print("objRead now: " + struct(o));
549	endifdef
550	}
551	if (hasOuter)
552	fixOuterRefs(o);
553	} else for (Map.Entry<S, O> e : fields.entrySet())
554	setDynObjectValue(dO, intern(e.getKey()), e.getValue());
555
556	if (o != null)
557	pcallOpt_noArgs(o, "_doneLoading");
558	}
559
560	void parseSet(final Set set, final unstructure_Receiver out) {
561	this.parseList(new ArrayList, new unstructure_Receiver {
562	void set(O o) {
563	set.addAll((L) o);
564	out.set(set);
565	}
566	});
567	}
568
569	void parseLisp(final unstructure_Receiver out) {
570	ifclass Lisp
571	consume("l");
572	consume("(");
573	final new ArrayList list;
574	stack.add(r {
575	if (eq(t(), ")")) {
576	consume(")");
577	out.set(Lisp((S) list.get(0), subList(list, 1)));
578	} else {
579	stack.add(this);
580	parse(new unstructure_Receiver {
581	void set(O o) {
582	list.add(o);
583	if (eq(t(), ",")) consume();
584	}
585	});
586	}
587	});
588	if (false) // skip fail line
589	endif
590
591	fail("class Lisp not included");
592	}
593
594	void parseBitSet(final unstructure_Receiver out) {
595	consume("bitset");
596	consume("{");
597	final new BitSet bs;
598	stack.add(r {
599	if (eq(t(), "}")) {
600	consume("}");
601	out.set(bs);
602	} else {
603	stack.add(this);
604	parse(new unstructure_Receiver {
605	void set(O o) {
606	bs.set((Integer) o);
607	if (eq(t(), ",")) consume();
608	}
609	});
610	}
611	});
612	}
613
614	void parseList(final L list, final unstructure_Receiver out) {
615	tokrefs.put(i, list);
616	consume("[");
617	stack.add(r {
618	if (eq(t(), "]")) {
619	consume();
620	ifdef unstructure_debug
621	print("Consumed list, next token: " + t());
622	endifdef
623	out.set(list);
624	} else {
625	stack.add(this);
626	parse(new unstructure_Receiver {
627	void set(O o) {
628	//if (debug) print("List element type: " + getClassName(o));
629	list.add(o);
630	if (eq(t(), ",")) consume();
631	}
632	});
633	}
634	});
635	}
636
637	void parseArray(unstructure_Receiver out) {
638	S _type = tpp();
639	int dims;
640
641	if (eq(t(), "S")) { // string array
642	_type = "S";
643	consume();
644	}
645
646	if (eq(t(), "/")) { // multi-dimensional array
647	consume();
648	dims = parseInt(tpp());
649	} else
650	dims = 1;
651
652	consume("{");
653	List list = new ArrayList;
654	S type = _type;
655
656	stack.add(r {
657	if (eq(t(), "}")) {
658	consume("}");
659	if (dims > 1) {
660	Class atype;
661	if (type.equals("intarray")) atype = int.class;
662	else if (type.equals("S")) atype = S.class;
663	else todo("multi-dimensional arrays of other types");
664
665	out.set(list.toArray((O[]) newMultiDimensionalOuterArray(atype, dims, l(list))));
666	} else
667	out.set(
668	type.equals("intarray") ? toIntArray(list)
669	: type.equals("dblarray") ? toDoubleArray(list)
670	: type.equals("S") ? toStringArray(list)
671	: list.toArray());
672	} else {
673	stack.add(this);
674	parse(new unstructure_Receiver {
675	void set(O o) {
676	list.add(o);
677	if (eq(t(), ",")) consume();
678	}
679	});
680	}
681	});
682	}
683
684	Object parseClass() {
685	consume("class");
686	consume("(");
687	S name = unquote(tpp());
688	consume(")");
689	Class c = allDynamic ? null : findAClass(name);
690	if (c != null) ret c;
691	new DynamicObject dO;
692	dO.className = "java.lang.Class";
693	name = dropPrefix(mcDollar, name);
694	dO.fieldValues.put("name", name);
695	ret dO;
696	}
697
698	Object parseBigInt() {
699	consume("bigint");
700	consume("(");
701	S val = tpp();
702	if (eq(val, "-"))
703	val = "-" + tpp();
704	consume(")");
705	ret new BigInteger(val);
706	}
707
708	Object parseDouble() {
709	consume("d");
710	consume("(");
711	S val = unquote(tpp());
712	consume(")");
713	ret Double.parseDouble(val);
714	}
715
716	Object parseFloat() {
717	consume("fl");
718	S val;
719	if (eq(t(), "(")) {
720	consume("(");
721	val = unquote(tpp());
722	consume(")");
723	} else {
724	val = unquote(tpp());
725	}
726	ret Float.parseFloat(val);
727	}
728
729	void parseHashSet(unstructure_Receiver out) {
730	consume("hashset");
731	parseSet(new HashSet, out);
732	}
733
734	void parseLinkedHashSet(unstructure_Receiver out) {
735	consume("lhs");
736	parseSet(new LinkedHashSet, out);
737	}
738
739	void parseTreeSet(unstructure_Receiver out) {
740	consume("treeset");
741	parseSet(new TreeSet, out);
742	}
743
744	void parseCISet(unstructure_Receiver out) {
745	consume("ciset");
746	parseSet(ciSet(), out);
747	}
748
749	void parseMap(unstructure_Receiver out) {
750	parseMap(new TreeMap, out);
751	}
752
753	O parseJava() {
754	S j = unquote(tpp());
755	new Matches m;
756	if (jmatch("java.awt.Color[r=,g=,b=*]", j, m))
757	ret nuObject("java.awt.Color", parseInt($1), parseInt($2), parseInt($3));
758	else {
759	warn("Unknown Java object: " + j);
760	null;
761	}
762	}
763
764	void parseMap(final Map map, final unstructure_Receiver out) {
765	consume("{");
766	stack.add(new Runnable {
767	bool v;
768	O key;
769
770	public void run() {
771	if (v) {
772	v = false;
773	stack.add(this);
774	if (!eq(tpp(), "="))
775	fail("= expected, got " + t() + " in map of size " + l(map));
776
777	parse(new unstructure_Receiver {
778	void set(O value) {
779	map.put(key, value);
780	ifdef unstructure_debug
781	print("parseMap: Got value " + getClassName(value) + ", next token: " + quote(t()));
782	endifdef
783	if (eq(t(), ",")) consume();
784	}
785	});
786	} else {
787	if (eq(t(), "}")) {
788	consume("}");
789	out.set(map);
790	} else {
791	v = true;
792	stack.add(this);
793	parse(new unstructure_Receiver {
794	void set(O o) {
795	key = o;
796	}
797	});
798	}
799	} // if v else
800	} // run()
801	});
802	}
803
804	/*void parseSub(unstructure_Receiver out) {
805	int n = l(stack);
806	parse(out);
807	while (l(stack) > n)
808	stack
809	}*/
810
811	void consume() { curT = tok.next(); ++i; }
812
813	void consume(S s) {
814	if (!eq(t(), s)) {
815	/*S prevToken = i-1 >= 0 ? tok.get(i-1) : "";
816	S nextTokens = join(tok.subList(i, Math.min(i+2, tok.size())));
817	fail(quote(s) + " expected: " + prevToken + " " + nextTokens + " (" + i + "/" + tok.size() + ")");*/
818	fail(quote(s) + " expected, got " + quote(t()));
819	}
820	consume();
821	}
822
823	// outer wrapper function getting first token and unwinding the stack
824	void parse_initial(unstructure_Receiver out) {
825	consume(); // get first token
826	parse(out);
827	while (nempty(stack))
828	popLast(stack).run();
829	}
830	}
831
832	ThreadLocal<Bool> tlLoading = dynamicObjectIsLoading_threadLocal();
833	Bool b = tlLoading!;
834	tlLoading.set(true);
835	try {
836	final new Var v;
837	new X x;
838	x.parse_initial(new unstructure_Receiver {
839	void set(O o) { v.set(o); }
840	});
841	unstructure_tokrefs = x.tokrefs.size();
842	ret v.get();
843	} finally {
844	tlLoading.set(b);
845	}
846	}
847
848	static boolean unstructure_debug;