static Object unstructure(String text) { ret unstructure(text, false); } static Object unstructure(String text, final boolean allDynamic) { ret unstructure(text, allDynamic, null); } static int structure_internStringsLongerThan = 50; static int unstructure_unquoteBufSize = 100; static int unstructure_tokrefs; // stats abstract sclass unstructure_Receiver { abstract void set(O o); } // classFinder: func(name) -> class (optional) static Object unstructure(String text, boolean allDynamic, O classFinder) { if (text == null) ret null; ret unstructure_tok(javaTokC_noMLS_iterator(text), allDynamic, classFinder); } static O unstructure_reader(BufferedReader reader) { ret unstructure_tok(javaTokC_noMLS_onReader(reader), false, null); } static O unstructure_tok(final Producer tok, final boolean allDynamic, final O _classFinder) { final boolean debug = unstructure_debug; final class X { int i = -1; final O classFinder = _classFinder != null ? _classFinder : _defaultClassFinder(); new HashMap refs; new HashMap tokrefs; new HashSet concepts; new HashMap classesMap; new L stack; S curT; char[] unquoteBuf = new char[unstructure_unquoteBufSize]; S unquote(S s) { ret unquoteUsingCharArray(s, unquoteBuf); } // look at current token S t() { ret curT; } // get current token, move to next S tpp() { S t = curT; consume(); ret t; } void parse(final unstructure_Receiver out) { S t = t(); int refID = 0; if (structure_isMarker(t, 0, l(t))) { refID = parseInt(t.substring(1)); consume(); } final int _refID = refID; // if (debug) print("parse: " + quote(t)); final int tokIndex = i; parse_inner(refID, tokIndex, new unstructure_Receiver { void set(O o) { if (_refID != 0) refs.put(_refID, o); if (o != null) tokrefs.put(tokIndex, o); out.set(o); } }); } void parse_inner(int refID, int tokIndex, final unstructure_Receiver out) { S t = t(); // if (debug) print("parse_inner: " + quote(t)); Class c = classesMap.get(t); if (c == null) { if (t.startsWith("\"")) { S s = internIfLongerThan(unquote(tpp()), structure_internStringsLongerThan); out.set(s); ret; } if (t.startsWith("'")) { out.set(unquoteCharacter(tpp())); ret; } if (t.equals("bigint")) { out.set(parseBigInt()); ret; } if (t.equals("d")) { out.set(parseDouble()); ret; } if (t.equals("fl")) { out.set(parseFloat()); ret; } if (t.equals("sh")) { consume(); t = tpp(); if (t.equals("-")) { t = tpp(); out.set((short) (-parseInt(t)); ret; } out.set((short) parseInt(t)); ret; } if (t.equals("-")) { consume(); t = tpp(); out.set(isLongConstant(t) ? (O) (-parseLong(t)) : (O) (-parseInt(t))); ret; } if (isInteger(t) || isLongConstant(t)) { consume(); //if (debug) print("isLongConstant " + quote(t) + " => " + isLongConstant(t)); if (isLongConstant(t)) { out.set(parseLong(t)); ret; } long l = parseLong(t); bool isInt = l == (int) l; ifdef unstructure_debug print("l=" + l + ", isInt: " + isInt); endifdef out.set(isInt ? (O) Integer.valueOf((int) l) : (O) Long.valueOf(l)); ret; } if (t.equals("false") || t.equals("f")) { consume(); out.set(false); ret; } if (t.equals("true") || t.equals("t")) { consume(); out.set(true); ret; } if (t.equals("-")) { consume(); t = tpp(); out.set(isLongConstant(t) ? (O) (-parseLong(t)) : (O) (-parseInt(t))); ret; } if (isInteger(t) || isLongConstant(t)) { consume(); //if (debug) print("isLongConstant " + quote(t) + " => " + isLongConstant(t)); if (isLongConstant(t)) { out.set(parseLong(t)); ret; } long l = parseLong(t); bool isInt = l == (int) l; ifdef unstructure_debug print("l=" + l + ", isInt: " + isInt); endifdef out.set(isInt ? (O) Integer.valueOf((int) l) : (O) Long.valueOf(l)); ret; } if (t.equals("File")) { consume(); File f = new File(unquote(tpp())); out.set(f); ret; } if (t.startsWith("r") && isInteger(t.substring(1))) { consume(); int ref = Integer.parseInt(t.substring(1)); O o = refs.get(ref); if (o == null) fail("unsatisfied back reference " + ref); out.set(o); ret; } if (t.startsWith("t") && isInteger(t.substring(1))) { consume(); int ref = Integer.parseInt(t.substring(1)); O o = tokrefs.get(ref); if (o == null) fail("unsatisfied token reference " + ref + " at " + tokIndex); out.set(o); ret; } if (t.equals("hashset")) ret with parseHashSet(out); if (t.equals("lhs")) ret with parseLinkedHashSet(out); if (t.equals("treeset")) ret with parseTreeSet(out); if (t.equals("ciset")) ret with parseCISet(out); if (eqOneOf(t, "hashmap", "hm")) { consume(); parseMap(new HashMap, out); ret; } if (t.equals("lhm")) { consume(); parseMap(new LinkedHashMap, out); ret; } if (t.equals("tm")) { consume(); parseMap(new TreeMap, out); ret; } if (t.equals("cimap")) { consume(); parseMap(ciMap(), out); ret; } if (t.equals("ll")) { consume(); ret with parseList(new LinkedList, out); } if (t.equals("syncLL")) { // legacy consume(); ret with parseList(synchroLinkedList(), out); } if (t.equals("sync")) { consume(); ret with parse(new unstructure_Receiver { void set(O value) { if (value instanceof Map) { ifndef Android // Java 7 if (value instanceof NavigableMap) ret with out.set(Collections.synchronizedNavigableMap((NavigableMap) value)); endifndef if (value instanceof SortedMap) ret with out.set(Collections.synchronizedSortedMap((SortedMap) value)); ret with out.set(Collections.synchronizedMap((Map) value)); } else ret with out.set(Collections.synchronizedList((L) value); } }); } if (t.equals("{")) { parseMap(out); ret; } if (t.equals("[")) { this.parseList(new ArrayList, out); ret; } if (t.equals("bitset")) { parseBitSet(out); ret; } if (t.equals("array") || t.equals("intarray")) { parseArray(out); ret; } if (t.equals("ba")) { consume(); S hex = unquote(tpp()); out.set(hexToBytes(hex)); ret; } if (t.equals("boolarray")) { consume(); int n = parseInt(tpp()); S hex = unquote(tpp()); out.set(boolArrayFromBytes(hexToBytes(hex), n)); ret; } if (t.equals("class")) { out.set(parseClass()); ret; } if (t.equals("l")) { parseLisp(out); ret; } if (t.equals("null")) { consume(); out.set(null); ret; } if (eq(t, "c")) { consume(); t = t(); assertTrue(isJavaIdentifier(t)); concepts.add(t); } // custom deserialization if (eq(t, "cu")) { consume(); t = tpp(); assertTrue(isJavaIdentifier(t)); S fullClassName = "main$" + t; c = classFinder != null ? (Class) callF(classFinder, fullClassName) : findClass_fullName(fullClassName); if (c == null) fail("Class not found: " + fullClassName); fO o = nuEmptyObject(c); out.set(o); parse(new unstructure_Receiver { void set(O value) { ifdef unstructure_debug print("Consumed custom object, next token: " + t()); endifdef call(o, "_deserialize", value); } }); ret; } } if (eq(t, "j")) { consume("j"); out.set(parseJava()); ret; } if (c == null && !isJavaIdentifier(t)) throw new RuntimeException("Unknown token " + (i+1) + ": " + t); // any other class name (or package name) consume(); S className, fullClassName; // Is it a package name? if (eq(t(), ".")) { consume(); className = fullClassName = t + "." + assertIdentifier(tpp()); } else { className = t; fullClassName = "main$" + t; } if (c == null) { // First, find class if (allDynamic) c = null; else c = classFinder != null ? (Class) callF(classFinder, fullClassName) : findClass_fullName(fullClassName); if (c != null) classesMap.put(className, c); } // Check if it has an outer reference bool hasBracket = eq(t(), "("); if (hasBracket) consume(); bool hasOuter = hasBracket && eq(t(), "this$1"); DynamicObject dO = null; O o = null; fS thingName = t; if (c != null) { o = hasOuter ? nuStubInnerObject(c, classFinder) : nuEmptyObject(c); if (o instanceof DynamicObject) dO = (DynamicObject) o; } else { if (concepts.contains(t) && (c = findClass("Concept")) != null) o = dO = (DynamicObject) nuEmptyObject(c); else dO = new DynamicObject; dO.className = className; ifdef unstructure_debug print("Made dynamic object " + t + " " + shortClassName(dO)); endifdef } // Save in references list early because contents of object // might link back to main object if (refID != 0) refs.put(refID, o != null ? o : dO); tokrefs.put(tokIndex, o != null ? o : dO); // NOW parse the fields! final new LinkedHashMap fields; // preserve order final O _o = o; final DynamicObject _dO = dO; if (hasBracket) { stack.add(r { ifdef unstructure_debug print("in object values, token: " + t()); endifdef if (eq(t(), ",")) consume(); if (eq(t(), ")")) { consume(")"); objRead(_o, _dO, fields, hasOuter); out.set(_o != null ? _o : _dO); } else { final S key = unquote(tpp()); S t = tpp(); if (!eq(t, "=")) fail("= expected, got " + t + " after " + quote(key) + " in object " + thingName /*+ " " + sfu(fields)*/); stack.add(this); parse(new unstructure_Receiver { void set(O value) { fields.put(key, value); /*ifdef unstructure_debug print("Got field value " + value + ", next token: " + t()); endifdef*/ //if (eq(t(), ",")) consume(); } }); } }); } else { objRead(o, dO, fields, hasOuter); out.set(o != null ? o : dO); } } void objRead(O o, DynamicObject dO, Map fields, bool hasOuter) { ifdef unstructure_debug print("objRead " + className(o) + " " + className(dO) + " " + struct(fields)); endifdef if (o != null) { if (dO != null) { ifdef unstructure_debug printStructure("setOptAllDyn", fields); endifdef setOptAllDyn(dO, fields); } else { setOptAll_pcall(o, fields); ifdef unstructure_debug print("objRead now: " + struct(o)); endifdef } if (hasOuter) fixOuterRefs(o); } else for (Map.Entry e : fields.entrySet()) setDynObjectValue(dO, intern(e.getKey()), e.getValue()); if (o != null) pcallOpt_noArgs(o, "_doneLoading"); } void parseSet(final Set set, final unstructure_Receiver out) { this.parseList(new ArrayList, new unstructure_Receiver { void set(O o) { set.addAll((L) o); out.set(set); } }); } void parseLisp(final unstructure_Receiver out) { ifclass Lisp consume("l"); consume("("); final new ArrayList list; stack.add(r { if (eq(t(), ")")) { consume(")"); out.set(Lisp((S) list.get(0), subList(list, 1))); } else { stack.add(this); parse(new unstructure_Receiver { void set(O o) { list.add(o); if (eq(t(), ",")) consume(); } }); } }); if (false) // skip fail line endif fail("class Lisp not included"); } void parseBitSet(final unstructure_Receiver out) { consume("bitset"); consume("{"); final new BitSet bs; stack.add(r { if (eq(t(), "}")) { consume("}"); out.set(bs); } else { stack.add(this); parse(new unstructure_Receiver { void set(O o) { bs.set((Integer) o); if (eq(t(), ",")) consume(); } }); } }); } void parseList(final L list, final unstructure_Receiver out) { consume("["); stack.add(r { if (eq(t(), "]")) { consume(); ifdef unstructure_debug print("Consumed list, next token: " + t()); endifdef out.set(list); } else { stack.add(this); parse(new unstructure_Receiver { void set(O o) { //if (debug) print("List element type: " + getClassName(o)); list.add(o); if (eq(t(), ",")) consume(); } }); } }); } void parseArray(final unstructure_Receiver out) { final S type = tpp(); consume("{"); final List list = new ArrayList; stack.add(r { if (eq(t(), "}")) { consume("}"); out.set(type.equals("intarray") ? toIntArray(list) : list.toArray()); } else { stack.add(this); parse(new unstructure_Receiver { void set(O o) { list.add(o); if (eq(t(), ",")) consume(); } }); } }); } Object parseClass() { consume("class"); consume("("); S name = unquote(tpp()); consume(")"); Class c = allDynamic ? null : classFinder != null ? (Class) callF(classFinder, name) : findClass_fullName(name); if (c != null) ret c; new DynamicObject dO; dO.className = "java.lang.Class"; name = dropPrefix("main$", name); dO.fieldValues.put("name", name); ret dO; } Object parseBigInt() { consume("bigint"); consume("("); S val = tpp(); if (eq(val, "-")) val = "-" + tpp(); consume(")"); ret new BigInteger(val); } Object parseDouble() { consume("d"); consume("("); S val = unquote(tpp()); consume(")"); ret Double.parseDouble(val); } Object parseFloat() { consume("fl"); S val; if (eq(t(), "(")) { consume("("); val = unquote(tpp()); consume(")"); } else { val = unquote(tpp()); } ret Float.parseFloat(val); } void parseHashSet(unstructure_Receiver out) { consume("hashset"); parseSet(new HashSet, out); } void parseLinkedHashSet(unstructure_Receiver out) { consume("lhs"); parseSet(new LinkedHashSet, out); } void parseTreeSet(unstructure_Receiver out) { consume("treeset"); parseSet(new TreeSet, out); } void parseCISet(unstructure_Receiver out) { consume("ciset"); parseSet(ciSet(), out); } void parseMap(unstructure_Receiver out) { parseMap(new TreeMap, out); } O parseJava() { S j = unquote(tpp()); new Matches m; if (jmatch("java.awt.Color[r=*,g=*,b=*]", j, m)) ret nuObject("java.awt.Color", parseInt($1), parseInt($2), parseInt($3)); else { warn("Unknown Java object: " + j); null; } } void parseMap(final Map map, final unstructure_Receiver out) { consume("{"); stack.add(new Runnable { bool v; O key; public void run() { if (v) { v = false; stack.add(this); if (!eq(tpp(), "=")) fail("= expected, got " + t() + " in map of size " + l(map)); parse(new unstructure_Receiver { void set(O value) { map.put(key, value); ifdef unstructure_debug print("parseMap: Got value " + getClassName(value) + ", next token: " + quote(t())); endifdef if (eq(t(), ",")) consume(); } }); } else { if (eq(t(), "}")) { consume("}"); out.set(map); } else { v = true; stack.add(this); parse(new unstructure_Receiver { void set(O o) { key = o; } }); } } // if v else } // run() }); } /*void parseSub(unstructure_Receiver out) { int n = l(stack); parse(out); while (l(stack) > n) stack }*/ void consume() { curT = tok.next(); ++i; } void consume(S s) { if (!eq(t(), s)) { /*S prevToken = i-1 >= 0 ? tok.get(i-1) : ""; S nextTokens = join(tok.subList(i, Math.min(i+2, tok.size()))); fail(quote(s) + " expected: " + prevToken + " " + nextTokens + " (" + i + "/" + tok.size() + ")");*/ fail(quote(s) + " expected, got " + quote(t())); } consume(); } // outer wrapper function getting first token and unwinding the stack void parse_initial(unstructure_Receiver out) { consume(); // get first token parse(out); while (nempty(stack)) popLast(stack).run(); } } Bool b = DynamicObject_loading!; DynamicObject_loading.set(true); try { final new Var v; new X x; x.parse_initial(new unstructure_Receiver { void set(O o) { v.set(o); } }); unstructure_tokrefs = x.tokrefs.size(); ret v.get(); } finally { DynamicObject_loading.set(b); } } static boolean unstructure_debug;