import java.util.*; import java.util.zip.*; import java.util.List; import java.util.regex.*; import java.util.concurrent.*; import java.util.concurrent.atomic.*; import java.util.concurrent.locks.*; import javax.swing.*; import javax.swing.event.*; import javax.swing.text.*; import javax.swing.table.*; import java.io.*; import java.net.*; import java.lang.reflect.*; import java.lang.ref.*; import java.lang.management.*; import java.security.*; import java.security.spec.*; import java.awt.*; import java.awt.event.*; import java.awt.image.*; import javax.imageio.*; import java.math.*; //packageName wikify //mainClassName Wikify import java.text.NumberFormat; import java.nio.charset.Charset; import org.eclipse.collections.impl.map.mutable.primitive.*; import org.eclipse.collections.impl.set.mutable.primitive.*; import org.eclipse.collections.api.tuple.primitive.*; import org.eclipse.collections.api.iterator.*; import java.nio.*; import java.nio.channels.*; import java.text.SimpleDateFormat; class main { static class Wikify { volatile Indexed indexed; String indexing; Q q = startQ(); int slowSearches; static class Indexed { String text; LCSearcher_v4_multiFile searcher; } int anyIndexOf(String fullText, String pattern) { if (empty(fullText)) return -1; Indexed _indexed = indexed; if (_indexed != null && fullText == _indexed.text) return _indexed.searcher.singleFileAnyIndexOf(pattern); // possibly index, do slow search possiblyIndexText(fullText); ++slowSearches; return main.indexOf(fullText, pattern); } synchronized void possiblyIndexText(String text) { if (indexing != null) return; indexing = text; q.add(new Runnable() { public void run() { try { try { print("Indexing " + nChars(text) + ": " + quote(shorten(text))); Indexed i = new Indexed(); i.text = text; i.searcher = wikifyString(text); setIndexed(i); } finally { indexing = null; } } catch (Exception __e) { throw rethrow(__e); } } public String toString() { return "try {\r\n print(\"Indexing \" + nChars(text) + \": \" + quote(shorten(text))..."; }}); } synchronized void setIndexed(Indexed i) { print("Indexed " + nChars(i.text)); indexed = i; } } static Q startQ() { return new Q(); } static Q startQ(String name) { return new Q(name); } static boolean empty(Collection c) { return c == null || c.isEmpty(); } static boolean empty(Iterable c) { return c == null || !c.iterator().hasNext(); } static boolean empty(CharSequence s) { return s == null || s.length() == 0; } static boolean empty(Map map) { return map == null || map.isEmpty(); } static boolean empty(Object[] o) { return o == null || o.length == 0; } static boolean empty(Object o) { if (o instanceof Collection) return empty((Collection) o); if (o instanceof String) return empty((String) o); if (o instanceof Map) return empty((Map) o); if (o instanceof Object[]) return empty((Object[]) o); if (o instanceof byte[]) return empty((byte[]) o); if (o == null) return true; throw fail("unknown type for 'empty': " + getType(o)); } static boolean empty(Iterator i) { return i == null || !i.hasNext(); } static boolean empty(double[] a) { return a == null || a.length == 0; } static boolean empty(float[] a) { return a == null || a.length == 0; } static boolean empty(int[] a) { return a == null || a.length == 0; } static boolean empty(long[] a) { return a == null || a.length == 0; } static boolean empty(byte[] a) { return a == null || a.length == 0; } static boolean empty(short[] a) { return a == null || a.length == 0; } static boolean empty(MultiMap mm) { return mm == null || mm.isEmpty(); } static boolean empty(File f) { return getFileSize(f) == 0; } static boolean empty(IntRange r) { return r == null || r.empty(); } static boolean empty(IntBuffer b) { return b == null || b.isEmpty(); } static boolean empty(LongBuffer b) { return b == null || b.isEmpty(); } static int indexOf(List l, A a, int startIndex) { if (l == null) return -1; int n = l(l); for (int i = startIndex; i < n; i++) if (eq(l.get(i), a)) return i; return -1; } static int indexOf(List l, int startIndex, A a) { return indexOf(l, a, startIndex); } static int indexOf(List l, A a) { if (l == null) return -1; return l.indexOf(a); } static int indexOf(String a, String b) { return a == null || b == null ? -1 : a.indexOf(b); } static int indexOf(String a, String b, int i) { return a == null || b == null ? -1 : a.indexOf(b, i); } static int indexOf(String a, char b) { return a == null ? -1 : a.indexOf(b); } static int indexOf(String a, int i, char b) { return indexOf(a, b, i); } static int indexOf(String a, char b, int i) { return a == null ? -1 : a.indexOf(b, i); } static int indexOf(String a, int i, String b) { return a == null || b == null ? -1 : a.indexOf(b, i); } static int indexOf(A[] x, A a) { int n = l(x); for (int i = 0; i < n; i++) if (eq(x[i], a)) return i; return -1; } static volatile StringBuffer local_log = new StringBuffer(); // not redirected static volatile Appendable print_log = local_log; // might be redirected, e.g. to main bot // in bytes - will cut to half that static volatile int print_log_max = 1024*1024; static volatile int local_log_max = 100*1024; static boolean print_silent = false; // total mute if set static Object print_byThread_lock = new Object(); static volatile ThreadLocal print_byThread; // special handling by thread - prefers F1 static volatile Object print_allThreads; static volatile Object print_preprocess; static void print() { print(""); } static A print(String s, A o) { print((endsWithLetterOrDigit(s) ? s + ": " : s) + o); return o; } // slightly overblown signature to return original object... static A print(A o) { ping_okInCleanUp(); if (print_silent) return o; String s = o + "\n"; print_noNewLine(s); return o; } static void print_noNewLine(String s) { Object f = getThreadLocal(print_byThread_dontCreate()); if (f == null) f = print_allThreads; if (f != null) // We do need the general callF machinery here as print_byThread is sometimes shared between modules if (isFalse( f instanceof F1 ? ((F1) f).get(s) : callF(f, s))) return; print_raw(s); } static void print_raw(String s) { if (print_preprocess != null) s = (String) callF(print_preprocess, s); s = fixNewLines(s); Appendable loc = local_log; Appendable buf = print_log; int loc_max = print_log_max; if (buf != loc && buf != null) { print_append(buf, s, print_log_max); loc_max = local_log_max; } if (loc != null) print_append(loc, s, loc_max); System.out.print(s); vmBus_send("printed",mc(), s); } static void print_autoRotate() { } static String nChars(long n) { return n2(n, "char"); } static String nChars(String s) { return nChars(l(s)); } static String quote(Object o) { if (o == null) return "null"; return quote(str(o)); } static String quote(String s) { if (s == null) return "null"; StringBuilder out = new StringBuilder((int) (l(s)*1.5+2)); quote_impl(s, out); return out.toString(); } static void quote_impl(String s, StringBuilder out) { out.append('"'); int l = s.length(); for (int i = 0; i < l; i++) { char c = s.charAt(i); if (c == '\\' || c == '"') out.append('\\').append(c); else if (c == '\r') out.append("\\r"); else if (c == '\n') out.append("\\n"); else if (c == '\t') out.append("\\t"); else if (c == '\0') out.append("\\0"); else out.append(c); } out.append('"'); } static int shorten_default = 100; static String shorten(CharSequence s) { return shorten(s, shorten_default); } static String shorten(CharSequence s, int max) { return shorten(s, max, "..."); } static String shorten(CharSequence s, int max, String shortener) { if (s == null) return ""; if (max < 0) return str(s); return s.length() <= max ? str(s) : subCharSequence(s, 0, min(s.length(), max-l(shortener))) + shortener; } static String shorten(int max, CharSequence s) { return shorten(s, max); } static LCSearcher_v4_multiFile wikifyString(String s) { LineCompCompressor comp = lcFullCompressText(s, "toUpper" , true); LineCompReader reader = lineCompReaderFromCompressor(comp); comp = null; LCSearcher_v4_compact searcher4 = lcSearcher4compactFromLCReader(reader); reader = null; // Wouldn't need the temporary file if we modified the save function to write to RAM File indexFile = createTempFileWithExtension(".qs3"); AutoCloseable __1 = tempDeleteFile(indexFile); try { save_LCSearcher_v1_compact_toQuickFile_v3(searcher4.lc, indexFile); searcher4 = null; RAMByteMemory64 mem = new RAMByteMemory64(indexFile); // this loads the file, so we can release it LCSearcher_v1_onDisk64_v3 lc_v3 = new LCSearcher_v1_onDisk64_v3(mem); LCSearcher_v4_multiFile mf = new LCSearcher_v4_multiFile(lc_v3); mf.getMem = () -> mem; mf.queriesToUpper = true; return mf; } finally { _close(__1); }} static RuntimeException rethrow(Throwable t) { if (t instanceof Error) _handleError((Error) t); throw t instanceof RuntimeException ? (RuntimeException) t : new RuntimeException(t); } static RuntimeException rethrow(String msg, Throwable t) { throw new RuntimeException(msg, t); } static AutoCloseable tempInterceptPrintIfNotIntercepted(F1 f) { return print_byThread().get() == null ? tempInterceptPrint(f) : null; } static RuntimeException fail() { throw new RuntimeException("fail"); } static RuntimeException fail(Throwable e) { throw asRuntimeException(e); } static RuntimeException fail(Object msg) { throw new RuntimeException(String.valueOf(msg)); } static RuntimeException fail(String msg) { throw new RuntimeException(msg == null ? "" : msg); } static RuntimeException fail(String msg, Throwable innerException) { throw new RuntimeException(msg, innerException); } static String getType(Object o) { return getClassName(o); } static long getFileSize(String path) { return path == null ? 0 : new File(path).length(); } static long getFileSize(File f) { return f == null ? 0 : f.length(); } static int l(Object[] a) { return a == null ? 0 : a.length; } static int l(boolean[] a) { return a == null ? 0 : a.length; } static int l(byte[] a) { return a == null ? 0 : a.length; } static int l(short[] a) { return a == null ? 0 : a.length; } static int l(long[] a) { return a == null ? 0 : a.length; } static int l(int[] a) { return a == null ? 0 : a.length; } static int l(float[] a) { return a == null ? 0 : a.length; } static int l(double[] a) { return a == null ? 0 : a.length; } static int l(char[] a) { return a == null ? 0 : a.length; } static int l(Collection c) { return c == null ? 0 : c.size(); } static int l(Iterator i) { return iteratorCount_int_close(i); } // consumes the iterator && closes it if possible static int l(Map m) { return m == null ? 0 : m.size(); } static int l(CharSequence s) { return s == null ? 0 : s.length(); } static long l(File f) { return f == null ? 0 : f.length(); } static int l(Object o) { return o == null ? 0 : o instanceof String ? l((String) o) : o instanceof Map ? l((Map) o) : o instanceof Collection ? l((Collection) o) : o instanceof Object[] ? l((Object[]) o) : o instanceof boolean[] ? l((boolean[]) o) : o instanceof byte[] ? l((byte[]) o) : o instanceof char[] ? l((char[]) o) : o instanceof short[] ? l((short[]) o) : o instanceof int[] ? l((int[]) o) : o instanceof float[] ? l((float[]) o) : o instanceof double[] ? l((double[]) o) : o instanceof long[] ? l((long[]) o) : (Integer) call(o, "size"); } static int l(IntRange r) { return r == null ? 0 : r.length(); } static long l(LongRange r) { return r == null ? 0 : r.length(); } static int l(IntBuffer b) { return b == null ? 0 : b.size(); } static int l(LongBuffer b) { return b == null ? 0 : b.size(); } static boolean eq(Object a, Object b) { return a == b || a != null && b != null && a.equals(b); } static boolean endsWithLetterOrDigit(String s) { return s != null && s.length() > 0 && Character.isLetterOrDigit(s.charAt(s.length()-1)); } static void ping_okInCleanUp() { if (ping_pauseAll || ping_anyActions) ping_impl(true); } // this syntax should be removed... static Object getThreadLocal(Object o, String name) { ThreadLocal t = (ThreadLocal) (getOpt(o, name)); return t != null ? t.get() : null; } static A getThreadLocal(ThreadLocal tl) { return tl == null ? null : tl.get(); } static A getThreadLocal(ThreadLocal tl, A defaultValue) { return or(getThreadLocal(tl), defaultValue); } static ThreadLocal print_byThread_dontCreate() { return print_byThread; } static boolean isFalse(Object o) { return eq(false, o); } static Map> callF_cache = newDangerousWeakHashMap(); static A callF(F0 f) { return f == null ? null : f.get(); } static B callF(F1 f, A a) { return f == null ? null : f.get(a); } static A callF(IF0 f) { return f == null ? null : f.get(); } static B callF(IF1 f, A a) { return f == null ? null : f.get(a); } static C callF(IF2 f, A a, B b) { return f == null ? null : f.get(a, b); } static void callF(VF1 f, A a) { if (f != null) f.get(a); } static Object callF(Object f, Object... args) { if (f instanceof String) return callMCWithVarArgs((String) f, args); // possible SLOWDOWN over callMC return safeCallF(f, args); } static Object safeCallF(Object f, Object... args) { if (f instanceof Runnable) { ((Runnable) f).run(); return null; } if (f == null) return null; Class c = f.getClass(); ArrayList methods; synchronized(callF_cache) { methods = callF_cache.get(c); if (methods == null) methods = callF_makeCache(c); } int n = l(methods); if (n == 0) { throw fail("No get method in " + getClassName(c)); } if (n == 1) return invokeMethod(methods.get(0), f, args); for (int i = 0; i < n; i++) { Method m = methods.get(i); if (call_checkArgs(m, args, false)) return invokeMethod(m, f, args); } throw fail("No matching get method in " + getClassName(c)); } // used internally static ArrayList callF_makeCache(Class c) { ArrayList l = new ArrayList(); Class _c = c; do { for (Method m : _c.getDeclaredMethods()) if (m.getName().equals("get")) { makeAccessible(m); l.add(m); } if (!l.isEmpty()) break; _c = _c.getSuperclass(); } while (_c != null); callF_cache.put(c, l); return l; } static String fixNewLines(String s) { int i = indexOf(s, '\r'); if (i < 0) return s; int l = s.length(); StringBuilder out = new StringBuilder(l); out.append(s, 0, i); for (; i < l; i++) { char c = s.charAt(i); if (c != '\r') out.append(c); else { out.append('\n'); if (i+1 < l && s.charAt(i+1) == '\n') ++i; } } return out.toString(); } static void print_append(Appendable buf, String s, int max) { try { synchronized(buf) { buf.append(s); if (buf instanceof StringBuffer) rotateStringBuffer(((StringBuffer) buf), max); else if (buf instanceof StringBuilder) rotateStringBuilder(((StringBuilder) buf), max); } } catch (Exception __e) { throw rethrow(__e); } } static void vmBus_send(String msg, Object... args) { Object arg = vmBus_wrapArgs(args); pcallFAll(vm_busListeners_live(), msg, arg); pcallFAll(vm_busListenersByMessage_live().get(msg), msg, arg); } static void vmBus_send(String msg) { vmBus_send(msg, (Object) null); } static Class mc() { return main.class; } static String n2(long l) { return formatWithThousands(l); } static String n2(Collection l) { return n2(l(l)); } static String n2(Map map) { return n2(l(map)); } static String n2(double l, String singular) { return n2(l, singular, singular + "s"); } static String n2(double l, String singular, String plural) { if (fraction(l) == 0) return n2((long) l, singular, plural); else return l + " " + plural; } static String n2(long l, String singular, String plural) { return n_fancy2(l, singular, plural); } static String n2(long l, String singular) { return n_fancy2(l, singular, singular + "s"); } static String n2(Collection l, String singular) { return n2(l(l), singular); } static String n2(Collection l, String singular, String plural) { return n_fancy2(l, singular, plural); } static String n2(Map m, String singular, String plural) { return n_fancy2(m, singular, plural); } static String n2(Map m, String singular) { return n2(l(m), singular); } static String n2(Object[] a, String singular) { return n2(l(a), singular); } static String n2(Object[] a, String singular, String plural) { return n_fancy2(a, singular, plural); } static String str(Object o) { return o == null ? "null" : o.toString(); } static String str(char[] c) { return new String(c); } static CharSequence subCharSequence(CharSequence s, int x) { return subCharSequence(s, x, s == null ? 0 : s.length()); } static CharSequence subCharSequence(CharSequence s, int x, int y) { if (s == null) return null; if (x < 0) x = 0; if (x >= s.length()) return ""; if (y < x) y = x; if (y > s.length()) y = s.length(); return s.subSequence(x, y); } static int min(int a, int b) { return Math.min(a, b); } static long min(long a, long b) { return Math.min(a, b); } static float min(float a, float b) { return Math.min(a, b); } static float min(float a, float b, float c) { return min(min(a, b), c); } static double min(double a, double b) { return Math.min(a, b); } static double min(double[] c) { double x = Double.MAX_VALUE; for (double d : c) x = Math.min(x, d); return x; } static float min(float[] c) { float x = Float.MAX_VALUE; for (float d : c) x = Math.min(x, d); return x; } static byte min(byte[] c) { byte x = 127; for (byte d : c) if (d < x) x = d; return x; } static short min(short[] c) { short x = 0x7FFF; for (short d : c) if (d < x) x = d; return x; } static int min(int[] c) { int x = Integer.MAX_VALUE; for (int d : c) if (d < x) x = d; return x; } static LineCompCompressor lcFullCompressText(String text, Object... __) { String fileName = optPar("fileName",__, "1"); LineCompCompressor compressor = new LineCompCompressor(); compressor.byteMode = true; compressor.fullCompression = true; compressor.toUpper = boolPar("toUpper",__); compressor.balancing = true; compressor.verify = false; compressor.loadTexts(litmap(fileName, text)); compressor.run(); return compressor; } static LineCompReader lineCompReaderFromCompressor(LineCompCompressor comp) { return new LineCompReader(bufferedReader(stringReader(comp.asText()))); } static LCSearcher_v4_compact lcSearcher4compactFromLCReader(LineCompReader lc, Object... __) { long _startTime_0 = sysNow(); LCSearcher_v1 searcher1 = lcSearcherFromLineCompReader(lc, (LCSearcher_v1) optPar("searcher1",__)); lc = null; done2_always("Searcher 1", _startTime_0); long _startTime_1 = sysNow(); LCSearcher_v1_compact searcher = new LCSearcher_v1_compact(); searcher.big = true; // print more updates searcher.load(searcher1); done2_always("Searcher 2", _startTime_1); return new LCSearcher_v4_compact(searcher); } static File createTempFileWithExtension(String ext) { return createTempFile("", addPrefixIfNempty(".", ext)); } static AutoCloseable tempDeleteFile(File f) { return f == null ? null : new AutoCloseable() { public String toString() { return "f.delete();"; } public void close() throws Exception { f.delete(); }}; } /* makes a .qsearch_v3 file Improvements in v3: -Don't write left pairs count for 1-pairs (=90% of pairs) (can be inferred from entry size) -Compress occurrence count & prod length */ static void save_LCSearcher_v1_compact_toQuickFile_v3(LCSearcher_v1_compact lc, File file) { try { print("Saving quickFile " + file); long iPointerArray; int[] symbolPointers; { long _startTime_0 = sysNow(); CountingOutputStream countingStream = new CountingOutputStream(bufferedOutputStream(newFileOutputStream(file))); try { DataOutputStream out = dataOutputStream(countingStream); out.write(toUTF8("QUICKSEARCH v3.\n")); assertEquals(16L, countingStream.getFilePointer()); // write counts out.writeInt(l(lc.literals)); out.writeInt(l(lc.pairs)); out.writeInt(l(lc.files)); // write raw literals for (char c : lc.literals) out.writeChar(c); if (odd(l(lc.literals))) out.writeChar(0); // align to 4 bytes // write files for (int i : lc.files) out.writeInt(i); // leave space for symbol pointer array iPointerArray = countingStream.getFilePointer(); symbolPointers = new int[lc.iFirstFile+1]; // one more to store end of last entry for (int i = 0; i < l(symbolPointers); i++) out.writeInt(0); // write literal + pair infos as compact structs to minimize cache misses for (int i = 0; i < lc.iFirstFile; i++) { symbolPointers[i] = (int) countingStream.getFilePointer(); // write pair contents if it's a pair // no point in compressing these, they are filling the whole int space // both prod length and occurrences are usually small. // prod length is <= the file length (100M), so can compress // into 1 to 4 bytes. // occurrence count can overflow in rare cases, so we just clip. if (i >= lc.iFirstPair) { out.writeLong(lc.pairs[i-lc.iFirstPair]); out.write(compress30BitUint(lc.getProdLength(i))); } // other fields apply to literals and pairs out.write(compress30BitUint(min(0x3FFFFFFF, lc.occurrences[i]))); int[] idxL = lc.leftPairIndex[i], idxR = lc.rightPairIndex[i]; if (l(idxL) == 1 && l(idxR) == 0) out.writeInt(idxL[0]); else if (l(idxL) == 0 && l(idxR) == 1) out.writeInt(idxR[0] | 0x80000000); else { out.write(compress30BitUint(l(idxL))); for (int val : unnull(idxL)) out.writeInt(val); // don't have to store idxR length, it's inferred from position of next entry for (int val : unnull(idxR)) out.writeInt(val); } if ((i % oneMillion()) == 0) print(nSymbols(i) + " written"); } symbolPointers[lc.iFirstFile] = (int) countingStream.getFilePointer(); done2_always("Save quickFile", _startTime_0); } finally { _close(countingStream); }} { long _startTime_1 = sysNow(); RandomAccessFile out = new RandomAccessFile(file, "rw"); try { out.seek(iPointerArray); writeIntArrayToRandomAccessFile(out, symbolPointers); done2_always("Save pointer array", _startTime_1); } finally { _close(out); }} printFileInfo(file); } catch (Exception __e) { throw rethrow(__e); } } static void _close(AutoCloseable c) { if (c != null) try { c.close(); } catch (Throwable e) { // Some classes stupidly throw an exception on double-closing if (c instanceof javax.imageio.stream.ImageOutputStream) return; else throw rethrow(e); } } static void _handleError(Error e) { call(javax(), "_handleError", e); } static ThreadLocal print_byThread() { synchronized(print_byThread_lock) { if (print_byThread == null) print_byThread = new ThreadLocal(); } return print_byThread; } // f can return false to suppress regular printing // call print_raw within f to actually print something static AutoCloseable tempInterceptPrint(F1 f) { return tempSetThreadLocal(print_byThread(), f); } static RuntimeException asRuntimeException(Throwable t) { if (t instanceof Error) _handleError((Error) t); return t instanceof RuntimeException ? (RuntimeException) t : new RuntimeException(t); } static String getClassName(Object o) { return o == null ? "null" : o instanceof Class ? ((Class) o).getName() : o.getClass().getName(); } static int iteratorCount_int_close(Iterator i) { try { int n = 0; if (i != null) while (i.hasNext()) { i.next(); ++n; } if (i instanceof AutoCloseable) ((AutoCloseable) i).close(); return n; } catch (Exception __e) { throw rethrow(__e); } } static Object call(Object o) { return callF(o); } // varargs assignment fixer for a single string array argument static Object call(Object o, String method, String[] arg) { return call(o, method, new Object[] {arg}); } static Object call(Object o, String method, Object... args) { //ret call_cached(o, method, args); return call_withVarargs(o, method, args); } //sbool ping_actions_shareable = true; static volatile boolean ping_pauseAll = false; static int ping_sleep = 100; // poll pauseAll flag every 100 static volatile boolean ping_anyActions = false; static Map ping_actions = newWeakHashMap(); static ThreadLocal ping_isCleanUpThread = new ThreadLocal(); // always returns true static boolean ping() { if (ping_pauseAll || ping_anyActions) ping_impl(true /* XXX */); //ifndef LeanMode ping_impl(); endifndef return true; } // returns true when it slept static boolean ping_impl(boolean okInCleanUp) { try { if (ping_pauseAll && !isAWTThread()) { do Thread.sleep(ping_sleep); while (ping_pauseAll); return true; } if (ping_anyActions) { // don't allow sharing ping_actions if (!okInCleanUp && !isTrue(ping_isCleanUpThread.get())) failIfUnlicensed(); Object action = null; synchronized(ping_actions) { if (!ping_actions.isEmpty()) { action = ping_actions.get(currentThread()); if (action instanceof Runnable) ping_actions.remove(currentThread()); if (ping_actions.isEmpty()) ping_anyActions = false; } } if (action instanceof Runnable) ((Runnable) action).run(); else if (eq(action, "cancelled")) throw fail("Thread cancelled."); } return false; } catch (Exception __e) { throw rethrow(__e); } } static Object getOpt(Object o, String field) { return getOpt_cached(o, field); } static Object getOpt(String field, Object o) { return getOpt_cached(o, field); } static Object getOpt_raw(Object o, String field) { try { Field f = getOpt_findField(o.getClass(), field); if (f == null) return null; makeAccessible(f); return f.get(o); } catch (Exception __e) { throw rethrow(__e); } } // access of static fields is not yet optimized static Object getOpt(Class c, String field) { try { if (c == null) return null; Field f = getOpt_findStaticField(c, field); if (f == null) return null; makeAccessible(f); return f.get(null); } catch (Exception __e) { throw rethrow(__e); } } static Field getOpt_findStaticField(Class c, String field) { Class _c = c; do { for (Field f : _c.getDeclaredFields()) if (f.getName().equals(field) && (f.getModifiers() & java.lang.reflect.Modifier.STATIC) != 0) return f; _c = _c.getSuperclass(); } while (_c != null); return null; } static A or(A a, A b) { return a != null ? a : b; } static Map newDangerousWeakHashMap() { return _registerDangerousWeakMap(synchroMap(new WeakHashMap())); } // initFunction: voidfunc(Map) - is called initially, and after clearing the map static Map newDangerousWeakHashMap(Object initFunction) { return _registerDangerousWeakMap(synchroMap(new WeakHashMap()), initFunction); } static Object callMCWithVarArgs(String method, Object... args) { return call_withVarargs(mc(), method, args); } static Object invokeMethod(Method m, Object o, Object... args) { try { try { return m.invoke(o, args); } catch (InvocationTargetException e) { throw rethrow(getExceptionCause(e)); } catch (IllegalArgumentException e) { throw new IllegalArgumentException(e.getMessage() + " - was calling: " + m + ", args: " + joinWithSpace(classNames(args))); } } catch (Exception __e) { throw rethrow(__e); } } static boolean call_checkArgs(Method m, Object[] args, boolean debug) { Class[] types = m.getParameterTypes(); if (types.length != args.length) { if (debug) print("Bad parameter length: " + args.length + " vs " + types.length); return false; } for (int i = 0; i < types.length; i++) { Object arg = args[i]; if (!(arg == null ? !types[i].isPrimitive() : isInstanceX(types[i], arg))) { if (debug) print("Bad parameter " + i + ": " + arg + " vs " + types[i]); return false; } } return true; } static Field makeAccessible(Field f) { try { f.setAccessible(true); } catch (Throwable e) { // Note: The error reporting only works with Java VM option --illegal-access=deny vmBus_send("makeAccessible_error",e, f); } return f; } static Method makeAccessible(Method m) { try { m.setAccessible(true); } catch (Throwable e) { vmBus_send("makeAccessible_error",e, m); } return m; } static Constructor makeAccessible(Constructor c) { try { c.setAccessible(true); } catch (Throwable e) { vmBus_send("makeAccessible_error",e, c); } return c; } static void rotateStringBuffer(StringBuffer buf, int max) { try { if (buf == null) return; synchronized(buf) { if (buf.length() <= max) return; try { int newLength = max/2; int ofs = buf.length()-newLength; String newString = buf.substring(ofs); buf.setLength(0); buf.append("[...] ").append(newString); } catch (Exception e) { buf.setLength(0); } buf.trimToSize(); } } catch (Exception __e) { throw rethrow(__e); } } static void rotateStringBuilder(StringBuilder buf, int max) { try { if (buf == null) return; synchronized(buf) { if (buf.length() <= max) return; try { int newLength = max/2; int ofs = buf.length()-newLength; String newString = buf.substring(ofs); buf.setLength(0); buf.append("[...] ").append(newString); } catch (Exception e) { buf.setLength(0); } buf.trimToSize(); } } catch (Exception __e) { throw rethrow(__e); } } static Object vmBus_wrapArgs(Object... args) { return empty(args) ? null : l(args) == 1 ? args[0] : args; } static void pcallFAll(Collection l, Object... args) { if (l != null) for (Object f : cloneList(l)) pcallF(f, args); } static void pcallFAll(Iterator it, Object... args) { while (it.hasNext()) pcallF(it.next(), args); } static Set vm_busListeners_live_cache; static Set vm_busListeners_live() { if (vm_busListeners_live_cache == null) vm_busListeners_live_cache = vm_busListeners_live_load(); return vm_busListeners_live_cache; } static Set vm_busListeners_live_load() { return vm_generalIdentityHashSet("busListeners"); } static Map vm_busListenersByMessage_live_cache; static Map vm_busListenersByMessage_live() { if (vm_busListenersByMessage_live_cache == null) vm_busListenersByMessage_live_cache = vm_busListenersByMessage_live_load(); return vm_busListenersByMessage_live_cache; } static Map vm_busListenersByMessage_live_load() { return vm_generalHashMap("busListenersByMessage"); } static String formatWithThousands(long l) { return formatWithThousandsSeparator(l); } static double fraction(double d) { return d % 1; } static String n_fancy2(long l, String singular, String plural) { return formatWithThousandsSeparator(l) + " " + trim(l == 1 ? singular : plural); } static String n_fancy2(Collection l, String singular, String plural) { return n_fancy2(l(l), singular, plural); } static String n_fancy2(Map m, String singular, String plural) { return n_fancy2(l(m), singular, plural); } static String n_fancy2(Object[] a, String singular, String plural) { return n_fancy2(l(a), singular, plural); } static A optPar(ThreadLocal tl, A defaultValue) { A a = tl.get(); if (a != null) { tl.set(null); return a; } return defaultValue; } static A optPar(ThreadLocal tl) { return optPar(tl, null); } static Object optPar(Object[] params, String name) { return optParam(params, name); } static Object optPar(String name, Object[] params) { return optParam(params, name); } static Object optPar(String name, Map params) { return optParam(name, params); } static A optPar(Object[] params, String name, A defaultValue) { return optParam(params, name, defaultValue); } static A optPar(String name, Object[] params, A defaultValue) { return optParam(params, name, defaultValue); } static String fileName(File f) { return f == null ? null : f.getName(); } static boolean boolPar(ThreadLocal tl) { return boolOptParam(tl); } // defaults to false static boolean boolPar(Object[] __, String name) { return boolOptParam(__, name); } static boolean boolPar(String name, Object[] __) { return boolOptParam(__, name); } static boolean boolPar(String name, Map __) { return boolOptParam(name, __); } static boolean boolPar(String name, Object[] params, boolean defaultValue) { return optParam(params, name, defaultValue); } static String toUpper(String s) { return s == null ? null : s.toUpperCase(); } static List toUpper(Collection s) { return allToUpper(s); } static HashMap litmap(Object... x) { HashMap map = new HashMap(); litmap_impl(map, x); return map; } static void litmap_impl(Map map, Object... x) { if (x != null) for (int i = 0; i < x.length-1; i += 2) if (x[i+1] != null) map.put(x[i], x[i+1]); } static BufferedReader bufferedReader(Reader r) { return bufferedReader(r, 8192); } static BufferedReader bufferedReader(Reader r, int bufSize) { return r instanceof BufferedReader ? (BufferedReader) r : _registerIOWrap(new BufferedReader(r, bufSize), r); } static StringReader stringReader(String s) { return new StringReader(s); } static long sysNow() { ping(); return System.nanoTime()/1000000; } static LCSearcher_v1 lcSearcherFromLineCompReader(LineCompReader lc) { return lcSearcherFromLineCompReader(lc, null); } static LCSearcher_v1 lcSearcherFromLineCompReader(LineCompReader lc, LCSearcher_v1 searcher) { if (searcher == null) searcher = new LCSearcher_v1(); searcher.big = true; searcher.literals = listToHashMap(lmap(__18 -> first(__18),lc.literals)); searcher.iFirstPair = l(lc.literals); //searcher.productions = repNull(searcher.iFirstPair); for (long p : lc.pairs.asVirtualList()) { //searcher.productions.add(intPairToList(longToIntPair(p))); searcher.addPair(p); } searcher.iFirstFile = searcher.iFirstPair+l(lc.pairs); searcher.fileNames = new HashMap(); for (Map.Entry> __0 : _entrySet( lc.versions)) { String name = __0.getKey(); List enc = __0.getValue(); searcher.fileNames.put(searcher.addFile(enc), name); } searcher.prepare(); return searcher; } static long done2_always(long startTime, String desc) { long time = sysNow()-startTime; // BREAKING CHANGE: Now stores result for lastTiming() saveTiming_noPrint(time); print(desc + " [" + time + " ms]"); return time; } static long done2_always(String desc, long startTime) { return done2_always(startTime, desc); } static long done2_always(long startTime) { return done2_always(startTime, ""); } static File createTempFile() { return createTempFile("tmp", null); } static File createTempFile(String prefix, String suffix) { try { prefix = nohup_sanitize(prefix); suffix = nohup_sanitize(suffix); if (shouldKeepTempFiles()) return mkdirsForFile(javaxCachesDir("Temp/" + prefix + "-" + randomID() + "-" + suffix)); // File.createTempFile needs at least 3 prefix characters, so we // fíll them up for you File f = File.createTempFile(takeFirst(10, pad(prefix, 3, '-')), suffix); f.deleteOnExit(); return f; } catch (Exception __e) { throw rethrow(__e); } } static String addPrefixIfNempty(String prefix, String s) { return addPrefixIfNotEmpty(prefix, s); } static BufferedOutputStream bufferedOutputStream(OutputStream out) { if (out == null) return null; if (out instanceof BufferedOutputStream) return ((BufferedOutputStream) out); return new BufferedOutputStream(out, defaultBufferedOutputStreamSize()); } static FileOutputStream newFileOutputStream(File path) throws IOException { return newFileOutputStream(path.getPath()); } static FileOutputStream newFileOutputStream(String path) throws IOException { return newFileOutputStream(path, false); } static FileOutputStream newFileOutputStream(File path, boolean append) throws IOException { return newFileOutputStream(path.getPath(), append); } static FileOutputStream newFileOutputStream(String path, boolean append) throws IOException { mkdirsForFile(path); FileOutputStream f = new FileOutputStream(path, append); _registerIO(f, path, true); return f; } static DataOutputStream dataOutputStream(File f) { try { return new DataOutputStream(bufferedOutputStream(newFileOutputStream(f))); } catch (Exception __e) { throw rethrow(__e); } } static DataOutputStream dataOutputStream(OutputStream out) { try { return new DataOutputStream(out); } catch (Exception __e) { throw rethrow(__e); } } static byte[] toUTF8(String s) { return toUtf8(s); } static A assertEquals(Object x, A y) { return assertEquals(null, x, y); } static A assertEquals(String msg, Object x, A y) { if (assertVerbose()) return assertEqualsVerbose(msg, x, y); if (!(x == null ? y == null : x.equals(y))) throw fail((msg != null ? msg + ": " : "") + y + " != " + x); return y; } static boolean odd(int i) { return (i & 1) != 0; } static boolean odd(long i) { return (i & 1) != 0; } static boolean odd(BigInteger i) { return odd(toInt(i)); } static byte[] compress30BitUint(int i) { if (i < 0 || i >= 0x40000000) throw fail("Not a 30 bit uint: " + i); if (i < 0x40) return new byte[] { (byte) i }; if (i < 0x4000) return new byte[] { (byte) ((i >> 8) | 0x40), (byte) i }; if (i < 0x400000) return new byte[] { (byte) ((i >> 16) | 0x80), (byte) (i >> 8), (byte) i }; return new byte[] { (byte) ((i >> 24) | 0xC0), (byte) (i >> 16), (byte) (i >> 8), (byte) i }; } static String unnull(String s) { return s == null ? "" : s; } static Collection unnull(Collection l) { return l == null ? emptyList() : l; } static List unnull(List l) { return l == null ? emptyList() : l; } static int[] unnull(int[] l) { return l == null ? emptyIntArray() : l; } static char[] unnull(char[] l) { return l == null ? emptyCharArray() : l; } static double[] unnull(double[] l) { return l == null ? emptyDoubleArray() : l; } static Map unnull(Map l) { return l == null ? emptyMap() : l; } static Iterable unnull(Iterable i) { return i == null ? emptyList() : i; } static A[] unnull(A[] a) { return a == null ? (A[]) emptyObjectArray() : a; } static BitSet unnull(BitSet b) { return b == null ? new BitSet() : b; } //ifclass Symbol static Pair unnull(Pair p) { return p != null ? p : new Pair(null, null); } static long unnull(Long l) { return l == null ? 0L : l; } static int oneMillion() { return 1000000; } static String nSymbols(long n) { return n2(n, "symbol"); } static String nSymbols(Collection l) { return nSymbols(l(l)); } static String nSymbols(Map map) { return nSymbols(l(map)); } static void writeIntArrayToRandomAccessFile(RandomAccessFile raf, int[] array) { try { for (IntRange r : intRangeChunks(l(array), 1024)) raf.write(intArrayToBytes(subIntArray(array, r))); } catch (Exception __e) { throw rethrow(__e); } } static File printFileInfo(File f) { return printFileInfo("", f); } static File printFileInfo(String s, File f) { print(s, renderFileInfo(f)); return f; } static Class javax() { return getJavaX(); } static AutoCloseable tempSetThreadLocal(final ThreadLocal tl, A a) { if (tl == null) return null; final A prev = setThreadLocal(tl, a); return new AutoCloseable() { public String toString() { return "tl.set(prev);"; } public void close() throws Exception { tl.set(prev); }}; } static Object call_withVarargs(Object o, String method, Object... args) { try { if (o == null) return null; if (o instanceof Class) { Class c = (Class) o; _MethodCache cache = callOpt_getCache(c); Method me = cache.findStaticMethod(method, args); if (me != null) return invokeMethod(me, null, args); // try varargs List methods = cache.cache.get(method); if (methods != null) methodSearch: for (Method m : methods) { { if (!(m.isVarArgs())) continue; } { if (!(isStaticMethod(m))) continue; } Object[] newArgs = massageArgsForVarArgsCall(m, args); if (newArgs != null) return invokeMethod(m, null, newArgs); } throw fail("Method " + c.getName() + "." + method + "(" + joinWithComma(classNames(args)) + ") not found"); } else { Class c = o.getClass(); _MethodCache cache = callOpt_getCache(c); Method me = cache.findMethod(method, args); if (me != null) return invokeMethod(me, o, args); // try varargs List methods = cache.cache.get(method); if (methods != null) methodSearch: for (Method m : methods) { { if (!(m.isVarArgs())) continue; } Object[] newArgs = massageArgsForVarArgsCall(m, args); if (newArgs != null) return invokeMethod(m, o, newArgs); } throw fail("Method " + c.getName() + "." + method + "(" + joinWithComma(classNames(args)) + ") not found"); } } catch (Exception __e) { throw rethrow(__e); } } static Map newWeakHashMap() { return _registerWeakMap(synchroMap(new WeakHashMap())); } // TODO: test if android complains about this static boolean isAWTThread() { if (isAndroid()) return false; if (isHeadless()) return false; return isAWTThread_awt(); } static boolean isAWTThread_awt() { return SwingUtilities.isEventDispatchThread(); } static boolean isTrue(Object o) { if (o instanceof Boolean) return ((Boolean) o).booleanValue(); if (o == null) return false; if (o instanceof ThreadLocal) // TODO: remove this return isTrue(((ThreadLocal) o).get()); throw fail(getClassName(o)); } static void failIfUnlicensed() { assertTrue("license off", licensed()); } static Thread currentThread() { return Thread.currentThread(); } //static final Map> getOpt_cache = newDangerousWeakHashMap(f getOpt_special_init); static class getOpt_Map extends WeakHashMap { getOpt_Map() { if (getOpt_special == null) getOpt_special = new HashMap(); clear(); } public void clear() { super.clear(); //print("getOpt clear"); put(Class.class, getOpt_special); put(String.class, getOpt_special); } } static final Map> getOpt_cache = _registerDangerousWeakMap(synchroMap(new getOpt_Map())); //static final Map> getOpt_cache = _registerWeakMap(synchroMap(new getOpt_Map)); static HashMap getOpt_special; // just a marker /*static void getOpt_special_init(Map map) { map.put(Class.class, getOpt_special); map.put(S.class, getOpt_special); }*/ static Object getOpt_cached(Object o, String field) { try { if (o == null) return null; Class c = o.getClass(); HashMap map; synchronized(getOpt_cache) { map = getOpt_cache.get(c); if (map == null) map = getOpt_makeCache(c); } if (map == getOpt_special) { if (o instanceof Class) return getOpt((Class) o, field); /*if (o instanceof S) ret getOpt(getBot((S) o), field);*/ if (o instanceof Map) return ((Map) o).get(field); } Field f = map.get(field); if (f != null) return f.get(o); if (o instanceof DynamicObject) return mapGet2(((DynamicObject) o).fieldValues, field); return null; } catch (Exception __e) { throw rethrow(__e); } } // used internally - we are in synchronized block static HashMap getOpt_makeCache(Class c) { HashMap map; if (isSubtypeOf(c, Map.class)) map = getOpt_special; else { map = new HashMap(); if (!reflection_classesNotToScan().contains(c.getName())) { Class _c = c; do { for (Field f : _c.getDeclaredFields()) { makeAccessible(f); String name = f.getName(); if (!map.containsKey(name)) map.put(name, f); } _c = _c.getSuperclass(); } while (_c != null); } } if (getOpt_cache != null) getOpt_cache.put(c, map); return map; } static Field getOpt_findField(Class c, String field) { Class _c = c; do { for (Field f : _c.getDeclaredFields()) if (f.getName().equals(field)) return f; _c = _c.getSuperclass(); } while (_c != null); return null; } static List _registerDangerousWeakMap_preList; static A _registerDangerousWeakMap(A map) { return _registerDangerousWeakMap(map, null); } static A _registerDangerousWeakMap(A map, Object init) { callF(init, map); if (init instanceof String) { final String f = (String) init; init = new VF1() { public void get(Map map) { try { callMC(f, map) ; } catch (Exception __e) { throw rethrow(__e); } } public String toString() { return "callMC(f, map)"; }}; } if (javax() == null) { // We're in class init if (_registerDangerousWeakMap_preList == null) _registerDangerousWeakMap_preList = synchroList(); _registerDangerousWeakMap_preList.add(pair(map, init)); return map; } call(javax(), "_registerDangerousWeakMap", map, init); return map; } static void _onLoad_registerDangerousWeakMap() { assertNotNull(javax()); if (_registerDangerousWeakMap_preList == null) return; for (Pair p : _registerDangerousWeakMap_preList) _registerDangerousWeakMap(p.a, p.b); _registerDangerousWeakMap_preList = null; } static Map synchroMap() { return synchroHashMap(); } static Map synchroMap(Map map) { return Collections.synchronizedMap(map); } static Throwable getExceptionCause(Throwable e) { Throwable c = e.getCause(); return c != null ? c : e; } static String joinWithSpace(Iterable c) { return join(" ", c); } static String joinWithSpace(String... c) { return join(" ", c); } static List classNames(Collection l) { return getClassNames(l); } static List classNames(Object[] l) { return getClassNames(Arrays.asList(l)); } static boolean isInstanceX(Class type, Object arg) { if (type == boolean.class) return arg instanceof Boolean; if (type == int.class) return arg instanceof Integer; if (type == long.class) return arg instanceof Long; if (type == float.class) return arg instanceof Float; if (type == short.class) return arg instanceof Short; if (type == char.class) return arg instanceof Character; if (type == byte.class) return arg instanceof Byte; if (type == double.class) return arg instanceof Double; return type.isInstance(arg); } static ArrayList cloneList(Iterable l) { return l instanceof Collection ? cloneList((Collection) l) : asList(l); } static ArrayList cloneList(Collection l) { if (l == null) return new ArrayList(); synchronized(collectionMutex(l)) { return new ArrayList(l); } } static Object pcallF(Object f, Object... args) { return pcallFunction(f, args); } static A pcallF(F0 f) { try { return f == null ? null : f.get(); } catch (Throwable __e) { return null; } } static B pcallF(F1 f, A a) { try { return f == null ? null : f.get(a); } catch (Throwable __e) { return null; } } static void pcallF(VF1 f, A a) { try { if (f != null) f.get(a); } catch (Throwable __e) { _handleException(__e); } } static Set vm_generalIdentityHashSet(Object name) { synchronized(get(javax(), "generalMap")) { Set set = (Set) (vm_generalMap_get(name)); if (set == null) vm_generalMap_put(name, set = syncIdentityHashSet()); return set; } } static Map vm_generalHashMap(Object name) { synchronized(get(javax(), "generalMap")) { Map m = (Map) (vm_generalMap_get(name)); if (m == null) vm_generalMap_put(name, m = syncHashMap()); return m; } } static String formatWithThousandsSeparator(long l) { return NumberFormat.getInstance(new Locale("en_US")).format(l); } static String trim(String s) { return s == null ? null : s.trim(); } static String trim(StringBuilder buf) { return buf.toString().trim(); } static String trim(StringBuffer buf) { return buf.toString().trim(); } static A optParam(ThreadLocal tl, A defaultValue) { return optPar(tl, defaultValue); } static A optParam(ThreadLocal tl) { return optPar(tl); } static Object optParam(String name, Map params) { return mapGet(params, name); } // now also takes a map as single array entry static A optParam(Object[] opt, String name, A defaultValue) { int n = l(opt); if (n == 1 && opt[0] instanceof Map) { Map map = (Map) (opt[0]); return map.containsKey(name) ? (A) map.get(name) : defaultValue; } if (!even(l(opt))) throw fail("Odd parameter length"); for (int i = 0; i < l(opt); i += 2) if (eq(opt[i], name)) return (A) opt[i+1]; return defaultValue; } static Object optParam(Object[] opt, String name) { return optParam(opt, name, null); } static Object optParam(String name, Object[] params) { return optParam(params, name); } static boolean boolOptParam(ThreadLocal tl) { return isTrue(optPar(tl)); } // defaults to false static boolean boolOptParam(Object[] __, String name) { return isTrue(optParam(__, name)); } static boolean boolOptParam(String name, Object[] __) { return boolOptParam(__, name); } static boolean boolOptParam(String name, Map __) { return isTrue(optPar(name, __)); } static List allToUpper(Collection l) { List x = new ArrayList(l(l)); if (l != null) for (String s : l) x.add(upper(s)); return x; } static A _registerIOWrap(A wrapper, Object wrapped) { return wrapper; } static HashMap listToHashMap(List l) { if (l == null) return null; HashMap map = new HashMap(); int n = l(l); for (int i = 0; i < n; i++) map.put(i, l.get(i)); return map; } static List lmap(IF1 f, Iterable l) { return lambdaMap(f, l); } static List lmap(IF1 f, A[] l) { return lambdaMap(f, l); } static Object first(Object list) { return first((Iterable) list); } static A first(List list) { return empty(list) ? null : list.get(0); } static A first(A[] bla) { return bla == null || bla.length == 0 ? null : bla[0]; } static A first(IterableIterator i) { return first((Iterator) i); } static A first(Iterator i) { return i == null || !i.hasNext() ? null : i.next(); } static A first(Iterable i) { if (i == null) return null; Iterator it = i.iterator(); return it.hasNext() ? it.next() : null; } static Character first(String s) { return empty(s) ? null : s.charAt(0); } static Character first(CharSequence s) { return empty(s) ? null : s.charAt(0); } static A first(Pair p) { return p == null ? null : p.a; } static Byte first(byte[] l) { return empty(l) ? null : l[0]; } static int first(IntBuffer buf) { return buf.get(0); } static Set> _entrySet(Map map) { return map == null ? Collections.EMPTY_SET : map.entrySet(); } static ThreadLocal saveTiming_last = new ThreadLocal(); static void saveTiming(long ms) { print(ms + " ms"); saveTiming_noPrint(ms); } static void saveTiming_noPrint(long ms) { saveTiming_last.set(ms); } static String nohup_sanitize(String s) { return empty(s) ? s : takeFirst(50, s.replaceAll("[^.a-zA-Z0-9\\-_]", "")); } static IF0 shouldKeepTempFiles; static boolean shouldKeepTempFiles() { return shouldKeepTempFiles != null ? shouldKeepTempFiles.get() : shouldKeepTempFiles_base(); } final static boolean shouldKeepTempFiles_fallback(IF0 _f) { return _f != null ? _f.get() : shouldKeepTempFiles_base(); } static boolean shouldKeepTempFiles_base() { return false; } public static File mkdirsForFile(File file) { File dir = file.getParentFile(); if (dir != null) { // is null if file is in current dir dir.mkdirs(); if (!dir.isDirectory()) if (dir.isFile()) throw fail("Please delete the file " + f2s(dir) + " - it is supposed to be a directory!"); else throw fail("Unknown IO exception during mkdirs of " + f2s(file)); } return file; } public static String mkdirsForFile(String path) { mkdirsForFile(new File(path)); return path; } static File javaxCachesDir_dir; // can be set to work on different base dir static File javaxCachesDir() { return javaxCachesDir_dir != null ? javaxCachesDir_dir : new File(userHome(), "JavaX-Caches"); } static File javaxCachesDir(String sub) { return newFile(javaxCachesDir(), sub); } static int randomID_defaultLength = 12; static String randomID(int length) { return makeRandomID(length); } static String randomID(Random r, int length) { return makeRandomID(r, length); } static String randomID() { return randomID(randomID_defaultLength); } static String randomID(Random r) { return randomID(r, randomID_defaultLength); } static List takeFirst(List l, int n) { return l(l) <= n ? l : newSubListOrSame(l, 0, n); } static List takeFirst(int n, List l) { return takeFirst(l, n); } static String takeFirst(int n, String s) { return substring(s, 0, n); } static String takeFirst(String s, int n) { return substring(s, 0, n); } static CharSequence takeFirst(int n, CharSequence s) { return subCharSequence(s, 0, n); } static List takeFirst(int n, Iterable i) { if (i == null) return null; List l = new ArrayList(); Iterator it = i.iterator(); for (int _repeat_0 = 0; _repeat_0 < n; _repeat_0++) { if (it.hasNext()) l.add(it.next()); else break; } return l; } static int[] takeFirst(int n, int[] a) { return takeFirstOfIntArray(n, a); } static String pad(Object s, int l) { return pad(s, l, ' '); } static String pad(Object s, int l, char c) { String _s = str(s); if (lengthOfString(_s) >= l) return _s; return rep(c, l-lengthOfString(_s)) + _s; } static String addPrefixIfNotEmpty(String prefix, String s) { return empty(s) ? "" : prefix + s; } static int defaultBufferedOutputStreamSize() { return 65536; } static void _registerIO(Object object, String path, boolean opened) { } static byte[] toUtf8(String s) { try { return s.getBytes(utf8charset()); } catch (Exception __e) { throw rethrow(__e); } } static ThreadLocal assertVerbose_value = new ThreadLocal(); static void assertVerbose(boolean b) { assertVerbose_value.set(b); } static boolean assertVerbose() { return isTrue(assertVerbose_value.get()); } static A assertEqualsVerbose(Object x, A y) { assertEqualsVerbose((String) null, x, y); return y; } // x = expected, y = actual static A assertEqualsVerbose(String msg, Object x, A y) { if (!eq(x, y)) { throw fail((msg != null ? msg + ": " : "") + "expected: "+ x + ", got: " + y); } else print("OK" + (empty(msg) ? "" : " " + msg) + ": " + /*sfu*/(x)); return y; } static int toInt(Object o) { if (o == null) return 0; if (o instanceof Number) return ((Number) o).intValue(); if (o instanceof String) return parseInt(((String) o)); if (o instanceof Boolean) return boolToInt(((Boolean) o)); throw fail("woot not int: " + getClassName(o)); } static int toInt(long l) { if (l != (int) l) throw fail("Too large for int: " + l); return (int) l; } static ArrayList emptyList() { return new ArrayList(); //ret Collections.emptyList(); } static ArrayList emptyList(int capacity) { return new ArrayList(max(0, capacity)); } // Try to match capacity static ArrayList emptyList(Iterable l) { return l instanceof Collection ? emptyList(((Collection) l).size()) : emptyList(); } static ArrayList emptyList(Object[] l) { return emptyList(l(l)); } // get correct type at once static ArrayList emptyList(Class c) { return new ArrayList(); } static int[] emptyIntArray_a = new int[0]; static int[] emptyIntArray() { return emptyIntArray_a; } static char[] emptyCharArray = new char[0]; static char[] emptyCharArray() { return emptyCharArray; } static double[] emptyDoubleArray = new double[0]; static double[] emptyDoubleArray() { return emptyDoubleArray; } static Map emptyMap() { return new HashMap(); } static Object[] emptyObjectArray_a = new Object[0]; static Object[] emptyObjectArray() { return emptyObjectArray_a; } static List intRangeChunks(int end, int chunkSize) { return intRangeChunks(0, end, chunkSize); } static List intRangeChunks(int start, int end, int chunkSize) { List l = new ArrayList(); int i = start; while (i < end) { l.add(new IntRange(i, min(end, i+chunkSize))); i += chunkSize; } return l; } // gratuitous convenience syntax for processing sound samples static List intRangeChunks(short[] l, int chunkSize) { return intRangeChunks(l(l), chunkSize); } static byte[] intArrayToBytes(int[] a) { byte[] b = new byte[a.length*4]; for (int i = 0; i < a.length; i++) intToBytes_inArray(a[i], b, i*4); return b; } static int[] subIntArray(int[] b, int start) { return subIntArray(b, start, l(b)); } static int[] subIntArray(int[] b, int start, int end) { start = max(start, 0); end = min(end, l(b)); if (start == 0 && end == l(b)) return b; if (start >= end) return new int[0]; int[] x = new int[end-start]; System.arraycopy(b, start, x, 0, end-start); return x; } static int[] subIntArray(int[] a, IntRange r) { return r == null ? null : subIntArray(a, r.start, r.end); } static String renderFileInfo(File f) { return f == null ? "-" : f2s(f) + " " + (f.isFile() ? "(file, " + n2(fileSize(f)) + " bytes)" : f.isDirectory() ? "(dir)" : "(not found)"); } static Class __javax; static Class getJavaX() { try { return __javax; } catch (Exception __e) { throw rethrow(__e); } } static A setThreadLocal(ThreadLocal tl, A value) { if (tl == null) return null; A old = tl.get(); tl.set(value); return old; } static final Map callOpt_cache = newDangerousWeakHashMap(); static Object callOpt_cached(Object o, String methodName, Object... args) { try { if (o == null) return null; if (o instanceof Class) { Class c = (Class) o; _MethodCache cache = callOpt_getCache(c); // TODO: (super-rare) case where method exists static and non-static // with different args Method me = cache.findMethod(methodName, args); if (me == null || (me.getModifiers() & Modifier.STATIC) == 0) return null; return invokeMethod(me, null, args); } else { Class c = o.getClass(); _MethodCache cache = callOpt_getCache(c); Method me = cache.findMethod(methodName, args); if (me == null) return null; return invokeMethod(me, o, args); } } catch (Exception __e) { throw rethrow(__e); } } static _MethodCache callOpt_getCache(Class c) { synchronized(callOpt_cache) { _MethodCache cache = callOpt_cache.get(c); if (cache == null) callOpt_cache.put(c, cache = new _MethodCache(c)); return cache; } } static boolean isStaticMethod(Method m) { return methodIsStatic(m); } static Object[] massageArgsForVarArgsCall(Method m, Object[] args) { Class[] types = m.getParameterTypes(); int n = types.length-1, nArgs = args.length; if (nArgs < n) return null; for (int i = 0; i < n; i++) if (!argumentCompatibleWithType(args[i], types[i])) return null; Class varArgType = types[n].getComponentType(); for (int i = n; i < nArgs; i++) if (!argumentCompatibleWithType(args[i], varArgType)) return null; Object[] newArgs = new Object[n+1]; arraycopy(args, 0, newArgs, 0, n); Object[] varArgs = arrayOfType(varArgType, nArgs-n); arraycopy(args, n, varArgs, 0, nArgs-n); newArgs[n] = varArgs; return newArgs; } static String joinWithComma(Collection c) { return join(", ", c); } static String joinWithComma(String... c) { return join(", ", c); } static String joinWithComma(Pair p) { return p == null ? "" : joinWithComma(str(p.a), str(p.b)); } static List _registerWeakMap_preList; static A _registerWeakMap(A map) { if (javax() == null) { // We're in class init if (_registerWeakMap_preList == null) _registerWeakMap_preList = synchroList(); _registerWeakMap_preList.add(map); return map; } try { call(javax(), "_registerWeakMap", map); } catch (Throwable e) { printException(e); print("Upgrade JavaX!!"); } return map; } static void _onLoad_registerWeakMap() { assertNotNull(javax()); if (_registerWeakMap_preList == null) return; for (Object o : _registerWeakMap_preList) _registerWeakMap(o); _registerWeakMap_preList = null; } static int isAndroid_flag; static boolean isAndroid() { if (isAndroid_flag == 0) isAndroid_flag = System.getProperty("java.vendor").toLowerCase().indexOf("android") >= 0 ? 1 : -1; return isAndroid_flag > 0; } static Boolean isHeadless_cache; static boolean isHeadless() { if (isHeadless_cache != null) return isHeadless_cache; if (isAndroid()) return isHeadless_cache = true; if (GraphicsEnvironment.isHeadless()) return isHeadless_cache = true; // Also check if AWT actually works. // If DISPLAY variable is set but no X server up, this will notice. try { SwingUtilities.isEventDispatchThread(); return isHeadless_cache = false; } catch (Throwable e) { return isHeadless_cache = true; } } static void assertTrue(Object o) { if (!(eq(o, true) /*|| isTrue(pcallF(o))*/)) throw fail(str(o)); } static boolean assertTrue(String msg, boolean b) { if (!b) throw fail(msg); return b; } static boolean assertTrue(boolean b) { if (!b) throw fail("oops"); return b; } static volatile boolean licensed_yes = true; static boolean licensed() { if (!licensed_yes) return false; ping_okInCleanUp(); return true; } static void licensed_off() { licensed_yes = false; } static void clear(Collection c) { if (c != null) c.clear(); } static void clear(Map map) { if (map != null) map.clear(); } static void put(Map map, A a, B b) { if (map != null) map.put(a, b); } static void put(List l, int i, A a) { if (l != null && i >= 0 && i < l(l)) l.set(i, a); } static B mapGet2(Map map, A a) { return map == null ? null : map.get(a); } static B mapGet2(A a, Map map) { return map == null ? null : map.get(a); } static boolean isSubtypeOf(Class a, Class b) { return b.isAssignableFrom(a); // << always hated that method, let's replace it! } static Set reflection_classesNotToScan_value = litset( "jdk.internal.loader.URLClassPath" ); static Set reflection_classesNotToScan() { return reflection_classesNotToScan_value; } static HashMap> callMC_cache = new HashMap(); static String callMC_key; static Method callMC_value; // varargs assignment fixer for a single string array argument static Object callMC(String method, String[] arg) { return callMC(method, new Object[] {arg}); } static Object callMC(String method, Object... args) { try { Method me; if (callMC_cache == null) callMC_cache = new HashMap(); // initializer time workaround synchronized(callMC_cache) { me = method == callMC_key ? callMC_value : null; } if (me != null) try { return invokeMethod(me, null, args); } catch (IllegalArgumentException e) { throw new RuntimeException("Can't call " + me + " with arguments " + classNames(args), e); } List m; synchronized(callMC_cache) { m = callMC_cache.get(method); } if (m == null) { if (callMC_cache.isEmpty()) { callMC_makeCache(); m = callMC_cache.get(method); } if (m == null) throw fail("Method named " + method + " not found in main"); } int n = m.size(); if (n == 1) { me = m.get(0); synchronized(callMC_cache) { callMC_key = method; callMC_value = me; } try { return invokeMethod(me, null, args); } catch (IllegalArgumentException e) { throw new RuntimeException("Can't call " + me + " with arguments " + classNames(args), e); } } for (int i = 0; i < n; i++) { me = m.get(i); if (call_checkArgs(me, args, false)) return invokeMethod(me, null, args); } throw fail("No method called " + method + " with arguments (" + joinWithComma(getClasses(args)) + ") found in main"); } catch (Exception __e) { throw rethrow(__e); } } static void callMC_makeCache() { synchronized(callMC_cache) { callMC_cache.clear(); Class _c = (Class) mc(), c = _c; while (c != null) { for (Method m : c.getDeclaredMethods()) if ((m.getModifiers() & java.lang.reflect.Modifier.STATIC) != 0) { makeAccessible(m); multiMapPut(callMC_cache, m.getName(), m); } c = c.getSuperclass(); } } } static List synchroList() { return Collections.synchronizedList(new ArrayList()); } static List synchroList(List l) { return Collections.synchronizedList(l); } static Pair pair(A a, B b) { return new Pair(a, b); } static Pair pair(A a) { return new Pair(a, a); } static A assertNotNull(A a) { assertTrue(a != null); return a; } static A assertNotNull(String msg, A a) { assertTrue(msg, a != null); return a; } static Map synchroHashMap() { return Collections.synchronizedMap(new HashMap()); } public static String join(String glue, Iterable strings) { if (strings == null) return ""; if (strings instanceof Collection) { if (((Collection) strings).size() == 1) return str(first(((Collection) strings))); } StringBuilder buf = new StringBuilder(); Iterator i = strings.iterator(); if (i.hasNext()) { buf.append(i.next()); while (i.hasNext()) buf.append(glue).append(i.next()); } return buf.toString(); } public static String join(String glue, String... strings) { return join(glue, Arrays.asList(strings)); } static String join(Iterable strings) { return join("", strings); } static String join(Iterable strings, String glue) { return join(glue, strings); } public static String join(String[] strings) { return join("", strings); } static String join(String glue, Pair p) { return p == null ? "" : str(p.a) + glue + str(p.b); } static List getClassNames(Collection l) { List out = new ArrayList(); if (l != null) for (Object o : l) out.add(o == null ? null : getClassName(o)); return out; } // unclear semantics as to whether return null on null static ArrayList asList(A[] a) { return a == null ? new ArrayList() : new ArrayList(Arrays.asList(a)); } static ArrayList asList(int[] a) { if (a == null) return null; ArrayList l = emptyList(a.length); for (int i : a) l.add(i); return l; } static ArrayList asList(long[] a) { if (a == null) return null; ArrayList l = emptyList(a.length); for (long i : a) l.add(i); return l; } static ArrayList asList(float[] a) { if (a == null) return null; ArrayList l = emptyList(a.length); for (float i : a) l.add(i); return l; } static ArrayList asList(double[] a) { if (a == null) return null; ArrayList l = emptyList(a.length); for (double i : a) l.add(i); return l; } static ArrayList asList(Iterable s) { if (s instanceof ArrayList) return (ArrayList) s; ArrayList l = new ArrayList(); if (s != null) for (A a : s) l.add(a); return l; } static ArrayList asList(Producer p) { ArrayList l = new ArrayList(); A a; if (p != null) while ((a = p.next()) != null) l.add(a); return l; } static ArrayList asList(Enumeration e) { ArrayList l = new ArrayList(); if (e != null) while (e.hasMoreElements()) l.add(e.nextElement()); return l; } static Object collectionMutex(List l) { return l; } static Object collectionMutex(Object o) { if (o instanceof List) return o; String c = className(o); if (eq(c, "java.util.TreeMap$KeySet")) c = className(o = getOpt(o, "m")); else if (eq(c, "java.util.HashMap$KeySet")) c = className(o = get_raw(o, "this$0")); if (eqOneOf(c, "java.util.TreeMap$AscendingSubMap", "java.util.TreeMap$DescendingSubMap")) c = className(o = get_raw(o, "m")); return o; } static Object pcallFunction(Object f, Object... args) { try { return callFunction(f, args); } catch (Throwable __e) { _handleException(__e); } return null; } static volatile PersistableThrowable _handleException_lastException; static List _handleException_onException = synchroList(ll("printStackTrace2")); static void _handleException(Throwable e) { _handleException_lastException = persistableThrowable(e); Throwable e2 = innerException(e); if (e2.getClass() == RuntimeException.class && eq(e2.getMessage(), "Thread cancelled.") || e2 instanceof InterruptedException) return; for (Object f : cloneList(_handleException_onException)) try { callF(f, e); } catch (Throwable e3) { try { printStackTrace2(e3); // not using pcall here - it could lead to endless loops } catch (Throwable e4) { System.out.println(getStackTrace(e3)); System.out.println(getStackTrace(e4)); } } } static A get(List l, int idx) { return l != null && idx >= 0 && idx < l(l) ? l.get(idx) : null; } // seems to conflict with other signatures /*static B get(Map map, A key) { ret map != null ? map.get(key) : null; }*/ static A get(A[] l, int idx) { return idx >= 0 && idx < l(l) ? l[idx] : null; } // default to false static boolean get(boolean[] l, int idx) { return idx >= 0 && idx < l(l) ? l[idx] : false; } // get purpose 2: access a field by reflection or a map static Object get(Object o, String field) { try { if (o == null) return null; if (o instanceof Class) return get((Class) o, field); if (o instanceof Map) return ((Map) o).get(field); Field f = getOpt_findField(o.getClass(), field); if (f != null) { makeAccessible(f); return f.get(o); } if (o instanceof DynamicObject) return ((DynamicObject) o).fieldValues.get(field); } catch (Exception e) { throw asRuntimeException(e); } throw new RuntimeException("Field '" + field + "' not found in " + o.getClass().getName()); } static Object get_raw(String field, Object o) { return get_raw(o, field); } static Object get_raw(Object o, String field) { try { if (o == null) return null; Field f = get_findField(o.getClass(), field); makeAccessible(f); return f.get(o); } catch (Exception __e) { throw rethrow(__e); } } static Object get(Class c, String field) { try { Field f = get_findStaticField(c, field); makeAccessible(f); return f.get(null); } catch (Exception e) { throw new RuntimeException(e); } } static Field get_findStaticField(Class c, String field) { Class _c = c; do { for (Field f : _c.getDeclaredFields()) if (f.getName().equals(field) && (f.getModifiers() & java.lang.reflect.Modifier.STATIC) != 0) return f; _c = _c.getSuperclass(); } while (_c != null); throw new RuntimeException("Static field '" + field + "' not found in " + c.getName()); } static Field get_findField(Class c, String field) { Class _c = c; do { for (Field f : _c.getDeclaredFields()) if (f.getName().equals(field)) return f; _c = _c.getSuperclass(); } while (_c != null); throw new RuntimeException("Field '" + field + "' not found in " + c.getName()); } static Object get(String field, Object o) { return get(o, field); } static Object vm_generalMap_get(Object key) { return vm_generalMap().get(key); } static Object vm_generalMap_put(Object key, Object value) { return mapPutOrRemove(vm_generalMap(), key, value); } static Set syncIdentityHashSet() { return (Set) synchronizedSet(identityHashSet()); } static Map syncHashMap() { return synchroHashMap(); } static B mapGet(Map map, A a) { return map == null || a == null ? null : map.get(a); } static B mapGet(A a, Map map) { return map == null || a == null ? null : map.get(a); } static boolean even(int i) { return (i & 1) == 0; } static boolean even(long i) { return (i & 1) == 0; } static String upper(String s) { return s == null ? null : s.toUpperCase(); } static char upper(char c) { return Character.toUpperCase(c); } static List lambdaMap(IF1 f, Iterable l) { return map(l, f); } static List lambdaMap(IF1 f, A[] l) { return map(l, f); } static String f2s(File f) { return f == null ? null : f.getAbsolutePath(); } static String f2s(String s) { return f2s(newFile(s)); } static String f2s(java.nio.file.Path p) { return p == null ? null : f2s(p.toFile()); } static String _userHome; static String userHome() { if (_userHome == null) return actualUserHome(); return _userHome; } static File userHome(String path) { return new File(userDir(), path); } static File newFile(File base, String... names) { for (String name : names) base = new File(base, name); return base; } static File newFile(String name) { return name == null ? null : new File(name); } static File newFile(String base, String... names) { return newFile(newFile(base), names); } static String makeRandomID(int length) { return makeRandomID(length, defaultRandomGenerator()); } static String makeRandomID(int length, Random random) { char[] id = new char[length]; for (int i = 0; i < id.length; i++) id[i] = (char) ((int) 'a' + random.nextInt(26)); return new String(id); } static String makeRandomID(Random r, int length) { return makeRandomID(length, r); } static List newSubListOrSame(List l, int startIndex) { return newSubListOrSame(l, startIndex, l(l)); } static List newSubListOrSame(List l, int startIndex, int endIndex) { if (l == null) return null; int n = l(l); startIndex = max(0, startIndex); endIndex = min(n, endIndex); if (startIndex >= endIndex) return ll(); if (startIndex == 0 && endIndex == n) return l; return cloneList(l.subList(startIndex, endIndex)); } static List newSubListOrSame(List l, IntRange r) { return newSubListOrSame(l, r.start, r.end); } static String substring(String s, int x) { return substring(s, x, strL(s)); } static String substring(String s, int x, int y) { if (s == null) return null; if (x < 0) x = 0; int n = s.length(); if (y < x) y = x; if (y > n) y = n; if (x >= y) return ""; return s.substring(x, y); } static String substring(String s, IntRange r) { return r == null ? null : substring(s, r.start, r.end); } // convenience method for quickly dropping a prefix static String substring(String s, CharSequence l) { return substring(s, l(l)); } static int[] takeFirstOfIntArray(int[] b, int n) { return subIntArray(b, 0, n); } static int[] takeFirstOfIntArray(int n, int[] b) { return takeFirstOfIntArray(b, n); } static int lengthOfString(String s) { return s == null ? 0 : s.length(); } static String rep(int n, char c) { return repeat(c, n); } static String rep(char c, int n) { return repeat(c, n); } static List rep(A a, int n) { return repeat(a, n); } static List rep(int n, A a) { return repeat(n, a); } static Charset utf8charset_cache; static Charset utf8charset() { if (utf8charset_cache == null) utf8charset_cache = utf8charset_load(); return utf8charset_cache; } static Charset utf8charset_load() { return Charset.forName("UTF-8"); } static int parseInt(String s) { return emptyString(s) ? 0 : Integer.parseInt(s); } static int parseInt(char c) { return Integer.parseInt(str(c)); } static int boolToInt(boolean b) { return b ? 1 : 0; } static int max(int a, int b) { return Math.max(a, b); } static int max(int a, int b, int c) { return max(max(a, b), c); } static long max(int a, long b) { return Math.max((long) a, b); } static long max(long a, long b) { return Math.max(a, b); } static double max(int a, double b) { return Math.max((double) a, b); } static float max(float a, float b) { return Math.max(a, b); } static double max(double a, double b) { return Math.max(a, b); } static int max(Collection c) { int x = Integer.MIN_VALUE; for (int i : c) x = max(x, i); return x; } static double max(double[] c) { if (c.length == 0) return Double.MIN_VALUE; double x = c[0]; for (int i = 1; i < c.length; i++) x = Math.max(x, c[i]); return x; } static float max(float[] c) { if (c.length == 0) return Float.MAX_VALUE; float x = c[0]; for (int i = 1; i < c.length; i++) x = Math.max(x, c[i]); return x; } static byte max(byte[] c) { byte x = -128; for (byte d : c) if (d > x) x = d; return x; } static short max(short[] c) { short x = -0x8000; for (short d : c) if (d > x) x = d; return x; } static int max(int[] c) { int x = Integer.MIN_VALUE; for (int d : c) if (d > x) x = d; return x; } static void intToBytes_inArray(int i, byte[] array, int idx) { array[idx] = (byte) (i >>> 24); array[idx+1] = (byte) (i >>> 16); array[idx+2] = (byte) (i >>> 8); array[idx+3 ] = (byte) i; } static long fileSize(String path) { return getFileSize(path); } static long fileSize(File f) { return getFileSize(f); } static Throwable printStackTrace2(Throwable e) { // we go to system.out now - system.err is nonsense print(getStackTrace2(e)); return e; } static void printStackTrace2() { printStackTrace2(new Throwable()); } static void printStackTrace2(String msg) { printStackTrace2(new Throwable(msg)); } static boolean methodIsStatic(Method m) { return (m.getModifiers() & Modifier.STATIC) != 0; } static boolean argumentCompatibleWithType(Object arg, Class type) { return arg == null ? !type.isPrimitive() : isInstanceX(type, arg); } static void arraycopy(Object[] a, Object[] b) { if (a != null && b != null) arraycopy(a, 0, b, 0, Math.min(a.length, b.length)); } static void arraycopy(Object src, int srcPos, Object dest, int destPos, int n) { if (n != 0) System.arraycopy(src, srcPos, dest, destPos, n); } static A[] arrayOfType(Class type, int n) { return makeArray(type, n); } static A[] arrayOfType(int n, Class type) { return arrayOfType(type, n); } static A printException(A e) { printStackTrace(e); return e; } static HashSet litset(A... items) { return lithashset(items); } static List getClasses(Object[] array) { List l = emptyList(l(array)); for (Object o : array) l.add(_getClass(o)); return l; } static void multiMapPut(Map> map, A a, B b) { List l = map.get(a); if (l == null) map.put(a, l = new ArrayList()); l.add(b); } static void multiMapPut(MultiMap mm, A key, B value) { if (mm != null && key != null && value != null) mm.put(key, value); } static String className(Object o) { return getClassName(o); } static boolean eqOneOf(Object o, Object... l) { for (Object x : l) if (eq(o, x)) return true; return false; } static Object callFunction(Object f, Object... args) { return callF(f, args); } static List ll(A... a) { ArrayList l = new ArrayList(a.length); if (a != null) for (A x : a) l.add(x); return l; } static PersistableThrowable persistableThrowable(Throwable e) { return e == null ? null : new PersistableThrowable(e); } static Throwable innerException(Throwable e) { return getInnerException(e); } static String getStackTrace(Throwable throwable) { lastException(throwable); return getStackTrace_noRecord(throwable); } static String getStackTrace_noRecord(Throwable throwable) { StringWriter writer = new StringWriter(); throwable.printStackTrace(new PrintWriter(writer)); return hideCredentials(writer.toString()); } static String getStackTrace() { return getStackTrace_noRecord(new Throwable()); } static Map vm_generalMap_map; static Map vm_generalMap() { if (vm_generalMap_map == null) vm_generalMap_map = (Map) get(javax(), "generalMap"); return vm_generalMap_map; } static B mapPutOrRemove(Map map, A key, B value) { if (map != null && key != null) if (value != null) return map.put(key, value); else return map.remove(key); return null; } static Set synchronizedSet() { return synchroHashSet(); } static Set synchronizedSet(Set set) { return Collections.synchronizedSet(set); } static Set identityHashSet() { return Collections.newSetFromMap(new IdentityHashMap()); } static List map(Iterable l, Object f) { return map(f, l); } static List map(Object f, Iterable l) { List x = emptyList(l); if (l != null) for (Object o : l) x.add(callF(f, o)); return x; } static List map(Iterable l, F1 f) { return map(f, l); } static List map(F1 f, Iterable l) { List x = emptyList(l); if (l != null) for (A o : l) x.add(callF(f, o)); return x; } static List map(IF1 f, Iterable l) { return map(l, f); } static List map(Iterable l, IF1 f) { List x = emptyList(l); if (l != null) for (A o : l) x.add(f.get(o)); return x; } static List map(IF1 f, A[] l) { return map(l, f); } static List map(A[] l, IF1 f) { List x = emptyList(l); if (l != null) for (A o : l) x.add(f.get(o)); return x; } static List map(Object f, Object[] l) { return map(f, asList(l)); } static List map(Object[] l, Object f) { return map(f, l); } static List map(Object f, Map map) { return map(map, f); } // map: func(key, value) -> list element static List map(Map map, Object f) { List x = new ArrayList(); if (map != null) for (Object _e : map.entrySet()) { Map.Entry e = (Map.Entry) _e; x.add(callF(f, e.getKey(), e.getValue())); } return x; } static List map(Map map, IF2 f) { return map(map, (Object) f); } static String actualUserHome_value; static String actualUserHome() { if (actualUserHome_value == null) { if (isAndroid()) actualUserHome_value = "/storage/emulated/0/"; else actualUserHome_value = System.getProperty("user.home"); } return actualUserHome_value; } static File actualUserHome(String sub) { return newFile(new File(actualUserHome()), sub); } static File userDir() { return new File(userHome()); } static File userDir(String path) { return new File(userHome(), path); } static Random defaultRandomGenerator() { return ThreadLocalRandom.current(); } static int strL(String s) { return s == null ? 0 : s.length(); } static String repeat(char c, int n) { n = Math.max(n, 0); char[] chars = new char[n]; for (int i = 0; i < n; i++) chars[i] = c; return new String(chars); } static List repeat(A a, int n) { n = Math.max(n, 0); List l = new ArrayList(n); for (int i = 0; i < n; i++) l.add(a); return l; } static List repeat(int n, A a) { return repeat(a, n); } static boolean emptyString(String s) { return s == null || s.length() == 0; } static String getStackTrace2(Throwable e) { return hideCredentials(getStackTrace(unwrapTrivialExceptionWraps(e)) + replacePrefix("java.lang.RuntimeException: ", "FAIL: ", hideCredentials(str(innerException2(e)))) + "\n"); } static A[] makeArray(Class type, int n) { return (A[]) Array.newInstance(type, n); } static A printStackTrace(A e) { // we go to system.out now - system.err is nonsense print(getStackTrace(e)); return e; } static void printStackTrace() { printStackTrace(new Throwable()); } static void printStackTrace(String msg) { printStackTrace(new Throwable(msg)); } static void printStackTrace(String msg, Throwable e) { printStackTrace(new Throwable(msg, e)); } static HashSet lithashset(A... items) { HashSet set = new HashSet(); for (A a : items) set.add(a); return set; } static Class _getClass(String name) { try { return Class.forName(name); } catch (ClassNotFoundException e) { return null; // could optimize this } } static Class _getClass(Object o) { return o == null ? null : o instanceof Class ? (Class) o : o.getClass(); } static Class _getClass(Object realm, String name) { try { return getClass(realm).getClassLoader().loadClass(classNameToVM(name)); } catch (ClassNotFoundException e) { return null; // could optimize this } } static Throwable getInnerException(Throwable e) { if (e == null) return null; while (e.getCause() != null) e = e.getCause(); return e; } static Throwable getInnerException(Runnable r) { return getInnerException(getException(r)); } // PersistableThrowable doesn't hold GC-disturbing class references in backtrace static volatile PersistableThrowable lastException_lastException; static PersistableThrowable lastException() { return lastException_lastException; } static void lastException(Throwable e) { lastException_lastException = persistableThrowable(e); } static String hideCredentials(URL url) { return url == null ? null : hideCredentials(str(url)); } static String hideCredentials(String url) { try { if (startsWithOneOf(url, "http://", "https://") && isAGIBlueDomain(hostNameFromURL(url))) return url; } catch (Throwable e) { print("HideCredentials", e); } return url.replaceAll("([&?])(_pass|key|cookie)=[^&\\s\"]*", "$1$2="); } static String hideCredentials(Object o) { return hideCredentials(str(o)); } static Set synchroHashSet() { return Collections.synchronizedSet(new HashSet()); } static Throwable unwrapTrivialExceptionWraps(Throwable e) { if (e == null) return e; while (e.getClass() == RuntimeException.class && e.getCause() != null && eq(e.getMessage(), str(e.getCause()))) e = e.getCause(); return e; } static String replacePrefix(String prefix, String replacement, String s) { if (!startsWith(s, prefix)) return s; return replacement + substring(s, l(prefix)); } static Throwable innerException2(Throwable e) { if (e == null) return null; while (empty(e.getMessage()) && e.getCause() != null) e = e.getCause(); return e; } static Class getClass(String name) { try { return Class.forName(name); } catch (ClassNotFoundException e) { return null; } } static Class getClass(Object o) { return o instanceof Class ? (Class) o : o.getClass(); } static Class getClass(Object realm, String name) { try { try { return getClass(realm).getClassLoader().loadClass(classNameToVM(name)); } catch (ClassNotFoundException e) { return null; } } catch (Exception __e) { throw rethrow(__e); } } static String classNameToVM(String name) { return name.replace(".", "$"); } static Throwable getException(Runnable r) { try { callF(r); return null; } catch (Throwable e) { return e; } } static boolean startsWithOneOf(String s, String... l) { for (String x : l) if (startsWith(s, x)) return true; return false; } static boolean startsWithOneOf(String s, Matches m, String... l) { for (String x : l) if (startsWith(s, x, m)) return true; return false; } static boolean isAGIBlueDomain(String domain) { return domainIsUnder(domain, theAGIBlueDomain()); } static String hostNameFromURL(String url) { try { return new URL(url).getHost(); } catch (Exception __e) { throw rethrow(__e); } } static boolean startsWith(String a, String b) { return a != null && a.startsWith(unnull(b)); } static boolean startsWith(String a, char c) { return nemptyString(a) && a.charAt(0) == c; } static boolean startsWith(String a, String b, Matches m) { if (!startsWith(a, b)) return false; m.m = new String[] {substring(a, strL(b))}; return true; } static boolean startsWith(List a, List b) { if (a == null || listL(b) > listL(a)) return false; for (int i = 0; i < listL(b); i++) if (neq(a.get(i), b.get(i))) return false; return true; } static boolean domainIsUnder(String domain, String mainDomain) { return eqic(domain, mainDomain) || ewic(domain, "." + mainDomain); } static String theAGIBlueDomain() { return "agi.blue"; } static boolean nemptyString(String s) { return s != null && s.length() > 0; } static int listL(Collection l) { return l == null ? 0 : l.size(); } static boolean neq(Object a, Object b) { return !eq(a, b); } static boolean eqic(String a, String b) { if ((a == null) != (b == null)) return false; if (a == null) return true; return a.equalsIgnoreCase(b); } static boolean eqic(char a, char b) { if (a == b) return true; char u1 = Character.toUpperCase(a); char u2 = Character.toUpperCase(b); if (u1 == u2) return true; return Character.toLowerCase(u1) == Character.toLowerCase(u2); } static boolean ewic(String a, String b) { return endsWithIgnoreCase(a, b); } static boolean ewic(String a, String b, Matches m) { return endsWithIgnoreCase(a, b, m); } static String asString(Object o) { return o == null ? null : o.toString(); } static boolean endsWithIgnoreCase(String a, String b) { int la = l(a), lb = l(b); return la >= lb && regionMatchesIC(a, la-lb, b, 0, lb); } static boolean endsWithIgnoreCase(String a, String b, Matches m) { if (!endsWithIgnoreCase(a, b)) return false; m.m = new String[] { substring(a, 0, l(a)-l(b)) }; return true; } static boolean regionMatchesIC(String a, int offsetA, String b, int offsetB, int len) { return a != null && a.regionMatches(true, offsetA, b, offsetB, len); } // immutable, has strong refs final static class _MethodCache { final Class c; final HashMap> cache = new HashMap(); _MethodCache(Class c) { this.c = c; _init(); } void _init() { Class _c = c; while (_c != null) { for (Method m : _c.getDeclaredMethods()) if (!isAbstract(m) && !reflection_isForbiddenMethod(m)) multiMapPut(cache, m.getName(), makeAccessible(m)); _c = _c.getSuperclass(); } // add default methods - this might lead to a duplication // because the overridden method is also added, but it's not // a problem except for minimal performance loss. for (Class intf : allInterfacesImplementedBy(c)) for (Method m : intf.getDeclaredMethods()) if (m.isDefault() && !reflection_isForbiddenMethod(m)) multiMapPut(cache, m.getName(), makeAccessible(m)); } // Returns only matching methods Method findMethod(String method, Object[] args) { try { List m = cache.get(method); if (m == null) return null; int n = m.size(); for (int i = 0; i < n; i++) { Method me = m.get(i); if (call_checkArgs(me, args, false)) return me; } return null; } catch (Exception __e) { throw rethrow(__e); } } Method findStaticMethod(String method, Object[] args) { try { List m = cache.get(method); if (m == null) return null; int n = m.size(); for (int i = 0; i < n; i++) { Method me = m.get(i); if (isStaticMethod(me) && call_checkArgs(me, args, false)) return me; } return null; } catch (Exception __e) { throw rethrow(__e); } } }static abstract class VF1 implements IVF1 { public abstract void get(A a); }static class Matches { String[] m; Matches() {} Matches(String... m) { this.m = m;} String get(int i) { return i < m.length ? m[i] : null; } String unq(int i) { return unquote(get(i)); } String tlc(int i) { return unq(i).toLowerCase(); } boolean bool(int i) { return "true".equals(unq(i)); } String rest() { return m[m.length-1]; } // for matchStart int psi(int i) { return Integer.parseInt(unq(i)); } public String toString() { return "Matches(" + joinWithComma(quoteAll(asList(m))) + ")"; } public int hashCode() { return _hashCode(toList(m)); } public boolean equals(Object o) { return o instanceof Matches && arraysEqual(m, ((Matches) o).m); } } static class LineCompReader { List literals = new ArrayList(); IVF1 onPair; // if not null, don't save pairs, but stream them to onPair instead int[] literalOffsets; // where they start in file LongBuffer pairs = new LongBuffer(); LinkedHashMap> versions = new LinkedHashMap(); boolean byteMode = false; // internal, optional CountingInputStream countingInputStream; long fileSize; LineCompReader() {} // takes text or gzipped input file LineCompReader(File f) { load(f); } LineCompReader(InputStream in) { load(rawByteReader(in, 128*1024)); } LineCompReader(BufferedReader reader) { load(reader); } void load(File f) { fileSize = fileSize(f); countingInputStream = new CountingInputStream(bufferedFileInputStream(f)); BufferedReader reader = isGZipFile(f) ? rawByteReader(gzipInputStream(countingInputStream)) : rawByteReader(countingInputStream); try { //rawByteReader_possiblyGZipped(f); load(reader); } finally { _close(reader); }} void load(BufferedReader reader) { try { StringBuilder lineBuf = new StringBuilder(); String s = readLineIgnoreCR(reader, lineBuf); int ofs = l(s)+1; Matches m = new Matches(); if (startsWith(s, "BYTECOMP ", m)) byteMode = true; else if (!startsWith(s, "LINECOMP ", m)) throw fail("Not a LINECOMP file"); int nLiterals = parseInt(m.rest()); IntBuffer offsets = new IntBuffer(); for (int i = 0; i < nLiterals; i++) { String line = readLineIgnoreCR(reader, lineBuf); assertNotNull(line); literals.add(byteMode ? str(charFromHex(line)) : line); offsets.add(ofs); ofs += l(line)+1; } offsets.add(ofs); literalOffsets = offsets.toArray(); int n = 0; while (licensed()) { s = readLineIgnoreCR(reader, lineBuf); if (s == null || contains(s, "=")) break; try { int iSpace = s.indexOf(' '); long pair = twoIntsToLong( Integer.parseInt(s, 0, iSpace, 10), Integer.parseInt(s, iSpace+1, l(s), 10)); if (onPair != null) onPair.get(pair); else pairs.add(pair); if (((++n) % oneMillion()) == 0) { String percentage = ""; if (fileSize != 0 && countingInputStream != null) percentage = " (" + intPercentRatio(countingInputStream.getFilePointer(), fileSize) + "%)"; print(nPairs(n) + " read" + percentage); } } catch (Throwable _e) { print("On line " + (nLiterals + l(pairs))); throw rethrow(_e); } } pairs.trimToSize(); while (contains(s, "=")) { int i = indexOf(s, '='); versions.put(takeFirst(s, i), compactIntList(parseInts(splitAtSpace(substring(s, i+1))))); s = readLineIgnoreCR(reader, lineBuf); } } catch (Exception __e) { throw rethrow(__e); } } Set versions() { return keys(versions); } String getText(String version) { return textForVersion(version); } String textForVersion(String version) { List encoded = versions.get(version); if (encoded == null) return null; List buf = new ArrayList(); for (int idx : encoded) decode(idx, buf); return myFromLines(buf); } // name of first (or only) file String firstFile() { return first(versions()); } // text for first (or only) file String text() { return getText(firstFile()); } List encoding() { return versions.get(firstFile()); } String myFromLines(List l) { return byteMode ? join(l) : fromLines_rtrim(l); } void decode(int idx, List buf) { if (idx < l(literals)) buf.add(literals.get(idx)); else { long p = pairs.get(idx-l(literals)); decode(firstIntFromLong(p), buf); decode(secondIntFromLong(p), buf); } } // That was it! The rest of this file is just for calculating some stats. Map lineCountsForPairs = new HashMap(); Map byteCountsForPairs = new HashMap(); int lineCountForPointer(int idx) { return idx < l(literals) ? 1 : lineCountForPair(idx); } long byteCountForPointer(int idx) { return idx < l(literals) ? l(literals.get(idx))+1 : byteCountForPair(idx); } int lineCountForPair(int idx) { Integer c = lineCountsForPairs.get(idx); if (c == null) { long p = pairs.get(idx-l(literals)); c = lineCountForPointer(firstIntFromLong(p)) + lineCountForPointer(secondIntFromLong(p)); lineCountsForPairs.put(idx, c); } return c; } long byteCountForPair(int idx) { Long c = byteCountsForPairs.get(idx); if (c == null) { long p = pairs.get(idx-l(literals)); c = byteCountForPointer(firstIntFromLong(p)) + byteCountForPointer(secondIntFromLong(p)); byteCountsForPairs.put(idx, c); } return c; } int lineCountForVersion(String version) { List encoded = versions.get(version); if (encoded == null) return 0; int n = 0; for (int i : encoded) n += lineCountForPointer(i); return n; } long byteCountForVersion(String version) { List encoded = versions.get(version); if (encoded == null) return 0; long n = 0; for (int i : encoded) n += byteCountForPointer(i); return max(0, n-1); } long totalByteCount() { return longSum(lambdaMap(__19 -> byteCountForVersion(__19),versions())); } // now we can also save again void save(PrintWriter out) { out.println((byteMode ? "BYTECOMP " : "LINECOMP ") + l(literals)); for (String s : literals) out.println(byteMode ? charToHex(first(s)) : s); for (long p : pairs) out.println(firstIntFromLong(p) + " " + secondIntFromLong(p)); for (Map.Entry> __0 : _entrySet( versions)) { String id = __0.getKey(); List l = __0.getValue(); out.println(id + "=" + joinWithSpace(l)); } } }// it's unclear whether the end is inclusive or exclusive // (usually exclusive I guess) static class IntRange { int start, end; IntRange() {} IntRange(int start, int end) { this.end = end; this.start = start;} IntRange(IntRange r) { start = r.start; end = r.end; } public boolean equals(Object o) { return stdEq2(this, o); } public int hashCode() { return stdHash2(this); } final int length() { return end-start; } final boolean empty() { return start >= end; } final boolean isEmpty() { return start >= end; } static String _fieldOrder = "start end"; public String toString() { return "[" + start + ";" + end + "]"; } } // read-only, so far. should be thread-safe final static class RAMByteMemory64 implements IByteMemory64, AutoCloseable { File file; // the file we loaded long size; // file size in ints boolean bigEndian = true; boolean debug = false; // byte buffers int arrayShift = 30; // each buffer is 1 GB int arraySize = 1 << arrayShift; byte[][] arrays; RAMByteMemory64() {} RAMByteMemory64(File file) { load(file); } void load(File file) { try { this.file = file; size = fileSize(file); RandomAccessFile raf = newRandomAccessFile(file, "r"); try { arrays = new byte[toInt(rightShift_ceil(size, arrayShift))][]; FileChannel channel = raf.getChannel(); print("Allocating " + n2(size) + " bytes"); for (int i = 0; i < l(arrays); i++) { long pos = (long) i << arrayShift; int len = toInt(min(1 << arrayShift, size-pos)); arrays[i] = new byte[len]; } for (int i = 0; i < l(arrays); i++) { long pos = (long) i << arrayShift; int len = l(arrays[i]); print("Loading bytes " + longToHex(pos) + "-" + longToHex(pos+len) + " of " + file); MappedByteBuffer byteBuffer = channel.map(FileChannel.MapMode.READ_ONLY, pos, len); byteBuffer.get(arrays[i]); } print(size + " bytes loaded"); } finally { _close(raf); }} catch (Exception __e) { throw rethrow(__e); } } public void close() {} public byte getByte(long idx) { rangeCheck(idx, size); return arrays[(int) (idx >> arrayShift)][(((int) idx) & (arraySize-1))]; } public int getInt(long idx) { assertTrue("bigEndian", bigEndian); return ubyteToInt(getByte(idx)) << 24 | ubyteToInt(getByte(idx+1)) << 16 | ubyteToInt(getByte(idx+2)) << 8 | ubyteToInt(getByte(idx+3)); } public void set(long idx, int val) { checkWritable(); throw fail("todo"); } void checkWritable() { throw fail("read-only"); } public long size() { return size; } public void ensureSize(int size) { this.size = max(this.size, size); } void importVirtual(Object mem) { copyFields(mem, this); } }static abstract class F1 { abstract B get(A a); }final static class LCSearcher_v1_onDisk64_v3 extends ILCCompactIndex implements AutoCloseable { // important: set this if the file uses a special comparator CharComparator charComparator; // flags user can set boolean debug = false; // switch print statements on/off LCSearcher_Protocol protocol; boolean doSubProtocol = false; IVF1_Long onMemAccess; // initialize this to get IO stats (TODO: actually call) // all-important memory field IByteMemory64 mem; // tuning parameter int intBufferEmptyCapacity = 8; // data we always hold in memory char version; // file version ('1' or '2') int iFirstPair, iFirstFile, nProds; char[] literals; Map literalIndex = new HashMap(); int[] files; // starting at iFirstFile MultiMap symbolToFile = new MultiMap(); // values start at iFirstFile (I think) // where symbol pointer table starts long iSymbolPointers; // beef up symbol pointers to 64 bit long[] symbolPtrHighBytes; int highBytesShift = 22; // we make a new high byte every 1 << 22 entries LCSearcher_v1_onDisk64_v3() {} LCSearcher_v1_onDisk64_v3(File f) { load(f); } LCSearcher_v1_onDisk64_v3(IByteMemory64 mem) { load(mem); } void load(File f) { BufferedDiskByteMemory64 mem = new BufferedDiskByteMemory64(f); mem.bigEndian = true; load(mem); } void load(IByteMemory64 mem) { this.mem = mem; String versionString = fromUtf8(intArrayToBytes(readIntArray(0, 4))); assertStartsWith(versionString, "QUICKSEARCH v"); assertEndsWith(versionString, "\n"); version = versionString.charAt(l(versionString)-3); if (debug) print("File signature verifies."); iFirstPair = memGet(4*4); iFirstFile = iFirstPair+memGet(5*4); nProds = iFirstFile+memGet(6*4); long ptr = 7*4; if (debug) printVars_str("iFirstPair", iFirstPair, "iFirstFile", iFirstFile, "nProds", nProds); // read raw literals literals = new char[iFirstPair]; for (int i = 0; i < l(literals); i++) { literals[i] = (char) memGet(ptr+i*2-2); literalIndex.put(literals[i], i); } ptr += roundUpTo(4, l(literals)*2); if (debug) printStruct("literals", literals); // read files files = readIntArray(ptr, nProds-iFirstFile); for (int i = 0; i < l(files); i++) symbolToFile.put(files[i], i+iFirstFile); ptr += (nProds-iFirstFile)*4; if (debug) printStruct("files", files); // init symbol table iSymbolPointers = ptr; makeHighBytes(); } long indexOfOccurrences(int symbol) { long ptr = symbolPtr(symbol); if (symbol >= iFirstPair) { ptr += 2*4; // skip pair contents ptr += compressed30BitUint_lengthFromFirstByte(mem.getByte(ptr)); // skip prod length } return ptr; } // adapted to v3 public int getProdLength(int i) { if (i >= nProds) throw fail("Symbol index out of range: " + i); if (i < iFirstPair) return 1; return readCompressed30BitUint_IByteMemory64(mem, symbolPtr(i)+2*4); } // adapted to v3 public int getOccurrences(int i) { if (i >= nProds) throw fail("Symbol index out of range: " + i); long ptr = indexOfOccurrences(i); return readCompressed30BitUint_IByteMemory64(mem, ptr); } long indexOfUnion(int i) { long ptr = indexOfOccurrences(i); return ptr+compressed30BitUint_lengthFromFirstByte(mem.getByte(ptr)); } // i == iFirstFile is OK because we need to know where last symbol ends long symbolPtr(int i) { if (i < 0 || i > iFirstFile) throw fail("Symbol out of range: " + i + " / " + iFirstFile); int ptr = symbolPtr_raw(i); int iHigh = i >> highBytesShift; long fullPtr = symbolPtrHighBytes[iHigh+(ptr < 0 ? 0 : 1)] | uintToLong(ptr); return fullPtr; } int symbolPtr_raw(int i) { return memGet(iSymbolPointers+i*4L); } String its(int idx) { return itemToString(idx); } public String itemToString(int idx) { return withStringBuilder(getProdLength(idx), buf -> itemToString_step(buf, idx)); } // i = symbol index long getPair(int i) { long _ptr = symbolPtr(i); long p; try { p = memGetLong(_ptr); } catch (Throwable _e) { print("Was getting pair " + intToHex(i) + " at " + longToHex(_ptr)); throw rethrow(_e); } return p; } void itemToString_step(StringBuilder buf, int idx) { if (idx < iFirstPair) { buf.append(literals[idx]); return; } long p = getPair(idx); try { itemToString_step(buf, firstIntFromLong(p)); itemToString_step(buf, secondIntFromLong(p)); } catch (Throwable _e) { print("Was expanding pair: " + intToHex(idx) + " [" + longToHex(p) + "]"); throw rethrow(_e); } } public void close() { { cleanUp(mem); mem = null; } } public int lookupLiteral(char c) { return or(literalIndex.get(c), -1); } public int lookupLiteral(String c) { return lookupLiteral(stringToChar(c)); } public int getLiteral(int i) { return (int) literals[i]; } List symbolToFiles(int symbol) { return symbolToFile.getOpt(symbol); } int[] getLeftPairs(int symbol) { return readIntArray(leftPairsRange(symbol)); } int[] readIntArray(LongRange r) { return r == null ? null : readIntArray(r.start, toInt(r.length() >> 2)); } int[] readIntArray(long start, int len) { int[] a = new int[len]; for (int i = 0; i < len; i++) a[i] = memGet(start+i*4L); return a; } int memGet(long ptr) { if (onMemAccess != null) onMemAccess.get(ptr); return mem.getInt(ptr); } long memGetLong(long ptr) { return twoIntsToLong(memGet(ptr), memGet(ptr+4)); } // adapted to v3 LongRange leftPairsRange(int symbol) { if (symbol >= nProds) throw fail("Symbol index out of range: " + symbol); long ptr = indexOfUnion(symbol); long lengthOfUnion = symbolPtr(symbol+1)-ptr; if (lengthOfUnion != 4) { int n = readCompressed30BitUint_IByteMemory64(mem, ptr); return longRangeWithLength(ptr+compressed30BitUint_lengthForValue(n), n*4); } // one parent case - check if it's a left parent if (mem.getInt(ptr) >= 0) return new LongRange(ptr, ptr+4); else return null; } // adapted to v3 LongRange rightPairsRange(int symbol) { if (symbol >= nProds) throw fail("Symbol index out of range: " + symbol); long ptr = indexOfUnion(symbol); long nextSymbol = symbolPtr(symbol+1); long lengthOfUnion = nextSymbol-ptr; if (lengthOfUnion != 4) { int nLeftPairs = readCompressed30BitUint_IByteMemory64(mem, ptr); ptr += compressed30BitUint_lengthForValue(nLeftPairs) + nLeftPairs*4L; return new LongRange(ptr, nextSymbol); } // one parent case - check if it's a right parent if (mem.getInt(ptr) < 0) return new LongRange(ptr, ptr+4); else return null; } int[] getRightPairs(int symbol) { int[] a = readIntArray(rightPairsRange(symbol)); if (l(a) == 1) a[0] &= 0x7FFFFFFF; return a; } public IntegerIterator getLeftPairsEndingWith(int symbol, String from) { return getLeftPairsEndingWith(symbol, from, null); } public IntegerIterator getLeftPairsEndingWith(int symbol, String from, String to) { LongRange r = leftPairsRange(symbol); if (r == null) { if (protocol != null) protocol.searched(this, false, symbol, reversed(from), null); return null; } if (empty(from)) return memArrayIntegerIterator_symbols(r); IntBuffer out = newIntBuffer(); dictionarySearch(out, false, 0, r.start, (int) (r.length() >> 2), from); if (protocol != null) protocol.searched(this, false, symbol, reversed(from), out); return out.integerIterator(); } public IntegerIterator getRightPairsStartingWith(int symbol, String from) { return getRightPairsStartingWith(symbol, from, null); } public IntegerIterator getRightPairsStartingWith(int symbol, String from, String to) { LongRange r = rightPairsRange(symbol); if (r == null) { if (protocol != null) protocol.searched(this, true, symbol, from, null); return null; } if (empty(from)) return memArrayIntegerIterator_symbols(r); IntBuffer out = newIntBuffer(); dictionarySearch(out, true, 0, r.start, (int) (r.length() >> 2), from); if (protocol != null) protocol.searched(this, true, symbol, from, out); return out.integerIterator(); } void dictionarySearch(IntBuffer out, boolean toTheRight, int knownCharacters, long ptr, int size, String prefix) { ping(); LCSearcher_Protocol protocol = doSubProtocol ? this.protocol : null; if (protocol != null) protocol.log("dictionarySearch " + (toTheRight ? "right" : "left") + " ptr=" + ptr + ", size " + size + ", knownCharacters=" + knownCharacters + ", prefix=" + quote(prefix)); if (size == 0) return; // Nothing to do for size 0 if (size == 1) { // only one symbol to check int sym = memGet(ptr) & 0x7FFFFFFF; int part = toTheRight ? pairRight(sym) : pairLeft(sym); if (protocol != null) protocol.log("Single symbol: " + symbolToProtocol(part)); // see how many new characters we have in common with the search string int common = lCommonPrefixOfCharIterators( itemCharIteratorWithSkip(toTheRight, part, knownCharacters), characterIterator(prefix, knownCharacters), charComparator); int have = knownCharacters+common; int need = min(getProdLength(part), l(prefix)); if (have < need) { // some characters not matched, it's a fail if (protocol != null) protocol.log("Symbol check fail (" + have + "/" + need + "): " + symbolToProtocol(part)); return; } // Add symbol. We like it if (protocol != null) protocol.log("Adding symbol: " + symbolToProtocol(sym)); out.add(sym); return; } // More than one symbol to check. Get common prefix length of first & last symbol in range int sym1 = memGet(ptr); int sym2 = memGet(ptr+(size-1)*4L); int part1 = !toTheRight ? pairLeft(sym1) : pairRight(sym1); int part2 = !toTheRight ? pairLeft(sym2) : pairRight(sym2); CharacterIterator it1 = itemCharIteratorWithSkip(toTheRight, part1, knownCharacters); CharacterIterator it2 = itemCharIteratorWithSkip(toTheRight, part2, knownCharacters); int lPrefix = prefix.length(), iPrefix = knownCharacters; if (protocol != null) protocol.log("Range: " + symbolToProtocol(part1, "toTheRight", toTheRight) + " to " + symbolToProtocol(part2, "toTheRight", toTheRight) + " [size " + size + "]"); int common = 0; char c1 = 0, c2 = 0, cPrefix = 0; boolean grabbed = false; // did we grab 3 characters already while (it1.hasNext() && it2.hasNext() && iPrefix < lPrefix) { c1 = it1.next(); c2 = it2.next(); cPrefix = prefix.charAt(iPrefix++); /*if (protocol != null) protocol.log("grabbed " + quote(c1) + ", " + quote(c2) + ", " + quote(cPrefix));*/ if (compareChars(charComparator, c1, c2) == 0 && compareChars(charComparator, c1, cPrefix) == 0) ++common; else { grabbed = true; break; } } if (protocol != null && common != 0) print("New common chars: " + common); if (!grabbed && iPrefix >= lPrefix) { // prefix exhausted - add all elements for (int i = 0; i < size; i++) { int sym = memGet(ptr+i*4L); if (protocol != null) protocol.log("Adding symbol: " + symbolToProtocol(sym)); out.add(sym); } return; } // prefix not exhausted, check next character in each iterator if (!grabbed) cPrefix = prefix.charAt(iPrefix++); boolean hasNext1 = grabbed || it1.hasNext(); if (hasNext1 && !grabbed) c1 = it1.next(); boolean hasNext2 = grabbed || it2.hasNext(); if (hasNext2 && !grabbed) c2 = it2.next(); if (hasNext1 && hasNext2 && compareChars(charComparator, c2, cPrefix) < 0) { // whole range is above prefix, skip if (protocol != null) protocol.log("Range is above prefix: c1=" + (hasNext1 ? quote(c1) + " [" + (int) c1 + "]" : "none") + ", c2=" + (hasNext2 ? quote(c2) : "none") + ", cPrefix=" + cPrefix + ", " + symbolToProtocol(sym1) + "/" + symbolToProtocol(sym2)); return; } if (hasNext1 && hasNext2 && compareChars(charComparator, c1, cPrefix) > 0) { // whole range is below prefix, skip if (protocol != null) protocol.log("Range is below prefix: " + symbolToProtocol(sym1) + "/" + symbolToProtocol(sym2)); return; } // subdivide knownCharacters += common; int split = size/2; if (protocol != null) protocol.log("Subdividing. split point " + split); dictionarySearch(out, toTheRight, knownCharacters, ptr, split, prefix); dictionarySearch(out, toTheRight, knownCharacters, ptr+split*4L, size-split, prefix); } class ItemCharIterator extends CharacterIterator { boolean toTheRight = true; IntBuffer stack = newIntBuffer(); ItemCharIterator(boolean toTheRight, int symbol) { this.toTheRight = toTheRight; stack.add(symbol); } ItemCharIterator(int symbol) { stack.add(symbol); } ItemCharIterator(int symbol, int skip) { this(true, symbol, skip); } ItemCharIterator(boolean toTheRight, int symbol, int skip) { this.toTheRight = toTheRight; stack.add(symbol); while (skip > 0 && !stack.isEmpty()) { int idx = stack.popLast(); int len = getProdLength(idx); if (skip >= len) { skip -= len; continue; } // skip over symbol completely // Now we know it's a pair (because skip >= 1) long p = getPair(idx); int a = firstIntFromLong(p), b = secondIntFromLong(p); if (!toTheRight) { { int __1 = a; a = b; b = __1; } } len = getProdLength(a); if (skip >= len) { // we can skip a, just post b stack.add(b); skip -= len; continue; } // we have to process both a & b (push in reverse order) stack.add(b); stack.add(a); } } public boolean hasNext() { return !stack.isEmpty(); } public char next() { if (stack.isEmpty()) throw fail("no more elements"); while (true) { int idx = stack.popLast(); if (idx < iFirstPair) return literals[idx]; long p = getPair(idx); int a = firstIntFromLong(p), b = secondIntFromLong(p); if (!toTheRight) { { int __2 = a; a = b; b = __2; } } stack.add(b); stack.add(a); } } } CharacterIterator itemCharIterator(int idx) { return new ItemCharIterator(idx); } CharacterIterator itemCharIteratorWithSkip(int symbol, int skip) { return new ItemCharIterator(symbol, skip); } CharacterIterator itemCharIteratorWithSkip(boolean toTheRight, int symbol, int skip) { return new ItemCharIterator(toTheRight, symbol, skip); } CharacterIterator itemCharIteratorWithSkip(boolean toTheRight, int symbol) { return new ItemCharIterator(toTheRight, symbol); } CharacterIterator itemReverseCharIterator(int idx) { return new ItemCharIterator(false, idx); } CharacterIterator itemReverseCharIteratorWithSkip(int symbol, int skip) { return new ItemCharIterator(false, symbol, skip); } int numberOfSymbols() { return iFirstFile; } void makeHighBytes() { int step = 1 << highBytesShift, lastPtr = 0; int steps = idiv_ceil(numberOfSymbols(), step); symbolPtrHighBytes = new long[steps+1]; long high = (iSymbolPointers+(numberOfSymbols()+1L)*4) & ~0xFFFFFFFFL; int idx = 0; for (int i = 0; i < numberOfSymbols(); i += step) { int ptr = symbolPtr_raw(i); if (uintToLong(ptr) < uintToLong(lastPtr)) { assertTrue(lastPtr < 0); //print("INC POINT " + idx + " (last: " + intToHex(lastPtr) + ")"); high += 0x100000000L; } /*if (idx >= 0)*/ symbolPtrHighBytes[idx] = high; ++idx; lastPtr = ptr; } symbolPtrHighBytes[steps] = high; printStruct("symbolPtrHighBytes", symbolPtrHighBytes); } String symbolToProtocol(int symbol, Object... __) { boolean toTheRight = optPar("toTheRight",__, true); boolean withIdx = boolPar("withIdx",__); int max = 40; CharacterIterator it = toTheRight ? itemCharIterator(symbol) : itemReverseCharIterator(symbol); String s = charIteratorToString_max(max+1, it); if (l(s) > max) return quote(shorten(s, max)) + " [length " + getProdLength(symbol) + (!withIdx ? "" : ", idx " + symbol) + "]"; else return quote(s) + (!withIdx ? "" : " [idx " + symbol + "]"); } void assertLeftPairsAreSorted(int symbol) { int[] lpairs = getLeftPairs(symbol); print("pairs", l(lpairs)); String last = ""; for (int i = 0; i < l(lpairs); i++) { int sym = lpairs[i]; String s = reversed(itemToString(sym)); print(i + "/" + l(lpairs) + ": " + quote(s)); assertTrue(cmp(s, last) >= 0); last = s; } } // file starts at 0 int getFile(int file) { return files[file]; } int nFiles() { return l(files); } boolean isPair(int symbol) { return symbol >= iFirstPair && symbol < iFirstFile; } // iFile starts at 0 String filePartToString(int iFile, int from, int to) { return charIteratorToString_max(to-from, itemCharIteratorWithSkip(getFile(iFile), from)); } // iFile starts at 0 int getFileLength(int iFile) { return getProdLength(getFile(iFile)); } IntegerIterator memArrayIntegerIterator_symbols(LongRange r) { return new IntegerIterator() { long i = r.start, end = r.end; public boolean hasNext() { return i < end; } public int next() { int val = memGet(i) & 0x7FFFFFFF; i += 4; return val; } }; } int numLeftPairs(int symbol) { return (int) (l(leftPairsRange(symbol)) >> 2); } int numRightPairs(int symbol) { return (int) (l(rightPairsRange(symbol)) >> 2); } int nLiterals() { return iFirstPair; } /*sclass SymbolInfo { int symbol; int left, right; // pair contents (if pair) int length, occurrences; bool oneParent; // symbol with only one parent long leftPairsStart, leftPairsEnd; long rightPairsStart, rightPairsEnd; }*/ IntBuffer newIntBuffer() { return new IntBuffer(intBufferEmptyCapacity); } }static class LCSearcher_v4_compact implements IFieldsToList{ LCSearcher_v1_compact lc; LCSearcher_v4_compact() {} LCSearcher_v4_compact(LCSearcher_v1_compact lc) { this.lc = lc;} public String toString() { return shortClassName(this) + "(" + lc + ")"; }public Object[] _fieldsToList() { return new Object[] {lc}; } // flags user can set boolean log = false; boolean importMagic = false; // unused boolean skipBorderChars = false; // unused boolean stopAfterFirstSymbol = false; // internal String query; int lQuery; char[] queryChars; int[] queryLiterals; String its(int i) { return lc.itemToString(i); } // return false if a literal wasn't found boolean processQuery(String query) { // store query, convert to literals this.query = query; if ((lQuery = l(query)) == 0) return false; queryChars = toCharArray(query); queryLiterals = new int[lQuery]; for (int iChar = 0; iChar < lQuery; iChar++) { char c = queryChars[iChar]; if ((queryLiterals[iChar] = lc.lookupLiteral(c)) < 0) { if (log) { print("Missing literal: " + quote(c)); //print("Have literals: " + lc.literalIndex); } return false; } } return true; } public UpScan rawScan(String query) { if (!processQuery(query)) return null; UpScan scan = new UpScan(); scan.run(); return scan; } // represents a symbol positioned at a certain offset in the query // (which may be negative if it starts to the left of the query) static class OfsAndSymbol implements IFieldsToList{ int ofs; int symbol; OfsAndSymbol() {} OfsAndSymbol(int ofs, int symbol) { this.symbol = symbol; this.ofs = ofs;} public String toString() { return shortClassName(this) + "(" + ofs + ", " + symbol + ")"; } public boolean equals(Object o) { if (!(o instanceof OfsAndSymbol)) return false; OfsAndSymbol __1 = (OfsAndSymbol) o; return ofs == __1.ofs && symbol == __1.symbol; } public int hashCode() { int h = -159510445; h = boostHashCombine(h, _hashCode(ofs)); h = boostHashCombine(h, _hashCode(symbol)); return h; } public Object[] _fieldsToList() { return new Object[] {ofs, symbol}; } } static interface SearchResult { int resultCount(); public int[] indices_unorderedIntArray(); public int[] indices_orderedIntArray(); void _continue(); } class UpScan implements SearchResult { LongBuffer queue = new LongBuffer(); // twoIntsToLong(ofs, symbol) LongHashSet seen = new LongHashSet(); // twoIntsToLong(ofs, symbol) LongBuffer fullSymbols = new LongBuffer(); String[] needLeft_from = new String[lQuery+1], needLeft_to = new String[lQuery+1]; String[] needRight_from = new String[lQuery+1], needRight_to = new String[lQuery+1]; public int resultCount() { int n = fullSymbols.size(), sum = 0; for (int i = 0; i < n; i++) sum += lc.getOccurrences(secondIntFromLong(fullSymbols.get(i))); return sum; } Collection seen() { List out = new ArrayList(); org.eclipse.collections.api.iterator.LongIterator it = seen.longIterator(); while (it.hasNext()) { long l = it.next(); out.add(new OfsAndSymbol(firstIntFromLong(l), secondIntFromLong(l))); } return out; } boolean addOfsAndSymbol(int ofs, int symbol) { if (!addToQueue(ofs, symbol)) return false; if (log) print("Candidate: " + quoteShorten(40, its(symbol)) + " at " + ofs); return true; } boolean addToQueue(int ofs, int symbol) { long l = twoIntsToLong(ofs, symbol); if (!seen.add(l)) return false; queue.add(l); return true; } void start() { int iChar = lQuery/2; // start somewhere (just not on the edges) if (addToQueue(iChar, queryLiterals[iChar]) && log) print("Base literal: " + queryChars[iChar] + " at " + iChar); } public void run() { start(); _continue(); } public void _continue() { while (!queue.isEmpty()) { long oas = queue.popLast(); int ofs = firstIntFromLong(oas), symbol = secondIntFromLong(oas); int _ofs = max(ofs, 0); int ofs2 = ofs+lc.getProdLength(symbol); // end of symbol in query int _ofs2 = min(ofs2, lQuery); if (_ofs == 0 && _ofs2 == lQuery) { fullSymbols.add(oas); if (stopAfterFirstSymbol) break; continue; } String needOnTheLeft_from = needLeft_from[_ofs], needOnTheLeft_to; if (needOnTheLeft_from == null) { needOnTheLeft_from = needLeft_from[_ofs] = reversedSubstring(query, 0, _ofs); needOnTheLeft_to = needLeft_to[_ofs] = needOnTheLeft_from + Character.MAX_VALUE; } else needOnTheLeft_to = needLeft_to[_ofs]; String needOnTheRight_from = needRight_from[_ofs2], needOnTheRight_to; if (needOnTheRight_from == null) { needOnTheRight_from = needRight_from[_ofs2] = substring(query, _ofs2); needOnTheRight_to = needRight_to[_ofs2] = needOnTheRight_from + Character.MAX_VALUE; } else needOnTheRight_to = needRight_to[_ofs2]; if (log) printVars_str("UpScan right", "oas" , quote(its(symbol)) + " at " + ofs, "needOnTheRight_from", needOnTheRight_from); // right pairs IntegerIterator setRight = lc.getRightPairsStartingWith(symbol, needOnTheRight_from, needOnTheRight_to); if (log) { print("setRight", setRight); //print(allRightPairs := lc.getRightPairs(symbol)); } if (setRight != null) { int n = 0; while (setRight.hasNext()) { addOfsAndSymbol(ofs, setRight.next()); ++n; } //if (ofs2-ofs == 1) print("right for " + queryChars[ofs] + ": " + n); } if (log) printVars_str("UpScan left", "oas" , quote(its(symbol)) + " at " + ofs, "needOnTheLeft_from", needOnTheLeft_from, "needOnTheLeft_to", needOnTheLeft_to); // left pairs //Cl leftAll = lc.getLeftPairs(symbol); //print(leftAll := lmap its(leftAll)); IntegerIterator setLeft = lc.getLeftPairsEndingWith(symbol, needOnTheLeft_from, needOnTheLeft_to); if (setLeft != null) { if (log) print("setLeft", setLeft); int n = 0; while (setLeft.hasNext()) { int i = setLeft.next(); addOfsAndSymbol(ofs2-lc.getProdLength(i), i); ++n; } //if (ofs2-ofs == 1) print("left for " + queryChars[ofs] + ": " + n); } } } public int[] indices_unorderedIntArray() { //print("full", l(fullSymbols) + ": " + map(i -> firstIntFromLong(i) + "/" + quote(lc.its(secondIntFromLong(i))), fullSymbols)); LongBuffer queue = new LongBuffer(fullSymbols); IntBuffer result = new IntBuffer(); while (!queue.isEmpty()) { long oas = queue.popLast(); int ofs = firstIntFromLong(oas), symbol = secondIntFromLong(oas); List files = lc.symbolToFiles(symbol); if (files != null) { if (log) print("File level: " + -ofs); result.add(-ofs); } for (int sym : unnull(lc.getLeftPairs(symbol))) { int ofs2 = ofs-lc.getProdLength(lc.pairLeft(sym)); if (log) print(ofs2 + "/" + quoteShorten(20, lc.its(sym))); long p = twoIntsToLong(ofs2, sym); queue.add(p); } for (int sym : unnull(lc.getRightPairs(symbol))) { if (log) print(ofs + "/" + quoteShorten(20, lc.its(sym))); long p = twoIntsToLong(ofs, sym); queue.add(p); } } return result.toArray(); } public int[] indices_orderedIntArray() { return sortIntArrayInPlace(indices_unorderedIntArray()); } } int[] indicesOf(String query) { SearchResult result = search(query); return result == null ? null : result.indices_orderedIntArray(); } int resultCount(String query) { SearchResult result = search(query); return result == null ? 0 : result.resultCount(); } UpScan search(String query) { if (!processQuery(query)) return null; UpScan scan = new UpScan(); scan.run(); return scan; } void printStats() { } }static class LCSearcher_v4_multiFile implements IFieldsToList{ ILCCompactIndex lc; LCSearcher_v4_multiFile() {} LCSearcher_v4_multiFile(ILCCompactIndex lc) { this.lc = lc;}public Object[] _fieldsToList() { return new Object[] {lc}; } // flags user can set boolean log = false; boolean queriesToUpper = false; transient IF0 getMem; Object getMem() { return getMem != null ? getMem.get() : getMem_base(); } final Object getMem_fallback(IF0 _f) { return _f != null ? _f.get() : getMem_base(); } Object getMem_base() { return null; } // for statistical purposes ILCCompactIndex getILCCompactIndex() { return lc; } String its(int i) { return lc.itemToString(i); } // represents a symbol positioned at a certain offset in the query // (which may be negative if it starts to the left of the query) static class OfsAndSymbol implements IFieldsToList{ int ofs; int symbol; OfsAndSymbol() {} OfsAndSymbol(int ofs, int symbol) { this.symbol = symbol; this.ofs = ofs;} public String toString() { return shortClassName(this) + "(" + ofs + ", " + symbol + ")"; } public boolean equals(Object o) { if (!(o instanceof OfsAndSymbol)) return false; OfsAndSymbol __1 = (OfsAndSymbol) o; return ofs == __1.ofs && symbol == __1.symbol; } public int hashCode() { int h = -159510445; h = boostHashCombine(h, _hashCode(ofs)); h = boostHashCombine(h, _hashCode(symbol)); return h; } public Object[] _fieldsToList() { return new Object[] {ofs, symbol}; } } static interface SearchResult { int resultCount(); public long[] multiFileIndices_unorderedArray(); public long[] multiFileIndices_orderedArray(); void _continue(); LongIterator unorderedIterator(); int nSymbolsLookedAt(); } class Search implements SearchResult { LCSearcher_Protocol protocol; String query; int lQuery; char[] queryChars; int[] queryLiterals; boolean stopAfterFirstSymbol = false; //new LongBuffer queue; // twoIntsToLong(ofs, symbol) //LongPriorityQueue queue = new(queueComparator()); // twoIntsToLong(ofs, symbol) ILongQueue queue = new LongPriorityQueue(queueComparator()); LongHashSet seen = new LongHashSet(); // twoIntsToLong(ofs, symbol) LongBuffer fullSymbols = new LongBuffer(); String[] needLeft; String[] needRight; void switchToSimpleQueue() { queue = new LongBuffer(); } LongComparator queueComparator() { // prioritize shorter symbols /*ret (a, b) -> cmp( lc.getProdLength(secondIntFromLong(a)), lc.getProdLength(secondIntFromLong(b)));*/ // prioritize symbols contained in query return (a, b) -> cmp(charsOutsideOfQuery(a), charsOutsideOfQuery(b)); } int charsOutsideOfQuery(long oas) { int ofs = firstIntFromLong(oas), symbol = secondIntFromLong(oas); return max(-ofs, 0) + max(ofs+lc.getProdLength(symbol)-lQuery, 0); } // return false if a literal wasn't found boolean processQuery(String query) { // store query, convert to literals if (queriesToUpper) query = upper(query); this.query = query; if ((lQuery = l(query)) == 0) return false; queryChars = toCharArray(query); queryLiterals = new int[lQuery]; for (int iChar = 0; iChar < lQuery; iChar++) { char c = queryChars[iChar]; if ((queryLiterals[iChar] = lc.lookupLiteral(c)) < 0) { if (protocol != null) protocol.log("Missing literal: " + quote(c)); //print("Have literals: " + lc.literalIndex); return false; } } needLeft = new String[lQuery+1]; needRight = new String[lQuery+1]; return true; } public int resultCount() { int n = fullSymbols.size(), sum = 0; print("Getting result count for " + nSymbols(n)); for (int i = 0; i < n; i++) { sum += lc.getOccurrences(secondIntFromLong(fullSymbols.get(i))); if (divides(10000, i)) print("Results after " + nSymbols(i) + ": " + n2(sum)); } return sum; } Collection seen() { List out = new ArrayList(); org.eclipse.collections.api.iterator.LongIterator it = seen.longIterator(); while (it.hasNext()) { ping(); long l = it.next(); out.add(new OfsAndSymbol(firstIntFromLong(l), secondIntFromLong(l))); } return out; } // this is called from other threads. no biggie though public int nSymbolsLookedAt() { return seen.size(); } boolean addOfsAndSymbol(int ofs, int symbol) { if (!addToQueue(ofs, symbol)) return false; if (log) print("Candidate: " + quoteShorten(40, its(symbol)) + " at " + ofs); if (protocol != null) { int lPair = lc.getProdLength(symbol); String overlapped = rpad(rep('_', ofs) + substring(query, ofs, ofs+lPair), lQuery, '_'); protocol.log("Found pair " + overlapped + " " + lc.symbolToProtocol(symbol, "withIdx" , true) + " at " + ofs); } return true; } boolean addToQueue(int ofs, int symbol) { long l = twoIntsToLong(ofs, symbol); if (!seen.add(l)) return false; if (ofs <= 0 && ofs+lc.getProdLength(symbol) >= lQuery) { if (protocol != null) protocol.log("FOUND: " + lc.symbolToProtocol(symbol) /*+ " at [" + ofs + "-" + ofs2 + "]"*/); fullSymbols.add(twoIntsToLong(ofs, symbol)); return false; } queue.add(l); return true; } boolean start(String query) { if (!processQuery(query)) return false; int iChar = lQuery/2; // start somewhere (just not on the edges) addToQueue(iChar, queryLiterals[iChar]); if (protocol != null) protocol.log("Base literal: " + queryChars[iChar] + " at " + iChar); return true; } public void _continue() { if (stopAfterFirstSymbol) while (!queue.isEmpty()) { ping(); step(); if (!fullSymbols.isEmpty()) break; } else while (!queue.isEmpty()) { ping(); step(); } } void step() { long oas = queue.poll/*popLast*/(); int ofs = firstIntFromLong(oas), symbol = secondIntFromLong(oas); int _ofs = max(ofs, 0); int ofs2 = ofs+lc.getProdLength(symbol); // end of symbol in query int _ofs2 = min(ofs2, lQuery); // LEFT SEARCH String needOnTheLeft = needLeft[_ofs]; if (needOnTheLeft == null) needOnTheLeft = needLeft[_ofs] = reversedSubstring(query, 0, _ofs); if (log) printVars_str("Search left", "oas" , quote(its(symbol)) + " at " + ofs, "needOnTheLeft", needOnTheLeft); // left pairs IntegerIterator setLeft = lc.getLeftPairsEndingWith(symbol, needOnTheLeft); if (setLeft != null) { if (log) print("setLeft", setLeft); //int n = 0; while (setLeft.hasNext()) { ping(); int i = setLeft.next(); addOfsAndSymbol(ofs2-lc.getProdLength(i), i); //++n; } //if (ofs2-ofs == 1) print("left for " + queryChars[ofs] + ": " + n); } // RIGHT SEARCH String needOnTheRight = needRight[_ofs2]; if (needOnTheRight == null) needOnTheRight = needRight[_ofs2] = substring(query, _ofs2); if (log) printVars_str("Search right", "oas" , quote(its(symbol)) + " at " + ofs, "needOnTheRight", needOnTheRight); // right pairs IntegerIterator setRight = lc.getRightPairsStartingWith(symbol, needOnTheRight); if (log) { print("setRight", setRight); //print(allRightPairs := lc.getRightPairs(symbol)); } if (setRight != null) { //int n = 0; while (setRight.hasNext()) { ping(); addOfsAndSymbol(ofs, setRight.next()); //++n; } //if (ofs2-ofs == 1) print("right for " + queryChars[ofs] + ": " + n); } } // end of step public long[] multiFileIndices_unorderedArray() { //print("full", l(fullSymbols) + ": " + map(i -> firstIntFromLong(i) + "/" + quote(lc.its(secondIntFromLong(i))), fullSymbols)); LongBuffer queue2 = new LongBuffer(fullSymbols); LongBuffer result = new LongBuffer(); while (!queue2.isEmpty()) { ping(); long oas = queue2.popLast(); int ofs = firstIntFromLong(oas), symbol = secondIntFromLong(oas); List files = lc.symbolToFiles(symbol); if (files != null) { if (log) print("File level: " + -ofs); result.add(twoIntsToLong(first(files), -ofs)); } for (int sym : unnull(lc.getLeftPairs(symbol))) { int ofs2 = ofs-lc.getProdLength(lc.pairLeft(sym)); if (log) print(ofs2 + "/" + quoteShorten(20, lc.its(sym))); queue2.add(twoIntsToLong(ofs2, sym)); } for (int sym : unnull(lc.getRightPairs(symbol))) { if (log) print(ofs + "/" + quoteShorten(20, lc.its(sym))); queue2.add(twoIntsToLong(ofs, sym)); } } return result.toArray(); } public LongIterator unorderedIterator() { LongBuffer queue2 = new LongBuffer(fullSymbols); return new LongIterator() { public boolean hasNext() { return !queue2.isEmpty(); } public long next() { while (true) { ping(); long oas = queue2.popLast(); int ofs = firstIntFromLong(oas), symbol = secondIntFromLong(oas); List files = lc.symbolToFiles(symbol); if (files != null) { if (log) print("File level: " + -ofs); return twoIntsToLong(first(files), -ofs); } for (int sym : unnull(lc.getLeftPairs(symbol))) { int ofs2 = ofs-lc.getProdLength(lc.pairLeft(sym)); if (log) print(ofs2 + "/" + quoteShorten(20, lc.its(sym))); queue2.add(twoIntsToLong(ofs2, sym)); } for (int sym : unnull(lc.getRightPairs(symbol))) { if (log) print(ofs + "/" + quoteShorten(20, lc.its(sym))); queue2.add(twoIntsToLong(ofs, sym)); } } } }; } public LongIterator unorderedIterator_lazy() { return new LongIterator() { int fullSymbolsSeen = 0; LongBuffer myQueue = new LongBuffer(); long value = -1; public boolean hasNext() { if (value == -1) grab(); return value >= 0; } void grab() { processQueue(); if (value >= 0) return; fillQueue(); processQueue(); } void fillQueue() { while (fullSymbols.size() <= fullSymbolsSeen && !queue.isEmpty()) { ping(); step(); } if (fullSymbols.size() > fullSymbolsSeen) myQueue.add(fullSymbols.get(fullSymbolsSeen++)); } void processQueue() { while (!myQueue.isEmpty()) { ping(); long oas = myQueue.popLast(); int ofs = firstIntFromLong(oas), symbol = secondIntFromLong(oas); List files = lc.symbolToFiles(symbol); if (files != null) { if (log) print("File level: " + -ofs); { value = twoIntsToLong(first(files), -ofs); return; } } for (int sym : unnull(lc.getLeftPairs(symbol))) { int ofs2 = ofs-lc.getProdLength(lc.pairLeft(sym)); if (log) print(ofs2 + "/" + quoteShorten(20, lc.its(sym))); myQueue.add(twoIntsToLong(ofs2, sym)); } for (int sym : unnull(lc.getRightPairs(symbol))) { if (log) print(ofs + "/" + quoteShorten(20, lc.its(sym))); myQueue.add(twoIntsToLong(ofs, sym)); } } } public long next() { long v = value; value = -1; return v; } }; } public long[] multiFileIndices_orderedArray() { return sortLongArrayInPlace(multiFileIndices_unorderedArray()); } } long[] multiFileIndicesOf(String query) { return multiFileIndicesOf(query, null); } long[] multiFileIndicesOf(String query, LCSearcher_Protocol protocol) { SearchResult result = search(query, false, protocol); return result == null ? null : result.multiFileIndices_orderedArray(); } int[] singleFileIndicesOf(String query) { return singleFileIndicesOf(query, null); } int[] singleFileIndicesOf(String query, LCSearcher_Protocol protocol) { return secondIntFromLong_onArray(multiFileIndicesOf(query, protocol)); } int singleFileAnyIndexOf(String query) { LongIterator it = resultIterator_unordered(query); return it.hasNext() ? secondIntFromLong(it.next()) : -1; } long[] multiFileIndicesOf_unordered(String query) { SearchResult result = search(query, false); return result == null ? null : result.multiFileIndices_unorderedArray(); } int[] singleFileIndicesOf_unordered(String query) { return secondIntFromLong_onArray(multiFileIndicesOf_unordered(query)); } boolean contains(String query) { return contains(query, null); } boolean contains(String query, LCSearcher_Protocol protocol) { return anyResults(query, protocol); } boolean anyResults(String query) { return anyResults(query, null); } boolean anyResults(String query, LCSearcher_Protocol protocol) { Search result = search(query, true, protocol); return result != null && !result.fullSymbols.isEmpty(); } int resultCount(String query) { return resultCount(query, null); } int resultCount(String query, LCSearcher_Protocol protocol) { SearchResult result = search_simpleQueue(query, protocol); return result == null ? 0 : result.resultCount(); } LongIterator resultIterator_unordered(String query) { Search result = search_noContinue(query); return result == null ? null : result.unorderedIterator_lazy(); } Search search(String query, boolean stopAfterFirstSymbol) { return search(query, stopAfterFirstSymbol, null); } Search search(String query, boolean stopAfterFirstSymbol, LCSearcher_Protocol protocol) { Search search = new Search(); search.protocol = protocol; search.stopAfterFirstSymbol = stopAfterFirstSymbol; if (!search.start(query)) return null; search._continue(); return search; } // don't actually run the search Search search_noContinue(String query) { return search_noContinue(query, null); } Search search_noContinue(String query, LCSearcher_Protocol protocol) { Search scan = new Search(); scan.protocol = protocol; if (!scan.start(query)) return null; return scan; } // use simple queue (when you need all full symbols anyway) Search search_simpleQueue(String query) { return search_simpleQueue(query, null); } Search search_simpleQueue(String query, LCSearcher_Protocol protocol) { Search search = new Search(); search.protocol = protocol; search.switchToSimpleQueue(); if (!search.start(query)) return null; search._continue(); return search; } // ofs = start position of pair static class ShouldFind implements IFieldsToList{ static final String _fieldOrder = "pair ofs desc"; int pair; int ofs; ShouldFind() {} ShouldFind(int pair, int ofs) { this.ofs = ofs; this.pair = pair;} public boolean equals(Object o) { if (!(o instanceof ShouldFind)) return false; ShouldFind __2 = (ShouldFind) o; return pair == __2.pair && ofs == __2.ofs; } public int hashCode() { int h = 1701202476; h = boostHashCombine(h, _hashCode(pair)); h = boostHashCombine(h, _hashCode(ofs)); return h; } public Object[] _fieldsToList() { return new Object[] {pair, ofs}; } String desc; public String toString() { return nempty(desc) ? desc : pair + "/" + ofs; } } List analyzeOccurrence(int iFile, int start, int len, LCSearcher_Protocol protocol) { String text = lc.filePartToString(iFile, start, start+len); LCSearcher_TreeWalker2 walker = new LCSearcher_TreeWalker2(lc); walker.debug = true; walker.scan(iFile, intRangeWithLen(start, len)); List out = new ArrayList(); for (IntPair p : walker.result) { int pair = p.a, ofs = p.b; int lPair = lc.getProdLength(pair); int left = lc.pairLeft(pair), right = lc.pairRight(pair); int lLeft = lc.getProdLength(left); String overlapped = rpad(rep('_', ofs) + substring(text, ofs, ofs+lPair), len, '_'); String desc = "Should find pair " + overlapped + " " + lc.symbolToProtocol(pair, "withIdx" , true) + " at " + ofs + ": " + lc.symbolToProtocol(left, "withIdx" , true) + " + " + lc.symbolToProtocol(right, "withIdx" , true); if (protocol != null) protocol.log(desc); out.add(set(new ShouldFind(pair, ofs), "desc", desc)); } return out; } int offsetFromResult(long l) { return secondIntFromLong(l); } }static class Q implements AutoCloseable { String name = "Unnamed Queue"; List q = synchroLinkedList(); ReliableSingleThread rst = new ReliableSingleThread(new Runnable() { public void run() { try { _run() ; } catch (Exception __e) { throw rethrow(__e); } } public String toString() { return "_run()"; }}); volatile boolean retired = false; Q() {} Q(String name) { this.name = name;} void add(Runnable r) { q.add(r); _trigger(); } void addInFront(Runnable r) { q.add(0, r); _trigger(); } void _trigger() { rst.name = name; rst.go(); } void add(Object r) { add(toRunnable(r)); } void _run() { Runnable r; while (licensed() && !retired && (r = syncPopFirst(q)) != null) { try { r.run(); } catch (Throwable __e) { _handleException(__e); } } } public void close() { retired = true; } // TODO: interrupt thread void done() {} // legacy function boolean isEmpty() { return q.isEmpty(); } int size() { return q.size(); } Object mutex() { return q; } // clients can synchronize on this List snapshot() { return cloneList(q); } }static class CountingOutputStream extends FilterOutputStream { long counter; CountingOutputStream(OutputStream out) { super(out); } @Override public void write(int b) throws IOException { ++counter; out.write(b); } @Override public void write(byte[] b) throws IOException { counter += b.length; out.write(b, 0, b.length); } @Override public void write(byte[] b, int off, int len) throws IOException { if (len == 0) return; counter += len; out.write(b, off, len); } long getFilePointer() { return counter; } }static interface IF0 { A get(); }static class LCSearcher_v1 { // user needs to fill these: transient Map literals; //transient LL productions; LongBuffer pairs = new LongBuffer(); IntBuffer files = new IntBuffer(); transient int iFirstPair, iFirstFile; CharComparator charComparator; // optional, used for sorting pairs transient Map fileNames; // optional int maxQueryLength = 32; boolean debug, big; // switch print statements on/off // internal structures begin here transient IntIntHashMap literals2 = new IntIntHashMap(); // compact/faster version of literals transient Map literalIndex; transient IntIntHashMap prodLengths; // prod index -> length of production in characters // not used anymore transient /*new*/ MultiMap prodIndex; // keys: item looked for, value: pair(prod index, position in prod) // stats int actualLeftPairSearches, actualRightPairSearches; transient IntBuffer[] leftPairIndex; // index of where a symbol appears on right side of a pair // index of where a symbol appears on left side of a pair transient IntBuffer[] rightPairIndex; //transient LCSortedPairIndex pairIndex; // the normal pair index (2 symbols -> pair symbol) transient boolean prepared = false; void prepare() { { if (prepared) return; prepared = true; } buildIndex(); } void buildIndex() { prodLengths = new IntIntHashMap(); // need this for making prodLengths for (Map.Entry e : literals.entrySet()) literals2.put(e.getKey(), (int) e.getValue()); // make a pair index for compressing queries pairs.trimToSize(); //making(big, +pairIndex = new LCSortedPairIndex(pairs.toArray(), iFirstPair); // build the reconstitution index making(big, "literalIndex", literalIndex = reverseMap(literals)); //if (debug) printVars_str(+iFirstFile, +productions); making(big, "leftPairIndex, rightPairIndex"); leftPairIndex = new IntBuffer[iFirstFile]; rightPairIndex = new IntBuffer[iFirstFile]; for (int iProd = iFirstPair; iProd < iFirstFile; iProd++) { long p = pairs.get(iProd-iFirstPair); int i = secondIntFromLong(p); IntBuffer buf = leftPairIndex[i]; if (buf == null) leftPairIndex[i] = buf = new IntBuffer(); buf.add(iProd); i = firstIntFromLong(p); buf = rightPairIndex[i]; if (buf == null) rightPairIndex[i] = buf = new IntBuffer(); buf.add(iProd); if (((iProd-iFirstPair) % oneMillion()) == 0) print(nPairs(iProd-iFirstPair) + " processed"); } if (big) print("Sorting leftPairIndex"); AtomicInteger count = new AtomicInteger(); int chunkSize = 1000; parallelDo(map(intRangeChunks(iFirstFile, chunkSize), chunk -> new Runnable() { public void run() { try { for (int i = chunk.start; i < chunk.end; i++) { IntBuffer buf = leftPairIndex[i]; if (buf == null) continue; sortIntBuffer_direct(buf, (__1, __2) -> compareItemsFromEnd_distinct(__1, __2)); buf.trimToSize(); int n2 = incAtomicInt(count, l(buf)); if (n2/oneMillion() > (n2-l(buf))/oneMillion()) print(n2(n2, "left pair") + " sorted"); } } catch (Exception __e) { throw rethrow(__e); } } public String toString() { return "for (int i = chunk.start; i < chunk.end; i++) {\r\n IntBuffer buf = left..."; }}), "queueSize" , oneMillion()); if (big) print("Sorting rightPairIndex"); count.set(0); parallelDo(map(intRangeChunks(iFirstFile, chunkSize), chunk -> new Runnable() { public void run() { try { for (int i = chunk.start; i < chunk.end; i++) { IntBuffer buf = rightPairIndex[i]; if (buf == null) continue; sortIntBuffer_direct(buf, (__3, __4) -> compareItems_distinct(__3, __4)); buf.trimToSize(); int n2 = incAtomicInt(count, l(buf)); if (n2/oneMillion() > (n2-l(buf))/oneMillion()) print(n2(n2, "right pair") + " sorted"); } } catch (Exception __e) { throw rethrow(__e); } } public String toString() { return "for (int i = chunk.start; i < chunk.end; i++) {\r\n IntBuffer buf = rightP..."; }}), "queueSize" , oneMillion()); if (debug) print("literalIndex", literalIndex); if (debug) print("leftPairIndex", leftPairIndex); if (debug) print("rightPairIndex", rightPairIndex); } public int getProdLength(int i) { throw fail("no longer implemented"); /*if (i < iFirstPair) ret 1; // literal if (i >= iFirstFile) ret getProdLength(files.get(i-iFirstFile)); int x = prodLengths.getIfAbsent(i, -1); if (x >= 0) ret x; prodLengths.put(i, x = intSum(lmap getProdLength(productions.get(i)))); ret x;*/ } public Collection files() { return asList(countIterator_exclusive(iFirstFile, nProductions())); } public int nProductions() { return iFirstFile+l(files); } CharacterIterator itemCharIterator(int idx) { return new ItemCharIterator(idx); } CharacterIterator itemReverseCharIterator(int idx) { return new ItemReverseCharIterator(idx); } // compare two items lexicographically public int compareItems(int idx1, int idx2) { return compareCharIterators(itemCharIterator(idx1), itemCharIterator(idx2), charComparator); } public int compareItemsFromEnd(int idx1, int idx2) { return compareCharIterators(itemReverseCharIterator(idx1), itemReverseCharIterator(idx2), charComparator); } // same comparison functions, but keeps distinct symbols non-equal public int compareItems_distinct(int idx1, int idx2) { int cmp = compareItems(idx1, idx2); if (cmp != 0) return cmp; return idx1-idx2; } public int compareItemsFromEnd_distinct(int idx1, int idx2) { int cmp = compareItemsFromEnd(idx1, idx2); if (cmp != 0) return cmp; return idx1-idx2; } class ItemCharIterator extends CharacterIterator { IntBuffer stack = new IntBuffer(); ItemCharIterator(int idx) { stack.add(idx); } public boolean hasNext() { return !stack.isEmpty(); } public char next() { if (stack.isEmpty()) throw fail("no more elements"); while (true) { int idx = stack.popLast(); if (idx >= iFirstPair) { long p = pairs.get(idx-iFirstPair); stack.add(secondIntFromLong(p)); stack.add(firstIntFromLong(p)); } else return (char) literals2.get(idx); } } } class ItemReverseCharIterator extends CharacterIterator { IntBuffer stack = new IntBuffer(); ItemReverseCharIterator(int idx) { stack.add(idx); } public boolean hasNext() { return !stack.isEmpty(); } public char next() { if (stack.isEmpty()) throw fail("no more elements"); while (true) { int idx = stack.popLast(); if (idx >= iFirstPair) { long p = pairs.get(idx-iFirstPair); stack.add(firstIntFromLong(p)); stack.add(secondIntFromLong(p)); } else return (char) literals2.get(idx); } } } public List getProduction(int idx) { if (idx >= iFirstFile) return ll(files.get(idx-iFirstFile)); if (idx >= iFirstPair) return intPairToList(longToIntPair(pairs.get(idx-iFirstPair))); return null; } void addPair(long p) { pairs.add(p); } int addFile(List encoding) { assertEquals(1, l(encoding)); files.add(first(encoding)); return iFirstFile+l(files)-1; } }static interface IF2 { C get(A a, B b); }static interface IF1 { B get(A a); }static class PersistableThrowable { String className; String msg; String stacktrace; PersistableThrowable() {} PersistableThrowable(Throwable e) { if (e == null) className = "Crazy Null Error"; else { className = getClassName(e).replace('/', '.'); msg = e.getMessage(); stacktrace = getStackTrace_noRecord(e); } } public String toString() { return nempty(msg) ? className + ": " + msg : className; } }final static class LCSearcher_v1_compact /*extends ILCCompactIndex*/ { // flags user can set boolean debug, big; // switch print statements on/off boolean calculateOccurrences = true; // adds another int per symbol // all the data char[] literals; Map literalIndex; int iFirstPair, iFirstFile, nProds; long[] pairs; int[] prodLengths; // length of production in characters int[] files; // starting at iFirstFile int[][] leftPairIndex; // index of where a symbol appears on right side of a pair int[][] rightPairIndex; // index of where a symbol appears on left side of a pair MultiMap symbolToFile = new MultiMap(); // optional: occurrence count for each symbol int[] occurrences; // stats transient int actualLeftPairSearches, actualRightPairSearches; LCSearcher_v1_compact() {} LCSearcher_v1_compact(LCSearcher_v1 v1) { load(v1); } // destroys v1 in the process void load(LCSearcher_v1 v1) { v1.prepare(); iFirstPair = v1.iFirstPair; iFirstFile = v1.iFirstFile; nProds = max(v1.files())+1; if (big) print("nProds", nProds); literals = new char[iFirstPair]; for (Map.Entry __0 : _entrySet( v1.literals)) { int idx = __0.getKey(); char c = __0.getValue(); literals[idx] = c; } literalIndex = v1.literalIndex; making(big, "pairs"); pairs = new long[iFirstFile-iFirstPair]; for (int i = iFirstPair; i < iFirstFile; i++) { List p = v1.getProduction(i); pairs[i-iFirstPair] = twoIntsToLong(first(p), second(p)); } making(big, "files"); files = new int[nProds-iFirstFile]; for (int i : v1.files()) { Integer symbol = first(v1.getProduction(i)); if (symbol != null) { files[i-iFirstFile] = symbol; symbolToFile.put(symbol, i); } } making(big, "prodLengths"); prodLengths = new int[nProds]; for (int i = 0; i < nProds; i++) { int len; if (i < iFirstPair) len = 1; else if (i < iFirstFile) len = prodLengths[pairLeft(i)] + prodLengths[pairRight(i)]; else len = prodLengths[files[i-iFirstFile]]; prodLengths[i] = len; } making(big, "leftPairIndex"); leftPairIndex = new int[iFirstFile][]; for (int i = 0; i < iFirstFile; i++) { leftPairIndex[i] = intBufferToArray(v1.leftPairIndex[i]); v1.leftPairIndex[i] = null; } making(big, "rightPairIndex"); rightPairIndex = new int[iFirstFile][]; for (int i = 0; i < iFirstFile; i++) { rightPairIndex[i] = intBufferToArray(v1.rightPairIndex[i]); v1.rightPairIndex[i] = null; } if (calculateOccurrences) { making(big, "occurrences"); occurrences = new int[iFirstFile]; for (int i = iFirstFile; i < nProds; i++) occurrences[files[i-iFirstFile]]++; for (int i = iFirstFile-1; i >= 0; i--) { int sum = 0; for (int sym : unnull(leftPairIndex[i])) sum += occurrences[sym]; for (int sym : unnull(rightPairIndex[i])) sum += occurrences[sym]; occurrences[i] += sum; } } } long getPair(int idx) { return pairs[idx-iFirstPair]; } int pairLeft(int idx) { return firstIntFromLong(pairs[idx-iFirstPair]); } int pairRight(int idx) { return secondIntFromLong(pairs[idx-iFirstPair]); } List queryToLiterals(String query) { List queryList = characters(query); List queryLiterals = new ArrayList(); for (Character c : queryList) { Integer iLit = literalIndex.get(c); if (iLit == null) { if (debug) print("literal not found"); return null; } queryLiterals.add(iLit); } return queryLiterals; } public int getProdLength(int i) { return prodLengths[i]; } class ItemCharIterator extends CharacterIterator { IntBuffer stack = new IntBuffer(); ItemCharIterator(int idx) { stack.add(idx); } public boolean hasNext() { return !stack.isEmpty(); } public char next() { if (stack.isEmpty()) throw fail("no more elements"); while (true) { int idx = stack.popLast(); if (idx < iFirstPair) return literals[idx]; long p = pairs[idx-iFirstPair]; stack.add(secondIntFromLong(p)); stack.add(firstIntFromLong(p)); } } } class ItemReverseCharIterator extends CharacterIterator { IntBuffer stack = new IntBuffer(); ItemReverseCharIterator(int idx) { stack.add(idx); } public boolean hasNext() { return !stack.isEmpty(); } public char next() { if (stack.isEmpty()) throw fail("no more elements"); while (true) { int idx = stack.popLast(); if (idx < iFirstPair) return literals[idx]; long p = pairs[idx-iFirstPair]; stack.add(firstIntFromLong(p)); stack.add(secondIntFromLong(p)); } } } CharacterIterator itemCharIterator(int idx) { return new ItemCharIterator(idx); } CharacterIterator itemReverseCharIterator(int idx) { return new ItemReverseCharIterator(idx); } String itemsToString(List idx) { return withStringBuilder(buf -> { for (int i : unnull(idx)) itemToString_step(buf, i); }); } String its(int idx) { return itemToString(idx); } public String itemToString(int idx) { return withStringBuilder(getProdLength(idx), buf -> itemToString_step(buf, idx)); } void itemToString_step(StringBuilder buf, int idx) { if (idx < iFirstPair) { buf.append(literals[idx]); return; } long p = pairs[idx-iFirstPair]; itemToString_step(buf, firstIntFromLong(p)); itemToString_step(buf, secondIntFromLong(p)); } public int lookupLiteral(char c) { return or(literalIndex.get(c), -1); } public int getLiteral(int i) { return (int) literals[i]; } int[] getLeftPairs(int symbol) { return leftPairIndex[symbol]; } int[] getRightPairs(int symbol) { return rightPairIndex[symbol]; } // newer version (faster) // with from = reversed(s), to = from + Char.MAX_VALUE public IntegerIterator getLeftPairsEndingWith(int symbol, String from, String to) { int[] set = leftPairIndex[symbol]; if (debug) print(formatFunctionCall("getLeftPairsEndingWith",symbol, from) + ": " + map(p -> its(pairLeft(p)), intArrayToList(set))); if (set == null) return null; if (from.isEmpty()) return integerIterator(set); ++actualLeftPairSearches; IntBuffer out = new IntBuffer(); getLeftPairsEndingWith_collect(out, set, 0, set.length, from, to); return out.integerIterator(); } // older version public IntegerIterator getLeftPairsEndingWith(int symbol, String s) { int[] set = leftPairIndex[symbol]; if (debug) print(formatFunctionCall("getLeftPairsEndingWith",symbol, s) + ": " + map(p -> its(pairLeft(p)), intArrayToList(set))); if (set == null) return null; if (s.isEmpty()) return integerIterator(set); // floor and ceiling (inclusive) of the range we are looking for String from = reversed(s), to = from + Character.MAX_VALUE; ++actualLeftPairSearches; IntBuffer out = new IntBuffer(); getLeftPairsEndingWith_collect(out, set, 0, set.length, from, to); return out.integerIterator(); } // newer version (faster) // with from = s, to = from + Char.MAX_VALUE public IntegerIterator getRightPairsStartingWith(int symbol, String from, String to) { // let's mess with the inner structures of CompactTreeSet // We can also make a clone of it in case the original changes int[] set = rightPairIndex[symbol]; if (debug) print(formatFunctionCall("getRightPairsStartingWith",symbol, from) + ": " + map(p -> its(pairRight(p)), intArrayToList(set))); if (set == null) return null; if (from.isEmpty()) return integerIterator(set); ++actualRightPairSearches; IntBuffer out = new IntBuffer(); getRightPairsStartingWith_collect(out, set, 0, set.length, from, to); return out.integerIterator(); } // older version public IntegerIterator getRightPairsStartingWith(int symbol, String s) { // let's mess with the inner structures of CompactTreeSet // We can also make a clone of it in case the original changes int[] set = rightPairIndex[symbol]; if (debug) print(formatFunctionCall("getRightPairsStartingWith",symbol, s) + ": " + map(p -> its(pairRight(p)), intArrayToList(set))); if (set == null) return null; if (s.isEmpty()) return integerIterator(set); // floor and ceiling (inclusive) of the range we are looking for String from = s, to = s + Character.MAX_VALUE; ++actualRightPairSearches; IntBuffer out = new IntBuffer(); getRightPairsStartingWith_collect(out, set, 0, set.length, from, to); return out.integerIterator(); } // binary search void getLeftPairsEndingWith_collect(IntBuffer out, int[] set, int start, int end, String from, String to) { if (end <= start) return; // get mid symbol int mid = (start + end) >>> 1; int symbol = pairLeft(set[mid]); // compare with beginning and end of range int cmp1 = compareCharIterators_maxLen(itemReverseCharIterator(symbol), characterIterator(from), min(getProdLength(symbol), l(from))); // prefix trick int cmp2 = compareCharIterators(itemReverseCharIterator(symbol), characterIterator(to)); boolean searchLeft = cmp2 >= 0; // search left unless right of range boolean searchRight = cmp1 <= 0; // search right unless left of range if (!searchLeft) { // Bug fix for "ba" in [aa, baaa]? // Scan left branch if there was at least one common character char firstChar = itemReverseCharIterator(symbol).next(); if (debug) print("bug fix: " + firstChar + "/" + first(from)); if (eq(firstChar, first(from))) searchLeft = true; } if (debug) printVars_str("leftSymbol" , its(symbol), "from", from, "to", to, "start", start, "mid", mid, "end", end, "cmp1", cmp1, "cmp2", cmp2, "searchLeft", searchLeft, "searchRight", searchRight); if (searchLeft) getLeftPairsEndingWith_collect(out, set, start, mid, from, to); if (cmp1 == 0) out.add(set[mid]); // add found pair if (searchRight) getLeftPairsEndingWith_collect(out, set, mid+1, end, from, to); } // subroutine 1: collect elements in range (look for range start) void getRightPairsStartingWith_collect(IntBuffer out, int[] set, int start, int end, String from, String to) { if (end <= start) return; // get mid symbol int mid = (start + end) >>> 1; int symbol = pairRight(set[mid]); // compare with beginning and end of range int cmp1 = compareCharIterators_maxLen(itemCharIterator(symbol), characterIterator(from), min(getProdLength(symbol), l(from))); // prefix trick int cmp2 = compareCharIterators(itemCharIterator(symbol), characterIterator(to)); boolean searchLeft = cmp2 >= 0; // search left unless right of range boolean searchRight = cmp1 <= 0; // search right unless left of range if (!searchLeft) { // Bug fix for "ba" in [aa, baaa]? // Scan left branch if there was at least one common character char firstChar = itemCharIterator(symbol).next(); if (debug) print("bug fix: " + firstChar + "/" + first(from)); if (eq(firstChar, first(from))) searchLeft = true; } if (debug) printVars_str("rightSymbol" , its(symbol), "from", from, "to", to, "start", start, "mid", mid, "end", end, "cmp1", cmp1, "cmp2", cmp2, "searchLeft", searchLeft, "searchRight", searchRight); if (searchLeft) getRightPairsStartingWith_collect(out, set, start, mid, from, to); if (cmp1 == 0) out.add(set[mid]); // add found pair if (searchRight) getRightPairsStartingWith_collect(out, set, mid+1, end, from, to); } List symbolToFiles(int symbol) { return symbolToFile.getOpt(symbol); } // -1 if unknown int getOccurrences(int symbol) { return occurrences != null ? occurrences[symbol] : -1; } int fileLength(int file) { return getProdLength(files[file-iFirstFile]); } int firstFileLength() { return fileLength(iFirstFile); } int getFileSymbol(int file) { return files[file-iFirstFile]; } }// This is using the implementation that seems to work fastest // in reality, although it probably is O(n log n) due to the use // of TreeMaps. // // The theoretically best implementation is O(n) (replacing the // TreeMap with a linked list). final static class LineCompCompressor { int safety = 0; abstract static class Chunk { abstract String text(List chunks); IntPair getPair() { return null; } } static class CPair extends Chunk implements IFieldsToList{ int i1; int i2; CPair() {} CPair(int i1, int i2) { this.i2 = i2; this.i1 = i1;} public String toString() { return shortClassName(this) + "(" + i1 + ", " + i2 + ")"; } public boolean equals(Object o) { if (!(o instanceof CPair)) return false; CPair __3 = (CPair) o; return i1 == __3.i1 && i2 == __3.i2; } public int hashCode() { int h = 64355773; h = boostHashCombine(h, _hashCode(i1)); h = boostHashCombine(h, _hashCode(i2)); return h; } public Object[] _fieldsToList() { return new Object[] {i1, i2}; } CPair(IntPair p) { i1 = p.a; i2 = p.b; } String text(List chunks) { return linesLL_rtrim(chunks.get(i1).text(chunks), chunks.get(i2).text(chunks)); } IntPair getPair() { return new IntPair(i1, i2); } } static class CPrim extends Chunk implements IFieldsToList{ String s; CPrim() {} CPrim(String s) { this.s = s;} public String toString() { return shortClassName(this) + "(" + s + ")"; } public boolean equals(Object o) { if (!(o instanceof CPrim)) return false; CPrim __4 = (CPrim) o; return eq(s, __4.s); } public int hashCode() { int h = 64372105; h = boostHashCombine(h, _hashCode(s)); return h; } public Object[] _fieldsToList() { return new Object[] {s}; } String text(List chunks) { return s; } } boolean verbose = false, verboseCompressionSteps = false; boolean verboseStats = false; boolean sortLines = true; boolean verify = true; boolean byteMode = false; // compress bytes instead of lines boolean caseInsensitive = false; // for line mode boolean toUpper = false; // for byte mode: convert all characters to upper case boolean recordPairLengths = false; // for "balanced" compression boolean balancing = false; // try to lower tree heights by compressing "shorter" pairs first boolean fullCompression = false; // keep compressing until every file is a single symbol LineComp_PairIndex pairIndex = new LineComp_PairIndex(); Map> textIDToLines; List allUniqueLines; List chunks = new ArrayList(); int primChunks; Map lineIndex; Map> simpleEncodings; // temporary Map> finalEncodings; LongIntHashMap existingPairsIndex = new LongIntHashMap(); // formerly Map int round, printStatsEvery = /*10000*/0, printStatsInFirstRounds = 10; int printStatsEveryNSeconds = /*10*/60; // 60 for slower computers // initializing fullIndex = new Map will make things slow and use a lot of memory // and I thought there might be a case where it improves the compression // - but maybe there isn't? Map fullIndex; SimpleIntList pairLengths; LineCompCompressor() {} LineCompCompressor(Map texts) { loadTexts(texts); } // key = version ID, values = text void loadTexts(Map texts) { textIDToLines = mapValuesToLinkedHashMap(__45 -> myToLines(__45),texts); } List myToLines(String s) { List l = byteMode ? lambdaMap(__20 -> charToHex(__20),characters(toUpper ? upper(s) : s)) : toLines_nOnly_reversible(s); return l; } String myFromLines(List l) { return byteMode ? join(mapI("hexToChar",l)) // TODO: optimize even more : fromLines_rtrim(l); } public void run() { if (balancing) { recordPairLengths = true; pairIndex.pairComparatorForBalancing = (p1, p2) -> { int l1 = pairLength(p1), l2 = pairLength(p2); if (l1 != l2) return l1-l2; // return arbitrary deterministic order, but not 0 if (p1.a != p2.a) return p1.a-p2.a; return p1.b-p2.b; }; } if (recordPairLengths) pairLengths = new SimpleIntList(); List allLines = concatLists(values(textIDToLines)); if (verboseCompressionSteps) print("Uniquifying " + nLines(allLines)); allUniqueLines = uniquify(allLines); if (verboseCompressionSteps) print("Have " + n2(allUniqueLines, "unique line")); allLines = null; // allow me to forget if (sortLines) sortInPlace(allUniqueLines); if (verboseCompressionSteps) print("Sorted " + nLines(allUniqueLines)); for (String line : allUniqueLines) chunks.add(new CPrim(line)); primChunks = l(chunks); List _lines = collect("s",chunks); lineIndex = caseInsensitive ? ciListIndex(_lines) : listIndex(_lines); _lines = null; // simple encoding (only direct line references) simpleEncodings = mapValues(textIDToLines, (IF1, List>) (lines -> wrapIntArrayAsImmutableList(mapToIntArray(lines, line -> lineIndex.get(line))))); //printAndCheckEncodings(simpleEncodings); if (verboseCompressionSteps) print("Have simple encodings"); finalEncodings = compressPairs(); if (verbose || verify) printAndCheckEncodings(finalEncodings); } void saveAsTextFile(File f) { String out = exportEncoding(finalEncodings); saveTextFile(f, out); if (verify) checkDecompression(f, textIDToLines); } void checkDecompression(File file, Map> textIDToLines) { BufferedReader reader = bufferedUtf8Reader(file); try { LineCompReader lcr = new LineCompReader(reader); assertEquals(keysList(textIDToLines), asList(lcr.versions())); for (String version : keys(textIDToLines)) assertEquals(lcr.textForVersion(version), lines_rtrim(textIDToLines.get(version))); if (verbose) print("Decompression OK for " + nVersions(textIDToLines)); } finally { _close(reader); }} String asText() { return exportEncoding(finalEncodings); } String exportEncoding(Map> encodings) { List buf = new ArrayList(); buf.add((byteMode ? "BYTECOMP " : "LINECOMP ") + primChunks); // magic signature for (Chunk c : chunks) { if (c instanceof CPair) buf.add(((CPair) c).i1 + " " + ((CPair) c).i2); else buf.add(((CPrim) c).s); } for (Map.Entry> __1 : _entrySet( encodings)) { String id = __1.getKey(); List l = __1.getValue(); buf.add(id + "=" + joinWithSpace(l)); } return lines_rtrim(buf); } Map> compressPairs() { pairIndex.verbose = verboseCompressionSteps || verbose; pairIndex.init(); for (Map.Entry> __5 : _entrySet( simpleEncodings)) { String version = __5.getKey(); List l = __5.getValue(); ping(); pairIndex.newChain(version, l); } pairIndex.haveChains(); simpleEncodings = null; // forget return compressChains(keys(pairIndex.chains)); } Map> compressChains(Collection files) { NotTooOften nto = new NotTooOften(printStatsEveryNSeconds*1000); IntPair toCompress; // Compress only most popular pair in one step while ((toCompress = fullCompression ? pairIndex.mostPopularPair() : pairIndex.mostPopularDuplicate()) != null) { ping(); if (verbose) { print("Chains: " + mapValuesToList(pairIndex.chains, c -> c.toList())); //print("Highest bucket (" + pairIndex.highestBucketNr() + "): " + pairIndex.mostPopularDuplicates()); print("Buckets: " + pairIndex.byCount); } if (safety > 0 && --safety <= 0) throw fail("safety"); ++round; if (verboseStats) if (printStatsEvery > 0 && (round % printStatsEvery) == 0 || round <= printStatsInFirstRounds || nto.get()) printStats(); int count = pairIndex.getCount(toCompress); Integer idx = makeCPair(toCompress); int dups = pairIndex.numberOfDuplicates(); if (verboseCompressionSteps) print("Compressing pair " + toCompress + " (count=" + count + ") -> " + idx + ", " + (dups-1) + " remaining"); pairIndex.replacePair(toCompress.a, toCompress.b, idx); } if (verboseStats) printStats(); // reconvert to normal list return mapToValues(f -> pairIndex.chains.get(f).toList(), files); } // TODO: check if pair is already there Integer makeCPair(IntPair p) { long l = intPairToLong(p); String text = null; if (fullIndex != null) { text = itemToString(p.a) + itemToString(p.b); Integer idx = fullIndex.get(text); if (idx != null) return idx; } int idx = existingPairsIndex.getIfAbsent(l, -1); if (idx < 0) { idx = addAndReturnIndex(chunks, new CPair(p)); if (recordPairLengths) listPut(pairLengths, idx, itemLength(p.a) + itemLength(p.b), 1); existingPairsIndex.put(l, idx); if (fullIndex != null) fullIndex.put(text, idx); } return idx; } void printAndCheckEncodings(Map> encodings) { for (Map.Entry> __0 : _entrySet( encodings)) { String id = __0.getKey(); List encoded = __0.getValue(); if (verbose) print(id + ": " + joinWithSpace(encoded)); String x = myFromLines(textIDToLines.get(id)), y = decode(encoded); if (caseInsensitive) assertEqualsIC(x, y); else assertEquals_quote(x, y); } } String decode(List encoded) { List l = lambdaMap(__21 -> chunkText(__21),encoded); if (byteMode) l = lines(lines(l)); return myFromLines(l); } // for non-byte mode String chunkText(int idx) { return chunks.get(idx).text(chunks); } void printStats() { print("Round: " + round); pairIndex.printStats(); } // byteMode only String itemToString(int idx) { StringBuilder buf = new StringBuilder(); itemToString_step(buf, idx); return str(buf); } void itemToString_step(StringBuilder buf, int idx) { Chunk c = chunks.get(idx); if (c instanceof CPair) { itemToString_step(buf, ((CPair) c).i1); itemToString_step(buf, ((CPair) c).i2); } else { String s = ((CPrim) c).s; buf.append(byteMode ? charFromHex(s) : s + "\n"); } } int nLiterals() { return primChunks; } int nItems() { return l(chunks); } IntPair getPair(int idx) { return chunks.get(idx).getPair(); } int itemLength(int idx) { return idx >= l(pairLengths) ? 1 : pairLengths.get(idx); } int pairLength(IntPair p) { return p == null ? 0 : itemLength(p.a) + itemLength(p.b); } List literals() { return lmap(__22 -> itemToString(__22),countIterator(primChunks)); } }static class Pair implements Comparable> { A a; B b; Pair() {} Pair(A a, B b) { this.b = b; this.a = a;} public int hashCode() { return hashCodeFor(a) + 2*hashCodeFor(b); } public boolean equals(Object o) { if (o == this) return true; if (!(o instanceof Pair)) return false; Pair t = (Pair) o; return eq(a, t.a) && eq(b, t.b); } public String toString() { return "<" + a + ", " + b + ">"; } public int compareTo(Pair p) { if (p == null) return 1; int i = ((Comparable) a).compareTo(p.a); if (i != 0) return i; return ((Comparable) b).compareTo(p.b); } } static interface CharComparator { int compare(char a, char b); }static class SimpleIntList extends RandomAccessAbstractList { int[] data = EMPTY_ELEMENTDATA; int size; private static final int[] EMPTY_ELEMENTDATA = {}; public void add(int i) { ensureCapacityInternal(size+1); data[size++] = i; } public boolean add(Integer i) { add((int) i); return true; } public Integer set(int i, Integer value) { if (i >= size) throw fail("Index out of range: " + i); int old = data[i]; data[i] = value; return old; } public int size() { return size; } public Integer get(int i) { if (i >= 0 && i < size) return data[i]; throw new IndexOutOfBoundsException(i + "/" + size); } void ensureCapacityInternal(int minCapacity) { if (data.length < minCapacity) grow(minCapacity); } private void grow(int minCapacity) { int oldCapacity = data.length; int newCapacity = oldCapacity + (oldCapacity >> 1); if (newCapacity - minCapacity < 0) newCapacity = minCapacity; data = Arrays.copyOf(data, newCapacity); } // always returns a fresh copy int[] toIntArray() { int[] a = new int[size]; System.arraycopy(data, 0, a, 0, size); return a; } }abstract static class IntegerIterator { abstract boolean hasNext(); abstract int next(); }static class MultiMap { Map> data = new HashMap>(); MultiMap() {} MultiMap(boolean useTreeMap) { if (useTreeMap) data = new TreeMap(); } MultiMap(MultiMap map) { putAll(map); } MultiMap(Map> data) { this.data = data;} void put(A key, B value) { synchronized(data) { List list = data.get(key); if (list == null) data.put(key, list = _makeEmptyList()); list.add(value); }} void add(A key, B value) { put(key, value); } void addAll(A key, Collection values) { putAll(key, values); } void addAllIfNotThere(A key, Collection values) { synchronized(data) { for (B value : values) setPut(key, value); }} void setPut(A key, B value) { synchronized(data) { if (!containsPair(key, value)) put(key, value); }} boolean containsPair(A key, B value) { synchronized(data) { return get(key).contains(value); }} void putAll(A key, Collection values) { synchronized(data) { if (nempty(values)) getActual(key).addAll(values); }} void removeAll(A key, Collection values) { synchronized(data) { for (B value : values) remove(key, value); }} List get(A key) { synchronized(data) { List list = data.get(key); return list == null ? Collections. emptyList() : list; }} List getOpt(A key) { synchronized(data) { return data.get(key); }} // returns actual mutable live list // creates the list if not there List getActual(A key) { synchronized(data) { List list = data.get(key); if (list == null) data.put(key, list = _makeEmptyList()); return list; }} void clean(A key) { synchronized(data) { List list = data.get(key); if (list != null && list.isEmpty()) data.remove(key); }} Set keySet() { synchronized(data) { return data.keySet(); }} Set keys() { synchronized(data) { return data.keySet(); }} void remove(A key) { synchronized(data) { data.remove(key); }} void remove(A key, B value) { synchronized(data) { List list = data.get(key); if (list != null) { list.remove(value); if (list.isEmpty()) data.remove(key); } }} void clear() { synchronized(data) { data.clear(); }} boolean containsKey(A key) { synchronized(data) { return data.containsKey(key); }} B getFirst(A key) { synchronized(data) { List list = get(key); return list.isEmpty() ? null : list.get(0); }} void addAll(MultiMap map) { putAll(map); } void putAll(MultiMap map) { synchronized(data) { for (A key : map.keySet()) putAll(key, map.get(key)); }} void putAll(Map map) { synchronized(data) { if (map != null) for (Map.Entry e : map.entrySet()) put(e.getKey(), e.getValue()); }} int keysSize() { synchronized(data) { return l(data); }} // full size - note: expensive operation int size() { synchronized(data) { int n = 0; for (List l : data.values()) n += l(l); return n; }} // expensive operation List reverseGet(B b) { synchronized(data) { List l = new ArrayList(); for (A key : data.keySet()) if (data.get(key).contains(b)) l.add(key); return l; }} Map> asMap() { synchronized(data) { return cloneMap(data); }} boolean isEmpty() { synchronized(data) { return data.isEmpty(); }} // override in subclasses List _makeEmptyList() { return new ArrayList(); } // returns live lists Collection> allLists() { synchronized(data) { return new ArrayList(data.values()); } } Collection> values() { return allLists(); } List allValues() { return concatLists(data.values()); } Object mutex() { return data; } public String toString() { return "mm" + str(data); } }final static class LongRange { long start, end; LongRange() {} LongRange(long start, long end) { this.end = end; this.start = start;} public boolean equals(Object o) { if (o instanceof LongRange) return start == ((LongRange) o).start && end == ((LongRange) o).end; return false; } public int hashCode() { return boostHashCombine(hashOfLong(start), hashOfLong(end)); } long length() { return end-start; } static String _fieldOrder = "start end"; public String toString() { return "[" + start + ";" + end + "]"; } } final static class LineComp_PairIndex { boolean verbose = false; boolean inOrder = true; //MultiSetMap nodes; LongObjectHashMap> nodes = new LongObjectHashMap(); MultiSetMap byCount // values are pairs = null; // init later //= multiSetMap_innerHashSet_outerRevTreeMap(); // makes a difference in selecting a pair if there are multiple with the same count // = multiSetMap_innerCompactHashSet_outerRevTreeMap(); // same but more compact //= multiSetMap_innerTreeSet_outerRevTreeMap(); // initial version that kind of pointlessly sorts the same-count nodes lexicographically LinkedHashMap chains = new LinkedHashMap(); Ch firstChain; int totalPairCount; int initialNodesSetCapacity = 1; List> onAddingPair, onRemovingPair; LongIntHashMap existingPairsIndex; // This queue checks all newly made pairs to see if they already // have a symbol. LinkedList compressionQueue = new LinkedList(); Comparator pairComparatorForBalancing; void init() { /*nodes = inOrder ? multiSetMap_innerLinkedHashSet_outerHashMap() : multiSetMap_innerHashSet_outerHashMap();*/ } // indicate end of chain-making phase. // we still more chains accept after this though, I think (for late input) void haveChains() { if (byCount == null) { //time "Making byCount" { if (pairComparatorForBalancing != null) byCount = multiSetMap_innerCustomTreeSet_outerRevTreeMap(pairComparatorForBalancing); else byCount = multiSetMap_innerCompactHashSet_outerRevTreeMap(); for (Set set : nodes.values()) { IntPair p = first(set).pair(); int n = set.size(); if (verbose) print("Adding node " + p + " (" + n + ")"); byCount.put(n, p); } //} } } // a "chain" of nodes (one input file) static class Ch { Node tail; Ch(LineComp_PairIndex pi, List l) { int count = 0; for (int i : unnull(l)) { add(pi, i); if (((++count) % oneMillion()) == 0) print("Added " + n2(count) + " entries to chain"); } } List toList() { List l = new ArrayList(); Node node = tail; while (node != null) { l.add(node.value); node = node.prev; } return reverseInPlace(l); } void add(LineComp_PairIndex pi, int i) { Node n = new Node(); n.ch = this; n.value = i; n.prev = tail; if (tail != null) tail.next = n; tail = n; pi.addToIndex(n.prev); pi.processQueue(); } } static class Node { Ch ch; int value; Node prev, next; int a() { return value; } int b() { return next == null ? -1 : next.value; } IntPair pair() { return next == null ? null : new IntPair(a(), b()); } public String toString() { return stringIf(prev != null, "<") + value + (next == null ? "" : next.value + stringIf(next.next != null, ">")); } } IntPair nodeToPair(Node n) { return n == null ? null : n.pair(); } // add node to pair index void addToIndex(Node n) { IntPair p = nodeToPair(n); if (p == null) return; callFAll(onAddingPair, n); int count = nodes_getSize(p); nodes_add(p, n); if (byCount != null) { // not in init phase if (count != 0) byCount.remove(count, p); byCount.put(count+1, p); } if (existingPairsIndex != null) compressionQueue.add(n); } // remove node from pair index void removeFromIndex(Node n) { IntPair p = nodeToPair(n); if (p == null) return; callFAll(onRemovingPair, n); int count = nodes_getSize(p); if (count == 0) throw fail("Can't remove pair " + p); nodes_remove(p, n); byCount.remove(count, p); if (--count > 0) byCount.put(count, p); } // return all pairs in "most popular" bucket Collection mostPopularDuplicates() { Integer key = firstKey(byCount); return toInt(key) < 2 ? null : byCount.get(key); } IntPair mostPopularDuplicate() { return toInt(firstKey(byCount)) < 2 ? null : firstValue(byCount); } IntPair mostPopularPair() { return firstValue(byCount); } int highestBucketNr() { return or0(firstKey(byCount)); } // counts buckets actually (lower value than totalPairCount) int numberOfDuplicates() { return byCount.size()-l(byCount.get(1)); } int getCount(IntPair p) { return nodes_getSize(p); } void replacePair(int pa, int pb, int replaceWith) { IntPair p = new IntPair(pa, pb); Set set = nodes.get(intPairToLong(p)); if (set != null) for (Node n : cloneList(set)) { { if (n.a() != pa || n.b() != pb) continue; } // nodes might have been deleted or changed replacePair(n, replaceWith); } } void replacePair(Node node, int replaceWith) { removeFromIndex(node.prev); removeFromIndex(node); removeFromIndex(node.next); node.value = replaceWith; deleteNode(node.next); addToIndex(node.prev); addToIndex(node); processQueue(); } void processQueue() { if (existingPairsIndex != null) { Node n; while ((n = popFirst(compressionQueue)) != null) checkIfPairExists(n); } } void checkIfPairExists(Node node) { if (existingPairsIndex == null) return; if (node.value < 0) return; IntPair pair = nodeToPair(node); if (pair == null) return; //print("Checking pair " + pair); int i = existingPairsIndex.getIfAbsent(intPairToLong(pair), -1); if (i >= 0) { //print(" Replacing with: " + i); replacePair(node, i); } } void deleteNode(Node node) { if (node.next != null) node.next.prev = node.prev; else getChain(node).tail = node.prev; if (node.prev != null) node.prev.next = node.next; node.value = -1; // mark invalid } Ch newChain(String version, List encoding) { Ch ch = new Ch(this, encoding); chains.put(version, ch); if (firstChain == null) firstChain = ch; return ch; } int nodes_getSize(IntPair p) { return l(nodes.get(intPairToLong(p))); } void nodes_add(IntPair p, Node n) { ++totalPairCount; long l = intPairToLong(p); LinkedHashSet set = nodes.get(l); if (set == null) nodes.put(l, set = new LinkedHashSet(initialNodesSetCapacity)); set.add(n); } void nodes_remove(IntPair p, Node n) { --totalPairCount; long l = intPairToLong(p); LinkedHashSet set = nodes.get(l); if (set == null || !set.remove(n)) throw fail("Consistency error"); if (set.isEmpty()) nodes.remove(l); } // also shrinks sets... void printStats() { print("Different pairs: " + l(nodes)); print("Different counts: " + byCount.keysSize() + " (highest count: " + firstKey(byCount) + ")"); long count = 0, waste = 0, newWaste = 0; long count1 = 0, waste1 = 0; for (HashSet set : nodes.values()) { count1 += set.size(); int capacity = hashSetCapacity(set); waste1 += capacity-set.size(); //if (capacity > set.size()*3) set.shrink(); // TODO } if (pairComparatorForBalancing == null) for (CompactHashSet set : (Collection) (Collection) values(byCount.data)) { count += set.size(); waste += set.capacity()-set.size(); set.shrinkToFactor(0.5); newWaste += set.capacity()-set.size(); } print("Nodes set count: " + count1 + ". Waste: " + waste1); //print("Set count: " + count + ". Waste: " + waste + (waste == newWaste ? "" : " => " + newWaste)); print("Duplicates: " + n2(totalPairCount-byCount.count(1))); } Ch getChain(Node node) { return node.ch; } /*public LineComp_PairIndex clone() { }*/ }static class ReliableSingleThread implements Runnable { boolean _isTransient() { return true; } Object runnable; // usually a Runnable. is allowed to call trigger() itself String name = "Single Thread"; boolean cancelBeforeTrigger = false; // always cancel running thread and wait for it to end before starting new operation F0 enter; // optional ownership marker, e.g. for DynModules int cancelTimeOut = 10000; boolean trigger = false; Thread thread; WeakReference threadBeingCancelled; ReliableSingleThread(Object runnable) { this.runnable = runnable;} void trigger() { go(); } synchronized void go() { if (cancelBeforeTrigger) cancel(); trigger = true; if (!running()) { AutoCloseable __1 = callF(enter); try { thread = startThread(name, new Runnable() { public void run() { try { AutoCloseable __2 = callF(enter); try { _run(); } finally { _close(__2); }} catch (Exception __e) { throw rethrow(__e); } } public String toString() { return "temp callF(enter);\r\n _run();"; }}); } finally { _close(__1); }} } public void run() { go(); } void get() { go(); } // so you can use the ! syntax synchronized boolean running() { return thread != null; } // use only if this is the last time you trigger this void triggerAndWait() { trigger(); waitUntilDone(); } void waitUntilDone() { while (running()) sleep(1); } void _run() { try { while (licensed()) { Thread oldThread; synchronized(this) { if (!trigger) { thread = null; break; } oldThread = getWeakRef(threadBeingCancelled); trigger = false; } if (oldThread != null && oldThread != currentThread()) oldThread.join(cancelTimeOut); pcallF(runnable); } } catch (Exception __e) { throw rethrow(__e); } } synchronized void cancel() { if (thread == null) return; threadBeingCancelled = new WeakReference(thread); cancelAndInterruptThread(thread); thread = null; } void cancelAndTrigger() { cancel(); trigger(); } synchronized boolean triggered() { return trigger; } void cleanMeUp() { cancel(); } ReliableSingleThread cancelBeforeTrigger() { cancelBeforeTrigger = true; return this; } }abstract static class ILCCompactIndex { String its(int idx) { return itemToString(idx); } abstract String itemToString(int idx); abstract int numLeftPairs(int symbol); abstract int numRightPairs(int symbol); abstract int[] getLeftPairs(int symbol); abstract int[] getRightPairs(int symbol); abstract int getProdLength(int idx); abstract int getOccurrences(int idx); abstract int lookupLiteral(char c); int pairLeft(int idx) { return firstIntFromLong(getPair(idx)); } int pairRight(int idx) { return secondIntFromLong(getPair(idx)); } abstract long getPair(int idx); abstract int numberOfSymbols(); abstract String filePartToString(int iFile, int from, int to); abstract IntegerIterator getRightPairsStartingWith(int symbol, String from); abstract IntegerIterator getLeftPairsEndingWith(int symbol, String from); abstract List symbolToFiles(int symbol); // values start at numberOfSymbols() abstract String symbolToProtocol(int symbol, Object... __); abstract int getFile(int iFile); // iFile starts at 0 abstract int getFileLength(int iFile); // iFile starts at 0 abstract int nFiles(); abstract int nLiterals(); abstract boolean isPair(int symbol); abstract CharacterIterator itemCharIteratorWithSkip(int symbol, int skip); } // read-only, so far. should be thread-safe final static class BufferedDiskByteMemory64 implements IByteMemory64, AutoCloseable { File file; long size; // file size in ints RandomAccessFile raf; boolean bigEndian = true; boolean writable = false; boolean debug = false; // byte buffers int byteBufferShift = 30; // each buffer is 1 GB int byteBufferSize = 1 << byteBufferShift; MappedByteBuffer[] byteBuffers; BufferedDiskByteMemory64(File file) { this(file, false); this.file = file; } BufferedDiskByteMemory64(File file, boolean writable) { this.writable = writable; this.file = file; load(file, writable); } void load(File file) { load(file, false); } void load(File file, boolean writable) { try { this.file = file; this.writable = writable; size = fileSize(file); raf = newRandomAccessFile(file, writable ? "rw" : "r"); byteBuffers = new MappedByteBuffer[toInt(rightShift_ceil(size, byteBufferShift))]; FileChannel channel = raf.getChannel(); for (int i = 0; i < l(byteBuffers); i++) { long pos = (long) i << byteBufferShift; int len = toInt(min(1 << byteBufferShift, size-pos)); //print("Mapping bytes " + longToHex(pos) + "-" + longToHex(pos+len)); byteBuffers[i] = channel.map(FileChannel.MapMode.READ_ONLY, pos, len); byteBuffers[i].order(bigEndian ? ByteOrder.BIG_ENDIAN : ByteOrder.LITTLE_ENDIAN); } } catch (Exception __e) { throw rethrow(__e); } } public void close() { { cleanUp(raf); raf = null; } } public byte getByte(long idx) { rangeCheck(idx, size); return byteBuffers[(int) (idx >> byteBufferShift)].get(((int) idx) & (byteBufferSize-1)); } public int getInt(long idx) { int ofs = ((int) idx) & (byteBufferSize-1); // check if we are crossing a buffer boundary if (ofs > byteBufferSize-4) return getInt_manual(idx); try { return byteBuffers[(int) (idx >> byteBufferShift)].getInt(ofs); } catch (Throwable _e) { print("idx", idx); throw rethrow(_e); } } public int getInt_manual(long idx) { assertTrue("bigEndian", bigEndian); return ubyteToInt(getByte(idx)) << 24 | ubyteToInt(getByte(idx+1)) << 16 | ubyteToInt(getByte(idx+2)) << 8 | ubyteToInt(getByte(idx+3)); } public void set(long idx, int val) { checkWritable(); throw fail("todo"); } void checkWritable() { if (!writable) throw fail("read-only"); } public long size() { return size; } public void ensureSize(int size) { this.size = max(this.size, size); } }static class LongBuffer implements Iterable, ILongQueue { long[] data; int size; LongBuffer() {} LongBuffer(int size) { if (size != 0) data = new long[size]; } LongBuffer(Iterable l) { this(l(l)); addAll(l); } public void add(long i) { if (size >= lLongArray(data)) { data = resizeLongArray(data, Math.max(1, toInt(Math.min(maximumSafeArraySize(), lLongArray(data)*2L)))); if (size >= data.length) throw fail("LongBuffer too large: " + size); } data[size++] = i; } void addAll(Iterable l) { if (l != null) for (long i : l) add(i); } long[] toArray() { return size == 0 ? null : resizeLongArray(data, size); } List toList() { return longArrayToList(data, 0, size); } List asVirtualList() { return listFromFunction(__46 -> get(__46),size); } void reset() { size = 0; } void clear() { reset(); } int size() { return size; } public boolean isEmpty() { return size == 0; } long get(int idx) { if (idx >= size) throw fail("Index out of range: " + idx + "/" + size); return data[idx]; } void set(int idx, long value) { if (idx >= size) throw fail("Index out of range: " + idx + "/" + size); data[idx] = value; } long popLast() { if (size == 0) throw fail("empty buffer"); return data[--size]; } long last() { return data[size-1]; } long nextToLast() { return data[size-2]; } public String toString() { return squareBracket(joinWithSpace(toList())); } public Iterator iterator() { return new IterableIterator() { int i = 0; public boolean hasNext() { return i < size; } public Long next() { if (!hasNext()) throw fail("Index out of bounds: " + i); return data[i++]; } }; } void trimToSize() { data = resizeLongArray(data, size); } // don't rely on return value if buffer is empty public long poll() { return size == 0 ? -1 : data[--size]; } }/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ // A native long priority queue // LongComparator support added by Stefan Reich final static class LongPriorityQueue implements ILongQueue { protected int size; // number of elements currently in the queue protected int currentCapacity; // number of elements the queue can hold w/o expanding protected int maxSize = Integer.MAX_VALUE; // max number of elements allowed in the queue protected long[] heap; protected final long sentinel; // represents a null return value LongComparator comparator; // may be null // too many constructors follow LongPriorityQueue(long sentinel) { this(4, sentinel); } LongPriorityQueue(LongComparator comparator) { this(-1); this.comparator = comparator; } LongPriorityQueue(long sentinel, LongComparator comparator) { this(sentinel); this.comparator = comparator; } LongPriorityQueue(int initialSize, long sentinel) { this.sentinel = sentinel; initialize(initialSize); } LongPriorityQueue(int initialSize, int maxSize, long sentinel) { this.sentinel = sentinel; this.maxSize = maxSize; initialize(initialSize); } protected void initialize(int sz) { int heapSize; if (0 == sz) // We allocate 1 extra to avoid if statement in top() heapSize = 2; else { // NOTE: we add +1 because all access to heap is // 1-based not 0-based. heap[0] is unused. heapSize = Math.max(sz, sz + 1); // handle overflow } heap = new long[heapSize]; currentCapacity = sz; } public int getCurrentCapacity() { return currentCapacity; } public void resize(int sz) { int heapSize; if (sz > maxSize) { maxSize = sz; } if (0 == sz) // We allocate 1 extra to avoid if statement in top() heapSize = 2; else { heapSize = Math.max(sz, sz + 1); // handle overflow } heap = Arrays.copyOf(heap, heapSize); currentCapacity = sz; } /** * Adds an object to a PriorityQueue in log(size) time. If one tries to add * more objects than maxSize from initialize an * {@link ArrayIndexOutOfBoundsException} is thrown. */ public /*long*/void add(long element) { if (size >= currentCapacity) { int newSize = Math.min(currentCapacity <<1, maxSize); if (newSize < currentCapacity) newSize = Integer.MAX_VALUE; // handle overflow resize(newSize); } size++; heap[size] = element; upHeap(); } /** * Adds an object to a PriorityQueue in log(size) time. If one tries to add * more objects than the current capacity, an * {@link ArrayIndexOutOfBoundsException} is thrown. */ public void addNoCheck(long element) { ++size; heap[size] = element; upHeap(); } /** * Adds an object to a PriorityQueue in log(size) time. * It returns the smallest object (if any) that was * dropped off the heap because it was full, or * the sentinel value. * * This can be * the given parameter (in case it is smaller than the * full heap's minimum, and couldn't be added), or another * object that was previously the smallest value in the * heap and now has been replaced by a larger one, or null * if the queue wasn't yet full with maxSize elements. */ public long insertWithOverflow(long element) { if (size < maxSize) { add(element); return sentinel; } else if (compare(element, heap[1]) > 0) { long ret = heap[1]; heap[1] = element; updateTop(); return ret; } else { return element; } } /** inserts the element and returns true if this element caused another element * to be dropped from the queue. */ public boolean insert(long element) { if (size < maxSize) { add(element); return false; } else if (compare(element, heap[1]) > 0) { // long ret = heap[1]; heap[1] = element; updateTop(); return true; } else { return false; } } /** Returns the least element of the PriorityQueue in constant time. */ public long top() { return heap[1]; } /** Removes and returns the least element of the PriorityQueue in log(size) time. Only valid if size() > 0. */ public long pop() { long result = heap[1]; // save first value heap[1] = heap[size]; // move last to first size--; downHeap(); // adjust heap return result; } // return sentinel if empty or pop() public long poll() { return isEmpty() ? sentinel : pop(); } /** * Should be called when the Object at top changes values. * @return the new 'top' element. */ public long updateTop() { downHeap(); return heap[1]; } /** Returns the number of elements currently stored in the PriorityQueue. */ public int size() { return size; } /** Returns the array used to hold the heap, with the smallest item at array[1] * and the last (but not necessarily largest) at array[size()]. This is *not* * fully sorted. */ public long[] getInternalArray() { return heap; } /** Pops the smallest n items from the heap, placing them in the internal array at * arr[size] through arr[size-(n-1)] with the smallest (first element popped) * being at arr[size]. The internal array is returned. */ public long[] sort(int n) { while (--n >= 0) { long result = heap[1]; // save first value heap[1] = heap[size]; // move last to first heap[size] = result; // place it last size--; downHeap(); // adjust heap } return heap; } /** Removes all entries from the PriorityQueue. */ public void clear() { size = 0; } private void upHeap() { int i = size; long node = heap[i]; // save bottom node int j = i >>> 1; while (j > 0 && compare(node, heap[j]) < 0) { heap[i] = heap[j]; // shift parents down i = j; j = j >>> 1; } heap[i] = node; // install saved node } private void downHeap() { int i = 1; long node = heap[i]; // save top node int j = i << 1; // find smaller child int k = j + 1; if (k <= size && compare(heap[k], heap[j]) < 0) { j = k; } while (j <= size && compare(heap[j], node) < 0) { heap[i] = heap[j]; // shift up child i = j; j = i << 1; k = j + 1; if (k <= size && compare(heap[k], heap[j]) < 0) { j = k; } } heap[i] = node; // install saved node } int compare(long a, long b) { return comparator == null ? cmp(a, b) : comparator.compare(a, b); } public boolean isEmpty() { return size == 0; } }static class CountingInputStream extends FilterInputStream { long counter; CountingInputStream(InputStream in) { super(in); } @Override public int read() throws IOException { int c = in.read(); if (c >= 0) ++counter; return c; } @Override public int read(byte[] b) throws IOException { int n = in.read(b); counter += max(n, 0); return n; } @Override public int read(byte[] b, int off, int len) throws IOException { int n = in.read(b, off, len); counter += max(n, 0); return n; } long getFilePointer() { return counter; } }static interface LongComparator { int compare(long a, long b); }static class LCSearcher_TreeWalker2 implements IFieldsToList{ ILCCompactIndex lc; LCSearcher_TreeWalker2() {} LCSearcher_TreeWalker2(ILCCompactIndex lc) { this.lc = lc;} public String toString() { return shortClassName(this) + "(" + lc + ")"; }public Object[] _fieldsToList() { return new Object[] {lc}; } boolean debug = false; IntRange queryRange; List result = new ArrayList(); // pair(pair symbol, offset in query) void scan(int iFile, IntRange queryRange) { this.queryRange = queryRange; int sym = lc.getFile(iFile); // get file's top symbol // We are covering the whole file with the current symbol IntRange range = new IntRange(0, lc.getProdLength(sym)); scan(sym, range, queryRange); } void scan(int sym, IntRange symRange, IntRange queryRange) { if (!intRangesOverlapNempty(symRange, queryRange)) return; if (!lc.isPair(sym)) return; int left = lc.pairLeft(sym), right = lc.pairRight(sym); int lLeft = lc.getProdLength(left); IntRange rLeft = intRangeWithLen(symRange.start, lLeft); IntRange rRight = intRange(symRange.start+lLeft, symRange.end); // if one of the children completely covers the query, // don't write down the mother pair. if (!intRangeIsPartOf(queryRange, rLeft) && !intRangeIsPartOf(queryRange, rRight)) gotPair(sym, symRange); scan(left, rLeft, queryRange); scan(right, rRight, queryRange); } transient IVF2 gotPair; void gotPair(int pair, IntRange range) { if (gotPair != null) gotPair.get(pair, range); else gotPair_base(pair, range); } final void gotPair_fallback(IVF2 _f, int pair, IntRange range) { if (_f != null) _f.get(pair, range); else gotPair_base(pair, range); } void gotPair_base(int pair, IntRange range) { result.add(intPair(pair, range.start-queryRange.start)); if (debug) print("Got pair: " + lc.symbolToProtocol(pair, "withIdx" , true) + " at " + range); } }static interface ILongQueue { public boolean isEmpty(); public void add(long element); // return value is undefined if queue is empty public long poll(); }static final class IntPair implements Comparable , IFieldsToList{ int a; int b; IntPair() {} IntPair(int a, int b) { this.b = b; this.a = a;} public String toString() { return shortClassName(this) + "(" + a + ", " + b + ")"; }public Object[] _fieldsToList() { return new Object[] {a, b}; } public boolean equals(Object o) { if (!(o instanceof IntPair)) return false; IntPair x = (IntPair) o; return a == x.a && b == x.b; } public int hashCode() { int h = -672893111; h = boostHashCombine(h, _hashCode(a)); h = boostHashCombine(h, _hashCode(b)); return h; } public int compareTo(IntPair p) { if (p == null) return 1; int pa = p.a; if (a < pa) return -1; if (a > pa) return 1; return Integer.compare(b, p.b); } }static class LCSearcher_Protocol { List log = new ArrayList(); int exitAfter; boolean print = false; LCSearcher_Protocol() {} LCSearcher_Protocol(boolean print) { this.print = print;} transient IVF1 onLog; void onLog(String s) { if (onLog != null) onLog.get(s); else onLog_base(s); } final void onLog_fallback(IVF1 _f, String s) { if (_f != null) _f.get(s); else onLog_base(s); } void onLog_base(String s) {} void log(String s) { log.add(s); if (print) print(s); onLog(s); if (l(log) == exitAfter) quickFail("Protocol full: " + exitAfter); } transient IVF5 searched; void searched(ILCCompactIndex lc, boolean toTheRight, int symbol, String rest, IntBuffer results) { if (searched != null) searched.get(lc, toTheRight, symbol, rest, results); else searched_base(lc, toTheRight, symbol, rest, results); } final void searched_fallback(IVF5 _f, ILCCompactIndex lc, boolean toTheRight, int symbol, String rest, IntBuffer results) { if (_f != null) _f.get(lc, toTheRight, symbol, rest, results); else searched_base(lc, toTheRight, symbol, rest, results); } void searched_base(ILCCompactIndex lc, boolean toTheRight, int symbol, String rest, IntBuffer results) { if (toTheRight) log("Searched " + lc.symbolToProtocol(symbol) + " > " + quote(rest) + " >> " + nResults(l(results))); else log("Searched " + quote(rest) + " < " + lc.symbolToProtocol(symbol) + " >> " + nResults(l(results))); } String asText() { return lines_rtrim(log); } }static interface IFieldsToList { Object[] _fieldsToList(); }abstract static class CharacterIterator { abstract boolean hasNext(); abstract char next(); }static class IntBuffer implements Iterable { int[] data; int size; IntBuffer() {} IntBuffer(int size) { if (size != 0) data = new int[size]; } IntBuffer(Iterable l) { this(l(l)); addAll(l); } void add(int i) { if (size >= lIntArray(data)) { data = resizeIntArray(data, Math.max(1, toInt(Math.min(maximumSafeArraySize(), lIntArray(data)*2L)))); if (size >= data.length) throw fail("IntBuffer too large: " + size); } data[size++] = i; } void addAll(Iterable l) { if (l != null) for (int i : l) add(i); } int[] toArray() { return size == 0 ? null : resizeIntArray(data, size); } List toList() { return intArrayToList(data, 0, size); } List asVirtualList() { return new RandomAccessAbstractList() { public int size() { return size; } public Integer get(int i) { return IntBuffer.this.get(i); } public Integer set(int i, Integer val) { Integer a = get(i); data[i] = val; return a; } }; } void reset() { size = 0; } void clear() { reset(); } int size() { return size; } boolean isEmpty() { return size == 0; } int get(int idx) { if (idx >= size) throw fail("Index out of range: " + idx + "/" + size); return data[idx]; } void set(int idx, int value) { if (idx >= size) throw fail("Index out of range: " + idx + "/" + size); data[idx] = value; } int popLast() { if (size == 0) throw fail("empty buffer"); return data[--size]; } int last() { return data[size-1]; } int nextToLast() { return data[size-2]; } public String toString() { return squareBracket(joinWithSpace(toList())); } public Iterator iterator() { return new IterableIterator() { int i = 0; public boolean hasNext() { return i < size; } public Integer next() { //if (!hasNext()) fail("Index out of bounds: " + i); return data[i++]; } }; } public IntegerIterator integerIterator() { return new IntegerIterator() { int i = 0; public boolean hasNext() { return i < size; } public int next() { //if (!hasNext()) fail("Index out of bounds: " + i); return data[i++]; } public String toString() { return "Iterator@" + i + " over " + IntBuffer.this; } }; } void trimToSize() { data = resizeIntArray(data, size); } }static class NotTooOften { long minDelay; long lastTime; NotTooOften() {} NotTooOften(long minDelay) { this.minDelay = minDelay;} boolean canDoAgain_willDoIfTrue() { if (lastTime == 0 || now() >= lastTime+minDelay) { lastTime = now(); return true; } return false; } boolean yo() { return canDoAgain_willDoIfTrue(); } boolean get() { return canDoAgain_willDoIfTrue(); } }static interface IByteMemory64 { byte getByte(long idx); int getInt(long idx); //void setByte(long idx, byte val); long size(); default int[] readIntArray(long start, int len) { int[] a = new int[len]; for (int i = 0; i < len; i++) a[i] = getInt(start+(((long) i) << 2)); return a; } default int[] readIntArray(LongRange r) { return r == null ? null : readIntArray(r.start, toInt(r.length())); } default long getLong(long idx) { return twoIntsToLong(getInt(idx), getInt(idx+4)); } /*default void setLong(long idx, long val) { setInt(idx, firstIntFromLong(val)); setInt(idx+4, secondIntFromLong(val)); }*/ }static interface IVF1_Long { void get(long l); }static interface IVF1 { void get(A a); }abstract static class LongIterator { abstract boolean hasNext(); abstract long next(); } // uses hash sets as inner sets unless subclassed // uses a hash map as the outer map by default static class MultiSetMap { Map> data = new HashMap>(); int size; MultiSetMap() {} MultiSetMap(boolean useTreeMap) { if (useTreeMap) data = new TreeMap(); } MultiSetMap(MultiSetMap map) { putAll(map); } MultiSetMap(Map> data) { this.data = data;} boolean put(A key, B value) { synchronized(data) { Set set = data.get(key); if (set == null) data.put(key, set = _makeEmptySet()); if (!set.add(value)) return false; { ++size; return true; } }} boolean add(A key, B value) { return put(key, value); } void addAll(A key, Collection values) { synchronized(data) { putAll(key, values); }} void addAllIfNotThere(A key, Collection values) { synchronized(data) { for (B value : values) setPut(key, value); }} void setPut(A key, B value) { synchronized(data) { if (!containsPair(key, value)) put(key, value); }} boolean containsPair(A key, B value) { synchronized(data) { return get(key).contains(value); }} void putAll(A key, Collection values) { synchronized(data) { for (B value : values) put(key, value); }} void removeAll(A key, Collection values) { synchronized(data) { for (B value : values) remove(key, value); }} Set get(A key) { synchronized(data) { Set set = data.get(key); return set == null ? Collections. emptySet() : set; }} // return null if empty Set getOpt(A key) { synchronized(data) { return data.get(key); }} // returns actual mutable live set // creates the set if not there Set getActual(A key) { synchronized(data) { Set set = data.get(key); if (set == null) data.put(key, set = _makeEmptySet()); return set; }} // TODO: this looks unnecessary void clean(A key) { synchronized(data) { Set list = data.get(key); if (list != null && list.isEmpty()) data.remove(key); }} Set keySet() { synchronized(data) { return data.keySet(); }} Set keys() { synchronized(data) { return data.keySet(); }} void remove(A key) { synchronized(data) { size -= l(data.get(key)); data.remove(key); }} void remove(A key, B value) { synchronized(data) { Set set = data.get(key); if (set != null) { if (set.remove(value)) { --size; if (set.isEmpty()) data.remove(key); } } }} void clear() { synchronized(data) { data.clear(); size = 0; }} boolean containsKey(A key) { synchronized(data) { return data.containsKey(key); }} B getFirst(A key) { synchronized(data) { return first(get(key)); }} void addAll(MultiSetMap map) { putAll(map); } void putAll(MultiSetMap map) { synchronized(data) { for (A key : map.keySet()) putAll(key, map.get(key)); }} void putAll(Map map) { synchronized(data) { if (map != null) for (Map.Entry e : map.entrySet()) put(e.getKey(), e.getValue()); }} int keysSize() { synchronized(data) { return l(data); }} // full size int size() { synchronized(data) { return size; }} // count values for key int getSize(A key) { return l(data.get(key)); } int count(A key) { return getSize(key); } // expensive operation Set reverseGet(B b) { synchronized(data) { Set l = new HashSet(); for (A key : data.keySet()) if (data.get(key).contains(b)) l.add(key); return l; }} Map> asMap() { synchronized(data) { return cloneMap(data); }} boolean isEmpty() { synchronized(data) { return data.isEmpty(); }} // override in subclasses Set _makeEmptySet() { return new HashSet(); } Collection> allLists() { synchronized(data) { return new HashSet(data.values()); } } List allValues() { return concatLists(values(data)); } Object mutex() { return data; } public String toString() { return "mm" + str(data); } Pair firstEntry() { if (empty(data)) return null; Map.Entry> entry = data.entrySet().iterator().next(); return pair(entry.getKey(), first(entry.getValue())); } }static abstract class F0 { abstract A get(); }static interface IVF2 { void get(A a, B b); }// you still need to implement hasNext() and next() static abstract class IterableIterator implements Iterator, Iterable { public Iterator iterator() { return this; } public void remove() { unsupportedOperation(); } }abstract static class RandomAccessAbstractList extends AbstractList implements RandomAccess { }/* * #! * Ontopia Engine * #- * Copyright (C) 2001 - 2013 The Ontopia Project * #- * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. * !# */ // modified by Stefan Reich // Implements the Set interface more compactly than // java.util.HashSet by using a closed hashtable. static class CompactHashSet extends java.util.AbstractSet { protected final static int INITIAL_SIZE = 3; public final static double LOAD_FACTOR = 0.75; protected final static Object nullObject = new Object(); protected final static Object deletedObject = new Object(); protected int elements; protected int freecells; protected A[] objects; CompactHashSet() { this(INITIAL_SIZE); } CompactHashSet(int size) { // NOTE: If array size is 0, we get a // "java.lang.ArithmeticException: / by zero" in add(Object). objects = (A[]) new Object[(size==0 ? 1 : size)]; elements = 0; freecells = objects.length; } CompactHashSet(Collection c) { this(c.size()); addAll(c); } @Override public Iterator iterator() { return new CompactHashIterator(); } @Override public int size() { return elements; } @Override public boolean isEmpty() { return elements == 0; } @Override public boolean contains(Object o) { return find(o) != null; } synchronized A find(Object o) { if (o == null) o = nullObject; int hash = o.hashCode(); int index = (hash & 0x7FFFFFFF) % objects.length; int offset = 1; // search for the object (continue while !null and !this object) while(objects[index] != null && !(objects[index].hashCode() == hash && objects[index].equals(o))) { index = ((index + offset) & 0x7FFFFFFF) % objects.length; offset = offset*2 + 1; if (offset == -1) offset = 2; } return objects[index]; } boolean removeIfSame(Object o) { A value = find(o); if (value == o) { remove(value); return true; } return false; } @Override synchronized public boolean add(Object o) { if (o == null) o = nullObject; int hash = o.hashCode(); int index = (hash & 0x7FFFFFFF) % objects.length; int offset = 1; int deletedix = -1; // search for the object (continue while !null and !this object) while(objects[index] != null && !(objects[index].hashCode() == hash && objects[index].equals(o))) { // if there's a deleted object here we can put this object here, // provided it's not in here somewhere else already if (objects[index] == deletedObject) deletedix = index; index = ((index + offset) & 0x7FFFFFFF) % objects.length; offset = offset*2 + 1; if (offset == -1) offset = 2; } if (objects[index] == null) { // wasn't present already if (deletedix != -1) // reusing a deleted cell index = deletedix; else freecells--; elements++; // here we face a problem regarding generics: // add(A o) is not possible because of the null Object. We cant do 'new A()' or '(A) new Object()' // so adding an empty object is a problem here // If (! o instanceof A) : This will cause a class cast exception // If (o instanceof A) : This will work fine objects[index] = (A) o; // do we need to rehash? if (1 - (freecells / (double) objects.length) > LOAD_FACTOR) rehash(); return true; } else // was there already return false; } @Override synchronized public boolean remove(Object o) { if (o == null) o = nullObject; int hash = o.hashCode(); int index = (hash & 0x7FFFFFFF) % objects.length; int offset = 1; // search for the object (continue while !null and !this object) while(objects[index] != null && !(objects[index].hashCode() == hash && objects[index].equals(o))) { index = ((index + offset) & 0x7FFFFFFF) % objects.length; offset = offset*2 + 1; if (offset == -1) offset = 2; } // we found the right position, now do the removal if (objects[index] != null) { // we found the object // same problem here as with add objects[index] = (A) deletedObject; elements--; return true; } else // we did not find the object return false; } @Override synchronized public void clear() { elements = 0; for (int ix = 0; ix < objects.length; ix++) objects[ix] = null; freecells = objects.length; } @Override synchronized public Object[] toArray() { Object[] result = new Object[elements]; Object[] objects = this.objects; int pos = 0; for (int i = 0; i < objects.length; i++) if (objects[i] != null && objects[i] != deletedObject) { if (objects[i] == nullObject) result[pos++] = null; else result[pos++] = objects[i]; } // unchecked because it should only contain A return result; } // not sure if this needs to have generics @Override synchronized public T[] toArray(T[] a) { int size = elements; if (a.length < size) a = (T[])java.lang.reflect.Array.newInstance( a.getClass().getComponentType(), size); A[] objects = this.objects; int pos = 0; for (int i = 0; i < objects.length; i++) if (objects[i] != null && objects[i] != deletedObject) { if (objects[i] == nullObject) a[pos++] = null; else a[pos++] = (T) objects[i]; } return a; } protected void rehash() { int garbagecells = objects.length - (elements + freecells); if (garbagecells / (double) objects.length > 0.05) // rehash with same size rehash(objects.length); else // rehash with increased capacity rehash(objects.length*2 + 1); } protected void rehash(int newCapacity) { int oldCapacity = objects.length; @SuppressWarnings("unchecked") A[] newObjects = (A[]) new Object[newCapacity]; for (int ix = 0; ix < oldCapacity; ix++) { Object o = objects[ix]; if (o == null || o == deletedObject) continue; int hash = o.hashCode(); int index = (hash & 0x7FFFFFFF) % newCapacity; int offset = 1; // search for the object while(newObjects[index] != null) { // no need to test for duplicates index = ((index + offset) & 0x7FFFFFFF) % newCapacity; offset = offset*2 + 1; if (offset == -1) offset = 2; } newObjects[index] = (A) o; } objects = newObjects; freecells = objects.length - elements; } private class CompactHashIterator implements Iterator { private int index; private int lastReturned = -1; @SuppressWarnings("empty-statement") public CompactHashIterator() { synchronized(CompactHashSet.this) { for (index = 0; index < objects.length && (objects[index] == null || objects[index] == deletedObject); index++) ; } } @Override public boolean hasNext() { synchronized(CompactHashSet.this) { return index < objects.length; } } @SuppressWarnings("empty-statement") @Override public T next() { synchronized(CompactHashSet.this) { /*if (modCount != expectedModCount) throw new ConcurrentModificationException();*/ int length = objects.length; if (index >= length) { lastReturned = -2; throw new NoSuchElementException(); } lastReturned = index; for (index += 1; index < length && (objects[index] == null || objects[index] == deletedObject); index++) ; if (objects[lastReturned] == nullObject) return null; else return (T) objects[lastReturned]; } } @Override public void remove() { synchronized(CompactHashSet.this) { if (lastReturned == -1 || lastReturned == -2) throw new IllegalStateException(); // delete object if (objects[lastReturned] != null && objects[lastReturned] != deletedObject) { objects[lastReturned] = (A) deletedObject; elements--; } } } } synchronized int capacity() { return objects.length; } // returns true if there was a shrink synchronized boolean shrinkToFactor(double factor) { if (factor > LOAD_FACTOR) throw fail("Shrink factor must be equal to or smaller than load factor: " + factor + " / " + LOAD_FACTOR); int newCapacity = max(INITIAL_SIZE, iround(size()/factor)); if (newCapacity >= capacity()) return false; rehash(newCapacity); return true; } }static interface IVF5 { void get(A a, B b, C c, D d, E e); } static char hexToChar(String s) { return charFromHex(s); } static boolean isAbstract(Class c) { return (c.getModifiers() & Modifier.ABSTRACT) != 0; } static boolean isAbstract(Method m) { return (m.getModifiers() & Modifier.ABSTRACT) != 0; } static boolean reflection_isForbiddenMethod(Method m) { return m.getDeclaringClass() == Object.class && eqOneOf(m.getName(), "finalize", "clone", "registerNatives"); } static Set allInterfacesImplementedBy(Class c) { if (c == null) return null; HashSet set = new HashSet(); allInterfacesImplementedBy_find(c, set); return set; } static void allInterfacesImplementedBy_find(Class c, Set set) { if (c.isInterface() && !set.add(c)) return; do { for (Class intf : c.getInterfaces()) allInterfacesImplementedBy_find(intf, set); } while ((c = c.getSuperclass()) != null); } static Method findMethod(Object o, String method, Object... args) { return findMethod_cached(o, method, args); } static boolean findMethod_checkArgs(Method m, Object[] args, boolean debug) { Class[] types = m.getParameterTypes(); if (types.length != args.length) { if (debug) System.out.println("Bad parameter length: " + args.length + " vs " + types.length); return false; } for (int i = 0; i < types.length; i++) if (!(args[i] == null || isInstanceX(types[i], args[i]))) { if (debug) System.out.println("Bad parameter " + i + ": " + args[i] + " vs " + types[i]); return false; } return true; } static Method findStaticMethod(Class c, String method, Object... args) { Class _c = c; while (c != null) { for (Method m : c.getDeclaredMethods()) { if (!m.getName().equals(method)) continue; if ((m.getModifiers() & Modifier.STATIC) == 0 || !findStaticMethod_checkArgs(m, args)) continue; return m; } c = c.getSuperclass(); } return null; } static boolean findStaticMethod_checkArgs(Method m, Object[] args) { Class[] types = m.getParameterTypes(); if (types.length != args.length) return false; for (int i = 0; i < types.length; i++) if (!(args[i] == null || isInstanceX(types[i], args[i]))) return false; return true; } static String unquote(String s) { if (s == null) return null; if (startsWith(s, '[')) { int i = 1; while (i < s.length() && s.charAt(i) == '=') ++i; if (i < s.length() && s.charAt(i) == '[') { String m = s.substring(1, i); if (s.endsWith("]" + m + "]")) return s.substring(i+1, s.length()-i-1); } } if (s.length() > 1) { char c = s.charAt(0); if (c == '\"' || c == '\'') { int l = endsWith(s, c) ? s.length()-1 : s.length(); StringBuilder sb = new StringBuilder(l-1); for (int i = 1; i < l; i++) { char ch = s.charAt(i); if (ch == '\\') { char nextChar = (i == l - 1) ? '\\' : s.charAt(i + 1); // Octal escape? if (nextChar >= '0' && nextChar <= '7') { String code = "" + nextChar; i++; if ((i < l - 1) && s.charAt(i + 1) >= '0' && s.charAt(i + 1) <= '7') { code += s.charAt(i + 1); i++; if ((i < l - 1) && s.charAt(i + 1) >= '0' && s.charAt(i + 1) <= '7') { code += s.charAt(i + 1); i++; } } sb.append((char) Integer.parseInt(code, 8)); continue; } switch (nextChar) { case '\"': ch = '\"'; break; case '\\': ch = '\\'; break; case 'b': ch = '\b'; break; case 'f': ch = '\f'; break; case 'n': ch = '\n'; break; case 'r': ch = '\r'; break; case 't': ch = '\t'; break; case '\'': ch = '\''; break; // Hex Unicode: u???? case 'u': if (i >= l - 5) { ch = 'u'; break; } int code = Integer.parseInt( "" + s.charAt(i + 2) + s.charAt(i + 3) + s.charAt(i + 4) + s.charAt(i + 5), 16); sb.append(Character.toChars(code)); i += 5; continue; default: ch = nextChar; // added by Stefan } i++; } sb.append(ch); } return sb.toString(); } } return s; // not quoted - return original } static List quoteAll(Collection l) { List x = new ArrayList(); for (String s : l) x.add(quote(s)); return x; } static int _hashCode(Object a) { return a == null ? 0 : a.hashCode(); } static ArrayList toList(A[] a) { return asList(a); } static ArrayList toList(int[] a) { return asList(a); } static ArrayList toList(Set s) { return asList(s); } static ArrayList toList(Iterable s) { return asList(s); } static boolean arraysEqual(Object[] a, Object[] b) { if (a.length != b.length) return false; for (int i = 0; i < a.length; i++) if (neq(a[i], b[i])) return false; return true; } static Object load(String varName) { readLocally(varName); return get(mc(), varName); } static Object load(String progID, String varName) { readLocally(progID, varName); return get(mc(), varName); } static BufferedReader rawByteReader(InputStream in) { return rawByteReader(in, 8192); } static BufferedReader rawByteReader(InputStream in, int bufSize) { try { return bufferedReader(_registerIOWrap(new InputStreamReader(in, "ISO-8859-1"), in), bufSize); } catch (Exception __e) { throw rethrow(__e); } } static BufferedReader rawByteReader(File f) { try { return rawByteReader(newFileInputStream(f)); } catch (Exception __e) { throw rethrow(__e); } } static BufferedReader rawByteReader(File f, int bufSize) { try { return rawByteReader(newFileInputStream(f), bufSize); } catch (Exception __e) { throw rethrow(__e); } } static BufferedInputStream bufferedFileInputStream(File f) { return bufferedInputStream(f); } static BufferedInputStream bufferedFileInputStream(File f, int bufSize) { try { return new BufferedInputStream(new FileInputStream(f), bufSize); } catch (Exception __e) { throw rethrow(__e); } } static byte[] isGZipFile_magic = bytesFromHex("1f8b08"); static boolean isGZipFile(File f) { return byteArraysEqual(isGZipFile_magic, loadBeginningOfBinaryFile(f, l(isGZipFile_magic))); } static GZIPInputStream gzipInputStream(File f) { return newGZIPInputStream(f); } static GZIPInputStream gzipInputStream(InputStream in) { return newGZIPInputStream(in); } static String readLineIgnoreCR(Reader reader) { return readLineIgnoreCR(reader, null); } static String readLineIgnoreCR(Reader reader, StringBuilder buf) { try { int ch; if (buf == null) buf = new StringBuilder(); else buf.setLength(0); while ((ch = reader.read()) >= 0) { if (ch == '\n') break; buf.append((char) ch); } return str(buf); } catch (Exception __e) { throw rethrow(__e); } } static char charFromHex(String s) { return (char) hexToInt(s); } static boolean contains(Collection c, Object o) { return c != null && c.contains(o); } static boolean contains(Object[] x, Object o) { if (x != null) for (Object a : x) if (eq(a, o)) return true; return false; } static boolean contains(String s, char c) { return s != null && s.indexOf(c) >= 0; } static boolean contains(String s, String b) { return s != null && s.indexOf(b) >= 0; } static boolean contains(BitSet bs, int i) { return bs != null && bs.get(i); } static boolean contains(Producer p, A a) { if (p != null && a != null) while (true) { A x = p.next(); if (x == null) break; if (eq(x, a)) return true; } return false; } static long twoIntsToLong(int a, int b) { return (((long) a) << 32) | (((long) b) & 0xFFFFFFFF); } static int intPercentRatio(double x, double y) { return ratioToIntPercent(x, y); } static String nPairs(long n) { return n2(n, "pair"); } static String nPairs(Collection l) { return nPairs(l(l)); } static String nPairs(Map map) { return nPairs(l(map)); } static List compactIntList(List l) { return l == null ? null : new IntBuffer(l).asVirtualList(); } static List parseInts(Iterable l) { return lambdaMap(__47 -> parseInt(__47),l); } static List splitAtSpace(String s) { return empty(s) ? emptyList() : asList(s.split("\\s+")); } static Set keys(Map map) { return map == null ? new HashSet() : map.keySet(); } // convenience shortcut for keys_gen static Set keys(Object map) { return keys((Map) map); } static Set keys(MultiMap mm) { return mm.keySet(); } static Set keys(MultiSetMap mm) { return mm.keySet(); } static String getText(final AbstractButton c) { return c == null ? "" : (String) swingAndWait(new F0() { public Object get() { try { return c.getText(); } catch (Exception __e) { throw rethrow(__e); } } public String toString() { return "ret c.getText();"; }}); } static String getText(final JTextComponent c) { return c == null ? "" : (String) swingAndWait(new F0() { public Object get() { try { return c.getText(); } catch (Exception __e) { throw rethrow(__e); } } public String toString() { return "ret c.getText();"; }}); } static String getText(final JLabel l) { return l == null ? "" : (String) swingAndWait(new F0() { public Object get() { try { return l.getText(); } catch (Exception __e) { throw rethrow(__e); } } public String toString() { return "ret l.getText();"; }}); } // returns the contents of text field for editable combo box static String getText(final JComboBox cb) { if (cb == null) return null; if (isEditableComboBox(cb)) return unnull((String) cb.getEditor().getItem()); else return str(cb.getSelectedItem()); } static String fromLines_rtrim(Collection lines) { return rtrim_fromLines(lines); } static int firstIntFromLong(long l) { return (int) (l >> 32); } static int secondIntFromLong(long l) { return (int) l; } static long longSum(Iterable l) { long sum = 0; for (Long i : unnull(l)) if (i != null) sum += i; return sum; } static String charToHex(char c) { return bytesToHex(charToBytes(c)); } static boolean stdEq2(Object a, Object b) { if (a == null) return b == null; if (b == null) return false; if (a.getClass() != b.getClass()) return false; for (String field : allFields(a)) if (neq(getOpt(a, field), getOpt(b, field))) return false; return true; } static int stdHash2(Object a) { if (a == null) return 0; return stdHash(a, toStringArray(allFields(a))); } static RandomAccessFile newRandomAccessFile(File path, String mode) { try { boolean forWrite = mode.indexOf('w') >= 0; if (forWrite) mkdirsForFile(path); RandomAccessFile f = new RandomAccessFile(path, mode); callJavaX("registerIO", f, path, forWrite); return f; } catch (Exception __e) { throw rethrow(__e); } } static long rightShift_ceil(long a, int shift) { return (a+(1 << shift)-1) >> shift; } static String longToHex(long l) { return bytesToHex(longToBytes(l)); } static void rangeCheck(int i, int n) { if (i < 0 || i >= n) throw fail("Range check fail: " + i + "/" + n); } static void rangeCheck(long i, long n) { if (i < 0 || i >= n) throw fail("Range check fail: " + i + "/" + n); } static int ubyteToInt(byte b) { return b & 0x0FF; } static A copyFields(Object x, A y, String... fields) { if (empty(fields)) { // assume we should copy all fields Map map = objectToMap(x); for (String field : map.keySet()) setOpt(y, field, map.get(field)); } else for (String field : fields) { Object o = getOpt(x, field); if (o != null) setOpt(y, field, o); } return y; } static A copyFields(Object x, A y, Collection fields) { return copyFields(x, y, asStringArray(fields)); } static String fromUtf8(byte[] bytes) { try { return bytes == null ? null : new String(bytes, utf8charset()); } catch (Exception __e) { throw rethrow(__e); } } static void assertStartsWith(List a, List b) { if (!startsWith(a, b)) throw fail(a + " does not start with " + b); } static void assertStartsWith(String a, String b) { if (!startsWith(a, b)) throw fail(quote(a) + " does not start with " + quote(b)); } static void assertEndsWith(String a, String b) { if (!endsWith(a, b)) throw fail(quote(a) + " does not end with " + quote(b)); } // Use like this: printVars_str(+x, +y); // Or: printVars("bla", +x); // Or: printVars bla(+x); static void printVars_str(Object... params) { String s = ""; if (odd(l(params))) { s = str(first(params)); if (endsWithLetterOrDigit(s)) s += ": "; params = dropFirst(params); } print(s + renderVars_str(params)); } static int roundUpTo(int n, int x) { return (x+n-1)/n*n; } static long roundUpTo(long n, long x) { return (x+n-1)/n*n; } static A printStruct(String prefix, A a) { printStructure(prefix, a); return a; } static A printStruct(A a) { printStructure(a); return a; } static int compressed30BitUint_lengthFromFirstByte(byte b) { return (ubyteToInt(b) >> 6)+1; } static int readCompressed30BitUint_IByteMemory64(IByteMemory64 mem, long ptr) { int i = ubyteToInt(mem.getByte(ptr)); if (i < 0x40) return i; int j = ubyteToInt(mem.getByte(++ptr)); if (i < 0x80) return ((i & 0x3F) << 8) | j; int k = ubyteToInt(mem.getByte(++ptr)); if (i < 0xC0) return ((i & 0x3F) << 16) | (j << 8) | k; return ((i & 0x3F) << 24) | (j << 16) | (k << 8) | ubyteToInt(mem.getByte(++ptr)); } static long uintToLong(int i) { return i & 0xFFFFFFFFL; } static String withStringBuilder(IVF1 f) { StringBuilder buf = new StringBuilder(); if (f != null) f.get(buf); return str(buf); } static String withStringBuilder(int expectedLength, IVF1 f) { StringBuilder buf = new StringBuilder(expectedLength); if (f != null) f.get(buf); return str(buf); } static String intToHex(int i) { return bytesToHex(intToBytes(i)); } static boolean cleanUp_interruptThreads = false; // experimental static void cleanUp(Object c) { if (c == null) return; if (c instanceof AutoCloseable) { close_pcall(((AutoCloseable) c)); return; } if (c instanceof java.util.Timer) { ((java.util.Timer) c).cancel(); return; } if (c instanceof Collection) { cleanUp((Collection) c); return; } if (c instanceof Map) { for (Object o : keys((Map) c)) cleanUp(o); for (Object o : values((Map) c)) cleanUp(o); ((Map) c).clear(); return; } //if (!(c instanceof Class)) ret; try { // revoke license preCleanUp(c); // unpause setOpt(c, "ping_pauseAll", false); // call custom cleanMeUp() and cleanMeUp_*() functions innerCleanUp(c); // Java spec says finalize should only be called by GC, // but we care to differ. // Edit: Not anymore (illegal access warnings) /*if (isTrue(vmMap_get('callFinalize))) pcallOpt(c, "finalize");*/ // remove all virtual bots (hope this works) List androids = (List) getOpt(c, "record_list"); for (Object android : unnull(androids)) pcallOpt(android, "dispose"); // heck we'll dispose anything // sub-cleanup List classes = (List) (getOpt(c, "hotwire_classes")); if (classes != null) for (WeakReference cc : classes) { try { cleanUp(cc.get()); } catch (Throwable __e) { _handleException(__e); }} // interrupt all threads (experimental, they might be doing cleanup?) if (cleanUp_interruptThreads) { List threads = registeredThreads(c); if (nempty(threads)) { print("cleanUp: Interrupting " + n2(threads, "thread") + ": " + joinWithComma(allToString(threads))); interruptThreads(threads); } } } catch (Throwable __e) { _handleException(__e); } setOpt(c, "cleaningUp_flag" , false); if (c instanceof Class && ((Class) c).getName().equals("main")) retireClassLoader(((Class) c).getClassLoader()); } static void cleanUp(Collection l) { if (l == null) return; for (Object c : l) cleanUp(c); l.clear(); } static char stringToChar(String s) { if (l(s) != 1) throw fail("bad stringToChar: " + s); return firstChar(s); } static LongRange longRangeWithLength(long start, long len) { return new LongRange(start, start+len); } static int compressed30BitUint_lengthForValue(int i) { if (i < 0x40) return 1; if (i < 0x4000) return 2; if (i < 0x400000) return 3; return 4; } static List reversed(Collection l) { return reversedList(l); } static List reversed(A[] l) { return reversedList(asList(l)); } static String reversed(String s) { return reversedString(s); } static int lCommonPrefixOfCharIterators(CharacterIterator it1, CharacterIterator it2) { int n = 0; while (true) { if (!it1.hasNext() || !it2.hasNext()) return n; char a = it1.next(), b = it2.next(); if (a != b) return n; ++n; } } static int lCommonPrefixOfCharIterators(CharacterIterator it1, CharacterIterator it2, CharComparator comparator) { if (comparator == null) return lCommonPrefixOfCharIterators(it1, it2); int n = 0; while (true) { if (!it1.hasNext() || !it2.hasNext()) return n; char a = it1.next(), b = it2.next(); if (comparator.compare(a, b) != 0) return n; ++n; } } static CharacterIterator characterIterator(String s) { return characterIterator(s, 0); } static CharacterIterator characterIterator(String s, int from) { return new CharacterIterator() { int n = l(s); int i = from; public boolean hasNext() { return i < n; } public char next() { return s.charAt(i++); } }; } static CharacterIterator characterIterator(String s, int from, int _n) { return new CharacterIterator() { int n = min(l(s), from+_n); int i = from; public boolean hasNext() { return i < n; } public char next() { return s.charAt(i++); } }; } static CharacterIterator characterIterator(char[] s, int from, int _n) { return new CharacterIterator() { int n = min(l(s), from+_n); int i = from; public boolean hasNext() { return i < n; } public char next() { return s[i++]; } }; } static int compareChars(CharComparator comp, char a, char b) { return comp == null ? a-b : comp.compare(a, b); } static int idiv_ceil(int a, int b) { return (a+b-1)/b; } static int idiv_ceil(long a, long b) { return toInt_checked((a+b-1)/b); } static String charIteratorToString_max(CharacterIterator it, int max) { if (it == null) return null; StringBuilder buf = new StringBuilder(); while (max-- > 0 && it.hasNext()) buf.append(it.next()); return str(buf); } static String charIteratorToString_max(int max, CharacterIterator it) { return charIteratorToString_max(it, max); } static int cmp(Number a, Number b) { return a == null ? b == null ? 0 : -1 : cmp(a.doubleValue(), b.doubleValue()); } static int cmp(double a, double b) { return a < b ? -1 : a == b ? 0 : 1; } static int cmp(int a, int b) { return a < b ? -1 : a == b ? 0 : 1; } static int cmp(long a, long b) { return a < b ? -1 : a == b ? 0 : 1; } static int cmp(Object a, Object b) { if (a == null) return b == null ? 0 : -1; if (b == null) return 1; return ((Comparable) a).compareTo(b); } static String shortClassName(Object o) { if (o == null) return null; Class c = o instanceof Class ? (Class) o : o.getClass(); String name = c.getName(); return shortenClassName(name); } static char[] toCharArray(String s) { return asChars(s); } static int boostHashCombine(int a, int b) { return a ^ (b + 0x9e3779b9 + (a << 6) + (a >> 2)); } static String quoteShorten(int len, String s) { return quote(shorten(len, s)); } static String reversedSubstring(String s, int x, int y) { if (s == null) return null; if (x < 0) x = 0; int n = s.length(); if (y < x) y = x; if (y > n) y = n; if (x >= y) return ""; // TODO: Is this actually faster than reversed(substring(...))? char[] chars = new char[y-x]; int j = y-x; for (int i = x; i < y; i++) chars[--j] = s.charAt(i); return String.valueOf(chars); } static int[] sortIntArrayInPlace(int[] a) { if (a != null) Arrays.sort(a); return a; } static boolean divides(long a, long b) { return (b % a) == 0; } static String rpad(String s, int l) { return rpad(s, l, ' '); } static String rpad(String s, int l, char c) { return lengthOfString(s) >= l ? s : s + rep(c, l-lengthOfString(s)); } static String rpad(int l, String s) { return rpad(s, l); } static long[] sortLongArrayInPlace(long[] a) { if (a != null) Arrays.sort(a); return a; } static int[] secondIntFromLong_onArray(long[] a) { if (a == null) return null; int n = a.length; int[] x = new int[n]; for (int i = 0; i < n; i++) x[i] = secondIntFromLong(a[i]); return x; } static boolean nempty(Collection c) { return !empty(c); } static boolean nempty(CharSequence s) { return !empty(s); } static boolean nempty(Object[] o) { return !empty(o); } static boolean nempty(byte[] o) { return !empty(o); } static boolean nempty(int[] o) { return !empty(o); } static boolean nempty(Map m) { return !empty(m); } static boolean nempty(Iterator i) { return i != null && i.hasNext(); } static boolean nempty(Object o) { return !empty(o); } static boolean nempty(IntRange r) { return !empty(r); } static boolean nempty(IntBuffer b) { return b != null && !b.isEmpty(); } static boolean nempty(LongBuffer b) { return b != null && !b.isEmpty(); } static IntRange intRangeWithLen(int start, int len) { return new IntRange(start, start+len); } static A set(A o, String field, Object value) { if (o == null) return null; if (o instanceof Class) set((Class) o, field, value); else try { Field f = set_findField(o.getClass(), field); makeAccessible(f); smartSet(f, o, value); } catch (Exception e) { throw new RuntimeException(e); } return o; } static void set(Class c, String field, Object value) { if (c == null) return; try { Field f = set_findStaticField(c, field); makeAccessible(f); smartSet(f, null, value); } catch (Exception e) { throw new RuntimeException(e); } } static Field set_findStaticField(Class c, String field) { Class _c = c; do { for (Field f : _c.getDeclaredFields()) if (f.getName().equals(field) && (f.getModifiers() & java.lang.reflect.Modifier.STATIC) != 0) return f; _c = _c.getSuperclass(); } while (_c != null); throw new RuntimeException("Static field '" + field + "' not found in " + c.getName()); } static Field set_findField(Class c, String field) { Class _c = c; do { for (Field f : _c.getDeclaredFields()) if (f.getName().equals(field)) return f; _c = _c.getSuperclass(); } while (_c != null); throw new RuntimeException("Field '" + field + "' not found in " + c.getName()); } static List synchroLinkedList() { return Collections.synchronizedList(new LinkedList()); } static Class _run(String progID, String... args) { Class main = hotwire(progID); callMain(main, args); return main; } static void add(BitSet bs, int i) { bs.set(i); } static boolean add(Collection c, A a) { return c != null && c.add(a); } static void add(Container c, Component x) { addToContainer(c, x); } static Runnable toRunnable(final Object o) { if (o instanceof Runnable) return (Runnable) o; return new Runnable() { public void run() { try { callF(o) ; } catch (Exception __e) { throw rethrow(__e); } } public String toString() { return "callF(o)"; }}; } static A syncPopFirst(List l) { if (empty(l)) return null; synchronized(l) { A a = first(l); l.remove(0); return a; } } static void making(boolean print, String name) { making(print, name, null); } static void making(boolean print, String name, Object object) { if (print) making(name, object); } static void making(String name) { making(name, null); } static void making(String name, Object object) { printWithTime("Making " + name); } static Map reverseMap(Map map) { Map rmap = new HashMap(); for (A key : keys(map)) rmap.put(map.get(key), key); return rmap; } static void parallelDo(Collection l, Object... __) { parallelDo(iterator(l), l(l), __); } static void parallelDo(Iterator it, int count, Object... __) { try { int queueSize = optPar("queueSize",__, 500); int poolSize = optPar("poolSize",__, coresToUse_fixed()); NotifyingBlockingThreadPoolExecutor e = new NotifyingBlockingThreadPoolExecutor(poolSize, queueSize, 15, TimeUnit.SECONDS); // TODO: if poolSize == 1, do it in this thread Var error = new Var(); try { for (A o : iterable(it)) { if (o == null) continue; e.execute(new Runnable() { public void run() { try { try { try { o.run(); } catch (Throwable e) { error.set(e); } } catch (Throwable __e) { _handleException(__e); } } catch (Exception __e) { throw rethrow(__e); } } public String toString() { return "pcall {\r\n try {\r\n o.run();\r\n } catch (Throwable e) {\r\n..."; }}); } e.shutdown(); e.awaitTermination(1, TimeUnit.DAYS); } finally { e.shutdown(); } if (error.has()) throw rethrow(error.get()); } catch (Exception __e) { throw rethrow(__e); } } static void sortIntBuffer_direct(IntBuffer buf, IntComparator comparator) { if (buf == null) return; timSortIntArrayWithComparator(buf.data, buf.size(), comparator); } static int incAtomicInt(AtomicInteger i) { return i.incrementAndGet(); } static int incAtomicInt(AtomicInteger i, int delta) { return i.addAndGet(delta); } static IterableIterator countIterator_exclusive(int b) { return countIterator_exclusive(0, b); } static IterableIterator countIterator_exclusive(int a, int b) { return new IterableIterator() { int i = a; public boolean hasNext() { return i < b; } public Integer next() { return i++; } }; } static IterableIterator countIterator_exclusive(int b, IF1 f) { return countIterator_exclusive(0, b, f); } static IterableIterator countIterator_exclusive(int a, int b, IF1 f) { return mapI_if1(f, countIterator_exclusive(a, b)); } static String nProductions(long n) { return n2(n, "production"); } static String nProductions(Collection l) { return nProductions(l(l)); } static String nProductions(Map map) { return nProductions(l(map)); } static int compareCharIterators(Iterator it1, Iterator it2) { while (true) { if (!it1.hasNext()) return it2.hasNext() ? -1 : 0; if (!it2.hasNext()) return 1; char a = it1.next(), b = it2.next(); if (a < b) return -1; if (a > b) return 1; } } static int compareCharIterators(CharacterIterator it1, CharacterIterator it2) { while (true) { if (!it1.hasNext()) return it2.hasNext() ? -1 : 0; if (!it2.hasNext()) return 1; char a = it1.next(), b = it2.next(); if (a < b) return -1; if (a > b) return 1; } } static int compareCharIterators(CharacterIterator it1, CharacterIterator it2, CharComparator comparator) { if (comparator == null) return compareCharIterators(it1, it2); while (true) { if (!it1.hasNext()) return it2.hasNext() ? -1 : 0; if (!it2.hasNext()) return 1; char a = it1.next(), b = it2.next(); int cmp = comparator.compare(a, b); if (cmp != 0) return cmp; } } static List intPairToList(IntPair p) { return p == null ? null : ll(p.a, p.b); } static IntPair longToIntPair(long l) { return new IntPair(firstIntFromLong(l), secondIntFromLong(l)); } static A second(List l) { return get(l, 1); } static A second(Iterable l) { if (l == null) return null; Iterator it = iterator(l); if (!it.hasNext()) return null; it.next(); return it.hasNext() ? it.next() : null; } static A second(A[] bla) { return bla == null || bla.length <= 1 ? null : bla[1]; } static B second(Pair p) { return p == null ? null : p.b; } static A second(Producer p) { if (p == null) return null; if (p.next() == null) return null; return p.next(); } static char second(String s) { return charAt(s, 1); } static int[] intBufferToArray(IntBuffer buf) { return buf == null ? null : buf.toArray(); } static List characters(final String s) { return stringAsCharacterList(s); } static String formatFunctionCall(String fname, Object... args) { return fname + "(" + joinWithComma(allToString(args)) + ")"; } static ArrayList intArrayToList(int[] a) { if (a == null) return null; return intArrayToList(a, 0, a.length); } // no range checking on from/to in the interest of S P E E E E E EEED static ArrayList intArrayToList(int[] a, int from, int to) { if (a == null) return null; ArrayList < Integer > l = new ArrayList<>(to-from); for (int i = from; i < to; i++) l.add(a[i]); return l; } static IntegerIterator integerIterator(int[] l) { return l == null ? null : new IntegerIterator() { int i = 0; public boolean hasNext() { return i < l.length; } public int next() { return l[i++]; } }; } static int compareCharIterators_maxLen(CharacterIterator it1, CharacterIterator it2, int n) { for (int i = 0; i < n; i++) { if (!it1.hasNext()) return it2.hasNext() ? -1 : 0; if (!it2.hasNext()) return 1; char a = it1.next(), b = it2.next(); if (a < b) return -1; if (a > b) return 1; } return 0; } static long fileLength(String path) { return getFileSize(path); } static long fileLength(File f) { return getFileSize(f); } static String linesLL_rtrim(Object... x) { return lines_rtrim(ll(x)); } static LinkedHashMap mapValuesToLinkedHashMap(Object func, Map map) { LinkedHashMap m = new LinkedHashMap(); for (Object key : keys(map)) m.put(key, callF(func, map.get(key))); return m; } static LinkedHashMap mapValuesToLinkedHashMap(Map map, IF1 f) { return mapValuesToLinkedHashMap(f, map); } static LinkedHashMap mapValuesToLinkedHashMap(IF1 f, Map map) { return mapValuesToLinkedHashMap((Object) f, map); } static LinkedHashMap mapValuesToLinkedHashMap(Map map, Object func) { return mapValuesToLinkedHashMap(func, map); } static List toLines_nOnly_reversible(String s) { List lines = new ArrayList(); if (s == null) return lines; int start = 0, n = s.length(); if (n != 0) while (true) { int i = smartIndexOf(s, '\n', start); lines.add(s.substring(start, i)); if (i == n) break; start = i+1; } return lines; } static class mapI_It extends IterableIterator { Object f; Iterator i; mapI_It() {} mapI_It(Object f, Iterator i) { this.i = i; this.f = f;} public boolean hasNext() { return i.hasNext(); } public Object next() { return callF(f, i.next()); } public String toString() { return formatFunctionCall("mapI", f, i); } } // apply a function to an iterator static IterableIterator mapI(final Object f, final Iterator i) { return new mapI_It(f, i); } static IterableIterator mapI(IterableIterator i, Object f) { return mapI((Iterator) i, f); } static IterableIterator mapI(Object f, IterableIterator i) { return mapI((Iterator) i, f); } static IterableIterator mapI(Iterator i, Object f) { return mapI(f, i); } static IterableIterator mapI(Iterable i, IF1 f) { return mapI(i, (Object) f); } static IterableIterator mapI(Iterator i, IF1 f) { return mapI(f, i); } static IterableIterator mapI(IterableIterator i, IF1 f) { return mapI((Iterator) i, f); } static IterableIterator mapI(IF1 f, Iterable i) { return mapI(i, (Object) f); } static IterableIterator mapI(Iterable i, Object f) { return mapI(f, i.iterator()); } static IterableIterator mapI(Object f, Iterable i) { return mapI(i, f); } static List concatLists(Iterable... lists) { List l = new ArrayList(); if (lists != null) for (Iterable list : lists) addAll(l, list); return l; } static List concatLists(Collection> lists) { List l = new ArrayList(); if (lists != null) for (Iterable list : lists) addAll(l, list); return l; } static Collection values(Map map) { return map == null ? emptyList() : map.values(); } // convenience shortcut for values_gen static Collection values(Object map) { return values((Map) map); } static Collection values(MultiMap mm) { return mm == null ? emptyList() : concatLists(values(mm.data)); } static String nLines(long n) { return n2(n, "line"); } static String nLines(Collection l) { return nLines(l(l)); } static String nLines(String s) { return nLines(countLines(s)); } static List uniquify(Collection l) { return uniquifyList(l); } static List sortInPlace(List l, final Object comparator) { return sortedInPlace(l, comparator); } static List sortInPlace(List l) { return sortedInPlace(l); } static List collect(Iterable c, String field) { return collectField(c, field); } static List collect(String field, Iterable c) { return collectField(c, field); } /*ifclass Concept static L collect(Class c, S field) { ret collect(list(c), field); } endif TODO: make translator ignore stuff in ifclass until resolved */ static Map ciListIndex(List l) { return listIndexCI(l); } static Map listIndex(List l) { Map map = new HashMap(); for (int i = 0; i < l(l); i++) map.put(l.get(i), i); return map; } static Map mapValues(Object func, Map map) { Map m = similarEmptyMap(map); for (Object key : keys(map)) m.put(key, callF(func, map.get(key))); return m; } static Map mapValues(Map map, IF1 f) { return mapValues(f, map); } static Map mapValues(IF1 f, Map map) { return mapValues((Object) f, map); } static Map mapValues(Map map, Object func) { return mapValues(func, map); } static List wrapIntArrayAsImmutableList(int[] l) { return new RandomAccessAbstractList() { public int size() { return l.length; } public Integer get(int i) { return l[i]; } }; } static int[] mapToIntArray(Object f, Collection l) { return toIntArray(map(f, l)); } static int[] mapToIntArray(Object f, Object[] l) { return toIntArray(map(f, l)); } static int[] mapToIntArray(Collection l, IF1 f) { return mapToIntArray((Object) f, l); } /** writes safely (to temp file, then rename) */ static File saveTextFile(String fileName, String contents) throws IOException { CriticalAction action = beginCriticalAction("Saving file " + fileName + " (" + l(contents) + " chars)"); try { File file = new File(fileName); mkdirsForFile(file); String tempFileName = fileName + "_temp"; File tempFile = new File(tempFileName); if (contents != null) { if (tempFile.exists()) try { String saveName = tempFileName + ".saved." + now(); copyFile(tempFile, new File(saveName)); } catch (Throwable e) { printStackTrace(e); } FileOutputStream fileOutputStream = newFileOutputStream(tempFile.getPath()); OutputStreamWriter outputStreamWriter = new OutputStreamWriter(fileOutputStream, "UTF-8"); PrintWriter printWriter = new PrintWriter(outputStreamWriter); printWriter.print(contents); printWriter.close(); } if (file.exists() && !file.delete()) throw new IOException("Can't delete " + fileName); if (contents != null) if (!tempFile.renameTo(file)) throw new IOException("Can't rename " + tempFile + " to " + file); vmBus_send("wroteFile", file); return file; } finally { action.done(); } } static File saveTextFile(File fileName, String contents) { try { saveTextFile(fileName.getPath(), contents); return fileName; } catch (Exception __e) { throw rethrow(__e); } } static BufferedReader bufferedUtf8Reader(InputStream in) { return utf8BufferedReader(in); } static BufferedReader bufferedUtf8Reader(File f) { return utf8BufferedReader(f); } static List keysList(Map map) { return cloneListSynchronizingOn(keys(map), map); } static String lines_rtrim(Collection lines) { return rtrim_fromLines(lines); } static String nVersions(long n) { return n2(n, "version"); } static String nVersions(Collection l) { return nVersions(l(l)); } static String nVersions(Map map) { return nVersions(l(map)); } static List mapValuesToList(Object func, Map map) { List out = emptyList(l(map)); for (Object key : keys(map)) out.add(callF(func, map.get(key))); return out; } static List mapValuesToList(IF1 f, Map map) { return mapValuesToList((Object) f, map); } static List mapValuesToList(Map map, IF1 f) { return mapValuesToList(f, map); } static List mapValuesToList(Map map, Object func) { return mapValuesToList(func, map); } static Map mapToValues(Iterable l, Object f) { return mapKeyAndFunction(l, f); } static Map mapToValues(Object f, Iterable l) { return mapKeyAndFunction(f, l); } static Map mapToValues(Iterable l, IF1 f) { return mapKeyAndFunction(f, l); } static Map mapToValues(IF1 f, Iterable l) { return mapKeyAndFunction(f, l); } static Map mapToValues(Map map, IF2 f) { return mapKeyAndFunction(map, f); } static long intPairToLong(IntPair p) { return p == null ? 0 : (((long) p.a) << 32) | (((long) p.b) & 0xFFFFFFFF); } static int addAndReturnIndex(List l, A a) { if (l == null) return -1; int idx = l.size(); l.add(a); return idx; } static void listPut(List l, int i, A a, A emptyElement) { listSet(l, i, a, emptyElement); } static void listPut(List l, int i, A a) { listSet(l, i, a); } static void assertEqualsIC(String x, String y) { assertEqic(x, y); } static void assertEqualsIC(String msg, String x, String y) { assertEqic(msg, x, y); } static String assertEquals_quote(String x, String y) { return assertEquals_quote(null, x, y); } static String assertEquals_quote(String msg, String x, String y) { if (neq(x, y)) throw fail((msg != null ? msg + ": " : "") + quote(y) + " != " + quote(x)); return y; } static String lines(Iterable lines) { return fromLines(lines); } static String lines(Object[] lines) { return fromLines(asList(lines)); } static List lines(String s) { return toLines(s); } static IterableIterator countIterator(int b) { return countIterator(0, b); } static IterableIterator countIterator(int a, int b) { return countIterator_exclusive(a, b); } static IterableIterator countIterator(int b, IF1 f) { return countIterator(0, b, f); } static IterableIterator countIterator(int a, int b, IF1 f) { return countIterator_exclusive(a, b, f); } static int hashCodeFor(Object a) { return a == null ? 0 : a.hashCode(); } static Map putAll(Map a, Map b) { if (a != null && b != null) a.putAll(b); return a; } static void remove(List l, int i) { if (l != null && i >= 0 && i < l(l)) l.remove(i); } static void remove(Collection l, A a) { if (l != null) l.remove(a); } static Set keySet(Map map) { return map == null ? new HashSet() : map.keySet(); } static Set keySet(Object map) { return keys((Map) map); } static Set keySet(MultiMap mm) { return mm.keySet(); } static A reverseGet(List l, int idx) { if (l == null || idx < 0) return null; int n = l(l); return idx < n ? l.get(n-1-idx) : null; } static Map cloneMap(Map map) { if (map == null) return new HashMap(); // assume mutex is equal to map synchronized(map) { return map instanceof TreeMap ? new TreeMap((TreeMap) map) // copies comparator : map instanceof LinkedHashMap ? new LinkedHashMap(map) : new HashMap(map); } } static List cloneMap(Iterable l, IF1 f) { List x = emptyList(l); if (l != null) for (A o : cloneList(l)) x.add(f.get(o)); return x; } static int hashOfLong(long l) { return Long.hashCode(l); } static int length(Object[] array) { return array == null ? 0 : array.length; } static int length(List list) { return list == null ? 0 : list.size(); } static int length(String s) { return s == null ? 0 : s.length(); } static MultiSetMap multiSetMap_innerCustomTreeSet_outerRevTreeMap(Comparator innerComparator) { MultiSetMap m = multiSetMap_innerTreeSet(innerComparator); m.data = revTreeMap(); return m; } static MultiSetMap multiSetMap_innerCompactHashSet_outerRevTreeMap() { return new MultiSetMap(descTreeMap()) { Set _makeEmptySet() { return new CompactHashSet(); } }; } static List reverseInPlace(List l) { return reverseList(l); } static String a(String noun) { if (eq(noun, "")) return "?"; return ("aeiou".indexOf(noun.charAt(0)) >= 0 ? "an " : "a ") + noun; } static String a(String contents, Object... params) { return htag("a", contents, params); } static String b(Object contents, Object... params) { return tag("b", contents, params); } static String stringIf(boolean b, String s) { return stringIfTrue(b, s); } static List callFAll(Collection l, Object... args) { return callF_all(l, args); } static A firstKey(Map map) { return first(keys(map)); } static A firstKey(MultiSetMap map) { return map == null ? null : firstKey(map.data); } static B firstValue(Map map) { return first(values(map)); } static B firstValue(MultiSetMap map) { return map == null ? null : first(firstValue(map.data)); } static int or0(Integer i) { return i == null ? 0 : i; } static long or0(Long l) { return l == null ? 0L : l; } static double or0(Double d) { return d == null ? 0.0 : d; } static A popFirst(List l) { if (empty(l)) return null; A a = first(l); l.remove(0); return a; } static A popFirst(Collection l) { if (empty(l)) return null; A a = first(l); l.remove(a); return a; } static List popFirst(int n, List l) { List part = cloneSubList(l, 0, n); removeSubList(l, 0, n); return part; } static int hashSetCapacity(HashSet set) { return (int) call(get(set, "map"), "capacity"); } static Thread startThread(Object runnable) { return startThread(defaultThreadName(), runnable); } static Thread startThread(String name, Object runnable) { runnable = wrapAsActivity(runnable); return startThread(newThread(toRunnable(runnable), name)); } static Thread startThread(Thread t) { _registerThread(t); t.start(); return t; } static volatile boolean sleep_noSleep = false; static void sleep(long ms) { ping(); if (ms < 0) return; // allow spin locks if (isAWTThread() && ms > 100) throw fail("Should not sleep on AWT thread"); try { Thread.sleep(ms); } catch (Exception e) { throw new RuntimeException(e); } } static void sleep() { try { if (sleep_noSleep) throw fail("nosleep"); print("Sleeping."); sleepQuietly(); } catch (Exception __e) { throw rethrow(__e); } } static A getWeakRef(Reference ref) { return ref == null ? null : ref.get(); } static void cancelAndInterruptThread(Thread t) { if (t == null) return; cancelThread(t); t.interrupt(); } static void addAll(Collection c, Iterable b) { if (c != null && b != null) for (A a : b) c.add(a); } static boolean addAll(Collection c, Collection b) { return c != null && b != null && c.addAll(b); } static boolean addAll(Collection c, B... b) { return c != null && c.addAll(Arrays.asList(b)); } static Map addAll(Map a, Map b) { if (a != null) a.putAll(b); return a; } static int lLongArray(long[] a) { return a == null ? 0 : a.length; } static long[] resizeLongArray(long[] a, int n) { if (n == lLongArray(a)) return a; long[] b = new long[n]; arraycopy(a, 0, b, 0, Math.min(lLongArray(a), n)); return b; } static int maximumSafeArraySize() { return Integer.MAX_VALUE-8; } static ArrayList longArrayToList(long[] a) { if (a == null) return null; return longArrayToList(a, 0, a.length); } // no range checking on from/to in the interest of S P E E E E E EEED static ArrayList longArrayToList(long[] a, int from, int to) { if (a == null) return null; ArrayList < Long > l = new ArrayList<>(to-from); for (int i = from; i < to; i++) l.add(a[i]); return l; } static List listFromFunction(int n, IF1 f) { return new RandomAccessAbstractList() { public int size() { return n; } public A get(int i) { return f.get(i); } }; } static List listFromFunction(IF1 f, int n) { return listFromFunction(n, f); } static A popLast(List l) { return liftLast(l); } static List popLast(int n, List l) { return liftLast(n, l); } static A last(List l) { return empty(l) ? null : l.get(l.size()-1); } static char last(String s) { return empty(s) ? '#' : s.charAt(l(s)-1); } static int last(int[] a) { return l(a) != 0 ? a[l(a)-1] : 0; } static double last(double[] a) { return l(a) != 0 ? a[l(a)-1] : 0; } static A last(A[] a) { return l(a) != 0 ? a[l(a)-1] : null; } static A last(Iterator it) { A a = null; while (it.hasNext()) { ping(); a = it.next(); } return a; } static A last(Collection l) { if (l == null) return null; if (l instanceof List) return (A) last(((List) l)); if (l instanceof SortedSet) return (A) last(((SortedSet) l)); Iterator it = iterator(l); A a = null; while (it.hasNext()) { ping(); a = it.next(); } return a; } static A last(SortedSet l) { return l == null ? null : l.last(); } static A nextToLast(List l) { return get(l, l(l)-2); } static String squareBracket(String s) { return "[" + s + "]"; } static Iterator iterator(Iterable c) { return c == null ? emptyIterator() : c.iterator(); } static boolean isEmpty(Collection c) { return c == null || c.isEmpty(); } static boolean isEmpty(CharSequence s) { return s == null || s.length() == 0; } static boolean isEmpty(Object[] a) { return a == null || a.length == 0; } static boolean isEmpty(byte[] a) { return a == null || a.length == 0; } static boolean isEmpty(Map map) { return map == null || map.isEmpty(); } static boolean intRangesOverlapNempty(IntRange a, IntRange b) { return nempty(intersectIntRanges(a, b)); } static IntRange intRange(int start, int end) { return new IntRange(start, end); } static boolean intRangeIsPartOf(IntRange a, IntRange b) { return a.start >= b.start && a.end <= b.end; } static IntPair intPair(int a, int b) { return new IntPair(a, b); } static RuntimeException quickFail() { throw new QuickException("fail"); } static RuntimeException quickFail(Throwable e) { throw asQuickException(e); } static RuntimeException quickFail(Object msg) { throw new QuickException(String.valueOf(msg)); } static RuntimeException quickFail(String msg) { throw new QuickException(unnull(msg)); } static RuntimeException quickFail(String msg, Throwable innerException) { throw new QuickException(msg, innerException); } static double log(double d) { return Math.log(d); } static String nResults(int n) { return n2(n, "result"); } static String nResults(Collection l) { return nResults(l(l)); } static int lIntArray(int[] a) { return a == null ? 0 : a.length; } static int[] resizeIntArray(int[] a, int n) { if (n == lIntArray(a)) return a; int[] b = new int[n]; arraycopy(a, 0, b, 0, Math.min(lIntArray(a), n)); return b; } static long now_virtualTime; static long now() { return now_virtualTime != 0 ? now_virtualTime : System.currentTimeMillis(); } static int getInt(List c, int i) { return toInt(get(c, i)); } static int getInt(Map map, Object key) { return toInt(mapGet(map, key)); } static int getInt(Object o, String field) { return toInt(getOpt(o, (String) field)); } static int getInt(String field, Object o) { return getInt(o, field); } static long getLong(Object o, String field) { return toLong(getOpt(o, field)); } static long getLong(String field, Object o) { return getLong(o, field); } static Set emptySet() { return new HashSet(); } static Map.Entry firstEntry(Map map) { return empty(map) ? null : first(map.entrySet()); } static UnsupportedOperationException unsupportedOperation() { throw new UnsupportedOperationException(); } static String find(String pattern, String text) { Matcher matcher = Pattern.compile(pattern).matcher(text); if (matcher.find()) return matcher.group(1); return null; } static A find(Collection c, Object... data) { for (A x : c) if (checkFields(x, data)) return x; return null; } static int iround(double d) { return (int) Math.round(d); } static int iround(Number n) { return iround(toDouble(n)); } static Method findMethod_cached(Object o, String method, Object... args) { try { if (o == null) return null; if (o instanceof Class) { _MethodCache cache = callOpt_getCache(((Class) o)); List methods = cache.cache.get(method); if (methods != null) for (Method m : methods) if (isStaticMethod(m) && findMethod_checkArgs(m, args, false)) return m; return null; } else { _MethodCache cache = callOpt_getCache(o.getClass()); List methods = cache.cache.get(method); if (methods != null) for (Method m : methods) if (findMethod_checkArgs(m, args, false)) return m; return null; } } catch (Exception __e) { throw rethrow(__e); } } static boolean endsWith(String a, String b) { return a != null && a.endsWith(b); } static boolean endsWith(String a, char c) { return nempty(a) && lastChar(a) == c; } static boolean endsWith(String a, String b, Matches m) { if (!endsWith(a, b)) return false; m.m = new String[] {dropLast(l(b), a)}; return true; } static void readLocally(String progID, String varNames) { readLocally2(mc(), progID, varNames); } static void readLocally(String varNames) { readLocally2(mc(), programID(), varNames); } static void readLocally2(Object obj, String varNames) { readLocally2(obj, programID(), varNames); } static int readLocally_stringLength; static ThreadLocal readLocally2_allDynamic = new ThreadLocal(); static ThreadLocal readLocally2_classFinder = new ThreadLocal(); // read a string variable from standard storage // does not overwrite variable contents if there is no file static void readLocally2(Object obj, String progID, String varNames) { try { boolean allDynamic = isTrue(getAndClearThreadLocal(readLocally2_allDynamic)); for (String variableName : javaTokC(varNames)) { File textFile = new File(programDir(progID), variableName + ".text"); String value = loadTextFile(textFile); if (value != null) set(main.class, variableName, value); else { File structureFile = new File(programDir(progID), variableName + ".structure"); value = loadTextFile(structureFile); if (value == null) { File structureGZFile = new File(programDir(progID), variableName + ".structure.gz"); if (!structureGZFile.isFile()) return; //value = loadGZTextFile(structureGZFile); InputStream fis = new FileInputStream(structureGZFile); try { GZIPInputStream gis = newGZIPInputStream(fis); InputStreamReader reader = new InputStreamReader(gis, "UTF-8"); BufferedReader bufferedReader = new BufferedReader(reader); //O o = unstructure_reader(bufferedReader); Object o = unstructure_tok(javaTokC_noMLS_onReader(bufferedReader), allDynamic, readLocally2_classFinder.get()); readLocally_set(obj, variableName, o); return; } finally { _close(fis); }} readLocally_stringLength = l(value); if (nempty(value)) readLocally_set(obj, variableName, unstructure(value, allDynamic, readLocally2_classFinder.get())); } } } catch (Exception __e) { throw rethrow(__e); } } static void readLocally_set(Object c, String varName, Object value) { Object oldValue = get(c, varName); if (oldValue instanceof List && !(oldValue instanceof ArrayList) && value != null) { // Assume it's a synchroList. value = synchroList((List) value); } set(c, varName, value); } static FileInputStream newFileInputStream(File path) throws IOException { return newFileInputStream(path.getPath()); } static FileInputStream newFileInputStream(String path) throws IOException { FileInputStream f = new FileInputStream(path); _registerIO(f, path, true); return f; } static int bufferedInputStream_bufferSize = 65536; static BufferedInputStream bufferedInputStream(int bufSize, File f) { try { return bufferedInputStream(bufSize, newFileInputStream(f)); } catch (Exception __e) { throw rethrow(__e); } } static BufferedInputStream bufferedInputStream(File f) { try { return bufferedInputStream(newFileInputStream(f)); } catch (Exception __e) { throw rethrow(__e); } } static BufferedInputStream bufferedInputStream(InputStream in) { return new BufferedInputStream(in, bufferedInputStream_bufferSize); } static BufferedInputStream bufferedInputStream(int bufSize, InputStream in) { return new BufferedInputStream(in, bufSize); } static byte[] bytesFromHex(String s) { return hexToBytes(s); } static boolean byteArraysEqual(byte[] a, byte[] b) { return Arrays.equals(a, b); } static byte[] loadBeginningOfBinaryFile(File file, int maxBytes) { return loadBinaryFilePart(file, 0, maxBytes); } static GZIPInputStream newGZIPInputStream(File f) { return gzInputStream(f); } static GZIPInputStream newGZIPInputStream(InputStream in) { return gzInputStream(in); } static int hexToInt(String s) { return Integer.parseInt(s, 16); } static int ratioToIntPercent(double x, double y) { return roundToInt(x*100/y); } static void swingAndWait(Runnable r) { try { if (isAWTThread()) r.run(); else EventQueue.invokeAndWait(addThreadInfoToRunnable(r)); } catch (Exception __e) { throw rethrow(__e); } } static Object swingAndWait(final Object f) { if (isAWTThread()) return callF(f); else { final Var result = new Var(); swingAndWait(new Runnable() { public void run() { try { result.set(callF(f)); } catch (Exception __e) { throw rethrow(__e); } } public String toString() { return "result.set(callF(f));"; }}); return result.get(); } } static boolean isEditableComboBox(final JComboBox cb) { return cb != null && swing(new F0() { public Boolean get() { try { return cb.isEditable(); } catch (Exception __e) { throw rethrow(__e); } } public String toString() { return "ret cb.isEditable();"; }}); } static String rtrim_fromLines(Collection lines) { StringBuilder buf = new StringBuilder(); if (lines != null) { boolean first = true; for (Object line : lines) { if (first) first = false; else buf.append('\n'); buf.append(str(line)); } } return buf.toString(); } public static String bytesToHex(byte[] bytes) { return bytesToHex(bytes, 0, bytes.length); } public static String bytesToHex(byte[] bytes, int ofs, int len) { StringBuilder stringBuilder = new StringBuilder(len*2); for (int i = 0; i < len; i++) { String s = "0" + Integer.toHexString(bytes[ofs+i]); stringBuilder.append(s.substring(s.length()-2, s.length())); } return stringBuilder.toString(); } static byte[] charToBytes(char c) { return new byte[] { (byte) (c >>> 8), (byte) c }; } static Set allFields(Object o) { TreeSet fields = new TreeSet(); Class _c = _getClass(o); do { for (Field f : _c.getDeclaredFields()) fields.add(f.getName()); _c = _c.getSuperclass(); } while (_c != null); return fields; } static int stdHash(Object a, String... fields) { if (a == null) return 0; int hash = getClassName(a).hashCode(); for (String field : fields) hash = boostHashCombine(hash, hashCode(getOpt(a, field))); return hash; } static String[] toStringArray(Collection c) { String[] a = new String[l(c)]; Iterator it = c.iterator(); for (int i = 0; i < l(a); i++) a[i] = it.next(); return a; } static String[] toStringArray(Object o) { if (o instanceof String[]) return (String[]) o; else if (o instanceof Collection) return toStringArray((Collection) o); else throw fail("Not a collection or array: " + getClassName(o)); } static Object callJavaX(String method, Object... args) { return callOpt(getJavaX(), method, args); } static byte[] longToBytes(long l) { return new byte[] { (byte) (l >>> 56), (byte) (l >>> 48), (byte) (l >>> 40), (byte) (l >>> 32), (byte) (l >>> 24), (byte) (l >>> 16), (byte) (l >>> 8), (byte) l}; } // o is either a map already (string->object) or an arbitrary object, // in which case its fields are converted into a map. static Map objectToMap(Object o) { try { if (o instanceof Map) return (Map) o; TreeMap map = new TreeMap(); Class c = o.getClass(); while (c != Object.class) { Field[] fields = c.getDeclaredFields(); for (final Field field : fields) { if ((field.getModifiers() & Modifier.STATIC) != 0) continue; field.setAccessible(true); final Object value = field.get(o); if (value != null) map.put(field.getName(), value); } c = c.getSuperclass(); } // XXX NEW - hopefully this doesn't break anything if (o instanceof DynamicObject) map.putAll(((DynamicObject) o).fieldValues); return map; } catch (Exception __e) { throw rethrow(__e); } } // same for a collection (convert each element) static List> objectToMap(Iterable l) { if (l == null) return null; List x = new ArrayList(); for (Object o : l) x.add(objectToMap(o)); return x; } static Field setOpt_findField(Class c, String field) { HashMap map; synchronized(getOpt_cache) { map = getOpt_cache.get(c); if (map == null) map = getOpt_makeCache(c); } return map.get(field); } static void setOpt(Object o, String field, Object value) { try { if (o == null) return; Class c = o.getClass(); HashMap map; if (getOpt_cache == null) map = getOpt_makeCache(c); // in class init else synchronized(getOpt_cache) { map = getOpt_cache.get(c); if (map == null) map = getOpt_makeCache(c); } if (map == getOpt_special) { if (o instanceof Class) { setOpt((Class) o, field, value); return; } // It's probably a subclass of Map. Use raw method setOpt_raw(o, field, value); return; } Field f = map.get(field); if (f != null) { smartSet(f, o, value); return; } // possible improvement: skip setAccessible if (o instanceof DynamicObject) setDyn(((DynamicObject) o), field, value); } catch (Exception __e) { throw rethrow(__e); } } static void setOpt(Class c, String field, Object value) { if (c == null) return; try { Field f = setOpt_findStaticField(c, field); // TODO: optimize if (f != null) smartSet(f, null, value); } catch (Exception e) { throw new RuntimeException(e); } } static Field setOpt_findStaticField(Class c, String field) { Class _c = c; do { for (Field f : _c.getDeclaredFields()) if (f.getName().equals(field) && (f.getModifiers() & java.lang.reflect.Modifier.STATIC) != 0) { makeAccessible(f); return f; } _c = _c.getSuperclass(); } while (_c != null); return null; } static String[] asStringArray(Collection c) { return toStringArray(c); } static String[] asStringArray(Object o) { return toStringArray(o); } static String[] dropFirst(int n, String[] a) { return drop(n, a); } static String[] dropFirst(String[] a) { return drop(1, a); } static Object[] dropFirst(Object[] a) { return drop(1, a); } static List dropFirst(List l) { return dropFirst(1, l); } static List dropFirst(int n, Iterable i) { return dropFirst(n, toList(i)); } static List dropFirst(Iterable i) { return dropFirst(toList(i)); } static List dropFirst(int n, List l) { return n <= 0 ? l : new ArrayList(l.subList(Math.min(n, l.size()), l.size())); } static List dropFirst(List l, int n) { return dropFirst(n, l); } static String dropFirst(int n, String s) { return substring(s, n); } static String dropFirst(String s, int n) { return substring(s, n); } static String dropFirst(String s) { return substring(s, 1); } // Use like this: renderVars(+x, +y) static String renderVars_str(Object... params) { List l = new ArrayList(); for (int i = 0; i+1 < l(params); i += 2) l.add(params[i] + "=" + params[i+1]); return trim(joinWithComma(l)); } static A printStructure(String prefix, A o) { if (endsWithLetter(prefix)) prefix += ": "; print(prefix + structureForUser(o)); return o; } static A printStructure(A o) { print(structureForUser(o)); return o; } static byte[] intToBytes(int i) { return new byte[] { (byte) (i >>> 24), (byte) (i >>> 16), (byte) (i >>> 8), (byte) i}; } static void close_pcall(AutoCloseable c) { if (c != null) { try { c.close(); } catch (Throwable __e) { _handleException(__e); }} } static void preCleanUp(Object c) { if (c instanceof Collection) { for (Object o : ((Collection) c)) preCleanUp(o); return; } callOpt(c, "licensed_off"); setOpt(c, "ping_anyActions" , true); // so ping notices setOpt(c, "cleaningUp_flag" , true); } static void innerCleanUp(Object c) { // call custom cleanMeUp() and cleanMeUp_*() functions if (!isFalse(pcallOpt(c, "cleanMeUp"))) for (String name : sorted(methodsStartingWith(c, "cleanMeUp_"))) try { callOpt(c, name); } catch (Throwable e) { print("Error cleaning up: " + programID(c)); _handleException(e); } } static void innerCleanUp() { innerCleanUp(mc()); } static Object pcallOpt(Object o, String method, Object... args) { try { return callOpt(o, method, args); } catch (Throwable __e) { _handleException(__e); } return null; } static List registeredThreads(Object o) { Map map = (Map) (getOpt(o, "_registerThread_threads")); if (map == null) return ll(); map.size(); // force clean-up synchronized(map) { return asList(keys(map)); } } static List registeredThreads() { _registerThread_threads.size(); // force clean-up return asList(keys(_registerThread_threads)); } static List allToString(Iterable c) { List l = new ArrayList(); for (Object o : unnull(c)) l.add(str(o)); return l; } static List allToString(Object[] c) { List l = new ArrayList(); for (Object o : unnull(c)) l.add(str(o)); return l; } static void interruptThreads(Collection threads) { for (Thread t : unnull(threads)) interruptThread(t); } static void interruptThreads(Class mainClass) { interruptThreads(registeredThreads(mainClass)); } static void retireClassLoader(ClassLoader cl) { if (isJavaXClassLoader(cl)) setOptAll(cl, "retired" , true, "retiredMarker" , new DefunctClassLoader()); } static char firstChar(String s) { return s.charAt(0); } static List reversedList(Collection l) { List x = cloneList(l); Collections.reverse(x); return x; } static String reversedString(String s) { return reverseString(s); } static int toInt_checked(long l) { if (l != (int) l) throw fail("Too large for int: " + l); return (int) l; } static String shortenClassName(String name) { if (name == null) return null; int i = lastIndexOf(name, "$"); if (i < 0) i = lastIndexOf(name, "."); return i < 0 ? name : substring(name, i+1); } static char[] asChars(String s) { return s == null ? null : s.toCharArray(); } static void smartSet(Field f, Object o, Object value) throws Exception { try { f.set(o, value); } catch (Exception e) { Class type = f.getType(); // take care of common case (long to int) if (type == int.class && value instanceof Long) value = ((Long) value).intValue(); if (type == LinkedHashMap.class && value instanceof Map) { f.set(o, asLinkedHashMap((Map) value)); return; } throw e; } } static Class hotwire(String src) { assertFalse(_inCore()); Class j = getJavaX(); if (isAndroid()) { synchronized(j) { // hopefully this goes well... List libraries = new ArrayList(); File srcDir = (File) call(j, "transpileMain", src, libraries); if (srcDir == null) throw fail("transpileMain returned null (src=" + quote(src) + ")"); Object androidContext = get(j, "androidContext"); return (Class) call(j, "loadx2android", srcDir, src); } } else { Class c = (Class) (call(j, "hotwire", src)); hotwire_copyOver(c); return c; } } static A callMain(A c, String... args) { callOpt(c, "main", new Object[] {args}); return c; } static void callMain() { callMain(mc()); } static void addToContainer(Container a, Component... b) { if (a == null) return; { swing(new Runnable() { public void run() { try { for (Component c : unnull(b)) if (c != null) a.add(c); } catch (Exception __e) { throw rethrow(__e); } } public String toString() { return "for (Component c : unnull(b))\r\n if (c != null) \r\n a.add(c);"; }}); } } static A printWithTime(A a) { return printWithTime("", a); } static A printWithTime(String s, A a) { print(hmsWithColons() + ": " + s, a); return a; } static int coresToUse_fixed() { return max(1, coresToUse()); } static Iterable iterable(final Iterator i) { return new Iterable() { public Iterator iterator() { return i; } }; } // not actually Timsort, but works well // This code has been contributed by 29AjayKumar // from: https://www.geeksforgeeks.org/timsort/ static final int timSortIntArrayWithComparator_RUN = 32; // this function sorts array from left index to // to right index which is of size atmost RUN static void timSortIntArrayWithComparator_insertionSort(int[] arr, IntComparator comparator, int left, int right) { for (int i = left + 1; i <= right; i++) { int temp = arr[i]; int j = i - 1; while (j >= left && comparator.compare(arr[j], temp) > 0) { arr[j + 1] = arr[j]; j--; } arr[j + 1] = temp; } } // merge function merges the sorted runs static void timSortIntArrayWithComparator_merge(int[] arr, IntComparator comparator, int l, int m, int r) { // original array is broken in two parts // left and right array int len1 = m - l + 1, len2 = r - m; int[] left = new int[len1]; int[] right = new int[len2]; for (int x = 0; x < len1; x++) { left[x] = arr[l + x]; } for (int x = 0; x < len2; x++) { right[x] = arr[m + 1 + x]; } int i = 0; int j = 0; int k = l; // after comparing, we merge those two array // in larger sub array while (i < len1 && j < len2) { if (comparator.compare(left[i], right[j]) <= 0) { arr[k] = left[i]; i++; } else { arr[k] = right[j]; j++; } k++; } // copy remaining elements of left, if any while (i < len1) { arr[k] = left[i]; k++; i++; } // copy remaining element of right, if any while (j < len2) { arr[k] = right[j]; k++; j++; } } // iterative Timsort function to sort the // array[0...n-1] (similar to merge sort) static void timSortIntArrayWithComparator(int[] arr, IntComparator comparator) { timSortIntArrayWithComparator(arr, lIntArray(arr), comparator); } static void timSortIntArrayWithComparator(int[] arr, int n, IntComparator comparator) { // Sort individual subarrays of size RUN for (int i = 0; i < n; i += timSortIntArrayWithComparator_RUN) { timSortIntArrayWithComparator_insertionSort(arr, comparator, i, Math.min((i + 31), (n - 1))); } // start merging from size RUN (or 32). It will merge // to form size 64, then 128, 256 and so on .... for (int size = timSortIntArrayWithComparator_RUN; size < n; size = 2 * size) { // pick starting point of left sub array. We // are going to merge arr[left..left+size-1] // and arr[left+size, left+2*size-1] // After every merge, we increase left by 2*size for (int left = 0; left < n; left += 2 * size) { // find ending point of left sub array // mid+1 is starting point of right sub array int mid = Math.min(left + size - 1, n - 1); int right = Math.min(left + 2 * size - 1, n - 1); // merge sub array arr[left.....mid] & // arr[mid+1....right] timSortIntArrayWithComparator_merge(arr, comparator, left, mid, right); } } } static class mapI_if1_It extends IterableIterator { IF1 f; Iterator i; mapI_if1_It() {} mapI_if1_It(IF1 f, Iterator i) { this.i = i; this.f = f;} public boolean hasNext() { return i.hasNext(); } public B next() { return f.get(i.next()); } public String toString() { return formatFunctionCall("mapI_if1", f, i); } } static IterableIterator mapI_if1(IF1 f, Iterable i) { return new mapI_if1_It(f, i.iterator()); } static IterableIterator mapI_if1(Iterable i, IF1 f) { return mapI_if1(f, i); } static char charAt(String s, int i) { return s != null && i >= 0 && i < s.length() ? s.charAt(i) : '\0'; } static List stringAsCharacterList(final String s) { if (s == null) return null; return new RandomAccessAbstractList() { final int l = l(s); public int size() { return l; } public Character get(int i) { return s.charAt(i); } }; } // returns l(s) if not found static int smartIndexOf(String s, String sub, int i) { if (s == null) return 0; i = s.indexOf(sub, min(i, l(s))); return i >= 0 ? i : l(s); } static int smartIndexOf(String s, int i, char c) { return smartIndexOf(s, c, i); } static int smartIndexOf(String s, char c, int i) { if (s == null) return 0; i = s.indexOf(c, min(i, l(s))); return i >= 0 ? i : l(s); } static int smartIndexOf(String s, String sub) { return smartIndexOf(s, sub, 0); } static int smartIndexOf(String s, char c) { return smartIndexOf(s, c, 0); } static int smartIndexOf(List l, A sub) { return smartIndexOf(l, sub, 0); } static int smartIndexOf(List l, int start, A sub) { return smartIndexOf(l, sub, start); } static int smartIndexOf(List l, A sub, int start) { int i = indexOf(l, sub, start); return i < 0 ? l(l) : i; } static int countLines(String s) { return l(toLines(s)); // yeah could be optimized :-) } static List uniquifyList(Collection l) { if (l == null) return null; if (l(l) < 2) return asList(l); HashSet set = new HashSet(); List out = new ArrayList(); for (A a : l) if (set.add(a)) out.add(a); return out; } static List sortedInPlace(List l, final Object comparator) { sort(l, makeComparator(comparator)); return l; } static List sortedInPlace(List l) { sort(l); return l; } static List collectField(Iterable c, String field) { List l = new ArrayList(); if (c != null) for (Object a : c) l.add(getOpt(a, field)); return l; } static List collectField(String field, Iterable c) { return collectField(c, field); } static Map listIndexCI(List l) { Map map = ciMap(); for (int i = 0; i < l(l); i++) map.put(l.get(i), i); return map; } static Map similarEmptyMap(Map m) { if (m instanceof TreeMap) return new TreeMap(((TreeMap) m).comparator()); if (m instanceof LinkedHashMap) return new LinkedHashMap(); // default to a hash map return new HashMap(); } static int[] toIntArray(Collection l) { int[] a = new int[l(l)]; int i = 0; if (a.length != 0) for (int x : l) a[i++] = x; return a; } static List beginCriticalAction_inFlight = synchroList(); static class CriticalAction { String description; CriticalAction() {} CriticalAction(String description) { this.description = description;} void done() { beginCriticalAction_inFlight.remove(this); } } static CriticalAction beginCriticalAction(String description) { ping(); CriticalAction c = new CriticalAction(description); beginCriticalAction_inFlight.add(c); return c; } static void cleanMeUp_beginCriticalAction() { int n = 0; while (nempty(beginCriticalAction_inFlight)) { int m = l(beginCriticalAction_inFlight); if (m != n) { n = m; try { print("Waiting for " + n2(n, "critical actions") + ": " + join(", ", collect(beginCriticalAction_inFlight, "description"))); } catch (Throwable __e) { _handleException(__e); } } sleepInCleanUp(10); } } static File copyFile(File src, File dest) { try { FileInputStream inputStream = new FileInputStream(src.getPath()); FileOutputStream outputStream = newFileOutputStream(dest.getPath()); try { copyStream(inputStream, outputStream); inputStream.close(); } finally { outputStream.close(); } return dest; } catch (Exception __e) { throw rethrow(__e); } } static BufferedReader utf8BufferedReader(InputStream in) { return utf8bufferedReader(in); } static BufferedReader utf8BufferedReader(File f) { return utf8bufferedReader(f); } static ArrayList cloneListSynchronizingOn(Collection l, Object mutex) { if (l == null) return new ArrayList(); synchronized(mutex) { return new ArrayList(l); } } static Map mapKeyAndFunction(Iterable l, Object f) { return mapKeyAndFunction(f, l); } static Map mapKeyAndFunction(Object f, Iterable l) { HashMap map = new HashMap(); if (l != null) for (Object o : l) map.put(o, callF(f, o)); return map; } static Map mapKeyAndFunction(Map map, IF2 f) { HashMap map2 = new HashMap(); if (map != null) for (Map.Entry __0 : _entrySet( map)) { A key = __0.getKey(); B value = __0.getValue(); map2.put(key, callF(f, key, value)); } return map2; } static Map mapKeyAndFunction(Iterable l, IF1 f) { return mapKeyAndFunction(f, l); } static void listSet(List l, int i, A a, A emptyElement) { if (i < 0) return; while (i >= l(l)) l.add(emptyElement); l.set(i, a); } static void listSet(List l, int i, A a) { listSet(l, i, a, null); } static void assertEqic(String x, String y) { assertEqic(null, x, y); } static void assertEqic(String msg, String x, String y) { if (!eqic(x, y)) throw fail((msg != null ? msg + ": " : "") + quote(x) + " != " + quote(y) + " [ic]"); } // usually L static String fromLines(Iterable lines) { StringBuilder buf = new StringBuilder(); if (lines != null) for (Object line : lines) buf.append(str(line)).append('\n'); return buf.toString(); } static String fromLines(String... lines) { return fromLines(asList(lines)); } static IterableIterator toLines(File f) { return linesFromFile(f); } static List toLines(String s) { List lines = new ArrayList(); if (s == null) return lines; int start = 0; while (true) { int i = toLines_nextLineBreak(s, start); if (i < 0) { if (s.length() > start) lines.add(s.substring(start)); break; } lines.add(s.substring(start, i)); if (s.charAt(i) == '\r' && i+1 < s.length() && s.charAt(i+1) == '\n') i += 2; else ++i; start = i; } return lines; } static int toLines_nextLineBreak(String s, int start) { int n = s.length(); for (int i = start; i < n; i++) { char c = s.charAt(i); if (c == '\r' || c == '\n') return i; } return -1; } static MultiSetMap multiSetMap_innerTreeSet() { return multiSetMap_innerTreeSet(null); } static MultiSetMap multiSetMap_innerTreeSet(Comparator innerComparator) { return new MultiSetMap() { Set _makeEmptySet() { return new TreeSet(innerComparator); } }; } static TreeMap revTreeMap() { return new TreeMap(reverseComparator()); } static TreeMap descTreeMap() { return revTreeMap(); } static List reverseList(List l) { Collections.reverse(l); return l; } static String htag(String tag) { return htag(tag, ""); } static String htag(String tag, Object contents, Object... params) { String openingTag = hopeningTag(tag, params); String s = str(contents); if (empty(s) && neqic(tag, "script")) return dropLast(openingTag) + "/>"; return openingTag + s + ""; } static String tag(String tag) { return htag(tag); } static String tag(String tag, Object contents, Object... params) { return htag(tag, str(contents), params); } static String tag(String tag, StringBuilder contents, Object... params) { return htag(tag, contents, params); } static String tag(String tag, StringBuffer contents, Object... params) { return htag(tag, contents, params); } static String stringIfTrue(boolean b, String s) { return b ? s : ""; } static List callF_all(Collection l, Object... args) { return map(l, f -> callF(f, args)); } static List cloneSubList(List l, int startIndex, int endIndex) { return newSubList(l, startIndex, endIndex); } static List cloneSubList(List l, int startIndex) { return newSubList(l, startIndex); } static void removeSubList(List l, int from, int to) { if (l != null) subList(l, from, to).clear(); } static void removeSubList(List l, int from) { if (l != null) subList(l, from).clear(); } static String defaultThreadName_name; static String defaultThreadName() { if (defaultThreadName_name == null) defaultThreadName_name = "A thread by " + programID(); return defaultThreadName_name; } static Runnable wrapAsActivity(Object r) { return toRunnable(r); } // runnable = Runnable or String (method name) static Thread newThread(Object runnable) { return new Thread(_topLevelErrorHandling(toRunnable(runnable))); } static Thread newThread(Object runnable, String name) { if (name == null) name = defaultThreadName(); return new Thread(_topLevelErrorHandling(toRunnable(runnable)), name); } static Thread newThread(String name, Object runnable) { return newThread(runnable, name); } static Map _registerThread_threads; static Object _onRegisterThread; // voidfunc(Thread) static Thread _registerThread(Thread t) { if (_registerThread_threads == null) _registerThread_threads = newWeakHashMap(); _registerThread_threads.put(t, true); vm_generalWeakSubMap("thread2mc").put(t, weakRef(mc())); callF(_onRegisterThread, t); return t; } static void _registerThread() { _registerThread(Thread.currentThread()); } static Object sleepQuietly_monitor = new Object(); static void sleepQuietly() { try { assertFalse(isAWTThread()); synchronized(sleepQuietly_monitor) { sleepQuietly_monitor.wait(); } } catch (Exception __e) { throw rethrow(__e); } } static void cancelThread(Thread t) { if (t == null) return; ping(); /*O mc = getWeakRef((WeakReference) vm_generalWeakSubMap("thread2mc").get(t)); ifdef cancelThread_verbose print("cancelThread: mc=" + mc); endifdef if (mc != null) { Map ping_actions = cast get(mc, 'ping_actions); synchronized(ping_actions) { ping_actions.put(t, "cancelled"); set(mc, ping_anyActions := true); } } else*/ synchronized(ping_actions) { ping_actions.put(t, "cancelled"); ping_anyActions = true; } } static A liftLast(List l) { if (empty(l)) return null; int i = l(l)-1; A a = l.get(i); l.remove(i); return a; } static List liftLast(int n, List l) { int i = l(l)-n; List part = cloneSubList(l, i); removeSubList(l, i); return part; } static Iterator emptyIterator() { return Collections.emptyIterator(); } static IntRange intersectIntRanges(IntRange a, IntRange b) { int start = max(a.start, b.start); int end = min(a.end, b.end); return start <= end ? new IntRange(start, end) : null; } static RuntimeException asQuickException(Throwable t) { return t instanceof RuntimeException ? (RuntimeException) t : new QuickException(t); } static long toLong(Object o) { if (o instanceof Number) return ((Number) o).longValue(); if (o instanceof String) return parseLong((String) o); return 0; } static boolean checkFields(Object x, Object... data) { for (int i = 0; i < l(data); i += 2) if (neq(getOpt(x, (String) data[i]), data[i+1])) return false; return true; } static double toDouble(Object o) { if (o instanceof Number) return ((Number) o).doubleValue(); if (o instanceof BigInteger) return ((BigInteger) o).doubleValue(); if (o instanceof String) return parseDouble(((String) o)); if (o == null) return 0.0; throw fail(o); } static char lastChar(String s) { return empty(s) ? '\0' : s.charAt(l(s)-1); } static A[] dropLast(A[] a) { return dropLast(a, 1); } static A[] dropLast(A[] a, int n) { if (a == null) return null; n = Math.min(n, a.length); A[] b = arrayOfSameType(a, a.length-n); System.arraycopy(a, 0, b, 0, b.length); return b; } static List dropLast(List l) { return subList(l, 0, l(l)-1); } static List dropLast(int n, List l) { return subList(l, 0, l(l)-n); } static List dropLast(Iterable l) { return dropLast(asList(l)); } static String dropLast(String s) { return substring(s, 0, l(s)-1); } static String dropLast(String s, int n) { return substring(s, 0, l(s)-n); } static String dropLast(int n, String s) { return dropLast(s, n); } static String programID() { return getProgramID(); } static String programID(Object o) { return getProgramID(o); } static A getAndClearThreadLocal(ThreadLocal tl) { A a = tl.get(); tl.set(null); return a; } static List javaTokC(String s) { if (s == null) return null; int l = s.length(); ArrayList tok = new ArrayList(); int i = 0; while (i < l) { int j = i; char c, d; // scan for whitespace while (j < l) { c = s.charAt(j); d = j+1 >= l ? '\0' : s.charAt(j+1); if (c == ' ' || c == '\t' || c == '\r' || c == '\n') ++j; else if (c == '/' && d == '*') { do ++j; while (j < l && !s.substring(j, Math.min(j+2, l)).equals("*/")); j = Math.min(j+2, l); } else if (c == '/' && d == '/') { do ++j; while (j < l && "\r\n".indexOf(s.charAt(j)) < 0); } else break; } i = j; if (i >= l) break; c = s.charAt(i); d = i+1 >= l ? '\0' : s.charAt(i+1); // scan for non-whitespace if (c == '\'' || c == '"') { char opener = c; ++j; while (j < l) { if (s.charAt(j) == opener || s.charAt(j) == '\n') { // end at \n to not propagate unclosed string literal errors ++j; break; } else if (s.charAt(j) == '\\' && j+1 < l) j += 2; else ++j; } } else if (Character.isJavaIdentifierStart(c)) do ++j; while (j < l && (Character.isJavaIdentifierPart(s.charAt(j)) || "'".indexOf(s.charAt(j)) >= 0)); // for stuff like "don't" else if (Character.isDigit(c)) { do ++j; while (j < l && Character.isDigit(s.charAt(j))); if (j < l && s.charAt(j) == 'L') ++j; // Long constants like 1L } else if (c == '[' && d == '[') { do ++j; while (j+1 < l && !s.substring(j, j+2).equals("]]")); j = Math.min(j+2, l); } else if (c == '[' && d == '=' && i+2 < l && s.charAt(i+2) == '[') { do ++j; while (j+2 < l && !s.substring(j, j+3).equals("]=]")); j = Math.min(j+3, l); } else ++j; tok.add(javaTok_substringC(s, i, j)); i = j; } return tok; } static File programDir_mine; // set this to relocate program's data static File programDir() { return programDir(getProgramID()); } static File programDir(String snippetID) { boolean me = sameSnippetID(snippetID, programID()); if (programDir_mine != null && me) return programDir_mine; File dir = new File(javaxDataDir(), formatSnippetIDOpt(snippetID)); if (me) { String c = caseID(); if (nempty(c)) dir = newFile(dir, c); } return dir; } static File programDir(String snippetID, String subPath) { return new File(programDir(snippetID), subPath); } static String loadTextFile(String fileName) { return loadTextFile(fileName, null); } static String loadTextFile(File f, String defaultContents) { return loadTextFile(f, defaultContents, "UTF-8"); } static String loadTextFile(File f, String defaultContents, String encoding) { try { checkFileNotTooBigToRead(f); if (f == null || !f.exists()) return defaultContents; FileInputStream fileInputStream = new FileInputStream(f); InputStreamReader inputStreamReader = new InputStreamReader(fileInputStream, encoding); return loadTextFile(inputStreamReader); } catch (Exception __e) { throw rethrow(__e); } } public static String loadTextFile(File fileName) { return loadTextFile(fileName, null); } static String loadTextFile(String fileName, String defaultContents) { return fileName == null ? defaultContents : loadTextFile(newFile(fileName), defaultContents); } static String loadTextFile(Reader reader) throws IOException { StringBuilder builder = new StringBuilder(); try { char[] buffer = new char[1024]; int n; while (-1 != (n = reader.read(buffer))) builder.append(buffer, 0, n); } finally { reader.close(); } return str(builder); } static Object unstructure(String text) { return unstructure(text, false); } static Object unstructure(String text, final boolean allDynamic) { return unstructure(text, allDynamic, null); } static int structure_internStringsLongerThan = 50; static int unstructure_unquoteBufSize = 100; static int unstructure_tokrefs; // stats abstract static class unstructure_Receiver { abstract void set(Object o); } // classFinder: func(name) -> class (optional) static Object unstructure(String text, boolean allDynamic, Object classFinder) { if (text == null) return null; return unstructure_tok(javaTokC_noMLS_iterator(text), allDynamic, classFinder); } static Object unstructure_reader(BufferedReader reader) { return unstructure_tok(javaTokC_noMLS_onReader(reader), false, null); } static Object unstructure_tok(final Producer tok, final boolean allDynamic, final Object _classFinder) { final boolean debug = unstructure_debug; final class X { int i = -1; final Object classFinder = _classFinder != null ? _classFinder : _defaultClassFinder(); HashMap refs = new HashMap(); HashMap tokrefs = new HashMap(); HashSet concepts = new HashSet(); HashMap classesMap = new HashMap(); List stack = new ArrayList(); String curT; char[] unquoteBuf = new char[unstructure_unquoteBufSize]; Class findAClass(String fullClassName) { return classFinder != null ? (Class) callF(classFinder, fullClassName) : findClass_fullName(fullClassName); } String unquote(String s) { return unquoteUsingCharArray(s, unquoteBuf); } // look at current token String t() { return curT; } // get current token, move to next String tpp() { String t = curT; consume(); return t; } void parse(final unstructure_Receiver out) { String t = t(); int refID = 0; if (structure_isMarker(t, 0, l(t))) { refID = parseInt(t.substring(1)); consume(); } final int _refID = refID; // if (debug) print("parse: " + quote(t)); final int tokIndex = i; parse_inner(refID, tokIndex, new unstructure_Receiver() { void set(Object o) { if (_refID != 0) refs.put(_refID, o); if (o != null) tokrefs.put(tokIndex, o); out.set(o); } }); } void parse_inner(int refID, int tokIndex, final unstructure_Receiver out) { String t = t(); // if (debug) print("parse_inner: " + quote(t)); Class c = classesMap.get(t); if (c == null) { if (t.startsWith("\"")) { String s = internIfLongerThan(unquote(tpp()), structure_internStringsLongerThan); out.set(s); return; } if (t.startsWith("'")) { out.set(unquoteCharacter(tpp())); return; } if (t.equals("bigint")) { out.set(parseBigInt()); return; } if (t.equals("d")) { out.set(parseDouble()); return; } if (t.equals("fl")) { out.set(parseFloat()); return; } if (t.equals("sh")) { consume(); t = tpp(); if (t.equals("-")) { t = tpp(); out.set((short) (-parseInt(t))); return; } out.set((short) parseInt(t)); return; } if (t.equals("-")) { consume(); t = tpp(); out.set(isLongConstant(t) ? (Object) (-parseLong(t)) : (Object) (-parseInt(t))); return; } if (isInteger(t) || isLongConstant(t)) { consume(); //if (debug) print("isLongConstant " + quote(t) + " => " + isLongConstant(t)); if (isLongConstant(t)) { out.set(parseLong(t)); return; } long l = parseLong(t); boolean isInt = l == (int) l; out.set(isInt ? (Object) Integer.valueOf((int) l) : (Object) Long.valueOf(l)); return; } if (t.equals("false") || t.equals("f")) { consume(); out.set(false); return; } if (t.equals("true") || t.equals("t")) { consume(); out.set(true); return; } if (t.equals("-")) { consume(); t = tpp(); out.set(isLongConstant(t) ? (Object) (-parseLong(t)) : (Object) (-parseInt(t))); return; } if (isInteger(t) || isLongConstant(t)) { consume(); //if (debug) print("isLongConstant " + quote(t) + " => " + isLongConstant(t)); if (isLongConstant(t)) { out.set(parseLong(t)); return; } long l = parseLong(t); boolean isInt = l == (int) l; out.set(isInt ? (Object) Integer.valueOf((int) l) : (Object) Long.valueOf(l)); return; } if (t.equals("File")) { consume(); File f = new File(unquote(tpp())); out.set(f); return; } if (t.startsWith("r") && isInteger(t.substring(1))) { consume(); int ref = Integer.parseInt(t.substring(1)); Object o = refs.get(ref); if (o == null) throw fail("unsatisfied back reference " + ref); out.set(o); return; } if (t.startsWith("t") && isInteger(t.substring(1))) { consume(); int ref = Integer.parseInt(t.substring(1)); Object o = tokrefs.get(ref); if (o == null) throw fail("unsatisfied token reference " + ref + " at " + tokIndex); out.set(o); return; } if (t.equals("hashset")) { parseHashSet(out); return; } if (t.equals("lhs")) { parseLinkedHashSet(out); return; } if (t.equals("treeset")) { parseTreeSet(out); return; } if (t.equals("ciset")) { parseCISet(out); return; } if (eqOneOf(t, "hashmap", "hm")) { consume(); parseMap(new HashMap(), out); return; } if (t.equals("lhm")) { consume(); parseMap(new LinkedHashMap(), out); return; } if (t.equals("tm")) { consume(); parseMap(new TreeMap(), out); return; } if (t.equals("cimap")) { consume(); parseMap(ciMap(), out); return; } if (t.equals("ll")) { consume(); { parseList(new LinkedList(), out); return; } } if (t.equals("syncLL")) { // legacy consume(); { parseList(synchroLinkedList(), out); return; } } if (t.equals("sync")) { consume(); { parse(new unstructure_Receiver() { void set(Object value) { if (value instanceof Map) { // Java 7 if (value instanceof NavigableMap) { out.set(Collections.synchronizedNavigableMap((NavigableMap) value)); return; } if (value instanceof SortedMap) { out.set(Collections.synchronizedSortedMap((SortedMap) value)); return; } { out.set(Collections.synchronizedMap((Map) value)); return; } } else { out.set(Collections.synchronizedList((List) value)); return; } } }); return; } } if (t.equals("{")) { parseMap(out); return; } if (t.equals("[")) { this.parseList(new ArrayList(), out); return; } if (t.equals("bitset")) { parseBitSet(out); return; } if (t.equals("array") || t.equals("intarray") || t.equals("dblarray")) { parseArray(out); return; } if (t.equals("ba")) { consume(); String hex = unquote(tpp()); out.set(hexToBytes(hex)); return; } if (t.equals("boolarray")) { consume(); int n = parseInt(tpp()); String hex = unquote(tpp()); out.set(boolArrayFromBytes(hexToBytes(hex), n)); return; } if (t.equals("class")) { out.set(parseClass()); return; } if (t.equals("l")) { parseLisp(out); return; } if (t.equals("null")) { consume(); out.set(null); return; } if (eq(t, "c")) { consume(); t = t(); assertTrue(isJavaIdentifier(t)); concepts.add(t); } // custom deserialization (new static method method) if (eq(t, "cu")) { consume(); t = tpp(); assertTrue(isJavaIdentifier(t)); String fullClassName = "main$" + t; Class _c = findAClass(fullClassName); if (_c == null) throw fail("Class not found: " + fullClassName); parse(new unstructure_Receiver() { void set(Object value) { out.set(call(_c, "_deserialize", value)); } }); return; } } if (eq(t, "j")) { consume("j"); out.set(parseJava()); return; } if (c == null && !isJavaIdentifier(t)) throw new RuntimeException("Unknown token " + (i+1) + ": " + quote(t)); // any other class name (or package name) consume(); String className, fullClassName; // Is it a package name? if (eq(t(), ".")) { consume(); className = fullClassName = t + "." + assertIdentifier(tpp()); } else { className = t; fullClassName = "main$" + t; } if (c == null) { // First, find class if (allDynamic) c = null; else c = findAClass(fullClassName); if (c != null) classesMap.put(className, c); } // Check if it has an outer reference boolean hasBracket = eq(t(), "("); if (hasBracket) consume(); boolean hasOuter = hasBracket && eq(t(), "this$1"); DynamicObject dO = null; Object o = null; final String thingName = t; if (c != null) { o = hasOuter ? nuStubInnerObject(c, classFinder) : nuEmptyObject(c); if (o instanceof DynamicObject) dO = (DynamicObject) o; } else { if (concepts.contains(t) && (c = findAClass("main$Concept")) != null) o = dO = (DynamicObject) nuEmptyObject(c); else dO = new DynamicObject(); dO.className = className; } // Save in references list early because contents of object // might link back to main object if (refID != 0) refs.put(refID, o != null ? o : dO); tokrefs.put(tokIndex, o != null ? o : dO); // NOW parse the fields! final LinkedHashMap fields = new LinkedHashMap(); // preserve order final Object _o = o; final DynamicObject _dO = dO; if (hasBracket) { stack.add(new Runnable() { public void run() { try { if (eq(t(), ",")) consume(); if (eq(t(), ")")) { consume(")"); objRead(_o, _dO, fields, hasOuter); out.set(_o != null ? _o : _dO); } else { final String key = unquote(tpp()); String t = tpp(); if (!eq(t, "=")) throw fail("= expected, got " + t + " after " + quote(key) + " in object " + thingName /*+ " " + sfu(fields)*/); stack.add(this); parse(new unstructure_Receiver() { void set(Object value) { fields.put(key, value); /*ifdef unstructure_debug print("Got field value " + value + ", next token: " + t()); endifdef*/ //if (eq(t(), ",")) consume(); } }); } } catch (Exception __e) { throw rethrow(__e); } } public String toString() { return "ifdef unstructure_debug\r\n print(\"in object values, token: \" + t())..."; }}); } else { objRead(o, dO, fields, hasOuter); out.set(o != null ? o : dO); } } void objRead(Object o, DynamicObject dO, Map fields, boolean hasOuter) { if (o != null) { if (dO != null) { setOptAllDyn_pcall(dO, fields); } else { setOptAll_pcall(o, fields); } if (hasOuter) fixOuterRefs(o); } else for (Map.Entry e : fields.entrySet()) setDynObjectValue(dO, intern(e.getKey()), e.getValue()); if (o != null) pcallOpt_noArgs(o, "_doneLoading"); } void parseSet(final Set set, final unstructure_Receiver out) { this.parseList(new ArrayList(), new unstructure_Receiver() { void set(Object o) { set.addAll((List) o); out.set(set); } }); } void parseLisp(final unstructure_Receiver out) { throw fail("class Lisp not included"); } void parseBitSet(final unstructure_Receiver out) { consume("bitset"); consume("{"); final BitSet bs = new BitSet(); stack.add(new Runnable() { public void run() { try { if (eq(t(), "}")) { consume("}"); out.set(bs); } else { stack.add(this); parse(new unstructure_Receiver() { void set(Object o) { bs.set((Integer) o); if (eq(t(), ",")) consume(); } }); } } catch (Exception __e) { throw rethrow(__e); } } public String toString() { return "if (eq(t(), \"}\")) {\r\n consume(\"}\");\r\n out.set(bs);\r\n ..."; }}); } void parseList(final List list, final unstructure_Receiver out) { tokrefs.put(i, list); consume("["); stack.add(new Runnable() { public void run() { try { if (eq(t(), "]")) { consume(); out.set(list); } else { stack.add(this); parse(new unstructure_Receiver() { void set(Object o) { //if (debug) print("List element type: " + getClassName(o)); list.add(o); if (eq(t(), ",")) consume(); } }); } } catch (Exception __e) { throw rethrow(__e); } } public String toString() { return "if (eq(t(), \"]\")) {\r\n consume();\r\n ifdef unstructure_debug\r..."; }}); } void parseArray(final unstructure_Receiver out) { final String type = tpp(); consume("{"); final List list = new ArrayList(); stack.add(new Runnable() { public void run() { try { if (eq(t(), "}")) { consume("}"); out.set( type.equals("intarray") ? toIntArray(list) : type.equals("dblarray") ? toDoubleArray(list) : list.toArray()); } else { stack.add(this); parse(new unstructure_Receiver() { void set(Object o) { list.add(o); if (eq(t(), ",")) consume(); } }); } } catch (Exception __e) { throw rethrow(__e); } } public String toString() { return "if (eq(t(), \"}\")) {\r\n consume(\"}\");\r\n out.set(\r\n ..."; }}); } Object parseClass() { consume("class"); consume("("); String name = unquote(tpp()); consume(")"); Class c = allDynamic ? null : findAClass(name); if (c != null) return c; DynamicObject dO = new DynamicObject(); dO.className = "java.lang.Class"; name = dropPrefix("main$", name); dO.fieldValues.put("name", name); return dO; } Object parseBigInt() { consume("bigint"); consume("("); String val = tpp(); if (eq(val, "-")) val = "-" + tpp(); consume(")"); return new BigInteger(val); } Object parseDouble() { consume("d"); consume("("); String val = unquote(tpp()); consume(")"); return Double.parseDouble(val); } Object parseFloat() { consume("fl"); String val; if (eq(t(), "(")) { consume("("); val = unquote(tpp()); consume(")"); } else { val = unquote(tpp()); } return Float.parseFloat(val); } void parseHashSet(unstructure_Receiver out) { consume("hashset"); parseSet(new HashSet(), out); } void parseLinkedHashSet(unstructure_Receiver out) { consume("lhs"); parseSet(new LinkedHashSet(), out); } void parseTreeSet(unstructure_Receiver out) { consume("treeset"); parseSet(new TreeSet(), out); } void parseCISet(unstructure_Receiver out) { consume("ciset"); parseSet(ciSet(), out); } void parseMap(unstructure_Receiver out) { parseMap(new TreeMap(), out); } Object parseJava() { String j = unquote(tpp()); Matches m = new Matches(); if (jmatch("java.awt.Color[r=*,g=*,b=*]", j, m)) return nuObject("java.awt.Color", parseInt(m.unq(0)), parseInt(m.unq(1)), parseInt(m.unq(2))); else { warn("Unknown Java object: " + j); return null; } } void parseMap(final Map map, final unstructure_Receiver out) { consume("{"); stack.add(new Runnable() { boolean v = false; Object key; public void run() { if (v) { v = false; stack.add(this); if (!eq(tpp(), "=")) throw fail("= expected, got " + t() + " in map of size " + l(map)); parse(new unstructure_Receiver() { void set(Object value) { map.put(key, value); if (eq(t(), ",")) consume(); } }); } else { if (eq(t(), "}")) { consume("}"); out.set(map); } else { v = true; stack.add(this); parse(new unstructure_Receiver() { void set(Object o) { key = o; } }); } } // if v else } // run() }); } /*void parseSub(unstructure_Receiver out) { int n = l(stack); parse(out); while (l(stack) > n) stack }*/ void consume() { curT = tok.next(); ++i; } void consume(String s) { if (!eq(t(), s)) { /*S prevToken = i-1 >= 0 ? tok.get(i-1) : ""; S nextTokens = join(tok.subList(i, Math.min(i+2, tok.size()))); fail(quote(s) + " expected: " + prevToken + " " + nextTokens + " (" + i + "/" + tok.size() + ")");*/ throw fail(quote(s) + " expected, got " + quote(t())); } consume(); } // outer wrapper function getting first token and unwinding the stack void parse_initial(unstructure_Receiver out) { consume(); // get first token parse(out); while (nempty(stack)) popLast(stack).run(); } } ThreadLocal tlLoading = dynamicObjectIsLoading_threadLocal(); Boolean b = tlLoading.get(); tlLoading.set(true); try { final Var v = new Var(); X x = new X(); x.parse_initial(new unstructure_Receiver() { void set(Object o) { v.set(o); } }); unstructure_tokrefs = x.tokrefs.size(); return v.get(); } finally { tlLoading.set(b); } } static boolean unstructure_debug = false; static Producer javaTokC_noMLS_onReader(final BufferedReader r) { final class X implements Producer { StringBuilder buf = new StringBuilder(); // stores from "i" char c, d, e = 'x'; // just not '\0' X() { // fill c, d and e nc(); nc(); nc(); } // get next character(s) into c, d and e void nc() { try { c = d; d = e; if (e == '\0') return; int i = r.read(); e = i < 0 ? '\0' : i == '\0' ? '_' // shouldn't happen anymore : (char) i; } catch (Exception __e) { throw rethrow(__e); } } void ncSave() { if (c != '\0') { buf.append(c); nc(); } } public String next() { // scan for whitespace while (c != '\0') { if (c == ' ' || c == '\t' || c == '\r' || c == '\n') nc(); else if (c == '/' && d == '*') { do nc(); while (c != '\0' && !(c == '*' && d == '/')); nc(); nc(); } else if (c == '/' && d == '/') { do nc(); while (c != '\0' && "\r\n".indexOf(c) < 0); } else break; } if (c == '\0') return null; // scan for non-whitespace if (c == '\'' || c == '"') { char opener = c; ncSave(); while (c != '\0') { if (c == opener || c == '\n') { // end at \n to not propagate unclosed string literal errors ncSave(); break; } else if (c == '\\') { ncSave(); ncSave(); } else ncSave(); } } else if (Character.isJavaIdentifierStart(c)) do ncSave(); while (Character.isJavaIdentifierPart(c) || c == '\''); // for stuff like "don't" else if (Character.isDigit(c)) { do ncSave(); while (Character.isDigit(c)); if (c == 'L') ncSave(); // Long constants like 1L } else ncSave(); String t = buf.toString(); buf.setLength(0); return t; } } return new X(); } static byte[] hexToBytes(String s) { if (odd(l(s))) throw fail("Hex string has odd length: " + quote(shorten(10, s))); int n = l(s) / 2; byte[] bytes = new byte[n]; for (int i = 0; i < n; i++) { int a = parseHexChar(s.charAt(i*2)); int b = parseHexChar(s.charAt(i*2+1)); if (a < 0 || b < 0) throw fail("Bad hex byte: " + quote(substring(s, i*2, i*2+2)) + " at " + i*2 + "/" + l(s)); bytes[i] = (byte) ((a << 4) | b); } return bytes; } static byte[] loadBinaryFilePart(File file, long start, long end) { try { RandomAccessFile raf = new RandomAccessFile(file, "r"); int n = toInt(min(raf.length(), end-start)); byte[] buffer = new byte[n]; try { raf.seek(start); raf.readFully(buffer, 0, n); return buffer; } finally { raf.close(); } } catch (Exception __e) { throw rethrow(__e); } } static int gzInputStream_defaultBufferSize = 65536; static GZIPInputStream gzInputStream(File f) { try { return gzInputStream(new FileInputStream(f)); } catch (Exception __e) { throw rethrow(__e); } } static GZIPInputStream gzInputStream(File f, int bufferSize) { try { return gzInputStream(new FileInputStream(f), bufferSize); } catch (Exception __e) { throw rethrow(__e); } } static GZIPInputStream gzInputStream(InputStream in) { return gzInputStream(in, gzInputStream_defaultBufferSize); } static GZIPInputStream gzInputStream(InputStream in, int bufferSize) { try { return _registerIOWrap(new GZIPInputStream(in, gzInputStream_defaultBufferSize), in); } catch (Exception __e) { throw rethrow(__e); } } static int roundToInt(double d) { return (int) Math.round(d); } static Runnable addThreadInfoToRunnable(final Object r) { final Object info = _threadInfo(); return info == null ? asRunnable(r) : new Runnable() { public void run() { try { _inheritThreadInfo(info); callF(r); } catch (Exception __e) { throw rethrow(__e); } } public String toString() { return "_inheritThreadInfo(info); callF(r);"; }}; } static Object swing(Object f) { return swingAndWait(f); } static A swing(F0 f) { return (A) swingAndWait(f); } static A swing(IF0 f) { return (A) swingAndWait(f); } static int hashCode(Object a) { return a == null ? 0 : a.hashCode(); } static Object callOpt(Object o) { return callF(o); } static A callOpt(Object o, String method, Object... args) { return (A) callOpt_withVarargs(o, method, args); } static void setOpt_raw(Object o, String field, Object value) { try { if (o == null) return; if (o instanceof Class) setOpt_raw((Class) o, field, value); else { Field f = setOpt_raw_findField(o.getClass(), field); if (f != null) { makeAccessible(f); smartSet(f, o, value); } } } catch (Exception __e) { throw rethrow(__e); } } static void setOpt_raw(Class c, String field, Object value) { try { if (c == null) return; Field f = setOpt_raw_findStaticField(c, field); if (f != null) { makeAccessible(f); smartSet(f, null, value); } } catch (Exception __e) { throw rethrow(__e); } } static Field setOpt_raw_findStaticField(Class c, String field) { Class _c = c; do { for (Field f : _c.getDeclaredFields()) if (f.getName().equals(field) && (f.getModifiers() & java.lang.reflect.Modifier.STATIC) != 0) return f; _c = _c.getSuperclass(); } while (_c != null); return null; } static Field setOpt_raw_findField(Class c, String field) { Class _c = c; do { for (Field f : _c.getDeclaredFields()) if (f.getName().equals(field)) return f; _c = _c.getSuperclass(); } while (_c != null); return null; } static A setDyn(A o, String key, Object value) { if (o == null) return o; setDynObjectValue(o, key, value); return o; } static String[] drop(int n, String[] a) { n = Math.min(n, a.length); String[] b = new String[a.length-n]; System.arraycopy(a, n, b, 0, b.length); return b; } static Object[] drop(int n, Object[] a) { n = Math.min(n, a.length); Object[] b = new Object[a.length-n]; System.arraycopy(a, n, b, 0, b.length); return b; } static boolean endsWithLetter(String s) { return nempty(s) && isLetter(last(s)); } static String structureForUser(Object o) { return beautifyStructure(struct_noStringSharing(o)); } static List sorted(Collection c, final Object comparator) { List l = cloneList(c); sort(l, makeComparator(comparator)); return l; } static List sorted(Collection c) { List l = cloneList(c); sort(l); return l; } static List methodsStartingWith(Object o, final String prefix) { return filter(allMethodNames(o), new F1() { public Object get(String s) { try { return startsWith(s, prefix); } catch (Exception __e) { throw rethrow(__e); } } public String toString() { return "startsWith(s, prefix)"; }}); } static boolean interruptThread_verbose = false; static void interruptThread(Thread t) { if (t == null) return; if (interruptThread_verbose) print("Interrupting thread " + t); // note reason in global map vm_threadInterruptionReasonsMap().put(t, getStackTrace()); t.interrupt(); URLConnection c = (URLConnection) (vm_generalSubMap("URLConnection per thread").get(t)); if (c != null) { try { print("Closing URLConnection of interrupted thread."); call(c, "disconnect"); } catch (Throwable __e) { _handleException(__e); }} } static boolean isJavaXClassLoader(ClassLoader cl) { return startsWithOneOf(className(cl), "main$JavaXClassLoader", "x30$JavaXClassLoader"); } static void setOptAll(Object o, Map fields) { if (fields == null) return; for (String field : keys(fields)) setOpt/*_flex*/(o, field, fields.get(field)); } static void setOptAll(Object o, Object... values) { //values = expandParams(c.getClass(), values); warnIfOddCount(values); for (int i = 0; i+1 < l(values); i += 2) { String field = (String) values[i]; Object value = values[i+1]; setOpt(o, field, value); } } static String reverseString(String s) { return new StringBuilder(s).reverse().toString(); } static int lastIndexOf(String a, String b) { return a == null || b == null ? -1 : a.lastIndexOf(b); } static int lastIndexOf(String a, char b) { return a == null ? -1 : a.lastIndexOf(b); } // starts searching from i-1 static int lastIndexOf(List l, int i, A a) { if (l == null) return -1; for (i = min(l(l), i)-1; i >= 0; i--) if (eq(l.get(i), a)) return i; return -1; } static int lastIndexOf(List l, A a) { if (l == null) return -1; for (int i = l(l)-1; i >= 0; i--) if (eq(l.get(i), a)) return i; return -1; } static LinkedHashMap asLinkedHashMap(Map map) { if (map instanceof LinkedHashMap) return (LinkedHashMap) map; LinkedHashMap m = new LinkedHashMap(); if (map != null) synchronized(collectionMutex(map)) { m.putAll(map); } return m; } static void assertFalse(Object o) { if (!(eq(o, false) /*|| isFalse(pcallF(o))*/)) throw fail(str(o)); } static boolean assertFalse(boolean b) { if (b) throw fail("oops"); return b; } static boolean assertFalse(String msg, boolean b) { if (b) throw fail(msg); return b; } static boolean _inCore() { return false; } static List hotwire_copyOver_after = synchroList(); static void hotwire_copyOver(Class c) { // TODO: make a mechanism for making such "inheritable" fields for (String field : ll("print_log", "print_silent", "androidContext", "_userHome")) setOptIfNotNull(c, field, getOpt(mc(), field)); setOptIfNotNull(c, "mainBot" , getMainBot()); setOpt(c, "creator_class" , new WeakReference(mc())); pcallFAll(hotwire_copyOver_after, c); } static String hmsWithColons() { return hmsWithColons(now()); } static String hmsWithColons(long time) { return new SimpleDateFormat("HH:mm:ss").format(time); } static ThreadLocal coresToUse_tl = new ThreadLocal(); static int coresToUse() { { Integer i = coresToUse_tl.get(); if (i != null) return i; } return max(1, numberOfCores()-1); } static void sort(T[] a, Comparator c) { if (a != null) Arrays.sort(a, c); } static void sort(T[] a) { if (a != null) Arrays.sort(a); } static void sort(int[] a) { if (a != null) Arrays.sort(a); } static void sort(List a, Comparator c) { if (a != null) Collections.sort(a, c); } static void sort(List a) { if (a != null) Collections.sort(a); } static Comparator makeComparator(final Object f) { if (f instanceof Comparator) return (Comparator) f; return new Comparator() { public int compare(Object a, Object b) { return (Integer) callF(f, a, b); } }; } static TreeMap ciMap() { return caseInsensitiveMap(); } static void sleepInCleanUp(long ms) { try { if (ms < 0) return; Thread.sleep(ms); } catch (Exception __e) { throw rethrow(__e); } } static void copyStream(InputStream in, OutputStream out) { try { byte[] buf = new byte[65536]; while (true) { int n = in.read(buf); if (n <= 0) return; out.write(buf, 0, n); } } catch (Exception __e) { throw rethrow(__e); } } static BufferedReader utf8bufferedReader(InputStream in) { try { return bufferedReader(_registerIOWrap(new InputStreamReader(in, "UTF-8"), in)); } catch (Exception __e) { throw rethrow(__e); } } static BufferedReader utf8bufferedReader(File f) { try { return utf8bufferedReader(newFileInputStream(f)); } catch (Exception __e) { throw rethrow(__e); } } static CloseableIterableIterator linesFromFile(File f) { try { if (!f.exists()) return emptyCloseableIterableIterator(); if (ewic(f.getName(), ".gz")) return linesFromReader(utf8bufferedReader(newGZIPInputStream(f))); return linesFromReader(utf8bufferedReader(f)); } catch (Exception __e) { throw rethrow(__e); } } static CloseableIterableIterator linesFromFile(String path) { return linesFromFile(newFile(path)); } static Comparator reverseComparator(Comparator c) { return (a, b) -> c.compare(b, a); } static Comparator reverseComparator() { return (a, b) -> cmp(b, a); } static String hopeningTag(String tag, Map params) { return hopeningTag(tag, mapToParams(params)); } static String hopeningTag(String tag, Object... params) { StringBuilder buf = new StringBuilder(); buf.append("<" + tag); for (int i = 0; i < l(params); i += 2) { String name = (String) get(params, i); Object val = get(params, i+1); if (nempty(name) && val != null) { if (val == html_valueLessParam()) buf.append(" " + name); else { String s = str(val); if (!empty(s)) buf.append(" " + name + "=" + htmlQuote(s)); } } } buf.append(">"); return str(buf); } static boolean neqic(String a, String b) { return !eqic(a, b); } static boolean neqic(char a, char b) { return !eqic(a, b); } static List newSubList(List l, int startIndex, int endIndex) { return cloneList(subList(l, startIndex, endIndex)); } static List newSubList(List l, int startIndex) { return cloneList(subList(l, startIndex)); } static List subList(List l, int startIndex) { return subList(l, startIndex, l(l)); } static List subList(int startIndex, int endIndex, List l) { return subList(l, startIndex, endIndex); } static List subList(List l, int startIndex, int endIndex) { if (l == null) return null; int n = l(l); startIndex = Math.max(0, startIndex); endIndex = Math.min(n, endIndex); if (startIndex >= endIndex) return ll(); if (startIndex == 0 && endIndex == n) return l; return l.subList(startIndex, endIndex); } static List subList(List l, IntRange r) { return subList(l, r.start, r.end); } static Runnable _topLevelErrorHandling(final Runnable runnable) { final Object info = _threadInfo(); return new Runnable() { public void run() { try { try { _threadInheritInfo(info); runnable.run(); } catch (Throwable __e) { _handleException(__e); } } catch (Exception __e) { throw rethrow(__e); } } public String toString() { return "pcall {\r\n _threadInheritInfo(info);\r\n runnable.run();\r\n }"; }}; } static Map vm_generalWeakSubMap(Object name) { synchronized(get(javax(), "generalMap")) { Map map = (Map) (vm_generalMap_get(name)); if (map == null) vm_generalMap_put(name, map = newWeakMap()); return map; } } static WeakReference weakRef(A a) { return newWeakReference(a); } static long parseLong(String s) { if (empty(s)) return 0; return Long.parseLong(dropSuffix("L", s)); } static long parseLong(Object s) { return Long.parseLong((String) s); } static double parseDouble(String s) { return Double.parseDouble(s); } static A[] arrayOfSameType(A[] a, int n) { return newObjectArrayOfSameType(a, n); } static String programID; static String getProgramID() { return nempty(programID) ? formatSnippetIDOpt(programID) : "?"; } // TODO: ask JavaX instead static String getProgramID(Class c) { String id = (String) getOpt(c, "programID"); if (nempty(id)) return formatSnippetID(id); return "?"; } static String getProgramID(Object o) { return getProgramID(getMainClass(o)); } static String javaTok_substringC(String s, int i, int j) { return s.substring(i, j); } static boolean sameSnippetID(String a, String b) { if (!isSnippetID(a) || !isSnippetID(b)) return false; return parseSnippetID(a) == parseSnippetID(b); } static File javaxDataDir_dir; // can be set to work on different base dir static File javaxDataDir() { return javaxDataDir_dir != null ? javaxDataDir_dir : new File(userHome(), "JavaX-Data"); } static File javaxDataDir(String... subs) { return newFile(javaxDataDir(), subs); } static String formatSnippetIDOpt(String s) { return isSnippetID(s) ? formatSnippetID(s) : s; } static volatile String caseID_caseID; static String caseID() { return caseID_caseID; } static void caseID(String id) { caseID_caseID = id; } static ThreadLocal> checkFileNotTooBigToRead_tl = new ThreadLocal(); static void checkFileNotTooBigToRead(File f) { callF(checkFileNotTooBigToRead_tl.get(), f); } static Producer javaTokC_noMLS_iterator(final String s) { return javaTokC_noMLS_iterator(s, 0); } static Producer javaTokC_noMLS_iterator(final String s, final int startIndex) { return new Producer() { final int l = s.length(); int i = startIndex; public String next() { if (i >= l) return null; int j = i; char c, d; // scan for whitespace while (j < l) { c = s.charAt(j); d = j+1 >= l ? '\0' : s.charAt(j+1); if (c == ' ' || c == '\t' || c == '\r' || c == '\n') ++j; else if (c == '/' && d == '*') { do ++j; while (j < l && !s.substring(j, Math.min(j+2, l)).equals("*/")); j = Math.min(j+2, l); } else if (c == '/' && d == '/') { do ++j; while (j < l && "\r\n".indexOf(s.charAt(j)) < 0); } else break; } i = j; if (i >= l) return null; c = s.charAt(i); d = i+1 >= l ? '\0' : s.charAt(i+1); // scan for non-whitespace if (c == '\'' || c == '"') { char opener = c; ++j; while (j < l) { if (s.charAt(j) == opener || s.charAt(j) == '\n') { // end at \n to not propagate unclosed string literal errors ++j; break; } else if (s.charAt(j) == '\\' && j+1 < l) j += 2; else ++j; } } else if (Character.isJavaIdentifierStart(c)) do ++j; while (j < l && Character.isJavaIdentifierPart(s.charAt(j))); else if (Character.isDigit(c)) { do ++j; while (j < l && Character.isDigit(s.charAt(j))); if (j < l && s.charAt(j) == 'L') ++j; // Long constants like 1L } else ++j; String t = quickSubstring(s, i, j); i = j; return t; } }; } static Object _defaultClassFinder_value = defaultDefaultClassFinder(); static Object _defaultClassFinder() { return _defaultClassFinder_value; } static HashMap findClass_fullName_cache = new HashMap(); // returns null on not found // this is the simple version that is not case-tolerant static Class findClass_fullName(String name) { synchronized(findClass_fullName_cache) { if (findClass_fullName_cache.containsKey(name)) return findClass_fullName_cache.get(name); Class c; try { c = Class.forName(name); } catch (ClassNotFoundException e) { c = null; } findClass_fullName_cache.put(name, c); return c; } } static String unquoteUsingCharArray(String s, char[] buf) { if (s == null) return null; if (startsWith(s, '[')) { int i = 1; while (i < s.length() && s.charAt(i) == '=') ++i; if (i < s.length() && s.charAt(i) == '[') { String m = s.substring(1, i); if (s.endsWith("]" + m + "]")) return s.substring(i+1, s.length()-i-1); } } if (s.length() > 1) { char c = s.charAt(0); if (c == '\"' || c == '\'') { int l = endsWith(s, c) ? s.length()-1 : s.length(); if (l > buf.length) return unquote(s); // fallback int n = 0; for (int i = 1; i < l; i++) { char ch = s.charAt(i); if (ch == '\\') { char nextChar = (i == l - 1) ? '\\' : s.charAt(i + 1); // Octal escape? if (nextChar >= '0' && nextChar <= '7') { String code = "" + nextChar; i++; if ((i < l - 1) && s.charAt(i + 1) >= '0' && s.charAt(i + 1) <= '7') { code += s.charAt(i + 1); i++; if ((i < l - 1) && s.charAt(i + 1) >= '0' && s.charAt(i + 1) <= '7') { code += s.charAt(i + 1); i++; } } buf[n++] = (char) Integer.parseInt(code, 8); continue; } switch (nextChar) { case '\"': ch = '\"'; break; case '\\': ch = '\\'; break; case 'b': ch = '\b'; break; case 'f': ch = '\f'; break; case 'n': ch = '\n'; break; case 'r': ch = '\r'; break; case 't': ch = '\t'; break; case '\'': ch = '\''; break; // Hex Unicode: u???? case 'u': if (i >= l - 5) { ch = 'u'; break; } int code = Integer.parseInt( "" + s.charAt(i + 2) + s.charAt(i + 3) + s.charAt(i + 4) + s.charAt(i + 5), 16); char[] x = Character.toChars(code); int lx = x.length; for (int j = 0; j < lx; j++) buf[n++] = x[j]; i += 5; continue; default: ch = nextChar; // added by Stefan } i++; } buf[n++] = ch; } return new String(buf, 0, n); } } return s; // not quoted - return original } static boolean structure_isMarker(String s, int i, int j) { if (i >= j) return false; if (s.charAt(i) != 'm') return false; ++i; while (i < j) { char c = s.charAt(i); if (c < '0' || c > '9') return false; ++i; } return true; } static String internIfLongerThan(String s, int l) { return s == null ? null : l(s) >= l ? intern(s) : s; } static char unquoteCharacter(String s) { assertTrue(s.startsWith("'") && s.length() > 1); return unquote("\"" + s.substring(1, s.endsWith("'") ? s.length()-1 : s.length()) + "\"").charAt(0); } static BigInteger parseBigInt(String s) { return new BigInteger(s); } static float parseFloat(String s) { return Float.parseFloat(s); } static boolean isLongConstant(String s) { if (!s.endsWith("L")) return false; s = s.substring(0, l(s)-1); return isInteger(s); } static boolean isInteger(String s) { int n = l(s); if (n == 0) return false; int i = 0; if (s.charAt(0) == '-') if (++i >= n) return false; while (i < n) { char c = s.charAt(i); if (c < '0' || c > '9') return false; ++i; } return true; } static List parseList(String s) { return (List) safeUnstructure(s); } static boolean[] boolArrayFromBytes(byte[] a, int n) { boolean[] b = new boolean[n]; int m = min(n, l(a)*8); for (int i = 0; i < m; i++) b[i] = (a[i/8] & 1 << (i & 7)) != 0; return b; } static boolean isJavaIdentifier(String s) { if (empty(s) || !Character.isJavaIdentifierStart(s.charAt(0))) return false; for (int i = 1; i < s.length(); i++) if (!Character.isJavaIdentifierPart(s.charAt(i))) return false; return true; } static String assertIdentifier(String s) { return assertIsIdentifier(s); } static String assertIdentifier(String msg, String s) { return assertIsIdentifier(msg, s); } static A nuStubInnerObject(Class c) { return nuStubInnerObject(c, null); } static A nuStubInnerObject(Class c, Object classFinder) { try { Class outerType = getOuterClass(c, classFinder); Constructor m = c.getDeclaredConstructor(outerType); makeAccessible(m); return (A) m.newInstance(new Object[] {null}); } catch (Exception __e) { throw rethrow(__e); } } static Map nuEmptyObject_cache = newDangerousWeakHashMap(); static A nuEmptyObject(Class c) { try { Constructor ctr; synchronized(nuEmptyObject_cache) { ctr = nuEmptyObject_cache.get(c); if (ctr == null) { nuEmptyObject_cache.put(c, ctr = nuEmptyObject_findConstructor(c)); makeAccessible(ctr); } } try { return (A) ctr.newInstance(); } catch (InstantiationException e) { if (empty(e.getMessage())) if ((c.getModifiers() & Modifier.ABSTRACT) != 0) throw fail("Can't instantiate abstract class " + className(c), e); else throw fail("Can't instantiate " + className(c), e); else throw rethrow(e); } } catch (Exception __e) { throw rethrow(__e); } } static Constructor nuEmptyObject_findConstructor(Class c) { for (Constructor m : c.getDeclaredConstructors()) if (m.getParameterTypes().length == 0) return m; throw fail("No default constructor declared in " + c.getName()); } static void setOptAllDyn_pcall(DynamicObject o, Map fields) { if (fields == null) return; HashMap fieldMap = instanceFieldsMap(o); for (Map.Entry e : fields.entrySet()) { try { String field = e.getKey(); Object val = e.getValue(); boolean has = fieldMap.containsKey(field); if (has) setOpt(o, field, val); else { o.fieldValues = syncMapPut2_createLinkedHashMap(o.fieldValues, intern(field), val); } } catch (Throwable __e) { _handleException(__e); }} } static void setOptAll_pcall(Object o, Map fields) { if (fields == null) return; for (String field : keys(fields)) try { setOpt(o, field, fields.get(field)); } catch (Throwable __e) { print(exceptionToStringShort(__e)); } } static void setOptAll_pcall(Object o, Object... values) { //values = expandParams(c.getClass(), values); warnIfOddCount(values); for (int i = 0; i+1 < l(values); i += 2) { String field = (String) values[i]; Object value = values[i+1]; try { setOpt(o, field, value); } catch (Throwable __e) { print(exceptionToStringShort(__e)); } } } static void fixOuterRefs(Object o) { try { if (o == null) return; Field[] l = thisDollarOneFields(o.getClass()); if (l.length <= 1) return; Object father = null; for (Field f : l) { father = f.get(o); if (father != null) break; } if (father == null) return; for (Field f : l) f.set(o, father); } catch (Exception __e) { throw rethrow(__e); } } static void setDynObjectValue(DynamicObject o, String field, Object value) { o.fieldValues = syncMapPut2_createLinkedHashMap(o.fieldValues, field, value); } static String intern(String s) { return fastIntern(s); } static void pcallOpt_noArgs(Object o, String method) { try { callOpt_noArgs(o, method); } catch (Throwable __e) { _handleException(__e); } } static double[] toDoubleArray(Collection l) { double[] a = new double[l(l)]; int i = 0; if (a.length != 0) for (double x : l) a[i++] = x; return a; } static String dropPrefix(String prefix, String s) { return s == null ? null : s.startsWith(prefix) ? s.substring(l(prefix)) : s; } static TreeSet ciSet() { return caseInsensitiveSet(); } static boolean jmatch(String pat, String s) { return jmatch(pat, s, null); } static boolean jmatch(String pat, String s, Matches matches) { if (s == null) return false; return jmatch(pat, javaTok(s), matches); } static boolean jmatch(String pat, List toks) { return jmatch(pat, toks, null); } static boolean jmatch(String pat, List toks, Matches matches) { List tokpat = javaTok(pat); String[] m = match2(tokpat, toks); //print(structure(tokpat) + " on " + structure(toks) + " => " + structure(m)); if (m == null) return false; else { if (matches != null) matches.m = m; return true; } } static Object nuObject(String className, Object... args) { try { return nuObject(classForName(className), args); } catch (Exception __e) { throw rethrow(__e); } } // too ambiguous - maybe need to fix some callers /*static O nuObject(O realm, S className, O... args) { ret nuObject(_getClass(realm, className), args); }*/ static A nuObject(Class c, Object... args) { try { if (args.length == 0) return nuObjectWithoutArguments(c); // cached! Constructor m = nuObject_findConstructor(c, args); makeAccessible(m); return (A) m.newInstance(args); } catch (Exception __e) { throw rethrow(__e); } } static Constructor nuObject_findConstructor(Class c, Object... args) { for (Constructor m : c.getDeclaredConstructors()) { if (!nuObject_checkArgs(m.getParameterTypes(), args, false)) continue; return m; } throw fail("Constructor " + c.getName() + getClasses(args) + " not found" + (args.length == 0 && (c.getModifiers() & java.lang.reflect.Modifier.STATIC) == 0 ? " - hint: it's a non-static class!" : "")); } static boolean nuObject_checkArgs(Class[] types, Object[] args, boolean debug) { if (types.length != args.length) { if (debug) System.out.println("Bad parameter length: " + args.length + " vs " + types.length); return false; } for (int i = 0; i < types.length; i++) if (!(args[i] == null || isInstanceX(types[i], args[i]))) { if (debug) System.out.println("Bad parameter " + i + ": " + args[i] + " vs " + types[i]); return false; } return true; } static boolean warn_on = true; static ThreadLocal> warn_warnings = new ThreadLocal(); static void warn(String s) { if (warn_on) print("Warning: " + s); } static void warn(String s, List warnings) { warn(s); if (warnings != null) warnings.add(s); addToCollection(warn_warnings.get(), s); } static ThreadLocal dynamicObjectIsLoading_threadLocal() { return DynamicObject_loading; } static int parseHexChar(char c) { if (c >= '0' && c <= '9') return charDiff(c, '0'); if (c >= 'a' && c <= 'f') return charDiff(c, 'a')+10; if (c >= 'A' && c <= 'F') return charDiff(c, 'A')+10; return -1; } static List> _threadInfo_makers = synchroList(); static Object _threadInfo() { if (empty(_threadInfo_makers)) return null; HashMap map = new HashMap(); pcallFAll(_threadInfo_makers, map); return map; } static Runnable asRunnable(Object o) { return toRunnable(o); } static void _inheritThreadInfo(Object info) { _threadInheritInfo(info); } static Object callOpt_withVarargs(Object o, String method, Object... args) { try { if (o == null) return null; if (o instanceof Class) { Class c = (Class) o; _MethodCache cache = callOpt_getCache(c); Method me = cache.findMethod(method, args); if (me == null) { // TODO: varargs return null; } if ((me.getModifiers() & Modifier.STATIC) == 0) return null; return invokeMethod(me, null, args); } else { Class c = o.getClass(); _MethodCache cache = callOpt_getCache(c); Method me = cache.findMethod(method, args); if (me != null) return invokeMethod(me, o, args); // try varargs List methods = cache.cache.get(method); if (methods != null) methodSearch: for (Method m : methods) { { if (!(m.isVarArgs())) continue; } Object[] newArgs = massageArgsForVarArgsCall(m, args); if (newArgs != null) return invokeMethod(m, o, newArgs); } return null; } } catch (Exception __e) { throw rethrow(__e); } } static boolean isLetter(char c) { return Character.isLetter(c); } static String beautifyStructure(String s) { List tok = javaTokForStructure(s); structure_addTokenMarkers(tok); jreplace(tok, "lhm", ""); return join(tok); } static String struct_noStringSharing(Object o) { structure_Data d = new structure_Data(); d.noStringSharing = true; return structure(o, d); } static List filter(Iterable c, Object pred) { if (pred instanceof F1) return filter(c, (F1) pred); List x = new ArrayList(); if (c != null) for (Object o : c) if (isTrue(callF(pred, o))) x.add(o); return x; } static List filter(Object pred, Iterable c) { return filter(c, pred); } static List filter(Iterable c, F1 pred) { List x = new ArrayList(); if (c != null) for (B o : c) if (pred.get(o)) x.add(o); return x; } static List filter(F1 pred, Iterable c) { return filter(c, pred); } //ifclass IF1 static List filter(Iterable c, IF1 pred) { List x = new ArrayList(); if (c != null) for (B o : c) if (pred.get(o)) x.add(o); return x; } static List filter(B[] c, IF1 pred) { List x = new ArrayList(); if (c != null) for (B o : c) if (pred.get(o)) x.add(o); return x; } static List filter(IF1 pred, Iterable c) { return filter(c, pred); } //endif static List allMethodNames(Object o) { Class c = _getClass(o); TreeSet names = new TreeSet(); while (c != null) { for (Method m : c.getDeclaredMethods()) names.add(m.getName()); c = c.getSuperclass(); } return asList(names); } static Map vm_threadInterruptionReasonsMap() { return vm_generalWeakSubMap("Thread interruption reasons"); } static Map vm_generalSubMap(Object name) { synchronized(get(javax(), "generalMap")) { Map map = (Map) (vm_generalMap_get(name)); if (map == null) vm_generalMap_put(name, map = synchroMap()); return map; } } static void warnIfOddCount(Object... list) { if (odd(l(list))) printStackTrace("Odd list size: " + list); } static void setOptIfNotNull(Object o, String field, Object value) { if (value != null) setOpt(o, field, value); } static Object mainBot; static Object getMainBot() { return mainBot; } static volatile int numberOfCores_value; static int numberOfCores() { if (numberOfCores_value == 0) numberOfCores_value = Runtime.getRuntime().availableProcessors(); return numberOfCores_value; } static TreeMap caseInsensitiveMap() { return new TreeMap(caseInsensitiveComparator()); } static CloseableIterableIterator emptyCloseableIterableIterator_instance = new CloseableIterableIterator() { public Object next() { throw fail(); } public boolean hasNext() { return false; } }; static CloseableIterableIterator emptyCloseableIterableIterator() { return emptyCloseableIterableIterator_instance; } static CloseableIterableIterator linesFromReader(Reader r) { final BufferedReader br = bufferedReader(r); return iteratorFromFunction_f0_autoCloseable(new F0() { public String get() { try { return readLineFromReaderWithClose(br); } catch (Exception __e) { throw rethrow(__e); } } public String toString() { return "ret readLineFromReaderWithClose(br);"; }}, _wrapIOCloseable(r)); } static Object[] mapToParams(Map map) { return mapToObjectArray(map); } static Object html_valueLessParam_cache; static Object html_valueLessParam() { if (html_valueLessParam_cache == null) html_valueLessParam_cache = html_valueLessParam_load(); return html_valueLessParam_cache; } static Object html_valueLessParam_load() { return new Object(); } static String htmlQuote(String s) { return "\"" + htmlencode_forParams(s) + "\""; } static List> _threadInheritInfo_retrievers = synchroList(); static void _threadInheritInfo(Object info) { if (info == null) return; pcallFAll(_threadInheritInfo_retrievers, (Map) info); } static Map newWeakMap() { return newWeakHashMap(); } static WeakReference newWeakReference(A a) { return a == null ? null : new WeakReference(a); } static String dropSuffix(String suffix, String s) { return s.endsWith(suffix) ? s.substring(0, l(s)-l(suffix)) : s; } static A[] newObjectArrayOfSameType(A[] a, int n) { return (A[]) Array.newInstance(a.getClass().getComponentType(), n); } static String formatSnippetID(String id) { return "#" + parseSnippetID(id); } static String formatSnippetID(long id) { return "#" + id; } static Class getMainClass() { return mc(); } static Class getMainClass(Object o) { try { if (o == null) return null; if (o instanceof Class && eq(((Class) o).getName(), "x30")) return (Class) o; return (o instanceof Class ? (Class) o : o.getClass()).getClassLoader().loadClass("main"); } catch (Exception __e) { throw rethrow(__e); } } public static boolean isSnippetID(String s) { try { parseSnippetID(s); return true; } catch (RuntimeException e) { return false; } } public static long parseSnippetID(String snippetID) { long id = Long.parseLong(shortenSnippetID(snippetID)); if (id == 0) throw fail("0 is not a snippet ID"); return id; } static String quickSubstring(String s, int i, int j) { if (i == j) return ""; return s.substring(i, j); } static Object defaultDefaultClassFinder() { return new F1() { public Class get(String name) { Class c = findClass_fullName(name); if (c != null) return c; if (name.startsWith("loadableUtils.utils$")) return findClass_fullName("main" + name.substring(19)); return null; } }; } static Object safeUnstructure(String s) { return unstructure(s, true); } static String assertIsIdentifier(String s) { if (!isIdentifier(s)) throw fail("Not an identifier: " + quote(s)); return s; } static String assertIsIdentifier(String msg, String s) { if (!isIdentifier(s)) throw fail(msg + " - Not an identifier: " + quote(s)); return s; } static Class getOuterClass(Class c) { return getOuterClass(c, null); } static Class getOuterClass(Class c, Object classFinder) { try { String s = c.getName(); int i = s.lastIndexOf('$'); String name = substring(s, 0, i); if (classFinder != null) return (Class) callF(classFinder, name); return Class.forName(name); } catch (Exception __e) { throw rethrow(__e); } } static HashMap instanceFieldsMap(Object o) { Class c = o.getClass(); HashMap map; synchronized(getOpt_cache) { map = getOpt_cache.get(c); if (map == null) map = getOpt_makeCache(c); } return map; } static LinkedHashMap syncMapPut2_createLinkedHashMap(LinkedHashMap map, A key, B value) { if (key != null) if (value != null) { if (map == null) map = new LinkedHashMap(); synchronized(collectionMutex(map)) { map.put(key, value); } } else if (map != null) synchronized(collectionMutex(map)) { map.remove(key); } return map; } static String exceptionToStringShort(Throwable e) { lastException(e); e = getInnerException(e); String msg = hideCredentials(unnull(e.getMessage())); if (msg.indexOf("Error") < 0 && msg.indexOf("Exception") < 0) return baseClassName(e) + prependIfNempty(": ", msg); else return msg; } static Map thisDollarOneFields_cache = newDangerousWeakHashMap(); static Field[] thisDollarOneFields(Class c) { synchronized(thisDollarOneFields_cache) { Field[] l = thisDollarOneFields_cache.get(c); if (l == null) thisDollarOneFields_cache.put(c, l = thisDollarOneFields_uncached(c)); return l; } } static Field[] thisDollarOneFields_uncached(Class c) { List fields = new ArrayList(); do { for (Field f : c.getDeclaredFields()) if (f.getName().equals("this$1")) fields.add(makeAccessible(f)); c = c.getSuperclass(); } while (c != null); return toArray(new Field[l(fields)], fields); } static Method fastIntern_method; static String fastIntern(String s) { try { if (s == null) return null; if (fastIntern_method == null) { fastIntern_method = findMethodNamed(javax(), "internPerProgram"); if (fastIntern_method == null) upgradeJavaXAndRestart(); } return (String) fastIntern_method.invoke(null, s); } catch (Exception __e) { throw rethrow(__e); } } static Map> callOpt_noArgs_cache = newDangerousWeakHashMap(); static Object callOpt_noArgs(Object o, String method) { try { if (o == null) return null; if (o instanceof Class) return callOpt(o, method); // not optimized Class c = o.getClass(); HashMap map; synchronized(callOpt_noArgs_cache) { map = callOpt_noArgs_cache.get(c); if (map == null) map = callOpt_noArgs_makeCache(c); } Method m = map.get(method); return m != null ? m.invoke(o) : null; } catch (Exception __e) { throw rethrow(__e); } } // used internally - we are in synchronized block static HashMap callOpt_noArgs_makeCache(Class c) { HashMap map = new HashMap(); Class _c = c; do { for (Method m : c.getDeclaredMethods()) if (m.getParameterTypes().length == 0 && !reflection_isForbiddenMethod(m)) { makeAccessible(m); String name = m.getName(); if (!map.containsKey(name)) map.put(name, m); } _c = _c.getSuperclass(); } while (_c != null); callOpt_noArgs_cache.put(c, map); return map; } static TreeSet caseInsensitiveSet() { return caseInsensitiveSet_treeSet(); } static TreeSet caseInsensitiveSet(Collection c) { return caseInsensitiveSet_treeSet(c); } // TODO: extended multi-line strings static int javaTok_n, javaTok_elements; static boolean javaTok_opt = false; static List javaTok(String s) { ++javaTok_n; ArrayList tok = new ArrayList(); int l = s == null ? 0 : s.length(); int i = 0, n = 0; while (i < l) { int j = i; char c, d; // scan for whitespace while (j < l) { c = s.charAt(j); d = j+1 >= l ? '\0' : s.charAt(j+1); if (c == ' ' || c == '\t' || c == '\r' || c == '\n') ++j; else if (c == '/' && d == '*') { do ++j; while (j < l && !s.substring(j, Math.min(j+2, l)).equals("*/")); j = Math.min(j+2, l); } else if (c == '/' && d == '/') { do ++j; while (j < l && "\r\n".indexOf(s.charAt(j)) < 0); } else break; } tok.add(javaTok_substringN(s, i, j)); ++n; i = j; if (i >= l) break; c = s.charAt(i); d = i+1 >= l ? '\0' : s.charAt(i+1); // scan for non-whitespace // Special JavaX syntax: 'identifier if (c == '\'' && Character.isJavaIdentifierStart(d) && i+2 < l && "'\\".indexOf(s.charAt(i+2)) < 0) { j += 2; while (j < l && Character.isJavaIdentifierPart(s.charAt(j))) ++j; } else if (c == '\'' || c == '"') { char opener = c; ++j; while (j < l) { int c2 = s.charAt(j); if (c2 == opener || c2 == '\n' && opener == '\'') { // allow multi-line strings, but not for ' ++j; break; } else if (c2 == '\\' && j+1 < l) j += 2; else ++j; } } else if (Character.isJavaIdentifierStart(c)) do ++j; while (j < l && (Character.isJavaIdentifierPart(s.charAt(j)) || s.charAt(j) == '\'')); // for stuff like "don't" else if (Character.isDigit(c)) { do ++j; while (j < l && Character.isDigit(s.charAt(j))); if (j < l && s.charAt(j) == 'L') ++j; // Long constants like 1L } else if (c == '[' && d == '[') { do ++j; while (j+1 < l && !s.substring(j, j+2).equals("]]")); j = Math.min(j+2, l); } else if (c == '[' && d == '=' && i+2 < l && s.charAt(i+2) == '[') { do ++j; while (j+2 < l && !s.substring(j, j+3).equals("]=]")); j = Math.min(j+3, l); } else ++j; tok.add(javaTok_substringC(s, i, j)); ++n; i = j; } if ((tok.size() % 2) == 0) tok.add(""); javaTok_elements += tok.size(); return tok; } static List javaTok(List tok) { return javaTokWithExisting(join(tok), tok); } // match2 matches multiple "*" (matches a single token) wildcards and zero or one "..." wildcards (matches multiple tokens) static String[] match2(List pat, List tok) { // standard case (no ...) int i = pat.indexOf("..."); if (i < 0) return match2_match(pat, tok); pat = new ArrayList(pat); // We're modifying it, so copy first pat.set(i, "*"); while (pat.size() < tok.size()) { pat.add(i, "*"); pat.add(i+1, ""); // doesn't matter } return match2_match(pat, tok); } static String[] match2_match(List pat, List tok) { List result = new ArrayList(); if (pat.size() != tok.size()) { return null; } for (int i = 1; i < pat.size(); i += 2) { String p = pat.get(i), t = tok.get(i); if (eq(p, "*")) result.add(t); else if (!equalsIgnoreCase(unquote(p), unquote(t))) // bold change - match quoted and unquoted now. TODO: should remove return null; } return result.toArray(new String[result.size()]); } static Map classForName_cache = synchroHashMap(); static Class classForName(String name) { try { if (classForName_cache == null) return Class.forName(name); // in class init Class c = classForName_cache.get(name); if (c == null) classForName_cache.put(name, c = Class.forName(name)); return c; } catch (Exception __e) { throw rethrow(__e); } } static Map nuObjectWithoutArguments_cache = newDangerousWeakHashMap(); static Object nuObjectWithoutArguments(String className) { try { return nuObjectWithoutArguments(classForName(className)); } catch (Exception __e) { throw rethrow(__e); } } static A nuObjectWithoutArguments(Class c) { try { if (nuObjectWithoutArguments_cache == null) // in class init return (A) nuObjectWithoutArguments_findConstructor(c).newInstance(); Constructor m = nuObjectWithoutArguments_cache.get(c); if (m == null) nuObjectWithoutArguments_cache.put(c, m = nuObjectWithoutArguments_findConstructor(c)); return (A) m.newInstance(); } catch (Exception __e) { throw rethrow(__e); } } static Constructor nuObjectWithoutArguments_findConstructor(Class c) { for (Constructor m : c.getDeclaredConstructors()) if (empty(m.getParameterTypes())) { makeAccessible(m); return m; } throw fail("No default constructor found in " + c.getName()); } static boolean addToCollection(Collection c, A a) { return c != null && c.add(a); } static int charDiff(char a, char b) { return (int) a-(int) b; } static int charDiff(String a, char b) { return charDiff(stringToChar(a), b); } static List javaTokForStructure(String s) { return javaTok_noMLS(s); } static String structure_addTokenMarkers(String s) { return join(structure_addTokenMarkers(javaTokForStructure(s))); } static List structure_addTokenMarkers(List tok) { // find references TreeSet refs = new TreeSet(); for (int i = 1; i < l(tok); i += 2) { String t = tok.get(i); if (t.startsWith("t") && isInteger(t.substring(1))) refs.add(parseInt(t.substring(1))); } if (empty(refs)) return tok; // add markers for (int i : refs) { int idx = i*2+1; if (idx >= l(tok)) continue; // broken structure String t = ""; if (endsWithLetterOrDigit(tok.get(idx-1))) t = " "; tok.set(idx, t + "m" + i + " " + tok.get(idx)); } return tok; } static String jreplace(String s, String in, String out) { return jreplace(s, in, out, null); } static String jreplace(String s, String in, String out, Object condition) { List tok = javaTok(s); return jreplace(tok, in, out, condition) ? join(tok) : s; } // leaves tok properly tokenized // returns true iff anything was replaced static boolean jreplace(List tok, String in, String out) { return jreplace(tok, in, out, false, true, null); } static boolean jreplace(List tok, String in, String out, Object condition) { return jreplace(tok, in, out, false, true, condition); } static boolean jreplace(List tok, String in, String out, IF2, Integer, Boolean> condition) { return jreplace(tok, in, out, (Object) condition); } static boolean jreplace(List tok, String in, String out, boolean ignoreCase, boolean reTok, Object condition) { String[] toks = javaTokForJFind_array(in); int lTokin = toks.length*2+1; boolean anyChange = false; int i = -1; for (int n = 0; n < 10000; n++) { // TODO: don't need this check anymore i = findCodeTokens(tok, i+1, ignoreCase, toks, condition); if (i < 0) return anyChange; List subList = tok.subList(i-1, i+lTokin-1); // N to N String expansion = jreplaceExpandRefs(out, subList); int end = i+lTokin-2; clearAllTokens(tok, i, end); // C to C tok.set(i, expansion); if (reTok) // would this ever be false?? reTok(tok, i, end); i = end; anyChange = true; } throw fail("woot? 10000! " + quote(in) + " => " + quote(out)); } static boolean jreplace_debug = false; static boolean structure_showTiming, structure_checkTokenCount; static String structure(Object o) { return structure(o, new structure_Data()); } static String structure(Object o, structure_Data d) { StringWriter sw = new StringWriter(); d.out = new PrintWriter(sw); structure_go(o, d); String s = str(sw); if (structure_checkTokenCount) { print("token count=" + d.n); assertEquals("token count", l(javaTokC(s)), d.n); } return s; } static void structure_go(Object o, structure_Data d) { structure_1(o, d); while (nempty(d.stack)) popLast(d.stack).run(); } static void structureToPrintWriter(Object o, PrintWriter out) { structure_Data d = new structure_Data(); d.out = out; structure_go(o, d); } // leave to false, unless unstructure() breaks static boolean structure_allowShortening = false; // info on how to serialize objects of a certain class static class structure_ClassInfo { List fields; Method customSerializer; boolean special, nullInstances; } static class structure_Data { PrintWriter out; int stringSizeLimit; int shareStringsLongerThan = 20; boolean noStringSharing = false; IdentityHashMap seen = new IdentityHashMap(); //new BitSet refd; HashMap strings = new HashMap(); HashSet concepts = new HashSet(); HashMap infoByClass = new HashMap(); HashMap persistenceInfo = new HashMap(); int n; // token count List stack = new ArrayList(); // append single token structure_Data append(String token) { out.print(token); ++n; return this; } structure_Data append(int i) { out.print(i); ++n; return this; } // append multiple tokens structure_Data append(String token, int tokCount) { out.print(token); n += tokCount; return this; } // extend last token structure_Data app(String token) { out.print(token); return this; } structure_Data app(int i) { out.print(i); return this; } } static void structure_1(final Object o, final structure_Data d) { try { if (o == null) { d.append("null"); return; } Class c = o.getClass(); boolean concept = false; structure_ClassInfo info = d.infoByClass.get(c); if (info == null) { d.infoByClass.put(c, info = new structure_ClassInfo()); if ((info.customSerializer = findMethodNamed(c, "_serialize")) != null) info.special = true; } List lFields = info.fields; if (lFields == null) { // these are never back-referenced (for readability) if (o instanceof Number) { PrintWriter out = d.out; if (o instanceof Integer) { int i = ((Integer) o).intValue(); out.print(i); d.n += i < 0 ? 2 : 1; return; } if (o instanceof Long) { long l = ((Long) o).longValue(); out.print(l); out.print("L"); d.n += l < 0 ? 2 : 1; return; } if (o instanceof Short) { short s = ((Short) o).shortValue(); d.append("sh "); out.print(s); d.n += s < 0 ? 2 : 1; return; } if (o instanceof Float) { d.append("fl ", 2); quoteToPrintWriter(str(o), out); return; } if (o instanceof Double) { d.append("d(", 3); quoteToPrintWriter(str(o), out); d.append(")"); return; } if (o instanceof BigInteger) { out.print("bigint("); out.print(o); out.print(")"); d.n += ((BigInteger) o).signum() < 0 ? 5 : 4; return; } } if (o instanceof Boolean) { d.append(((Boolean) o).booleanValue() ? "t" : "f"); return; } if (o instanceof Character) { d.append(quoteCharacter((Character) o)); return; } if (o instanceof File) { d.append("File ").append(quote(((File) o).getPath())); return; } // referencable objects follow Integer ref = d.seen.get(o); if (o instanceof String && ref == null) ref = d.strings.get((String) o); if (ref != null) { /*d.refd.set(ref);*/ d.append("t").app(ref); return; } if (!(o instanceof String)) d.seen.put(o, d.n); // record token number else { String s = d.stringSizeLimit != 0 ? shorten((String) o, d.stringSizeLimit) : (String) o; if (!d.noStringSharing) { if (d.shareStringsLongerThan == Integer.MAX_VALUE) d.seen.put(o, d.n); if (l(s) >= d.shareStringsLongerThan) d.strings.put(s, d.n); } quoteToPrintWriter(s, d.out); d.n++; return; } if (o instanceof Set) { /*O set2 = unwrapSynchronizedSet(o); if (set2 != o) { d.append("sync"); o = set2; } TODO */ if (((Set) o) instanceof TreeSet) { d.append(isCISet_gen(((Set) o)) ? "ciset" : "treeset"); structure_1(new ArrayList(((Set) o)), d); return; } // assume it's a HashSet or LinkedHashSet d.append(((Set) o) instanceof LinkedHashSet ? "lhs" : "hashset"); structure_1(new ArrayList(((Set) o)), d); return; } String name = c.getName(); if (o instanceof Collection && !isJavaXClassName(name) /* && neq(name, "main$Concept$RefL") */) { // it's a list if (name.equals("java.util.Collections$SynchronizedList") || name.equals("java.util.Collections$SynchronizedRandomAccessList")) { d.append("sync "); { structure_1(unwrapSynchronizedList(((List) o)), d); return; } } else if (name.equals("java.util.LinkedList")) d.append("ll"); d.append("["); final int l = d.n; final Iterator it = ((Collection) o).iterator(); d.stack.add(new Runnable() { public void run() { try { if (!it.hasNext()) d.append("]"); else { d.stack.add(this); if (d.n != l) d.append(", "); structure_1(it.next(), d); } } catch (Exception __e) { throw rethrow(__e); } } public String toString() { return "if (!it.hasNext())\r\n d.append(\"]\");\r\n else {\r\n d.sta..."; }}); return; } if (o instanceof Map && !startsWith(name, "main$")) { if (o instanceof LinkedHashMap) d.append("lhm"); else if (o instanceof HashMap) d.append("hm"); else if (o instanceof TreeMap) d.append(isCIMap_gen(((TreeMap) o)) ? "cimap" : "tm"); else if (name.equals("java.util.Collections$SynchronizedMap") || name.equals("java.util.Collections$SynchronizedSortedMap") || name.equals("java.util.Collections$SynchronizedNavigableMap")) { d.append("sync "); { structure_1(unwrapSynchronizedMap(((Map) o)), d); return; } } d.append("{"); final int l = d.n; final Iterator it = ((Map) o).entrySet().iterator(); d.stack.add(new Runnable() { boolean v = false; Map.Entry e; public void run() { if (v) { d.append("="); v = false; d.stack.add(this); structure_1(e.getValue(), d); } else { if (!it.hasNext()) d.append("}"); else { e = (Map.Entry) it.next(); v = true; d.stack.add(this); if (d.n != l) d.append(", "); structure_1(e.getKey(), d); } } } }); return; } if (c.isArray()) { if (o instanceof byte[]) { d.append("ba ").append(quote(bytesToHex((byte[]) o))); return; } final int n = Array.getLength(o); if (o instanceof boolean[]) { String hex = boolArrayToHex((boolean[]) o); int i = l(hex); while (i > 0 && hex.charAt(i-1) == '0' && hex.charAt(i-2) == '0') i -= 2; d.append("boolarray ").append(n).app(" ").append(quote(substring(hex, 0, i))); return; } String atype = "array"/*, sep = ", "*/; // sep is not used yet if (o instanceof int[]) { //ret "intarray " + quote(intArrayToHex((int[]) o)); atype = "intarray"; //sep = " "; } else if (o instanceof double[]) { atype = "dblarray"; //sep = " "; } d.append(atype).append("{"); d.stack.add(new Runnable() { int i; public void run() { if (i >= n) d.append("}"); else { d.stack.add(this); if (i > 0) d.append(", "); structure_1(Array.get(o, i++), d); } } }); return; } if (o instanceof Class) { d.append("class(", 2).append(quote(((Class) o).getName())).append(")"); return; } if (o instanceof Throwable) { d.append("exception(", 2).append(quote(((Throwable) o).getMessage())).append(")"); return; } if (o instanceof BitSet) { BitSet bs = (BitSet) o; d.append("bitset{", 2); int l = d.n; for (int i = bs.nextSetBit(0); i >= 0; i = bs.nextSetBit(i+1)) { if (d.n != l) d.append(", "); d.append(i); } d.append("}"); return; } // Need more cases? This should cover all library classes... if (name.startsWith("java.") || name.startsWith("javax.")) { d.append("j ").append(quote(str(o))); return; // Hm. this is not unstructure-able } /*if (name.equals("main$Lisp")) { fail("lisp not supported right now"); }*/ if (info.special) { if (info.customSerializer != null) { // custom serialization (_serialize method) Object o2 = invokeMethod(info.customSerializer, o); d.append("cu "); String shortName = dropPrefix("main$", name); d.append(shortName); d.out.append(' '); structure_1(o2, d); return; } else if (info.nullInstances) { d.append("null"); return; } else throw fail("unknown special type"); } String dynName = shortDynamicClassName(o); if (concept && !d.concepts.contains(dynName)) { d.concepts.add(dynName); d.append("c "); } // serialize an object with fields. // first, collect all fields and values in fv. TreeSet fields = new TreeSet(new Comparator() { public int compare(Field a, Field b) { return stdcompare(a.getName(), b.getName()); } }); Class cc = c; while (cc != Object.class) { for (Field field : getDeclaredFields_cached(cc)) { String fieldName = field.getName(); if (fieldName.equals("_persistenceInfo")) d.persistenceInfo.put(c, field); if ((field.getModifiers() & (java.lang.reflect.Modifier.STATIC | java.lang.reflect.Modifier.TRANSIENT)) != 0) continue; fields.add(field); // put special cases here...? } cc = cc.getSuperclass(); } // TODO: S fieldOrder = getOpt(c, "_fieldOrder"); lFields = asList(fields); // Render this$1 first because unstructure needs it for constructor call. int n = l(lFields); for (int i = 0; i < n; i++) { Field f = lFields.get(i); if (f.getName().equals("this$1")) { lFields.remove(i); lFields.add(0, f); break; } } info.fields = lFields; } // << if (lFields == null) else { // ref handling for lFields != null Integer ref = d.seen.get(o); if (ref != null) { /*d.refd.set(ref);*/ d.append("t").app(ref); return; } d.seen.put(o, d.n); // record token number } Field persistenceInfoField = (Field) (d.persistenceInfo.get(c)); Map persistenceInfo = persistenceInfoField == null ? null : (Map) persistenceInfoField.get(o); LinkedHashMap fv = new LinkedHashMap(); for (Field f : lFields) { Object value; try { value = f.get(o); } catch (Exception e) { value = "?"; } if (value != null && (persistenceInfo == null || !Boolean.FALSE.equals(persistenceInfo.get(f.getName())))) fv.put(f.getName(), value); } String name = c.getName(); String shortName = dropPrefix("main$", name); // TODO: drop loadableUtils also if (startsWithDigit(shortName)) shortName = name; // for anonymous classes // Now we have fields & values. Process fieldValues if it's a DynamicObject. // omit field "className" if equal to class's name if (concept && eq(fv.get("className"), shortName)) fv.remove("className"); if (o instanceof DynamicObject) { putAll(fv, (Map) fv.get("fieldValues")); fv.remove("fieldValues"); shortName = shortDynamicClassName(o); fv.remove("className"); } String singleField = fv.size() == 1 ? first(fv.keySet()) : null; d.append(shortName); d.n += countDots(shortName)*2; // correct token count final int l = d.n; final Iterator it = fv.entrySet().iterator(); d.stack.add(new Runnable() { public void run() { try { if (!it.hasNext()) { if (d.n != l) d.append(")"); } else { Map.Entry e = (Map.Entry) it.next(); d.append(d.n == l ? "(" : ", "); d.append((String) e.getKey()).append("="); d.stack.add(this); structure_1(e.getValue(), d); } } catch (Exception __e) { throw rethrow(__e); } } public String toString() { return "if (!it.hasNext()) {\r\n if (d.n != l)\r\n d.append(\")\");\r\n } else..."; }}); } catch (Exception __e) { throw rethrow(__e); } } static Comparator caseInsensitiveComparator() { return betterCIComparator(); } static CloseableIterableIterator iteratorFromFunction_f0_autoCloseable(final F0 f, final AutoCloseable closeable) { class IFF2 extends CloseableIterableIterator { A a; boolean done = false; public boolean hasNext() { getNext(); return !done; } public A next() { getNext(); if (done) throw fail(); A _a = a; a = null; return _a; } void getNext() { if (done || a != null) return; a = f.get(); done = a == null; } public void close() throws Exception { if (closeable != null) closeable.close(); } }; return new IFF2(); } static String readLineFromReaderWithClose(BufferedReader r) { try { String s = r.readLine(); if (s == null) r.close(); return s; } catch (Exception __e) { throw rethrow(__e); } } static AutoCloseable _wrapIOCloseable(final AutoCloseable c) { return c == null ? null : new AutoCloseable() { public String toString() { return "c.close();\r\n _registerIO(c, null, false);"; } public void close() throws Exception { c.close(); _registerIO(c, null, false); }}; } static Object[] mapToObjectArray(Map map) { List l = new ArrayList(); for (Object o : keys(map)) { l.add(o); l.add(map.get(o)); } return toObjectArray(l); } static Object[] mapToObjectArray(Object f, Collection l) { int n = l(l); Object[] array = new Object[n]; if (n != 0) { Iterator it = iterator(l); for (int i = 0; i < n; i++) array[i] = callF(f, it.next()); } return array; } static Object[] mapToObjectArray(Object f, Object[] l) { int n = l(l); Object[] array = new Object[n]; for (int i = 0; i < n; i++) array[i] = callF(f, l[i]); return array; } static Object[] mapToObjectArray(Collection l, IF1 f) { return mapToObjectArray(f, l); } static String htmlencode_forParams(String s) { if (s == null) return ""; StringBuilder out = new StringBuilder(Math.max(16, s.length())); for (int i = 0; i < s.length(); i++) { char c = s.charAt(i); /*if (c >= 0x100) out.append("&#x").append(charToHex(c)).append(';'); else*/ if (c > 127 || c == '"' || c == '<' || c == '>') { out.append("&#"); out.append((int) c); out.append(';'); } else out.append(c); } return out.toString(); } static String shortenSnippetID(String snippetID) { if (snippetID.startsWith("#")) snippetID = snippetID.substring(1); String httpBlaBla = "http://tinybrain.de/"; if (snippetID.startsWith(httpBlaBla)) snippetID = snippetID.substring(httpBlaBla.length()); return "" + parseLong(snippetID); } static boolean isIdentifier(String s) { return isJavaIdentifier(s); } static String baseClassName(String className) { return substring(className, className.lastIndexOf('.')+1); } static String baseClassName(Object o) { return baseClassName(getClassName(o)); } static String prependIfNempty(String prefix, String s) { return empty(s) ? unnull(s) : prefix + s; } static Object[] toArray(Collection c) { return toObjectArray(c); } static A[] toArray(Iterable c, Class type) { A[] a = arrayOfType(l(c), type); if (a.length == 0) return a; asList(c).toArray(a); return a; } // array must have correct length and will be filled static A[] toArray(A[] array, Collection c) { if (array == null || c == null) return null; asList(c).toArray(array); return array; } // This is a bit rough... finds static and non-static methods. static Method findMethodNamed(Object obj, String method) { if (obj == null) return null; if (obj instanceof Class) return findMethodNamed((Class) obj, method); return findMethodNamed(obj.getClass(), method); } static Method findMethodNamed(Class c, String method) { while (c != null) { for (Method m : c.getDeclaredMethods()) if (m.getName().equals(method)) { makeAccessible(m); return m; } c = c.getSuperclass(); } return null; } static void upgradeJavaXAndRestart() { run("#1001639"); restart(); sleep(); } static TreeSet caseInsensitiveSet_treeSet() { return new TreeSet(caseInsensitiveComparator()); } static TreeSet caseInsensitiveSet_treeSet(Collection c) { return toCaseInsensitiveSet_treeSet(c); } static String javaTok_substringN(String s, int i, int j) { if (i == j) return ""; if (j == i+1 && s.charAt(i) == ' ') return " "; return s.substring(i, j); } static List javaTokWithExisting(String s, List existing) { ++javaTok_n; int nExisting = javaTok_opt && existing != null ? existing.size() : 0; ArrayList tok = existing != null ? new ArrayList(nExisting) : new ArrayList(); int l = s.length(); int i = 0, n = 0; while (i < l) { int j = i; char c, d; // scan for whitespace while (j < l) { c = s.charAt(j); d = j+1 >= l ? '\0' : s.charAt(j+1); if (c == ' ' || c == '\t' || c == '\r' || c == '\n') ++j; else if (c == '/' && d == '*') { do ++j; while (j < l && !s.substring(j, Math.min(j+2, l)).equals("*/")); j = Math.min(j+2, l); } else if (c == '/' && d == '/') { do ++j; while (j < l && "\r\n".indexOf(s.charAt(j)) < 0); } else break; } if (n < nExisting && javaTokWithExisting_isCopyable(existing.get(n), s, i, j)) tok.add(existing.get(n)); else tok.add(javaTok_substringN(s, i, j)); ++n; i = j; if (i >= l) break; c = s.charAt(i); d = i+1 >= l ? '\0' : s.charAt(i+1); // scan for non-whitespace // Special JavaX syntax: 'identifier if (c == '\'' && Character.isJavaIdentifierStart(d) && i+2 < l && "'\\".indexOf(s.charAt(i+2)) < 0) { j += 2; while (j < l && Character.isJavaIdentifierPart(s.charAt(j))) ++j; } else if (c == '\'' || c == '"') { char opener = c; ++j; while (j < l) { if (s.charAt(j) == opener /*|| s.charAt(j) == '\n'*/) { // allow multi-line strings ++j; break; } else if (s.charAt(j) == '\\' && j+1 < l) j += 2; else ++j; } } else if (Character.isJavaIdentifierStart(c)) do ++j; while (j < l && (Character.isJavaIdentifierPart(s.charAt(j)) || "'".indexOf(s.charAt(j)) >= 0)); // for stuff like "don't" else if (Character.isDigit(c)) { do ++j; while (j < l && Character.isDigit(s.charAt(j))); if (j < l && s.charAt(j) == 'L') ++j; // Long constants like 1L } else if (c == '[' && d == '[') { do ++j; while (j+1 < l && !s.substring(j, j+2).equals("]]")); j = Math.min(j+2, l); } else if (c == '[' && d == '=' && i+2 < l && s.charAt(i+2) == '[') { do ++j; while (j+2 < l && !s.substring(j, j+3).equals("]=]")); j = Math.min(j+3, l); } else ++j; if (n < nExisting && javaTokWithExisting_isCopyable(existing.get(n), s, i, j)) tok.add(existing.get(n)); else tok.add(javaTok_substringC(s, i, j)); ++n; i = j; } if ((tok.size() % 2) == 0) tok.add(""); javaTok_elements += tok.size(); return tok; } static boolean javaTokWithExisting_isCopyable(String t, String s, int i, int j) { return t.length() == j-i && s.regionMatches(i, t, 0, j-i); // << could be left out, but that's brave } static boolean equalsIgnoreCase(String a, String b) { return eqic(a, b); } static boolean equalsIgnoreCase(char a, char b) { return eqic(a, b); } static List javaTok_noMLS(String s) { ArrayList tok = new ArrayList(); int l = s == null ? 0 : s.length(); int i = 0, n = 0; while (i < l) { int j = i; char c, d; // scan for whitespace while (j < l) { c = s.charAt(j); d = j+1 >= l ? '\0' : s.charAt(j+1); if (c == ' ' || c == '\t' || c == '\r' || c == '\n') ++j; else if (c == '/' && d == '*') { do ++j; while (j < l && !s.substring(j, Math.min(j+2, l)).equals("*/")); j = Math.min(j+2, l); } else if (c == '/' && d == '/') { do ++j; while (j < l && "\r\n".indexOf(s.charAt(j)) < 0); } else break; } tok.add(javaTok_substringN(s, i, j)); ++n; i = j; if (i >= l) break; c = s.charAt(i); d = i+1 >= l ? '\0' : s.charAt(i+1); // scan for non-whitespace if (c == '\'' || c == '"') { char opener = c; ++j; while (j < l) { int c2 = s.charAt(j); if (c2 == opener || c2 == '\n' && opener == '\'') { // allow multi-line strings, but not for ' ++j; break; } else if (c2 == '\\' && j+1 < l) j += 2; else ++j; } } else if (Character.isJavaIdentifierStart(c)) do ++j; while (j < l && Character.isJavaIdentifierPart(s.charAt(j))); else if (Character.isDigit(c)) { do ++j; while (j < l && Character.isDigit(s.charAt(j))); if (j < l && s.charAt(j) == 'L') ++j; // Long constants like 1L } else ++j; tok.add(javaTok_substringC(s, i, j)); ++n; i = j; } if ((tok.size() % 2) == 0) tok.add(""); return tok; } static Map javaTokForJFind_array_cache = synchronizedMRUCache(1000); static String[] javaTokForJFind_array(String s) { String[] tok = javaTokForJFind_array_cache.get(s); if (tok == null) javaTokForJFind_array_cache.put(s, tok = codeTokensAsStringArray(jfind_preprocess(javaTok(s)))); return tok; } static int findCodeTokens(List tok, String... tokens) { return findCodeTokens(tok, 1, false, tokens); } static int findCodeTokens(List tok, boolean ignoreCase, String... tokens) { return findCodeTokens(tok, 1, ignoreCase, tokens); } static int findCodeTokens(List tok, int startIdx, boolean ignoreCase, String... tokens) { return findCodeTokens(tok, startIdx, ignoreCase, tokens, null); } static HashSet findCodeTokens_specials = lithashset("*", "", "", "", "\\*"); static int findCodeTokens_bails, findCodeTokens_nonbails; static interface findCodeTokens_Matcher { boolean get(String token); } static int findCodeTokens(List tok, int startIdx, boolean ignoreCase, String[] tokens, Object condition) { int end = tok.size()-tokens.length*2+2, nTokens = tokens.length; int i = startIdx | 1; // bail out early if first token not found (works great with IndexedList) String firstToken = tokens[0]; if (!ignoreCase && !findCodeTokens_specials.contains(firstToken)) { // quickly scan for first token while (i < end && !firstToken.equals(tok.get(i))) i += 2; } findCodeTokens_Matcher[] matchers = new findCodeTokens_Matcher[nTokens]; for (int j = 0; j < nTokens; j++) { String p = tokens[j]; findCodeTokens_Matcher matcher; if (p.equals("*")) matcher = t -> true; else if (p.equals("")) matcher = t -> isQuoted(t); else if (p.equals("")) matcher = t -> isIdentifier(t); else if (p.equals("")) matcher = t -> isInteger(t); else if (p.equals("\\*")) matcher = t -> t.equals("*"); else if (ignoreCase) matcher = t -> eqic(p, t); else matcher = t -> t.equals(p); matchers[j] = matcher; } outer: for (; i < end; i += 2) { for (int j = 0; j < nTokens; j++) if (!matchers[j].get(tok.get(i+j*2))) continue outer; if (condition == null || checkTokCondition(condition, tok, i-1)) // pass N index return i; } return -1; } // "$1" is first code token, "$2" second code token etc. static String jreplaceExpandRefs(String s, List tokref) { if (!contains(s, '$')) return s; List tok = javaTok(s); for (int i = 1; i < l(tok); i += 2) { String t = tok.get(i); if (t.startsWith("$") && isInteger(t.substring(1))) { String x = tokref.get(-1+parseInt(t.substring(1))*2); tok.set(i, x); } else if (t.equals("\\")) { tok.set(i, ""); i += 2; } } return join(tok); } static void clearAllTokens(List tok) { for (int i = 0; i < tok.size(); i++) tok.set(i, ""); } static void clearAllTokens(List tok, int i, int j) { for (; i < j; i++) tok.set(i, ""); } static List reTok(List tok) { replaceCollection(tok, javaTok(tok)); return tok; } static List reTok(List tok, int i) { return reTok(tok, i, i+1); } static List reTok(List tok, int i, int j) { // extend i to an "N" token // and j to "C" (so j-1 is an "N" token) i = max(i & ~1, 0); j = min(l(tok), j | 1); if (i >= j) return tok; List t = javaTok(joinSubList(tok, i, j)); replaceListPart(tok, i, j, t); // fallback to safety // reTok(tok); return tok; } static List reTok(List tok, IntRange r) { if (r != null) reTok(tok, r.start, r.end); return tok; } static void quoteToPrintWriter(String s, PrintWriter out) { if (s == null) { out.print("null"); return; } out.print('"'); int l = s.length(); for (int i = 0; i < l; i++) { char c = s.charAt(i); if (c == '\\' || c == '"') { out.print('\\'); out.print(c); } else if (c == '\r') out.print("\\r"); else if (c == '\n') out.print("\\n"); else if (c == '\0') out.print("\\0"); else out.print(c); } out.print('"'); } static String quoteCharacter(char c) { if (c == '\'') return "'\\''"; if (c == '\\') return "'\\\\'"; if (c == '\r') return "'\\r'"; if (c == '\n') return "'\\n'"; if (c == '\t') return "'\\t'"; return "'" + c + "'"; } static boolean isCISet_gen(Iterable l) { return l instanceof TreeSet && className(((TreeSet) l).comparator()).contains("CIComp"); } static boolean isJavaXClassName(String s) { return startsWithOneOf(s, "main$", "loadableUtils."); } static List unwrapSynchronizedList(List l) { if (eqOneOf(className(l), "java.util.Collections$SynchronizedList", "java.util.Collections$SynchronizedRandomAccessList")) return (List) get_raw(l, "list"); return l; } static boolean isCIMap_gen(Map map) { return map instanceof TreeMap && className(((TreeMap) map).comparator()).contains("CIComp"); } static Map unwrapSynchronizedMap(Map map) { if (eqOneOf(className(map), "java.util.Collections$SynchronizedMap", "java.util.Collections$SynchronizedSortedMap", "java.util.Collections$SynchronizedNavigableMap")) return (Map) get_raw(map, "m"); return map; } static String boolArrayToHex(boolean[] a) { return bytesToHex(boolArrayToBytes(a)); } static String shortDynamicClassName(Object o) { if (o instanceof DynamicObject && ((DynamicObject) o).className != null) return ((DynamicObject) o).className; return shortClassName(o); } static int stdcompare(Number a, Number b) { return cmp(a, b); } static int stdcompare(String a, String b) { return cmp(a, b); } static int stdcompare(long a, long b) { return a < b ? -1 : a > b ? 1 : 0; } static int stdcompare(Object a, Object b) { return cmp(a, b); } static Map getDeclaredFields_cache = newDangerousWeakHashMap(); static Field[] getDeclaredFields_cached(Class c) { Field[] fields; synchronized(getDeclaredFields_cache) { fields = getDeclaredFields_cache.get(c); if (fields == null) { getDeclaredFields_cache.put(c, fields = c.getDeclaredFields()); for (Field f : fields) makeAccessible(f); } } return fields; } static boolean startsWithDigit(String s) { return nempty(s) && isDigit(s.charAt(0)); } static int countDots(String s) { int n = l(s), count = 0; for (int i = 0; i < n; i++) if (s.charAt(i) == '.') ++count; return count; } static betterCIComparator_C betterCIComparator_instance; static betterCIComparator_C betterCIComparator() { if (betterCIComparator_instance == null) betterCIComparator_instance = new betterCIComparator_C(); return betterCIComparator_instance; } final static class betterCIComparator_C implements Comparator { public int compare(String s1, String s2) { if (s1 == null) return s2 == null ? 0 : -1; if (s2 == null) return 1; int n1 = s1.length(); int n2 = s2.length(); int min = Math.min(n1, n2); for (int i = 0; i < min; i++) { char c1 = s1.charAt(i); char c2 = s2.charAt(i); if (c1 != c2) { c1 = Character.toUpperCase(c1); c2 = Character.toUpperCase(c2); if (c1 != c2) { c1 = Character.toLowerCase(c1); c2 = Character.toLowerCase(c2); if (c1 != c2) { // No overflow because of numeric promotion return c1 - c2; } } } } return n1 - n2; } } static Object[] toObjectArray(Collection c) { List l = asList(c); return l.toArray(new Object[l.size()]); } static Class run(String progID, String... args) { Class main = hotwire(progID); callMain(main, args); return main; } static void restart() { Object j = getJavaX(); call(j, "cleanRestart", get(j, "fullArgs")); } static TreeSet toCaseInsensitiveSet_treeSet(Iterable c) { if (isCISet(c)) return (TreeSet) c; TreeSet set = caseInsensitiveSet_treeSet(); addAll(set, c); return set; } static TreeSet toCaseInsensitiveSet_treeSet(String... x) { TreeSet set = caseInsensitiveSet_treeSet(); addAll(set, x); return set; } static Map synchronizedMRUCache(int maxSize) { return synchroMap(new MRUCache(maxSize)); } static String[] codeTokensAsStringArray(List tok) { int n = max(0, (l(tok)-1)/2); String[] out = new String[n]; for (int i = 0; i < n; i++) out[i] = tok.get(i*2+1); return out; } static int jfind(String s, String in) { return jfind(javaTok(s), in); } static int jfind(List tok, String in) { return jfind(tok, 1, in); } static int jfind(List tok, int startIdx, String in) { return jfind(tok, startIdx, in, null); } static int jfind(List tok, String in, Object condition) { return jfind(tok, 1, in, condition); } static int jfind(List tok, int startIdx, String in, Object condition) { //LS tokin = jfind_preprocess(javaTok(in)); return jfind(tok, startIdx, javaTokForJFind_array(in), condition); } // assumes you preprocessed tokin static int jfind(List tok, List tokin) { return jfind(tok, 1, tokin); } static int jfind(List tok, int startIdx, List tokin) { return jfind(tok, startIdx, tokin, null); } static int jfind(List tok, int startIdx, String[] tokinC, Object condition) { return findCodeTokens(tok, startIdx, false, tokinC, condition); } static int jfind(List tok, int startIdx, List tokin, Object condition) { return jfind(tok, startIdx, codeTokensAsStringArray(tokin), condition); } static List jfind_preprocess(List tok) { for (String type : litlist("quoted", "id", "int")) replaceSublist(tok, ll("<", "", type, "", ">"), ll("<" + type + ">")); replaceSublist(tok, ll("\\", "", "*"), ll("\\*")); return tok; } // supports the usual quotings (", variable length double brackets) except ' quoting static boolean isQuoted(String s) { if (isNormalQuoted(s)) return true; // use the exact version return isMultilineQuoted(s); } static boolean checkTokCondition(Object condition, List tok, int i) { if (condition instanceof TokCondition) return ((TokCondition) condition).get(tok, i); return checkCondition(condition, tok, i); } static void replaceCollection(Collection dest, Collection src) { if (dest == src) return; dest.clear(); dest.addAll(src); } static String joinSubList(List l, int i, int j) { return join(subList(l, i, j)); } static String joinSubList(List l, int i) { return join(subList(l, i)); } static String joinSubList(List l, IntRange r) { return r == null ? null : joinSubList(l, r.start, r.end); } static void replaceListPart(List l, int i, int j, List l2) { replaceSublist(l, i, j, l2); } static byte[] boolArrayToBytes(boolean[] a) { byte[] b = new byte[(l(a)+7)/8]; for (int i = 0; i < l(a); i++) if (a[i]) b[i/8] |= 1 << (i & 7); return b; } static boolean isDigit(char c) { return Character.isDigit(c); } static boolean isCISet(Iterable l) { return l instanceof TreeSet && ((TreeSet) l).comparator() == caseInsensitiveComparator(); } static ArrayList litlist(A... a) { ArrayList l = new ArrayList(a.length); for (A x : a) l.add(x); return l; } // syntax 1: replace all occurrences of x in l with y static List replaceSublist(List l, List x, List y) { if (x == null) return l; int i = 0; while (true) { i = indexOfSubList(l, x, i); if (i < 0) break; replaceSublist(l, i, i+l(x), y); i += l(y); } return l; } // syntax 2: splice l at fromIndex-toIndex and replace middle part with y static List replaceSublist(List l, int fromIndex, int toIndex, List y) { int n = y.size(), toIndex_new = fromIndex+n; if (toIndex_new < toIndex) { removeSubList(l, toIndex_new, toIndex); copyListPart(y, 0, l, fromIndex, n); } else { copyListPart(y, 0, l, fromIndex, toIndex-fromIndex); if (toIndex_new > toIndex) l.addAll(toIndex, subList(y, toIndex-fromIndex)); } return l; } static List replaceSublist(List l, IntRange r, List y) { return replaceSublist(l, r.start, r.end, y); } static boolean isNormalQuoted(String s) { int l = l(s); if (!(l >= 2 && s.charAt(0) == '"' && lastChar(s) == '"')) return false; int j = 1; while (j < l) if (s.charAt(j) == '"') return j == l-1; else if (s.charAt(j) == '\\' && j+1 < l) j += 2; else ++j; return false; } static boolean isMultilineQuoted(String s) { if (!startsWith(s, "[")) return false; int i = 1; while (i < s.length() && s.charAt(i) == '=') ++i; return i < s.length() && s.charAt(i) == '['; } static boolean checkCondition(Object condition, Object... args) { return isTrue(callF(condition, args)); } static boolean checkCondition(IF1 condition, A arg) { return isTrue(callF(condition, arg)); } static int indexOfSubList(List x, List y) { return indexOfSubList(x, y, 0); } static int indexOfSubList(List x, List y, int i) { outer: for (; i+l(y) <= l(x); i++) { for (int j = 0; j < l(y); j++) if (neq(x.get(i+j), y.get(j))) continue outer; return i; } return -1; } static int indexOfSubList(List x, A[] y, int i) { outer: for (; i+l(y) <= l(x); i++) { for (int j = 0; j < l(y); j++) if (neq(x.get(i+j), y[j])) continue outer; return i; } return -1; } static void copyListPart(List a, int i1, List b, int i2, int n) { if (a == null || b == null) return; for (int i = 0; i < n; i++) b.set(i2+i, a.get(i1+i)); } // elements are put to front when added (not when accessed) static class MRUCache extends LinkedHashMap { int maxSize = 10; MRUCache() {} MRUCache(int maxSize) { this.maxSize = maxSize;} protected boolean removeEldestEntry(Map.Entry eldest) { return size() > maxSize; } Object _serialize() { return ll(maxSize, cloneLinkedHashMap(this)); } static MRUCache _deserialize(List l) { MRUCache m = new MRUCache(); m.maxSize = (int) first(l); m.putAll((LinkedHashMap) second(l)); return m; } }static class QuickException extends RuntimeException { public Throwable fillInStackTrace() { return this; } QuickException() {} QuickException(Throwable e) { super(e); } QuickException(String msg) { super(msg); } QuickException(String msg, Throwable e) { super(msg, e); } }static abstract class CloseableIterableIterator extends IterableIterator implements AutoCloseable { public void close() throws Exception {} }static interface Producer { public A next(); // null when end }static ThreadLocal DynamicObject_loading = new ThreadLocal(); static class DynamicObject { String className; // just the name, without the "main$" LinkedHashMap fieldValues; DynamicObject() {} // className = just the name, without the "main$" DynamicObject(String className) { this.className = className;} Map _map() { return fieldValues; } public String toString() { return getClass() == DynamicObject.class ? "dyn " + className : super.toString(); } }static class Var implements IVar { Var() {} Var(A v) { this.v = v;} A v; // you can access this directly if you use one thread public synchronized void set(A a) { if (v != a) { v = a; notifyAll(); } } public synchronized A get() { return v; } public synchronized boolean has() { return v != null; } public synchronized void clear() { v = null; } public String toString() { return str(get()); } }static class DefunctClassLoader {}static interface IntComparator { int compare(int a, int b); }static abstract class TokCondition { abstract boolean get(List tok, int i); // i = N Index }static class NotifyingBlockingThreadPoolExecutor extends ThreadPoolExecutor { private AtomicInteger tasksInProcess = new AtomicInteger(); private Synchronizer synchronizer = new Synchronizer(); public NotifyingBlockingThreadPoolExecutor(int poolSize, int queueSize, long keepAliveTime, TimeUnit keepAliveTimeUnit, long maxBlockingTime, TimeUnit maxBlockingTimeUnit, Callable blockingTimeCallback) { super(poolSize, // Core size poolSize, // Max size keepAliveTime, keepAliveTimeUnit, new ArrayBlockingQueue(Math.max(poolSize, queueSize)), new BlockThenRunPolicy(maxBlockingTime, maxBlockingTimeUnit, blockingTimeCallback)); super.allowCoreThreadTimeOut(true); } public NotifyingBlockingThreadPoolExecutor(int poolSize, int queueSize, long keepAliveTime, TimeUnit unit) { super(poolSize, // Core size poolSize, // Max size keepAliveTime, unit, new ArrayBlockingQueue(Math.max(poolSize, queueSize)), // not smaller than the poolSize (to avoid redundant threads) new BlockThenRunPolicy()); // When super invokes the reject method this class will ensure a blocking try. super.allowCoreThreadTimeOut(true); // Time out the core threads. } @Override public void execute(Runnable task) { // count a new task in process tasksInProcess.incrementAndGet(); try { super.execute(task); } catch(RuntimeException e) { // specifically handle RejectedExecutionException tasksInProcess.decrementAndGet(); throw e; } catch(Error e) { tasksInProcess.decrementAndGet(); throw e; } } @Override protected void afterExecute(Runnable r, Throwable t) { super.afterExecute(r, t); synchronized(this) { tasksInProcess.decrementAndGet(); if (tasksInProcess.intValue() == 0) { synchronizer.signalAll(); } } } @Override public void setCorePoolSize(int corePoolSize) { super.setCorePoolSize(corePoolSize); super.setMaximumPoolSize(corePoolSize); } @Override public void setMaximumPoolSize(int maximumPoolSize) { throw new UnsupportedOperationException("setMaximumPoolSize is not supported."); } public void setRejectedExecutionHandler(RejectedExecutionHandler handler) { throw new UnsupportedOperationException("setRejectedExecutionHandler is not allowed on this class."); } public void await() throws InterruptedException { synchronizer.await(); } public boolean await(long timeout, TimeUnit timeUnit) throws InterruptedException { return synchronizer.await(timeout, timeUnit); } private class Synchronizer { private final Lock lock = new ReentrantLock(); private final Condition done = lock.newCondition(); private boolean isDone = false; private void signalAll() { lock.lock(); try { isDone = true; done.signalAll(); } finally { lock.unlock(); } } public void await() throws InterruptedException { lock.lock(); try { while (!isDone) { done.await(); } } finally { isDone = false; lock.unlock(); } } public boolean await(long timeout, TimeUnit timeUnit) throws InterruptedException { boolean await_result = false; lock.lock(); boolean localIsDone = false; try { await_result = done.await(timeout, timeUnit); } finally { localIsDone = isDone; isDone = false; lock.unlock(); } return await_result && localIsDone; } } private static class BlockThenRunPolicy implements RejectedExecutionHandler { private long maxBlockingTime; private TimeUnit maxBlockingTimeUnit; private Callable blockingTimeCallback; public BlockThenRunPolicy(long maxBlockingTime, TimeUnit maxBlockingTimeUnit, Callable blockingTimeCallback) { this.maxBlockingTime = maxBlockingTime; this.maxBlockingTimeUnit = maxBlockingTimeUnit; this.blockingTimeCallback = blockingTimeCallback; } public BlockThenRunPolicy() { } @Override public void rejectedExecution(Runnable task, ThreadPoolExecutor executor) { BlockingQueue workQueue = executor.getQueue(); boolean taskSent = false; while (!taskSent) { if (executor.isShutdown()) { throw new RejectedExecutionException( "ThreadPoolExecutor has shutdown while attempting to offer a new task."); } try { if(blockingTimeCallback != null) { if (workQueue.offer(task, maxBlockingTime, maxBlockingTimeUnit)) { taskSent = true; } else { // task was not accepted - call the Callback Boolean result = null; try { result = blockingTimeCallback.call(); } catch(Exception e) { throw new RejectedExecutionException(e); } if(result == false) { throw new RejectedExecutionException("User decided to stop waiting for task insertion"); } else { continue; } } } else { workQueue.put(task); taskSent = true; } } catch (InterruptedException e) { } } } } } static interface IVar extends IF0 { void set(A a); A get(); default boolean has() { return get() != null; } default void clear() { set(null); } } static LinkedHashMap cloneLinkedHashMap(Map map) { return map == null ? new LinkedHashMap() : new LinkedHashMap(map); } }