Warning : session_start(): open(/var/lib/php/sessions/sess_3k5bv81qvggnr7aq2rgetqikt7, O_RDWR) failed: No space left on device (28) in /var/www/tb-usercake/models/config.php on line 51 Warning : session_start(): Failed to read session data: files (path: /var/lib/php/sessions) in /var/www/tb-usercake/models/config.php on line 51 i = returnTimed(() -> wikify.anyIndexOf(" bla bla bla", "bla"));
print(wikify.slowSearches + " | " + i);
sleep(100);
}
}
static Q startQ() {
return new Q();
}
static Q startQ(String name) {
return new Q(name);
}
static boolean empty(Collection c) { return c == null || c.isEmpty(); }
static boolean empty(Iterable c) { return c == null || !c.iterator().hasNext(); }
static boolean empty(CharSequence s) { return s == null || s.length() == 0; }
static boolean empty(Map map) { return map == null || map.isEmpty(); }
static boolean empty(Object[] o) { return o == null || o.length == 0; }
static boolean empty(Object o) {
if (o instanceof Collection) return empty((Collection) o);
if (o instanceof String) return empty((String) o);
if (o instanceof Map) return empty((Map) o);
if (o instanceof Object[]) return empty((Object[]) o);
if (o instanceof byte[]) return empty((byte[]) o);
if (o == null) return true;
throw fail("unknown type for 'empty': " + getType(o));
}
static boolean empty(Iterator i) { return i == null || !i.hasNext(); }
static boolean empty(double[] a) { return a == null || a.length == 0; }
static boolean empty(float[] a) { return a == null || a.length == 0; }
static boolean empty(int[] a) { return a == null || a.length == 0; }
static boolean empty(long[] a) { return a == null || a.length == 0; }
static boolean empty(byte[] a) { return a == null || a.length == 0; }
static boolean empty(short[] a) { return a == null || a.length == 0; }
static boolean empty(MultiMap mm) { return mm == null || mm.isEmpty(); }
static boolean empty(File f) { return getFileSize(f) == 0; }
static boolean empty(IntRange r) { return r == null || r.empty(); }
static boolean empty(IntBuffer b) { return b == null || b.isEmpty(); }
static boolean empty(LongBuffer b) { return b == null || b.isEmpty(); }
static int indexOf(List l, A a, int startIndex) {
if (l == null) return -1;
int n = l(l);
for (int i = startIndex; i < n; i++)
if (eq(l.get(i), a))
return i;
return -1;
}
static int indexOf(List l, int startIndex, A a) {
return indexOf(l, a, startIndex);
}
static int indexOf(List l, A a) {
if (l == null) return -1;
return l.indexOf(a);
}
static int indexOf(String a, String b) {
return a == null || b == null ? -1 : a.indexOf(b);
}
static int indexOf(String a, String b, int i) {
return a == null || b == null ? -1 : a.indexOf(b, i);
}
static int indexOf(String a, char b) {
return a == null ? -1 : a.indexOf(b);
}
static int indexOf(String a, int i, char b) {
return indexOf(a, b, i);
}
static int indexOf(String a, char b, int i) {
return a == null ? -1 : a.indexOf(b, i);
}
static int indexOf(String a, int i, String b) {
return a == null || b == null ? -1 : a.indexOf(b, i);
}
static int indexOf(A[] x, A a) {
int n = l(x);
for (int i = 0; i < n; i++)
if (eq(x[i], a))
return i;
return -1;
}
static volatile StringBuffer local_log = new StringBuffer(); // not redirected
static volatile Appendable print_log = local_log; // might be redirected, e.g. to main bot
// in bytes - will cut to half that
static volatile int print_log_max = 1024*1024;
static volatile int local_log_max = 100*1024;
static boolean print_silent = false; // total mute if set
static Object print_byThread_lock = new Object();
static volatile ThreadLocal print_byThread; // special handling by thread - prefers F1
static volatile Object print_allThreads;
static volatile Object print_preprocess;
static void print() {
print("");
}
static A print(String s, A o) {
print((endsWithLetterOrDigit(s) ? s + ": " : s) + o);
return o;
}
// slightly overblown signature to return original object...
static A print(A o) {
ping_okInCleanUp();
if (print_silent) return o;
String s = o + "\n";
print_noNewLine(s);
return o;
}
static void print_noNewLine(String s) {
Object f = getThreadLocal(print_byThread_dontCreate());
if (f == null) f = print_allThreads;
if (f != null)
// We do need the general callF machinery here as print_byThread is sometimes shared between modules
if (isFalse(
f instanceof F1 ? ((F1) f).get(s) :
callF(f, s))) return;
print_raw(s);
}
static void print_raw(String s) {
if (print_preprocess != null) s = (String) callF(print_preprocess, s);
s = fixNewLines(s);
Appendable loc = local_log;
Appendable buf = print_log;
int loc_max = print_log_max;
if (buf != loc && buf != null) {
print_append(buf, s, print_log_max);
loc_max = local_log_max;
}
if (loc != null)
print_append(loc, s, loc_max);
System.out.print(s);
vmBus_send("printed",mc(), s);
}
static void print_autoRotate() {
}
static String nChars(long n) { return n2(n, "char"); }
static String nChars(String s) { return nChars(l(s)); }
static String quote(Object o) {
if (o == null) return "null";
return quote(str(o));
}
static String quote(String s) {
if (s == null) return "null";
StringBuilder out = new StringBuilder((int) (l(s)*1.5+2));
quote_impl(s, out);
return out.toString();
}
static void quote_impl(String s, StringBuilder out) {
out.append('"');
int l = s.length();
for (int i = 0; i < l; i++) {
char c = s.charAt(i);
if (c == '\\' || c == '"')
out.append('\\').append(c);
else if (c == '\r')
out.append("\\r");
else if (c == '\n')
out.append("\\n");
else if (c == '\t')
out.append("\\t");
else if (c == '\0')
out.append("\\0");
else
out.append(c);
}
out.append('"');
}
static int shorten_default = 100;
static String shorten(CharSequence s) { return shorten(s, shorten_default); }
static String shorten(CharSequence s, int max) {
return shorten(s, max, "...");
}
static String shorten(CharSequence s, int max, String shortener) {
if (s == null) return "";
if (max < 0) return str(s);
return s.length() <= max ? str(s) : subCharSequence(s, 0, min(s.length(), max-l(shortener))) + shortener;
}
static String shorten(int max, CharSequence s) { return shorten(s, max); }
static LCSearcher_v4_multiFile wikifyString(String s) {
LineCompCompressor comp = lcFullCompressText(s, "toUpper" , true);
LineCompReader reader = lineCompReaderFromCompressor(comp);
comp = null;
LCSearcher_v4_compact searcher4 = lcSearcher4compactFromLCReader(reader);
reader = null;
// Wouldn't need the temporary file if we modified the save function to write to RAM
File indexFile = createTempFileWithExtension(".qs3");
AutoCloseable __1 = tempDeleteFile(indexFile); try {
save_LCSearcher_v1_compact_toQuickFile_v3(searcher4.lc, indexFile);
searcher4 = null;
RAMByteMemory64 mem = new RAMByteMemory64(indexFile); // this loads the file, so we can release it
LCSearcher_v1_onDisk64_v3 lc_v3 = new LCSearcher_v1_onDisk64_v3(mem);
LCSearcher_v4_multiFile mf = new LCSearcher_v4_multiFile(lc_v3);
mf.getMem = () -> mem;
mf.queriesToUpper = true;
return mf;
} finally { _close(__1); }}
static RuntimeException rethrow(Throwable t) {
if (t instanceof Error)
_handleError((Error) t);
throw t instanceof RuntimeException ? (RuntimeException) t : new RuntimeException(t);
}
static RuntimeException rethrow(String msg, Throwable t) {
throw new RuntimeException(msg, t);
}
static Timed returnTimed(IF0 f) {
long time = sysNow();
A a = f.get();
return new Timed(a, sysNow()-time);
}
static volatile boolean sleep_noSleep = false;
static void sleep(long ms) {
ping();
if (ms < 0) return;
// allow spin locks
if (isAWTThread() && ms > 100) throw fail("Should not sleep on AWT thread");
try {
Thread.sleep(ms);
} catch (Exception e) { throw new RuntimeException(e); }
}
static void sleep() { try {
if (sleep_noSleep) throw fail("nosleep");
print("Sleeping.");
sleepQuietly();
} catch (Exception __e) { throw rethrow(__e); } }
static AutoCloseable tempInterceptPrintIfNotIntercepted(F1 f) {
return print_byThread().get() == null ? tempInterceptPrint(f) : null;
}
static RuntimeException fail() { throw new RuntimeException("fail"); }
static RuntimeException fail(Throwable e) { throw asRuntimeException(e); }
static RuntimeException fail(Object msg) { throw new RuntimeException(String.valueOf(msg)); }
static RuntimeException fail(String msg) { throw new RuntimeException(msg == null ? "" : msg); }
static RuntimeException fail(String msg, Throwable innerException) { throw new RuntimeException(msg, innerException); }
static String getType(Object o) {
return getClassName(o);
}
static long getFileSize(String path) {
return path == null ? 0 : new File(path).length();
}
static long getFileSize(File f) {
return f == null ? 0 : f.length();
}
static int l(Object[] a) { return a == null ? 0 : a.length; }
static int l(boolean[] a) { return a == null ? 0 : a.length; }
static int l(byte[] a) { return a == null ? 0 : a.length; }
static int l(short[] a) { return a == null ? 0 : a.length; }
static int l(long[] a) { return a == null ? 0 : a.length; }
static int l(int[] a) { return a == null ? 0 : a.length; }
static int l(float[] a) { return a == null ? 0 : a.length; }
static int l(double[] a) { return a == null ? 0 : a.length; }
static int l(char[] a) { return a == null ? 0 : a.length; }
static int l(Collection c) { return c == null ? 0 : c.size(); }
static int l(Iterator i) { return iteratorCount_int_close(i); } // consumes the iterator && closes it if possible
static int l(Map m) { return m == null ? 0 : m.size(); }
static int l(CharSequence s) { return s == null ? 0 : s.length(); }
static long l(File f) { return f == null ? 0 : f.length(); }
static int l(Object o) {
return o == null ? 0
: o instanceof String ? l((String) o)
: o instanceof Map ? l((Map) o)
: o instanceof Collection ? l((Collection) o)
: o instanceof Object[] ? l((Object[]) o)
: o instanceof boolean[] ? l((boolean[]) o)
: o instanceof byte[] ? l((byte[]) o)
: o instanceof char[] ? l((char[]) o)
: o instanceof short[] ? l((short[]) o)
: o instanceof int[] ? l((int[]) o)
: o instanceof float[] ? l((float[]) o)
: o instanceof double[] ? l((double[]) o)
: o instanceof long[] ? l((long[]) o)
: (Integer) call(o, "size");
}
static int l(IntRange r) { return r == null ? 0 : r.length(); }
static long l(LongRange r) { return r == null ? 0 : r.length(); }
static int l(IntBuffer b) { return b == null ? 0 : b.size(); }
static int l(LongBuffer b) { return b == null ? 0 : b.size(); }
static boolean eq(Object a, Object b) {
return a == b || a != null && b != null && a.equals(b);
}
static boolean endsWithLetterOrDigit(String s) {
return s != null && s.length() > 0 && Character.isLetterOrDigit(s.charAt(s.length()-1));
}
static void ping_okInCleanUp() {
if (ping_pauseAll || ping_anyActions)
ping_impl(true);
}
// this syntax should be removed...
static Object getThreadLocal(Object o, String name) {
ThreadLocal t = (ThreadLocal) (getOpt(o, name));
return t != null ? t.get() : null;
}
static A getThreadLocal(ThreadLocal tl) {
return tl == null ? null : tl.get();
}
static A getThreadLocal(ThreadLocal tl, A defaultValue) {
return or(getThreadLocal(tl), defaultValue);
}
static ThreadLocal print_byThread_dontCreate() {
return print_byThread;
}
static boolean isFalse(Object o) {
return eq(false, o);
}
static Map> callF_cache = newDangerousWeakHashMap();
static A callF(F0 f) {
return f == null ? null : f.get();
}
static B callF(F1 f, A a) {
return f == null ? null : f.get(a);
}
static A callF(IF0 f) {
return f == null ? null : f.get();
}
static B callF(IF1 f, A a) {
return f == null ? null : f.get(a);
}
static C callF(IF2 f, A a, B b) {
return f == null ? null : f.get(a, b);
}
static void callF(VF1 f, A a) {
if (f != null) f.get(a);
}
static Object callF(Object f, Object... args) {
if (f instanceof String)
return callMCWithVarArgs((String) f, args); // possible SLOWDOWN over callMC
return safeCallF(f, args);
}
static Object safeCallF(Object f, Object... args) {
if (f instanceof Runnable) {
((Runnable) f).run();
return null;
}
if (f == null) return null;
Class c = f.getClass();
ArrayList methods;
synchronized(callF_cache) {
methods = callF_cache.get(c);
if (methods == null)
methods = callF_makeCache(c);
}
int n = l(methods);
if (n == 0) {
throw fail("No get method in " + getClassName(c));
}
if (n == 1) return invokeMethod(methods.get(0), f, args);
for (int i = 0; i < n; i++) {
Method m = methods.get(i);
if (call_checkArgs(m, args, false))
return invokeMethod(m, f, args);
}
throw fail("No matching get method in " + getClassName(c));
}
// used internally
static ArrayList callF_makeCache(Class c) {
ArrayList l = new ArrayList();
Class _c = c;
do {
for (Method m : _c.getDeclaredMethods())
if (m.getName().equals("get")) {
makeAccessible(m);
l.add(m);
}
if (!l.isEmpty()) break;
_c = _c.getSuperclass();
} while (_c != null);
callF_cache.put(c, l);
return l;
}
static String fixNewLines(String s) {
int i = indexOf(s, '\r');
if (i < 0) return s;
int l = s.length();
StringBuilder out = new StringBuilder(l);
out.append(s, 0, i);
for (; i < l; i++) {
char c = s.charAt(i);
if (c != '\r')
out.append(c);
else {
out.append('\n');
if (i+1 < l && s.charAt(i+1) == '\n') ++i;
}
}
return out.toString();
}
static void print_append(Appendable buf, String s, int max) { try {
synchronized(buf) {
buf.append(s);
if (buf instanceof StringBuffer)
rotateStringBuffer(((StringBuffer) buf), max);
else if (buf instanceof StringBuilder)
rotateStringBuilder(((StringBuilder) buf), max);
}
} catch (Exception __e) { throw rethrow(__e); } }
static void vmBus_send(String msg, Object... args) {
Object arg = vmBus_wrapArgs(args);
pcallFAll(vm_busListeners_live(), msg, arg);
pcallFAll(vm_busListenersByMessage_live().get(msg), msg, arg);
}
static void vmBus_send(String msg) {
vmBus_send(msg, (Object) null);
}
static Class mc() {
return main.class;
}
static String n2(long l) { return formatWithThousands(l); }
static String n2(Collection l) { return n2(l(l)); }
static String n2(Map map) { return n2(l(map)); }
static String n2(double l, String singular) {
return n2(l, singular, singular + "s");
}
static String n2(double l, String singular, String plural) {
if (fraction(l) == 0)
return n2((long) l, singular, plural);
else
return l + " " + plural;
}
static String n2(long l, String singular, String plural) {
return n_fancy2(l, singular, plural);
}
static String n2(long l, String singular) {
return n_fancy2(l, singular, singular + "s");
}
static String n2(Collection l, String singular) {
return n2(l(l), singular);
}
static String n2(Collection l, String singular, String plural) {
return n_fancy2(l, singular, plural);
}
static String n2(Map m, String singular, String plural) {
return n_fancy2(m, singular, plural);
}
static String n2(Map m, String singular) {
return n2(l(m), singular);
}
static String n2(Object[] a, String singular) { return n2(l(a), singular); }
static String n2(Object[] a, String singular, String plural) { return n_fancy2(a, singular, plural); }
static String str(Object o) {
return o == null ? "null" : o.toString();
}
static String str(char[] c) {
return new String(c);
}
static CharSequence subCharSequence(CharSequence s, int x) {
return subCharSequence(s, x, s == null ? 0 : s.length());
}
static CharSequence subCharSequence(CharSequence s, int x, int y) {
if (s == null) return null;
if (x < 0) x = 0;
if (x >= s.length()) return "";
if (y < x) y = x;
if (y > s.length()) y = s.length();
return s.subSequence(x, y);
}
static int min(int a, int b) {
return Math.min(a, b);
}
static long min(long a, long b) {
return Math.min(a, b);
}
static float min(float a, float b) { return Math.min(a, b); }
static float min(float a, float b, float c) { return min(min(a, b), c); }
static double min(double a, double b) {
return Math.min(a, b);
}
static double min(double[] c) {
double x = Double.MAX_VALUE;
for (double d : c) x = Math.min(x, d);
return x;
}
static float min(float[] c) {
float x = Float.MAX_VALUE;
for (float d : c) x = Math.min(x, d);
return x;
}
static byte min(byte[] c) {
byte x = 127;
for (byte d : c) if (d < x) x = d;
return x;
}
static short min(short[] c) {
short x = 0x7FFF;
for (short d : c) if (d < x) x = d;
return x;
}
static int min(int[] c) {
int x = Integer.MAX_VALUE;
for (int d : c) if (d < x) x = d;
return x;
}
static LineCompCompressor lcFullCompressText(String text, Object... __) {
String fileName = optPar("fileName",__, "1");
LineCompCompressor compressor = new LineCompCompressor();
compressor.byteMode = true;
compressor.fullCompression = true;
compressor.toUpper = boolPar("toUpper",__);
compressor.balancing = true;
compressor.verify = false;
compressor.loadTexts(litmap(fileName, text));
compressor.run();
return compressor;
}
static LineCompReader lineCompReaderFromCompressor(LineCompCompressor comp) {
return new LineCompReader(bufferedReader(stringReader(comp.asText())));
}
static LCSearcher_v4_compact lcSearcher4compactFromLCReader(LineCompReader lc, Object... __) {
long _startTime_0 = sysNow();
LCSearcher_v1 searcher1 = lcSearcherFromLineCompReader(lc,
(LCSearcher_v1) optPar("searcher1",__));
lc = null;
done2_always("Searcher 1", _startTime_0);
long _startTime_1 = sysNow();
LCSearcher_v1_compact searcher = new LCSearcher_v1_compact();
searcher.big = true; // print more updates
searcher.load(searcher1);
done2_always("Searcher 2", _startTime_1);
return new LCSearcher_v4_compact(searcher);
}
static File createTempFileWithExtension(String ext) {
return createTempFile("", addPrefixIfNempty(".", ext));
}
static AutoCloseable tempDeleteFile(File f) {
return f == null ? null : new AutoCloseable() { public String toString() { return "f.delete();"; } public void close() throws Exception { f.delete(); }};
}
/* makes a .qsearch_v3 file
Improvements in v3:
-Don't write left pairs count for 1-pairs (=90% of pairs)
(can be inferred from entry size)
-Compress occurrence count & prod length
*/
static void save_LCSearcher_v1_compact_toQuickFile_v3(LCSearcher_v1_compact lc, File file) { try {
print("Saving quickFile " + file);
long iPointerArray;
int[] symbolPointers;
{ long _startTime_0 = sysNow();
CountingOutputStream countingStream = new CountingOutputStream(bufferedOutputStream(newFileOutputStream(file))); try {
DataOutputStream out = dataOutputStream(countingStream);
out.write(toUTF8("QUICKSEARCH v3.\n"));
assertEquals(16L, countingStream.getFilePointer());
// write counts
out.writeInt(l(lc.literals));
out.writeInt(l(lc.pairs));
out.writeInt(l(lc.files));
// write raw literals
for (char c : lc.literals)
out.writeChar(c);
if (odd(l(lc.literals))) out.writeChar(0); // align to 4 bytes
// write files
for (int i : lc.files)
out.writeInt(i);
// leave space for symbol pointer array
iPointerArray = countingStream.getFilePointer();
symbolPointers = new int[lc.iFirstFile+1]; // one more to store end of last entry
for (int i = 0; i < l(symbolPointers); i++)
out.writeInt(0);
// write literal + pair infos as compact structs to minimize cache misses
for (int i = 0; i < lc.iFirstFile; i++) {
symbolPointers[i] = (int) countingStream.getFilePointer();
// write pair contents if it's a pair
// no point in compressing these, they are filling the whole int space
// both prod length and occurrences are usually small.
// prod length is <= the file length (100M), so can compress
// into 1 to 4 bytes.
// occurrence count can overflow in rare cases, so we just clip.
if (i >= lc.iFirstPair) {
out.writeLong(lc.pairs[i-lc.iFirstPair]);
out.write(compress30BitUint(lc.getProdLength(i)));
}
// other fields apply to literals and pairs
out.write(compress30BitUint(min(0x3FFFFFFF, lc.occurrences[i])));
int[] idxL = lc.leftPairIndex[i], idxR = lc.rightPairIndex[i];
if (l(idxL) == 1 && l(idxR) == 0)
out.writeInt(idxL[0]);
else if (l(idxL) == 0 && l(idxR) == 1)
out.writeInt(idxR[0] | 0x80000000);
else {
out.write(compress30BitUint(l(idxL)));
for (int val : unnull(idxL)) out.writeInt(val);
// don't have to store idxR length, it's inferred from position of next entry
for (int val : unnull(idxR)) out.writeInt(val);
}
if ((i % oneMillion()) == 0) print(nSymbols(i) + " written");
}
symbolPointers[lc.iFirstFile] = (int) countingStream.getFilePointer();
done2_always("Save quickFile", _startTime_0); } finally { _close(countingStream); }}
{ long _startTime_1 = sysNow();
RandomAccessFile out = new RandomAccessFile(file, "rw"); try {
out.seek(iPointerArray);
writeIntArrayToRandomAccessFile(out, symbolPointers);
done2_always("Save pointer array", _startTime_1); } finally { _close(out); }}
printFileInfo(file);
} catch (Exception __e) { throw rethrow(__e); } }
static void _close(AutoCloseable c) {
if (c != null) try {
c.close();
} catch (Throwable e) {
// Some classes stupidly throw an exception on double-closing
if (c instanceof javax.imageio.stream.ImageOutputStream)
return;
else throw rethrow(e);
}
}
static void _handleError(Error e) {
call(javax(), "_handleError", e);
}
static long sysNow() {
ping();
return System.nanoTime()/1000000;
}
//sbool ping_actions_shareable = true;
static volatile boolean ping_pauseAll = false;
static int ping_sleep = 100; // poll pauseAll flag every 100
static volatile boolean ping_anyActions = false;
static Map ping_actions = newWeakHashMap();
static ThreadLocal ping_isCleanUpThread = new ThreadLocal();
// always returns true
static boolean ping() {
if (ping_pauseAll || ping_anyActions) ping_impl(true /* XXX */);
//ifndef LeanMode ping_impl(); endifndef
return true;
}
// returns true when it slept
static boolean ping_impl(boolean okInCleanUp) { try {
if (ping_pauseAll && !isAWTThread()) {
do
Thread.sleep(ping_sleep);
while (ping_pauseAll);
return true;
}
if (ping_anyActions) { // don't allow sharing ping_actions
if (!okInCleanUp && !isTrue(ping_isCleanUpThread.get()))
failIfUnlicensed();
Object action = null;
synchronized(ping_actions) {
if (!ping_actions.isEmpty()) {
action = ping_actions.get(currentThread());
if (action instanceof Runnable)
ping_actions.remove(currentThread());
if (ping_actions.isEmpty()) ping_anyActions = false;
}
}
if (action instanceof Runnable)
((Runnable) action).run();
else if (eq(action, "cancelled"))
throw fail("Thread cancelled.");
}
return false;
} catch (Exception __e) { throw rethrow(__e); } }
// TODO: test if android complains about this
static boolean isAWTThread() {
if (isAndroid()) return false;
if (isHeadless()) return false;
return isAWTThread_awt();
}
static boolean isAWTThread_awt() {
return SwingUtilities.isEventDispatchThread();
}
static Object sleepQuietly_monitor = new Object();
static void sleepQuietly() { try {
assertFalse(isAWTThread());
synchronized(sleepQuietly_monitor) { sleepQuietly_monitor.wait(); }
} catch (Exception __e) { throw rethrow(__e); } }
static ThreadLocal print_byThread() {
synchronized(print_byThread_lock) {
if (print_byThread == null)
print_byThread = new ThreadLocal();
}
return print_byThread;
}
// f can return false to suppress regular printing
// call print_raw within f to actually print something
static AutoCloseable tempInterceptPrint(F1 f) {
return tempSetThreadLocal(print_byThread(), f);
}
static RuntimeException asRuntimeException(Throwable t) {
if (t instanceof Error)
_handleError((Error) t);
return t instanceof RuntimeException ? (RuntimeException) t : new RuntimeException(t);
}
static String getClassName(Object o) {
return o == null ? "null" : o instanceof Class ? ((Class) o).getName() : o.getClass().getName();
}
static int iteratorCount_int_close(Iterator i) { try {
int n = 0;
if (i != null) while (i.hasNext()) { i.next(); ++n; }
if (i instanceof AutoCloseable) ((AutoCloseable) i).close();
return n;
} catch (Exception __e) { throw rethrow(__e); } }
static Object call(Object o) {
return callF(o);
}
// varargs assignment fixer for a single string array argument
static Object call(Object o, String method, String[] arg) {
return call(o, method, new Object[] {arg});
}
static Object call(Object o, String method, Object... args) {
//ret call_cached(o, method, args);
return call_withVarargs(o, method, args);
}
static Object getOpt(Object o, String field) {
return getOpt_cached(o, field);
}
static Object getOpt(String field, Object o) {
return getOpt_cached(o, field);
}
static Object getOpt_raw(Object o, String field) { try {
Field f = getOpt_findField(o.getClass(), field);
if (f == null) return null;
makeAccessible(f);
return f.get(o);
} catch (Exception __e) { throw rethrow(__e); } }
// access of static fields is not yet optimized
static Object getOpt(Class c, String field) { try {
if (c == null) return null;
Field f = getOpt_findStaticField(c, field);
if (f == null) return null;
makeAccessible(f);
return f.get(null);
} catch (Exception __e) { throw rethrow(__e); } }
static Field getOpt_findStaticField(Class c, String field) {
Class _c = c;
do {
for (Field f : _c.getDeclaredFields())
if (f.getName().equals(field) && (f.getModifiers() & java.lang.reflect.Modifier.STATIC) != 0)
return f;
_c = _c.getSuperclass();
} while (_c != null);
return null;
}
static A or(A a, A b) {
return a != null ? a : b;
}
static Map newDangerousWeakHashMap() {
return _registerDangerousWeakMap(synchroMap(new WeakHashMap()));
}
// initFunction: voidfunc(Map) - is called initially, and after clearing the map
static Map newDangerousWeakHashMap(Object initFunction) {
return _registerDangerousWeakMap(synchroMap(new WeakHashMap()), initFunction);
}
static Object callMCWithVarArgs(String method, Object... args) {
return call_withVarargs(mc(), method, args);
}
static Object invokeMethod(Method m, Object o, Object... args) { try {
try {
return m.invoke(o, args);
} catch (InvocationTargetException e) {
throw rethrow(getExceptionCause(e));
} catch (IllegalArgumentException e) {
throw new IllegalArgumentException(e.getMessage() + " - was calling: " + m + ", args: " + joinWithSpace(classNames(args)));
}
} catch (Exception __e) { throw rethrow(__e); } }
static boolean call_checkArgs(Method m, Object[] args, boolean debug) {
Class[] types = m.getParameterTypes();
if (types.length != args.length) {
if (debug)
print("Bad parameter length: " + args.length + " vs " + types.length);
return false;
}
for (int i = 0; i < types.length; i++) {
Object arg = args[i];
if (!(arg == null ? !types[i].isPrimitive()
: isInstanceX(types[i], arg))) {
if (debug)
print("Bad parameter " + i + ": " + arg + " vs " + types[i]);
return false;
}
}
return true;
}
static Field makeAccessible(Field f) {
try {
f.setAccessible(true);
} catch (Throwable e) {
// Note: The error reporting only works with Java VM option --illegal-access=deny
vmBus_send("makeAccessible_error",e, f);
}
return f;
}
static Method makeAccessible(Method m) {
try {
m.setAccessible(true);
} catch (Throwable e) {
vmBus_send("makeAccessible_error",e, m);
}
return m;
}
static Constructor makeAccessible(Constructor c) {
try {
c.setAccessible(true);
} catch (Throwable e) {
vmBus_send("makeAccessible_error",e, c);
}
return c;
}
static void rotateStringBuffer(StringBuffer buf, int max) { try {
if (buf == null) return;
synchronized(buf) {
if (buf.length() <= max) return;
try {
int newLength = max/2;
int ofs = buf.length()-newLength;
String newString = buf.substring(ofs);
buf.setLength(0);
buf.append("[...] ").append(newString);
} catch (Exception e) {
buf.setLength(0);
}
buf.trimToSize();
}
} catch (Exception __e) { throw rethrow(__e); } }
static void rotateStringBuilder(StringBuilder buf, int max) { try {
if (buf == null) return;
synchronized(buf) {
if (buf.length() <= max) return;
try {
int newLength = max/2;
int ofs = buf.length()-newLength;
String newString = buf.substring(ofs);
buf.setLength(0);
buf.append("[...] ").append(newString);
} catch (Exception e) {
buf.setLength(0);
}
buf.trimToSize();
}
} catch (Exception __e) { throw rethrow(__e); } }
static Object vmBus_wrapArgs(Object... args) {
return empty(args) ? null
: l(args) == 1 ? args[0]
: args;
}
static void pcallFAll(Collection l, Object... args) {
if (l != null) for (Object f : cloneList(l)) pcallF(f, args);
}
static void pcallFAll(Iterator it, Object... args) {
while (it.hasNext()) pcallF(it.next(), args);
}
static Set vm_busListeners_live_cache;
static Set vm_busListeners_live() { if (vm_busListeners_live_cache == null) vm_busListeners_live_cache = vm_busListeners_live_load(); return vm_busListeners_live_cache; }
static Set vm_busListeners_live_load() {
return vm_generalIdentityHashSet("busListeners");
}
static Map vm_busListenersByMessage_live_cache;
static Map vm_busListenersByMessage_live() { if (vm_busListenersByMessage_live_cache == null) vm_busListenersByMessage_live_cache = vm_busListenersByMessage_live_load(); return vm_busListenersByMessage_live_cache; }
static Map vm_busListenersByMessage_live_load() {
return vm_generalHashMap("busListenersByMessage");
}
static String formatWithThousands(long l) {
return formatWithThousandsSeparator(l);
}
static double fraction(double d) {
return d % 1;
}
static String n_fancy2(long l, String singular, String plural) {
return formatWithThousandsSeparator(l) + " " + trim(l == 1 ? singular : plural);
}
static String n_fancy2(Collection l, String singular, String plural) {
return n_fancy2(l(l), singular, plural);
}
static String n_fancy2(Map m, String singular, String plural) {
return n_fancy2(l(m), singular, plural);
}
static String n_fancy2(Object[] a, String singular, String plural) {
return n_fancy2(l(a), singular, plural);
}
static A optPar(ThreadLocal tl, A defaultValue) {
A a = tl.get();
if (a != null) {
tl.set(null);
return a;
}
return defaultValue;
}
static A optPar(ThreadLocal tl) {
return optPar(tl, null);
}
static Object optPar(Object[] params, String name) {
return optParam(params, name);
}
static Object optPar(String name, Object[] params) {
return optParam(params, name);
}
static Object optPar(String name, Map params) {
return optParam(name, params);
}
static A optPar(Object[] params, String name, A defaultValue) {
return optParam(params, name, defaultValue);
}
static A optPar(String name, Object[] params, A defaultValue) {
return optParam(params, name, defaultValue);
}
static String fileName(File f) {
return f == null ? null : f.getName();
}
static boolean boolPar(ThreadLocal tl) {
return boolOptParam(tl);
}
// defaults to false
static boolean boolPar(Object[] __, String name) {
return boolOptParam(__, name);
}
static boolean boolPar(String name, Object[] __) {
return boolOptParam(__, name);
}
static boolean boolPar(String name, Map __) {
return boolOptParam(name, __);
}
static boolean boolPar(String name, Object[] params, boolean defaultValue) {
return optParam(params, name, defaultValue);
}
static String toUpper(String s) {
return s == null ? null : s.toUpperCase();
}
static List toUpper(Collection s) {
return allToUpper(s);
}
static HashMap litmap(Object... x) {
HashMap map = new HashMap();
litmap_impl(map, x);
return map;
}
static void litmap_impl(Map map, Object... x) {
if (x != null) for (int i = 0; i < x.length-1; i += 2)
if (x[i+1] != null)
map.put(x[i], x[i+1]);
}
static BufferedReader bufferedReader(Reader r) { return bufferedReader(r, 8192); }
static BufferedReader bufferedReader(Reader r, int bufSize) {
return r instanceof BufferedReader ? (BufferedReader) r : _registerIOWrap(new BufferedReader(r, bufSize), r);
}
static StringReader stringReader(String s) {
return new StringReader(s);
}
static LCSearcher_v1 lcSearcherFromLineCompReader(LineCompReader lc) { return lcSearcherFromLineCompReader(lc, null); }
static LCSearcher_v1 lcSearcherFromLineCompReader(LineCompReader lc, LCSearcher_v1 searcher) {
if (searcher == null) searcher = new LCSearcher_v1();
searcher.big = true;
searcher.literals = listToHashMap(lmap(__21 -> first(__21),lc.literals));
searcher.iFirstPair = l(lc.literals);
//searcher.productions = repNull(searcher.iFirstPair);
for (long p : lc.pairs.asVirtualList()) {
//searcher.productions.add(intPairToList(longToIntPair(p)));
searcher.addPair(p);
}
searcher.iFirstFile = searcher.iFirstPair+l(lc.pairs);
searcher.fileNames = new HashMap();
for (Map.Entry> __0 : _entrySet( lc.versions))
{ String name = __0.getKey(); List enc = __0.getValue(); searcher.fileNames.put(searcher.addFile(enc), name); }
searcher.prepare();
return searcher;
}
static long done2_always(long startTime, String desc) {
long time = sysNow()-startTime;
// BREAKING CHANGE: Now stores result for lastTiming()
saveTiming_noPrint(time);
print(desc + " [" + time + " ms]");
return time;
}
static long done2_always(String desc, long startTime) {
return done2_always(startTime, desc);
}
static long done2_always(long startTime) {
return done2_always(startTime, "");
}
static File createTempFile() {
return createTempFile("tmp", null);
}
static File createTempFile(String prefix, String suffix) { try {
prefix = nohup_sanitize(prefix);
suffix = nohup_sanitize(suffix);
if (shouldKeepTempFiles())
return mkdirsForFile(javaxCachesDir("Temp/" + prefix + "-" + randomID() + "-" + suffix));
// File.createTempFile needs at least 3 prefix characters, so we
// fíll them up for you
File f = File.createTempFile(takeFirst(10, pad(prefix, 3, '-')), suffix);
f.deleteOnExit();
return f;
} catch (Exception __e) { throw rethrow(__e); } }
static String addPrefixIfNempty(String prefix, String s) {
return addPrefixIfNotEmpty(prefix, s);
}
static BufferedOutputStream bufferedOutputStream(OutputStream out) {
if (out == null) return null;
if (out instanceof BufferedOutputStream) return ((BufferedOutputStream) out);
return new BufferedOutputStream(out, defaultBufferedOutputStreamSize());
}
static FileOutputStream newFileOutputStream(File path) throws IOException {
return newFileOutputStream(path.getPath());
}
static FileOutputStream newFileOutputStream(String path) throws IOException {
return newFileOutputStream(path, false);
}
static FileOutputStream newFileOutputStream(File path, boolean append) throws IOException {
return newFileOutputStream(path.getPath(), append);
}
static FileOutputStream newFileOutputStream(String path, boolean append) throws IOException {
mkdirsForFile(path);
FileOutputStream f = new FileOutputStream(path, append);
_registerIO(f, path, true);
return f;
}
static DataOutputStream dataOutputStream(File f) { try {
return new DataOutputStream(bufferedOutputStream(newFileOutputStream(f)));
} catch (Exception __e) { throw rethrow(__e); } }
static DataOutputStream dataOutputStream(OutputStream out) { try {
return new DataOutputStream(out);
} catch (Exception __e) { throw rethrow(__e); } }
static byte[] toUTF8(String s) {
return toUtf8(s);
}
static A assertEquals(Object x, A y) {
return assertEquals(null, x, y);
}
static A assertEquals(String msg, Object x, A y) {
if (assertVerbose()) return assertEqualsVerbose(msg, x, y);
if (!(x == null ? y == null : x.equals(y)))
throw fail((msg != null ? msg + ": " : "") + y + " != " + x);
return y;
}
static boolean odd(int i) {
return (i & 1) != 0;
}
static boolean odd(long i) {
return (i & 1) != 0;
}
static boolean odd(BigInteger i) { return odd(toInt(i)); }
static byte[] compress30BitUint(int i) {
if (i < 0 || i >= 0x40000000) throw fail("Not a 30 bit uint: " + i);
if (i < 0x40)
return new byte[] { (byte) i };
if (i < 0x4000)
return new byte[] {
(byte) ((i >> 8) | 0x40),
(byte) i
};
if (i < 0x400000)
return new byte[] {
(byte) ((i >> 16) | 0x80),
(byte) (i >> 8),
(byte) i
};
return new byte[] {
(byte) ((i >> 24) | 0xC0),
(byte) (i >> 16),
(byte) (i >> 8),
(byte) i
};
}
static String unnull(String s) {
return s == null ? "" : s;
}
static Collection unnull(Collection l) {
return l == null ? emptyList() : l;
}
static List unnull(List l) { return l == null ? emptyList() : l; }
static int[] unnull(int[] l) { return l == null ? emptyIntArray() : l; }
static char[] unnull(char[] l) { return l == null ? emptyCharArray() : l; }
static double[] unnull(double[] l) { return l == null ? emptyDoubleArray() : l; }
static Map unnull(Map l) {
return l == null ? emptyMap() : l;
}
static Iterable unnull(Iterable i) {
return i == null ? emptyList() : i;
}
static A[] unnull(A[] a) {
return a == null ? (A[]) emptyObjectArray() : a;
}
static BitSet unnull(BitSet b) {
return b == null ? new BitSet() : b;
}
//ifclass Symbol
static Pair unnull(Pair p) {
return p != null ? p : new Pair(null, null);
}
static long unnull(Long l) { return l == null ? 0L : l; }
static int oneMillion() {
return 1000000;
}
static String nSymbols(long n) { return n2(n, "symbol"); }
static String nSymbols(Collection l) { return nSymbols(l(l)); }
static String nSymbols(Map map) { return nSymbols(l(map)); }
static void writeIntArrayToRandomAccessFile(RandomAccessFile raf, int[] array) { try {
for (IntRange r : intRangeChunks(l(array), 1024))
raf.write(intArrayToBytes(subIntArray(array, r)));
} catch (Exception __e) { throw rethrow(__e); } }
static File printFileInfo(File f) { return printFileInfo("", f); }
static File printFileInfo(String s, File f) {
print(s, renderFileInfo(f));
return f;
}
static Class javax() {
return getJavaX();
}
static Map newWeakHashMap() {
return _registerWeakMap(synchroMap(new WeakHashMap()));
}
static boolean isTrue(Object o) {
if (o instanceof Boolean)
return ((Boolean) o).booleanValue();
if (o == null) return false;
if (o instanceof ThreadLocal) // TODO: remove this
return isTrue(((ThreadLocal) o).get());
throw fail(getClassName(o));
}
static void failIfUnlicensed() {
assertTrue("license off", licensed());
}
static Thread currentThread() {
return Thread.currentThread();
}
static int isAndroid_flag;
static boolean isAndroid() {
if (isAndroid_flag == 0)
isAndroid_flag = System.getProperty("java.vendor").toLowerCase().indexOf("android") >= 0 ? 1 : -1;
return isAndroid_flag > 0;
}
static Boolean isHeadless_cache;
static boolean isHeadless() {
if (isHeadless_cache != null) return isHeadless_cache;
if (isAndroid()) return isHeadless_cache = true;
if (GraphicsEnvironment.isHeadless()) return isHeadless_cache = true;
// Also check if AWT actually works.
// If DISPLAY variable is set but no X server up, this will notice.
try {
SwingUtilities.isEventDispatchThread();
return isHeadless_cache = false;
} catch (Throwable e) { return isHeadless_cache = true; }
}
static void assertFalse(Object o) {
if (!(eq(o, false) /*|| isFalse(pcallF(o))*/))
throw fail(str(o));
}
static boolean assertFalse(boolean b) {
if (b) throw fail("oops");
return b;
}
static boolean assertFalse(String msg, boolean b) {
if (b) throw fail(msg);
return b;
}
static AutoCloseable tempSetThreadLocal(final ThreadLocal tl, A a) {
if (tl == null) return null;
final A prev = setThreadLocal(tl, a);
return new AutoCloseable() { public String toString() { return "tl.set(prev);"; } public void close() throws Exception { tl.set(prev); }};
}
static Object call_withVarargs(Object o, String method, Object... args) { try {
if (o == null) return null;
if (o instanceof Class) {
Class c = (Class) o;
_MethodCache cache = callOpt_getCache(c);
Method me = cache.findStaticMethod(method, args);
if (me != null)
return invokeMethod(me, null, args);
// try varargs
List methods = cache.cache.get(method);
if (methods != null) methodSearch: for (Method m : methods) {
{ if (!(m.isVarArgs())) continue; }
{ if (!(isStaticMethod(m))) continue; }
Object[] newArgs = massageArgsForVarArgsCall(m, args);
if (newArgs != null)
return invokeMethod(m, null, newArgs);
}
throw fail("Method " + c.getName() + "." + method + "(" + joinWithComma(classNames(args)) + ") not found");
} else {
Class c = o.getClass();
_MethodCache cache = callOpt_getCache(c);
Method me = cache.findMethod(method, args);
if (me != null)
return invokeMethod(me, o, args);
// try varargs
List methods = cache.cache.get(method);
if (methods != null) methodSearch: for (Method m : methods) {
{ if (!(m.isVarArgs())) continue; }
Object[] newArgs = massageArgsForVarArgsCall(m, args);
if (newArgs != null)
return invokeMethod(m, o, newArgs);
}
throw fail("Method " + c.getName() + "." + method + "(" + joinWithComma(classNames(args)) + ") not found");
}
} catch (Exception __e) { throw rethrow(__e); } }
//static final Map> getOpt_cache = newDangerousWeakHashMap(f getOpt_special_init);
static class getOpt_Map extends WeakHashMap {
getOpt_Map() {
if (getOpt_special == null) getOpt_special = new HashMap();
clear();
}
public void clear() {
super.clear();
//print("getOpt clear");
put(Class.class, getOpt_special);
put(String.class, getOpt_special);
}
}
static final Map> getOpt_cache = _registerDangerousWeakMap(synchroMap(new getOpt_Map()));
//static final Map> getOpt_cache = _registerWeakMap(synchroMap(new getOpt_Map));
static HashMap getOpt_special; // just a marker
/*static void getOpt_special_init(Map map) {
map.put(Class.class, getOpt_special);
map.put(S.class, getOpt_special);
}*/
static Object getOpt_cached(Object o, String field) { try {
if (o == null) return null;
Class c = o.getClass();
HashMap map;
synchronized(getOpt_cache) {
map = getOpt_cache.get(c);
if (map == null)
map = getOpt_makeCache(c);
}
if (map == getOpt_special) {
if (o instanceof Class)
return getOpt((Class) o, field);
/*if (o instanceof S)
ret getOpt(getBot((S) o), field);*/
if (o instanceof Map)
return ((Map) o).get(field);
}
Field f = map.get(field);
if (f != null) return f.get(o);
if (o instanceof DynamicObject)
return mapGet2(((DynamicObject) o).fieldValues, field);
return null;
} catch (Exception __e) { throw rethrow(__e); } }
// used internally - we are in synchronized block
static HashMap getOpt_makeCache(Class c) {
HashMap map;
if (isSubtypeOf(c, Map.class))
map = getOpt_special;
else {
map = new HashMap();
if (!reflection_classesNotToScan().contains(c.getName())) {
Class _c = c;
do {
for (Field f : _c.getDeclaredFields()) {
makeAccessible(f);
String name = f.getName();
if (!map.containsKey(name))
map.put(name, f);
}
_c = _c.getSuperclass();
} while (_c != null);
}
}
if (getOpt_cache != null) getOpt_cache.put(c, map);
return map;
}
static Field getOpt_findField(Class c, String field) {
Class _c = c;
do {
for (Field f : _c.getDeclaredFields())
if (f.getName().equals(field))
return f;
_c = _c.getSuperclass();
} while (_c != null);
return null;
}
static List _registerDangerousWeakMap_preList;
static A _registerDangerousWeakMap(A map) {
return _registerDangerousWeakMap(map, null);
}
static A _registerDangerousWeakMap(A map, Object init) {
callF(init, map);
if (init instanceof String) {
final String f = (String) init;
init = new VF1() { public void get(Map map) { try { callMC(f, map) ; } catch (Exception __e) { throw rethrow(__e); } }
public String toString() { return "callMC(f, map)"; }};
}
if (javax() == null) {
// We're in class init
if (_registerDangerousWeakMap_preList == null) _registerDangerousWeakMap_preList = synchroList();
_registerDangerousWeakMap_preList.add(pair(map, init));
return map;
}
call(javax(), "_registerDangerousWeakMap", map, init);
return map;
}
static void _onLoad_registerDangerousWeakMap() {
assertNotNull(javax());
if (_registerDangerousWeakMap_preList == null) return;
for (Pair p : _registerDangerousWeakMap_preList)
_registerDangerousWeakMap(p.a, p.b);
_registerDangerousWeakMap_preList = null;
}
static Map synchroMap() {
return synchroHashMap();
}
static Map synchroMap(Map map) {
return Collections.synchronizedMap(map);
}
static Throwable getExceptionCause(Throwable e) {
Throwable c = e.getCause();
return c != null ? c : e;
}
static String joinWithSpace(Iterable c) {
return join(" ", c);
}
static String joinWithSpace(String... c) {
return join(" ", c);
}
static List classNames(Collection l) {
return getClassNames(l);
}
static List classNames(Object[] l) {
return getClassNames(Arrays.asList(l));
}
static boolean isInstanceX(Class type, Object arg) {
if (type == boolean.class) return arg instanceof Boolean;
if (type == int.class) return arg instanceof Integer;
if (type == long.class) return arg instanceof Long;
if (type == float.class) return arg instanceof Float;
if (type == short.class) return arg instanceof Short;
if (type == char.class) return arg instanceof Character;
if (type == byte.class) return arg instanceof Byte;
if (type == double.class) return arg instanceof Double;
return type.isInstance(arg);
}
static ArrayList cloneList(Iterable l) {
return l instanceof Collection ? cloneList((Collection) l) : asList(l);
}
static ArrayList cloneList(Collection l) {
if (l == null) return new ArrayList();
synchronized(collectionMutex(l)) {
return new ArrayList (l);
}
}
static Object pcallF(Object f, Object... args) {
return pcallFunction(f, args);
}
static A pcallF(F0 f) { try {
return f == null ? null : f.get();
} catch (Throwable __e) { return null; } }
static B pcallF(F1 f, A a) { try {
return f == null ? null : f.get(a);
} catch (Throwable __e) { return null; } }
static void pcallF(VF1 f, A a) {
try {
if (f != null) f.get(a);
} catch (Throwable __e) { _handleException(__e); }
}
static Set vm_generalIdentityHashSet(Object name) {
synchronized(get(javax(), "generalMap")) {
Set set = (Set) (vm_generalMap_get(name));
if (set == null)
vm_generalMap_put(name, set = syncIdentityHashSet());
return set;
}
}
static Map vm_generalHashMap(Object name) {
synchronized(get(javax(), "generalMap")) {
Map m = (Map) (vm_generalMap_get(name));
if (m == null)
vm_generalMap_put(name, m = syncHashMap());
return m;
}
}
static String formatWithThousandsSeparator(long l) {
return NumberFormat.getInstance(new Locale("en_US")).format(l);
}
static String trim(String s) { return s == null ? null : s.trim(); }
static String trim(StringBuilder buf) { return buf.toString().trim(); }
static String trim(StringBuffer buf) { return buf.toString().trim(); }
static A optParam(ThreadLocal tl, A defaultValue) {
return optPar(tl, defaultValue);
}
static A optParam(ThreadLocal tl) {
return optPar(tl);
}
static Object optParam(String name, Map params) {
return mapGet(params, name);
}
// now also takes a map as single array entry
static A optParam(Object[] opt, String name, A defaultValue) {
int n = l(opt);
if (n == 1 && opt[0] instanceof Map) {
Map map = (Map) (opt[0]);
return map.containsKey(name) ? (A) map.get(name) : defaultValue;
}
if (!even(l(opt))) throw fail("Odd parameter length");
for (int i = 0; i < l(opt); i += 2)
if (eq(opt[i], name))
return (A) opt[i+1];
return defaultValue;
}
static Object optParam(Object[] opt, String name) {
return optParam(opt, name, null);
}
static Object optParam(String name, Object[] params) {
return optParam(params, name);
}
static boolean boolOptParam(ThreadLocal tl) {
return isTrue(optPar(tl));
}
// defaults to false
static boolean boolOptParam(Object[] __, String name) {
return isTrue(optParam(__, name));
}
static boolean boolOptParam(String name, Object[] __) {
return boolOptParam(__, name);
}
static boolean boolOptParam(String name, Map __) {
return isTrue(optPar(name, __));
}
static List allToUpper(Collection l) {
List x = new ArrayList(l(l));
if (l != null) for (String s : l) x.add(upper(s));
return x;
}
static A _registerIOWrap(A wrapper, Object wrapped) {
return wrapper;
}
static HashMap listToHashMap(List l) {
if (l == null) return null;
HashMap map = new HashMap();
int n = l(l);
for (int i = 0; i < n; i++)
map.put(i, l.get(i));
return map;
}
static List lmap(IF1 f, Iterable l) {
return lambdaMap(f, l);
}
static List lmap(IF1 f, A[] l) {
return lambdaMap(f, l);
}
static Object first(Object list) {
return first((Iterable) list);
}
static A first(List list) {
return empty(list) ? null : list.get(0);
}
static A first(A[] bla) {
return bla == null || bla.length == 0 ? null : bla[0];
}
static A first(IterableIterator i) {
return first((Iterator ) i);
}
static A first(Iterator i) {
return i == null || !i.hasNext() ? null : i.next();
}
static A first(Iterable i) {
if (i == null) return null;
Iterator it = i.iterator();
return it.hasNext() ? it.next() : null;
}
static Character first(String s) { return empty(s) ? null : s.charAt(0); }
static Character first(CharSequence s) { return empty(s) ? null : s.charAt(0); }
static A first(Pair p) {
return p == null ? null : p.a;
}
static Byte first(byte[] l) {
return empty(l) ? null : l[0];
}
static int first(IntBuffer buf) {
return buf.get(0);
}
static Set> _entrySet(Map map) {
return map == null ? Collections.EMPTY_SET : map.entrySet();
}
static ThreadLocal saveTiming_last = new ThreadLocal();
static void saveTiming(long ms) {
print(ms + " ms");
saveTiming_noPrint(ms);
}
static void saveTiming_noPrint(long ms) {
saveTiming_last.set(ms);
}
static String nohup_sanitize(String s) {
return empty(s) ? s : takeFirst(50, s.replaceAll("[^.a-zA-Z0-9\\-_]", ""));
}
static IF0 shouldKeepTempFiles;
static boolean shouldKeepTempFiles() { return shouldKeepTempFiles != null ? shouldKeepTempFiles.get() : shouldKeepTempFiles_base(); }
final static boolean shouldKeepTempFiles_fallback(IF0 _f) { return _f != null ? _f.get() : shouldKeepTempFiles_base(); }
static boolean shouldKeepTempFiles_base() {
return false;
}
public static File mkdirsForFile(File file) {
File dir = file.getParentFile();
if (dir != null) { // is null if file is in current dir
dir.mkdirs();
if (!dir.isDirectory())
if (dir.isFile()) throw fail("Please delete the file " + f2s(dir) + " - it is supposed to be a directory!");
else throw fail("Unknown IO exception during mkdirs of " + f2s(file));
}
return file;
}
public static String mkdirsForFile(String path) {
mkdirsForFile(new File(path));
return path;
}
static File javaxCachesDir_dir; // can be set to work on different base dir
static File javaxCachesDir() {
return javaxCachesDir_dir != null ? javaxCachesDir_dir : new File(userHome(), "JavaX-Caches");
}
static File javaxCachesDir(String sub) {
return newFile(javaxCachesDir(), sub);
}
static int randomID_defaultLength = 12;
static String randomID(int length) {
return makeRandomID(length);
}
static String randomID(Random r, int length) {
return makeRandomID(r, length);
}
static String randomID() {
return randomID(randomID_defaultLength);
}
static String randomID(Random r) {
return randomID(r, randomID_defaultLength);
}
static List takeFirst(List l, int n) {
return l(l) <= n ? l : newSubListOrSame(l, 0, n);
}
static List takeFirst(int n, List l) {
return takeFirst(l, n);
}
static String takeFirst(int n, String s) { return substring(s, 0, n); }
static String takeFirst(String s, int n) { return substring(s, 0, n); }
static CharSequence takeFirst(int n, CharSequence s) { return subCharSequence(s, 0, n); }
static List takeFirst(int n, Iterable i) {
if (i == null) return null;
List l = new ArrayList();
Iterator it = i.iterator();
for (int _repeat_0 = 0; _repeat_0 < n; _repeat_0++) { if (it.hasNext()) l.add(it.next()); else break; }
return l;
}
static int[] takeFirst(int n, int[] a) {
return takeFirstOfIntArray(n, a);
}
static String pad(Object s, int l) {
return pad(s, l, ' ');
}
static String pad(Object s, int l, char c) {
String _s = str(s);
if (lengthOfString(_s) >= l) return _s;
return rep(c, l-lengthOfString(_s)) + _s;
}
static String addPrefixIfNotEmpty(String prefix, String s) {
return empty(s) ? "" : prefix + s;
}
static int defaultBufferedOutputStreamSize() {
return 65536;
}
static void _registerIO(Object object, String path, boolean opened) {
}
static byte[] toUtf8(String s) { try {
return s.getBytes(utf8charset());
} catch (Exception __e) { throw rethrow(__e); } }
static ThreadLocal assertVerbose_value = new ThreadLocal();
static void assertVerbose(boolean b) {
assertVerbose_value.set(b);
}
static boolean assertVerbose() { return isTrue(assertVerbose_value.get()); }
static A assertEqualsVerbose(Object x, A y) {
assertEqualsVerbose((String) null, x, y);
return y;
}
// x = expected, y = actual
static A assertEqualsVerbose(String msg, Object x, A y) {
if (!eq(x, y)) {
throw fail((msg != null ? msg + ": " : "") + "expected: "+ x + ", got: " + y);
} else
print("OK" + (empty(msg) ? "" : " " + msg) + ": " + /*sfu*/(x));
return y;
}
static int toInt(Object o) {
if (o == null) return 0;
if (o instanceof Number)
return ((Number) o).intValue();
if (o instanceof String)
return parseInt(((String) o));
if (o instanceof Boolean)
return boolToInt(((Boolean) o));
throw fail("woot not int: " + getClassName(o));
}
static int toInt(long l) {
if (l != (int) l) throw fail("Too large for int: " + l);
return (int) l;
}
static ArrayList emptyList() {
return new ArrayList();
//ret Collections.emptyList();
}
static ArrayList emptyList(int capacity) {
return new ArrayList(max(0, capacity));
}
// Try to match capacity
static ArrayList emptyList(Iterable l) {
return l instanceof Collection ? emptyList(((Collection) l).size()) : emptyList();
}
static ArrayList emptyList(Object[] l) {
return emptyList(l(l));
}
// get correct type at once
static ArrayList emptyList(Class c) {
return new ArrayList();
}
static int[] emptyIntArray_a = new int[0];
static int[] emptyIntArray() { return emptyIntArray_a; }
static char[] emptyCharArray = new char[0];
static char[] emptyCharArray() { return emptyCharArray; }
static double[] emptyDoubleArray = new double[0];
static double[] emptyDoubleArray() { return emptyDoubleArray; }
static Map emptyMap() {
return new HashMap();
}
static Object[] emptyObjectArray_a = new Object[0];
static Object[] emptyObjectArray() { return emptyObjectArray_a; }
static List intRangeChunks(int end, int chunkSize) {
return intRangeChunks(0, end, chunkSize);
}
static List intRangeChunks(int start, int end, int chunkSize) {
List l = new ArrayList();
int i = start;
while (i < end) {
l.add(new IntRange(i, min(end, i+chunkSize)));
i += chunkSize;
}
return l;
}
// gratuitous convenience syntax for processing sound samples
static List intRangeChunks(short[] l, int chunkSize) { return intRangeChunks(l(l), chunkSize); }
static byte[] intArrayToBytes(int[] a) {
byte[] b = new byte[a.length*4];
for (int i = 0; i < a.length; i++)
intToBytes_inArray(a[i], b, i*4);
return b;
}
static int[] subIntArray(int[] b, int start) {
return subIntArray(b, start, l(b));
}
static int[] subIntArray(int[] b, int start, int end) {
start = max(start, 0); end = min(end, l(b));
if (start == 0 && end == l(b)) return b;
if (start >= end) return new int[0];
int[] x = new int[end-start];
System.arraycopy(b, start, x, 0, end-start);
return x;
}
static int[] subIntArray(int[] a, IntRange r) {
return r == null ? null : subIntArray(a, r.start, r.end);
}
static String renderFileInfo(File f) {
return f == null ? "-" : f2s(f) + " "
+ (f.isFile() ? "(file, " + n2(fileSize(f)) + " bytes)"
: f.isDirectory() ? "(dir)" : "(not found)");
}
static Class __javax;
static Class getJavaX() { try {
return __javax;
} catch (Exception __e) { throw rethrow(__e); } }
static List _registerWeakMap_preList;
static A _registerWeakMap(A map) {
if (javax() == null) {
// We're in class init
if (_registerWeakMap_preList == null) _registerWeakMap_preList = synchroList();
_registerWeakMap_preList.add(map);
return map;
}
try {
call(javax(), "_registerWeakMap", map);
} catch (Throwable e) {
printException(e);
print("Upgrade JavaX!!");
}
return map;
}
static void _onLoad_registerWeakMap() {
assertNotNull(javax());
if (_registerWeakMap_preList == null) return;
for (Object o : _registerWeakMap_preList)
_registerWeakMap(o);
_registerWeakMap_preList = null;
}
static void assertTrue(Object o) {
if (!(eq(o, true) /*|| isTrue(pcallF(o))*/))
throw fail(str(o));
}
static boolean assertTrue(String msg, boolean b) {
if (!b)
throw fail(msg);
return b;
}
static boolean assertTrue(boolean b) {
if (!b)
throw fail("oops");
return b;
}
static volatile boolean licensed_yes = true;
static boolean licensed() {
if (!licensed_yes) return false;
ping_okInCleanUp();
return true;
}
static void licensed_off() {
licensed_yes = false;
}
static A setThreadLocal(ThreadLocal tl, A value) {
if (tl == null) return null;
A old = tl.get();
tl.set(value);
return old;
}
static final Map callOpt_cache = newDangerousWeakHashMap();
static Object callOpt_cached(Object o, String methodName, Object... args) { try {
if (o == null) return null;
if (o instanceof Class) {
Class c = (Class) o;
_MethodCache cache = callOpt_getCache(c);
// TODO: (super-rare) case where method exists static and non-static
// with different args
Method me = cache.findMethod(methodName, args);
if (me == null || (me.getModifiers() & Modifier.STATIC) == 0) return null;
return invokeMethod(me, null, args);
} else {
Class c = o.getClass();
_MethodCache cache = callOpt_getCache(c);
Method me = cache.findMethod(methodName, args);
if (me == null) return null;
return invokeMethod(me, o, args);
}
} catch (Exception __e) { throw rethrow(__e); } }
static _MethodCache callOpt_getCache(Class c) {
synchronized(callOpt_cache) {
_MethodCache cache = callOpt_cache.get(c);
if (cache == null)
callOpt_cache.put(c, cache = new _MethodCache(c));
return cache;
}
}
static boolean isStaticMethod(Method m) {
return methodIsStatic(m);
}
static Object[] massageArgsForVarArgsCall(Method m, Object[] args) {
Class[] types = m.getParameterTypes();
int n = types.length-1, nArgs = args.length;
if (nArgs < n) return null;
for (int i = 0; i < n; i++)
if (!argumentCompatibleWithType(args[i], types[i]))
return null;
Class varArgType = types[n].getComponentType();
for (int i = n; i < nArgs; i++)
if (!argumentCompatibleWithType(args[i], varArgType))
return null;
Object[] newArgs = new Object[n+1];
arraycopy(args, 0, newArgs, 0, n);
Object[] varArgs = arrayOfType(varArgType, nArgs-n);
arraycopy(args, n, varArgs, 0, nArgs-n);
newArgs[n] = varArgs;
return newArgs;
}
static String joinWithComma(Collection c) {
return join(", ", c);
}
static String joinWithComma(String... c) {
return join(", ", c);
}
static String joinWithComma(Pair p) {
return p == null ? "" : joinWithComma(str(p.a), str(p.b));
}
static void clear(Collection c) {
if (c != null) c.clear();
}
static void clear(Map map) {
if (map != null) map.clear();
}
static void put(Map map, A a, B b) {
if (map != null) map.put(a, b);
}
static void put(List l, int i, A a) {
if (l != null && i >= 0 && i < l(l)) l.set(i, a);
}
static B mapGet2(Map map, A a) {
return map == null ? null : map.get(a);
}
static B mapGet2(A a, Map map) {
return map == null ? null : map.get(a);
}
static boolean isSubtypeOf(Class a, Class b) {
return b.isAssignableFrom(a); // << always hated that method, let's replace it!
}
static Set reflection_classesNotToScan_value = litset(
"jdk.internal.loader.URLClassPath"
);
static Set reflection_classesNotToScan() {
return reflection_classesNotToScan_value;
}
static HashMap> callMC_cache = new HashMap();
static String callMC_key;
static Method callMC_value;
// varargs assignment fixer for a single string array argument
static Object callMC(String method, String[] arg) {
return callMC(method, new Object[] {arg});
}
static Object callMC(String method, Object... args) { try {
Method me;
if (callMC_cache == null) callMC_cache = new HashMap(); // initializer time workaround
synchronized(callMC_cache) {
me = method == callMC_key ? callMC_value : null;
}
if (me != null) try {
return invokeMethod(me, null, args);
} catch (IllegalArgumentException e) {
throw new RuntimeException("Can't call " + me + " with arguments " + classNames(args), e);
}
List m;
synchronized(callMC_cache) {
m = callMC_cache.get(method);
}
if (m == null) {
if (callMC_cache.isEmpty()) {
callMC_makeCache();
m = callMC_cache.get(method);
}
if (m == null) throw fail("Method named " + method + " not found in main");
}
int n = m.size();
if (n == 1) {
me = m.get(0);
synchronized(callMC_cache) {
callMC_key = method;
callMC_value = me;
}
try {
return invokeMethod(me, null, args);
} catch (IllegalArgumentException e) {
throw new RuntimeException("Can't call " + me + " with arguments " + classNames(args), e);
}
}
for (int i = 0; i < n; i++) {
me = m.get(i);
if (call_checkArgs(me, args, false))
return invokeMethod(me, null, args);
}
throw fail("No method called " + method + " with arguments (" + joinWithComma(getClasses(args)) + ") found in main");
} catch (Exception __e) { throw rethrow(__e); } }
static void callMC_makeCache() {
synchronized(callMC_cache) {
callMC_cache.clear();
Class _c = (Class) mc(), c = _c;
while (c != null) {
for (Method m : c.getDeclaredMethods())
if ((m.getModifiers() & java.lang.reflect.Modifier.STATIC) != 0) {
makeAccessible(m);
multiMapPut(callMC_cache, m.getName(), m);
}
c = c.getSuperclass();
}
}
}
static List synchroList() {
return Collections.synchronizedList(new ArrayList ());
}
static List synchroList(List l) {
return Collections.synchronizedList(l);
}
static Pair pair(A a, B b) {
return new Pair(a, b);
}
static Pair pair(A a) {
return new Pair(a, a);
}
static A assertNotNull(A a) {
assertTrue(a != null);
return a;
}
static A assertNotNull(String msg, A a) {
assertTrue(msg, a != null);
return a;
}
static Map synchroHashMap() {
return Collections.synchronizedMap(new HashMap());
}
public static String join(String glue, Iterable strings) {
if (strings == null) return "";
if (strings instanceof Collection) {
if (((Collection) strings).size() == 1) return str(first(((Collection) strings)));
}
StringBuilder buf = new StringBuilder();
Iterator i = strings.iterator();
if (i.hasNext()) {
buf.append(i.next());
while (i.hasNext())
buf.append(glue).append(i.next());
}
return buf.toString();
}
public static String join(String glue, String... strings) {
return join(glue, Arrays.asList(strings));
}
static String join(Iterable strings) {
return join("", strings);
}
static String join(Iterable strings, String glue) {
return join(glue, strings);
}
public static String join(String[] strings) {
return join("", strings);
}
static String join(String glue, Pair p) {
return p == null ? "" : str(p.a) + glue + str(p.b);
}
static List getClassNames(Collection l) {
List out = new ArrayList();
if (l != null) for (Object o : l)
out.add(o == null ? null : getClassName(o));
return out;
}
// unclear semantics as to whether return null on null
static ArrayList asList(A[] a) {
return a == null ? new ArrayList () : new ArrayList (Arrays.asList(a));
}
static ArrayList asList(int[] a) {
if (a == null) return null;
ArrayList l = emptyList(a.length);
for (int i : a) l.add(i);
return l;
}
static ArrayList asList(long[] a) {
if (a == null) return null;
ArrayList l = emptyList(a.length);
for (long i : a) l.add(i);
return l;
}
static ArrayList asList(float[] a) {
if (a == null) return null;
ArrayList l = emptyList(a.length);
for (float i : a) l.add(i);
return l;
}
static ArrayList asList(double[] a) {
if (a == null) return null;
ArrayList l = emptyList(a.length);
for (double i : a) l.add(i);
return l;
}
static ArrayList asList(Iterable s) {
if (s instanceof ArrayList) return (ArrayList) s;
ArrayList l = new ArrayList();
if (s != null)
for (A a : s)
l.add(a);
return l;
}
static ArrayList asList(Producer p) {
ArrayList l = new ArrayList();
A a;
if (p != null) while ((a = p.next()) != null)
l.add(a);
return l;
}
static ArrayList asList(Enumeration e) {
ArrayList l = new ArrayList();
if (e != null)
while (e.hasMoreElements())
l.add(e.nextElement());
return l;
}
static Object collectionMutex(List l) {
return l;
}
static Object collectionMutex(Object o) {
if (o instanceof List) return o;
String c = className(o);
if (eq(c, "java.util.TreeMap$KeySet"))
c = className(o = getOpt(o, "m"));
else if (eq(c, "java.util.HashMap$KeySet"))
c = className(o = get_raw(o, "this$0"));
if (eqOneOf(c, "java.util.TreeMap$AscendingSubMap", "java.util.TreeMap$DescendingSubMap"))
c = className(o = get_raw(o, "m"));
return o;
}
static Object pcallFunction(Object f, Object... args) {
try { return callFunction(f, args); } catch (Throwable __e) { _handleException(__e); }
return null;
}
static volatile PersistableThrowable _handleException_lastException;
static List _handleException_onException = synchroList(ll("printStackTrace2"));
static void _handleException(Throwable e) {
_handleException_lastException = persistableThrowable(e);
Throwable e2 = innerException(e);
if (e2.getClass() == RuntimeException.class && eq(e2.getMessage(), "Thread cancelled.") || e2 instanceof InterruptedException)
return;
for (Object f : cloneList(_handleException_onException)) try {
callF(f, e);
} catch (Throwable e3) {
try {
printStackTrace2(e3); // not using pcall here - it could lead to endless loops
} catch (Throwable e4) {
System.out.println(getStackTrace(e3));
System.out.println(getStackTrace(e4));
}
}
}
static A get(List l, int idx) {
return l != null && idx >= 0 && idx < l(l) ? l.get(idx) : null;
}
// seems to conflict with other signatures
/*static B get(Map map, A key) {
ret map != null ? map.get(key) : null;
}*/
static A get(A[] l, int idx) {
return idx >= 0 && idx < l(l) ? l[idx] : null;
}
// default to false
static boolean get(boolean[] l, int idx) {
return idx >= 0 && idx < l(l) ? l[idx] : false;
}
// get purpose 2: access a field by reflection or a map
static Object get(Object o, String field) {
try {
if (o == null) return null;
if (o instanceof Class) return get((Class) o, field);
if (o instanceof Map)
return ((Map) o).get(field);
Field f = getOpt_findField(o.getClass(), field);
if (f != null) {
makeAccessible(f);
return f.get(o);
}
if (o instanceof DynamicObject)
return ((DynamicObject) o).fieldValues.get(field);
} catch (Exception e) {
throw asRuntimeException(e);
}
throw new RuntimeException("Field '" + field + "' not found in " + o.getClass().getName());
}
static Object get_raw(String field, Object o) {
return get_raw(o, field);
}
static Object get_raw(Object o, String field) { try {
if (o == null) return null;
Field f = get_findField(o.getClass(), field);
makeAccessible(f);
return f.get(o);
} catch (Exception __e) { throw rethrow(__e); } }
static Object get(Class c, String field) {
try {
Field f = get_findStaticField(c, field);
makeAccessible(f);
return f.get(null);
} catch (Exception e) {
throw new RuntimeException(e);
}
}
static Field get_findStaticField(Class c, String field) {
Class _c = c;
do {
for (Field f : _c.getDeclaredFields())
if (f.getName().equals(field) && (f.getModifiers() & java.lang.reflect.Modifier.STATIC) != 0)
return f;
_c = _c.getSuperclass();
} while (_c != null);
throw new RuntimeException("Static field '" + field + "' not found in " + c.getName());
}
static Field get_findField(Class c, String field) {
Class _c = c;
do {
for (Field f : _c.getDeclaredFields())
if (f.getName().equals(field))
return f;
_c = _c.getSuperclass();
} while (_c != null);
throw new RuntimeException("Field '" + field + "' not found in " + c.getName());
}
static Object get(String field, Object o) {
return get(o, field);
}
static Object vm_generalMap_get(Object key) {
return vm_generalMap().get(key);
}
static Object vm_generalMap_put(Object key, Object value) {
return mapPutOrRemove(vm_generalMap(), key, value);
}
static Set syncIdentityHashSet() {
return (Set) synchronizedSet(identityHashSet());
}
static Map syncHashMap() {
return synchroHashMap();
}
static B mapGet(Map map, A a) {
return map == null || a == null ? null : map.get(a);
}
static B mapGet(A a, Map map) {
return map == null || a == null ? null : map.get(a);
}
static boolean even(int i) {
return (i & 1) == 0;
}
static boolean even(long i) {
return (i & 1) == 0;
}
static String upper(String s) {
return s == null ? null : s.toUpperCase();
}
static char upper(char c) {
return Character.toUpperCase(c);
}
static List lambdaMap(IF1 f, Iterable l) {
return map(l, f);
}
static List lambdaMap(IF1 f, A[] l) {
return map(l, f);
}
static String f2s(File f) {
return f == null ? null : f.getAbsolutePath();
}
static String f2s(String s) { return f2s(newFile(s)); }
static String f2s(java.nio.file.Path p) {
return p == null ? null : f2s(p.toFile());
}
static String _userHome;
static String userHome() {
if (_userHome == null)
return actualUserHome();
return _userHome;
}
static File userHome(String path) {
return new File(userDir(), path);
}
static File newFile(File base, String... names) {
for (String name : names) base = new File(base, name);
return base;
}
static File newFile(String name) {
return name == null ? null : new File(name);
}
static File newFile(String base, String... names) {
return newFile(newFile(base), names);
}
static String makeRandomID(int length) {
return makeRandomID(length, defaultRandomGenerator());
}
static String makeRandomID(int length, Random random) {
char[] id = new char[length];
for (int i = 0; i < id.length; i++)
id[i] = (char) ((int) 'a' + random.nextInt(26));
return new String(id);
}
static String makeRandomID(Random r, int length) {
return makeRandomID(length, r);
}
static List newSubListOrSame(List l, int startIndex) {
return newSubListOrSame(l, startIndex, l(l));
}
static List newSubListOrSame(List l, int startIndex, int endIndex) {
if (l == null) return null;
int n = l(l);
startIndex = max(0, startIndex);
endIndex = min(n, endIndex);
if (startIndex >= endIndex) return ll();
if (startIndex == 0 && endIndex == n) return l;
return cloneList(l.subList(startIndex, endIndex));
}
static List newSubListOrSame(List l, IntRange r) {
return newSubListOrSame(l, r.start, r.end);
}
static String substring(String s, int x) {
return substring(s, x, strL(s));
}
static String substring(String s, int x, int y) {
if (s == null) return null;
if (x < 0) x = 0;
int n = s.length();
if (y < x) y = x;
if (y > n) y = n;
if (x >= y) return "";
return s.substring(x, y);
}
static String substring(String s, IntRange r) {
return r == null ? null : substring(s, r.start, r.end);
}
// convenience method for quickly dropping a prefix
static String substring(String s, CharSequence l) {
return substring(s, l(l));
}
static int[] takeFirstOfIntArray(int[] b, int n) {
return subIntArray(b, 0, n);
}
static int[] takeFirstOfIntArray(int n, int[] b) {
return takeFirstOfIntArray(b, n);
}
static int lengthOfString(String s) {
return s == null ? 0 : s.length();
}
static String rep(int n, char c) {
return repeat(c, n);
}
static String rep(char c, int n) {
return repeat(c, n);
}
static List rep(A a, int n) {
return repeat(a, n);
}
static List rep(int n, A a) {
return repeat(n, a);
}
static Charset utf8charset_cache;
static Charset utf8charset() { if (utf8charset_cache == null) utf8charset_cache = utf8charset_load(); return utf8charset_cache; }
static Charset utf8charset_load() {
return Charset.forName("UTF-8");
}
static int parseInt(String s) {
return emptyString(s) ? 0 : Integer.parseInt(s);
}
static int parseInt(char c) {
return Integer.parseInt(str(c));
}
static int boolToInt(boolean b) {
return b ? 1 : 0;
}
static int max(int a, int b) { return Math.max(a, b); }
static int max(int a, int b, int c) { return max(max(a, b), c); }
static long max(int a, long b) { return Math.max((long) a, b); }
static long max(long a, long b) { return Math.max(a, b); }
static double max(int a, double b) { return Math.max((double) a, b); }
static float max(float a, float b) { return Math.max(a, b); }
static double max(double a, double b) { return Math.max(a, b); }
static int max(Collection c) {
int x = Integer.MIN_VALUE;
for (int i : c) x = max(x, i);
return x;
}
static double max(double[] c) {
if (c.length == 0) return Double.MIN_VALUE;
double x = c[0];
for (int i = 1; i < c.length; i++) x = Math.max(x, c[i]);
return x;
}
static float max(float[] c) {
if (c.length == 0) return Float.MAX_VALUE;
float x = c[0];
for (int i = 1; i < c.length; i++) x = Math.max(x, c[i]);
return x;
}
static byte max(byte[] c) {
byte x = -128;
for (byte d : c) if (d > x) x = d;
return x;
}
static short max(short[] c) {
short x = -0x8000;
for (short d : c) if (d > x) x = d;
return x;
}
static int max(int[] c) {
int x = Integer.MIN_VALUE;
for (int d : c) if (d > x) x = d;
return x;
}
static void intToBytes_inArray(int i, byte[] array, int idx) {
array[idx] = (byte) (i >>> 24);
array[idx+1] = (byte) (i >>> 16);
array[idx+2] = (byte) (i >>> 8);
array[idx+3 ] = (byte) i;
}
static long fileSize(String path) { return getFileSize(path); }
static long fileSize(File f) { return getFileSize(f); }
static A printException(A e) {
printStackTrace(e);
return e;
}
static Throwable printStackTrace2(Throwable e) {
// we go to system.out now - system.err is nonsense
print(getStackTrace2(e));
return e;
}
static void printStackTrace2() {
printStackTrace2(new Throwable());
}
static void printStackTrace2(String msg) {
printStackTrace2(new Throwable(msg));
}
static boolean methodIsStatic(Method m) {
return (m.getModifiers() & Modifier.STATIC) != 0;
}
static boolean argumentCompatibleWithType(Object arg, Class type) {
return arg == null ? !type.isPrimitive() : isInstanceX(type, arg);
}
static void arraycopy(Object[] a, Object[] b) {
if (a != null && b != null)
arraycopy(a, 0, b, 0, Math.min(a.length, b.length));
}
static void arraycopy(Object src, int srcPos, Object dest, int destPos, int n) {
if (n != 0)
System.arraycopy(src, srcPos, dest, destPos, n);
}
static A[] arrayOfType(Class type, int n) {
return makeArray(type, n);
}
static A[] arrayOfType(int n, Class type) {
return arrayOfType(type, n);
}
static HashSet litset(A... items) {
return lithashset(items);
}
static List getClasses(Object[] array) {
List l = emptyList(l(array));
for (Object o : array) l.add(_getClass(o));
return l;
}
static void multiMapPut(Map> map, A a, B b) {
List l = map.get(a);
if (l == null)
map.put(a, l = new ArrayList());
l.add(b);
}
static void multiMapPut(MultiMap mm, A key, B value) {
if (mm != null && key != null && value != null) mm.put(key, value);
}
static String className(Object o) {
return getClassName(o);
}
static boolean eqOneOf(Object o, Object... l) {
for (Object x : l) if (eq(o, x)) return true; return false;
}
static Object callFunction(Object f, Object... args) {
return callF(f, args);
}
static List ll(A... a) {
ArrayList l = new ArrayList(a.length);
if (a != null) for (A x : a) l.add(x);
return l;
}
static PersistableThrowable persistableThrowable(Throwable e) {
return e == null ? null : new PersistableThrowable(e);
}
static Throwable innerException(Throwable e) {
return getInnerException(e);
}
static String getStackTrace(Throwable throwable) {
lastException(throwable);
return getStackTrace_noRecord(throwable);
}
static String getStackTrace_noRecord(Throwable throwable) {
StringWriter writer = new StringWriter();
throwable.printStackTrace(new PrintWriter(writer));
return hideCredentials(writer.toString());
}
static String getStackTrace() {
return getStackTrace_noRecord(new Throwable());
}
static Map vm_generalMap_map;
static Map vm_generalMap() {
if (vm_generalMap_map == null)
vm_generalMap_map = (Map) get(javax(), "generalMap");
return vm_generalMap_map;
}
static B mapPutOrRemove(Map map, A key, B value) {
if (map != null && key != null)
if (value != null) return map.put(key, value);
else return map.remove(key);
return null;
}
static Set synchronizedSet() {
return synchroHashSet();
}
static Set synchronizedSet(Set set) {
return Collections.synchronizedSet(set);
}
static Set identityHashSet() {
return Collections.newSetFromMap(new IdentityHashMap());
}
static List map(Iterable l, Object f) { return map(f, l); }
static List map(Object f, Iterable l) {
List x = emptyList(l);
if (l != null) for (Object o : l)
x.add(callF(f, o));
return x;
}
static List map(Iterable l, F1 f) { return map(f, l); }
static List map(F1 f, Iterable l) {
List x = emptyList(l);
if (l != null) for (A o : l)
x.add(callF(f, o));
return x;
}
static List map(IF1 f, Iterable l) { return map(l, f); }
static List map(Iterable l, IF1 f) {
List x = emptyList(l);
if (l != null) for (A o : l)
x.add(f.get(o));
return x;
}
static List map(IF1 f, A[] l) { return map(l, f); }
static List map(A[] l, IF1 f) {
List x = emptyList(l);
if (l != null) for (A o : l)
x.add(f.get(o));
return x;
}
static List map(Object f, Object[] l) { return map(f, asList(l)); }
static List map(Object[] l, Object f) { return map(f, l); }
static List map(Object f, Map map) {
return map(map, f);
}
// map: func(key, value) -> list element
static List map(Map map, Object f) {
List x = new ArrayList();
if (map != null) for (Object _e : map.entrySet()) {
Map.Entry e = (Map.Entry) _e;
x.add(callF(f, e.getKey(), e.getValue()));
}
return x;
}
static List map(Map map, IF2 f) {
return map(map, (Object) f);
}
static String actualUserHome_value;
static String actualUserHome() {
if (actualUserHome_value == null) {
if (isAndroid())
actualUserHome_value = "/storage/emulated/0/";
else
actualUserHome_value = System.getProperty("user.home");
}
return actualUserHome_value;
}
static File actualUserHome(String sub) {
return newFile(new File(actualUserHome()), sub);
}
static File userDir() {
return new File(userHome());
}
static File userDir(String path) {
return new File(userHome(), path);
}
static Random defaultRandomGenerator() {
return ThreadLocalRandom.current();
}
static int strL(String s) {
return s == null ? 0 : s.length();
}
static String repeat(char c, int n) {
n = Math.max(n, 0);
char[] chars = new char[n];
for (int i = 0; i < n; i++)
chars[i] = c;
return new String(chars);
}
static List repeat(A a, int n) {
n = Math.max(n, 0);
List l = new ArrayList(n);
for (int i = 0; i < n; i++)
l.add(a);
return l;
}
static List repeat(int n, A a) {
return repeat(a, n);
}
static boolean emptyString(String s) {
return s == null || s.length() == 0;
}
static A printStackTrace(A e) {
// we go to system.out now - system.err is nonsense
print(getStackTrace(e));
return e;
}
static void printStackTrace() {
printStackTrace(new Throwable());
}
static void printStackTrace(String msg) {
printStackTrace(new Throwable(msg));
}
static void printStackTrace(String msg, Throwable e) {
printStackTrace(new Throwable(msg, e));
}
static String getStackTrace2(Throwable e) {
return hideCredentials(getStackTrace(unwrapTrivialExceptionWraps(e)) + replacePrefix("java.lang.RuntimeException: ", "FAIL: ",
hideCredentials(str(innerException2(e)))) + "\n");
}
static A[] makeArray(Class type, int n) {
return (A[]) Array.newInstance(type, n);
}
static HashSet lithashset(A... items) {
HashSet set = new HashSet();
for (A a : items) set.add(a);
return set;
}
static Class _getClass(String name) {
try {
return Class.forName(name);
} catch (ClassNotFoundException e) {
return null; // could optimize this
}
}
static Class _getClass(Object o) {
return o == null ? null
: o instanceof Class ? (Class) o : o.getClass();
}
static Class _getClass(Object realm, String name) {
try {
return getClass(realm).getClassLoader().loadClass(classNameToVM(name));
} catch (ClassNotFoundException e) {
return null; // could optimize this
}
}
static Throwable getInnerException(Throwable e) {
if (e == null) return null;
while (e.getCause() != null)
e = e.getCause();
return e;
}
static Throwable getInnerException(Runnable r) {
return getInnerException(getException(r));
}
// PersistableThrowable doesn't hold GC-disturbing class references in backtrace
static volatile PersistableThrowable lastException_lastException;
static PersistableThrowable lastException() {
return lastException_lastException;
}
static void lastException(Throwable e) {
lastException_lastException = persistableThrowable(e);
}
static String hideCredentials(URL url) { return url == null ? null : hideCredentials(str(url)); }
static String hideCredentials(String url) {
try {
if (startsWithOneOf(url, "http://", "https://") && isAGIBlueDomain(hostNameFromURL(url))) return url;
} catch (Throwable e) {
print("HideCredentials", e);
}
return url.replaceAll("([&?])(_pass|key|cookie)=[^&\\s\"]*", "$1$2=");
}
static String hideCredentials(Object o) {
return hideCredentials(str(o));
}
static Set synchroHashSet() {
return Collections.synchronizedSet(new HashSet ());
}
static Throwable unwrapTrivialExceptionWraps(Throwable e) {
if (e == null) return e;
while (e.getClass() == RuntimeException.class
&& e.getCause() != null && eq(e.getMessage(), str(e.getCause())))
e = e.getCause();
return e;
}
static String replacePrefix(String prefix, String replacement, String s) {
if (!startsWith(s, prefix)) return s;
return replacement + substring(s, l(prefix));
}
static Throwable innerException2(Throwable e) {
if (e == null) return null;
while (empty(e.getMessage()) && e.getCause() != null)
e = e.getCause();
return e;
}
static Class getClass(String name) {
try {
return Class.forName(name);
} catch (ClassNotFoundException e) {
return null;
}
}
static Class getClass(Object o) {
return o instanceof Class ? (Class) o : o.getClass();
}
static Class getClass(Object realm, String name) { try {
try {
return getClass(realm).getClassLoader().loadClass(classNameToVM(name));
} catch (ClassNotFoundException e) {
return null;
}
} catch (Exception __e) { throw rethrow(__e); } }
static String classNameToVM(String name) {
return name.replace(".", "$");
}
static Throwable getException(Runnable r) {
try {
callF(r);
return null;
} catch (Throwable e) {
return e;
}
}
static boolean startsWithOneOf(String s, String... l) {
for (String x : l) if (startsWith(s, x)) return true; return false;
}
static boolean startsWithOneOf(String s, Matches m, String... l) {
for (String x : l) if (startsWith(s, x, m)) return true; return false;
}
static boolean isAGIBlueDomain(String domain) {
return domainIsUnder(domain, theAGIBlueDomain());
}
static String hostNameFromURL(String url) { try {
return new URL(url).getHost();
} catch (Exception __e) { throw rethrow(__e); } }
static boolean startsWith(String a, String b) {
return a != null && a.startsWith(unnull(b));
}
static boolean startsWith(String a, char c) {
return nemptyString(a) && a.charAt(0) == c;
}
static boolean startsWith(String a, String b, Matches m) {
if (!startsWith(a, b)) return false;
m.m = new String[] {substring(a, strL(b))};
return true;
}
static boolean startsWith(List a, List b) {
if (a == null || listL(b) > listL(a)) return false;
for (int i = 0; i < listL(b); i++)
if (neq(a.get(i), b.get(i)))
return false;
return true;
}
static boolean domainIsUnder(String domain, String mainDomain) {
return eqic(domain, mainDomain) || ewic(domain, "." + mainDomain);
}
static String theAGIBlueDomain() {
return "agi.blue";
}
static boolean nemptyString(String s) {
return s != null && s.length() > 0;
}
static int listL(Collection l) {
return l == null ? 0 : l.size();
}
static boolean neq(Object a, Object b) {
return !eq(a, b);
}
static boolean eqic(String a, String b) {
if ((a == null) != (b == null)) return false;
if (a == null) return true;
return a.equalsIgnoreCase(b);
}
static boolean eqic(char a, char b) {
if (a == b) return true;
char u1 = Character.toUpperCase(a);
char u2 = Character.toUpperCase(b);
if (u1 == u2) return true;
return Character.toLowerCase(u1) == Character.toLowerCase(u2);
}
static boolean ewic(String a, String b) {
return endsWithIgnoreCase(a, b);
}
static boolean ewic(String a, String b, Matches m) {
return endsWithIgnoreCase(a, b, m);
}
static String asString(Object o) {
return o == null ? null : o.toString();
}
static boolean endsWithIgnoreCase(String a, String b) {
int la = l(a), lb = l(b);
return la >= lb && regionMatchesIC(a, la-lb, b, 0, lb);
}
static boolean endsWithIgnoreCase(String a, String b, Matches m) {
if (!endsWithIgnoreCase(a, b)) return false;
m.m = new String[] { substring(a, 0, l(a)-l(b)) };
return true;
}
static boolean regionMatchesIC(String a, int offsetA, String b, int offsetB, int len) {
return a != null && a.regionMatches(true, offsetA, b, offsetB, len);
}
// immutable, has strong refs
final static class _MethodCache {
final Class c;
final HashMap> cache = new HashMap();
_MethodCache(Class c) {
this.c = c; _init(); }
void _init() {
Class _c = c;
while (_c != null) {
for (Method m : _c.getDeclaredMethods())
if (!isAbstract(m) && !reflection_isForbiddenMethod(m))
multiMapPut(cache, m.getName(), makeAccessible(m));
_c = _c.getSuperclass();
}
// add default methods - this might lead to a duplication
// because the overridden method is also added, but it's not
// a problem except for minimal performance loss.
for (Class intf : allInterfacesImplementedBy(c))
for (Method m : intf.getDeclaredMethods())
if (m.isDefault() && !reflection_isForbiddenMethod(m))
multiMapPut(cache, m.getName(), makeAccessible(m));
}
// Returns only matching methods
Method findMethod(String method, Object[] args) { try {
List m = cache.get(method);
if (m == null) return null;
int n = m.size();
for (int i = 0; i < n; i++) {
Method me = m.get(i);
if (call_checkArgs(me, args, false))
return me;
}
return null;
} catch (Exception __e) { throw rethrow(__e); } }
Method findStaticMethod(String method, Object[] args) { try {
List m = cache.get(method);
if (m == null) return null;
int n = m.size();
for (int i = 0; i < n; i++) {
Method me = m.get(i);
if (isStaticMethod(me) && call_checkArgs(me, args, false))
return me;
}
return null;
} catch (Exception __e) { throw rethrow(__e); } }
}static abstract class VF1 implements IVF1 {
public abstract void get(A a);
}static class Matches {
String[] m;
Matches() {}
Matches(String... m) {
this.m = m;}
String get(int i) { return i < m.length ? m[i] : null; }
String unq(int i) { return unquote(get(i)); }
String tlc(int i) { return unq(i).toLowerCase(); }
boolean bool(int i) { return "true".equals(unq(i)); }
String rest() { return m[m.length-1]; } // for matchStart
int psi(int i) { return Integer.parseInt(unq(i)); }
public String toString() { return "Matches(" + joinWithComma(quoteAll(asList(m))) + ")"; }
public int hashCode() { return _hashCode(toList(m)); }
public boolean equals(Object o) { return o instanceof Matches && arraysEqual(m, ((Matches) o).m); }
}
static class Timed extends Var {
long ms;
Timed() {}
Timed(A value, long ms) { super(value);
this.ms = ms; }
public String toString() { return "[" + ms + " ms] " + super.toString(); }
}static class LineCompReader {
List literals = new ArrayList();
IVF1 onPair; // if not null, don't save pairs, but stream them to onPair instead
int[] literalOffsets; // where they start in file
LongBuffer pairs = new LongBuffer();
LinkedHashMap> versions = new LinkedHashMap();
boolean byteMode = false;
// internal, optional
CountingInputStream countingInputStream;
long fileSize;
LineCompReader() {}
// takes text or gzipped input file
LineCompReader(File f) { load(f); }
LineCompReader(InputStream in) { load(rawByteReader(in, 128*1024)); }
LineCompReader(BufferedReader reader) { load(reader); }
void load(File f) {
fileSize = fileSize(f);
countingInputStream = new CountingInputStream(bufferedFileInputStream(f));
BufferedReader reader = isGZipFile(f)
? rawByteReader(gzipInputStream(countingInputStream))
: rawByteReader(countingInputStream); try {
//rawByteReader_possiblyGZipped(f);
load(reader);
} finally { _close(reader); }}
void load(BufferedReader reader) { try {
StringBuilder lineBuf = new StringBuilder();
String s = readLineIgnoreCR(reader, lineBuf);
int ofs = l(s)+1;
Matches m = new Matches();
if (startsWith(s, "BYTECOMP ", m)) byteMode = true;
else if (!startsWith(s, "LINECOMP ", m))
throw fail("Not a LINECOMP file");
int nLiterals = parseInt(m.rest());
IntBuffer offsets = new IntBuffer();
for (int i = 0; i < nLiterals; i++) {
String line = readLineIgnoreCR(reader, lineBuf);
assertNotNull(line);
literals.add(byteMode ? str(charFromHex(line)) : line);
offsets.add(ofs);
ofs += l(line)+1;
}
offsets.add(ofs);
literalOffsets = offsets.toArray();
int n = 0;
while (licensed()) {
s = readLineIgnoreCR(reader, lineBuf);
if (s == null || contains(s, "=")) break;
try {
int iSpace = s.indexOf(' ');
long pair = twoIntsToLong(
Integer.parseInt(s, 0, iSpace, 10),
Integer.parseInt(s, iSpace+1, l(s), 10));
if (onPair != null) onPair.get(pair);
else pairs.add(pair);
if (((++n) % oneMillion()) == 0) {
String percentage = "";
if (fileSize != 0 && countingInputStream != null)
percentage = " (" + intPercentRatio(countingInputStream.getFilePointer(), fileSize) + "%)";
print(nPairs(n) + " read" + percentage);
}
} catch (Throwable _e) {
print("On line " + (nLiterals + l(pairs)));
throw rethrow(_e); }
}
pairs.trimToSize();
while (contains(s, "=")) {
int i = indexOf(s, '=');
versions.put(takeFirst(s, i), compactIntList(parseInts(splitAtSpace(substring(s, i+1)))));
s = readLineIgnoreCR(reader, lineBuf);
}
} catch (Exception __e) { throw rethrow(__e); } }
Set versions() { return keys(versions); }
String getText(String version) { return textForVersion(version); }
String textForVersion(String version) {
List encoded = versions.get(version);
if (encoded == null) return null;
List buf = new ArrayList();
for (int idx : encoded)
decode(idx, buf);
return myFromLines(buf);
}
// name of first (or only) file
String firstFile() { return first(versions()); }
// text for first (or only) file
String text() { return getText(firstFile()); }
List encoding() { return versions.get(firstFile()); }
String myFromLines(List l) {
return byteMode
? join(l)
: fromLines_rtrim(l);
}
void decode(int idx, List buf) {
if (idx < l(literals))
buf.add(literals.get(idx));
else {
long p = pairs.get(idx-l(literals));
decode(firstIntFromLong(p), buf);
decode(secondIntFromLong(p), buf);
}
}
// That was it! The rest of this file is just for calculating some stats.
Map lineCountsForPairs = new HashMap();
Map byteCountsForPairs = new HashMap();
int lineCountForPointer(int idx) {
return idx < l(literals) ? 1 : lineCountForPair(idx);
}
long byteCountForPointer(int idx) {
return idx < l(literals) ? l(literals.get(idx))+1 : byteCountForPair(idx);
}
int lineCountForPair(int idx) {
Integer c = lineCountsForPairs.get(idx);
if (c == null) {
long p = pairs.get(idx-l(literals));
c = lineCountForPointer(firstIntFromLong(p)) + lineCountForPointer(secondIntFromLong(p));
lineCountsForPairs.put(idx, c);
}
return c;
}
long byteCountForPair(int idx) {
Long c = byteCountsForPairs.get(idx);
if (c == null) {
long p = pairs.get(idx-l(literals));
c = byteCountForPointer(firstIntFromLong(p)) + byteCountForPointer(secondIntFromLong(p));
byteCountsForPairs.put(idx, c);
}
return c;
}
int lineCountForVersion(String version) {
List encoded = versions.get(version);
if (encoded == null) return 0;
int n = 0;
for (int i : encoded) n += lineCountForPointer(i);
return n;
}
long byteCountForVersion(String version) {
List encoded = versions.get(version);
if (encoded == null) return 0;
long n = 0;
for (int i : encoded) n += byteCountForPointer(i);
return max(0, n-1);
}
long totalByteCount() {
return longSum(lambdaMap(__22 -> byteCountForVersion(__22),versions()));
}
// now we can also save again
void save(PrintWriter out) {
out.println((byteMode ? "BYTECOMP " : "LINECOMP ") + l(literals));
for (String s : literals)
out.println(byteMode ? charToHex(first(s)) : s);
for (long p : pairs)
out.println(firstIntFromLong(p) + " " + secondIntFromLong(p));
for (Map.Entry> __0 : _entrySet( versions))
{ String id = __0.getKey(); List l = __0.getValue(); out.println(id + "=" + joinWithSpace(l)); }
}
}// it's unclear whether the end is inclusive or exclusive
// (usually exclusive I guess)
static class IntRange {
int start, end;
IntRange() {}
IntRange(int start, int end) {
this.end = end;
this.start = start;}
IntRange(IntRange r) { start = r.start; end = r.end; }
public boolean equals(Object o) { return stdEq2(this, o); }
public int hashCode() { return stdHash2(this); }
final int length() { return end-start; }
final boolean empty() { return start >= end; }
final boolean isEmpty() { return start >= end; }
static String _fieldOrder = "start end";
public String toString() { return "[" + start + ";" + end + "]"; }
}
// read-only, so far. should be thread-safe
final static class RAMByteMemory64 implements IByteMemory64, AutoCloseable {
File file; // the file we loaded
long size; // file size in ints
boolean bigEndian = true;
boolean debug = false;
// byte buffers
int arrayShift = 30; // each buffer is 1 GB
int arraySize = 1 << arrayShift;
byte[][] arrays;
RAMByteMemory64() {}
RAMByteMemory64(File file) {
load(file);
}
void load(File file) { try {
this.file = file;
size = fileSize(file);
RandomAccessFile raf = newRandomAccessFile(file, "r"); try {
arrays = new byte[toInt(rightShift_ceil(size, arrayShift))][];
FileChannel channel = raf.getChannel();
print("Allocating " + n2(size) + " bytes");
for (int i = 0; i < l(arrays); i++) {
long pos = (long) i << arrayShift;
int len = toInt(min(1 << arrayShift, size-pos));
arrays[i] = new byte[len];
}
for (int i = 0; i < l(arrays); i++) {
long pos = (long) i << arrayShift;
int len = l(arrays[i]);
print("Loading bytes " + longToHex(pos) + "-" + longToHex(pos+len) + " of " + file);
MappedByteBuffer byteBuffer = channel.map(FileChannel.MapMode.READ_ONLY, pos, len);
byteBuffer.get(arrays[i]);
}
print(size + " bytes loaded");
} finally { _close(raf); }} catch (Exception __e) { throw rethrow(__e); } }
public void close() {}
public byte getByte(long idx) {
rangeCheck(idx, size);
return arrays[(int) (idx >> arrayShift)][(((int) idx) & (arraySize-1))];
}
public int getInt(long idx) {
assertTrue("bigEndian", bigEndian);
return ubyteToInt(getByte(idx)) << 24 |
ubyteToInt(getByte(idx+1)) << 16 |
ubyteToInt(getByte(idx+2)) << 8 |
ubyteToInt(getByte(idx+3));
}
public void set(long idx, int val) {
checkWritable();
throw fail("todo");
}
void checkWritable() {
throw fail("read-only");
}
public long size() {
return size;
}
public void ensureSize(int size) {
this.size = max(this.size, size);
}
void importVirtual(Object mem) {
copyFields(mem, this);
}
}static abstract class F1 {
abstract B get(A a);
}final static class LCSearcher_v1_onDisk64_v3 extends ILCCompactIndex implements AutoCloseable {
// important: set this if the file uses a special comparator
CharComparator charComparator;
// flags user can set
boolean debug = false; // switch print statements on/off
LCSearcher_Protocol protocol;
boolean doSubProtocol = false;
IVF1_Long onMemAccess; // initialize this to get IO stats (TODO: actually call)
// all-important memory field
IByteMemory64 mem;
// tuning parameter
int intBufferEmptyCapacity = 8;
// data we always hold in memory
char version; // file version ('1' or '2')
int iFirstPair, iFirstFile, nProds;
char[] literals;
Map literalIndex = new HashMap();
int[] files; // starting at iFirstFile
MultiMap symbolToFile = new MultiMap(); // values start at iFirstFile (I think)
// where symbol pointer table starts
long iSymbolPointers;
// beef up symbol pointers to 64 bit
long[] symbolPtrHighBytes;
int highBytesShift = 22; // we make a new high byte every 1 << 22 entries
LCSearcher_v1_onDisk64_v3() {}
LCSearcher_v1_onDisk64_v3(File f) { load(f); }
LCSearcher_v1_onDisk64_v3(IByteMemory64 mem) { load(mem); }
void load(File f) {
BufferedDiskByteMemory64 mem = new BufferedDiskByteMemory64(f);
mem.bigEndian = true;
load(mem);
}
void load(IByteMemory64 mem) {
this.mem = mem;
String versionString = fromUtf8(intArrayToBytes(readIntArray(0, 4)));
assertStartsWith(versionString, "QUICKSEARCH v");
assertEndsWith(versionString, "\n");
version = versionString.charAt(l(versionString)-3);
if (debug) print("File signature verifies.");
iFirstPair = memGet(4*4);
iFirstFile = iFirstPair+memGet(5*4);
nProds = iFirstFile+memGet(6*4);
long ptr = 7*4;
if (debug) printVars_str("iFirstPair", iFirstPair, "iFirstFile", iFirstFile, "nProds", nProds);
// read raw literals
literals = new char[iFirstPair];
for (int i = 0; i < l(literals); i++) {
literals[i] = (char) memGet(ptr+i*2-2);
literalIndex.put(literals[i], i);
}
ptr += roundUpTo(4, l(literals)*2);
if (debug) printStruct("literals", literals);
// read files
files = readIntArray(ptr, nProds-iFirstFile);
for (int i = 0; i < l(files); i++)
symbolToFile.put(files[i], i+iFirstFile);
ptr += (nProds-iFirstFile)*4;
if (debug) printStruct("files", files);
// init symbol table
iSymbolPointers = ptr;
makeHighBytes();
}
long indexOfOccurrences(int symbol) {
long ptr = symbolPtr(symbol);
if (symbol >= iFirstPair) {
ptr += 2*4; // skip pair contents
ptr += compressed30BitUint_lengthFromFirstByte(mem.getByte(ptr)); // skip prod length
}
return ptr;
}
// adapted to v3
public int getProdLength(int i) {
if (i >= nProds) throw fail("Symbol index out of range: " + i);
if (i < iFirstPair) return 1;
return readCompressed30BitUint_IByteMemory64(mem, symbolPtr(i)+2*4);
}
// adapted to v3
public int getOccurrences(int i) {
if (i >= nProds) throw fail("Symbol index out of range: " + i);
long ptr = indexOfOccurrences(i);
return readCompressed30BitUint_IByteMemory64(mem, ptr);
}
long indexOfUnion(int i) {
long ptr = indexOfOccurrences(i);
return ptr+compressed30BitUint_lengthFromFirstByte(mem.getByte(ptr));
}
// i == iFirstFile is OK because we need to know where last symbol ends
long symbolPtr(int i) {
if (i < 0 || i > iFirstFile) throw fail("Symbol out of range: " + i + " / " + iFirstFile);
int ptr = symbolPtr_raw(i);
int iHigh = i >> highBytesShift;
long fullPtr = symbolPtrHighBytes[iHigh+(ptr < 0 ? 0 : 1)] | uintToLong(ptr);
return fullPtr;
}
int symbolPtr_raw(int i) {
return memGet(iSymbolPointers+i*4L);
}
String its(int idx) { return itemToString(idx); }
public String itemToString(int idx) {
return withStringBuilder(getProdLength(idx), buf -> itemToString_step(buf, idx));
}
// i = symbol index
long getPair(int i) {
long _ptr = symbolPtr(i);
long p;
try {
p = memGetLong(_ptr);
} catch (Throwable _e) {
print("Was getting pair " + intToHex(i) + " at " + longToHex(_ptr));
throw rethrow(_e); }
return p;
}
void itemToString_step(StringBuilder buf, int idx) {
if (idx < iFirstPair) { buf.append(literals[idx]); return; }
long p = getPair(idx);
try {
itemToString_step(buf, firstIntFromLong(p));
itemToString_step(buf, secondIntFromLong(p));
} catch (Throwable _e) {
print("Was expanding pair: " + intToHex(idx) + " [" + longToHex(p) + "]");
throw rethrow(_e); }
}
public void close() {
{ cleanUp(mem); mem = null; }
}
public int lookupLiteral(char c) {
return or(literalIndex.get(c), -1);
}
public int lookupLiteral(String c) {
return lookupLiteral(stringToChar(c));
}
public int getLiteral(int i) {
return (int) literals[i];
}
List symbolToFiles(int symbol) {
return symbolToFile.getOpt(symbol);
}
int[] getLeftPairs(int symbol) {
return readIntArray(leftPairsRange(symbol));
}
int[] readIntArray(LongRange r) {
return r == null ? null : readIntArray(r.start, toInt(r.length() >> 2));
}
int[] readIntArray(long start, int len) {
int[] a = new int[len];
for (int i = 0; i < len; i++)
a[i] = memGet(start+i*4L);
return a;
}
int memGet(long ptr) {
if (onMemAccess != null) onMemAccess.get(ptr);
return mem.getInt(ptr);
}
long memGetLong(long ptr) {
return twoIntsToLong(memGet(ptr), memGet(ptr+4));
}
// adapted to v3
LongRange leftPairsRange(int symbol) {
if (symbol >= nProds) throw fail("Symbol index out of range: " + symbol);
long ptr = indexOfUnion(symbol);
long lengthOfUnion = symbolPtr(symbol+1)-ptr;
if (lengthOfUnion != 4) {
int n = readCompressed30BitUint_IByteMemory64(mem, ptr);
return longRangeWithLength(ptr+compressed30BitUint_lengthForValue(n), n*4);
}
// one parent case - check if it's a left parent
if (mem.getInt(ptr) >= 0) return new LongRange(ptr, ptr+4); else return null;
}
// adapted to v3
LongRange rightPairsRange(int symbol) {
if (symbol >= nProds) throw fail("Symbol index out of range: " + symbol);
long ptr = indexOfUnion(symbol);
long nextSymbol = symbolPtr(symbol+1);
long lengthOfUnion = nextSymbol-ptr;
if (lengthOfUnion != 4) {
int nLeftPairs = readCompressed30BitUint_IByteMemory64(mem, ptr);
ptr += compressed30BitUint_lengthForValue(nLeftPairs) + nLeftPairs*4L;
return new LongRange(ptr, nextSymbol);
}
// one parent case - check if it's a right parent
if (mem.getInt(ptr) < 0) return new LongRange(ptr, ptr+4); else return null;
}
int[] getRightPairs(int symbol) {
int[] a = readIntArray(rightPairsRange(symbol));
if (l(a) == 1)
a[0] &= 0x7FFFFFFF;
return a;
}
public IntegerIterator getLeftPairsEndingWith(int symbol, String from) { return getLeftPairsEndingWith(symbol, from, null); }
public IntegerIterator getLeftPairsEndingWith(int symbol, String from, String to) {
LongRange r = leftPairsRange(symbol);
if (r == null) {
if (protocol != null) protocol.searched(this, false, symbol, reversed(from), null);
return null;
}
if (empty(from)) return memArrayIntegerIterator_symbols(r);
IntBuffer out = newIntBuffer();
dictionarySearch(out, false, 0, r.start, (int) (r.length() >> 2), from);
if (protocol != null) protocol.searched(this, false, symbol, reversed(from), out);
return out.integerIterator();
}
public IntegerIterator getRightPairsStartingWith(int symbol, String from) { return getRightPairsStartingWith(symbol, from, null); }
public IntegerIterator getRightPairsStartingWith(int symbol, String from, String to) {
LongRange r = rightPairsRange(symbol);
if (r == null) {
if (protocol != null) protocol.searched(this, true, symbol, from, null);
return null;
}
if (empty(from)) return memArrayIntegerIterator_symbols(r);
IntBuffer out = newIntBuffer();
dictionarySearch(out, true, 0, r.start, (int) (r.length() >> 2), from);
if (protocol != null) protocol.searched(this, true, symbol, from, out);
return out.integerIterator();
}
void dictionarySearch(IntBuffer out, boolean toTheRight, int knownCharacters, long ptr, int size, String prefix) {
ping();
LCSearcher_Protocol protocol = doSubProtocol ? this.protocol : null;
if (protocol != null)
protocol.log("dictionarySearch " + (toTheRight ? "right" : "left") + " ptr=" + ptr + ", size " + size + ", knownCharacters=" + knownCharacters + ", prefix=" + quote(prefix));
if (size == 0) return; // Nothing to do for size 0
if (size == 1) { // only one symbol to check
int sym = memGet(ptr) & 0x7FFFFFFF;
int part = toTheRight ? pairRight(sym) : pairLeft(sym);
if (protocol != null) protocol.log("Single symbol: " + symbolToProtocol(part));
// see how many new characters we have in common with the search string
int common = lCommonPrefixOfCharIterators(
itemCharIteratorWithSkip(toTheRight, part, knownCharacters),
characterIterator(prefix, knownCharacters), charComparator);
int have = knownCharacters+common;
int need = min(getProdLength(part), l(prefix));
if (have < need) { // some characters not matched, it's a fail
if (protocol != null) protocol.log("Symbol check fail (" + have + "/" + need + "): " + symbolToProtocol(part));
return;
}
// Add symbol. We like it
if (protocol != null) protocol.log("Adding symbol: " + symbolToProtocol(sym));
out.add(sym);
return;
}
// More than one symbol to check. Get common prefix length of first & last symbol in range
int sym1 = memGet(ptr);
int sym2 = memGet(ptr+(size-1)*4L);
int part1 = !toTheRight ? pairLeft(sym1) : pairRight(sym1);
int part2 = !toTheRight ? pairLeft(sym2) : pairRight(sym2);
CharacterIterator it1 = itemCharIteratorWithSkip(toTheRight, part1, knownCharacters);
CharacterIterator it2 = itemCharIteratorWithSkip(toTheRight, part2, knownCharacters);
int lPrefix = prefix.length(), iPrefix = knownCharacters;
if (protocol != null) protocol.log("Range: " + symbolToProtocol(part1, "toTheRight", toTheRight) + " to " + symbolToProtocol(part2, "toTheRight", toTheRight) + " [size " + size + "]");
int common = 0;
char c1 = 0, c2 = 0, cPrefix = 0;
boolean grabbed = false; // did we grab 3 characters already
while (it1.hasNext() && it2.hasNext() && iPrefix < lPrefix) {
c1 = it1.next();
c2 = it2.next();
cPrefix = prefix.charAt(iPrefix++);
/*if (protocol != null)
protocol.log("grabbed " + quote(c1) + ", " + quote(c2) + ", " + quote(cPrefix));*/
if (compareChars(charComparator, c1, c2) == 0 && compareChars(charComparator, c1, cPrefix) == 0)
++common;
else {
grabbed = true;
break;
}
}
if (protocol != null && common != 0)
print("New common chars: " + common);
if (!grabbed && iPrefix >= lPrefix) { // prefix exhausted - add all elements
for (int i = 0; i < size; i++) {
int sym = memGet(ptr+i*4L);
if (protocol != null)
protocol.log("Adding symbol: " + symbolToProtocol(sym));
out.add(sym);
}
return;
}
// prefix not exhausted, check next character in each iterator
if (!grabbed) cPrefix = prefix.charAt(iPrefix++);
boolean hasNext1 = grabbed || it1.hasNext();
if (hasNext1 && !grabbed) c1 = it1.next();
boolean hasNext2 = grabbed || it2.hasNext();
if (hasNext2 && !grabbed) c2 = it2.next();
if (hasNext1 && hasNext2 && compareChars(charComparator, c2, cPrefix) < 0) { // whole range is above prefix, skip
if (protocol != null)
protocol.log("Range is above prefix: c1="
+ (hasNext1 ? quote(c1) + " [" + (int) c1 + "]" : "none")
+ ", c2=" + (hasNext2 ? quote(c2) : "none")
+ ", cPrefix=" + cPrefix + ", " + symbolToProtocol(sym1) + "/" + symbolToProtocol(sym2));
return;
}
if (hasNext1 && hasNext2 && compareChars(charComparator, c1, cPrefix) > 0) { // whole range is below prefix, skip
if (protocol != null)
protocol.log("Range is below prefix: " + symbolToProtocol(sym1) + "/" + symbolToProtocol(sym2));
return;
}
// subdivide
knownCharacters += common;
int split = size/2;
if (protocol != null)
protocol.log("Subdividing. split point " + split);
dictionarySearch(out, toTheRight, knownCharacters, ptr, split, prefix);
dictionarySearch(out, toTheRight, knownCharacters, ptr+split*4L, size-split, prefix);
}
class ItemCharIterator extends CharacterIterator {
boolean toTheRight = true;
IntBuffer stack = newIntBuffer();
ItemCharIterator(boolean toTheRight, int symbol) {
this.toTheRight = toTheRight; stack.add(symbol); }
ItemCharIterator(int symbol) { stack.add(symbol); }
ItemCharIterator(int symbol, int skip) { this(true, symbol, skip); }
ItemCharIterator(boolean toTheRight, int symbol, int skip) {
this.toTheRight = toTheRight;
stack.add(symbol);
while (skip > 0 && !stack.isEmpty()) {
int idx = stack.popLast();
int len = getProdLength(idx);
if (skip >= len) { skip -= len; continue; } // skip over symbol completely
// Now we know it's a pair (because skip >= 1)
long p = getPair(idx);
int a = firstIntFromLong(p), b = secondIntFromLong(p);
if (!toTheRight) { { int __1 = a; a = b; b = __1; } }
len = getProdLength(a);
if (skip >= len) { // we can skip a, just post b
stack.add(b);
skip -= len;
continue;
}
// we have to process both a & b (push in reverse order)
stack.add(b);
stack.add(a);
}
}
public boolean hasNext() { return !stack.isEmpty(); }
public char next() {
if (stack.isEmpty()) throw fail("no more elements");
while (true) {
int idx = stack.popLast();
if (idx < iFirstPair) return literals[idx];
long p = getPair(idx);
int a = firstIntFromLong(p), b = secondIntFromLong(p);
if (!toTheRight) { { int __2 = a; a = b; b = __2; } }
stack.add(b);
stack.add(a);
}
}
}
CharacterIterator itemCharIterator(int idx) {
return new ItemCharIterator(idx);
}
CharacterIterator itemCharIteratorWithSkip(int symbol, int skip) {
return new ItemCharIterator(symbol, skip);
}
CharacterIterator itemCharIteratorWithSkip(boolean toTheRight, int symbol, int skip) {
return new ItemCharIterator(toTheRight, symbol, skip);
}
CharacterIterator itemCharIteratorWithSkip(boolean toTheRight, int symbol) {
return new ItemCharIterator(toTheRight, symbol);
}
CharacterIterator itemReverseCharIterator(int idx) {
return new ItemCharIterator(false, idx);
}
CharacterIterator itemReverseCharIteratorWithSkip(int symbol, int skip) {
return new ItemCharIterator(false, symbol, skip);
}
int numberOfSymbols() { return iFirstFile; }
void makeHighBytes() {
int step = 1 << highBytesShift, lastPtr = 0;
int steps = idiv_ceil(numberOfSymbols(), step);
symbolPtrHighBytes = new long[steps+1];
long high = (iSymbolPointers+(numberOfSymbols()+1L)*4) & ~0xFFFFFFFFL;
int idx = 0;
for (int i = 0; i < numberOfSymbols(); i += step) {
int ptr = symbolPtr_raw(i);
if (uintToLong(ptr) < uintToLong(lastPtr)) {
assertTrue(lastPtr < 0);
//print("INC POINT " + idx + " (last: " + intToHex(lastPtr) + ")");
high += 0x100000000L;
}
/*if (idx >= 0)*/ symbolPtrHighBytes[idx] = high;
++idx;
lastPtr = ptr;
}
symbolPtrHighBytes[steps] = high;
printStruct("symbolPtrHighBytes", symbolPtrHighBytes);
}
String symbolToProtocol(int symbol, Object... __) {
boolean toTheRight = optPar("toTheRight",__, true);
boolean withIdx = boolPar("withIdx",__);
int max = 40;
CharacterIterator it = toTheRight
? itemCharIterator(symbol)
: itemReverseCharIterator(symbol);
String s = charIteratorToString_max(max+1, it);
if (l(s) > max)
return quote(shorten(s, max)) + " [length " + getProdLength(symbol) + (!withIdx ? "" : ", idx " + symbol) + "]";
else
return quote(s) + (!withIdx ? "" : " [idx " + symbol + "]");
}
void assertLeftPairsAreSorted(int symbol) {
int[] lpairs = getLeftPairs(symbol);
print("pairs", l(lpairs));
String last = "";
for (int i = 0; i < l(lpairs); i++) {
int sym = lpairs[i];
String s = reversed(itemToString(sym));
print(i + "/" + l(lpairs) + ": " + quote(s));
assertTrue(cmp(s, last) >= 0);
last = s;
}
}
// file starts at 0
int getFile(int file) {
return files[file];
}
int nFiles() { return l(files); }
boolean isPair(int symbol) { return symbol >= iFirstPair && symbol < iFirstFile; }
// iFile starts at 0
String filePartToString(int iFile, int from, int to) {
return charIteratorToString_max(to-from, itemCharIteratorWithSkip(getFile(iFile), from));
}
// iFile starts at 0
int getFileLength(int iFile) {
return getProdLength(getFile(iFile));
}
IntegerIterator memArrayIntegerIterator_symbols(LongRange r) {
return new IntegerIterator() {
long i = r.start, end = r.end;
public boolean hasNext() { return i < end; }
public int next() { int val = memGet(i) & 0x7FFFFFFF; i += 4; return val; }
};
}
int numLeftPairs(int symbol) { return (int) (l(leftPairsRange(symbol)) >> 2); }
int numRightPairs(int symbol) { return (int) (l(rightPairsRange(symbol)) >> 2); }
int nLiterals() { return iFirstPair; }
/*sclass SymbolInfo {
int symbol;
int left, right; // pair contents (if pair)
int length, occurrences;
bool oneParent; // symbol with only one parent
long leftPairsStart, leftPairsEnd;
long rightPairsStart, rightPairsEnd;
}*/
IntBuffer newIntBuffer() { return new IntBuffer(intBufferEmptyCapacity); }
}static class LCSearcher_v4_compact implements IFieldsToList{
LCSearcher_v1_compact lc;
LCSearcher_v4_compact() {}
LCSearcher_v4_compact(LCSearcher_v1_compact lc) {
this.lc = lc;}
public String toString() { return shortClassName(this) + "(" + lc + ")"; }public Object[] _fieldsToList() { return new Object[] {lc}; }
// flags user can set
boolean log = false;
boolean importMagic = false; // unused
boolean skipBorderChars = false; // unused
boolean stopAfterFirstSymbol = false;
// internal
String query;
int lQuery;
char[] queryChars;
int[] queryLiterals;
String its(int i) { return lc.itemToString(i); }
// return false if a literal wasn't found
boolean processQuery(String query) {
// store query, convert to literals
this.query = query;
if ((lQuery = l(query)) == 0) return false;
queryChars = toCharArray(query);
queryLiterals = new int[lQuery];
for (int iChar = 0; iChar < lQuery; iChar++) {
char c = queryChars[iChar];
if ((queryLiterals[iChar] = lc.lookupLiteral(c)) < 0) {
if (log) {
print("Missing literal: " + quote(c));
//print("Have literals: " + lc.literalIndex);
}
return false;
}
}
return true;
}
public UpScan rawScan(String query) {
if (!processQuery(query)) return null;
UpScan scan = new UpScan();
scan.run();
return scan;
}
// represents a symbol positioned at a certain offset in the query
// (which may be negative if it starts to the left of the query)
static class OfsAndSymbol implements IFieldsToList{
int ofs;
int symbol;
OfsAndSymbol() {}
OfsAndSymbol(int ofs, int symbol) {
this.symbol = symbol;
this.ofs = ofs;}
public String toString() { return shortClassName(this) + "(" + ofs + ", " + symbol + ")"; }
public boolean equals(Object o) {
if (!(o instanceof OfsAndSymbol)) return false;
OfsAndSymbol __1 = (OfsAndSymbol) o;
return ofs == __1.ofs && symbol == __1.symbol;
}
public int hashCode() {
int h = -159510445;
h = boostHashCombine(h, _hashCode(ofs));
h = boostHashCombine(h, _hashCode(symbol));
return h;
}
public Object[] _fieldsToList() { return new Object[] {ofs, symbol}; }
}
static interface SearchResult {
int resultCount();
public int[] indices_unorderedIntArray();
public int[] indices_orderedIntArray();
void _continue();
}
class UpScan implements SearchResult {
LongBuffer queue = new LongBuffer(); // twoIntsToLong(ofs, symbol)
LongHashSet seen = new LongHashSet(); // twoIntsToLong(ofs, symbol)
LongBuffer fullSymbols = new LongBuffer();
String[] needLeft_from = new String[lQuery+1], needLeft_to = new String[lQuery+1];
String[] needRight_from = new String[lQuery+1], needRight_to = new String[lQuery+1];
public int resultCount() {
int n = fullSymbols.size(), sum = 0;
for (int i = 0; i < n; i++)
sum += lc.getOccurrences(secondIntFromLong(fullSymbols.get(i)));
return sum;
}
Collection seen() {
List out = new ArrayList();
org.eclipse.collections.api.iterator.LongIterator it = seen.longIterator();
while (it.hasNext()) {
long l = it.next();
out.add(new OfsAndSymbol(firstIntFromLong(l), secondIntFromLong(l)));
}
return out;
}
boolean addOfsAndSymbol(int ofs, int symbol) {
if (!addToQueue(ofs, symbol)) return false;
if (log)
print("Candidate: " + quoteShorten(40, its(symbol)) + " at " + ofs);
return true;
}
boolean addToQueue(int ofs, int symbol) {
long l = twoIntsToLong(ofs, symbol);
if (!seen.add(l)) return false;
queue.add(l);
return true;
}
void start() {
int iChar = lQuery/2; // start somewhere (just not on the edges)
if (addToQueue(iChar, queryLiterals[iChar]) && log)
print("Base literal: " + queryChars[iChar] + " at " + iChar);
}
public void run() {
start();
_continue();
}
public void _continue() {
while (!queue.isEmpty()) {
long oas = queue.popLast();
int ofs = firstIntFromLong(oas), symbol = secondIntFromLong(oas);
int _ofs = max(ofs, 0);
int ofs2 = ofs+lc.getProdLength(symbol); // end of symbol in query
int _ofs2 = min(ofs2, lQuery);
if (_ofs == 0 && _ofs2 == lQuery) {
fullSymbols.add(oas);
if (stopAfterFirstSymbol) break;
continue;
}
String needOnTheLeft_from = needLeft_from[_ofs], needOnTheLeft_to;
if (needOnTheLeft_from == null) {
needOnTheLeft_from = needLeft_from[_ofs] = reversedSubstring(query, 0, _ofs);
needOnTheLeft_to = needLeft_to[_ofs] = needOnTheLeft_from + Character.MAX_VALUE;
} else
needOnTheLeft_to = needLeft_to[_ofs];
String needOnTheRight_from = needRight_from[_ofs2], needOnTheRight_to;
if (needOnTheRight_from == null) {
needOnTheRight_from = needRight_from[_ofs2] = substring(query, _ofs2);
needOnTheRight_to = needRight_to[_ofs2] = needOnTheRight_from + Character.MAX_VALUE;
} else
needOnTheRight_to = needRight_to[_ofs2];
if (log) printVars_str("UpScan right", "oas" , quote(its(symbol)) + " at " + ofs, "needOnTheRight_from", needOnTheRight_from);
// right pairs
IntegerIterator setRight = lc.getRightPairsStartingWith(symbol, needOnTheRight_from, needOnTheRight_to);
if (log) {
print("setRight", setRight);
//print(allRightPairs := lc.getRightPairs(symbol));
}
if (setRight != null) {
int n = 0;
while (setRight.hasNext()) {
addOfsAndSymbol(ofs, setRight.next());
++n;
}
//if (ofs2-ofs == 1) print("right for " + queryChars[ofs] + ": " + n);
}
if (log) printVars_str("UpScan left", "oas" , quote(its(symbol)) + " at " + ofs, "needOnTheLeft_from", needOnTheLeft_from, "needOnTheLeft_to", needOnTheLeft_to);
// left pairs
//Cl leftAll = lc.getLeftPairs(symbol);
//print(leftAll := lmap its(leftAll));
IntegerIterator setLeft = lc.getLeftPairsEndingWith(symbol, needOnTheLeft_from, needOnTheLeft_to);
if (setLeft != null) {
if (log) print("setLeft", setLeft);
int n = 0;
while (setLeft.hasNext()) {
int i = setLeft.next();
addOfsAndSymbol(ofs2-lc.getProdLength(i), i);
++n;
}
//if (ofs2-ofs == 1) print("left for " + queryChars[ofs] + ": " + n);
}
}
}
public int[] indices_unorderedIntArray() {
//print("full", l(fullSymbols) + ": " + map(i -> firstIntFromLong(i) + "/" + quote(lc.its(secondIntFromLong(i))), fullSymbols));
LongBuffer queue = new LongBuffer(fullSymbols);
IntBuffer result = new IntBuffer();
while (!queue.isEmpty()) {
long oas = queue.popLast();
int ofs = firstIntFromLong(oas), symbol = secondIntFromLong(oas);
List files = lc.symbolToFiles(symbol);
if (files != null) {
if (log) print("File level: " + -ofs);
result.add(-ofs);
}
for (int sym : unnull(lc.getLeftPairs(symbol))) {
int ofs2 = ofs-lc.getProdLength(lc.pairLeft(sym));
if (log) print(ofs2 + "/" + quoteShorten(20, lc.its(sym)));
long p = twoIntsToLong(ofs2, sym);
queue.add(p);
}
for (int sym : unnull(lc.getRightPairs(symbol))) {
if (log) print(ofs + "/" + quoteShorten(20, lc.its(sym)));
long p = twoIntsToLong(ofs, sym);
queue.add(p);
}
}
return result.toArray();
}
public int[] indices_orderedIntArray() {
return sortIntArrayInPlace(indices_unorderedIntArray());
}
}
int[] indicesOf(String query) {
SearchResult result = search(query);
return result == null ? null : result.indices_orderedIntArray();
}
int resultCount(String query) {
SearchResult result = search(query);
return result == null ? 0 : result.resultCount();
}
UpScan search(String query) {
if (!processQuery(query)) return null;
UpScan scan = new UpScan();
scan.run();
return scan;
}
void printStats() {
}
}static class LCSearcher_v4_multiFile implements IFieldsToList{
ILCCompactIndex lc;
LCSearcher_v4_multiFile() {}
LCSearcher_v4_multiFile(ILCCompactIndex lc) {
this.lc = lc;}public Object[] _fieldsToList() { return new Object[] {lc}; }
// flags user can set
boolean log = false;
boolean queriesToUpper = false;
transient IF0 getMem;
Object getMem() { return getMem != null ? getMem.get() : getMem_base(); }
final Object getMem_fallback(IF0 _f) { return _f != null ? _f.get() : getMem_base(); }
Object getMem_base() { return null; } // for statistical purposes
ILCCompactIndex getILCCompactIndex() { return lc; }
String its(int i) { return lc.itemToString(i); }
// represents a symbol positioned at a certain offset in the query
// (which may be negative if it starts to the left of the query)
static class OfsAndSymbol implements IFieldsToList{
int ofs;
int symbol;
OfsAndSymbol() {}
OfsAndSymbol(int ofs, int symbol) {
this.symbol = symbol;
this.ofs = ofs;}
public String toString() { return shortClassName(this) + "(" + ofs + ", " + symbol + ")"; }
public boolean equals(Object o) {
if (!(o instanceof OfsAndSymbol)) return false;
OfsAndSymbol __1 = (OfsAndSymbol) o;
return ofs == __1.ofs && symbol == __1.symbol;
}
public int hashCode() {
int h = -159510445;
h = boostHashCombine(h, _hashCode(ofs));
h = boostHashCombine(h, _hashCode(symbol));
return h;
}
public Object[] _fieldsToList() { return new Object[] {ofs, symbol}; }
}
static interface SearchResult {
int resultCount();
public long[] multiFileIndices_unorderedArray();
public long[] multiFileIndices_orderedArray();
void _continue();
LongIterator unorderedIterator();
int nSymbolsLookedAt();
}
class Search implements SearchResult {
LCSearcher_Protocol protocol;
String query;
int lQuery;
char[] queryChars;
int[] queryLiterals;
boolean stopAfterFirstSymbol = false;
//new LongBuffer queue; // twoIntsToLong(ofs, symbol)
//LongPriorityQueue queue = new(queueComparator()); // twoIntsToLong(ofs, symbol)
ILongQueue queue = new LongPriorityQueue(queueComparator());
LongHashSet seen = new LongHashSet(); // twoIntsToLong(ofs, symbol)
LongBuffer fullSymbols = new LongBuffer();
String[] needLeft;
String[] needRight;
void switchToSimpleQueue() {
queue = new LongBuffer();
}
LongComparator queueComparator() {
// prioritize shorter symbols
/*ret (a, b) -> cmp(
lc.getProdLength(secondIntFromLong(a)),
lc.getProdLength(secondIntFromLong(b)));*/
// prioritize symbols contained in query
return (a, b) -> cmp(charsOutsideOfQuery(a), charsOutsideOfQuery(b));
}
int charsOutsideOfQuery(long oas) {
int ofs = firstIntFromLong(oas), symbol = secondIntFromLong(oas);
return max(-ofs, 0) + max(ofs+lc.getProdLength(symbol)-lQuery, 0);
}
// return false if a literal wasn't found
boolean processQuery(String query) {
// store query, convert to literals
if (queriesToUpper) query = upper(query);
this.query = query;
if ((lQuery = l(query)) == 0) return false;
queryChars = toCharArray(query);
queryLiterals = new int[lQuery];
for (int iChar = 0; iChar < lQuery; iChar++) {
char c = queryChars[iChar];
if ((queryLiterals[iChar] = lc.lookupLiteral(c)) < 0) {
if (protocol != null) protocol.log("Missing literal: " + quote(c));
//print("Have literals: " + lc.literalIndex);
return false;
}
}
needLeft = new String[lQuery+1];
needRight = new String[lQuery+1];
return true;
}
public int resultCount() {
int n = fullSymbols.size(), sum = 0;
print("Getting result count for " + nSymbols(n));
for (int i = 0; i < n; i++) {
sum += lc.getOccurrences(secondIntFromLong(fullSymbols.get(i)));
if (divides(10000, i))
print("Results after " + nSymbols(i) + ": " + n2(sum));
}
return sum;
}
Collection seen() {
List out = new ArrayList();
org.eclipse.collections.api.iterator.LongIterator it = seen.longIterator();
while (it.hasNext()) { ping();
long l = it.next();
out.add(new OfsAndSymbol(firstIntFromLong(l), secondIntFromLong(l)));
}
return out;
}
// this is called from other threads. no biggie though
public int nSymbolsLookedAt() { return seen.size(); }
boolean addOfsAndSymbol(int ofs, int symbol) {
if (!addToQueue(ofs, symbol)) return false;
if (log)
print("Candidate: " + quoteShorten(40, its(symbol)) + " at " + ofs);
if (protocol != null) {
int lPair = lc.getProdLength(symbol);
String overlapped = rpad(rep('_', ofs)
+ substring(query, ofs, ofs+lPair), lQuery, '_');
protocol.log("Found pair " + overlapped + " " + lc.symbolToProtocol(symbol, "withIdx" , true) + " at " + ofs);
}
return true;
}
boolean addToQueue(int ofs, int symbol) {
long l = twoIntsToLong(ofs, symbol);
if (!seen.add(l)) return false;
if (ofs <= 0 && ofs+lc.getProdLength(symbol) >= lQuery) {
if (protocol != null) protocol.log("FOUND: " + lc.symbolToProtocol(symbol) /*+ " at [" + ofs + "-" + ofs2 + "]"*/);
fullSymbols.add(twoIntsToLong(ofs, symbol));
return false;
}
queue.add(l);
return true;
}
boolean start(String query) {
if (!processQuery(query)) return false;
int iChar = lQuery/2; // start somewhere (just not on the edges)
addToQueue(iChar, queryLiterals[iChar]);
if (protocol != null) protocol.log("Base literal: " + queryChars[iChar] + " at " + iChar);
return true;
}
public void _continue() {
if (stopAfterFirstSymbol)
while (!queue.isEmpty()) { ping();
step();
if (!fullSymbols.isEmpty()) break;
}
else
while (!queue.isEmpty()) { ping(); step(); }
}
void step() {
long oas = queue.poll/*popLast*/();
int ofs = firstIntFromLong(oas), symbol = secondIntFromLong(oas);
int _ofs = max(ofs, 0);
int ofs2 = ofs+lc.getProdLength(symbol); // end of symbol in query
int _ofs2 = min(ofs2, lQuery);
// LEFT SEARCH
String needOnTheLeft = needLeft[_ofs];
if (needOnTheLeft == null)
needOnTheLeft = needLeft[_ofs] = reversedSubstring(query, 0, _ofs);
if (log) printVars_str("Search left", "oas" , quote(its(symbol)) + " at " + ofs, "needOnTheLeft", needOnTheLeft);
// left pairs
IntegerIterator setLeft = lc.getLeftPairsEndingWith(symbol, needOnTheLeft);
if (setLeft != null) {
if (log) print("setLeft", setLeft);
//int n = 0;
while (setLeft.hasNext()) { ping();
int i = setLeft.next();
addOfsAndSymbol(ofs2-lc.getProdLength(i), i);
//++n;
}
//if (ofs2-ofs == 1) print("left for " + queryChars[ofs] + ": " + n);
}
// RIGHT SEARCH
String needOnTheRight = needRight[_ofs2];
if (needOnTheRight == null)
needOnTheRight = needRight[_ofs2] = substring(query, _ofs2);
if (log) printVars_str("Search right", "oas" , quote(its(symbol)) + " at " + ofs, "needOnTheRight", needOnTheRight);
// right pairs
IntegerIterator setRight = lc.getRightPairsStartingWith(symbol, needOnTheRight);
if (log) {
print("setRight", setRight);
//print(allRightPairs := lc.getRightPairs(symbol));
}
if (setRight != null) {
//int n = 0;
while (setRight.hasNext()) { ping();
addOfsAndSymbol(ofs, setRight.next());
//++n;
}
//if (ofs2-ofs == 1) print("right for " + queryChars[ofs] + ": " + n);
}
} // end of step
public long[] multiFileIndices_unorderedArray() {
//print("full", l(fullSymbols) + ": " + map(i -> firstIntFromLong(i) + "/" + quote(lc.its(secondIntFromLong(i))), fullSymbols));
LongBuffer queue2 = new LongBuffer(fullSymbols);
LongBuffer result = new LongBuffer();
while (!queue2.isEmpty()) { ping();
long oas = queue2.popLast();
int ofs = firstIntFromLong(oas), symbol = secondIntFromLong(oas);
List files = lc.symbolToFiles(symbol);
if (files != null) {
if (log) print("File level: " + -ofs);
result.add(twoIntsToLong(first(files), -ofs));
}
for (int sym : unnull(lc.getLeftPairs(symbol))) {
int ofs2 = ofs-lc.getProdLength(lc.pairLeft(sym));
if (log) print(ofs2 + "/" + quoteShorten(20, lc.its(sym)));
queue2.add(twoIntsToLong(ofs2, sym));
}
for (int sym : unnull(lc.getRightPairs(symbol))) {
if (log) print(ofs + "/" + quoteShorten(20, lc.its(sym)));
queue2.add(twoIntsToLong(ofs, sym));
}
}
return result.toArray();
}
public LongIterator unorderedIterator() {
LongBuffer queue2 = new LongBuffer(fullSymbols);
return new LongIterator() {
public boolean hasNext() {
return !queue2.isEmpty();
}
public long next() {
while (true) { ping();
long oas = queue2.popLast();
int ofs = firstIntFromLong(oas), symbol = secondIntFromLong(oas);
List files = lc.symbolToFiles(symbol);
if (files != null) {
if (log) print("File level: " + -ofs);
return twoIntsToLong(first(files), -ofs);
}
for (int sym : unnull(lc.getLeftPairs(symbol))) {
int ofs2 = ofs-lc.getProdLength(lc.pairLeft(sym));
if (log) print(ofs2 + "/" + quoteShorten(20, lc.its(sym)));
queue2.add(twoIntsToLong(ofs2, sym));
}
for (int sym : unnull(lc.getRightPairs(symbol))) {
if (log) print(ofs + "/" + quoteShorten(20, lc.its(sym)));
queue2.add(twoIntsToLong(ofs, sym));
}
}
}
};
}
public LongIterator unorderedIterator_lazy() {
return new LongIterator() {
int fullSymbolsSeen = 0;
LongBuffer myQueue = new LongBuffer();
long value = -1;
public boolean hasNext() {
if (value == -1) grab();
return value >= 0;
}
void grab() {
processQueue();
if (value >= 0) return;
fillQueue();
processQueue();
}
void fillQueue() {
while (fullSymbols.size() <= fullSymbolsSeen && !queue.isEmpty())
{ ping(); step(); }
if (fullSymbols.size() > fullSymbolsSeen)
myQueue.add(fullSymbols.get(fullSymbolsSeen++));
}
void processQueue() {
while (!myQueue.isEmpty()) { ping();
long oas = myQueue.popLast();
int ofs = firstIntFromLong(oas), symbol = secondIntFromLong(oas);
List files = lc.symbolToFiles(symbol);
if (files != null) {
if (log) print("File level: " + -ofs);
{ value = twoIntsToLong(first(files), -ofs); return; }
}
for (int sym : unnull(lc.getLeftPairs(symbol))) {
int ofs2 = ofs-lc.getProdLength(lc.pairLeft(sym));
if (log) print(ofs2 + "/" + quoteShorten(20, lc.its(sym)));
myQueue.add(twoIntsToLong(ofs2, sym));
}
for (int sym : unnull(lc.getRightPairs(symbol))) {
if (log) print(ofs + "/" + quoteShorten(20, lc.its(sym)));
myQueue.add(twoIntsToLong(ofs, sym));
}
}
}
public long next() {
long v = value;
value = -1;
return v;
}
};
}
public long[] multiFileIndices_orderedArray() {
return sortLongArrayInPlace(multiFileIndices_unorderedArray());
}
}
long[] multiFileIndicesOf(String query) { return multiFileIndicesOf(query, null); }
long[] multiFileIndicesOf(String query, LCSearcher_Protocol protocol) {
SearchResult result = search(query, false, protocol);
return result == null ? null : result.multiFileIndices_orderedArray();
}
int[] singleFileIndicesOf(String query) { return singleFileIndicesOf(query, null); }
int[] singleFileIndicesOf(String query, LCSearcher_Protocol protocol) {
return secondIntFromLong_onArray(multiFileIndicesOf(query, protocol));
}
int singleFileAnyIndexOf(String query) {
LongIterator it = resultIterator_unordered(query);
return it.hasNext() ? secondIntFromLong(it.next()) : -1;
}
long[] multiFileIndicesOf_unordered(String query) {
SearchResult result = search(query, false);
return result == null ? null : result.multiFileIndices_unorderedArray();
}
int[] singleFileIndicesOf_unordered(String query) {
return secondIntFromLong_onArray(multiFileIndicesOf_unordered(query));
}
boolean contains(String query) { return contains(query, null); }
boolean contains(String query, LCSearcher_Protocol protocol) { return anyResults(query, protocol); }
boolean anyResults(String query) { return anyResults(query, null); }
boolean anyResults(String query, LCSearcher_Protocol protocol) {
Search result = search(query, true, protocol);
return result != null && !result.fullSymbols.isEmpty();
}
int resultCount(String query) { return resultCount(query, null); }
int resultCount(String query, LCSearcher_Protocol protocol) {
SearchResult result = search_simpleQueue(query, protocol);
return result == null ? 0 : result.resultCount();
}
LongIterator resultIterator_unordered(String query) {
Search result = search_noContinue(query);
return result == null ? null : result.unorderedIterator_lazy();
}
Search search(String query, boolean stopAfterFirstSymbol) { return search(query, stopAfterFirstSymbol, null); }
Search search(String query, boolean stopAfterFirstSymbol, LCSearcher_Protocol protocol) {
Search search = new Search();
search.protocol = protocol;
search.stopAfterFirstSymbol = stopAfterFirstSymbol;
if (!search.start(query)) return null;
search._continue();
return search;
}
// don't actually run the search
Search search_noContinue(String query) { return search_noContinue(query, null); }
Search search_noContinue(String query, LCSearcher_Protocol protocol) {
Search scan = new Search();
scan.protocol = protocol;
if (!scan.start(query)) return null;
return scan;
}
// use simple queue (when you need all full symbols anyway)
Search search_simpleQueue(String query) { return search_simpleQueue(query, null); }
Search search_simpleQueue(String query, LCSearcher_Protocol protocol) {
Search search = new Search();
search.protocol = protocol;
search.switchToSimpleQueue();
if (!search.start(query)) return null;
search._continue();
return search;
}
// ofs = start position of pair
static class ShouldFind implements IFieldsToList{
static final String _fieldOrder = "pair ofs desc";
int pair;
int ofs;
ShouldFind() {}
ShouldFind(int pair, int ofs) {
this.ofs = ofs;
this.pair = pair;}
public boolean equals(Object o) {
if (!(o instanceof ShouldFind)) return false;
ShouldFind __2 = (ShouldFind) o;
return pair == __2.pair && ofs == __2.ofs;
}
public int hashCode() {
int h = 1701202476;
h = boostHashCombine(h, _hashCode(pair));
h = boostHashCombine(h, _hashCode(ofs));
return h;
}
public Object[] _fieldsToList() { return new Object[] {pair, ofs}; }
String desc;
public String toString() {
return nempty(desc) ? desc : pair + "/" + ofs;
}
}
List analyzeOccurrence(int iFile, int start, int len, LCSearcher_Protocol protocol) {
String text = lc.filePartToString(iFile, start, start+len);
LCSearcher_TreeWalker2 walker = new LCSearcher_TreeWalker2(lc);
walker.debug = true;
walker.scan(iFile, intRangeWithLen(start, len));
List out = new ArrayList();
for (IntPair p : walker.result) {
int pair = p.a, ofs = p.b;
int lPair = lc.getProdLength(pair);
int left = lc.pairLeft(pair), right = lc.pairRight(pair);
int lLeft = lc.getProdLength(left);
String overlapped = rpad(rep('_', ofs)
+ substring(text, ofs, ofs+lPair), len, '_');
String desc = "Should find pair " + overlapped + " " + lc.symbolToProtocol(pair, "withIdx" , true) + " at " + ofs + ": " + lc.symbolToProtocol(left, "withIdx" , true) + " + " + lc.symbolToProtocol(right, "withIdx" , true);
if (protocol != null) protocol.log(desc);
out.add(set(new ShouldFind(pair, ofs), "desc", desc));
}
return out;
}
int offsetFromResult(long l) { return secondIntFromLong(l); }
}static class Q implements AutoCloseable {
String name = "Unnamed Queue";
List q = synchroLinkedList();
ReliableSingleThread rst = new ReliableSingleThread(new Runnable() { public void run() { try { _run() ;
} catch (Exception __e) { throw rethrow(__e); } } public String toString() { return "_run()"; }});
volatile boolean retired = false;
Q() {}
Q(String name) {
this.name = name;}
void add(Runnable r) {
q.add(r);
_trigger();
}
void addInFront(Runnable r) {
q.add(0, r);
_trigger();
}
void _trigger() {
rst.name = name;
rst.go();
}
void add(Object r) {
add(toRunnable(r));
}
void _run() {
Runnable r;
while (licensed() && !retired && (r = syncPopFirst(q)) != null) {
try { r.run(); } catch (Throwable __e) { _handleException(__e); }
}
}
public void close() { retired = true; } // TODO: interrupt thread
void done() {} // legacy function
boolean isEmpty() { return q.isEmpty(); }
int size() { return q.size(); }
Object mutex() { return q; } // clients can synchronize on this
List snapshot() { return cloneList(q); }
}static class CountingOutputStream extends FilterOutputStream {
long counter;
CountingOutputStream(OutputStream out) { super(out); }
@Override
public void write(int b) throws IOException {
++counter;
out.write(b);
}
@Override
public void write(byte[] b) throws IOException {
counter += b.length;
out.write(b, 0, b.length);
}
@Override
public void write(byte[] b, int off, int len) throws IOException {
if (len == 0) return;
counter += len;
out.write(b, off, len);
}
long getFilePointer() { return counter; }
}static interface IF0 {
A get();
}static class LCSearcher_v1 {
// user needs to fill these:
transient Map literals;
//transient LL productions;
LongBuffer pairs = new LongBuffer();
IntBuffer files = new IntBuffer();
transient int iFirstPair, iFirstFile;
CharComparator charComparator; // optional, used for sorting pairs
transient Map fileNames; // optional
int maxQueryLength = 32;
boolean debug, big; // switch print statements on/off
// internal structures begin here
transient IntIntHashMap literals2 = new IntIntHashMap(); // compact/faster version of literals
transient Map literalIndex;
transient IntIntHashMap prodLengths; // prod index -> length of production in characters
// not used anymore
transient /*new*/ MultiMap prodIndex; // keys: item looked for, value: pair(prod index, position in prod)
// stats
int actualLeftPairSearches, actualRightPairSearches;
transient IntBuffer[] leftPairIndex; // index of where a symbol appears on right side of a pair
// index of where a symbol appears on left side of a pair
transient IntBuffer[] rightPairIndex;
//transient LCSortedPairIndex pairIndex; // the normal pair index (2 symbols -> pair symbol)
transient boolean prepared = false;
void prepare() {
{ if (prepared) return; prepared = true; }
buildIndex();
}
void buildIndex() {
prodLengths = new IntIntHashMap();
// need this for making prodLengths
for (Map.Entry e : literals.entrySet())
literals2.put(e.getKey(), (int) e.getValue());
// make a pair index for compressing queries
pairs.trimToSize();
//making(big, +pairIndex = new LCSortedPairIndex(pairs.toArray(), iFirstPair);
// build the reconstitution index
making(big, "literalIndex", literalIndex = reverseMap(literals));
//if (debug) printVars_str(+iFirstFile, +productions);
making(big, "leftPairIndex, rightPairIndex");
leftPairIndex = new IntBuffer[iFirstFile];
rightPairIndex = new IntBuffer[iFirstFile];
for (int iProd = iFirstPair; iProd < iFirstFile; iProd++) {
long p = pairs.get(iProd-iFirstPair);
int i = secondIntFromLong(p);
IntBuffer buf = leftPairIndex[i];
if (buf == null) leftPairIndex[i] = buf = new IntBuffer();
buf.add(iProd);
i = firstIntFromLong(p);
buf = rightPairIndex[i];
if (buf == null) rightPairIndex[i] = buf = new IntBuffer();
buf.add(iProd);
if (((iProd-iFirstPair) % oneMillion()) == 0) print(nPairs(iProd-iFirstPair) + " processed");
}
if (big) print("Sorting leftPairIndex");
AtomicInteger count = new AtomicInteger();
int chunkSize = 1000;
parallelDo(map(intRangeChunks(iFirstFile, chunkSize), chunk -> new Runnable() { public void run() { try {
for (int i = chunk.start; i < chunk.end; i++) {
IntBuffer buf = leftPairIndex[i];
if (buf == null) continue;
sortIntBuffer_direct(buf, (__1, __2) -> compareItemsFromEnd_distinct(__1, __2));
buf.trimToSize();
int n2 = incAtomicInt(count, l(buf));
if (n2/oneMillion() > (n2-l(buf))/oneMillion()) print(n2(n2, "left pair") + " sorted");
}
} catch (Exception __e) { throw rethrow(__e); } } public String toString() { return "for (int i = chunk.start; i < chunk.end; i++) {\r\n IntBuffer buf = left..."; }}), "queueSize" , oneMillion());
if (big) print("Sorting rightPairIndex");
count.set(0);
parallelDo(map(intRangeChunks(iFirstFile, chunkSize), chunk -> new Runnable() { public void run() { try {
for (int i = chunk.start; i < chunk.end; i++) {
IntBuffer buf = rightPairIndex[i];
if (buf == null) continue;
sortIntBuffer_direct(buf, (__3, __4) -> compareItems_distinct(__3, __4));
buf.trimToSize();
int n2 = incAtomicInt(count, l(buf));
if (n2/oneMillion() > (n2-l(buf))/oneMillion()) print(n2(n2, "right pair") + " sorted");
}
} catch (Exception __e) { throw rethrow(__e); } } public String toString() { return "for (int i = chunk.start; i < chunk.end; i++) {\r\n IntBuffer buf = rightP..."; }}), "queueSize" , oneMillion());
if (debug) print("literalIndex", literalIndex);
if (debug) print("leftPairIndex", leftPairIndex);
if (debug) print("rightPairIndex", rightPairIndex);
}
public int getProdLength(int i) {
throw fail("no longer implemented");
/*if (i < iFirstPair) ret 1; // literal
if (i >= iFirstFile)
ret getProdLength(files.get(i-iFirstFile));
int x = prodLengths.getIfAbsent(i, -1);
if (x >= 0) ret x;
prodLengths.put(i, x = intSum(lmap getProdLength(productions.get(i))));
ret x;*/
}
public Collection files() { return asList(countIterator_exclusive(iFirstFile, nProductions())); }
public int nProductions() { return iFirstFile+l(files); }
CharacterIterator itemCharIterator(int idx) {
return new ItemCharIterator(idx);
}
CharacterIterator itemReverseCharIterator(int idx) {
return new ItemReverseCharIterator(idx);
}
// compare two items lexicographically
public int compareItems(int idx1, int idx2) {
return compareCharIterators(itemCharIterator(idx1), itemCharIterator(idx2), charComparator);
}
public int compareItemsFromEnd(int idx1, int idx2) {
return compareCharIterators(itemReverseCharIterator(idx1), itemReverseCharIterator(idx2), charComparator);
}
// same comparison functions, but keeps distinct symbols non-equal
public int compareItems_distinct(int idx1, int idx2) {
int cmp = compareItems(idx1, idx2);
if (cmp != 0) return cmp;
return idx1-idx2;
}
public int compareItemsFromEnd_distinct(int idx1, int idx2) {
int cmp = compareItemsFromEnd(idx1, idx2);
if (cmp != 0) return cmp;
return idx1-idx2;
}
class ItemCharIterator extends CharacterIterator {
IntBuffer stack = new IntBuffer();
ItemCharIterator(int idx) { stack.add(idx); }
public boolean hasNext() { return !stack.isEmpty(); }
public char next() {
if (stack.isEmpty()) throw fail("no more elements");
while (true) {
int idx = stack.popLast();
if (idx >= iFirstPair) {
long p = pairs.get(idx-iFirstPair);
stack.add(secondIntFromLong(p));
stack.add(firstIntFromLong(p));
} else
return (char) literals2.get(idx);
}
}
}
class ItemReverseCharIterator extends CharacterIterator {
IntBuffer stack = new IntBuffer();
ItemReverseCharIterator(int idx) { stack.add(idx); }
public boolean hasNext() { return !stack.isEmpty(); }
public char next() {
if (stack.isEmpty()) throw fail("no more elements");
while (true) {
int idx = stack.popLast();
if (idx >= iFirstPair) {
long p = pairs.get(idx-iFirstPair);
stack.add(firstIntFromLong(p));
stack.add(secondIntFromLong(p));
} else
return (char) literals2.get(idx);
}
}
}
public List getProduction(int idx) {
if (idx >= iFirstFile)
return ll(files.get(idx-iFirstFile));
if (idx >= iFirstPair)
return intPairToList(longToIntPair(pairs.get(idx-iFirstPair)));
return null;
}
void addPair(long p) {
pairs.add(p);
}
int addFile(List encoding) {
assertEquals(1, l(encoding));
files.add(first(encoding));
return iFirstFile+l(files)-1;
}
}static interface IF2 {
C get(A a, B b);
}static interface IF1 {
B get(A a);
}static class PersistableThrowable {
String className;
String msg;
String stacktrace;
PersistableThrowable() {}
PersistableThrowable(Throwable e) {
if (e == null)
className = "Crazy Null Error";
else {
className = getClassName(e).replace('/', '.');
msg = e.getMessage();
stacktrace = getStackTrace_noRecord(e);
}
}
public String toString() {
return nempty(msg) ? className + ": " + msg : className;
}
}final static class LCSearcher_v1_compact /*extends ILCCompactIndex*/ {
// flags user can set
boolean debug, big; // switch print statements on/off
boolean calculateOccurrences = true; // adds another int per symbol
// all the data
char[] literals;
Map literalIndex;
int iFirstPair, iFirstFile, nProds;
long[] pairs;
int[] prodLengths; // length of production in characters
int[] files; // starting at iFirstFile
int[][] leftPairIndex; // index of where a symbol appears on right side of a pair
int[][] rightPairIndex; // index of where a symbol appears on left side of a pair
MultiMap symbolToFile = new MultiMap();
// optional: occurrence count for each symbol
int[] occurrences;
// stats
transient int actualLeftPairSearches, actualRightPairSearches;
LCSearcher_v1_compact() {}
LCSearcher_v1_compact(LCSearcher_v1 v1) { load(v1); }
// destroys v1 in the process
void load(LCSearcher_v1 v1) {
v1.prepare();
iFirstPair = v1.iFirstPair;
iFirstFile = v1.iFirstFile;
nProds = max(v1.files())+1;
if (big) print("nProds", nProds);
literals = new char[iFirstPair];
for (Map.Entry __0 : _entrySet( v1.literals))
{ int idx = __0.getKey(); char c = __0.getValue(); literals[idx] = c; }
literalIndex = v1.literalIndex;
making(big, "pairs");
pairs = new long[iFirstFile-iFirstPair];
for (int i = iFirstPair; i < iFirstFile; i++) {
List p = v1.getProduction(i);
pairs[i-iFirstPair] = twoIntsToLong(first(p), second(p));
}
making(big, "files");
files = new int[nProds-iFirstFile];
for (int i : v1.files()) {
Integer symbol = first(v1.getProduction(i));
if (symbol != null) {
files[i-iFirstFile] = symbol;
symbolToFile.put(symbol, i);
}
}
making(big, "prodLengths");
prodLengths = new int[nProds];
for (int i = 0; i < nProds; i++) {
int len;
if (i < iFirstPair) len = 1;
else if (i < iFirstFile)
len = prodLengths[pairLeft(i)] + prodLengths[pairRight(i)];
else
len = prodLengths[files[i-iFirstFile]];
prodLengths[i] = len;
}
making(big, "leftPairIndex");
leftPairIndex = new int[iFirstFile][];
for (int i = 0; i < iFirstFile; i++) {
leftPairIndex[i] = intBufferToArray(v1.leftPairIndex[i]);
v1.leftPairIndex[i] = null;
}
making(big, "rightPairIndex");
rightPairIndex = new int[iFirstFile][];
for (int i = 0; i < iFirstFile; i++) {
rightPairIndex[i] = intBufferToArray(v1.rightPairIndex[i]);
v1.rightPairIndex[i] = null;
}
if (calculateOccurrences) {
making(big, "occurrences");
occurrences = new int[iFirstFile];
for (int i = iFirstFile; i < nProds; i++)
occurrences[files[i-iFirstFile]]++;
for (int i = iFirstFile-1; i >= 0; i--) {
int sum = 0;
for (int sym : unnull(leftPairIndex[i])) sum += occurrences[sym];
for (int sym : unnull(rightPairIndex[i])) sum += occurrences[sym];
occurrences[i] += sum;
}
}
}
long getPair(int idx) {
return pairs[idx-iFirstPair];
}
int pairLeft(int idx) {
return firstIntFromLong(pairs[idx-iFirstPair]);
}
int pairRight(int idx) {
return secondIntFromLong(pairs[idx-iFirstPair]);
}
List queryToLiterals(String query) {
List queryList = characters(query);
List queryLiterals = new ArrayList();
for (Character c : queryList) {
Integer iLit = literalIndex.get(c);
if (iLit == null) {
if (debug) print("literal not found");
return null;
}
queryLiterals.add(iLit);
}
return queryLiterals;
}
public int getProdLength(int i) {
return prodLengths[i];
}
class ItemCharIterator extends CharacterIterator {
IntBuffer stack = new IntBuffer();
ItemCharIterator(int idx) { stack.add(idx); }
public boolean hasNext() { return !stack.isEmpty(); }
public char next() {
if (stack.isEmpty()) throw fail("no more elements");
while (true) {
int idx = stack.popLast();
if (idx < iFirstPair) return literals[idx];
long p = pairs[idx-iFirstPair];
stack.add(secondIntFromLong(p));
stack.add(firstIntFromLong(p));
}
}
}
class ItemReverseCharIterator extends CharacterIterator {
IntBuffer stack = new IntBuffer();
ItemReverseCharIterator(int idx) { stack.add(idx); }
public boolean hasNext() { return !stack.isEmpty(); }
public char next() {
if (stack.isEmpty()) throw fail("no more elements");
while (true) {
int idx = stack.popLast();
if (idx < iFirstPair) return literals[idx];
long p = pairs[idx-iFirstPair];
stack.add(firstIntFromLong(p));
stack.add(secondIntFromLong(p));
}
}
}
CharacterIterator itemCharIterator(int idx) {
return new ItemCharIterator(idx);
}
CharacterIterator itemReverseCharIterator(int idx) {
return new ItemReverseCharIterator(idx);
}
String itemsToString(List idx) {
return withStringBuilder(buf -> {
for (int i : unnull(idx))
itemToString_step(buf, i);
});
}
String its(int idx) { return itemToString(idx); }
public String itemToString(int idx) {
return withStringBuilder(getProdLength(idx), buf -> itemToString_step(buf, idx));
}
void itemToString_step(StringBuilder buf, int idx) {
if (idx < iFirstPair) { buf.append(literals[idx]); return; }
long p = pairs[idx-iFirstPair];
itemToString_step(buf, firstIntFromLong(p));
itemToString_step(buf, secondIntFromLong(p));
}
public int lookupLiteral(char c) {
return or(literalIndex.get(c), -1);
}
public int getLiteral(int i) {
return (int) literals[i];
}
int[] getLeftPairs(int symbol) {
return leftPairIndex[symbol];
}
int[] getRightPairs(int symbol) {
return rightPairIndex[symbol];
}
// newer version (faster)
// with from = reversed(s), to = from + Char.MAX_VALUE
public IntegerIterator getLeftPairsEndingWith(int symbol, String from, String to) {
int[] set = leftPairIndex[symbol];
if (debug) print(formatFunctionCall("getLeftPairsEndingWith",symbol, from) + ": " + map(p -> its(pairLeft(p)), intArrayToList(set)));
if (set == null) return null;
if (from.isEmpty()) return integerIterator(set);
++actualLeftPairSearches;
IntBuffer out = new IntBuffer();
getLeftPairsEndingWith_collect(out, set, 0, set.length, from, to);
return out.integerIterator();
}
// older version
public IntegerIterator getLeftPairsEndingWith(int symbol, String s) {
int[] set = leftPairIndex[symbol];
if (debug) print(formatFunctionCall("getLeftPairsEndingWith",symbol, s) + ": " + map(p -> its(pairLeft(p)), intArrayToList(set)));
if (set == null) return null;
if (s.isEmpty()) return integerIterator(set);
// floor and ceiling (inclusive) of the range we are looking for
String from = reversed(s), to = from + Character.MAX_VALUE;
++actualLeftPairSearches;
IntBuffer out = new IntBuffer();
getLeftPairsEndingWith_collect(out, set, 0, set.length, from, to);
return out.integerIterator();
}
// newer version (faster)
// with from = s, to = from + Char.MAX_VALUE
public IntegerIterator getRightPairsStartingWith(int symbol, String from, String to) {
// let's mess with the inner structures of CompactTreeSet
// We can also make a clone of it in case the original changes
int[] set = rightPairIndex[symbol];
if (debug) print(formatFunctionCall("getRightPairsStartingWith",symbol, from) + ": " + map(p -> its(pairRight(p)), intArrayToList(set)));
if (set == null) return null;
if (from.isEmpty()) return integerIterator(set);
++actualRightPairSearches;
IntBuffer out = new IntBuffer();
getRightPairsStartingWith_collect(out, set, 0, set.length, from, to);
return out.integerIterator();
}
// older version
public IntegerIterator getRightPairsStartingWith(int symbol, String s) {
// let's mess with the inner structures of CompactTreeSet
// We can also make a clone of it in case the original changes
int[] set = rightPairIndex[symbol];
if (debug) print(formatFunctionCall("getRightPairsStartingWith",symbol, s) + ": " + map(p -> its(pairRight(p)), intArrayToList(set)));
if (set == null) return null;
if (s.isEmpty()) return integerIterator(set);
// floor and ceiling (inclusive) of the range we are looking for
String from = s, to = s + Character.MAX_VALUE;
++actualRightPairSearches;
IntBuffer out = new IntBuffer();
getRightPairsStartingWith_collect(out, set, 0, set.length, from, to);
return out.integerIterator();
}
// binary search
void getLeftPairsEndingWith_collect(IntBuffer out, int[] set, int start, int end, String from, String to) {
if (end <= start) return;
// get mid symbol
int mid = (start + end) >>> 1;
int symbol = pairLeft(set[mid]);
// compare with beginning and end of range
int cmp1 = compareCharIterators_maxLen(itemReverseCharIterator(symbol), characterIterator(from),
min(getProdLength(symbol), l(from))); // prefix trick
int cmp2 = compareCharIterators(itemReverseCharIterator(symbol), characterIterator(to));
boolean searchLeft = cmp2 >= 0; // search left unless right of range
boolean searchRight = cmp1 <= 0; // search right unless left of range
if (!searchLeft) {
// Bug fix for "ba" in [aa, baaa]?
// Scan left branch if there was at least one common character
char firstChar = itemReverseCharIterator(symbol).next();
if (debug) print("bug fix: " + firstChar + "/" + first(from));
if (eq(firstChar, first(from)))
searchLeft = true;
}
if (debug) printVars_str("leftSymbol" , its(symbol), "from", from, "to", to, "start", start, "mid", mid, "end", end, "cmp1", cmp1, "cmp2", cmp2, "searchLeft", searchLeft, "searchRight", searchRight);
if (searchLeft)
getLeftPairsEndingWith_collect(out, set, start, mid, from, to);
if (cmp1 == 0) out.add(set[mid]); // add found pair
if (searchRight)
getLeftPairsEndingWith_collect(out, set, mid+1, end, from, to); }
// subroutine 1: collect elements in range (look for range start)
void getRightPairsStartingWith_collect(IntBuffer out, int[] set, int start, int end, String from, String to) {
if (end <= start) return;
// get mid symbol
int mid = (start + end) >>> 1;
int symbol = pairRight(set[mid]);
// compare with beginning and end of range
int cmp1 = compareCharIterators_maxLen(itemCharIterator(symbol), characterIterator(from),
min(getProdLength(symbol), l(from))); // prefix trick
int cmp2 = compareCharIterators(itemCharIterator(symbol), characterIterator(to));
boolean searchLeft = cmp2 >= 0; // search left unless right of range
boolean searchRight = cmp1 <= 0; // search right unless left of range
if (!searchLeft) {
// Bug fix for "ba" in [aa, baaa]?
// Scan left branch if there was at least one common character
char firstChar = itemCharIterator(symbol).next();
if (debug) print("bug fix: " + firstChar + "/" + first(from));
if (eq(firstChar, first(from)))
searchLeft = true;
}
if (debug) printVars_str("rightSymbol" , its(symbol), "from", from, "to", to, "start", start, "mid", mid, "end", end, "cmp1", cmp1, "cmp2", cmp2, "searchLeft", searchLeft, "searchRight", searchRight);
if (searchLeft)
getRightPairsStartingWith_collect(out, set, start, mid, from, to);
if (cmp1 == 0) out.add(set[mid]); // add found pair
if (searchRight)
getRightPairsStartingWith_collect(out, set, mid+1, end, from, to);
}
List symbolToFiles(int symbol) {
return symbolToFile.getOpt(symbol);
}
// -1 if unknown
int getOccurrences(int symbol) {
return occurrences != null ? occurrences[symbol] : -1;
}
int fileLength(int file) {
return getProdLength(files[file-iFirstFile]);
}
int firstFileLength() { return fileLength(iFirstFile); }
int getFileSymbol(int file) {
return files[file-iFirstFile];
}
}// This is using the implementation that seems to work fastest
// in reality, although it probably is O(n log n) due to the use
// of TreeMaps.
//
// The theoretically best implementation is O(n) (replacing the
// TreeMap with a linked list).
final static class LineCompCompressor {
int safety = 0;
abstract static class Chunk {
abstract String text(List chunks);
IntPair getPair() { return null; }
}
static class CPair extends Chunk implements IFieldsToList{
int i1;
int i2;
CPair() {}
CPair(int i1, int i2) {
this.i2 = i2;
this.i1 = i1;}
public String toString() { return shortClassName(this) + "(" + i1 + ", " + i2 + ")"; }
public boolean equals(Object o) {
if (!(o instanceof CPair)) return false;
CPair __3 = (CPair) o;
return i1 == __3.i1 && i2 == __3.i2;
}
public int hashCode() {
int h = 64355773;
h = boostHashCombine(h, _hashCode(i1));
h = boostHashCombine(h, _hashCode(i2));
return h;
}
public Object[] _fieldsToList() { return new Object[] {i1, i2}; }
CPair(IntPair p) { i1 = p.a; i2 = p.b; }
String text(List chunks) {
return linesLL_rtrim(chunks.get(i1).text(chunks), chunks.get(i2).text(chunks));
}
IntPair getPair() {
return new IntPair(i1, i2);
}
}
static class CPrim extends Chunk implements IFieldsToList{
String s;
CPrim() {}
CPrim(String s) {
this.s = s;}
public String toString() { return shortClassName(this) + "(" + s + ")"; }
public boolean equals(Object o) {
if (!(o instanceof CPrim)) return false;
CPrim __4 = (CPrim) o;
return eq(s, __4.s);
}
public int hashCode() {
int h = 64372105;
h = boostHashCombine(h, _hashCode(s));
return h;
}
public Object[] _fieldsToList() { return new Object[] {s}; }
String text(List chunks) { return s; }
}
boolean verbose = false, verboseCompressionSteps = false;
boolean verboseStats = false;
boolean sortLines = true;
boolean verify = true;
boolean byteMode = false; // compress bytes instead of lines
boolean caseInsensitive = false; // for line mode
boolean toUpper = false; // for byte mode: convert all characters to upper case
boolean recordPairLengths = false; // for "balanced" compression
boolean balancing = false; // try to lower tree heights by compressing "shorter" pairs first
boolean fullCompression = false; // keep compressing until every file is a single symbol
LineComp_PairIndex pairIndex = new LineComp_PairIndex();
Map> textIDToLines;
List allUniqueLines;
List chunks = new ArrayList();
int primChunks;
Map lineIndex;
Map