import java.util.*;
import java.util.zip.*;
import java.util.List;
import java.util.regex.*;
import java.util.concurrent.*;
import java.util.concurrent.atomic.*;
import javax.swing.*;
import javax.swing.event.*;
import javax.swing.text.*;
import javax.swing.table.*;
import java.io.*;
import java.net.*;
import java.lang.reflect.*;
import java.lang.ref.*;
import java.lang.management.*;
import java.security.*;
import java.security.spec.*;
import java.awt.*;
import java.awt.event.*;
import java.awt.image.*;
import javax.imageio.*;
import java.math.*;
public class main {


static String corpusID = "#1001010";
static int numSnippets = 300;
static boolean showGUI = true;
static int maxCharsGUI = 500000;
static boolean allTokens = true;

static Collector collector;
static List<Function> files;
static Map<Function, Set<Integer>> predicted;

// a file to learn from
static class Function {
  String id, name;
  List<String> tok;
}

// a predictor
static abstract class P {
  int seen;
  String file;
  
  abstract String read(String file, List<String> tok);
  abstract P derive(); // clone & reset counter for actual use
  abstract P clear();
  
  void prepare(String file) {
    if (!eq(file, this.file)) {
      seen = 0;
      this.file = file;
    }
  }
}

static class Chain extends P {
  List<P> list = new ArrayList<P>();
  
  Chain() {}
  Chain(List<P> list) {
  this.list = list;}
  Chain(P... a) { list = asList(a); }
  
  void add(P p) { list.add(p); }
  
  String read(String file, List<String> tok) {
    for (P p : list) {
      String s = p.read(file, tok);
      if (s != null) return s;
    }
    return null;
  }
  
  P derive() {
    Chain c = new Chain();
    for (P p : list)
      c.add(p.derive());
    return c;
  }
  
  P clear() {
    Chain c = new Chain();
    for (P p : list)
      c.add(p.clear());
    return c;
  }
}
  
static class Tuples extends P {
  Map<List<String>,String> map = new HashMap<List<String>,String>();
  int n;

  Tuples(int n) {
  this.n = n;
  }
  
  String read(String file, List<String> tok) {
    prepare(file);
    
    while (tok.size() > seen) {
      ++seen;
      if (seen > n)
        map.put(new ArrayList<String>(tok.subList(seen-n-1, seen-1)), tok.get(seen-1));
    }
    
    if (tok.size() >= n)
      return map.get(new ArrayList<String>(tok.subList(tok.size()-n, tok.size())));
      
    return null;
  }
  
  // slow...
  P oldDerive() {
    Tuples t = new Tuples(n);
    t.map.putAll(map);
    // t.seen == 0 which is ok
    return t;
  }
  
  // fast!
  P derive() {
    Tuples t = new Tuples(n);
    t.map = new DerivedHashMap<List<String>,String>(map);
    return t;
  }
  
  P clear() {
    return new Tuples(n);
  }
}
  static class DerivedHashMap<A, B> extends AbstractMap<A, B> {
    Map<A, B> base;
    HashMap<A, B> additions = new HashMap<A, B>();
    
    DerivedHashMap(Map<A, B> base) {
  this.base = base;}
    
    public B get(Object key) {
      B b = additions.get(key);
      if (b != null) return b;
      return base.get(key);
    }
    
    public B put(A key, B value) {
      return additions.put(key, value);
    }
    
    public Set<Map.Entry<A,B>> entrySet() {
      throw fail();
    }
  }
  
 // DerivedHashMap

static class Node {
  String token;
  float count;
  List<Node> next = new ArrayList<Node>();
  
  Node() {} // for clone method
  
  Node(String token) {
  this.token = token;}
  
  Node find(String token) {
    for (Node n : next)
      if (n.token.equals(token))
        return n;
    return null;
  }
  
  Node bestNext() {
    float bestCount = 0f;
    Node best = null;
    for (Node n : next)
      if (best == null || n.count > best.count) {
        best = n;
        bestCount = n.count;
      }
    return best;
  }
}

static class StartTree extends P {
  Node tree = new Node("");
  Node node;
  boolean nonmod;
  
  String read(String file, List<String> tok) {
    if (!eq(file, this.file)) {
      seen = 0;
      this.file = file;
      node = tree;
    }
    
    if (!nonmod) while (tok.size() > seen) {
      String t = tok.get(seen++);
      Node child = node.find(t);
      if (child == null)
        node.next.add(child = new Node(t));
      child.count++;
      node = child;
    }
    
    Node n = node.bestNext();
    return n != null ? n.token : null;
  }
  
  // it's a hack - derived predictor doesn't learn
  P derive() {
    //return (P) main.clone(this);
    StartTree p = new StartTree();
    p.nonmod = true;
    p.tree = tree;
    return p;
  }
  
  P clear() {
    return new StartTree();
  }
}

public static void main(String[] args) throws Exception {
  files = makeCorpus();
  print("Files in corpus: " + files.size());
  
  print("Learning...");
  collector = new Collector();
  //test(new Tuples(1));
  test(new StartTree());
  test(new Chain(new Tuples(4), new Tuples(3), new Tuples(2), new Tuples(1), new StartTree()));

  print("Learning done.");
  printVMSize();
  if (collector.winner != null && showGUI)
    window();
}

static int points = 0, total = 0;

// train & evaluate a predictor
static void test(P p) {
  int lastPercent = 0;
  predicted = new HashMap();
  points = 0;
  total = 0;
  for (int ii = 0; ii < files.size(); ii++) {
    Function f = files.get(ii);
    
    testFile(p, f);
    
    int percent = roundUpTo(10, (int) (ii*100L/files.size()));
    if (percent > lastPercent) {
      print("Learning " + percent + "% done.");
      lastPercent = percent;
    }
  }
  double score = points*100.0/total;
  collector.add(p, score);
}

static void testFile(P p, Function f) {
  TreeSet<Integer> pred = new TreeSet<Integer>();
  List<String> history = new ArrayList<String>();
  for (int i = allTokens ? 0 : 1; i < f.tok.size(); i += allTokens ? 1 : 2) {
    String t = f.tok.get(i);
    String x = p.read(f.name, history);
    boolean correct = t.equals(x);
    total += t.length();
    if (correct) {
      pred.add(i);
      points += t.length();
    }
    history.add(t);
  }
  predicted.put(f, pred);
}
static class SnippetDB {
  static List<String> fields;
  static List<List<String>> rows;
  
  SnippetDB() {}
  
  SnippetDB(String snippetIDOfGzippedMysqlDump) {
    importMySQLDump(gunzipToText(snippetIDOfGzippedMysqlDump));
  }
  
  void importMySQLDump(String dump) {
    System.out.println("Importing " + dump.length() + " chars.");
    Class importer = hotwire("#1000660");
    set(importer, "in", dump);
    callMain(importer);
    Map<String, List> tables = (Map) get(importer, "tables");
    Map<String, List> tableFields = (Map) get(importer, "tableFields");
    fields = tableFields.get("snippets_public");
    rows = tables.get("snippets_public");
  }
  
  String getField(List<String> row, String field) {
    int i = fields.indexOf(field);
    return i < 0 ? null : row.get(i);
  }
  
  List<List<String>> rowsOrderedBy(String field) {
    final int i = fields.indexOf(field);
    List<List<String>> list = new ArrayList<List<String>>(rows);
    sort(list, new Comparator<List<String>>() {
      public int compare(List<String> rowA, List<String> rowB) {
        String a = rowA.get(i), b = rowB.get(i);
        return a.compareTo(b);
      }
    });      
    return list;
  }
  
  public void finalize() {
    System.err.println("Note: SnippetDB released from memory.");
  }
} // SnippetDB

static List<Function> makeCorpus() {
  String name = getSnippetTitle(corpusID);
  if (name.toLowerCase().indexOf(".zip") >= 0)
    return makeCorpus_zip();
  else
    return makeCorpus_mysqldump();
}

static List<Function> makeCorpus_zip() { try {
 
  List<Function> files = new ArrayList<Function>();
  ZipFile zipFile = new ZipFile(loadLibrary(corpusID));
  Enumeration entries = zipFile.entries();

  while (entries.hasMoreElements() && files.size() < numSnippets) {
    ZipEntry entry = (ZipEntry) entries.nextElement(); 
    if (entry.isDirectory()) continue;
    //System.out.println("File found: " + entry.getName());

    InputStream fin = zipFile.getInputStream(entry);
    // TODO: try to skip binary files?
    
    InputStreamReader reader = new InputStreamReader(fin, "UTF-8");
    StringBuilder builder = new StringBuilder();
    BufferedReader bufferedReader = new BufferedReader(reader);
    String line;
    while ((line = bufferedReader.readLine()) != null)
      builder.append(line).append('\n');
    fin.close();
    String text = builder.toString();
    
    Function f = new Function();
    f.name = entry.getName();
    f.tok = internAll(javaTok(text));
    files.add(f);
  }
  
  zipFile.close();
  return files;

} catch (Throwable __e) { throw __e instanceof RuntimeException ? (RuntimeException) __e : new RuntimeException(__e); }}

static List<Function> makeCorpus_mysqldump() {
  List<Function> files = new ArrayList<Function>();
  SnippetDB db = new SnippetDB(corpusID);
  List<List<String>> rows = db.rowsOrderedBy("sn_created");
  for (int i = 0; i < Math.min(rows.size(), numSnippets); i++) {
    Function f = new Function();
    f.id = db.getField(rows.get(i), "sn_id");
    f.name = db.getField(rows.get(i), "sn_title");
    String text = db.getField(rows.get(i), "sn_text");
    f.tok = internAll(javaTok(text));
    files.add(f);
    ++i;
  }
  return files;
}

static class Collector {
  P winner;
  double bestScore = -1;
  Map<Function, Set<Integer>> predicted;

  void add(P p, double score) {
    if (winner == null || score > bestScore) {
      winner = p;
      bestScore = score;
      //S name = shorten(structure(p), 100);
      String name = p.getClass().getName();
      print("New best score: " + formatDouble(score, 2) + "% (" + name + ")");
      predicted = main.predicted;
    }
  }
}

static void window() {
  //final P p = collector.winner.clear();

  JFrame jf = new JFrame("Predicted = green");
  Container cp = jf.getContentPane();

  final JButton btnNext = new JButton("Next");
  
  final JTextPane pane = new JTextPane();
  //pane.setFont(loadFont("#1000993", 24));
  
  JScrollPane scrollPane = new JScrollPane(pane);
  cp.add(scrollPane, BorderLayout.CENTER);
  
  class X {
    int ii;
    
    void y() { try {
 
      ii = ii == 0 ? files.size()-1 : ii-1;
      Function f = files.get(ii);
      //testFile(p, f);
      Set<Integer> pred = collector.predicted.get(f);
      
      StyledDocument doc = new DefaultStyledDocument();

      List<String> tok = f.tok;
      int i = tok.size(), len = 0;
      while (len <= maxCharsGUI && i > 0) {
        --i;
        len += tok.get(i).length();
      }
      
      for (; i < tok.size(); i++) {
        if (tok.get(i).length() == 0) continue;
        boolean green = pred.contains(i);
        SimpleAttributeSet set = new SimpleAttributeSet();
        StyleConstants.setForeground(set, green ? Color.green : Color.gray);
        doc.insertString(doc.getLength(), tok.get(i), set);
      }
      
      pane.setDocument(doc);
      double score = getScore(pred, tok);
      btnNext.setText(f.name + " (" + (ii+1) + "/" + files.size() + ") - " + (int) score + " %");
    
} catch (Throwable __e) { throw __e instanceof RuntimeException ? (RuntimeException) __e : new RuntimeException(__e); }}
  }
  final X x = new X();
  
  btnNext.addActionListener(new java.awt.event.ActionListener() {
public void actionPerformed(java.awt.event.ActionEvent _evt) {

    x.y();
  
}});
  cp.add(btnNext, BorderLayout.NORTH);

  x.y();
  
  jf.setBounds(100, 100, 600, 600);
  jf.setVisible(true);    
}
static Object clone(Object o) {
  ArrayDeque<Runnable> q = new ArrayDeque<Runnable>();
  Object x = clone_impl(o, new IdentityHashMap(), q);
  while (!q.isEmpty())
    q.poll().run();
  return x;
}

static Object clone_impl(Object o, final IdentityHashMap seen, final ArrayDeque<Runnable> q) { try {
 
  if (o == null)
    return o;
    
  Object y = seen.get(o);
  if (y != null) return y;
  
  if (o instanceof List) {
    //print("Cloning list: " + o);
    final List l = new ArrayList();
    seen.put(o, l);
    for (final Object x : (List) o)
      q.add(new Runnable() {
public void run() { try {

        l.add(clone_impl(x, seen, q));
      
} catch (Exception __e) { throw __e instanceof RuntimeException ? (RuntimeException) __e : new RuntimeException(__e); }}});
    return l;
  }
  
  if (o instanceof Map) {
    final Map m = o instanceof TreeMap ? new TreeMap() : new HashMap();
    seen.put(o, m);
    for (Object entry : ((Map) o).entrySet()) {
      final Map.Entry e = (Map.Entry) entry;
      q.add(new Runnable() {
public void run() { try {

        m.put(clone_impl(e.getKey(), seen, q), clone_impl(e.getValue(), seen, q));
      
} catch (Exception __e) { throw __e instanceof RuntimeException ? (RuntimeException) __e : new RuntimeException(__e); }}});
    }
    return m;
  }
  
  if (o instanceof String || o instanceof Number || o instanceof Boolean)
    return o;
    
  if (o instanceof Object[]) {
    final Object[] l = (Object[]) o;
    final Object[] l2 = l.clone();
    seen.put(o, l2);
    for (int i = 0; i < l.length; i++) {
      final int _i = i;
      q.add(new Runnable() {
public void run() { try {

        l2[_i] = clone_impl(l[_i], seen, q);
      
} catch (Exception __e) { throw __e instanceof RuntimeException ? (RuntimeException) __e : new RuntimeException(__e); }}});
    }
    return l2;
  }
  
  // clone an arbitrary custom object

  // TODO: use Constructor object and setAccessible
  //print("Cloning custom: " + o);
  final Object clone = o.getClass().newInstance();
  seen.put(o, clone);
  Class c = o.getClass();
  while (c != Object.class) {
    Field[] fields = c.getDeclaredFields();
    for (final Field field : fields) {
      if ((field.getModifiers() & Modifier.STATIC) != 0)
        continue;
      field.setAccessible(true);
      final Object value = field.get(o);
      q.add(new Runnable() {
public void run() { try {

        field.set(clone, clone_impl(value, seen, q));
      
} catch (Exception __e) { throw __e instanceof RuntimeException ? (RuntimeException) __e : new RuntimeException(__e); }}});
    }
    c = c.getSuperclass();
  }
  return clone;

} catch (Throwable __e) { throw __e instanceof RuntimeException ? (RuntimeException) __e : new RuntimeException(__e); }}
 // clone function

static double getScore(Set<Integer> pred, List<String> tok) {
  int total = 0, score = 0;
  for (int i = 0; i < tok.size(); i++) {
    int n = tok.get(i).length();
    total += n;
    if (pred.contains(i))
      score += n;
  }
  return score*100.0/total;
}


static String getSnippetTitle(String id) { try {
 
  return loadPage(new URL("http://tinybrain.de:8080/tb-int/getfield.php?id=" + parseSnippetID(id) + "&field=title"));

} catch (Throwable __e) { throw __e instanceof RuntimeException ? (RuntimeException) __e : new RuntimeException(__e); }}
static File loadLibrary(String snippetID) {
  return loadBinarySnippet(snippetID);
}
  public static String formatDouble(double d, int digits) {
    String format = "0.";
    for (int i = 0; i < digits; i++) format += "#";
    return new java.text.DecimalFormat(format, new java.text.DecimalFormatSymbols(Locale.ENGLISH)).format(d);
  }

  static void set(Object o, String field, Object value) {
    if (o instanceof Class) set((Class) o, field, value);
    else try {
      Field f = set_findField(o.getClass(), field);
      smartSet(f, o, value);
    } catch (Exception e) {
      throw new RuntimeException(e);
    }
  }
  
  static void set(Class c, String field, Object value) {
    try {
      Field f = set_findStaticField(c, field);
      smartSet(f, null, value);
    } catch (Exception e) {
      throw new RuntimeException(e);
    }
  }
  
  static Field set_findField(Class<?> c, String field) {
    for (Field f : c.getDeclaredFields())
      if (f.getName().equals(field))
        return f;
    throw new RuntimeException("Field '" + field + "' not found in " + c.getName());
  }
  
  static Field set_findStaticField(Class<?> c, String field) {
    for (Field f : c.getDeclaredFields())
      if (f.getName().equals(field) && (f.getModifiers() & Modifier.STATIC) != 0)
        return f;
    throw new RuntimeException("Static field '" + field + "' not found in " + c.getName());
  }
static void callMain(Object c, String... args) {
  callOpt(c, "main", new Object[] {args});
}
static void printVMSize() {
  print("VM size: " + (Runtime.getRuntime().totalMemory()+1024*1024-1)/1024/1024 + " MB");
}
// replacement for class JavaTok
// maybe incomplete, might want to add floating point numbers
// todo also: extended multi-line strings

static List<String> javaTok(String s) {
  List<String> tok = new ArrayList<String>();
  int l = s.length();
  
  int i = 0;
  while (i < l) {
    int j = i;
    char c; String cc;
    
    // scan for whitespace
    while (j < l) {
      c = s.charAt(j);
      cc = s.substring(j, Math.min(j+2, l));
      if (c == ' ' || c == '\t' || c == '\r' || c == '\n')
        ++j;
      else if (cc.equals("/*")) {
        do ++j; while (j < l && !s.substring(j, Math.min(j+2, l)).equals("*/"));
        j = Math.min(j+2, l);
      } else if (cc.equals("//")) {
        do ++j; while (j < l && "\r\n".indexOf(s.charAt(j)) < 0);
      } else
        break;
    }
    
    tok.add(s.substring(i, j));
    i = j;
    if (i >= l) break;
    c = s.charAt(i); // cc is not needed in rest of loop body
    cc = s.substring(i, Math.min(i+2, l));

    // scan for non-whitespace
    if (c == '\'' || c == '"') {
      char opener = c;
      ++j;
      while (j < l) {
        if (s.charAt(j) == opener || s.charAt(j) == '\n') { // end at \n to not propagate unclosed string literal errors
          ++j;
          break;
        } else if (s.charAt(j) == '\\' && j+1 < l)
          j += 2;
        else
          ++j;
      }
    } else if (Character.isJavaIdentifierStart(c))
      do ++j; while (j < l && (Character.isJavaIdentifierPart(s.charAt(j)) || "'".indexOf(s.charAt(j)) >= 0)); // for stuff like "don't"
    else if (Character.isDigit(c)) {
      do ++j; while (j < l && Character.isDigit(s.charAt(j)));
      if (j < l && s.charAt(j) == 'L') ++j; // Long constants like 1L
    } else if (cc.equals("[[")) {
      do ++j; while (j+1 < l && !s.substring(j, j+2).equals("]]"));
      j = Math.min(j+2, l);
    } else if (cc.equals("[=") && i+2 < l && s.charAt(i+2) == '[') {
      do ++j; while (j+2 < l && !s.substring(j, j+3).equals("]=]"));
      j = Math.min(j+3, l);
    } else
      ++j;

    tok.add(s.substring(i, j));
    i = j;
  }
  
  if ((tok.size() % 2) == 0) tok.add("");
  return tok;
}

static List<String> javaTok(List<String> tok) {
  return javaTok(join(tok));
}
static <T> void sort(T[] a, Comparator<? super T> c) {
  Arrays.sort(a, c);
}

static <T> void sort(List<T> a, Comparator<? super T> c) {
  Collections.sort(a, c);
}
static boolean eq(Object a, Object b) {
  if (a == null) return b == null;
  if (a.equals(b)) return true;
  if (a instanceof BigInteger) {
    if (b instanceof Integer) return a.equals(BigInteger.valueOf((Integer) b));
    if (b instanceof Long) return a.equals(BigInteger.valueOf((Long) b));
  }
  return false;
}
static <A> ArrayList<A> asList(A[] a) {
  return new ArrayList<A>(Arrays.asList(a));
}

static ArrayList<Integer> asList(int[] a) {
  ArrayList<Integer> l = new ArrayList();
  for (int i : a) l.add(i);
  return l;
}

static <A> ArrayList<A> asList(Collection<A> s) {
  return s == null ? new ArrayList() 
    : s instanceof ArrayList ? (ArrayList) s : new ArrayList(s);
}
  static RuntimeException fail() {
    throw new RuntimeException("fail");
  }
  
  static RuntimeException fail(Object msg) {
    throw new RuntimeException(String.valueOf(msg));
  }
  
  static RuntimeException fail(String msg) {
    throw new RuntimeException(unnull(msg));
  }
    
  static RuntimeException fail(String msg, Object... args) {
    throw new RuntimeException(format(msg, args));
  }
static volatile StringBuffer local_log = new StringBuffer(); // not redirected
static volatile StringBuffer print_log = local_log; // might be redirected, e.g. to main bot

// in bytes - will cut to half that
static volatile int print_log_max = 1024*1024;
static volatile int local_log_max = 100*1024;

static void print() {
  print("");
}

// slightly overblown signature to return original object...
static <A> A print(A o) {
  String s = String.valueOf(o) + "\n";
  StringBuffer loc = local_log;
  StringBuffer buf = print_log;
  int loc_max = print_log_max;
  if (buf != loc && buf != null) {
    print_append(buf, s, print_log_max);
    loc_max = local_log_max;
  }
  if (loc != null) 
    print_append(loc, s, loc_max);
  System.out.print(s);
  return o;
}

static void print(long l) {
  print(String.valueOf(l));
}

static void print(char c) {
  print(String.valueOf(c));
}

static void print_append(StringBuffer buf, String s, int max) {
  synchronized(buf) {
    buf.append(s);
    max /= 2;
    if (buf.length() > max) try {
      int newLength = max/2;
      int ofs = buf.length()-newLength;
      String newString = buf.substring(ofs);
      buf.setLength(0);
      buf.append("[...] ").append(newString);
    } catch (Exception e) {
      buf.setLength(0);
    }
  }
}
static String gunzipToText(String dataID) { try {
 
  File file = loadBinarySnippet(dataID);

  InputStream fis = new FileInputStream(file);
  GZIPInputStream gis = new GZIPInputStream(fis);
  ByteArrayOutputStream fos = new ByteArrayOutputStream();
  byte[] buffer = new byte[1024];
  int len;
  while((len = gis.read(buffer)) != -1){
      fos.write(buffer, 0, len);
  }
  fis.close();
  fos.close();
  return new String(fos.toByteArray(), "UTF-8");

} catch (Throwable __e) { throw __e instanceof RuntimeException ? (RuntimeException) __e : new RuntimeException(__e); }}
static String find(String pattern, String text) {
  Matcher matcher = Pattern.compile(pattern).matcher(text);
  if (matcher.find())
    return matcher.group(1);
  return null;
}

static <A> A find(Collection<A> c, Object... data) {
  for (A x : c)
    if (checkFields(x, data))
      return x;
  return null;
}
// get purpose 1: access a list/array (safer version of x.get(y))

static <A> A get(List<A> l, int idx) {
  return idx >= 0 && idx < l(l) ? l.get(idx) : null;
}

static <A> A get(A[] l, int idx) {
  return idx >= 0 && idx < l(l) ? l[idx] : null;
}

// get purpose 2: access a field by reflection or a map

static Object get(Object o, String field) {
  if (o instanceof Class) return get((Class) o, field);
  
  if (o instanceof Map)
    return ((Map) o).get(field);
    
  if (o.getClass().getName().equals("main$DynamicObject"))
    return call(get_raw(o, "fieldValues"), "get", field);
    
  return get_raw(o, field);
}

static Object get_raw(Object o, String field) {
  try {
    Field f = get_findField(o.getClass(), field);
    f.setAccessible(true);
    return f.get(o);
  } catch (Exception e) {
    throw new RuntimeException(e);
  }
}

static Object get(Class c, String field) {
  try {
    Field f = get_findStaticField(c, field);
    f.setAccessible(true);
    return f.get(null);
  } catch (Exception e) {
    throw new RuntimeException(e);
  }
}

static Field get_findStaticField(Class<?> c, String field) {
  Class _c = c;
  do {
    for (Field f : _c.getDeclaredFields())
      if (f.getName().equals(field) && (f.getModifiers() & Modifier.STATIC) != 0)
        return f;
    _c = _c.getSuperclass();
  } while (_c != null);
  throw new RuntimeException("Static field '" + field + "' not found in " + c.getName());
}

static Field get_findField(Class<?> c, String field) {
  Class _c = c;
  do {
    for (Field f : _c.getDeclaredFields())
      if (f.getName().equals(field))
        return f;
    _c = _c.getSuperclass();
  } while (_c != null);
  throw new RuntimeException("Field '" + field + "' not found in " + c.getName());
}
static int roundUpTo(int n, int x) {
  return (x+n-1)/n*n;
}
static List<String> internAll(List<String> tok) {
  List<String> l = new ArrayList<String>();
  for (String t : tok)
    l.add(t.intern());
  return l;
}

  // compile JavaX source, load classes & return main class
  // src can be a snippet ID or actual source code
  // TODO: record injection?
  
  static Class<?> hotwire(String src) {
    try {
      Class j = getJavaX();
      
      synchronized(j) { // hopefully this goes well...
        List<File> libraries = new ArrayList<File>();
        File srcDir = (File) call(j, "transpileMain", src, libraries);
        if (srcDir == null)
          fail("transpileMain returned null (src=" + quote(src) + ")");
        
        Object androidContext = get(j, "androidContext");
        if (androidContext != null)
          return (Class) call(j, "loadx2android", srcDir, src);
          
        File classesDir = (File) call(j, "TempDirMaker_make");
        String javacOutput = (String) call(j, "compileJava", srcDir, libraries, classesDir);
        System.out.println(javacOutput);
        
        URL[] urls = new URL[libraries.size()+1];
        urls[0] = classesDir.toURI().toURL();
        for (int i = 0; i < libraries.size(); i++)
          urls[i+1] = libraries.get(i).toURI().toURL();
  
        // make class loader
        URLClassLoader classLoader = new URLClassLoader(urls);
    
        // load & return main class
        Class<?> theClass = classLoader.loadClass("main");
        
        callOpt(j, "registerSourceCode", theClass, loadTextFile(new File(srcDir, "main.java")));

        call(j, "setVars", theClass, isSnippetID(src) ? src: null);
        
        if (isSnippetID(src))
          callOpt(j, "addInstance", src, theClass);
          
        if (!_inCore())
          hotwire_copyOver(theClass);
  
        return theClass;
      }
    } catch (Exception e) {
      throw e instanceof RuntimeException ? (RuntimeException) e : new RuntimeException(e);
    }
  }

static void hotwire_copyOver(Class c) {
  synchronized(StringBuffer.class) {
    Object print_log = get(getMainClass(), "print_log");
    if (print_log != null)
      setOpt(c, "print_log", print_log);
      
    Object mainBot = getMainBot();
    if (mainBot != null)
      setOpt(c, "mainBot", mainBot);
  }
}
  static Object callOpt(Object o, String method, Object... args) {
    try {
      if (o == null) return null;
      if (o instanceof Class) {
        Method m = callOpt_findStaticMethod((Class) o, method, args, false);
        if (m == null) return null;
        m.setAccessible(true);
        return m.invoke(null, args);
      } else {
        Method m = callOpt_findMethod(o, method, args, false);
        if (m == null) return null;
        m.setAccessible(true);
        return m.invoke(o, args);
      }
    } catch (Exception e) {
      throw new RuntimeException(e);
    }
  }

  static Method callOpt_findStaticMethod(Class c, String method, Object[] args, boolean debug) {
    Class _c = c;
    while (c != null) {
      for (Method m : c.getDeclaredMethods()) {
        if (debug)
          System.out.println("Checking method " + m.getName() + " with " + m.getParameterTypes().length + " parameters");;
        if (!m.getName().equals(method)) {
          if (debug) System.out.println("Method name mismatch: " + method);
          continue;
        }

        if ((m.getModifiers() & Modifier.STATIC) == 0 || !callOpt_checkArgs(m, args, debug))
          continue;

        return m;
      }
      c = c.getSuperclass();
    }
    return null;
  }

  static Method callOpt_findMethod(Object o, String method, Object[] args, boolean debug) {
    Class c = o.getClass();
    while (c != null) {
      for (Method m : c.getDeclaredMethods()) {
        if (debug)
          System.out.println("Checking method " + m.getName() + " with " + m.getParameterTypes().length + " parameters");;
        if (m.getName().equals(method) && callOpt_checkArgs(m, args, debug))
          return m;
      }
      c = c.getSuperclass();
    }
    return null;
  }

  private static boolean callOpt_checkArgs(Method m, Object[] args, boolean debug) {
    Class<?>[] types = m.getParameterTypes();
    if (types.length != args.length) {
      if (debug)
        System.out.println("Bad parameter length: " + args.length + " vs " + types.length);
      return false;
    }
    for (int i = 0; i < types.length; i++)
      if (!(args[i] == null || isInstanceX(types[i], args[i]))) {
        if (debug)
          System.out.println("Bad parameter " + i + ": " + args[i] + " vs " + types[i]);
        return false;
      }
    return true;
  }



  static String unnull(String s) {
    return s == null ? "" : s;
  }
  
  static <A> List<A> unnull(List<A> l) {
    return l == null ? emptyList() : l;
  }
static boolean _inCore() {
  return false;
}
  public static boolean isSnippetID(String s) {
    try {
      parseSnippetID(s);
      return true;
    } catch (RuntimeException e) {
      return false;
    }
  }
public static long parseSnippetID(String snippetID) {
  long id = Long.parseLong(shortenSnippetID(snippetID));
  if (id == 0) fail("0 is not a snippet ID");
  return id;
}
  static String quote(String s) {
    if (s == null) return "null";
    return "\"" + s.replace("\\", "\\\\").replace("\"", "\\\"").replace("\r", "\\r").replace("\n", "\\n") + "\"";
  }
  
  static String quote(long l) {
    return quote("" + l);
  }
  
  static String quote(char c) {
    return quote("" + c);
  }

  public static String join(String glue, Iterable<String> strings) {
    StringBuilder buf = new StringBuilder();
    Iterator<String> i = strings.iterator();
    if (i.hasNext()) {
      buf.append(i.next());
      while (i.hasNext())
        buf.append(glue).append(i.next());
    }
    return buf.toString();
  }
  
  public static String join(String glue, String[] strings) {
    return join(glue, Arrays.asList(strings));
  }
  
  public static String join(Iterable<String> strings) {
    return join("", strings);
  }
  
  public static String join(String[] strings) {
    return join("", strings);
  }

static boolean checkFields(Object x, Object... data) {
  for (int i = 0; i < l(data); i += 2)
    if (neq(getOpt(x, (String) data[i]), data[i+1]))
      return false;
  return true;
}
static Class __javax;

static Class getJavaX() {
  return __javax;
}
  public static String loadTextFile(String fileName) {
    try {
      return loadTextFile(fileName, null);
    } catch (IOException e) {
      throw new RuntimeException(e);
    }
  }
  
  public static String loadTextFile(String fileName, String defaultContents) throws IOException {
    if (!new File(fileName).exists())
      return defaultContents;

    FileInputStream fileInputStream = new FileInputStream(fileName);
    InputStreamReader inputStreamReader = new InputStreamReader(fileInputStream, "UTF-8");
    return loadTextFile(inputStreamReader);
  }
  
  public static String loadTextFile(File fileName) {
    try {
      return loadTextFile(fileName, null);
    } catch (IOException e) {
      throw new RuntimeException(e);
    }
  }

  public static String loadTextFile(File fileName, String defaultContents) throws IOException {
    try {
      return loadTextFile(fileName.getPath(), defaultContents);
    } catch (IOException e) {
      throw new RuntimeException(e);
    }
  }
  
  public static String loadTextFile(Reader reader) throws IOException {
    StringBuilder builder = new StringBuilder();
    try {
      char[] buffer = new char[1024];
      int n;
      while (-1 != (n = reader.read(buffer)))
        builder.append(buffer, 0, n);
        
    } finally {
      reader.close();
    }
    return builder.toString();
  }
  static String format(String pat, Object... args) {
    return format3(pat, args);
  }

static void smartSet(Field f, Object o, Object value) throws Exception {
  f.setAccessible(true);
  
  // take care of common case (long to int)
  if (f.getType() == int.class && value instanceof Long)
    value = ((Long) value).intValue();
    
  f.set(o, value);
}
  static ThreadLocal<String> loadPage_charset = new ThreadLocal<String>();

  public static String loadPageSilently(String url) {
    try {
      return loadPageSilently(new URL(loadPage_preprocess(url)));
    } catch (IOException e) { throw new RuntimeException(e); }
  }

  public static String loadPageSilently(URL url) {
    try {
      IOException e = null;
      for (int tries = 0; tries < 60; tries++)
        try {
          URLConnection con = url.openConnection();
          return loadPage(con, url);
        } catch (IOException _e) {
          e = _e;
          print("Retrying because of: " + e);
          sleepSeconds(1);
        }
      throw e;
    } catch (IOException e) { throw new RuntimeException(e); }
  }

  static String loadPage_preprocess(String url) {  
    if (url.startsWith("tb/"))
      url = "tinybrain.de:8080/" + url;
    if (url.indexOf("://") < 0)
      url = "http://" + url;
    return url;
  }
  
  public static String loadPage(String url) {
    try {
      return loadPage(new URL(loadPage_preprocess(url)));
    } catch (IOException e) { throw new RuntimeException(e); }
  }
  
  public static String loadPage(URL url) {
    print("Loading: " + url.toExternalForm());
    return loadPageSilently(url);
  }

  public static String loadPage(URLConnection con, URL url) throws IOException {
    String computerID = getComputerID();
    if (computerID != null)
      try { con.setRequestProperty("X-ComputerID", computerID); } catch (Throwable e) {} // fails if within doPost
    String contentType = con.getContentType();
    if (contentType == null)
      throw new IOException("Page could not be read: " + url);
    //print("Content-Type: " + contentType);
    String charset = loadPage_charset == null ? null : loadPage_charset.get();
    if (charset == null) charset = loadPage_guessCharset(contentType);
    Reader r = new InputStreamReader(con.getInputStream(), charset);
    StringBuilder buf = new StringBuilder();
    while (true) {
      int ch = r.read();
      if (ch < 0)
        break;
      //Log.info("Chars read: " + buf.length());
      buf.append((char) ch);
    }
    return buf.toString();
  }
  
  static String loadPage_guessCharset(String contentType) {
    Pattern p = Pattern.compile("text/[a-z]+;\\s+charset=([^\\s]+)\\s*");
    Matcher m = p.matcher(contentType);
    /* If Content-Type doesn't match this pre-conception, choose default and hope for the best. */
    return m.matches() ? m.group(1) : "ISO-8859-1";
  }

static int l(Object[] array) {
  return array == null ? 0 : array.length;
}

static int l(byte[] array) {
  return array == null ? 0 : array.length;
}

static int l(int[] array) {
  return array == null ? 0 : array.length;
}

static int l(char[] array) {
  return array == null ? 0 : array.length;
}

static int l(Collection c) {
  return c == null ? 0 : c.size();
}

static int l(Map m) {
  return m == null ? 0 : m.size();
}

static int l(String s) {
  return s == null ? 0 : s.length();
} 


  // varargs assignment fixer for a single string array argument
  static Object call(Object o, String method, String[] arg) {
    return call(o, method, new Object[] {arg});
  }
  
  static Object call(Object o, String method, Object... args) {
    try {
      if (o instanceof Class) {
        Method m = call_findStaticMethod((Class) o, method, args, false);
        m.setAccessible(true);
        return m.invoke(null, args);
      } else {
        Method m = call_findMethod(o, method, args, false);
        m.setAccessible(true);
        return m.invoke(o, args);
      }
    } catch (Exception e) {
      throw e instanceof RuntimeException ? (RuntimeException) e : new RuntimeException(e);
    }
  }

  static Method call_findStaticMethod(Class c, String method, Object[] args, boolean debug) {
    Class _c = c;
    while (c != null) {
      for (Method m : c.getDeclaredMethods()) {
        if (debug)
          System.out.println("Checking method " + m.getName() + " with " + m.getParameterTypes().length + " parameters");;
        if (!m.getName().equals(method)) {
          if (debug) System.out.println("Method name mismatch: " + method);
          continue;
        }

        if ((m.getModifiers() & Modifier.STATIC) == 0 || !call_checkArgs(m, args, debug))
          continue;

        return m;
      }
      c = c.getSuperclass();
    }
    throw new RuntimeException("Method '" + method + "' (static) with " + args.length + " parameter(s) not found in " + _c.getName());
  }

  static Method call_findMethod(Object o, String method, Object[] args, boolean debug) {
    Class c = o.getClass();
    while (c != null) {
      for (Method m : c.getDeclaredMethods()) {
        if (debug)
          System.out.println("Checking method " + m.getName() + " with " + m.getParameterTypes().length + " parameters");;
        if (m.getName().equals(method) && call_checkArgs(m, args, debug))
          return m;
      }
      c = c.getSuperclass();
    }
    throw new RuntimeException("Method '" + method + "' (non-static) with " + args.length + " parameter(s) not found in " + o.getClass().getName());
  }

  private static boolean call_checkArgs(Method m, Object[] args, boolean debug) {
    Class<?>[] types = m.getParameterTypes();
    if (types.length != args.length) {
      if (debug)
        System.out.println("Bad parameter length: " + args.length + " vs " + types.length);
      return false;
    }
    for (int i = 0; i < types.length; i++)
      if (!(args[i] == null || isInstanceX(types[i], args[i]))) {
        if (debug)
          System.out.println("Bad parameter " + i + ": " + args[i] + " vs " + types[i]);
        return false;
      }
    return true;
  }


static File loadBinarySnippet(String snippetID) { try {
 
  long id = parseSnippetID(snippetID);
  File f = DiskSnippetCache_getLibrary(id);
  if (f == null) {
    byte[] data = loadDataSnippetImpl(snippetID);
    DiskSnippetCache_putLibrary(id, data);
    f = DiskSnippetCache_getLibrary(id);
  }
  return f;

} catch (Throwable __e) { throw __e instanceof RuntimeException ? (RuntimeException) __e : new RuntimeException(__e); }}

  static String format3(String pat, Object... args) {
    if (args.length == 0) return pat;
    
    List<String> tok = javaTokPlusPeriod(pat);
    int argidx = 0;
    for (int i = 1; i < tok.size(); i += 2)
      if (tok.get(i).equals("*"))
        tok.set(i, format3_formatArg(argidx < args.length ? args[argidx++] : "null"));
    return join(tok);
  }
  
  static String format3_formatArg(Object arg) {
    if (arg == null) return "null";
    if (arg instanceof String) {
      String s = (String) arg;
      return isIdentifier(s) || isNonNegativeInteger(s) ? s : quote(s);
    }
    if (arg instanceof Integer || arg instanceof Long) return String.valueOf(arg);
    return quote(structure(arg));
  }
  

static boolean neq(Object a, Object b) {
  return !eq(a, b);
}
static String shortenSnippetID(String snippetID) {
  if (snippetID.startsWith("#"))
    snippetID = snippetID.substring(1);
  String httpBlaBla = "http://tinybrain.de/";
  if (snippetID.startsWith(httpBlaBla))
    snippetID = snippetID.substring(httpBlaBla.length());
  return "" + parseLong(snippetID);
}
// extended over Class.isInstance() to handle primitive types
static boolean isInstanceX(Class type, Object arg) {
  if (type == boolean.class) return arg instanceof Boolean;
  if (type == int.class) return arg instanceof Integer;
  if (type == long.class) return arg instanceof Long;
  if (type == float.class) return arg instanceof Float;
  if (type == short.class) return arg instanceof Short;
  if (type == char.class) return arg instanceof Character;
  if (type == byte.class) return arg instanceof Byte;
  if (type == double.class) return arg instanceof Double;
  return type.isInstance(arg);
}
static void sleepSeconds(long s) {
  if (s > 0) sleep(s*1000);
}
static String getComputerID() { try {
 
  return computerID();

} catch (Throwable __e) { throw __e instanceof RuntimeException ? (RuntimeException) __e : new RuntimeException(__e); }}
static Object mainBot;

static Object getMainBot() {
  return mainBot;
}
  
  // Data files are immutable, use centralized cache
public static File DiskSnippetCache_getLibrary(long snippetID) throws IOException {
  File file = new File(getGlobalCache(), "data_" + snippetID + ".jar");
  return file.exists() ? file : null;
}

public static void DiskSnippetCache_putLibrary(long snippetID, byte[] data) throws IOException {
  saveBinaryFile(new File(getGlobalCache(), "data_" + snippetID).getPath() + ".jar", data);
}

static byte[] loadDataSnippetImpl(String snippetID) throws IOException {
  byte[] data;
  try {
    URL url = new URL("http://eyeocr.sourceforge.net/filestore/filestore.php?cmd=serve&file=blob_"
      + parseSnippetID(snippetID) + "&contentType=application/binary");
    System.err.println("Loading library: " + url);
    try {
      data = loadBinaryPage(url.openConnection());
    } catch (IOException e) {
      data = null;
    }
    
    if (data == null || data.length == 0) {
      url = new URL("http://data.tinybrain.de/blobs/"
        + parseSnippetID(snippetID));
      System.err.println("Loading library: " + url);
      data = loadBinaryPage(url.openConnection());
    }
    System.err.println("Bytes loaded: " + data.length);
  } catch (FileNotFoundException e) {
    throw new IOException("Binary snippet #" + snippetID + " not found or not public");
  }
  return data;
}
static List emptyList() {
  return new ArrayList();
  //ret Collections.emptyList();
}
static Class getMainClass() { try {
 
  return Class.forName("main");

} catch (Throwable __e) { throw __e instanceof RuntimeException ? (RuntimeException) __e : new RuntimeException(__e); }}

static Class getMainClass(Object o) { try {
 
  return (o instanceof Class ? (Class) o : o.getClass()).getClassLoader().loadClass("main");

} catch (Throwable __e) { throw __e instanceof RuntimeException ? (RuntimeException) __e : new RuntimeException(__e); }}
  static void setOpt(Object o, String field, Object value) {
    if (o instanceof Class) setOpt((Class) o, field, value);
    else try {
      Field f = setOpt_findField(o.getClass(), field);
      if (f != null)
        smartSet(f, o, value);
    } catch (Exception e) {
      throw new RuntimeException(e);
    }
  }
  
  static void setOpt(Class c, String field, Object value) {
    try {
      Field f = setOpt_findStaticField(c, field);
      if (f != null)
        smartSet(f, null, value);
    } catch (Exception e) {
      throw new RuntimeException(e);
    }
  }
  
  static Field setOpt_findField(Class<?> c, String field) {
    for (Field f : c.getDeclaredFields())
      if (f.getName().equals(field))
        return f;
    return null;
  }
  
  static Field setOpt_findStaticField(Class<?> c, String field) {
    for (Field f : c.getDeclaredFields())
      if (f.getName().equals(field) && (f.getModifiers() & Modifier.STATIC) != 0)
        return f;
    return null;
  }
static Object getOpt(Object o, String field) {
  if (o instanceof String) o = getBot ((String) o);
  if (o == null) return null;
  if (o instanceof Class) return getOpt((Class) o, field);
  
  if (o.getClass().getName().equals("main$DynamicObject"))
    return call(getOpt_raw(o, "fieldValues"), "get", field);
  
  if (o instanceof Map) return ((Map) o).get(field);
  
  return getOpt_raw(o, field);
}

static Object getOpt_raw(Object o, String field) {
  try {
    Field f = getOpt_findField(o.getClass(), field);
    if (f == null) return null;
    f.setAccessible(true);
    return f.get(o);
  } catch (Exception e) {
    throw new RuntimeException(e);
  }
}

static Object getOpt(Class c, String field) {
  try {
    Field f = getOpt_findStaticField(c, field);
    if (f == null) return null;
    f.setAccessible(true);
    return f.get(null);
  } catch (Exception e) {
    throw new RuntimeException(e);
  }
}

static Field getOpt_findStaticField(Class<?> c, String field) {
  Class _c = c;
  do {
    for (Field f : _c.getDeclaredFields())
      if (f.getName().equals(field) && (f.getModifiers() & Modifier.STATIC) != 0)
        return f;
    _c = _c.getSuperclass();
  } while (_c != null);
  return null;
}

static Field getOpt_findField(Class<?> c, String field) {
  Class _c = c;
  do {
    for (Field f : _c.getDeclaredFields())
      if (f.getName().equals(field))
        return f;
    _c = _c.getSuperclass();
  } while (_c != null);
  return null;
}

static boolean isIdentifier(String s) {
  return isJavaIdentifier(s);
}
static String _computerID;
public static String computerID() { try {
 
  if (_computerID == null) {
    File file = new File(userHome(), ".tinybrain/computer-id");
    _computerID = loadTextFile(file.getPath(), null);
    if (_computerID == null) {
      _computerID = makeRandomID(12);
      saveTextFile(file.getPath(), _computerID);
    }
  }
  return _computerID;

} catch (Throwable __e) { throw __e instanceof RuntimeException ? (RuntimeException) __e : new RuntimeException(__e); }}
static Object getBot(String botID) {
  return callOpt(getMainBot(), "getBot", botID);
}

static boolean isNonNegativeInteger(String s) {
  return s != null && Pattern.matches("\\d+", s);
}
// This is made for NL parsing.
// It's javaTok extended with "..." token, "$n" and "#n" and
// special quotes (which are converted to normal ones).

static List<String> javaTokPlusPeriod(String s) {
  List<String> tok = new ArrayList<String>();
  int l = s.length();
  
  int i = 0;
  while (i < l) {
    int j = i;
    char c; String cc;
    
    // scan for whitespace
    while (j < l) {
      c = s.charAt(j);
      cc = s.substring(j, Math.min(j+2, l));
      if (c == ' ' || c == '\t' || c == '\r' || c == '\n')
        ++j;
      else if (cc.equals("/*")) {
        do ++j; while (j < l && !s.substring(j, Math.min(j+2, l)).equals("*/"));
        j = Math.min(j+2, l);
      } else if (cc.equals("//")) {
        do ++j; while (j < l && "\r\n".indexOf(s.charAt(j)) < 0);
      } else
        break;
    }
    
    tok.add(s.substring(i, j));
    i = j;
    if (i >= l) break;
    c = s.charAt(i);
    cc = s.substring(i, Math.min(i+2, l));

    // scan for non-whitespace
    if (c == '\u201C' || c == '\u201D') c = '"'; // normalize quotes
    if (c == '\'' || c == '"') {
      char opener = c;
      ++j;
      while (j < l) {
        char _c = s.charAt(j);
        if (_c == '\u201C' || _c == '\u201D') _c = '"'; // normalize quotes
        if (_c == opener) {
          ++j;
          break;
        } else if (s.charAt(j) == '\\' && j+1 < l)
          j += 2;
        else
          ++j;
      }
      if (j-1 >= i+1) {
        tok.add(opener + s.substring(i+1, j-1) + opener);
        i = j;
        continue;
      }
    } else if (Character.isJavaIdentifierStart(c))
      do ++j; while (j < l && (Character.isJavaIdentifierPart(s.charAt(j)) || s.charAt(j) == '\'')); // for things like "this one's"
    else if (Character.isDigit(c))
      do ++j; while (j < l && Character.isDigit(s.charAt(j)));
    else if (cc.equals("[[")) {
      do ++j; while (j+1 < l && !s.substring(j, j+2).equals("]]"));
      j = Math.min(j+2, l);
    } else if (cc.equals("[=") && i+2 < l && s.charAt(i+2) == '[') {
      do ++j; while (j+2 < l && !s.substring(j, j+3).equals("]=]"));
      j = Math.min(j+3, l);
    } else if (s.substring(j, Math.min(j+3, l)).equals("..."))
      j += 3;
    else if (c == '$' || c == '#')
      do ++j; while (j < l && Character.isDigit(s.charAt(j)));
    else
      ++j;

    tok.add(s.substring(i, j));
    i = j;
  }
  
  if ((tok.size() % 2) == 0) tok.add("");
  return tok;
}

  /** writes safely (to temp file, then rename) */
  public static void saveBinaryFile(String fileName, byte[] contents) throws IOException {
    File file = new File(fileName);
    File parentFile = file.getParentFile();
    if (parentFile != null)
      parentFile.mkdirs();
    String tempFileName = fileName + "_temp";
    FileOutputStream fileOutputStream = new FileOutputStream(tempFileName);
    fileOutputStream.write(contents);
    fileOutputStream.close();
    if (file.exists() && !file.delete())
      throw new IOException("Can't delete " + fileName);

    if (!new File(tempFileName).renameTo(file))
      throw new IOException("Can't rename " + tempFileName + " to " + fileName);
  }

  static void saveBinaryFile(File fileName, byte[] contents) {
    try {
      saveBinaryFile(fileName.getPath(), contents);
    } catch (IOException e) {
      throw new RuntimeException(e);
    }
  }
static void sleep(long ms) {
  try {
    Thread.sleep(ms);
  } catch (Exception e) { throw new RuntimeException(e); }
}

static void sleep() { try {
 
  print("Sleeping.");
  synchronized(main.class) { main.class.wait(); }

} catch (Throwable __e) { throw __e instanceof RuntimeException ? (RuntimeException) __e : new RuntimeException(__e); }}
static byte[] loadBinaryPage(String url) throws IOException {
  return loadBinaryPage(new URL(url).openConnection());
}

public static byte[] loadBinaryPage(URLConnection con) throws IOException {
  //setHeaders(con);
  ByteArrayOutputStream buf = new ByteArrayOutputStream();
  InputStream inputStream = con.getInputStream();
  int n = 0;
  while (true) {
    int ch = inputStream.read();
    if (ch < 0)
      break;
    buf.write(ch);
    if (++n % 100000 == 0)
      System.err.println("  " + n + " bytes loaded.");
  }
  inputStream.close();
  return buf.toByteArray();
}

static String structure(Object o) {
  HashSet refd = new HashSet();
  return structure_2(structure_1(o, 0, new IdentityHashMap(), refd), refd);
}

// leave to false, unless unstructure() breaks
static boolean structure_allowShortening = false;

static String structure_1(Object o, int stringSizeLimit, IdentityHashMap<Object, Integer> seen, HashSet<Integer> refd) {
  if (o == null) return "null";
  
  // these are never back-referenced (for readability)
  
  if (o instanceof String)
    return quote(stringSizeLimit != 0 ? shorten((String) o, stringSizeLimit) : (String) o);
    
  if (o instanceof BigInteger)
    return "bigint(" + o + ")";
  
  if (o instanceof Double)
    return "d(" + quote(str(o)) + ")";
    
  if (o instanceof Long)
    return o + "L";
  
  if (o instanceof Integer)
    return str(o);
    
  if (o instanceof Boolean)
    return ((Boolean) o).booleanValue() ? "t" : "f";
    
  if (o instanceof Character)
    return quoteCharacter((Character) o);
  
  Integer ref = seen.get(o);
  if (ref != null) {
    refd.add(ref);
    return "r" + ref;
  }
      
  ref = seen.size()+1;
  seen.put(o, ref);
  String r = "m" + ref + " "; // marker

  String name = o.getClass().getName();

  StringBuilder buf = new StringBuilder();
  
  if (o instanceof HashSet)
    return r + "hashset " + structure_1(new ArrayList((Set) o), stringSizeLimit, seen, refd);

  if (o instanceof TreeSet)
    return r + "treeset " + structure_1(new ArrayList((Set) o), stringSizeLimit, seen, refd);
  
  if (o instanceof Collection) {
    for (Object x : (Collection) o) {
      if (buf.length() != 0) buf.append(", ");
      buf.append(structure_1(x, stringSizeLimit, seen, refd));
    }
    return r + "[" + buf + "]";
  }
  
  if (o instanceof Map) {
    for (Object e : ((Map) o).entrySet()) {
      if (buf.length() != 0) buf.append(", ");
      buf.append(structure_1(((Map.Entry) e).getKey(), stringSizeLimit, seen, refd));
      buf.append("=");
      buf.append(structure_1(((Map.Entry) e).getValue(), stringSizeLimit, seen, refd));
    }
    return r + (o instanceof HashMap ? "hashmap" : "") + "{" + buf + "}";
  }
  
  if (o.getClass().isArray()) {
    int n = Array.getLength(o);
    for (int i = 0; i < n; i++) {
      if (buf.length() != 0) buf.append(", ");
      buf.append(structure_1(Array.get(o, i), stringSizeLimit, seen, refd));
    }
    return r + "array{" + buf + "}";
  }

  if (o instanceof Class)
    return r + "class(" + quote(((Class) o).getName()) + ")";
    
  if (o instanceof Throwable)
    return r + "exception(" + quote(((Throwable) o).getMessage()) + ")";
    
  if (o instanceof BitSet) {
    BitSet bs = (BitSet) o;
    for (int i = bs.nextSetBit(0); i >= 0; i = bs.nextSetBit(i+1)) {
      if (buf.length() != 0) buf.append(", ");
       buf.append(i);
    }
    return "bitset{" + buf + "}";
  }
    
  // Need more cases? This should cover all library classes...
  if (name.startsWith("java.") || name.startsWith("javax."))
    return r + String.valueOf(o);
    
  String shortName = o.getClass().getName().replaceAll("^main\\$", "");
  
  if (shortName.equals("Lisp")) {
    buf.append("l(" + structure_1(getOpt(o, "head"), stringSizeLimit, seen, refd));
    List args = (List) ( getOpt(o, "args"));
    if (nempty(args))
      for (int i = 0; i < l(args); i++) {
        buf.append(", ");
        Object arg = args.get(i);
        
        // sweet shortening
        if (arg != null && eq(arg.getClass().getName(), "main$Lisp") && isTrue(call(arg, "isEmpty")))
          arg = get(arg, "head");
          
        buf.append(structure_1(arg, stringSizeLimit, seen, refd));
      }
    buf.append(")");
    return r + str(buf);
  }
    
  int numFields = 0;
  String fieldName = "";
  if (shortName.equals("DynamicObject")) {
    shortName = (String) get(o, "className");
    Map<String, Object> fieldValues = (Map) get(o, "fieldValues");
    
    for (String _fieldName : fieldValues.keySet()) {
      fieldName = _fieldName;
      Object value = fieldValues.get(fieldName);
      if (value != null) {
        if (buf.length() != 0) buf.append(", ");
        buf.append(fieldName + "=" + structure_1(value, stringSizeLimit, seen, refd));
      }
      ++numFields;
   }
  } else {
    // regular class
    
    Class c = o.getClass();
    while (c != Object.class) {
      Field[] fields = c.getDeclaredFields();
      for (Field field : fields) {
        if ((field.getModifiers() & Modifier.STATIC) != 0)
          continue;
        fieldName = field.getName();
        
        // skip outer object reference
        if (fieldName.indexOf("$") >= 0) continue;
  
        Object value;
        try {
          field.setAccessible(true);
          value = field.get(o);
        } catch (Exception e) {
          value = "?";
        }
        
        // put special cases here...
    
        if (value != null) {
          if (buf.length() != 0) buf.append(", ");
          buf.append(fieldName + "=" + structure_1(value, stringSizeLimit, seen, refd));
        }
        ++numFields;
      }
      c = c.getSuperclass();
    }
  }
  
  String b = buf.toString();
  
  if (numFields == 1 && structure_allowShortening)
    b = b.replaceAll("^" + fieldName + "=", ""); // drop field name if only one
  String s = shortName;
  if (buf.length() != 0)
    s += "(" + b + ")";
  return r + s;
}

// drop unused markers
static String structure_2(String s, HashSet<Integer> refd) {
  List<String> tok = javaTok(s);
  StringBuilder out = new StringBuilder();
  for (int i = 1; i < l(tok); i += 2) {
    String t = tok.get(i);
    if (t.startsWith("m") && isInteger(t.substring(1))
      && !refd.contains(parseInt(t.substring(1))))
      continue;
    out.append(t).append(tok.get(i+1));
  }
  return str(out);
}

static File getGlobalCache() {
  File file = new File(userHome(), ".tinybrain/snippet-cache");
  file.mkdirs();
  return file;
}

static long parseLong(String s) {
  return Long.parseLong(s);
}

static boolean isInteger(String s) {
  return s != null && Pattern.matches("\\-?\\d+", s);
}
static String shorten(String s, int max) {
  if (s == null) return "";
  return s.length() <= max ? s : s.substring(0, Math.min(s.length(), max)) + "...";
}
static String quoteCharacter(char c) {
  if (c == '\'') return "'\\''";
  if (c == '\\') return "'\\\\'";
  return "'" + c + "'";
}

static String str(Object o) {
  return String.valueOf(o);
}
  /** writes safely (to temp file, then rename) */
  public static void saveTextFile(String fileName, String contents) throws IOException {
    File file = new File(fileName);
    File parentFile = file.getParentFile();
    if (parentFile != null)
      parentFile.mkdirs();
    String tempFileName = fileName + "_temp";
    if (contents != null) {
      FileOutputStream fileOutputStream = new FileOutputStream(tempFileName);
      OutputStreamWriter outputStreamWriter = new OutputStreamWriter(fileOutputStream, "UTF-8");
      PrintWriter printWriter = new PrintWriter(outputStreamWriter);
      printWriter.print(contents);
      printWriter.close();
    }
    
    if (file.exists() && !file.delete())
      throw new IOException("Can't delete " + fileName);

    if (contents != null)
      if (!new File(tempFileName).renameTo(file))
        throw new IOException("Can't rename " + tempFileName + " to " + fileName);
  }
  
  public static void saveTextFile(File fileName, String contents) {
    try {
      saveTextFile(fileName.getPath(), contents);
    } catch (IOException e) {
      throw new RuntimeException(e);
    }
  }

static boolean isJavaIdentifier(String s) {
  if (s.length() == 0 || !Character.isJavaIdentifierStart(s.charAt(0)))
    return false;
  for (int i = 1; i < s.length(); i++)
    if (!Character.isJavaIdentifierPart(s.charAt(i)))
      return false;
  return true;
}
static String _userHome;
static String userHome() {
  if (_userHome == null) {
    if (isAndroid())
      _userHome = "/storage/sdcard0/";
    else
      _userHome = System.getProperty("user.home");
    //System.out.println("userHome: " + _userHome);
  }
  return _userHome;
}

static String makeRandomID(int length) {
  Random random = new Random();
  char[] id = new char[length];
  for (int i = 0; i < id.length; i++)
    id[i] = (char) ((int) 'a' + random.nextInt(26));
  return new String(id);
}
static boolean nempty(Collection c) {
  return !isEmpty(c);
}

static boolean nempty(CharSequence s) {
  return !isEmpty(s);
}

static boolean nempty(Object[] o) {
  return !isEmpty(o);
}

static int parseInt(String s) {
  return empty(s) ? 0 : Integer.parseInt(s);
}
static boolean isTrue(Object o) {
  return booleanValue(o);
}

static boolean empty(Collection c) {
  return isEmpty(c);
}

static boolean empty(String s) {
  return isEmpty(s);
}

static boolean empty(Map map) {
  return map == null || map.isEmpty();
}
static boolean booleanValue(Object o) {
  return eq(true, o);
}
static boolean isEmpty(Collection c) {
  return c == null || c.isEmpty();
}

static boolean isEmpty(CharSequence s) {
  return s == null || s.length() == 0;
}

static boolean isEmpty(Object[] a) {
  return a == null || a.length == 0;
}
static boolean isAndroid() { return System.getProperty("java.vendor").toLowerCase().indexOf("android") >= 0; }
}