import java.io.*;

import java.lang.reflect.Method;

import java.net.URL;

import java.net.URLClassLoader;

import java.net.URLConnection;

import java.net.URLEncoder;

import java.security.MessageDigest;

import java.security.NoSuchAlgorithmException;

import java.util.*;

import java.util.regex.Matcher;

import java.util.regex.Pattern;

/**

 JavaX runner version 16

 Changes to v15:

 -fixed caching bug

 -remove "hash match" messages on System.err

 -cache transpiled translators

 */

public class x16 {

  static final String version = "JavaX 16";

  static boolean verbose = false, translate = false, list = false, virtualizeTranslators = true;

  static String translateTo = null;

  static boolean preferCached = false, safeOnly = false, noID = false;

  static List<String[]> mainTranslators = new ArrayList<String[]>();

  private static Map<Long, String> memSnippetCache = new HashMap<Long, String>();

  private static int processesStarted, compilations;

  // snippet ID -> md5

  private static HashMap<Long, String> prefetched = new HashMap<Long, String>();

  private static File virtCache;

  // doesn't work yet

  private static Map<String, Class<?>> programCache = new HashMap<String, Class<?>>();

  static boolean cacheTranslators = false;

  // this should work (caches transpiled translators)

  private static HashMap<Long, Object[]> translationCache = new HashMap<Long, Object[]>();

  static boolean cacheTranspiledTranslators = true;

  public static void main(String[] args) throws Exception {

    File ioBaseDir = new File("."), inputDir = null, outputDir = null;

    String src = null;

    List<String> programArgs = new ArrayList<String>();

    for (int i = 0; i < args.length; i++) {

      String arg = args[i];

      if (arg.equals("-v") || arg.equals("-verbose"))

        verbose = true;

      else if (arg.equals("-finderror"))

        verbose = true;

      else if (arg.equals("-offline") || arg.equalsIgnoreCase("-prefercached"))

        preferCached = true;

      else if (arg.equals("-novirt"))

        virtualizeTranslators = false;

      else if (arg.equals("-safeonly"))

        safeOnly = true;

      else if (arg.equals("-noid"))

        noID = true;

      else if (arg.equals("-nocachetranspiled"))

        cacheTranspiledTranslators = false;

      else if (arg.equals("translate"))

        translate = true;

      else if (arg.equals("list")) {

        list = true;

        virtualizeTranslators = false; // so they are silenced

      } else if (arg.equals("run")) {

        // it's the default command anyway

      } else if (arg.startsWith("input="))

        inputDir = new File(arg.substring(6));

      else if (arg.startsWith("output="))

        outputDir = new File(arg.substring(7));

      else if (arg.equals("with"))

        mainTranslators.add(new String[] {args[++i], null});

      else if (translate && arg.equals("to"))

        translateTo = args[++i];

      else if (src == null) {

        //System.out.println("src=" + arg);

        src = arg;

      } else

        programArgs.add(arg);

    }

    if (src == null) src = ".";

    // Might actually want to write to 2 disk caches (global/per program).

    if (virtualizeTranslators && !preferCached)

      virtCache = TempDirMaker_make();

    if (inputDir != null) {

      ioBaseDir = TempDirMaker_make();

      System.out.println("Taking input from: " + inputDir.getAbsolutePath());

      System.out.println("Output is in: " + new File(ioBaseDir, "output").getAbsolutePath());

      copyInput(inputDir, new File(ioBaseDir, "input"));

    }

    javaxmain(src, ioBaseDir, translate, list, programArgs.toArray(new String[programArgs.size()]));

    if (outputDir != null) {

      copyInput(new File(ioBaseDir, "output"), outputDir);

      System.out.println("Output copied to: " + outputDir.getAbsolutePath());

    }

    if (verbose) {

      // print stats

      System.out.println("Processes started: " + processesStarted + ", compilations: " + compilations);

    }

  }

  public static void javaxmain(String src, File ioDir, boolean translate, boolean list,

                               String[] args) throws Exception {

    File srcDir;

    if (isSnippetID(src)) {

      prefetch(src);

      srcDir = loadSnippetAsMainJava(src);

    } else {

      srcDir = new File(src);

      // if the argument is a file, it is assumed to be main.java

      if (srcDir.isFile()) {

        srcDir = TempDirMaker_make();

        copy(new File(src), new File(srcDir, "main.java"));

      }

      if (!new File(srcDir, "main.java").exists()) {

        System.out.println("This is " + version  + ".\n" +

          "No main.java found, exiting");

        return;

      }

    }

    // translate

    List<File> libraries = new ArrayList<File>();

    File X = topLevelTranslate(srcDir, libraries);

    // save prefetch data

    if (isSnippetID(src))

      savePrefetchData(src);

    // list or run

    if (translate) {

      File to = X;

      if (translateTo != null)

        if (new File(translateTo).isDirectory())

          to = new File(translateTo, "main.java");

        else

          to = new File(translateTo);

      if (to != X)

        copy(new File(X, "main.java"), to);

      System.out.println("Program translated to: " + to.getAbsolutePath());

    } else if (list)

      System.out.println(loadTextFile(new File(X, "main.java").getPath(), null));

    else

      javax2(X, ioDir, false, false, libraries, args, null);

  }

  private static void prefetch(String mainSnippetID) throws IOException {

    long mainID = parseSnippetID(mainSnippetID);

    String s = mainID + " " + loadTextFile(new File(".tinybrain/prefetch/" + mainID + ".txt").getPath(), "");

    String[] ids = s.trim().split(" ");

    if (ids.length > 1) {

      String url = "http://tinybrain.de:8080/tb-int/prefetch.php?ids=" + URLEncoder.encode(s, "UTF-8");

      String data = loadPage(new URL(url));

      String[] split = data.split(" ");

      if (split.length == ids.length)

        for (int i = 0; i < ids.length; i++)

          prefetched.put(parseSnippetID(ids[i]), split[i]);

    }

  }

  private static void savePrefetchData(String mainSnippetID) throws IOException {

    List<String> ids = new ArrayList<String>();

    long mainID = parseSnippetID(mainSnippetID);

    for (long id : memSnippetCache.keySet())

      if (id != mainID)

        ids.add(String.valueOf(id));

    saveTextFile(new File(".tinybrain/prefetch/" + mainID + ".txt").getPath(), join(" ", ids));

  }

  static File topLevelTranslate(File srcDir, List<File> libraries_out) throws Exception {

    File X = srcDir;

    X = applyTranslators(X, mainTranslators, libraries_out);

    X = defaultTranslate(X, libraries_out);

    return X;

  }

  private static File defaultTranslate(File x, List<File> libraries_out) throws Exception {

    x = luaPrintToJavaPrint(x);

    x = repeatAutoTranslate(x, libraries_out);

    return x;

  }

  private static File repeatAutoTranslate(File x, List<File> libraries_out) throws Exception {

    while (true) {

      File y = autoTranslate(x, libraries_out);

      if (y == x)

        return x;

      x = y;

    }

  }

  private static File autoTranslate(File x, List<File> libraries_out) throws Exception {

    String main = loadTextFile(new File(x, "main.java").getPath(), null);

    List<String> lines = toLines(main);

    List<String[]> translators = findTranslators(lines);

    if (translators.isEmpty())

      return x;

    main = fromLines(lines);

    File newDir = TempDirMaker_make();

    saveTextFile(new File(newDir, "main.java").getPath(), main);

    return applyTranslators(newDir, translators, libraries_out);

  }

  private static List<String[]> findTranslators(List<String> lines) {

    List<String[]> translators = new ArrayList<String[]>();

    Pattern pattern = Pattern.compile("^!([0-9# \t]+)");

    Pattern pArgs = Pattern.compile("^\\s*\\((.*)\\)");

    for (ListIterator<String> iterator = lines.listIterator(); iterator.hasNext(); ) {

      String line = iterator.next();

      line = line.trim();

      Matcher matcher = pattern.matcher(line);

      if (matcher.find()) {

        String[] t = matcher.group(1).split("[ \t]+");

        String rest = line.substring(matcher.end());

        String arg = null;

        if (t.length == 1) {

          Matcher mArgs = pArgs.matcher(rest);

          if (mArgs.find())

            arg = mArgs.group(1);

        }

        for (String transi : t)

          translators.add(new String[]{transi, arg});

        iterator.remove();

      }

    }

    return translators;

  }

  public static List<String> toLines(String s) {

    List<String> lines = new ArrayList<String>();

    int start = 0;

    while (true) {

      int i = toLines_nextLineBreak(s, start);

      if (i < 0) {

        if (s.length() > start) lines.add(s.substring(start));

        break;

      }

      lines.add(s.substring(start, i));

      if (s.charAt(i) == '\r' && i+1 < s.length() && s.charAt(i+1) == '\n')

        i += 2;

      else

        ++i;

      start = i;

    }

    return lines;

  }

  private static int toLines_nextLineBreak(String s, int start) {

    for (int i = start; i < s.length(); i++) {

      char c = s.charAt(i);

      if (c == '\r' || c == '\n')

        return i;

    }

    return -1;

  }

  public static String fromLines(List<String> lines) {

    StringBuilder buf = new StringBuilder();

    for (String line : lines) {

      buf.append(line).append('\n');

    }

    return buf.toString();

  }

  private static File applyTranslators(File x, List<String[]> translators, List<File> libraries_out) throws Exception {

    for (String[] translator : translators)

      x = applyTranslator(x, translator[0], translator[1], libraries_out);

    return x;

  }

  // also takes a library

  private static File applyTranslator(File x, String translator, String arg, List<File> libraries_out) throws Exception {

    if (verbose)

      System.out.println("Using translator " + translator + " on sources in " + x.getPath());

    File newDir = runTranslatorOnInput(translator, null, arg, x, !verbose, libraries_out);

    if (!new File(newDir, "main.java").exists()) {

      throw new Exception("Translator " + translator + " did not generate main.java");

      // TODO: show translator output

    }

    if (verbose)

      System.out.println("Translated with " + translator + " from " + x.getPath() + " to " + newDir.getPath());

    x = newDir;

    return x;

  }

  private static File luaPrintToJavaPrint(File x) throws IOException {

    File newDir = TempDirMaker_make();

    String code = loadTextFile(new File(x, "main.java").getPath(), null);

    code = luaPrintToJavaPrint(code);

    if (verbose)

      System.out.println(code);

    saveTextFile(new File(newDir, "main.java").getPath(), code);

    return newDir;

  }

  public static String luaPrintToJavaPrint(String code) {

    return ("\n" + code).replaceAll(

      "(\n\\s*)print (\".*\")",

      "$1System.out.println($2);").substring(1);

  }

  public static File loadSnippetAsMainJava(String snippetID) throws IOException {

    checkProgramSafety(snippetID);

    File srcDir = TempDirMaker_make();

    saveTextFile(new File(srcDir, "main.java").getPath(), loadSnippet(snippetID));

    return srcDir;

  }

  public static File loadSnippetAsMainJavaVerified(String snippetID, String hash) throws IOException {

    checkProgramSafety(snippetID);

    File srcDir = TempDirMaker_make();

    saveTextFile(new File(srcDir, "main.java").getPath(), loadSnippetVerified(snippetID, hash));

    return srcDir;

  }

  /** returns output dir */

  private static File runTranslatorOnInput(String snippetID, String hash, String arg, File input,

                                           boolean silent,

                                           List<File> libraries_out) throws Exception {

    File libraryFile = DiskSnippetCache_getLibrary(parseSnippetID(snippetID));

    if (libraryFile != null) {

      loadLibrary(snippetID, libraries_out, libraryFile);

      return input;

    }

    String[] args = arg != null ? new String[]{arg} : new String[0];

    File srcDir = hash == null ? loadSnippetAsMainJava(snippetID)

      : loadSnippetAsMainJavaVerified(snippetID, hash);

    long mainJavaSize = new File(srcDir, "main.java").length();

    if (mainJavaSize == 0) { // no text in snippet? assume it's a library

      loadLibrary(snippetID, libraries_out, libraryFile);

      return input;

    }

    File ioBaseDir = TempDirMaker_make();

    /*Class<?> mainClass = programCache.get("" + parseSnippetID(snippetID));

    if (mainClass != null)

      return runCached(ioBaseDir, input, args);*/

    // Doesn't work yet because virtualized directories are hardcoded in translator...

    List<File> libraries = new ArrayList<File>();

    Object[] cached = translationCache.get(parseSnippetID(snippetID));

    if (cached != null) {

      //System.err.println("Taking translator " + snippetID + " from cache!");

      srcDir = (File) cached[0];

      libraries = (List<File>) cached[1];

    } else {

      srcDir = defaultTranslate(srcDir, libraries);

      if (cacheTranspiledTranslators)

        translationCache.put(parseSnippetID(snippetID), new Object[] {srcDir, libraries});

    }

    boolean runInProcess = false;

    if (virtualizeTranslators) {

      if (verbose) System.out.println("Virtualizing translator");

      //srcDir = applyTranslator(srcDir, "#2000351"); // I/O-virtualize the translator

      // that doesn't work because it recurses infinitely...

      // So we do it right here:

      String s = loadTextFile(new File(srcDir, "main.java").getPath(), null);

      s = s.replaceAll("new\\s+File\\(", "virtual.newFile(");

      s = s.replaceAll("new\\s+FileInputStream\\(", "virtual.newFileInputStream(");

      s = s.replaceAll("new\\s+FileOutputStream\\(", "virtual.newFileOutputStream(");

      s += "\n\n" + loadSnippet("#2000355"); // load class virtual

      // change baseDir

      s = s.replace("virtual_baseDir = \"\";",

        "virtual_baseDir = " + javaQuote(ioBaseDir.getAbsolutePath()) + ";");

      // forward snippet cache (virtualized one)

      File dir = virtCache != null ? virtCache : DiskSnippetCache_dir;

      s = s.replace("static File DiskSnippetCache_dir;",

        "static File DiskSnippetCache_dir = new File(" + javaQuote(dir.getAbsolutePath()) + ");");

      s = s.replace("static boolean preferCached = false;", "static boolean preferCached = true;");

      if (verbose) {

        System.out.println("==BEGIN VIRTUALIZED TRANSLATOR==");

        System.out.println(s);

        System.out.println("==END VIRTUALIZED TRANSLATOR==");

      }

      srcDir = TempDirMaker_make();

      saveTextFile(new File(srcDir, "main.java").getPath(), s);

      // TODO: silence translator also

      runInProcess = true;

    }

    return runJavaX(ioBaseDir, srcDir, input, silent, runInProcess, libraries,

      args, cacheTranslators ? "" + parseSnippetID(snippetID) : null);

  }

  static void checkProgramSafety(String snippetID) throws IOException {

    if (!safeOnly) return;

    URL url = new URL("http://tinybrain.de:8080/tb-int/is-javax-safe.php?id=" + parseSnippetID(snippetID));

    String text = loadPage(url);

    if (!text.startsWith("{\"safe\":\"1\"}"))

      throw new RuntimeException("Translator not safe: #" + parseSnippetID(snippetID));

  }

  private static void loadLibrary(String snippetID, List<File> libraries_out, File libraryFile) throws IOException {

    if (verbose)

      System.out.println("Assuming " + snippetID + " is a library.");

    if (libraryFile == null) {

      byte[] data = loadDataSnippetImpl(snippetID);

      DiskSnippetCache_putLibrary(parseSnippetID(snippetID), data);

      libraryFile = DiskSnippetCache_getLibrary(parseSnippetID(snippetID));

    }

    if (!libraries_out.contains(libraryFile))

      libraries_out.add(libraryFile);

  }

  private static byte[] loadDataSnippetImpl(String snippetID) throws IOException {

    byte[] data;

    try {

      URL url = new URL("http://eyeocr.sourceforge.net/filestore/filestore.php?cmd=serve&file=blob_"

        + parseSnippetID(snippetID) + "&contentType=application/binary");

      System.err.println("Loading library: " + url);

      data = loadBinaryPage(url.openConnection());

      if (verbose)

        System.err.println("Bytes loaded: " + data.length);

    } catch (FileNotFoundException e) {

      throw new IOException("Binary snippet #" + snippetID + " not found or not public");

    }

    return data;

  }

  /** returns output dir */

  private static File runJavaX(File ioBaseDir, File originalSrcDir, File originalInput,

                               boolean silent, boolean runInProcess,

                               List<File> libraries, String[] args, String cacheAs) throws Exception {

    File srcDir = new File(ioBaseDir, "src");

    File inputDir = new File(ioBaseDir, "input");

    File outputDir = new File(ioBaseDir, "output");

    copyInput(originalSrcDir, srcDir);

    copyInput(originalInput, inputDir);

    javax2(srcDir, ioBaseDir, silent, runInProcess, libraries, args, cacheAs);

    return outputDir;

  }

  private static void copyInput(File src, File dst) throws IOException {

    copyDirectory(src, dst);

  }

  public static boolean hasFile(File inputDir, String name) {

    return new File(inputDir, name).exists();

  }

  public static void copyDirectory(File src, File dst) throws IOException {

    if (verbose) System.out.println("Copying " + src.getAbsolutePath() + " to " + dst.getAbsolutePath());

    dst.mkdirs();

    File[] files = src.listFiles();

    if (files == null) return;

    for (File file : files) {

      File dst1 = new File(dst, file.getName());

      if (file.isDirectory())

        copyDirectory(file, dst1);

      else {

        if (verbose) System.out.println("Copying " + file.getAbsolutePath() + " to " + dst1.getAbsolutePath());

        copy(file, dst1);

      }

    }

  }

  /** Quickly copy a file without a progress bar or any other fancy GUI... :) */

  public static void copy(File src, File dest) throws IOException {

    FileInputStream inputStream = new FileInputStream(src);

    FileOutputStream outputStream = new FileOutputStream(dest);

    try {

      copy(inputStream, outputStream);

      inputStream.close();

    } finally {

      outputStream.close();

    }

  }

  public static void copy(InputStream in, OutputStream out) throws IOException {

    byte[] buf = new byte[65536];

    while (true) {

      int n = in.read(buf);

      if (n <= 0) return;

      out.write(buf, 0, n);

    }

  }

  /** writes safely (to temp file, then rename) */

  public static void saveTextFile(String fileName, String contents) throws IOException {

    File file = new File(fileName);

    File parentFile = file.getParentFile();

    if (parentFile != null)

      parentFile.mkdirs();

    String tempFileName = fileName + "_temp";

    FileOutputStream fileOutputStream = new FileOutputStream(tempFileName);

    OutputStreamWriter outputStreamWriter = new OutputStreamWriter(fileOutputStream, charsetForTextFiles);

    PrintWriter printWriter = new PrintWriter(outputStreamWriter);

    printWriter.print(contents);

    printWriter.close();

    if (file.exists() && !file.delete())

      throw new IOException("Can't delete " + fileName);

    if (!new File(tempFileName).renameTo(file))

      throw new IOException("Can't rename " + tempFileName + " to " + fileName);

  }

  /** writes safely (to temp file, then rename) */

  public static void saveBinaryFile(String fileName, byte[] contents) throws IOException {

    File file = new File(fileName);

    File parentFile = file.getParentFile();

    if (parentFile != null)

      parentFile.mkdirs();

    String tempFileName = fileName + "_temp";

    FileOutputStream fileOutputStream = new FileOutputStream(tempFileName);

    fileOutputStream.write(contents);

    fileOutputStream.close();

    if (file.exists() && !file.delete())

      throw new IOException("Can't delete " + fileName);

    if (!new File(tempFileName).renameTo(file))

      throw new IOException("Can't rename " + tempFileName + " to " + fileName);

  }

  public static String loadTextFile(String fileName, String defaultContents) throws IOException {

    if (!new File(fileName).exists())

      return defaultContents;

    FileInputStream fileInputStream = new FileInputStream(fileName);

    InputStreamReader inputStreamReader = new InputStreamReader(fileInputStream, charsetForTextFiles);

    return loadTextFile(inputStreamReader, (int) new File(fileName).length());

  }

  public static String loadTextFile(Reader reader, int length) throws IOException {

    try {

      char[] chars = new char[length];

      int n = reader.read(chars);

      return new String(chars, 0, n);

    } finally {

      reader.close();

    }

  }

  static File DiskSnippetCache_dir;

  public static void initDiskSnippetCache(File dir) {

    DiskSnippetCache_dir = dir;

    dir.mkdirs();

  }

  // Data files are immutable, use centralized cache

  public static synchronized File DiskSnippetCache_getLibrary(long snippetID) throws IOException {

    File file = new File(getGlobalCache(), "data_" + snippetID + ".jar");

    if (verbose)

      System.out.println("Checking data cache: " + file.getPath());

    return file.exists() ? file : null;

  }

  public static synchronized String DiskSnippetCache_get(long snippetID) throws IOException {

    return loadTextFile(DiskSnippetCache_getFile(snippetID).getPath(), null);

  }

  private static File DiskSnippetCache_getFile(long snippetID) {

    return new File(DiskSnippetCache_dir, "" + snippetID);

  }

  public static synchronized void DiskSnippetCache_put(long snippetID, String snippet) throws IOException {

    saveTextFile(DiskSnippetCache_getFile(snippetID).getPath(), snippet);

  }

  public static synchronized void DiskSnippetCache_putLibrary(long snippetID, byte[] data) throws IOException {

    saveBinaryFile(new File(getGlobalCache(), "data_" + snippetID).getPath() + ".jar", data);

  }

  public static File DiskSnippetCache_getDir() {

    return DiskSnippetCache_dir;

  }

  public static void initSnippetCache() {

    if (DiskSnippetCache_dir == null)

      initDiskSnippetCache(getGlobalCache());

  }

  private static File getGlobalCache() {

    File file = new File(System.getProperty("user.home"), ".tinybrain/snippet-cache");

    file.mkdirs();

    return file;

  }

  public static String loadSnippetVerified(String snippetID, String hash) throws IOException {

    String text = loadSnippet(snippetID);

    String realHash = getHash(text.getBytes("UTF-8"));

    if (!realHash.equals(hash)) {

      String msg;

      if (hash.isEmpty())

        msg = "Here's your hash for " + snippetID + ", please put in your program: " + realHash;

      else

        msg = "Hash mismatch for " + snippetID + ": " + realHash + " (new) vs " + hash + " - has tinybrain.de been hacked??";

      throw new RuntimeException(msg);

    }

    return text;

  }

  public static String getHash(byte[] data) {

    return bytesToHex(getFullFingerprint(data));

  }

  public static byte[] getFullFingerprint(byte[] data) {

    try {

      return MessageDigest.getInstance("MD5").digest(data);

    } catch (NoSuchAlgorithmException e) {

      throw new RuntimeException(e);

    }

  }

  public static String bytesToHex(byte[] bytes) {

    return bytesToHex(bytes, 0, bytes.length);

  }

  public static String bytesToHex(byte[] bytes, int ofs, int len) {

    StringBuilder stringBuilder = new StringBuilder(len*2);

    for (int i = 0; i < len; i++) {

      String s = "0" + Integer.toHexString(bytes[ofs+i]);

      stringBuilder.append(s.substring(s.length()-2, s.length()));

    }

    return stringBuilder.toString();

  }

  public static String loadSnippet(String snippetID) throws IOException {

    return loadSnippet(parseSnippetID(snippetID));

  }

  public static long parseSnippetID(String snippetID) {

    return Long.parseLong(shortenSnippetID(snippetID));

  }

  private static String shortenSnippetID(String snippetID) {

    if (snippetID.startsWith("#"))

      snippetID = snippetID.substring(1);

    String httpBlaBla = "http://tinybrain.de/";

    if (snippetID.startsWith(httpBlaBla))

      snippetID = snippetID.substring(httpBlaBla.length());

    return snippetID;

  }

  public static boolean isSnippetID(String snippetID) {

    snippetID = shortenSnippetID(snippetID);

    return isInteger(snippetID) && Long.parseLong(snippetID) != 0;

  }

  public static boolean isInteger(String s) {

    return Pattern.matches("\\-?\\d+", s);

  }

  public static String loadSnippet(long snippetID) throws IOException {

    String text = memSnippetCache.get(snippetID);

    if (text != null)

      return text;

    initSnippetCache();

    text = DiskSnippetCache_get(snippetID);

    if (preferCached && text != null)

      return text;

    String md5 = text != null ? md5(text) : "-";

    if (text != null) {

      String hash = prefetched.get(snippetID);

      if (hash != null) {

        if (md5.equals(hash)) {

          memSnippetCache.put(snippetID, text);

          return text;

        } else

          prefetched.remove(snippetID); // (maybe this is not necessary)

      }

    }

    try {

      /*URL url = new URL("http://tinybrain.de:8080/getraw.php?id=" + snippetID);

      text = loadPage(url);*/

      String theURL = "http://tinybrain.de:8080/getraw.php?id=" + snippetID + "&getmd5=1&utf8=1";

      if (text != null) {

        //System.err.println("MD5: " + md5);

        theURL += "&md5=" + md5;

      }

      URL url = new URL(theURL);

      String page = loadPage(url);

      if (page.startsWith("==*#*==")) {

        // same, keep text

        //System.err.println("Snippet unchanged, keeping.");

      } else {

        // drop md5 line

        int i = page.indexOf('\n');

        String hash = page.substring(0, i).trim();

        text = page.substring(i+1);

        String myHash = md5(text);

        if (myHash.equals(hash)) {

          //System.err.println("Hash match: " + hash);

        } else

          System.err.println("Hash mismatch");

      }

    } catch (FileNotFoundException e) {

      throw new IOException("Snippet #" + snippetID + " not found or not public");

    }

    memSnippetCache.put(snippetID, text);

    try {

      initSnippetCache();

      DiskSnippetCache_put(snippetID, text);

    } catch (IOException e) {

      System.err.println("Minor warning: Couldn't save snippet to cache ("  + DiskSnippetCache_getDir() + ")");

    }

    return text;

  }

  private static String md5(String text) {

    try {

      return bytesToHex(md5impl(text.getBytes("UTF-8"))); // maybe different than the way PHP does it...

    } catch (UnsupportedEncodingException e) {

      throw new RuntimeException(e);

    }

  }

  public static byte[] md5impl(byte[] data) {

    try {

      return MessageDigest.getInstance("MD5").digest(data);

    } catch (NoSuchAlgorithmException e) {

      throw new RuntimeException(e);

    }

  }

  private static String loadPage(URL url) throws IOException {

    System.err.println("Loading: " + url.toExternalForm());

    URLConnection con = url.openConnection();

    return loadPage(con, url);

  }

  public static String loadPage(URLConnection con, URL url) throws IOException {

    setHeaders(con);

    String contentType = con.getContentType();

    if (contentType == null)

      throw new IOException("Page could not be read: " + url);

    //Log.info("Content-Type: " + contentType);

    String charset = guessCharset(contentType);

    //System.err.println("Charset: " + charset);

    Reader r = new InputStreamReader(con.getInputStream(), charset);

    StringBuilder buf = new StringBuilder();

    while (true) {

      int ch = r.read();

      if (ch < 0)

        break;

      //Log.info("Chars read: " + buf.length());

      buf.append((char) ch);

    }

    return buf.toString();

  }

  public static byte[] loadBinaryPage(URLConnection con) throws IOException {

    setHeaders(con);

    ByteArrayOutputStream buf = new ByteArrayOutputStream();

    InputStream inputStream = con.getInputStream();

    while (true) {

      int ch = inputStream.read();

      if (ch < 0)

        break;

      buf.write(ch);

    }

    inputStream.close();

    return buf.toByteArray();

  }

  private static void setHeaders(URLConnection con) throws IOException {

    String computerID = getComputerID();

    if (computerID != null)

      con.setRequestProperty("X-ComputerID", computerID);

  }

  public static String guessCharset(String contentType) {

    Pattern p = Pattern.compile("text/html;\\s+charset=([^\\s]+)\\s*");

    Matcher m = p.matcher(contentType);

    /* If Content-Type doesn't match this pre-conception, choose default and hope for the best. */

    return m.matches() ? m.group(1) : "ISO-8859-1";

  }

  /** runs a transpiled set of sources */

  public static void javax2(File srcDir, File ioBaseDir, boolean silent, boolean runInProcess,

                            List<File> libraries, String[] args, String cacheAs) throws Exception {

    File classesDir = TempDirMaker_make();

    String javacOutput = compileJava(srcDir, libraries, classesDir);

    // run

    if (verbose) System.out.println("Running program (" + srcDir.getAbsolutePath()

      + ") on io dir " + ioBaseDir.getAbsolutePath() + (runInProcess ? "[in-process]" : "") + "\n");

    runProgram(javacOutput, classesDir, ioBaseDir, silent, runInProcess, libraries, args, cacheAs);

  }

  static String compileJava(File srcDir, List<File> libraries, File classesDir) throws IOException {

    ++compilations;

    // collect sources

    List<File> sources = new ArrayList<File>();

    if (verbose) System.out.println("Scanning for sources in " + srcDir.getPath());

    scanForSources(srcDir, sources, true);

    if (sources.isEmpty())

      throw new IOException("No sources found");

    // compile

    File optionsFile = File.createTempFile("javax", "");

    if (verbose) System.out.println("Compiling " + sources.size() + " source(s) to " + classesDir.getPath());

    String options = "-d " + bashQuote(classesDir.getPath());

    writeOptions(sources, libraries, optionsFile, options);

    classesDir.mkdirs();

    return invokeJavac(optionsFile);

  }

  private static void runProgram(String javacOutput, File classesDir, File ioBaseDir,

                                 boolean silent, boolean runInProcess,

                                 List<File> libraries, String[] args, String cacheAs) throws Exception {

    // print javac output if compile failed and it hasn't been printed yet

    boolean didNotCompile = !didCompile(classesDir);

    if (verbose || didNotCompile)

      System.out.println(javacOutput);

    if (didNotCompile)

      return;

    if (runInProcess

      || (ioBaseDir.getAbsolutePath().equals(new File(".").getAbsolutePath()) && !silent)) {

      runProgramQuick(classesDir, libraries, args, cacheAs);

      return;

    }

    boolean echoOK = false;

    // TODO: add libraries to class path

    String bashCmd = "(cd " + bashQuote(ioBaseDir.getAbsolutePath()) + " && (java -cp "

      + bashQuote(classesDir.getAbsolutePath()) + " main" + (echoOK ? "; echo ok" : "") + "))";

    if (verbose) System.out.println(bashCmd);

    String output = backtick(bashCmd);

    if (verbose || !silent)

      System.out.println(output);

  }

  static boolean didCompile(File classesDir) {

    return hasFile(classesDir, "main.class");

  }

  private static void runProgramQuick(File classesDir, List<File> libraries,

                                      String[] args, String cacheAs) throws Exception {

    // collect urls

    URL[] urls = new URL[libraries.size()+1];

    urls[0] = classesDir.toURI().toURL();

    for (int i = 0; i < libraries.size(); i++)

      urls[i+1] = libraries.get(i).toURI().toURL();

    // make class loader

    URLClassLoader classLoader = new URLClassLoader(urls);

    // load JavaX main class

    Class<?> mainClass = classLoader.loadClass("main");

    if (cacheAs != null)

      programCache.put(cacheAs, mainClass);

    // run main method

    Method main = mainClass.getMethod("main", String[].class);

    main.invoke(null, (Object) args);

  }

  private static String invokeJavac(File optionsFile) throws IOException {

    String output;

    try {

      output = invokeEcj(optionsFile);

    } catch (NoClassDefFoundError e) {

      if (verbose) {

        System.err.println("ecj not found - using javac");

        e.printStackTrace();

      }

      output = backtick("javac " + bashQuote("@" + optionsFile.getPath()));

    }

    if (verbose) System.out.println(output);

    return output;

  }

  // throws ClassNotFoundError if ecj is not in classpath

  static String invokeEcj(File optionsFile) {

    StringWriter writer = new StringWriter();

    PrintWriter printWriter = new PrintWriter(writer);

    org.eclipse.jdt.core.compiler.CompilationProgress progress = null;

    // add more eclipse options in the line below

    org.eclipse.jdt.core.compiler.batch.BatchCompiler.compile(

      new String[] { "@" + optionsFile.getPath(),

        "-source", "1.7",

        "-nowarn"

      },

      printWriter,

      printWriter,

      progress);

    return writer.toString();

  }

  private static void writeOptions(List<File> sources, List<File> libraries,

                                   File optionsFile, String moreOptions) throws IOException {

    FileWriter writer = new FileWriter(optionsFile);

    for (File source : sources)

      writer.write(bashQuote(source.getPath()) + " ");

    if (!libraries.isEmpty()) {

      List<String> cp = new ArrayList<String>();

      for (File lib : libraries)

        cp.add(lib.getAbsolutePath());

      writer.write("-cp " + bashQuote(join(File.pathSeparator, cp)) + " ");

    }

    writer.write(moreOptions);

    writer.close();

  }

  static void scanForSources(File source, List<File> sources, boolean topLevel) {

    if (source.isFile() && source.getName().endsWith(".java"))

      sources.add(source);

    else if (source.isDirectory() && !isSkippedDirectoryName(source.getName(), topLevel)) {

      File[] files = source.listFiles();

      for (File file : files)

        scanForSources(file, sources, false);

    }

  }

  private static boolean isSkippedDirectoryName(String name, boolean topLevel) {

    if (topLevel) return false; // input or output ok as highest directory (intentionally specified by user, not just found by a directory scan in which case we probably don't want it. it's more like heuristics actually.)

    return name.equalsIgnoreCase("input") || name.equalsIgnoreCase("output");

  }

  public static String backtick(String cmd) throws IOException {

    ++processesStarted;

    File outFile = File.createTempFile("_backtick", "");

    File scriptFile = File.createTempFile("_backtick", isWindows() ? ".bat" : "");

    String command = cmd + ">" + bashQuote(outFile.getPath()) + " 2>&1";

    //Log.info("[Backtick] " + command);

    try {

      saveTextFile(scriptFile.getPath(), command);

      String[] command2;

      if (isWindows())

        command2 = new String[] { scriptFile.getPath() };

      else

        command2 = new String[] { "/bin/bash", scriptFile.getPath() };

      Process process = Runtime.getRuntime().exec(command2);

      try {

        process.waitFor();

      } catch (InterruptedException e) {

        throw new RuntimeException(e);

      }

      process.exitValue();

      return loadTextFile(outFile.getPath(), "");

    } finally {

      scriptFile.delete();

    }

  }

  /** possibly improvable */

  public static String javaQuote(String text) {

    return bashQuote(text);

  }

  /** possibly improvable */

  public static String bashQuote(String text) {

    if (text == null) return null;

    return "\"" + text

      .replace("\\", "\\\\")

      .replace("\"", "\\\"")

      .replace("\n", "\\n")

      .replace("\r", "\\r") + "\"";

  }

  public final static String charsetForTextFiles = "UTF8";

  static long TempDirMaker_lastValue;

  public static File TempDirMaker_make() {

    File dir = new File(System.getProperty("user.home"), ".javax/" + TempDirMaker_newValue());

    dir.mkdirs();

    return dir;

  }

  private static long TempDirMaker_newValue() {

    long value;

    do

      value = System.currentTimeMillis();

    while (value == TempDirMaker_lastValue);

    TempDirMaker_lastValue = value;

    return value;

  }

  public static String join(String glue, Iterable<String> strings) {

    StringBuilder buf = new StringBuilder();

    Iterator<String> i = strings.iterator();

    if (i.hasNext()) {

      buf.append(i.next());

      while (i.hasNext())

        buf.append(glue).append(i.next());

    }

    return buf.toString();

  }

  public static boolean isWindows() {

    return System.getProperty("os.name").contains("Windows");

  }

  public static String makeRandomID(int length) {

    Random random = new Random();

    char[] id = new char[length];

    for (int i = 0; i< id.length; i++)

      id[i] = (char) ((int) 'a' + random.nextInt(26));

    return new String(id);

  }

  static String computerID;

  public static String getComputerID() throws IOException {

    if (noID) return null;

    if (computerID == null) {

      File file = new File(System.getProperty("user.home"), ".tinybrain/computer-id");

      computerID = loadTextFile(file.getPath(), null);

      if (computerID == null) {

        computerID = makeRandomID(12);

        saveTextFile(file.getPath(), computerID);

      }

      if (verbose)

        System.out.println("Local computer ID: " + computerID);

    }

    return computerID;

  }

}