Not logged in.  Login/Logout/Register | List snippets | | Create snippet | Upload image | Upload data

1061
LINES

< > BotCompany Repo | #2000469 // class _x16 (embeddable)

New Tinybrain snippet [tags: archive]

1  
import java.io.*;
2  
import java.lang.reflect.Method;
3  
import java.net.URL;
4  
import java.net.URLClassLoader;
5  
import java.net.URLConnection;
6  
import java.net.URLEncoder;
7  
import java.security.MessageDigest;
8  
import java.security.NoSuchAlgorithmException;
9  
import java.util.*;
10  
import java.util.regex.Matcher;
11  
import java.util.regex.Pattern;
12  
13  
/**
14  
 JavaX runner version 16
15  
16  
 Changes to v15:
17  
 -fixed caching bug
18  
 -remove "hash match" messages on System.err
19  
 -cache transpiled translators
20  
21  
 */
22  
23  
class _x16 {
24  
  static final String version = "JavaX 16";
25  
26  
  static boolean verbose = false, translate = false, list = false, virtualizeTranslators = true;
27  
  static String translateTo = null;
28  
  static boolean preferCached = false, safeOnly = false, noID = false;
29  
  static List<String[]> mainTranslators = new ArrayList<String[]>();
30  
  private static Map<Long, String> memSnippetCache = new HashMap<Long, String>();
31  
  private static int processesStarted, compilations;
32  
33  
  // snippet ID -> md5
34  
  private static HashMap<Long, String> prefetched = new HashMap<Long, String>();
35  
  private static File virtCache;
36  
37  
  // doesn't work yet
38  
  private static Map<String, Class<?>> programCache = new HashMap<String, Class<?>>();
39  
  static boolean cacheTranslators = false;
40  
41  
  // this should work (caches transpiled translators)
42  
  private static HashMap<Long, Object[]> translationCache = new HashMap<Long, Object[]>();
43  
  static boolean cacheTranspiledTranslators = true;
44  
45  
  public static void main(String[] args) throws Exception {
46  
    File ioBaseDir = new File("."), inputDir = null, outputDir = null;
47  
    String src = null;
48  
    List<String> programArgs = new ArrayList<String>();
49  
50  
    for (int i = 0; i < args.length; i++) {
51  
      String arg = args[i];
52  
      if (arg.equals("-v") || arg.equals("-verbose"))
53  
        verbose = true;
54  
      else if (arg.equals("-finderror"))
55  
        verbose = true;
56  
      else if (arg.equals("-offline") || arg.equalsIgnoreCase("-prefercached"))
57  
        preferCached = true;
58  
      else if (arg.equals("-novirt"))
59  
        virtualizeTranslators = false;
60  
      else if (arg.equals("-safeonly"))
61  
        safeOnly = true;
62  
      else if (arg.equals("-noid"))
63  
        noID = true;
64  
      else if (arg.equals("-nocachetranspiled"))
65  
        cacheTranspiledTranslators = false;
66  
      else if (arg.equals("translate"))
67  
        translate = true;
68  
      else if (arg.equals("list")) {
69  
        list = true;
70  
        virtualizeTranslators = false; // so they are silenced
71  
      } else if (arg.equals("run")) {
72  
        // it's the default command anyway
73  
      } else if (arg.startsWith("input="))
74  
        inputDir = new File(arg.substring(6));
75  
      else if (arg.startsWith("output="))
76  
        outputDir = new File(arg.substring(7));
77  
      else if (arg.equals("with"))
78  
        mainTranslators.add(new String[] {args[++i], null});
79  
      else if (translate && arg.equals("to"))
80  
        translateTo = args[++i];
81  
      else if (src == null) {
82  
        //System.out.println("src=" + arg);
83  
        src = arg;
84  
      } else
85  
        programArgs.add(arg);
86  
    }
87  
88  
    if (src == null) src = ".";
89  
90  
    // Might actually want to write to 2 disk caches (global/per program).
91  
    if (virtualizeTranslators && !preferCached)
92  
      virtCache = TempDirMaker_make();
93  
94  
    if (inputDir != null) {
95  
      ioBaseDir = TempDirMaker_make();
96  
      System.out.println("Taking input from: " + inputDir.getAbsolutePath());
97  
      System.out.println("Output is in: " + new File(ioBaseDir, "output").getAbsolutePath());
98  
      copyInput(inputDir, new File(ioBaseDir, "input"));
99  
    }
100  
101  
    javaxmain(src, ioBaseDir, translate, list, programArgs.toArray(new String[programArgs.size()]));
102  
103  
    if (outputDir != null) {
104  
      copyInput(new File(ioBaseDir, "output"), outputDir);
105  
      System.out.println("Output copied to: " + outputDir.getAbsolutePath());
106  
    }
107  
108  
    if (verbose) {
109  
      // print stats
110  
      System.out.println("Processes started: " + processesStarted + ", compilations: " + compilations);
111  
    }
112  
  }
113  
114  
  public static void javaxmain(String src, File ioDir, boolean translate, boolean list,
115  
                               String[] args) throws Exception {
116  
    File srcDir;
117  
    if (isSnippetID(src)) {
118  
      prefetch(src);
119  
      srcDir = loadSnippetAsMainJava(src);
120  
    } else {
121  
      srcDir = new File(src);
122  
123  
      // if the argument is a file, it is assumed to be main.java
124  
      if (srcDir.isFile()) {
125  
        srcDir = TempDirMaker_make();
126  
        copy(new File(src), new File(srcDir, "main.java"));
127  
      }
128  
129  
      if (!new File(srcDir, "main.java").exists()) {
130  
        System.out.println("This is " + version  + ".\n" +
131  
          "No main.java found, exiting");
132  
        return;
133  
      }
134  
    }
135  
136  
    // translate
137  
138  
    List<File> libraries = new ArrayList<File>();
139  
    File X = topLevelTranslate(srcDir, libraries);
140  
141  
    // save prefetch data
142  
    if (isSnippetID(src))
143  
      savePrefetchData(src);
144  
145  
    // list or run
146  
147  
    if (translate) {
148  
      File to = X;
149  
      if (translateTo != null)
150  
        if (new File(translateTo).isDirectory())
151  
          to = new File(translateTo, "main.java");
152  
        else
153  
          to = new File(translateTo);
154  
      if (to != X)
155  
        copy(new File(X, "main.java"), to);
156  
      System.out.println("Program translated to: " + to.getAbsolutePath());
157  
    } else if (list)
158  
      System.out.println(loadTextFile(new File(X, "main.java").getPath(), null));
159  
    else
160  
      javax2(X, ioDir, false, false, libraries, args, null);
161  
  }
162  
163  
  private static void prefetch(String mainSnippetID) throws IOException {
164  
    long mainID = parseSnippetID(mainSnippetID);
165  
    String s = mainID + " " + loadTextFile(new File(".tinybrain/prefetch/" + mainID + ".txt").getPath(), "");
166  
    String[] ids = s.trim().split(" ");
167  
    if (ids.length > 1) {
168  
      String url = "http://tinybrain.de:8080/tb-int/prefetch.php?ids=" + URLEncoder.encode(s, "UTF-8");
169  
      String data = loadPage(new URL(url));
170  
      String[] split = data.split(" ");
171  
      if (split.length == ids.length)
172  
        for (int i = 0; i < ids.length; i++)
173  
          prefetched.put(parseSnippetID(ids[i]), split[i]);
174  
    }
175  
  }
176  
177  
  private static void savePrefetchData(String mainSnippetID) throws IOException {
178  
    List<String> ids = new ArrayList<String>();
179  
    long mainID = parseSnippetID(mainSnippetID);
180  
181  
    for (long id : memSnippetCache.keySet())
182  
      if (id != mainID)
183  
        ids.add(String.valueOf(id));
184  
185  
    saveTextFile(new File(".tinybrain/prefetch/" + mainID + ".txt").getPath(), join(" ", ids));
186  
  }
187  
188  
  static File topLevelTranslate(File srcDir, List<File> libraries_out) throws Exception {
189  
    File X = srcDir;
190  
    X = applyTranslators(X, mainTranslators, libraries_out);
191  
    X = defaultTranslate(X, libraries_out);
192  
    return X;
193  
  }
194  
195  
  private static File defaultTranslate(File x, List<File> libraries_out) throws Exception {
196  
    x = luaPrintToJavaPrint(x);
197  
    x = repeatAutoTranslate(x, libraries_out);
198  
    return x;
199  
  }
200  
201  
  private static File repeatAutoTranslate(File x, List<File> libraries_out) throws Exception {
202  
    while (true) {
203  
      File y = autoTranslate(x, libraries_out);
204  
      if (y == x)
205  
        return x;
206  
      x = y;
207  
    }
208  
  }
209  
210  
  private static File autoTranslate(File x, List<File> libraries_out) throws Exception {
211  
    String main = loadTextFile(new File(x, "main.java").getPath(), null);
212  
    List<String> lines = toLines(main);
213  
    List<String[]> translators = findTranslators(lines);
214  
    if (translators.isEmpty())
215  
      return x;
216  
217  
    main = fromLines(lines);
218  
    File newDir = TempDirMaker_make();
219  
    saveTextFile(new File(newDir, "main.java").getPath(), main);
220  
    return applyTranslators(newDir, translators, libraries_out);
221  
  }
222  
223  
  private static List<String[]> findTranslators(List<String> lines) {
224  
    List<String[]> translators = new ArrayList<String[]>();
225  
    Pattern pattern = Pattern.compile("^!([0-9# \t]+)");
226  
    Pattern pArgs = Pattern.compile("^\\s*\\((.*)\\)");
227  
    for (ListIterator<String> iterator = lines.listIterator(); iterator.hasNext(); ) {
228  
      String line = iterator.next();
229  
      line = line.trim();
230  
      Matcher matcher = pattern.matcher(line);
231  
      if (matcher.find()) {
232  
        String[] t = matcher.group(1).split("[ \t]+");
233  
        String rest = line.substring(matcher.end());
234  
        String arg = null;
235  
        if (t.length == 1) {
236  
          Matcher mArgs = pArgs.matcher(rest);
237  
          if (mArgs.find())
238  
            arg = mArgs.group(1);
239  
        }
240  
        for (String transi : t)
241  
          translators.add(new String[]{transi, arg});
242  
        iterator.remove();
243  
      }
244  
    }
245  
    return translators;
246  
  }
247  
248  
  public static List<String> toLines(String s) {
249  
    List<String> lines = new ArrayList<String>();
250  
    int start = 0;
251  
    while (true) {
252  
      int i = toLines_nextLineBreak(s, start);
253  
      if (i < 0) {
254  
        if (s.length() > start) lines.add(s.substring(start));
255  
        break;
256  
      }
257  
258  
      lines.add(s.substring(start, i));
259  
      if (s.charAt(i) == '\r' && i+1 < s.length() && s.charAt(i+1) == '\n')
260  
        i += 2;
261  
      else
262  
        ++i;
263  
264  
      start = i;
265  
    }
266  
    return lines;
267  
  }
268  
269  
  private static int toLines_nextLineBreak(String s, int start) {
270  
    for (int i = start; i < s.length(); i++) {
271  
      char c = s.charAt(i);
272  
      if (c == '\r' || c == '\n')
273  
        return i;
274  
    }
275  
    return -1;
276  
  }
277  
278  
  public static String fromLines(List<String> lines) {
279  
    StringBuilder buf = new StringBuilder();
280  
    for (String line : lines) {
281  
      buf.append(line).append('\n');
282  
    }
283  
    return buf.toString();
284  
  }
285  
286  
  private static File applyTranslators(File x, List<String[]> translators, List<File> libraries_out) throws Exception {
287  
    for (String[] translator : translators)
288  
      x = applyTranslator(x, translator[0], translator[1], libraries_out);
289  
    return x;
290  
  }
291  
292  
  // also takes a library
293  
  private static File applyTranslator(File x, String translator, String arg, List<File> libraries_out) throws Exception {
294  
    if (verbose)
295  
      System.out.println("Using translator " + translator + " on sources in " + x.getPath());
296  
297  
    File newDir = runTranslatorOnInput(translator, null, arg, x, !verbose, libraries_out);
298  
299  
    if (!new File(newDir, "main.java").exists()) {
300  
      throw new Exception("Translator " + translator + " did not generate main.java");
301  
      // TODO: show translator output
302  
    }
303  
    if (verbose)
304  
      System.out.println("Translated with " + translator + " from " + x.getPath() + " to " + newDir.getPath());
305  
    x = newDir;
306  
    return x;
307  
  }
308  
309  
  private static File luaPrintToJavaPrint(File x) throws IOException {
310  
    File newDir = TempDirMaker_make();
311  
    String code = loadTextFile(new File(x, "main.java").getPath(), null);
312  
    code = luaPrintToJavaPrint(code);
313  
    if (verbose)
314  
      System.out.println(code);
315  
    saveTextFile(new File(newDir, "main.java").getPath(), code);
316  
    return newDir;
317  
  }
318  
319  
  public static String luaPrintToJavaPrint(String code) {
320  
    return ("\n" + code).replaceAll(
321  
      "(\n\\s*)print (\".*\")",
322  
      "$1System.out.println($2);").substring(1);
323  
  }
324  
325  
  public static File loadSnippetAsMainJava(String snippetID) throws IOException {
326  
    checkProgramSafety(snippetID);
327  
    File srcDir = TempDirMaker_make();
328  
    saveTextFile(new File(srcDir, "main.java").getPath(), loadSnippet(snippetID));
329  
    return srcDir;
330  
  }
331  
332  
  public static File loadSnippetAsMainJavaVerified(String snippetID, String hash) throws IOException {
333  
    checkProgramSafety(snippetID);
334  
    File srcDir = TempDirMaker_make();
335  
    saveTextFile(new File(srcDir, "main.java").getPath(), loadSnippetVerified(snippetID, hash));
336  
    return srcDir;
337  
  }
338  
339  
  /** returns output dir */
340  
  private static File runTranslatorOnInput(String snippetID, String hash, String arg, File input,
341  
                                           boolean silent,
342  
                                           List<File> libraries_out) throws Exception {
343  
    File libraryFile = DiskSnippetCache_getLibrary(parseSnippetID(snippetID));
344  
    if (libraryFile != null) {
345  
      loadLibrary(snippetID, libraries_out, libraryFile);
346  
      return input;
347  
    }
348  
349  
    String[] args = arg != null ? new String[]{arg} : new String[0];
350  
351  
    File srcDir = hash == null ? loadSnippetAsMainJava(snippetID)
352  
      : loadSnippetAsMainJavaVerified(snippetID, hash);
353  
    long mainJavaSize = new File(srcDir, "main.java").length();
354  
355  
    if (mainJavaSize == 0) { // no text in snippet? assume it's a library
356  
      loadLibrary(snippetID, libraries_out, libraryFile);
357  
      return input;
358  
    }
359  
360  
    File ioBaseDir = TempDirMaker_make();
361  
362  
    /*Class<?> mainClass = programCache.get("" + parseSnippetID(snippetID));
363  
    if (mainClass != null)
364  
      return runCached(ioBaseDir, input, args);*/
365  
    // Doesn't work yet because virtualized directories are hardcoded in translator...
366  
367  
    List<File> libraries = new ArrayList<File>();
368  
    Object[] cached = translationCache.get(parseSnippetID(snippetID));
369  
    if (cached != null) {
370  
      //System.err.println("Taking translator " + snippetID + " from cache!");
371  
      srcDir = (File) cached[0];
372  
      libraries = (List<File>) cached[1];
373  
    } else {
374  
      srcDir = defaultTranslate(srcDir, libraries);
375  
      if (cacheTranspiledTranslators)
376  
        translationCache.put(parseSnippetID(snippetID), new Object[] {srcDir, libraries});
377  
    }
378  
    boolean runInProcess = false;
379  
380  
    if (virtualizeTranslators) {
381  
      if (verbose) System.out.println("Virtualizing translator");
382  
383  
      //srcDir = applyTranslator(srcDir, "#2000351"); // I/O-virtualize the translator
384  
      // that doesn't work because it recurses infinitely...
385  
386  
      // So we do it right here:
387  
      String s = loadTextFile(new File(srcDir, "main.java").getPath(), null);
388  
      s = s.replaceAll("new\\s+File\\(", "virtual.newFile(");
389  
      s = s.replaceAll("new\\s+FileInputStream\\(", "virtual.newFileInputStream(");
390  
      s = s.replaceAll("new\\s+FileOutputStream\\(", "virtual.newFileOutputStream(");
391  
      s += "\n\n" + loadSnippet("#2000355"); // load class virtual
392  
393  
      // change baseDir
394  
      s = s.replace("virtual_baseDir = \"\";",
395  
        "virtual_baseDir = " + javaQuote(ioBaseDir.getAbsolutePath()) + ";");
396  
397  
      // forward snippet cache (virtualized one)
398  
      File dir = virtCache != null ? virtCache : DiskSnippetCache_dir;
399  
      s = s.replace("static File DiskSnippetCache_dir;",
400  
        "static File DiskSnippetCache_dir = new File(" + javaQuote(dir.getAbsolutePath()) + ");");
401  
      s = s.replace("static boolean preferCached = false;", "static boolean preferCached = true;");
402  
403  
      if (verbose) {
404  
        System.out.println("==BEGIN VIRTUALIZED TRANSLATOR==");
405  
        System.out.println(s);
406  
        System.out.println("==END VIRTUALIZED TRANSLATOR==");
407  
      }
408  
      srcDir = TempDirMaker_make();
409  
      saveTextFile(new File(srcDir, "main.java").getPath(), s);
410  
411  
      // TODO: silence translator also
412  
      runInProcess = true;
413  
    }
414  
415  
    return runJavaX(ioBaseDir, srcDir, input, silent, runInProcess, libraries,
416  
      args, cacheTranslators ? "" + parseSnippetID(snippetID) : null);
417  
  }
418  
419  
  static void checkProgramSafety(String snippetID) throws IOException {
420  
    if (!safeOnly) return;
421  
    URL url = new URL("http://tinybrain.de:8080/tb-int/is-javax-safe.php?id=" + parseSnippetID(snippetID));
422  
    String text = loadPage(url);
423  
    if (!text.startsWith("{\"safe\":\"1\"}"))
424  
      throw new RuntimeException("Translator not safe: #" + parseSnippetID(snippetID));
425  
  }
426  
427  
  private static void loadLibrary(String snippetID, List<File> libraries_out, File libraryFile) throws IOException {
428  
    if (verbose)
429  
      System.out.println("Assuming " + snippetID + " is a library.");
430  
431  
    if (libraryFile == null) {
432  
      byte[] data = loadDataSnippetImpl(snippetID);
433  
      DiskSnippetCache_putLibrary(parseSnippetID(snippetID), data);
434  
      libraryFile = DiskSnippetCache_getLibrary(parseSnippetID(snippetID));
435  
    }
436  
437  
    if (!libraries_out.contains(libraryFile))
438  
      libraries_out.add(libraryFile);
439  
  }
440  
441  
  private static byte[] loadDataSnippetImpl(String snippetID) throws IOException {
442  
    byte[] data;
443  
    try {
444  
      URL url = new URL("http://eyeocr.sourceforge.net/filestore/filestore.php?cmd=serve&file=blob_"
445  
        + parseSnippetID(snippetID) + "&contentType=application/binary");
446  
      System.err.println("Loading library: " + url);
447  
      data = loadBinaryPage(url.openConnection());
448  
      if (verbose)
449  
        System.err.println("Bytes loaded: " + data.length);
450  
    } catch (FileNotFoundException e) {
451  
      throw new IOException("Binary snippet #" + snippetID + " not found or not public");
452  
    }
453  
    return data;
454  
  }
455  
456  
  /** returns output dir */
457  
  private static File runJavaX(File ioBaseDir, File originalSrcDir, File originalInput,
458  
                               boolean silent, boolean runInProcess,
459  
                               List<File> libraries, String[] args, String cacheAs) throws Exception {
460  
    File srcDir = new File(ioBaseDir, "src");
461  
    File inputDir = new File(ioBaseDir, "input");
462  
    File outputDir = new File(ioBaseDir, "output");
463  
    copyInput(originalSrcDir, srcDir);
464  
    copyInput(originalInput, inputDir);
465  
    javax2(srcDir, ioBaseDir, silent, runInProcess, libraries, args, cacheAs);
466  
    return outputDir;
467  
  }
468  
469  
  private static void copyInput(File src, File dst) throws IOException {
470  
    copyDirectory(src, dst);
471  
  }
472  
473  
  public static boolean hasFile(File inputDir, String name) {
474  
    return new File(inputDir, name).exists();
475  
  }
476  
477  
  public static void copyDirectory(File src, File dst) throws IOException {
478  
    if (verbose) System.out.println("Copying " + src.getAbsolutePath() + " to " + dst.getAbsolutePath());
479  
    dst.mkdirs();
480  
    File[] files = src.listFiles();
481  
    if (files == null) return;
482  
    for (File file : files) {
483  
      File dst1 = new File(dst, file.getName());
484  
      if (file.isDirectory())
485  
        copyDirectory(file, dst1);
486  
      else {
487  
        if (verbose) System.out.println("Copying " + file.getAbsolutePath() + " to " + dst1.getAbsolutePath());
488  
        copy(file, dst1);
489  
      }
490  
    }
491  
  }
492  
493  
  /** Quickly copy a file without a progress bar or any other fancy GUI... :) */
494  
  public static void copy(File src, File dest) throws IOException {
495  
    FileInputStream inputStream = new FileInputStream(src);
496  
    FileOutputStream outputStream = new FileOutputStream(dest);
497  
    try {
498  
      copy(inputStream, outputStream);
499  
      inputStream.close();
500  
    } finally {
501  
      outputStream.close();
502  
    }
503  
  }
504  
505  
  public static void copy(InputStream in, OutputStream out) throws IOException {
506  
    byte[] buf = new byte[65536];
507  
    while (true) {
508  
      int n = in.read(buf);
509  
      if (n <= 0) return;
510  
      out.write(buf, 0, n);
511  
    }
512  
  }
513  
514  
  /** writes safely (to temp file, then rename) */
515  
  public static void saveTextFile(String fileName, String contents) throws IOException {
516  
    File file = new File(fileName);
517  
    File parentFile = file.getParentFile();
518  
    if (parentFile != null)
519  
      parentFile.mkdirs();
520  
    String tempFileName = fileName + "_temp";
521  
    FileOutputStream fileOutputStream = new FileOutputStream(tempFileName);
522  
    OutputStreamWriter outputStreamWriter = new OutputStreamWriter(fileOutputStream, charsetForTextFiles);
523  
    PrintWriter printWriter = new PrintWriter(outputStreamWriter);
524  
    printWriter.print(contents);
525  
    printWriter.close();
526  
    if (file.exists() && !file.delete())
527  
      throw new IOException("Can't delete " + fileName);
528  
529  
    if (!new File(tempFileName).renameTo(file))
530  
      throw new IOException("Can't rename " + tempFileName + " to " + fileName);
531  
  }
532  
533  
  /** writes safely (to temp file, then rename) */
534  
  public static void saveBinaryFile(String fileName, byte[] contents) throws IOException {
535  
    File file = new File(fileName);
536  
    File parentFile = file.getParentFile();
537  
    if (parentFile != null)
538  
      parentFile.mkdirs();
539  
    String tempFileName = fileName + "_temp";
540  
    FileOutputStream fileOutputStream = new FileOutputStream(tempFileName);
541  
    fileOutputStream.write(contents);
542  
    fileOutputStream.close();
543  
    if (file.exists() && !file.delete())
544  
      throw new IOException("Can't delete " + fileName);
545  
546  
    if (!new File(tempFileName).renameTo(file))
547  
      throw new IOException("Can't rename " + tempFileName + " to " + fileName);
548  
  }
549  
550  
  public static String loadTextFile(String fileName, String defaultContents) throws IOException {
551  
    if (!new File(fileName).exists())
552  
      return defaultContents;
553  
554  
    FileInputStream fileInputStream = new FileInputStream(fileName);
555  
    InputStreamReader inputStreamReader = new InputStreamReader(fileInputStream, charsetForTextFiles);
556  
    return loadTextFile(inputStreamReader, (int) new File(fileName).length());
557  
  }
558  
559  
  public static String loadTextFile(Reader reader, int length) throws IOException {
560  
    try {
561  
      char[] chars = new char[length];
562  
      int n = reader.read(chars);
563  
      return new String(chars, 0, n);
564  
    } finally {
565  
      reader.close();
566  
    }
567  
  }
568  
569  
  static File DiskSnippetCache_dir;
570  
571  
  public static void initDiskSnippetCache(File dir) {
572  
    DiskSnippetCache_dir = dir;
573  
    dir.mkdirs();
574  
  }
575  
576  
  // Data files are immutable, use centralized cache
577  
  public static synchronized File DiskSnippetCache_getLibrary(long snippetID) throws IOException {
578  
    File file = new File(getGlobalCache(), "data_" + snippetID + ".jar");
579  
    if (verbose)
580  
      System.out.println("Checking data cache: " + file.getPath());
581  
    return file.exists() ? file : null;
582  
  }
583  
584  
  public static synchronized String DiskSnippetCache_get(long snippetID) throws IOException {
585  
    return loadTextFile(DiskSnippetCache_getFile(snippetID).getPath(), null);
586  
  }
587  
588  
  private static File DiskSnippetCache_getFile(long snippetID) {
589  
    return new File(DiskSnippetCache_dir, "" + snippetID);
590  
  }
591  
592  
  public static synchronized void DiskSnippetCache_put(long snippetID, String snippet) throws IOException {
593  
    saveTextFile(DiskSnippetCache_getFile(snippetID).getPath(), snippet);
594  
  }
595  
596  
  public static synchronized void DiskSnippetCache_putLibrary(long snippetID, byte[] data) throws IOException {
597  
    saveBinaryFile(new File(getGlobalCache(), "data_" + snippetID).getPath() + ".jar", data);
598  
  }
599  
600  
  public static File DiskSnippetCache_getDir() {
601  
    return DiskSnippetCache_dir;
602  
  }
603  
604  
  public static void initSnippetCache() {
605  
    if (DiskSnippetCache_dir == null)
606  
      initDiskSnippetCache(getGlobalCache());
607  
  }
608  
609  
  private static File getGlobalCache() {
610  
    File file = new File(System.getProperty("user.home"), ".tinybrain/snippet-cache");
611  
    file.mkdirs();
612  
    return file;
613  
  }
614  
615  
  public static String loadSnippetVerified(String snippetID, String hash) throws IOException {
616  
    String text = loadSnippet(snippetID);
617  
    String realHash = getHash(text.getBytes("UTF-8"));
618  
    if (!realHash.equals(hash)) {
619  
      String msg;
620  
      if (hash.isEmpty())
621  
        msg = "Here's your hash for " + snippetID + ", please put in your program: " + realHash;
622  
      else
623  
        msg = "Hash mismatch for " + snippetID + ": " + realHash + " (new) vs " + hash + " - has tinybrain.de been hacked??";
624  
      throw new RuntimeException(msg);
625  
    }
626  
    return text;
627  
  }
628  
629  
  public static String getHash(byte[] data) {
630  
    return bytesToHex(getFullFingerprint(data));
631  
  }
632  
633  
  public static byte[] getFullFingerprint(byte[] data) {
634  
    try {
635  
      return MessageDigest.getInstance("MD5").digest(data);
636  
    } catch (NoSuchAlgorithmException e) {
637  
      throw new RuntimeException(e);
638  
    }
639  
  }
640  
641  
  public static String bytesToHex(byte[] bytes) {
642  
    return bytesToHex(bytes, 0, bytes.length);
643  
  }
644  
645  
  public static String bytesToHex(byte[] bytes, int ofs, int len) {
646  
    StringBuilder stringBuilder = new StringBuilder(len*2);
647  
    for (int i = 0; i < len; i++) {
648  
      String s = "0" + Integer.toHexString(bytes[ofs+i]);
649  
      stringBuilder.append(s.substring(s.length()-2, s.length()));
650  
    }
651  
    return stringBuilder.toString();
652  
  }
653  
654  
  public static String loadSnippet(String snippetID) throws IOException {
655  
    return loadSnippet(parseSnippetID(snippetID));
656  
  }
657  
658  
  public static long parseSnippetID(String snippetID) {
659  
    return Long.parseLong(shortenSnippetID(snippetID));
660  
  }
661  
662  
  private static String shortenSnippetID(String snippetID) {
663  
    if (snippetID.startsWith("#"))
664  
      snippetID = snippetID.substring(1);
665  
    String httpBlaBla = "http://tinybrain.de/";
666  
    if (snippetID.startsWith(httpBlaBla))
667  
      snippetID = snippetID.substring(httpBlaBla.length());
668  
    return snippetID;
669  
  }
670  
671  
  public static boolean isSnippetID(String snippetID) {
672  
    snippetID = shortenSnippetID(snippetID);
673  
    return isInteger(snippetID) && Long.parseLong(snippetID) != 0;
674  
  }
675  
676  
  public static boolean isInteger(String s) {
677  
    return Pattern.matches("\\-?\\d+", s);
678  
  }
679  
680  
  public static String loadSnippet(long snippetID) throws IOException {
681  
    String text = memSnippetCache.get(snippetID);
682  
    if (text != null)
683  
      return text;
684  
685  
    initSnippetCache();
686  
    text = DiskSnippetCache_get(snippetID);
687  
    if (preferCached && text != null)
688  
      return text;
689  
690  
    String md5 = text != null ? md5(text) : "-";
691  
    if (text != null) {
692  
      String hash = prefetched.get(snippetID);
693  
      if (hash != null) {
694  
        if (md5.equals(hash)) {
695  
          memSnippetCache.put(snippetID, text);
696  
          return text;
697  
        } else
698  
          prefetched.remove(snippetID); // (maybe this is not necessary)
699  
      }
700  
    }
701  
702  
    try {
703  
      /*URL url = new URL("http://tinybrain.de:8080/getraw.php?id=" + snippetID);
704  
      text = loadPage(url);*/
705  
      String theURL = "http://tinybrain.de:8080/getraw.php?id=" + snippetID + "&getmd5=1&utf8=1";
706  
      if (text != null) {
707  
        //System.err.println("MD5: " + md5);
708  
        theURL += "&md5=" + md5;
709  
      }
710  
      URL url = new URL(theURL);
711  
      String page = loadPage(url);
712  
      if (page.startsWith("==*#*==")) {
713  
        // same, keep text
714  
        //System.err.println("Snippet unchanged, keeping.");
715  
      } else {
716  
        // drop md5 line
717  
        int i = page.indexOf('\n');
718  
        String hash = page.substring(0, i).trim();
719  
        text = page.substring(i+1);
720  
721  
        String myHash = md5(text);
722  
        if (myHash.equals(hash)) {
723  
          //System.err.println("Hash match: " + hash);
724  
        } else
725  
          System.err.println("Hash mismatch");
726  
      }
727  
    } catch (FileNotFoundException e) {
728  
      throw new IOException("Snippet #" + snippetID + " not found or not public");
729  
    }
730  
731  
    memSnippetCache.put(snippetID, text);
732  
733  
    try {
734  
      initSnippetCache();
735  
      DiskSnippetCache_put(snippetID, text);
736  
    } catch (IOException e) {
737  
      System.err.println("Minor warning: Couldn't save snippet to cache ("  + DiskSnippetCache_getDir() + ")");
738  
    }
739  
740  
    return text;
741  
  }
742  
743  
  private static String md5(String text) {
744  
    try {
745  
      return bytesToHex(md5impl(text.getBytes("UTF-8"))); // maybe different than the way PHP does it...
746  
    } catch (UnsupportedEncodingException e) {
747  
      throw new RuntimeException(e);
748  
    }
749  
  }
750  
751  
  public static byte[] md5impl(byte[] data) {
752  
    try {
753  
      return MessageDigest.getInstance("MD5").digest(data);
754  
    } catch (NoSuchAlgorithmException e) {
755  
      throw new RuntimeException(e);
756  
    }
757  
  }
758  
759  
  private static String loadPage(URL url) throws IOException {
760  
    System.err.println("Loading: " + url.toExternalForm());
761  
    URLConnection con = url.openConnection();
762  
    return loadPage(con, url);
763  
  }
764  
765  
  public static String loadPage(URLConnection con, URL url) throws IOException {
766  
    setHeaders(con);
767  
    String contentType = con.getContentType();
768  
    if (contentType == null)
769  
      throw new IOException("Page could not be read: " + url);
770  
    //Log.info("Content-Type: " + contentType);
771  
    String charset = guessCharset(contentType);
772  
    //System.err.println("Charset: " + charset);
773  
    Reader r = new InputStreamReader(con.getInputStream(), charset);
774  
    StringBuilder buf = new StringBuilder();
775  
    while (true) {
776  
      int ch = r.read();
777  
      if (ch < 0)
778  
        break;
779  
      //Log.info("Chars read: " + buf.length());
780  
      buf.append((char) ch);
781  
    }
782  
    return buf.toString();
783  
  }
784  
785  
  public static byte[] loadBinaryPage(URLConnection con) throws IOException {
786  
    setHeaders(con);
787  
    ByteArrayOutputStream buf = new ByteArrayOutputStream();
788  
    InputStream inputStream = con.getInputStream();
789  
    while (true) {
790  
      int ch = inputStream.read();
791  
      if (ch < 0)
792  
        break;
793  
      buf.write(ch);
794  
    }
795  
    inputStream.close();
796  
    return buf.toByteArray();
797  
  }
798  
799  
  private static void setHeaders(URLConnection con) throws IOException {
800  
    String computerID = getComputerID();
801  
    if (computerID != null)
802  
      con.setRequestProperty("X-ComputerID", computerID);
803  
  }
804  
805  
  public static String guessCharset(String contentType) {
806  
    Pattern p = Pattern.compile("text/html;\\s+charset=([^\\s]+)\\s*");
807  
    Matcher m = p.matcher(contentType);
808  
    /* If Content-Type doesn't match this pre-conception, choose default and hope for the best. */
809  
    return m.matches() ? m.group(1) : "ISO-8859-1";
810  
  }
811  
812  
  /** runs a transpiled set of sources */
813  
  public static void javax2(File srcDir, File ioBaseDir, boolean silent, boolean runInProcess,
814  
                            List<File> libraries, String[] args, String cacheAs) throws Exception {
815  
    File classesDir = TempDirMaker_make();
816  
    String javacOutput = compileJava(srcDir, libraries, classesDir);
817  
818  
    // run
819  
820  
    if (verbose) System.out.println("Running program (" + srcDir.getAbsolutePath()
821  
      + ") on io dir " + ioBaseDir.getAbsolutePath() + (runInProcess ? "[in-process]" : "") + "\n");
822  
    runProgram(javacOutput, classesDir, ioBaseDir, silent, runInProcess, libraries, args, cacheAs);
823  
  }
824  
825  
  static String compileJava(File srcDir, List<File> libraries, File classesDir) throws IOException {
826  
    ++compilations;
827  
828  
    // collect sources
829  
830  
    List<File> sources = new ArrayList<File>();
831  
    if (verbose) System.out.println("Scanning for sources in " + srcDir.getPath());
832  
    scanForSources(srcDir, sources, true);
833  
    if (sources.isEmpty())
834  
      throw new IOException("No sources found");
835  
836  
    // compile
837  
838  
    File optionsFile = File.createTempFile("javax", "");
839  
    if (verbose) System.out.println("Compiling " + sources.size() + " source(s) to " + classesDir.getPath());
840  
    String options = "-d " + bashQuote(classesDir.getPath());
841  
    writeOptions(sources, libraries, optionsFile, options);
842  
    classesDir.mkdirs();
843  
    return invokeJavac(optionsFile);
844  
  }
845  
846  
  private static void runProgram(String javacOutput, File classesDir, File ioBaseDir,
847  
                                 boolean silent, boolean runInProcess,
848  
                                 List<File> libraries, String[] args, String cacheAs) throws Exception {
849  
    // print javac output if compile failed and it hasn't been printed yet
850  
    boolean didNotCompile = !didCompile(classesDir);
851  
    if (verbose || didNotCompile)
852  
      System.out.println(javacOutput);
853  
    if (didNotCompile)
854  
      return;
855  
856  
    if (runInProcess
857  
      || (ioBaseDir.getAbsolutePath().equals(new File(".").getAbsolutePath()) && !silent)) {
858  
      runProgramQuick(classesDir, libraries, args, cacheAs);
859  
      return;
860  
    }
861  
862  
    boolean echoOK = false;
863  
    // TODO: add libraries to class path
864  
    String bashCmd = "(cd " + bashQuote(ioBaseDir.getAbsolutePath()) + " && (java -cp "
865  
      + bashQuote(classesDir.getAbsolutePath()) + " main" + (echoOK ? "; echo ok" : "") + "))";
866  
    if (verbose) System.out.println(bashCmd);
867  
    String output = backtick(bashCmd);
868  
    if (verbose || !silent)
869  
      System.out.println(output);
870  
  }
871  
872  
  static boolean didCompile(File classesDir) {
873  
    return hasFile(classesDir, "main.class");
874  
  }
875  
876  
  private static void runProgramQuick(File classesDir, List<File> libraries,
877  
                                      String[] args, String cacheAs) throws Exception {
878  
    // collect urls
879  
    URL[] urls = new URL[libraries.size()+1];
880  
    urls[0] = classesDir.toURI().toURL();
881  
    for (int i = 0; i < libraries.size(); i++)
882  
      urls[i+1] = libraries.get(i).toURI().toURL();
883  
884  
    // make class loader
885  
    URLClassLoader classLoader = new URLClassLoader(urls);
886  
887  
    // load JavaX main class
888  
    Class<?> mainClass = classLoader.loadClass("main");
889  
890  
    if (cacheAs != null)
891  
      programCache.put(cacheAs, mainClass);
892  
893  
    // run main method
894  
    Method main = mainClass.getMethod("main", String[].class);
895  
    main.invoke(null, (Object) args);
896  
  }
897  
898  
  private static String invokeJavac(File optionsFile) throws IOException {
899  
    String output;
900  
    try {
901  
      output = invokeEcj(optionsFile);
902  
    } catch (NoClassDefFoundError e) {
903  
      if (verbose) {
904  
        System.err.println("ecj not found - using javac");
905  
        e.printStackTrace();
906  
      }
907  
      output = backtick("javac " + bashQuote("@" + optionsFile.getPath()));
908  
    }
909  
    if (verbose) System.out.println(output);
910  
    return output;
911  
  }
912  
913  
  // throws ClassNotFoundError if ecj is not in classpath
914  
  static String invokeEcj(File optionsFile) {
915  
    StringWriter writer = new StringWriter();
916  
    PrintWriter printWriter = new PrintWriter(writer);
917  
    org.eclipse.jdt.core.compiler.CompilationProgress progress = null;
918  
919  
    // add more eclipse options in the line below
920  
921  
    org.eclipse.jdt.core.compiler.batch.BatchCompiler.compile(
922  
      new String[] { "@" + optionsFile.getPath(),
923  
        "-source", "1.7",
924  
        "-nowarn"
925  
      },
926  
      printWriter,
927  
      printWriter,
928  
      progress);
929  
    return writer.toString();
930  
  }
931  
932  
  private static void writeOptions(List<File> sources, List<File> libraries,
933  
                                   File optionsFile, String moreOptions) throws IOException {
934  
    FileWriter writer = new FileWriter(optionsFile);
935  
    for (File source : sources)
936  
      writer.write(bashQuote(source.getPath()) + " ");
937  
    if (!libraries.isEmpty()) {
938  
      List<String> cp = new ArrayList<String>();
939  
      for (File lib : libraries)
940  
        cp.add(lib.getAbsolutePath());
941  
      writer.write("-cp " + bashQuote(join(File.pathSeparator, cp)) + " ");
942  
    }
943  
    writer.write(moreOptions);
944  
    writer.close();
945  
  }
946  
947  
  static void scanForSources(File source, List<File> sources, boolean topLevel) {
948  
    if (source.isFile() && source.getName().endsWith(".java"))
949  
      sources.add(source);
950  
    else if (source.isDirectory() && !isSkippedDirectoryName(source.getName(), topLevel)) {
951  
      File[] files = source.listFiles();
952  
      for (File file : files)
953  
        scanForSources(file, sources, false);
954  
    }
955  
  }
956  
957  
  private static boolean isSkippedDirectoryName(String name, boolean topLevel) {
958  
    if (topLevel) return false; // input or output ok as highest directory (intentionally specified by user, not just found by a directory scan in which case we probably don't want it. it's more like heuristics actually.)
959  
    return name.equalsIgnoreCase("input") || name.equalsIgnoreCase("output");
960  
  }
961  
962  
  public static String backtick(String cmd) throws IOException {
963  
    ++processesStarted;
964  
    File outFile = File.createTempFile("_backtick", "");
965  
    File scriptFile = File.createTempFile("_backtick", isWindows() ? ".bat" : "");
966  
967  
    String command = cmd + ">" + bashQuote(outFile.getPath()) + " 2>&1";
968  
    //Log.info("[Backtick] " + command);
969  
    try {
970  
      saveTextFile(scriptFile.getPath(), command);
971  
      String[] command2;
972  
      if (isWindows())
973  
        command2 = new String[] { scriptFile.getPath() };
974  
      else
975  
        command2 = new String[] { "/bin/bash", scriptFile.getPath() };
976  
      Process process = Runtime.getRuntime().exec(command2);
977  
      try {
978  
        process.waitFor();
979  
      } catch (InterruptedException e) {
980  
        throw new RuntimeException(e);
981  
      }
982  
      process.exitValue();
983  
      return loadTextFile(outFile.getPath(), "");
984  
    } finally {
985  
      scriptFile.delete();
986  
    }
987  
  }
988  
989  
  /** possibly improvable */
990  
  public static String javaQuote(String text) {
991  
    return bashQuote(text);
992  
  }
993  
994  
  /** possibly improvable */
995  
  public static String bashQuote(String text) {
996  
    if (text == null) return null;
997  
    return "\"" + text
998  
      .replace("\\", "\\\\")
999  
      .replace("\"", "\\\"")
1000  
      .replace("\n", "\\n")
1001  
      .replace("\r", "\\r") + "\"";
1002  
  }
1003  
1004  
  public final static String charsetForTextFiles = "UTF8";
1005  
1006  
  static long TempDirMaker_lastValue;
1007  
1008  
  public static File TempDirMaker_make() {
1009  
    File dir = new File(System.getProperty("user.home"), ".javax/" + TempDirMaker_newValue());
1010  
    dir.mkdirs();
1011  
    return dir;
1012  
  }
1013  
1014  
  private static long TempDirMaker_newValue() {
1015  
    long value;
1016  
    do
1017  
      value = System.currentTimeMillis();
1018  
    while (value == TempDirMaker_lastValue);
1019  
    TempDirMaker_lastValue = value;
1020  
    return value;
1021  
  }
1022  
1023  
  public static String join(String glue, Iterable<String> strings) {
1024  
    StringBuilder buf = new StringBuilder();
1025  
    Iterator<String> i = strings.iterator();
1026  
    if (i.hasNext()) {
1027  
      buf.append(i.next());
1028  
      while (i.hasNext())
1029  
        buf.append(glue).append(i.next());
1030  
    }
1031  
    return buf.toString();
1032  
  }
1033  
1034  
  public static boolean isWindows() {
1035  
    return System.getProperty("os.name").contains("Windows");
1036  
  }
1037  
1038  
  public static String makeRandomID(int length) {
1039  
    Random random = new Random();
1040  
    char[] id = new char[length];
1041  
    for (int i = 0; i< id.length; i++)
1042  
      id[i] = (char) ((int) 'a' + random.nextInt(26));
1043  
    return new String(id);
1044  
  }
1045  
1046  
  static String computerID;
1047  
  public static String getComputerID() throws IOException {
1048  
    if (noID) return null;
1049  
    if (computerID == null) {
1050  
      File file = new File(System.getProperty("user.home"), ".tinybrain/computer-id");
1051  
      computerID = loadTextFile(file.getPath(), null);
1052  
      if (computerID == null) {
1053  
        computerID = makeRandomID(12);
1054  
        saveTextFile(file.getPath(), computerID);
1055  
      }
1056  
      if (verbose)
1057  
        System.out.println("Local computer ID: " + computerID);
1058  
    }
1059  
    return computerID;
1060  
  }
1061  
}

Author comment

Began life as a copy of #2000468

download  show line numbers   

Snippet is not live.

Travelled to 12 computer(s): aoiabmzegqzx, bhatertpkbcr, cbybwowwnfue, gwrvuhgaqvyk, ishqpsrjomds, lpdgvwnxivlt, mqqgnosmbjvj, pyentgdyhuwx, pzhvpgtvlbxg, tslmcundralx, tvejysmllsmz, vouqrxazstgt

No comments. add comment

Snippet ID: #2000469
Snippet name: class _x16 (embeddable)
Eternal ID of this version: #2000469/1
Text MD5: aa6200da9f384af30b737a87cf0d1a36
Author: stefan
Category: javax
Type: New Tinybrain snippet
Public (visible to everyone): Yes
Archived (hidden from active list): No
Created/modified: 2015-07-29 18:08:07
Source code size: 38508 bytes / 1061 lines
Pitched / IR pitched: No / Yes
Views / Downloads: 602 / 124
Referenced in: [show references]