import javax.imageio.*; import java.awt.image.*; import java.awt.event.*; import java.awt.*; import java.security.NoSuchAlgorithmException; import java.security.MessageDigest; import java.lang.reflect.*; import java.net.*; import java.io.*; import javax.swing.text.*; import javax.swing.event.*; import javax.swing.*; import java.util.concurrent.*; import java.util.regex.*; import java.util.List; import java.util.zip.*; import java.util.*; // todo: hotwire programs only once /** JavaX runner version 19 Changes to v18: -safeTranslate -"list", "translate" and "run" ok in program args -TODO: find a solution for -v etc. -"-javac" option to force using javac -compiler errors now detected robustly, both with javac and ecj. -and some more... */ class _javax { static final String version = "JavaX 19"; static boolean verbose = false, translate = false, list = false, virtualizeTranslators = true; static String translateTo = null; static boolean preferCached = false, noID = false, noPrefetch = false; static boolean safeOnly = false, safeTranslate = false, javacOnly = false; static boolean runMainInProcess = true; static List mainTranslators = new ArrayList(); private static Map memSnippetCache = new HashMap(); private static int processesStarted, compilations; // snippet ID -> md5 private static HashMap prefetched = new HashMap(); private static File virtCache; // doesn't work yet private static Map> programCache = new HashMap>(); static boolean cacheTranslators = false; // this should work (caches transpiled translators) private static HashMap translationCache = new HashMap(); static boolean cacheTranspiledTranslators = true; // which snippets are available pre-transpiled server-side? private static Set hasTranspiledSet = new HashSet(); static boolean useServerTranspiled = true; static Object androidContext; static boolean android = isAndroid(); // Translators currently being translated (to detect recursions) private static Set translating = new HashSet(); static String lastOutput; public static void main(String[] args) throws Exception { File ioBaseDir = new File("."), inputDir = null, outputDir = null; String src = null; List programArgs = new ArrayList(); String programID; for (int i = 0; i < args.length; i++) { String arg = args[i]; if (arg.equals("-version")) { showVersion(); return; } if (arg.equals("-sysprop")) { showSystemProperties(); return; } if (arg.equals("-v") || arg.equals("-verbose")) verbose = true; else if (arg.equals("-finderror")) verbose = true; else if (arg.equals("-offline") || arg.equalsIgnoreCase("-prefercached")) preferCached = true; else if (arg.equals("-novirt")) virtualizeTranslators = false; else if (arg.equals("-safeonly")) safeOnly = true; else if (arg.equals("-safetranslate")) safeTranslate = true; else if (arg.equals("-noid")) noID = true; else if (arg.equals("-nocachetranspiled")) cacheTranspiledTranslators = false; else if (arg.equals("-javac")) javacOnly = true; else if (arg.equals("-localtranspile")) useServerTranspiled = false; else if (arg.equals("translate") && src == null) translate = true; else if (arg.equals("list") && src == null) { list = true; virtualizeTranslators = false; // so they are silenced } else if (arg.equals("run") && src == null) { // it's the default command anyway } else if (arg.startsWith("input=")) inputDir = new File(arg.substring(6)); else if (arg.startsWith("output=")) outputDir = new File(arg.substring(7)); else if (arg.equals("with")) mainTranslators.add(new String[] {args[++i], null}); else if (translate && arg.equals("to")) translateTo = args[++i]; else if (src == null) { //System.out.println("src=" + arg); src = arg; } else programArgs.add(arg); } cleanCache(); if (useServerTranspiled) noPrefetch = true; if (src == null) src = "."; // Might actually want to write to 2 disk caches (global/per program). if (virtualizeTranslators && !preferCached) virtCache = TempDirMaker_make(); if (inputDir != null) { ioBaseDir = TempDirMaker_make(); System.out.println("Taking input from: " + inputDir.getAbsolutePath()); System.out.println("Output is in: " + new File(ioBaseDir, "output").getAbsolutePath()); copyInput(inputDir, new File(ioBaseDir, "input")); } javaxmain(src, ioBaseDir, translate, list, programArgs.toArray(new String[programArgs.size()])); if (outputDir != null) { copyInput(new File(ioBaseDir, "output"), outputDir); System.out.println("Output copied to: " + outputDir.getAbsolutePath()); } if (verbose) { // print stats System.out.println("Processes started: " + processesStarted + ", compilations: " + compilations); } } public static void javaxmain(String src, File ioDir, boolean translate, boolean list, String[] args) throws Exception { String programID = isSnippetID(src) ? "" + parseSnippetID(src) : null; List libraries = new ArrayList(); File X = transpileMain(src, libraries); if (X == null) return; // list or run if (translate) { File to = X; if (translateTo != null) if (new File(translateTo).isDirectory()) to = new File(translateTo, "main.java"); else to = new File(translateTo); if (to != X) copy(new File(X, "main.java"), to); System.out.println("Program translated to: " + to.getAbsolutePath()); } else if (list) System.out.println(loadTextFile(new File(X, "main.java").getPath(), null)); else javax2(X, ioDir, false, runMainInProcess, libraries, args, null, programID); } static File transpileMain(String src, List libraries) throws Exception { File srcDir; boolean isTranspiled = false; if (isSnippetID(src)) { prefetch(src); long id = parseSnippetID(src); prefetched.remove(id); // hackfix to ensure transpiled main program is found. srcDir = loadSnippetAsMainJava(src); if (verbose) System.err.println("hasTranspiledSet: " + hasTranspiledSet); if (hasTranspiledSet.contains(id) && useServerTranspiled) { //System.err.println("Trying pretranspiled main program: #" + id); String transpiledSrc = getServerTranspiled("#" + id); int i = transpiledSrc.indexOf('\n'); String libs = transpiledSrc.substring(0, Math.max(0, i)); transpiledSrc = transpiledSrc.substring(i+1); if (!transpiledSrc.isEmpty()) { srcDir = TempDirMaker_make(); saveTextFile(new File(srcDir, "main.java").getPath(), transpiledSrc); isTranspiled = true; //translationCache.put(id, new Object[] {srcDir, libraries}); Matcher m = Pattern.compile("\\d+").matcher(libs); while (m.find()) { String libid = m.group(); File libraryFile = DiskSnippetCache_getLibrary(parseSnippetID(libid)); loadLibrary(libid, libraries, libraryFile); } } } } else { srcDir = new File(src); // if the argument is a file, it is assumed to be main.java if (srcDir.isFile()) { srcDir = TempDirMaker_make(); copy(new File(src), new File(srcDir, "main.java")); } if (!new File(srcDir, "main.java").exists()) { showVersion(); System.out.println("No main.java found, exiting"); return null; } } // translate File X = srcDir; if (!isTranspiled) { X = topLevelTranslate(X, libraries); System.err.println("Translated " + src); // save prefetch data if (isSnippetID(src)) savePrefetchData(src); } return X; } private static void prefetch(String mainSnippetID) throws IOException { if (noPrefetch) return; long mainID = parseSnippetID(mainSnippetID); String s = mainID + " " + loadTextFile(new File(userHome(), ".tinybrain/prefetch/" + mainID + ".txt").getPath(), ""); String[] ids = s.trim().split(" "); if (ids.length > 1) { String url = "http://tinybrain.de:8080/tb-int/prefetch.php?ids=" + URLEncoder.encode(s, "UTF-8"); String data = loadPage(new URL(url)); String[] split = data.split(" "); if (split.length == ids.length) for (int i = 0; i < ids.length; i++) prefetched.put(parseSnippetID(ids[i]), split[i]); } } static String userHome() { if (android) return ((File) call(androidContext, "getFilesDir")).getAbsolutePath(); else return System.getProperty("user.home"); } private static void savePrefetchData(String mainSnippetID) throws IOException { List ids = new ArrayList(); long mainID = parseSnippetID(mainSnippetID); for (long id : memSnippetCache.keySet()) if (id != mainID) ids.add(String.valueOf(id)); saveTextFile(new File(userHome(),".tinybrain/prefetch/" + mainID + ".txt").getPath(), join(" ", ids)); } static File topLevelTranslate(File srcDir, List libraries_out) throws Exception { File X = srcDir; X = applyTranslators(X, mainTranslators, libraries_out); // translators supplied on command line (unusual) // actual inner translation of the JavaX source X = defaultTranslate(X, libraries_out); return X; } private static File defaultTranslate(File x, List libraries_out) throws Exception { x = luaPrintToJavaPrint(x); x = repeatAutoTranslate(x, libraries_out); return x; } private static File repeatAutoTranslate(File x, List libraries_out) throws Exception { while (true) { File y = autoTranslate(x, libraries_out); if (y == x) return x; x = y; } } private static File autoTranslate(File x, List libraries_out) throws Exception { String main = loadTextFile(new File(x, "main.java").getPath(), null); List lines = toLines(main); List translators = findTranslators(lines); if (translators.isEmpty()) return x; main = fromLines(lines); File newDir = TempDirMaker_make(); saveTextFile(new File(newDir, "main.java").getPath(), main); return applyTranslators(newDir, translators, libraries_out); } private static List findTranslators(List lines) { List translators = new ArrayList(); Pattern pattern = Pattern.compile("^!([0-9# \t]+)"); Pattern pArgs = Pattern.compile("^\\s*\\((.*)\\)"); for (ListIterator iterator = lines.listIterator(); iterator.hasNext(); ) { String line = iterator.next(); line = line.trim(); Matcher matcher = pattern.matcher(line); if (matcher.find()) { String[] t = matcher.group(1).split("[ \t]+"); String rest = line.substring(matcher.end()); String arg = null; if (t.length == 1) { Matcher mArgs = pArgs.matcher(rest); if (mArgs.find()) arg = mArgs.group(1); } for (String transi : t) translators.add(new String[]{transi, arg}); iterator.remove(); } } return translators; } public static List toLines(String s) { List lines = new ArrayList(); int start = 0; while (true) { int i = toLines_nextLineBreak(s, start); if (i < 0) { if (s.length() > start) lines.add(s.substring(start)); break; } lines.add(s.substring(start, i)); if (s.charAt(i) == '\r' && i+1 < s.length() && s.charAt(i+1) == '\n') i += 2; else ++i; start = i; } return lines; } private static int toLines_nextLineBreak(String s, int start) { for (int i = start; i < s.length(); i++) { char c = s.charAt(i); if (c == '\r' || c == '\n') return i; } return -1; } public static String fromLines(List lines) { StringBuilder buf = new StringBuilder(); for (String line : lines) { buf.append(line).append('\n'); } return buf.toString(); } private static File applyTranslators(File x, List translators, List libraries_out) throws Exception { for (String[] translator : translators) x = applyTranslator(x, translator[0], translator[1], libraries_out); return x; } // also takes a library private static File applyTranslator(File x, String translator, String arg, List libraries_out) throws Exception { if (verbose) System.out.println("Using translator " + translator + " on sources in " + x.getPath()); File newDir = runTranslatorOnInput(translator, null, arg, x, !verbose, libraries_out); if (!new File(newDir, "main.java").exists()) { throw new Exception("Translator " + translator + " did not generate main.java"); // TODO: show translator output } if (verbose) System.out.println("Translated with " + translator + " from " + x.getPath() + " to " + newDir.getPath()); x = newDir; return x; } private static File luaPrintToJavaPrint(File x) throws IOException { File newDir = TempDirMaker_make(); String code = loadTextFile(new File(x, "main.java").getPath(), null); code = luaPrintToJavaPrint(code); if (verbose) System.out.println(code); saveTextFile(new File(newDir, "main.java").getPath(), code); return newDir; } public static String luaPrintToJavaPrint(String code) { return ("\n" + code).replaceAll( "(\n\\s*)print (\".*\")", "$1System.out.println($2);").substring(1); } public static File loadSnippetAsMainJava(String snippetID) throws IOException { checkProgramSafety(snippetID); File srcDir = TempDirMaker_make(); saveTextFile(new File(srcDir, "main.java").getPath(), loadSnippet(snippetID)); return srcDir; } public static File loadSnippetAsMainJavaVerified(String snippetID, String hash) throws IOException { checkProgramSafety(snippetID); File srcDir = TempDirMaker_make(); saveTextFile(new File(srcDir, "main.java").getPath(), loadSnippetVerified(snippetID, hash)); return srcDir; } /** returns output dir */ private static File runTranslatorOnInput(String snippetID, String hash, String arg, File input, boolean silent, List libraries_out) throws Exception { if (safeTranslate) checkProgramSafetyImpl(snippetID); long id = parseSnippetID(snippetID); File libraryFile = DiskSnippetCache_getLibrary(id); if (libraryFile != null) { loadLibrary(snippetID, libraries_out, libraryFile); return input; } String[] args = arg != null ? new String[]{arg} : new String[0]; File srcDir = hash == null ? loadSnippetAsMainJava(snippetID) : loadSnippetAsMainJavaVerified(snippetID, hash); long mainJavaSize = new File(srcDir, "main.java").length(); if (mainJavaSize == 0) { // no text in snippet? assume it's a library loadLibrary(snippetID, libraries_out, libraryFile); return input; } List libraries = new ArrayList(); Object[] cached = translationCache.get(id); if (cached != null) { //System.err.println("Taking translator " + snippetID + " from cache!"); srcDir = (File) cached[0]; libraries = (List) cached[1]; } else if (hasTranspiledSet.contains(id) && useServerTranspiled) { System.err.println("Trying pretranspiled translator: #" + snippetID); String transpiledSrc = getServerTranspiled(snippetID); transpiledSrc = transpiledSrc.substring(transpiledSrc.indexOf('\n')+1); // TODO: check for libraries if (!transpiledSrc.isEmpty()) { srcDir = TempDirMaker_make(); saveTextFile(new File(srcDir, "main.java").getPath(), transpiledSrc); translationCache.put(id, cached = new Object[] {srcDir, libraries}); } } File ioBaseDir = TempDirMaker_make(); /*Class mainClass = programCache.get("" + parseSnippetID(snippetID)); if (mainClass != null) return runCached(ioBaseDir, input, args);*/ // Doesn't work yet because virtualized directories are hardcoded in translator... if (cached == null) { System.err.println("Translating translator #" + id); if (translating.contains(id)) throw new RuntimeException("Recursive translator reference chain including #" + id); translating.add(id); try { srcDir = defaultTranslate(srcDir, libraries); } finally { translating.remove(id); } System.err.println("Translated translator #" + id); translationCache.put(id, new Object[]{srcDir, libraries}); } boolean runInProcess = false; if (virtualizeTranslators) { if (verbose) System.out.println("Virtualizing translator"); // TODO: don't virtualize class _javax (as included in, say, #636) //srcDir = applyTranslator(srcDir, "#2000351"); // I/O-virtualize the translator // that doesn't work because it recurses infinitely... // So we do it right here: String s = loadTextFile(new File(srcDir, "main.java").getPath(), null); s = s.replaceAll("new\\s+File\\(", "virtual.newFile("); s = s.replaceAll("new\\s+FileInputStream\\(", "virtual.newFileInputStream("); s = s.replaceAll("new\\s+FileOutputStream\\(", "virtual.newFileOutputStream("); s += "\n\n" + loadSnippet("#2000355"); // load class virtual // change baseDir s = s.replace("virtual_baseDir = \"\";", "virtual_baseDir " + "= " + javaQuote(ioBaseDir.getAbsolutePath()) + ";"); // extra + is necessary for Dumb TinyBrain :) // forward snippet cache (virtualized one) File dir = virtCache != null ? virtCache : DiskSnippetCache_dir; s = s.replace("static File DiskSnippetCache_dir" + ";", "static File DiskSnippetCache_dir " + "= new File(" + javaQuote(dir.getAbsolutePath()) + ");"); // extra + is necessary for Dumb TinyBrain :) s = s.replace("static boolean preferCached = false;", "static boolean preferCached = true;"); if (verbose) { System.out.println("==BEGIN VIRTUALIZED TRANSLATOR=="); System.out.println(s); System.out.println("==END VIRTUALIZED TRANSLATOR=="); } srcDir = TempDirMaker_make(); saveTextFile(new File(srcDir, "main.java").getPath(), s); // TODO: silence translator also runInProcess = true; } return runJavaX(ioBaseDir, srcDir, input, silent, runInProcess, libraries, args, cacheTranslators ? "" + id : null, "" + id); } private static String getServerTranspiled(String snippetID) throws IOException { long id = parseSnippetID(snippetID); URL url = new URL("http://tinybrain.de:8080/tb-int/get-transpiled.php?raw=1&withlibs=1&id=" + id); return loadPage(url); } static void checkProgramSafety(String snippetID) throws IOException { if (!safeOnly) return; checkProgramSafetyImpl(snippetID); } static void checkProgramSafetyImpl(String snippetID) throws IOException { URL url = new URL("http://tinybrain.de:8080/tb-int/is-javax-safe.php?id=" + parseSnippetID(snippetID)); String text = loadPage(url); if (!text.startsWith("{\"safe\":\"1\"}")) throw new RuntimeException("Program not safe: #" + parseSnippetID(snippetID)); } static void loadLibrary(String snippetID, List libraries_out, File libraryFile) throws IOException { if (verbose) System.out.println("Assuming " + snippetID + " is a library."); if (libraryFile == null) { byte[] data = loadDataSnippetImpl(snippetID); DiskSnippetCache_putLibrary(parseSnippetID(snippetID), data); libraryFile = DiskSnippetCache_getLibrary(parseSnippetID(snippetID)); } if (!libraries_out.contains(libraryFile)) libraries_out.add(libraryFile); } private static byte[] loadDataSnippetImpl(String snippetID) throws IOException { byte[] data; try { URL url = new URL("http://eyeocr.sourceforge.net/filestore/filestore.php?cmd=serve&file=blob_" + parseSnippetID(snippetID) + "&contentType=application/binary"); System.err.println("Loading library: " + url); data = loadBinaryPage(url.openConnection()); if (verbose) System.err.println("Bytes loaded: " + data.length); } catch (FileNotFoundException e) { throw new IOException("Binary snippet #" + snippetID + " not found or not public"); } return data; } /** returns output dir */ private static File runJavaX(File ioBaseDir, File originalSrcDir, File originalInput, boolean silent, boolean runInProcess, List libraries, String[] args, String cacheAs, String programID) throws Exception { File srcDir = new File(ioBaseDir, "src"); File inputDir = new File(ioBaseDir, "input"); File outputDir = new File(ioBaseDir, "output"); copyInput(originalSrcDir, srcDir); copyInput(originalInput, inputDir); javax2(srcDir, ioBaseDir, silent, runInProcess, libraries, args, cacheAs, programID); return outputDir; } private static void copyInput(File src, File dst) throws IOException { copyDirectory(src, dst); } public static boolean hasFile(File inputDir, String name) { return new File(inputDir, name).exists(); } public static void copyDirectory(File src, File dst) throws IOException { if (verbose) System.out.println("Copying " + src.getAbsolutePath() + " to " + dst.getAbsolutePath()); dst.mkdirs(); File[] files = src.listFiles(); if (files == null) return; for (File file : files) { File dst1 = new File(dst, file.getName()); if (file.isDirectory()) copyDirectory(file, dst1); else { if (verbose) System.out.println("Copying " + file.getAbsolutePath() + " to " + dst1.getAbsolutePath()); copy(file, dst1); } } } /** Quickly copy a file without a progress bar or any other fancy GUI... :) */ public static void copy(File src, File dest) throws IOException { FileInputStream inputStream = newFileInputStream(src); FileOutputStream outputStream = newFileOutputStream(dest); try { copy(inputStream, outputStream); inputStream.close(); } finally { outputStream.close(); } } static Object call(Object o, String method, Object... args) { try { Method m = call_findMethod(o, method, args, false); m.setAccessible(true); return m.invoke(o, args); } catch (Exception e) { throw new RuntimeException(e); } } static Object call(Class c, String method, Object... args) { try { Method m = call_findStaticMethod(c, method, args, false); m.setAccessible(true); return m.invoke(null, args); } catch (Exception e) { throw new RuntimeException(e); } } static Method call_findStaticMethod(Class c, String method, Object[] args, boolean debug) { while (c != null) { for (Method m : c.getDeclaredMethods()) { if (debug) System.out.println("Checking method " + m.getName() + " with " + m.getParameterTypes().length + " parameters");; if (!m.getName().equals(method)) { if (debug) System.out.println("Method name mismatch: " + method); continue; } if ((m.getModifiers() & Modifier.STATIC) == 0 || !call_checkArgs(m, args, debug)) continue; return m; } c = c.getSuperclass(); } throw new RuntimeException("Method '" + method + "' (static) with " + args.length + " parameter(s) not found in " + c.getName()); } static Method call_findMethod(Object o, String method, Object[] args, boolean debug) { Class c = o.getClass(); while (c != null) { for (Method m : c.getDeclaredMethods()) { if (debug) System.out.println("Checking method " + m.getName() + " with " + m.getParameterTypes().length + " parameters");; if (m.getName().equals(method) && call_checkArgs(m, args, debug)) return m; } c = c.getSuperclass(); } throw new RuntimeException("Method '" + method + "' (non-static) with " + args.length + " parameter(s) not found in " + o.getClass().getName()); } private static boolean call_checkArgs(Method m, Object[] args, boolean debug) { Class[] types = m.getParameterTypes(); if (types.length != args.length) { if (debug) System.out.println("checkArgs: Bad parameter length: " + args.length + " vs " + types.length); return false; } for (int i = 0; i < types.length; i++) if (!(args[i] == null || types[i].isInstance(args[i]))) { if (debug) System.out.println("checkArgs: Bad parameter " + i + ": " + args[i] + " vs " + types[i]); return false; } return true; } private static FileInputStream newFileInputStream(File f) throws FileNotFoundException { /*if (androidContext != null) return (FileInputStream) call(androidContext, "openFileInput", f.getPath()); else*/ return new // line break for Dumb TinyBrain :) FileInputStream(f); } private static FileOutputStream newFileOutputStream(File f) throws FileNotFoundException { /*if (androidContext != null) return (FileOutputStream) call(androidContext, "openFileOutput", f.getPath(), 0); else*/ return new // line break for Dumb TinyBrain :) FileOutputStream(f); } public static void copy(InputStream in, OutputStream out) throws IOException { byte[] buf = new byte[65536]; while (true) { int n = in.read(buf); if (n <= 0) return; out.write(buf, 0, n); } } /** writes safely (to temp file, then rename) */ public static void saveTextFile(String fileName, String contents) throws IOException { File file = new File(fileName); File parentFile = file.getParentFile(); if (parentFile != null) parentFile.mkdirs(); String tempFileName = fileName + "_temp"; FileOutputStream fileOutputStream = newFileOutputStream(new File(tempFileName)); OutputStreamWriter outputStreamWriter = new OutputStreamWriter(fileOutputStream, charsetForTextFiles); PrintWriter printWriter = new PrintWriter(outputStreamWriter); printWriter.print(contents); printWriter.close(); if (file.exists() && !file.delete()) throw new IOException("Can't delete " + fileName); if (!new File(tempFileName).renameTo(file)) throw new IOException("Can't rename " + tempFileName + " to " + fileName); } /** writes safely (to temp file, then rename) */ public static void saveBinaryFile(String fileName, byte[] contents) throws IOException { File file = new File(fileName); File parentFile = file.getParentFile(); if (parentFile != null) parentFile.mkdirs(); String tempFileName = fileName + "_temp"; FileOutputStream fileOutputStream = newFileOutputStream(new File(tempFileName)); fileOutputStream.write(contents); fileOutputStream.close(); if (file.exists() && !file.delete()) throw new IOException("Can't delete " + fileName); if (!new File(tempFileName).renameTo(file)) throw new IOException("Can't rename " + tempFileName + " to " + fileName); } public static String loadTextFile(String fileName, String defaultContents) throws IOException { if (!new File(fileName).exists()) return defaultContents; FileInputStream fileInputStream = newFileInputStream(new File(fileName)); InputStreamReader inputStreamReader = new InputStreamReader(fileInputStream, charsetForTextFiles); return loadTextFile(inputStreamReader, (int) new File(fileName).length()); } public static String loadTextFile(Reader reader, int length) throws IOException { try { char[] chars = new char[length]; int n = reader.read(chars); return new String(chars, 0, n); } finally { reader.close(); } } static File DiskSnippetCache_dir; public static void initDiskSnippetCache(File dir) { DiskSnippetCache_dir = dir; dir.mkdirs(); } // Data files are immutable, use centralized cache public static synchronized File DiskSnippetCache_getLibrary(long snippetID) throws IOException { File file = new File(getGlobalCache(), "data_" + snippetID + ".jar"); if (verbose) System.out.println("Checking data cache: " + file.getPath()); return file.exists() ? file : null; } public static synchronized String DiskSnippetCache_get(long snippetID) throws IOException { return loadTextFile(DiskSnippetCache_getFile(snippetID).getPath(), null); } private static File DiskSnippetCache_getFile(long snippetID) { return new File(DiskSnippetCache_dir, "" + snippetID); } public static synchronized void DiskSnippetCache_put(long snippetID, String snippet) throws IOException { saveTextFile(DiskSnippetCache_getFile(snippetID).getPath(), snippet); } public static synchronized void DiskSnippetCache_putLibrary(long snippetID, byte[] data) throws IOException { saveBinaryFile(new File(getGlobalCache(), "data_" + snippetID).getPath() + ".jar", data); } public static File DiskSnippetCache_getDir() { return DiskSnippetCache_dir; } public static void initSnippetCache() { if (DiskSnippetCache_dir == null) initDiskSnippetCache(getGlobalCache()); } private static File getGlobalCache() { File file = new File(userHome(), ".tinybrain/snippet-cache"); file.mkdirs(); return file; } public static String loadSnippetVerified(String snippetID, String hash) throws IOException { String text = loadSnippet(snippetID); String realHash = getHash(text.getBytes("UTF-8")); if (!realHash.equals(hash)) { String msg; if (hash.isEmpty()) msg = "Here's your hash for " + snippetID + ", please put in your program: " + realHash; else msg = "Hash mismatch for " + snippetID + ": " + realHash + " (new) vs " + hash + " - has tinybrain.de been hacked??"; throw new RuntimeException(msg); } return text; } public static String getHash(byte[] data) { return bytesToHex(getFullFingerprint(data)); } public static byte[] getFullFingerprint(byte[] data) { try { return MessageDigest.getInstance("MD5").digest(data); } catch (NoSuchAlgorithmException e) { throw new RuntimeException(e); } } public static String bytesToHex(byte[] bytes) { return bytesToHex(bytes, 0, bytes.length); } public static String bytesToHex(byte[] bytes, int ofs, int len) { StringBuilder stringBuilder = new StringBuilder(len*2); for (int i = 0; i < len; i++) { String s = "0" + Integer.toHexString(bytes[ofs+i]); stringBuilder.append(s.substring(s.length()-2, s.length())); } return stringBuilder.toString(); } public static String loadSnippet(String snippetID) throws IOException { return loadSnippet(parseSnippetID(snippetID)); } public static long parseSnippetID(String snippetID) { return Long.parseLong(shortenSnippetID(snippetID)); } private static String shortenSnippetID(String snippetID) { if (snippetID.startsWith("#")) snippetID = snippetID.substring(1); String httpBlaBla = "http://tinybrain.de/"; if (snippetID.startsWith(httpBlaBla)) snippetID = snippetID.substring(httpBlaBla.length()); return snippetID; } public static boolean isSnippetID(String snippetID) { snippetID = shortenSnippetID(snippetID); return isInteger(snippetID) && Long.parseLong(snippetID) != 0; } public static boolean isInteger(String s) { return Pattern.matches("\\-?\\d+", s); } public static String loadSnippet(long snippetID) throws IOException { String text = memSnippetCache.get(snippetID); if (text != null) { if (verbose) System.out.println("Getting " + snippetID + " from mem cache"); return text; } initSnippetCache(); text = DiskSnippetCache_get(snippetID); if (preferCached && text != null) { if (verbose) System.out.println("Getting " + snippetID + " from disk cache (preferCached)"); return text; } String md5 = text != null ? md5(text) : "-"; if (text != null) { String hash = prefetched.get(snippetID); if (hash != null) { if (md5.equals(hash)) { memSnippetCache.put(snippetID, text); if (verbose) System.out.println("Getting " + snippetID + " from prefetched"); return text; } else prefetched.remove(snippetID); // (maybe this is not necessary) } } try { /*URL url = new URL("http://tinybrain.de:8080/getraw.php?id=" + snippetID); text = loadPage(url);*/ String theURL = "http://tinybrain.de:8080/getraw.php?id=" + snippetID + "&getmd5=1&utf8=1&usetranspiled=1"; if (text != null) { //System.err.println("MD5: " + md5); theURL += "&md5=" + md5; } URL url = new URL(theURL); String page = loadPage(url); // parse & drop transpilation flag available line int i = page.indexOf('\n'); boolean hasTranspiled = page.substring(0, i).trim().equals("1"); if (hasTranspiled) hasTranspiledSet.add(snippetID); else hasTranspiledSet.remove(snippetID); page = page.substring(i+1); if (page.startsWith("==*#*==")) { // same, keep text //System.err.println("Snippet unchanged, keeping."); } else { // drop md5 line i = page.indexOf('\n'); String hash = page.substring(0, i).trim(); text = page.substring(i+1); String myHash = md5(text); if (myHash.equals(hash)) { //System.err.println("Hash match: " + hash); } else System.err.println("Hash mismatch"); } } catch (FileNotFoundException e) { e.printStackTrace(); throw new IOException("Snippet #" + snippetID + " not found or not public"); } memSnippetCache.put(snippetID, text); try { initSnippetCache(); DiskSnippetCache_put(snippetID, text); } catch (IOException e) { System.err.println("Minor warning: Couldn't save snippet to cache (" + DiskSnippetCache_getDir() + ")"); } return text; } private static String md5(String text) { try { return bytesToHex(md5impl(text.getBytes("UTF-8"))); // maybe different than the way PHP does it... } catch (UnsupportedEncodingException e) { throw new RuntimeException(e); } } public static byte[] md5impl(byte[] data) { try { return MessageDigest.getInstance("MD5").digest(data); } catch (NoSuchAlgorithmException e) { throw new RuntimeException(e); } } private static String loadPage(URL url) throws IOException { System.err.println("Loading: " + url.toExternalForm()); URLConnection con = url.openConnection(); return loadPage(con, url); } public static String loadPage(URLConnection con, URL url) throws IOException { setHeaders(con); String contentType = con.getContentType(); if (contentType == null) throw new IOException("Page could not be read: " + url); //Log.info("Content-Type: " + contentType); String charset = guessCharset(contentType); //System.err.println("Charset: " + charset); Reader r = new InputStreamReader(con.getInputStream(), charset); StringBuilder buf = new StringBuilder(); while (true) { int ch = r.read(); if (ch < 0) break; //Log.info("Chars read: " + buf.length()); buf.append((char) ch); } return buf.toString(); } public static byte[] loadBinaryPage(URLConnection con) throws IOException { setHeaders(con); return loadBinaryPage_noHeaders(con); } private static byte[] loadBinaryPage_noHeaders(URLConnection con) throws IOException { ByteArrayOutputStream buf = new ByteArrayOutputStream(); InputStream inputStream = con.getInputStream(); while (true) { int ch = inputStream.read(); if (ch < 0) break; buf.write(ch); } inputStream.close(); return buf.toByteArray(); } private static void setHeaders(URLConnection con) throws IOException { String computerID = getComputerID(); if (computerID != null) con.setRequestProperty("X-ComputerID", computerID); } public static String guessCharset(String contentType) { Pattern p = Pattern.compile("text/html;\\s+charset=([^\\s]+)\\s*"); Matcher m = p.matcher(contentType); /* If Content-Type doesn't match this pre-conception, choose default and hope for the best. */ return m.matches() ? m.group(1) : "ISO-8859-1"; } /** runs a transpiled set of sources */ public static void javax2(File srcDir, File ioBaseDir, boolean silent, boolean runInProcess, List libraries, String[] args, String cacheAs, String programID) throws Exception { if (android) javax2android(srcDir, args, programID); else { File classesDir = TempDirMaker_make(); String javacOutput = compileJava(srcDir, libraries, classesDir); // run if (verbose) System.out.println("Running program (" + srcDir.getAbsolutePath() + ") on io dir " + ioBaseDir.getAbsolutePath() + (runInProcess ? "[in-process]" : "") + "\n"); runProgram(javacOutput, classesDir, ioBaseDir, silent, runInProcess, libraries, args, cacheAs, programID); } } static Class loadx2android(File srcDir, String programID) throws Exception { // TODO: optimize if it's a loaded snippet anyway URL url = new URL("http://tinybrain.de:8080/dexcompile.php"); URLConnection conn = url.openConnection(); String postData = "src=" + URLEncoder.encode(loadTextFile(new File(srcDir, "main.java").getPath(), null), "UTF-8"); byte[] dexData = doPostBinary(postData, conn); if (!isDex(dexData)) throw new RuntimeException("Dex generation error: " + dexData.length + " bytes - " + new String(dexData, "UTF-8")); System.out.println("Dex loaded: " + dexData.length + "b"); File dexDir = TempDirMaker_make(); File dexFile = new File(dexDir, System.currentTimeMillis() + ".dex"); File dexOutputDir = TempDirMaker_make(); System.out.println("Saving dex to: " + dexDir.getAbsolutePath()); try { saveBinaryFile(dexFile.getPath(), dexData); } catch (Throwable e) { System.out.println("Whoa!"); throw new RuntimeException(e); } System.out.println("Getting parent class loader."); ClassLoader parentClassLoader = //ClassLoader.getSystemClassLoader(); // does not find support jar //getClass().getClassLoader(); // Let's try this... _javax.class.getClassLoader().getParent(); // XXX ! //System.out.println("Making DexClassLoader."); //DexClassLoader classLoader = new DexClassLoader(dexFile.getAbsolutePath(), dexOutputDir.getAbsolutePath(), null, // parentClassLoader); Class dcl = Class.forName("dalvik.system.DexClassLoader"); Object classLoader = dcl.getConstructors()[0].newInstance(dexFile.getAbsolutePath(), dexOutputDir.getAbsolutePath(), null, parentClassLoader); //System.out.println("Loading main class."); //Class theClass = classLoader.loadClass(mainClassName); Class theClass = (Class) call(classLoader, "loadClass", "main"); //System.out.println("Main class loaded."); try { set(theClass, "androidContext", androidContext); } catch (Throwable e) {} try { set(theClass, "programID", programID); } catch (Throwable e) {} return theClass; } static void javax2android(File srcDir, String[] args, String programID) throws Exception { Class theClass = loadx2android(srcDir, programID); Method main = null; try { main = call_findStaticMethod(theClass, "main", new Object[]{androidContext}, false); } catch (RuntimeException e) { } //System.out.println("main method for " + androidContext + " of " + theClass + ": " + main); if (main != null) { // old style main program that returns a View System.out.println("Calling main (old-style)"); Object view = main.invoke(null, androidContext); System.out.println("Calling setContentView with " + view); call(Class.forName("main"), "setContentViewInUIThread", view); //call(androidContext, "setContentView", view); System.out.println("Done."); } else { System.out.println("New-style main method running.\n\n====\n"); runMainMethod(args, theClass); } } static byte[] DEX_FILE_MAGIC = { 0x64, 0x65, 0x78, 0x0a, 0x30, 0x33, 0x35, 0x00 }; static boolean isDex(byte[] dexData) { if (dexData.length < DEX_FILE_MAGIC.length) return false; for (int i = 0; i < DEX_FILE_MAGIC.length; i++) if (dexData[i] != DEX_FILE_MAGIC[i]) return false; return true; } static byte[] doPostBinary(String urlParameters, URLConnection conn) throws IOException { // connect and do POST setHeaders(conn); conn.setDoOutput(true); OutputStreamWriter writer = new OutputStreamWriter(conn.getOutputStream()); writer.write(urlParameters); writer.flush(); byte[] contents = loadBinaryPage_noHeaders(conn); writer.close(); return contents; } static String compileJava(File srcDir, List libraries, File classesDir) throws IOException { ++compilations; // collect sources List sources = new ArrayList(); if (verbose) System.out.println("Scanning for sources in " + srcDir.getPath()); scanForSources(srcDir, sources, true); if (sources.isEmpty()) throw new IOException("No sources found"); // compile File optionsFile = File.createTempFile("javax", ""); if (verbose) System.out.println("Compiling " + sources.size() + " source(s) to " + classesDir.getPath()); String options = "-d " + bashQuote(classesDir.getPath()); writeOptions(sources, libraries, optionsFile, options); classesDir.mkdirs(); return invokeJavaCompiler(optionsFile); } private static void runProgram(String javacOutput, File classesDir, File ioBaseDir, boolean silent, boolean runInProcess, List libraries, String[] args, String cacheAs, String programID) throws Exception { // print javac output if compile failed and it hasn't been printed yet boolean didNotCompile = !didCompile(classesDir); if (verbose || didNotCompile) System.out.println(javacOutput); if (didNotCompile) return; if (runInProcess || (ioBaseDir.getAbsolutePath().equals(new File(".").getAbsolutePath()) && !silent)) { runProgramQuick(classesDir, libraries, args, cacheAs, programID); return; } boolean echoOK = false; // TODO: add libraries to class path String bashCmd = "(cd " + bashQuote(ioBaseDir.getAbsolutePath()) + " && (java -cp " + bashQuote(classesDir.getAbsolutePath()) + " main" + (echoOK ? "; echo ok" : "") + "))"; if (verbose) System.out.println(bashCmd); String output = backtick(bashCmd); lastOutput = output; if (verbose || !silent) System.out.println(output); } static boolean didCompile(File classesDir) { return hasFile(classesDir, "main.class"); } private static void runProgramQuick(File classesDir, List libraries, String[] args, String cacheAs, String programID) throws Exception { // collect urls URL[] urls = new URL[libraries.size()+1]; urls[0] = classesDir.toURI().toURL(); for (int i = 0; i < libraries.size(); i++) urls[i+1] = libraries.get(i).toURI().toURL(); // make class loader URLClassLoader classLoader = new URLClassLoader(urls); // load JavaX main class Class mainClass = classLoader.loadClass("main"); if (cacheAs != null) programCache.put(cacheAs, mainClass); try { set(mainClass, "programID", programID); } catch (Throwable e) {} runMainMethod(args, mainClass); } static void runMainMethod(Object args, Class mainClass) throws NoSuchMethodException, IllegalAccessException, InvocationTargetException { Method main = mainClass.getMethod("main", String[].class); main.invoke(null, args); } private static String invokeJavaCompiler(File optionsFile) throws IOException { String output; if (hasEcj() && !javacOnly) output = invokeEcj(optionsFile); else output = invokeJavac(optionsFile); if (verbose) System.out.println(output); return output; } private static boolean hasEcj() { try { Class.forName("org.eclipse.jdt.internal.compiler.batch.Main"); return true; } catch (ClassNotFoundException e) { return false; } } private static String invokeJavac(File optionsFile) throws IOException { String output; output = backtick("javac " + bashQuote("@" + optionsFile.getPath())); if (exitValue != 0) { System.out.println(output); throw new RuntimeException("javac returned errors."); } return output; } // throws ClassNotFoundException if ecj is not in classpath static String invokeEcj(File optionsFile) { try { StringWriter writer = new StringWriter(); PrintWriter printWriter = new PrintWriter(writer); // add more eclipse options in the line below String[] args = {"@" + optionsFile.getPath(), "-source", "1.7", "-nowarn" }; Class ecjClass = Class.forName("org.eclipse.jdt.internal.compiler.batch.Main"); Object main = newInstance(ecjClass, printWriter, printWriter, false); call(main, "compile", new Object[]{args}); int errors = (Integer) get(main, "globalErrorsCount"); String output = writer.toString(); if (errors != 0) { System.out.println(output); throw new RuntimeException("Java compiler returned errors."); } return output; } catch (Exception e) { throw e instanceof RuntimeException ? (RuntimeException) e : new RuntimeException(e); } } static Object get(Object o, String field) { try { Field f = findField(o.getClass(), field); f.setAccessible(true); return f.get(o); } catch (Exception e) { throw new RuntimeException(e); } } static Object newInstance(Class c, Object... args) { try { Constructor m = findConstructor(c, args); m.setAccessible(true); return m.newInstance(args); } catch (Throwable __e) { throw __e instanceof RuntimeException ? (RuntimeException) __e : new RuntimeException(__e); }} static Constructor findConstructor(Class c, Object... args) { for (Constructor m : c.getDeclaredConstructors()) { if (!checkArgs(m.getParameterTypes(), args, verbose)) continue; return m; } throw new RuntimeException("Constructor with " + args.length + " matching parameter(s) not found in " + c.getName()); } static boolean checkArgs(Class[] types, Object[] args, boolean debug) { if (types.length != args.length) { if (debug) System.out.println("Bad parameter length: " + args.length + " vs " + types.length); return false; } for (int i = 0; i < types.length; i++) if (!(args[i] == null || isInstanceX(types[i], args[i]))) { if (debug) System.out.println("Bad parameter " + i + ": " + args[i] + " vs " + types[i]); return false; } return true; } // extended to handle primitive types private static boolean isInstanceX(Class type, Object arg) { if (type == boolean.class) return arg instanceof Boolean; if (type == int.class) return arg instanceof Integer; if (type == long.class) return arg instanceof Long; if (type == float.class) return arg instanceof Float; if (type == short.class) return arg instanceof Short; if (type == char.class) return arg instanceof Character; if (type == byte.class) return arg instanceof Byte; return type.isInstance(arg); } private static void writeOptions(List sources, List libraries, File optionsFile, String moreOptions) throws IOException { FileWriter writer = new FileWriter(optionsFile); for (File source : sources) writer.write(bashQuote(source.getPath()) + " "); if (!libraries.isEmpty()) { List cp = new ArrayList(); for (File lib : libraries) cp.add(lib.getAbsolutePath()); writer.write("-cp " + bashQuote(join(File.pathSeparator, cp)) + " "); } writer.write(moreOptions); writer.close(); } static void scanForSources(File source, List sources, boolean topLevel) { if (source.isFile() && source.getName().endsWith(".java")) sources.add(source); else if (source.isDirectory() && !isSkippedDirectoryName(source.getName(), topLevel)) { File[] files = source.listFiles(); for (File file : files) scanForSources(file, sources, false); } } private static boolean isSkippedDirectoryName(String name, boolean topLevel) { if (topLevel) return false; // input or output ok as highest directory (intentionally specified by user, not just found by a directory scan in which case we probably don't want it. it's more like heuristics actually.) return name.equalsIgnoreCase("input") || name.equalsIgnoreCase("output"); } static int exitValue; public static String backtick(String cmd) throws IOException { ++processesStarted; File outFile = File.createTempFile("_backtick", ""); File scriptFile = File.createTempFile("_backtick", isWindows() ? ".bat" : ""); String command = cmd + ">" + bashQuote(outFile.getPath()) + " 2>&1"; //Log.info("[Backtick] " + command); try { saveTextFile(scriptFile.getPath(), command); String[] command2; if (isWindows()) command2 = new String[] { scriptFile.getPath() }; else command2 = new String[] { "/bin/bash", scriptFile.getPath() }; Process process = Runtime.getRuntime().exec(command2); try { process.waitFor(); } catch (InterruptedException e) { throw new RuntimeException(e); } exitValue = process.exitValue(); if (verbose) System.out.println("Process return code: " + exitValue); return loadTextFile(outFile.getPath(), ""); } finally { scriptFile.delete(); } } /** possibly improvable */ public static String javaQuote(String text) { return bashQuote(text); } /** possibly improvable */ public static String bashQuote(String text) { if (text == null) return null; return "\"" + text .replace("\\", "\\\\") .replace("\"", "\\\"") .replace("\n", "\\n") .replace("\r", "\\r") + "\""; } public final static String charsetForTextFiles = "UTF8"; static long TempDirMaker_lastValue; public static File TempDirMaker_make() { File dir = new File(userHome(), ".javax/" + TempDirMaker_newValue()); dir.mkdirs(); return dir; } private static long TempDirMaker_newValue() { long value; do value = System.currentTimeMillis(); while (value == TempDirMaker_lastValue); TempDirMaker_lastValue = value; return value; } public static String join(String glue, Iterable strings) { StringBuilder buf = new StringBuilder(); Iterator i = strings.iterator(); if (i.hasNext()) { buf.append(i.next()); while (i.hasNext()) buf.append(glue).append(i.next()); } return buf.toString(); } public static boolean isWindows() { return System.getProperty("os.name").contains("Windows"); } public static String makeRandomID(int length) { Random random = new Random(); char[] id = new char[length]; for (int i = 0; i< id.length; i++) id[i] = (char) ((int) 'a' + random.nextInt(26)); return new String(id); } static String computerID; public static String getComputerID() throws IOException { if (noID) return null; if (computerID == null) { File file = new File(userHome(), ".tinybrain/computer-id"); computerID = loadTextFile(file.getPath(), null); if (computerID == null) { computerID = makeRandomID(12); saveTextFile(file.getPath(), computerID); } if (verbose) System.out.println("Local computer ID: " + computerID); } return computerID; } static int fileDeletions; static void cleanCache() { if (verbose) System.out.println("Cleaning cache"); fileDeletions = 0; File javax = new File(userHome(), ".javax"); long now = System.currentTimeMillis(); File[] files = javax.listFiles(); if (files != null) for (File dir : files) { if (dir.isDirectory() && Pattern.compile("\\d+").matcher(dir.getName()).matches()) { long time = Long.parseLong(dir.getName()); long seconds = (now - time) / 1000; long minutes = seconds / 60; long hours = minutes / 60; if (hours >= 1) { //System.out.println("Can delete " + dir.getAbsolutePath() + ", age: " + hours + " h"); removeDir(dir); } } } if (verbose && fileDeletions != 0) System.out.println("Cleaned cache. File deletions: " + fileDeletions); } static void removeDir(File dir) { if (dir.getAbsolutePath().indexOf(".javax") < 0) // security check! return; for (File f : dir.listFiles()) { if (f.isDirectory()) removeDir(f); else { if (verbose) System.out.println("Deleting " + f.getAbsolutePath()); f.delete(); ++fileDeletions; } } dir.delete(); } static void showSystemProperties() { System.out.println("System properties:\n"); for (Map.Entry entry : System.getProperties().entrySet()) { System.out.println(" " + entry.getKey() + " = " + entry.getValue()); } System.out.println(); } static void showVersion() { //showSystemProperties(); boolean eclipseFound = hasEcj(); //String platform = System.getProperty("java.vendor") + " " + System.getProperty("java.runtime.name") + " " + System.getProperty("java.version"); String platform = System.getProperty("java.vm.name") + " " + System.getProperty("java.version"); String os = System.getProperty("os.name"), arch = System.getProperty("os.arch"); System.out.println("This is " + version + "."); System.out.println("[Details: " + (eclipseFound ? "Eclipse compiler (good)" : "javac (not so good)") + ", " + platform + ", " + arch + ", " + os + "]"); } static boolean isAndroid() { return System.getProperty("java.vendor").toLowerCase().indexOf("android") >= 0; } static void set(Class c, String field, Object value) { try { Field f = findStaticField(c, field); f.setAccessible(true); f.set(null, value); } catch (Exception e) { throw new RuntimeException(e); } } static Field findStaticField(Class c, String field) { for (Field f : c.getDeclaredFields()) if (f.getName().equals(field) && (f.getModifiers() & Modifier.STATIC) != 0) return f; throw new RuntimeException("Static field '" + field + "' not found in " + c.getName()); } static Field findField(Class c, String field) { for (Field f : c.getDeclaredFields()) if (f.getName().equals(field)) return f; throw new RuntimeException("Field '" + field + "' not found in " + c.getName()); } } //1000300 // Lexicon //1000515 // Lexicon, fixing class JavaTok { static String join(List cnc) { StringBuilder buf = new StringBuilder(); for (String s : cnc) buf.append(s); return buf.toString(); } static List split(String src) { Java20 lex = new Java20(); src = src.replace("\r\n", "\n"); LineNumberReader source = new LineNumberReader(new StringReader(src)); int lineNr = source.getLineNumber()+1; List list = new ArrayList(); try { for (Object a; (a = lex.grab(source)) != lex.$;) { String word = lex.word(); String q = quote(word); //System.out.println("grabbed at line " + lineNr + ": " + a + " " + q); lineNr = source.getLineNumber()+1; T t = new T(a, word); boolean isSpace = t.isSpace(); if (isSpace && list.size() > 0 && list.get(list.size()-1).isSpace()) list.get(list.size()-1).word += word; // merge spaces else list.add(t); } } catch (Lexicon.Exception e) { throw new RuntimeException(e); } List cnc = new ArrayList(); for (int i = 0; i < list.size(); ) { T t = list.get(i); boolean shouldBeSpace = (cnc.size() % 2) == 0; boolean isSpace = t.isSpace(); if (shouldBeSpace == isSpace) { cnc.add(t.word); ++i; } else if (shouldBeSpace) cnc.add(""); else { System.out.println(cncToLines(cnc)); throw new RuntimeException("TILT at " + cnc.size() + ": " + quote(t.word)); } } if ((cnc.size() % 2) == 0) cnc.add(""); return cnc; } static class T { Object a; String word; T(Object a, String word) { this.a = a; this.word = word; } boolean isSpace() { return a.equals("WHITE_SPACE") || a.equals("COMMENT"); } } static String cncToLines(List cnc) { StringBuilder out = new StringBuilder(); for (String token : cnc) out.append(quote(token) + "\n"); return out.toString(); } public static String quote(String s) { if (s == null) return "null"; return "\"" + s.replace("\\", "\\\\").replace("\"", "\\\"").replace("\r", "\\r").replace("\n", "\\n") + "\""; } static class Java20 extends Lexicon { Java20() { /** * Grammar for Java 2.0. * * Nonterminal - first letter uppercase * TERMINAL - all letters uppercase * keyword - all letters lowercase */ int INFINITY = -1; /** * 19.3 Terminals from section 3.6: White Space: [[:space:]] */ put("WHITE_SPACE", new Repetition(space(), 1, INFINITY)); /** * 19.3 Terminals from section 3.7: Comment */ put("COMMENT", new Union( // // Traditional Comment: /\*[^*]+(\*([^*/][^*]*)?)*\*/ // new Concatenation( new Singleton("/*"), new Concatenation( new Repetition(new NonMatch("*"), 1, INFINITY), new Concatenation( new Repetition( new Concatenation( new Singleton("*"), new Repetition(new Concatenation( new NonMatch("*/"), new Repetition(new NonMatch("*"), 0, INFINITY) ), 0, 1) ), 0, INFINITY ), new Singleton("*/") ))), new Union( /** * End Of Line Comment: //[^\n]*\n */ new Concatenation( new Singleton("//"), new Concatenation( new Repetition(new NonMatch("\n"), 0, INFINITY), new Singleton("\n") )), // // Documentation Comment: /\*\*(([^*/][^*]*)?\*)*/ // new Concatenation( new Singleton("/**"), new Concatenation( new Repetition( new Concatenation( new Repetition(new Concatenation( new NonMatch("*/"), new Repetition(new NonMatch("*"), 0, INFINITY) ), 0, 1), new Singleton("*") ), 0, INFINITY ), new Singleton("/") )) ))); put("IDENTIFIER", new Concatenation( new Union( alpha(), new Match("_$") ), new Repetition( new Union( alnum(), new Match("_$") ), 0, INFINITY ) )); /** * 19.3 Terminals from section 3.9: Keyword (recognized but not in the Java grammar) */ put("KEYWORD", new Union( new Singleton("const"), new Singleton("goto") )); /** * 19.3 Terminals from section 3.10.1: Integer Literal */ put("INTEGER_LITERAL", new Concatenation( new Union( /** * Decimal Integer Literal: 0|[1-9][[:digit:]]* */ new Singleton("0"), new Union( new Concatenation( new Range('1', '9'), new Repetition(digit(), 0, INFINITY) ), new Union( /** * Hexadecimal Integer Literal: 0[xX][[:xdigit:]]+ */ new Concatenation( new Singleton("0"), new Concatenation( new Match("xX"), new Repetition(xdigit(), 1, INFINITY) )), /** * Octal Integer Literal: 0[0-7]+ */ new Concatenation( new Singleton("0"), new Repetition(new Range('0', '7'), 1, INFINITY) ) ))), new Repetition(new Match("lL"), 0, 1) )); /** * 19.3 Terminals from section 3.10.2: Floating-Point Literal */ put("FLOATING_POINT_LITERAL", new Union( /** * [[:digit:]]+\.[[:digit:]]*([eE][-+]?[[:digit:]]+)?[fFdD]? */ new Concatenation( new Repetition(digit(), 1, INFINITY), new Concatenation( new Singleton("."), new Concatenation( new Repetition(digit(), 0, INFINITY), new Concatenation( new Repetition(new Concatenation( new Match("eE"), new Concatenation( new Repetition(new Match("-+"), 0, 1), new Repetition(digit(), 1, INFINITY) )), 0, 1), new Repetition(new Match("fFdD"), 0, 1) )))), new Union( /** * \.[[:digit:]]+([eE][-+]?[[:digit:]]+)?[fFdD]? */ new Concatenation( new Singleton("."), new Concatenation( new Repetition(digit(), 1, INFINITY), new Concatenation( new Repetition(new Concatenation( new Match("eE"), new Concatenation( new Repetition(new Match("-+"), 0, 1), new Repetition(digit(), 1, INFINITY) )), 0, 1), new Repetition(new Match("fFdD"), 0, 1) ))), new Union( /** * [[:digit:]]+[eE][-+]?[[:digit:]]+[fFdD]? */ new Concatenation( new Repetition(digit(), 1, INFINITY), new Concatenation( new Match("eE"), new Concatenation( new Repetition(new Match("-+"), 0, 1), new Concatenation( new Repetition(digit(), 1, INFINITY), new Repetition(new Match("fFdD"), 0, 1) )))), /** * [[:digit:]]+([eE][-+]?[[:digit:]]+)?[fFdD] */ new Concatenation( new Repetition(digit(), 1, INFINITY), new Concatenation( new Repetition(new Concatenation( new Match("eE"), new Concatenation( new Repetition(new Match("-+"), 0, 1), new Repetition(digit(), 1, INFINITY) )), 0, 1), new Match("fFdD") )) )))); /** * 19.3 Terminals from section 3.10.3: Boolean Literal */ put("BOOLEAN_LITERAL", new Union( new Singleton("true"), new Singleton("false") )); /** * 19.3 Terminals from section 3.10.4: Character Literal */ put("CHARACTER_LITERAL", new Concatenation( new Singleton("'"), new Concatenation( new Union( /** * Single Character: [^\r\n'\\] */ new NonMatch("\r\n'\\"), /** * Escape Sequence: \\([btnfr\"'\\]|[0-3]?[0-7]{1,2}) */ new Concatenation( new Singleton("\\"), new Union( new Match("btnfr\"'\\"), new Concatenation( new Repetition(new Range('0', '3'), 0, 1), new Repetition(new Range('0', '7'), 1, 2) ) ) ) ), new Singleton("'") ))); put("MULTILINE_LITERAL", new Concatenation( new Singleton("[["), new Concatenation( new Repetition( new Union( new NonMatch("]"), new Concatenation( new Singleton("]"), new NonMatch("]")) ), 0, INFINITY ), new Singleton("]]") ))); put("MULTILINE_LITERAL2", new Concatenation( new Singleton("[=["), new Concatenation( new Repetition( new Union( new NonMatch("]"), new Concatenation(new Singleton("]"), new Union( new NonMatch("="), new Concatenation(new Singleton("="), new NonMatch("]")))) ), 0, INFINITY ), new Singleton("]=]") ))); /** * 19.3 Terminals from section 3.10.5: String Literal */ put("STRING_LITERAL", new Concatenation( new Singleton("\""), new Concatenation( new Repetition( new Union( /** * Single Character: [^\r\n"\\] */ new NonMatch("\r\n\"\\"), /** * Escape Sequence: \\([btnfr\"'\\]|[0-3]?[0-7]{1,2}) */ new Concatenation( new Singleton("\\"), new Union( new Match("btnfr\"'\\"), new Union( new Concatenation( new Repetition(new Range('0', '3'), 0, 1), new Repetition(new Range('0', '7'), 1, 2) ), new Concatenation( new Singleton("u"), new Repetition(new Match("0123456789abcdefABCDEF"), 4, 4) ) ) ) ) ), 0, INFINITY ), new Singleton("\"") ))); /** * 19.3 Terminals section 3.10.7: Null Literal */ put("NULL_LITERAL", new Singleton("null")); // OK, it seems we have to add some more stuff... //put("OTHER1", new Match(";{}=,<>[]().+-:|&!")); //put("OTHER1", new NonMatch("")); // catch anything, one character at a time put("OTHER1", new NonMatch(" \t\r\n")); // catch any non-whitespace, one character at a time } } // class Java20 } /** *

This class implements a {@link Lexicon}.

* * @version 1.3 * @author © 1999-2009 Craig A. Rich <carich@csupomona.edu> */ class Lexicon { //Q /** *

The number of lexical NFA states constructed.

*/ private /*static*/ int QSize = 0; /** *

Creates a new state in the lexical NFA.

* * @return a new state in the lexical NFA. */ private /*static*/ Integer s() { return ++QSize; } //delta /** *

The transition relation of the lexical NFA.

*/ private /*static*/ final Stack> delta = new Stack>(); /** *

Puts a transition into the lexical NFA.

* * @param s the state from which the transition is made. * @param A the Alphabet on which the transition is made. * @param r the state to which the transition is made. */ private /*static*/ void put(Integer s, Alphabet A, Integer r) { if (Math.max(s,r) >= delta.size()) delta.setSize(Math.max(s,r)+1); Stack pairs = delta.get(s); if (pairs == null) delta.set(s, pairs = new Stack()); pairs.push(A); pairs.push(r); } //Set /** *

This class implements a {@link Lexicon.Set Set}.

* * @version 1.3 * @author © 1999-2009 Craig A. Rich <carich@csupomona.edu> * @param the element type. */ static class Set extends Stack { /** *

The null exclusion indicator. If true, add methods will not add null to this Set.

*/ private final boolean excludeNull; /** *

Constructs a Set with an initial capacity.

* * @param capacity the initial capacity. The magnitude of capacity is the initial capacity. The null exclusion indicator is initialized to true if capacity is negative. */ Set(int capacity) { super(); ensureCapacity(Math.abs(capacity)); excludeNull = (capacity < 0); } /** *

Adds an element to this Set. The element is not added if it occurs in this Set or it is null and the null exclusion indicator is true. The capacity is expanded if necessary.

* * @param element the element to add to this Set. * @return true if this Set is changed; false otherwise. */ public boolean add(E element) { if (excludeNull && element == null || contains(element)) return false; push(element); return true; } /** *

Adds a Set of elements to this Set. An element is not added if it occurs in this Set or it is null and the null exclusion indicator is true. The capacity is expanded if necessary.

* * @param index the index in S beyond which elements are added. * @param S the Set to add to this Set. * @return true if this Set is changed; false otherwise. */ boolean add(int index, Set String) { if (String == null) return false; boolean push = isEmpty(); boolean add = false; for (int i = index; i < String.size(); i++) { E element = String.get(i); if (!(excludeNull && element == null)) if (push) { push(element); add = true; } else if (add(element)) add = true; } return add; } /** *

Adds a Set of elements to this Set. An element is not added if it occurs in this Set or it is null and the null exclusion indicator is true. The capacity is expanded if necessary.

* * @param S the Set to add to this Set. * @return true if this Set is changed; false otherwise. */ boolean add(Set String) { return add(0, String); } public String toString() { StringBuffer result = new StringBuffer(80); result.append('{'); for (int i = 0; i < size(); i++) { if (i > 0) result.append(' '); result.append(get(i)); } result.append('}'); return result.toString(); } //Set } //I /** *

The initial states of the lexical NFA. When empty, there is a need to compute the current initial states. It is computed only on demand created by {@link #initial()}.

*/ private final Set I; //F /** *

The final states of the lexical NFA. A final state is mapped to the terminal it accepts in this Lexicon. When empty, there is a need to compute current final states. It is computed only on demand created by {@link #initial()}.

*/ private final Map Function; //Lexicon.transition /** *

Computes a transition using the lexical NFA.

* * @param S the states from which the transition is made. * @param a the character on which the transition is made. * @param R the states to which the transition is made. * @return the states to which the transition is made. */ private /*static*/ Set transition(Set String, char a, Set R) { R.clear(); for (Integer s : String) { Stack pairs = delta.get(s); if (pairs != null) for (int k = 0; k < pairs.size(); k += 2) { Alphabet A = (Alphabet)pairs.get(k); if (A != null) { Integer r = (Integer)pairs.get(k+1); if (A.contains(a)) R.add(r); } } } return R; } //Lexicon.closure /** *

Computes a reflexive transitive closure under empty transition using the lexical NFA. The closure is computed in place by a breadth-first search expanding S.

* * @param S the states whose reflexive transitive closure is computed under empty transition. * @return the reflexive transitive closure of S under empty transition. */ private /*static*/ Set closure(Set String) { for (int i = 0; i < String.size(); i++) { Integer s = String.get(i); Stack pairs = delta.get(s); if (pairs != null) for (int k = 0; k < pairs.size(); k += 2) { Alphabet A = (Alphabet)pairs.get(k); if (A == null) { Integer r = (Integer)pairs.get(k+1); String.add(r); } } } return String; } //Expression /** *

This class implements an {@link Lexicon.Expression Expression} expressing a regular language.

* * @version 1.3 * @author © 1999-2009 Craig A. Rich <carich@csupomona.edu> */ abstract public /*static*/ class Expression implements Cloneable { /** *

The initial state of the NFA constructed from this Expression.

*/ Integer i; /** *

The final state of the NFA constructed from this Expression.

*/ Integer f; /** *

Creates a clone of this Expression, and replicates the NFA constructed from this Expression.

* * @return a clone of this Expression. */ abstract public Object clone(); } //Alphabet /** *

This class implements an {@link Lexicon.Alphabet Alphabet} of character symbols.

* * @version 1.3 * @author © 1999-2009 Craig A. Rich <carich@csupomona.edu> */ abstract public /*static*/ class Alphabet extends Expression { /** *

Indicates whether a character occurs in this Alphabet.

* * @param a the character whose status is requested. * @return true if a occurs in this Alphabet; false otherwise. */ abstract boolean contains(char a); } //Match /** *

This class implements an {@link Lexicon.Alphabet Alphabet} containing some characters.

* * @version 1.3 * @author © 1999-2009 Craig A. Rich <carich@csupomona.edu> */ public /*static*/ class Match extends Alphabet { /** *

The {@link Character} or {@link String} representing this Alphabet.

*/ final Object A; /** *

Constructs an Alphabet containing some characters, and builds the NFA constructed from this Expression.

* * @param i the initial state of the NFA constructed. * @param A the {@link Character} or {@link String} of characters in this Alphabet. * @param f the final state of the NFA constructed. */ private Match(Integer i, Object A, Integer f) { this.A = A; put(this.i = i, this, this.f = f); } /** *

Constructs an Alphabet containing one character, and builds the NFA constructed from this Expression.

* * @param i the initial state of the NFA constructed. * @param a the character in this Alphabet. * @param f the final state of the NFA constructed. */ private Match(Integer i, char a, Integer f) { this(i, new Character(a), f); } /** *

Constructs an Alphabet containing one character, and builds the NFA constructed from this Expression.

* * @param a the character in this Alphabet. */ public Match(char a) { this(s(), a, s()); } /** *

Constructs an Alphabet containing some characters, and builds the NFA constructed from this Expression.

* * @param A the {@link Character} or {@link String} of characters in this Alphabet. */ public Match(Object A) { this(s(), A, s()); } /** *

Indicates whether a character occurs in this Alphabet.

* * @param a the character whose status is requested. * @return true if a occurs in this Alphabet; false otherwise. */ boolean contains(char a) { if (A instanceof Character) return (Character)A == a; if (A instanceof String) return ((String)A).indexOf(a) != -1; if (A instanceof Stack) for (Alphabet alphabet : (Stack)A) if (alphabet.contains(a)) return true; return false; } /** *

Creates a clone of this Alphabet, and replicates the NFA constructed from this Expression.

* * @return a clone of this Alphabet. */ public Object clone() { return new Match(A); } } //NonMatch /** *

This class implements an {@link Lexicon.Alphabet Alphabet} containing all except some characters.

* * @version 1.3 * @author © 1999-2009 Craig A. Rich <carich@csupomona.edu> */ public /*static*/ class NonMatch extends Match { /** *

Constructs an Alphabet containing all characters except one, and builds the NFA constructed from this Expression.

* * @param a the character not in this Alphabet. */ public NonMatch(char a) { super(a); } /** *

Constructs an Alphabet containing all characters except some, and builds the NFA constructed from this Expression.

* * @param A the {@link Character} or {@link String} of characters not in this Alphabet. */ public NonMatch(Object A) { super(A); } /** *

Indicates whether a character occurs in this Alphabet.

* * @param a the character whose status is requested. * @return true if a occurs in this Alphabet; false otherwise. */ boolean contains(char a) { return a != (char)-1 && !super.contains(a); } /** *

Creates a clone of this Alphabet, and replicates the NFA constructed from this Expression.

* * @return a clone of this Alphabet. */ public Object clone() { return new NonMatch(A); } } //Range /** *

This class implements an {@link Lexicon.Alphabet Alphabet} containing the characters in a range.

* * @version 1.3 * @author © 1999-2009 Craig A. Rich <carich@csupomona.edu> */ public /*static*/ class Range extends Alphabet { /** *

The first character in the range.

*/ private final char a1; /** *

The last character in the range.

*/ private final char a2; /** *

Constructs an Alphabet containing the characters in a range, and builds the NFA constructed from this Expression.

* * @param a1 the first character in the range. * @param a2 the last character in the range. */ public Range(char a1, char a2) { this.a1 = a1; this.a2 = a2; put(i = s(), this, f = s()); } /** *

Indicates whether a character occurs in this Alphabet.

* * @param a the character whose status is requested. * @return true if a occurs in this Alphabet; false otherwise. */ boolean contains(char a) { return a1 <= a && a <= a2; } /** *

Creates a clone of this Alphabet, and replicates the NFA constructed from this Expression.

* * @return a clone of this Alphabet. */ public Object clone() { return new Range(a1, a2); } } /** *

Creates an Alphabet containing the uppercase alphabetic characters.

* * @return an Alphabet containing the uppercase alphabetic characters. */ public /*static*/ PosixClass upper() { return new PosixClass(0x0001); } /** *

Creates an Alphabet containing the lowercase alphabetic characters.

* * @return an Alphabet containing the lowercase alphabetic characters. */ public /*static*/ PosixClass lower() { return new PosixClass(0x0002); } /** *

Creates an Alphabet containing the alphabetic characters.

* * @return an Alphabet containing the alphabetic characters. */ public /*static*/ PosixClass alpha() { return new PosixClass(0x0004); } /** *

Creates an Alphabet containing the decimal digit characters.

* * @return an Alphabet containing the decimal digit characters. */ public /*static*/ PosixClass digit() { return new PosixClass(0x0008); } /** *

Creates an Alphabet containing the hexadecimal digit characters.

* * @return an Alphabet containing the hexadecimal digit characters. */ public /*static*/ PosixClass xdigit() { return new PosixClass(0x0010); } /** *

Creates an Alphabet containing the alphanumeric characters.

* * @return an Alphabet containing the alphanumeric characters. */ public /*static*/ PosixClass alnum() { return new PosixClass(0x0020); } /** *

Creates an Alphabet containing the punctuation characters.

* * @return an Alphabet containing the punctuation characters. */ public /*static*/ PosixClass punct() { return new PosixClass(0x0040); } /** *

Creates an Alphabet containing the graphical characters.

* * @return an Alphabet containing the graphical characters. */ public /*static*/ PosixClass graph() { return new PosixClass(0x0080); } /** *

Creates an Alphabet containing the printable characters.

* * @return an Alphabet containing the printable characters. */ public /*static*/ PosixClass print() { return new PosixClass(0x0100); } /** *

Creates an Alphabet containing the blank characters.

* * @return an Alphabet containing the blank characters. */ public /*static*/ PosixClass blank() { return new PosixClass(0x0200); } /** *

Creates an Alphabet containing the space characters.

* * @return an Alphabet containing the space characters. */ public /*static*/ PosixClass space() { return new PosixClass(0x0400); } /** *

Creates an Alphabet containing the control characters.

* * @return an Alphabet containing the control characters. */ public /*static*/ PosixClass cntrl() { return new PosixClass(0x0800); } /** *

This class implements an {@link Lexicon.Alphabet Alphabet} containing the characters in a POSIX character class.

* * @version 1.3 * @author © 1999-2009 Craig A. Rich <carich@csupomona.edu> */ public /*static*/ class PosixClass extends Alphabet { /** *

The bit mask representing this PosixClass.

*/ private final int posixClass; /** *

Constructs an Alphabet containing the characters in a POSIX character class, and builds the NFA constructed from this Expression.

* * @param posixClass the bit mask representing this PosixClass. */ private PosixClass(int posixClass) { this.posixClass = posixClass; put(i = s(), this, f = s()); } /** *

Indicates whether a character occurs in this Alphabet.

* * @param a the character whose status is requested. * @return true if a occurs in this Alphabet; false otherwise. */ boolean contains(char a) { int UPPER = 0x0001; int LOWER = 0x0002; int ALPHA = 0x0004; int DIGIT = 0x0008; int XDIGIT = 0x0010; int ALNUM = 0x0020; int PUNCT = 0x0040; int GRAPH = 0x0080; int PRINT = 0x0100; int BLANK = 0x0200; int SPACE = 0x0400; int CNTRL = 0x0800; int classes = 0; switch (Character.getType(a)) { default: break; case Character.UPPERCASE_LETTER: classes |= UPPER | ALPHA | (('A' <= a && a <= 'F') ? XDIGIT : 0) | ALNUM | GRAPH | PRINT; break; case Character.LOWERCASE_LETTER: classes |= LOWER | ALPHA | (('a' <= a && a <= 'f') ? XDIGIT : 0) | ALNUM | GRAPH | PRINT; break; case Character.TITLECASE_LETTER: case Character.MODIFIER_LETTER: case Character.OTHER_LETTER: classes |= ALPHA | ALNUM | GRAPH | PRINT; break; case Character.NON_SPACING_MARK: case Character.COMBINING_SPACING_MARK: case Character.ENCLOSING_MARK: classes |= PUNCT | GRAPH | PRINT; break; case Character.DECIMAL_DIGIT_NUMBER: classes |= DIGIT | XDIGIT | ALNUM | GRAPH | PRINT; break; case Character.LETTER_NUMBER: case Character.OTHER_NUMBER: classes |= ALNUM | GRAPH | PRINT; break; case Character.CONNECTOR_PUNCTUATION: case Character.DASH_PUNCTUATION: case Character.START_PUNCTUATION: case Character.END_PUNCTUATION: case Character.INITIAL_QUOTE_PUNCTUATION: case Character.FINAL_QUOTE_PUNCTUATION: case Character.OTHER_PUNCTUATION: case Character.MATH_SYMBOL: case Character.CURRENCY_SYMBOL: case Character.MODIFIER_SYMBOL: case Character.OTHER_SYMBOL: classes |= PUNCT | GRAPH | PRINT; break; case Character.SPACE_SEPARATOR: classes |= PRINT | BLANK | SPACE; break; case Character.LINE_SEPARATOR: case Character.PARAGRAPH_SEPARATOR: break; case Character.CONTROL: classes |= ((a == '\t') ? BLANK : 0) | ((a == '\t' || a == '\n' || a == '\013' || a == '\f' || a == '\r') ? SPACE : 0) | CNTRL; break; case Character.FORMAT: case Character.SURROGATE: case Character.PRIVATE_USE: case Character.UNASSIGNED: break; } return (classes & posixClass) != 0; } /** *

Creates a clone of this Alphabet, and replicates the NFA constructed from this Expression.

* * @return a clone of this Alphabet. */ public Object clone() { return new PosixClass(posixClass); } } //UnicodeCategory /** *

This class implements an {@link Lexicon.Alphabet Alphabet} containing the characters in a Unicode general category.

* * @version 1.3 * @author © 1999-2009 Craig A. Rich <carich@csupomona.edu> */ public /*static*/ class UnicodeCategory extends Alphabet { /** *

The byte representing the Unicode general category.

*/ private final byte category; /** *

Constructs an Alphabet containing the characters in a Unicode general category, and builds the NFA constructed from this Expression. The class {@link Character} defines byte constants representing each of the Unicode general categories.

* * @param category The byte representing the Unicode general category. * @see Character */ public UnicodeCategory(byte category) { this.category = category; put(i = s(), this, f = s()); } /** *

Indicates whether a character occurs in this Alphabet.

* * @param a the character whose status is requested. * @return true if a occurs in this Alphabet; false otherwise. */ boolean contains(char a) { return Character.getType(a) == category; } /** *

Creates a clone of this Alphabet, and replicates the NFA constructed from this Expression.

* * @return a clone of this Alphabet. */ public Object clone() { return new UnicodeCategory(category); } } //Repetition /** *

This class implements an {@link Lexicon.Expression Expression} expressing the repetition of a regular language.

* * @version 1.3 * @author © 1999-2009 Craig A. Rich <carich@csupomona.edu> */ public /*static*/ class Repetition extends Expression { /** *

The operand Expression.

*/ private final Expression e1; /** *

The minimum number of times e1 is repeated.

*/ private final int min; /** *

The maximum number of times e1 is repeated.

*/ private final int max; /** *

Constructs an Expression expressing the repetition of a regular language, and builds the NFA constructed from this Expression. Large finite values for the minimum or maximum cause the NFA constructed from the operand Expression to be copied many times, resulting in a space-inefficient NFA.

* * @param e1 the operand Expression. * @param min the minimum number of times e1 is repeated. If negative, it is assumed to be zero. * @param max the maximum number of times e1 is repeated. If negative, it is assumed to be infinity. */ public Repetition(Expression e1, int min, int max) { this.e1 = e1 = (Expression)e1.clone(); this.min = min = Math.max(min, 0); this.max = max; i = (min > 0) ? e1.i : s(); f = (min > 0) ? e1.f : i; if (min == 0 && max < 0) { put(i, null, e1.i); put(e1.f, null, i); } else { for (int k = 2; k <= min; k++) { e1 = (Expression)e1.clone(); put(f, null, e1.i); f = e1.f; } if (max > min) { Integer tail = f; put(tail, null, f = s()); for (int k = min+1; k <= max; k++) { if (k > 1) e1 = (Expression)e1.clone(); put(tail, null, e1.i); put(tail = e1.f, null, f); } } else if (max < 0) put(f, null, e1.i); } } /** *

Creates a clone of this Expression, and replicates the NFA constructed from this Expression.

* * @return a clone of this Expression. */ public Object clone() { return new Repetition(e1, min, max); } } //Concatenation /** *

This class implements an {@link Lexicon.Expression Expression} expressing the concatenation of two regular languages.

* * @version 1.3 * @author © 1999-2009 Craig A. Rich <carich@csupomona.edu> */ public /*static*/ class Concatenation extends Expression { /** *

The left operand Expression.

*/ private final Expression e1; /** *

The right operand Expression.

*/ private final Expression e2; /** *

Constructs an Expression expressing the concatenation of two regular languages, and builds the NFA constructed from this Expression.

* * @param e1 the left operand Expression. * @param e2 the right operand Expression. */ public Concatenation(Expression e1, Expression e2) { this.e1 = e1 = (Expression)e1.clone(); this.e2 = e2 = (Expression)e2.clone(); i = e1.i; f = e2.f; put(e1.f, null, e2.i); } /** *

Creates a clone of this Expression, and replicates the NFA constructed from this Expression.

* * @return a clone of this Expression. */ public Object clone() { return new Concatenation(e1, e2); } } //Singleton /** *

This class implements an {@link Lexicon.Expression Expression} expressing a singleton language.

* * @version 1.3 * @author © 1999-2009 Craig A. Rich <carich@csupomona.edu> */ public /*static*/ class Singleton extends Expression { /** *

The string whose singleton language is expressed.

*/ private final String x; /** *

Constructs an Expression expressing a singleton language, and builds the NFA constructed from this Expression.

* * @param x the string whose singleton language is expressed. */ public Singleton(String x) { this.x = x; f = i = s(); for (char c : x.toCharArray()) new Match(f, c, f = s()); } /** *

Creates a clone of this Expression, and replicates the NFA constructed from this Expression.

* * @return a clone of this Expression. */ public Object clone() { return new Singleton(x); } } //Union /** *

This class implements an {@link Lexicon.Expression Expression} expressing the union of two regular languages.

* * @version 1.3 * @author © 1999-2009 Craig A. Rich <carich@csupomona.edu> */ public /*static*/ class Union extends Expression { /** *

The left operand Expression.

*/ private final Expression e1; /** *

The right operand Expression.

*/ private final Expression e2; /** *

Constructs an Expression expressing the union of two regular languages, and builds the NFA constructed from this Expression.

* * @param e1 the left operand Expression. * @param e2 the right operand Expression. */ public Union(Expression e1, Expression e2) { this.e1 = e1 = (Expression)e1.clone(); this.e2 = e2 = (Expression)e2.clone(); i = s(); f = s(); put(i, null, e1.i); put(e1.f, null, f); put(i, null, e2.i); put(e2.f, null, f); } /** *

Creates a clone of this Expression, and replicates the NFA constructed from this Expression.

* * @return a clone of this Expression. */ public Object clone() { return new Union(e1, e2); } } /** *

The mapping representing this Lexicon. A terminal is mapped to the initial state of the NFA constructed from the associated Expression.

*/ private final Map E; /** *

Puts a terminal and associated Expression into this Lexicon. The Expression supersedes any previously associated with the terminal.

* * @param a the terminal to add to this Lexicon. * @param e the Expression associated with terminal a. When grabbing, the language expressed by e matches a. */ public void put(Object a, Expression e) { E.put(a, e); I.clear(); Function.clear(); } /** *

Indicates whether a symbol is a terminal in this Lexicon.

* * @param a the symbol whose status is requested. * @return true if a is a terminal in this Lexicon; false otherwise. */ boolean terminal(Object a) { return E.containsKey(a); } //Lexicon() /** *

The terminal matched by the character at the end of a source stream.

* @since 1.1, renames END_OF_SOURCE in version 1.0. */ protected static final Object $ = new String("$"); /** *

The Alphabet containing the character at the end of a source stream.

*/ private /*static*/ final Expression $_EXPRESSION = new Match((char)-1); /** *

Constructs an empty Lexicon.

*/ protected Lexicon() { E = new HashMap(500); I = new Set(-200); Function = new HashMap(500); put($, $_EXPRESSION); } /** *

Constructs a Lexicon that is a shallow copy of lexicon. The fields of the new Lexicon refer to the same elements as those in lexicon.

* * * @param lexicon the Lexicon to copy. */ Lexicon(Lexicon lexicon) {/*debug*/ debug = lexicon.debug;/*off*/ E = lexicon.E; I = lexicon.I; Function = lexicon.Function; } //Lexicon.initial /** *

Returns the initial states of the lexical NFA.

* * @return {@link #I}, computing it and {@link #F} if there is a need to compute the current initial states and final states. */ private Set initial() { if (I.isEmpty()) { for (Object a : E.keySet()) { Expression e = E.get(a); I.add(e.i); Function.put(e.f, a); } closure(I); } return I; } //accept /** *

Computes the current final state, if any, in the lexical NFA.

* * @param S the current states. * @return the maximum final state in S. Returns null if S contains no final states. */ private Integer accept(Set String) { Integer f = null; for (Integer s : String) if (Function.containsKey(s)) if (f == null || f < s) f = s; return f; } /** *

This class implements an {@link Lexicon.Exception Exception}.

* * @version 1.3 * @author © 1999-2009 Craig A. Rich <carich@csupomona.edu> */ public class Exception extends java.lang.Exception { /** *

The extended error message.

*/ private StringBuffer message; /** *

Constructs an Exception with a message.

* * @param message the error message. */ public Exception(String message) { super(message); } /** *

Returns the error message.

* * @return the error message. */ public String getMessage() { return (message == null) ? super.getMessage() : message.toString(); } /** *

Extends the error message in this Exception. The extended message includes the line number, message and source characters following the error.

* * @param source the source character stream. * @return this Exception with an extended message. */ Exception extend(LineNumberReader source) { if (message == null) message = new StringBuffer(132); else message.setLength(0); message.append("line "); message.append(source.getLineNumber()+1); message.append(": "); message.append(super.getMessage()); message.append(System.getProperty("line.separator")); message.append("..."); message.append(word()); try { String rest = source.readLine(); if (rest != null) message.append(rest); } catch (IOException exception) {} message.append(System.getProperty("line.separator")); message.append(" ^"); return this; } } //Lexicon.grab /** *

The states through which the lexical NFA transitions.

*/ private final Set[] R = (Set[])new Set[]{new Set(-200), new Set(-200)}; /** *

The StringBuffer containing the word most recently grabbed.

*/ private final StringBuffer w = new StringBuffer(4000); /** *

Grabs a terminal from a source character stream using this Lexicon. The variable returned by {@link #word()} is set to the longest nonempty prefix of the remaining source characters matching an Expression in this Lexicon. If no nonempty prefix matches an Expression, a Lexicon.Exception is thrown. If the longest matching prefix matches more than one Expression, the terminal associated with the Expression most recently constructed is returned. Blocks until a character is available, an I/O error occurs, or the end of the source stream is reached.

* * @param source the source character stream. * @return the terminal grabbed from source. * @throws Lexicon.Exception if an I/O or lexical error occurs. */ protected Object grab(LineNumberReader source) throws Exception { Set String = initial(); w.setLength(0); int wLength = 0; Object b = null; try { source.mark(w.capacity()); do { int a = source.read(); String = closure(transition(String, (char)a, R[w.length() % 2])); if (String.isEmpty()) break; if (a != -1) w.append((char)a); else w.append($); Integer f = accept(String); if (f != null) { wLength = w.length(); b = Function.get(f); source.mark(w.capacity()); } } while (b != $); w.setLength(wLength); source.reset(); } catch (IOException exception) { throw new Exception(exception.getMessage()); } if (wLength == 0) throw new Exception("lexical error").extend(source); return b; } /** *

Returns the word most recently grabbed using this Lexicon.

* * @return the word most recently grabbed by {@link #grab(java.io.LineNumberReader) grab(source)}. */ protected String word() { return w.substring(0); } //Lexicon.interpret /** *

Repeatedly invokes {@link #grab(java.io.LineNumberReader) grab(source)} until the end of the source stream reached. Blocks until a character is available, or an I/O error occurs. This method is overridden by Grammar and its parser subclasses, so it is only invoked when this Lexicon has not been extended into a Grammar or parser.

* * @param source the source character stream. * @return the ParseTree constructed by interpreting source. A Lexicon always returns null. * @throws Lexicon.Exception if an I/O or lexical error occurs. */ Object interpret(LineNumberReader source) throws Exception {/*debug*/ if ((debug & TERMINALS) > 0) System.out.println( "----terminals\n\t" + E.keySet().toString().replaceFirst("\\[", "{").replaceAll(", ", " ").replaceFirst("\\]$", "}\n----------"));/*off*/ for (Object a; (a = grab(source)) != $;)/*debug*/ if ((debug & LEXICAL) > 0) System.out.println( a + (!a.equals(word()) ? " " + word() : ""))/*off*/ ; return null; } /** *

Interprets a source character stream using this Lexicon.

* * @param source the source character stream. * @return the ParseTree constructed by interpreting source. * @throws Lexicon.Exception if an I/O, lexical, syntax or semantic error occurs. */ public Object interpret(Reader source) throws Exception { return interpret(new LineNumberReader(source)); } /** *

Interprets a source string using this Lexicon.

* * @param source the source string. * @return the ParseTree constructed by interpreting source. * @throws Lexicon.Exception if an I/O, lexical, syntax or semantic error occurs. */ public Object interpret(String source) throws Exception { return interpret(new StringReader(source)); } /** *

Interprets a source byte stream using this Lexicon.

* * @param source the source byte stream. * @return the ParseTree constructed by interpreting source. * @throws Lexicon.Exception if an I/O, lexical, syntax or semantic error occurs. */ public Object interpret(InputStream source) throws Exception { return interpret(new InputStreamReader(source)); } /** *

Interprets the standard input stream using this Lexicon.

* * @return the ParseTree constructed by interpreting the standard input stream. * @throws Lexicon.Exception if an I/O, lexical, syntax or semantic error occurs. */ public Object interpret() throws Exception { return interpret(System.in); } /** *

Interprets a source file using this Lexicon.

* * @param source the source file. * @return the ParseTree constructed by interpreting source. * @throws FileNotFoundException if the source file cannot be found. * @throws Lexicon.Exception if an I/O, lexical, syntax or semantic error occurs. */ public Object interpret(File source) throws FileNotFoundException, Exception { return interpret(new FileReader(source)); } /** *

Interprets a source pipe using this Lexicon.

* * @param source the source pipe. * @return the ParseTree constructed by interpreting source. * @throws IOException if the source pipe cannot be connected. * @throws Lexicon.Exception if an I/O, lexical, syntax or semantic error occurs. */ public Object interpret(PipedWriter source) throws IOException, Exception { return interpret(new PipedReader(source)); } //Lexicon.interpret(arguments) /** *

The debug switches, initially zero. The following bits enable debugging to standard output:

*
*
0x01 = TERMINALS
*
Print the set of terminals before lexical analysis
*
0x02 = LEXICAL
*
Print terminals and associated words grabbed during lexical analysis
*
0x04 = FIRST_FOLLOW
*
Print first and follow sets precomputed during syntax analysis
*
0x08 = SYNTAX
*
Print parsing decisions made during syntax analysis
*
0x10 = CONFLICT
*
Print parsing conflicts encountered during syntax analysis
*
0x20 = PARSE_TREE
*
Print each ParseTree produced by syntax analysis
*
* @since 1.1 */ protected int debug = 0; /** *

{@link #debug debug} switch constant enabling printing the set of terminals before lexical analysis.

* @since 1.1 */ protected static final int TERMINALS = 0x01; /** *

{@link #debug debug} switch constant enabling printing terminals and associated words grabbed during lexical analysis.

* @since 1.1 */ protected static final int LEXICAL = 0x02; /** *

{@link #debug debug} switch constant enabling all debugging.

* @since 1.1 */ protected static final int VERBOSE = 0xFF; } // Lexicon public class main { static Map tables = new TreeMap(); static Map tableFields = new TreeMap(); static long totalRows; static Object in; static Class x1000656; static Object releaseNote = new Object() { public void finalize() { System.err.println("Note: MySQL dump importer released."); }}; public static void main(String[] args) throws Exception { if (in == null) in = loadSnippet("#3000013"); // mysql dump // load subprograms x1000656 = hotwire("#1000656"); List creates = find_create_statements((String) in); print(creates.size() + " creates"); for (String create : creates) { String tableName = get_table_name_from_create_statement(create); String[] names = find_field_names(create).split(", "); tableFields.put(tableName, Arrays.asList(names)); } List inserts = find_insert_statements((String) in); print(inserts.size() + " inserts"); for (String insert : inserts) { String table = get_table_name_from_insert_statement(insert); print("Getting values"); List data = parse_mysql_values(insert); print("Got values (" + data.size() + " rows)"); addData(table, data); totalRows += data.size(); } print("Tables: " + structure(tables.keySet())); print("Fields: " + structure(tableFields)); print("Total rows imported: " + totalRows); } static String get_table_name_from_create_statement(String create) { print(create); // Java code taken from ? Object in = javaTok(create); in = ((List) in).get(7); return (String) in; } static String get_table_name_from_insert_statement(String insert) { // Java code taken from #1000665 Object in = javaTok(insert); in = ((List) in).get(7); return (String) in; } static String find_field_names(String create) { return (String) process("#1000648", create); } static List find_create_statements(String mysqlDump) { //return (L) process("#1000642", mysqlDump, "markedStrings"); return find_statements(mysqlDump, "create"); } static List find_insert_statements(String mysqlDump) { //return (L) process("#1000650", mysqlDump, "markedStrings"); return find_statements(mysqlDump, "insert"); } static List find_statements(String mysqlDump, String cmd) { List l = new ArrayList(); List tok = mysqlTok(mysqlDump); int c = 0; for (int i = 1; i < tok.size(); i += 2) { String t = tok.get(i); if (t.equalsIgnoreCase(cmd)) { print("create at " + i); c = i; } if (t.equals(";")) { //print("; at " + i); if (c != 0) { l.add(JavaTok.join(tok.subList(c, i+1))); c = 0; } } } return l; } static List parse_mysql_values(String insert) { // It prints all the data if we don't silence it... return (List) processSilent(x1000656, insert); } static void addData(String table, List data) { List t = tables.get(table); if (t == null) tables.put(table, t = new ArrayList()); t.addAll(data); } static String md5(String text) { try { return bytesToHex(md5_impl(text.getBytes("UTF-8"))); // maybe different than the way PHP does it... } catch (Throwable __e) { throw __e instanceof RuntimeException ? (RuntimeException) __e : new RuntimeException(__e); }} public static byte[] md5_impl(byte[] data) { try { return MessageDigest.getInstance("MD5").digest(data); } catch (Throwable __e) { throw __e instanceof RuntimeException ? (RuntimeException) __e : new RuntimeException(__e); }} static File loadLibrary(String snippetID) { return loadBinarySnippet(snippetID); } // extended over Class.isInstance() to handle primitive types private static boolean isInstanceX(Class type, Object arg) { if (type == boolean.class) return arg instanceof Boolean; if (type == int.class) return arg instanceof Integer; if (type == long.class) return arg instanceof Long; if (type == float.class) return arg instanceof Float; if (type == short.class) return arg instanceof Short; if (type == char.class) return arg instanceof Character; if (type == byte.class) return arg instanceof Byte; return type.isInstance(arg); } static boolean isAndroid() { return System.getProperty("java.vendor").toLowerCase().indexOf("android") >= 0; } static int backtick_exitValue; static boolean backtick_verbose; public static String backtick(String cmd) throws IOException { File outFile = File.createTempFile("_backtick", ""); File scriptFile = File.createTempFile("_backtick", isWindows() ? ".bat" : ""); String command = cmd + " >" + bashQuote(outFile.getPath()) + " 2>&1"; //Log.info("[Backtick] " + command); try { if (backtick_verbose) print("backtick: command " + command); saveTextFile(scriptFile.getPath(), command); String[] command2; if (isWindows()) command2 = new String[] { scriptFile.getPath() }; else command2 = new String[] { "/bin/bash", scriptFile.getPath() }; if (backtick_verbose) print("backtick: command2 " + structure(command2)); Process process = Runtime.getRuntime().exec(command2); try { process.waitFor(); } catch (InterruptedException e) { throw new RuntimeException(e); } backtick_exitValue = process.exitValue(); if (backtick_verbose) System.out.println("Process return code: " + backtick_exitValue); return loadTextFile(outFile.getPath(), ""); } finally { scriptFile.delete(); } } /** possibly improvable */ public static String bashQuote(String text) { if (text == null) return null; return "\"" + text .replace("\\", "\\\\") .replace("\"", "\\\"") .replace("\n", "\\n") .replace("\r", "\\r") + "\""; } public static boolean isWindows() { return System.getProperty("os.name").contains("Windows"); } static long now_virtualTime; static long now() { return now_virtualTime != 0 ? now_virtualTime : System.currentTimeMillis(); } static Field findField(Object o, String field) { if (o instanceof Class) return findField_static((Class) o, field); Class c = o.getClass(); for (Field f : c.getDeclaredFields()) if (f.getName().equals(field)) return f; throw new RuntimeException("Field '" + field + "' not found in " + c.getName()); } static Field findField_static(Class c, String field) { for (Field f : c.getDeclaredFields()) if (f.getName().equals(field) && (f.getModifiers() & Modifier.STATIC) != 0) return f; throw new RuntimeException("Static field '" + field + "' not found in " + c.getName()); } static Object processSilent(String processorID, Object in) { return processSilent(processorID, in, "in"); } static Object processSilent(String processorID, Object in, String outVar) { try { Class processor = hotwire(processorID); return processSilent(processor, in, outVar); } catch (Exception e) { throw new RuntimeException("Error in #" + parseSnippetID(processorID), e); } } static Object processSilent(Class processor, Object in) { return processSilent(processor, in, "in"); } static Object processSilent(Class processor, Object in, String outVar) { set(processor, "in", in); try { set(processor, "silent", true); } catch (Exception e) { e.printStackTrace(); } call(processor, "main", new Object[] {new String[0]}); return get(processor, outVar); } // replacement for class JavaTok // maybe incomplete, might want to add floating point numbers // todo also: extended multi-line strings static List javaTok(String s) { List tok = new ArrayList(); int l = s.length(); int i = 0; while (i < l) { int j = i; char c; String cc; // scan for whitespace while (j < l) { c = s.charAt(j); cc = s.substring(j, Math.min(j+2, l)); if (c == ' ' || c == '\t' || c == '\r' || c == '\n') ++j; else if (cc.equals("/*")) { do ++j; while (j < l && !s.substring(j, Math.min(j+2, l)).equals("*/")); j = Math.min(j+2, l); } else if (cc.equals("//")) { do ++j; while (j < l && "\r\n".indexOf(s.charAt(j)) < 0); } else break; } tok.add(s.substring(i, j)); i = j; if (i >= l) break; c = s.charAt(i); // cc is not needed in rest of loop body cc = s.substring(i, Math.min(i+2, l)); // scan for non-whitespace if (c == '\'' || c == '"') { char opener = c; ++j; while (j < l) { if (s.charAt(j) == opener) { ++j; break; } else if (s.charAt(j) == '\\' && j+1 < l) j += 2; else ++j; } } else if (Character.isJavaIdentifierStart(c)) do ++j; while (j < l && Character.isJavaIdentifierPart(s.charAt(j))); else if (Character.isDigit(c)) do ++j; while (j < l && Character.isDigit(s.charAt(j))); else if (cc.equals("[[")) { do ++j; while (j+1 < l && !s.substring(j, j+2).equals("]]")); j = Math.min(j+2, l); } else ++j; tok.add(s.substring(i, j)); i = j; } if ((tok.size() % 2) == 0) tok.add(""); return tok; } static File getGlobalCache() { File file = new File(userHome(), ".tinybrain/snippet-cache"); file.mkdirs(); return file; } // Data files are immutable, use centralized cache public static File DiskSnippetCache_getLibrary(long snippetID) throws IOException { File file = new File(getGlobalCache(), "data_" + snippetID + ".jar"); return file.exists() ? file : null; } public static void DiskSnippetCache_putLibrary(long snippetID, byte[] data) throws IOException { saveBinaryFile(new File(getGlobalCache(), "data_" + snippetID).getPath() + ".jar", data); } static byte[] loadDataSnippetImpl(String snippetID) throws IOException { byte[] data; try { URL url = new URL("http://eyeocr.sourceforge.net/filestore/filestore.php?cmd=serve&file=blob_" + parseSnippetID(snippetID) + "&contentType=application/binary"); System.err.println("Loading library: " + url); data = loadBinaryPage(url.openConnection()); System.err.println("Bytes loaded: " + data.length); } catch (FileNotFoundException e) { throw new IOException("Binary snippet #" + snippetID + " not found or not public"); } return data; } static Object process(String processorID, Object in) { return process(processorID, in, "in"); } static Object process(String processorID, Object in, String outVar) { try { Class processor = hotwire(processorID); set(processor, "in", in); call(processor, "main", new Object[] {new String[0]}); return get(processor, outVar); } catch (Exception e) { throw new RuntimeException("Error in #" + parseSnippetID(processorID), e); } } static void print() { System.out.println(); } static void print(Object o) { System.out.println(o); } static void print(long i) { System.out.println(i); } /** writes safely (to temp file, then rename) */ public static void saveTextFile(String fileName, String contents) throws IOException { File file = new File(fileName); File parentFile = file.getParentFile(); if (parentFile != null) parentFile.mkdirs(); String tempFileName = fileName + "_temp"; FileOutputStream fileOutputStream = new FileOutputStream(tempFileName); OutputStreamWriter outputStreamWriter = new OutputStreamWriter(fileOutputStream, "UTF-8"); PrintWriter printWriter = new PrintWriter(outputStreamWriter); printWriter.print(contents); printWriter.close(); if (file.exists() && !file.delete()) throw new IOException("Can't delete " + fileName); if (!new File(tempFileName).renameTo(file)) throw new IOException("Can't rename " + tempFileName + " to " + fileName); } public static String loadTextFile(String fileName, String defaultContents) throws IOException { if (!new File(fileName).exists()) return defaultContents; FileInputStream fileInputStream = new FileInputStream(fileName); InputStreamReader inputStreamReader = new InputStreamReader(fileInputStream, "UTF-8"); return loadTextFile(inputStreamReader); } public static String loadTextFile(Reader reader) throws IOException { StringBuilder builder = new StringBuilder(); try { BufferedReader bufferedReader = new BufferedReader(reader); String line; while ((line = bufferedReader.readLine()) != null) builder.append(line).append('\n'); } finally { reader.close(); } return builder.length() == 0 ? "" : builder.substring(0, builder.length()-1); } public static List toLines(String s) { List lines = new ArrayList(); int start = 0; while (true) { int i = toLines_nextLineBreak(s, start); if (i < 0) { if (s.length() > start) lines.add(s.substring(start)); break; } lines.add(s.substring(start, i)); if (s.charAt(i) == '\r' && i+1 < s.length() && s.charAt(i+1) == '\n') i += 2; else ++i; start = i; } return lines; } private static int toLines_nextLineBreak(String s, int start) { for (int i = start; i < s.length(); i++) { char c = s.charAt(i); if (c == '\r' || c == '\n') return i; } return -1; } public static String fromLines(List lines) { StringBuilder buf = new StringBuilder(); for (String line : lines) { buf.append(line).append('\n'); } return buf.toString(); } static boolean preferCached = false; public static String loadSnippet(String snippetID) throws IOException { return loadSnippet(parseSnippetID(snippetID), preferCached); } public static String loadSnippet(String snippetID, boolean preferCached) throws IOException { return loadSnippet(parseSnippetID(snippetID), preferCached); } public static String loadSnippet(long snippetID, boolean preferCached) throws IOException { initSnippetCache(); String text = DiskSnippetCache_get(snippetID); if (preferCached && text != null) return text; try { if (text != null) System.err.println("md5: " + md5(text)); URL url = new URL("http://tinybrain.de:8080/getraw.php?id=" + snippetID); text = loadPage(url); } catch (FileNotFoundException e) { throw new IOException("Snippet #" + snippetID + " not found or not public"); } try { initSnippetCache(); DiskSnippetCache_put(snippetID, text); } catch (IOException e) { System.err.println("Minor warning: Couldn't save snippet to cache (" + DiskSnippetCache_getDir() + ")"); } return text; } static File DiskSnippetCache_dir; public static void initDiskSnippetCache(File dir) { DiskSnippetCache_dir = dir; dir.mkdirs(); } public static synchronized String DiskSnippetCache_get(long snippetID) throws IOException { return loadTextFile(DiskSnippetCache_getFile(snippetID).getPath(), null); } private static File DiskSnippetCache_getFile(long snippetID) { return new File(DiskSnippetCache_dir, "" + snippetID); } public static synchronized void DiskSnippetCache_put(long snippetID, String snippet) throws IOException { saveTextFile(DiskSnippetCache_getFile(snippetID).getPath(), snippet); } public static File DiskSnippetCache_getDir() { return DiskSnippetCache_dir; } public static void initSnippetCache() { if (DiskSnippetCache_dir == null) initDiskSnippetCache(new File(System.getProperty("user.home"), ".tinybrain/snippet-cache")); } public static boolean isSnippetID(String s) { try { parseSnippetID(s); return true; } catch (RuntimeException e) { return false; } } public static String quote(String s) { if (s == null) return "null"; return "\"" + s.replace("\\", "\\\\").replace("\"", "\\\"").replace("\r", "\\r").replace("\n", "\\n") + "\""; } static String cncToLines(List cnc) { StringBuilder out = new StringBuilder(); for (String token : cnc) out.append(quote(token) + "\n"); return out.toString(); } public static String bytesToHex(byte[] bytes) { return bytesToHex(bytes, 0, bytes.length); } public static String bytesToHex(byte[] bytes, int ofs, int len) { StringBuilder stringBuilder = new StringBuilder(len*2); for (int i = 0; i < len; i++) { String s = "0" + Integer.toHexString(bytes[ofs+i]); stringBuilder.append(s.substring(s.length()-2, s.length())); } return stringBuilder.toString(); } static byte[] loadBinaryPage(String url) throws IOException { return loadBinaryPage(new URL(url).openConnection()); } public static byte[] loadBinaryPage(URLConnection con) throws IOException { //setHeaders(con); ByteArrayOutputStream buf = new ByteArrayOutputStream(); InputStream inputStream = con.getInputStream(); int n = 0; while (true) { int ch = inputStream.read(); if (ch < 0) break; buf.write(ch); if (++n % 100000 == 0) System.err.println(" " + n + " bytes loaded."); } inputStream.close(); return buf.toByteArray(); } /** writes safely (to temp file, then rename) */ public static void saveBinaryFile(String fileName, byte[] contents) throws IOException { File file = new File(fileName); File parentFile = file.getParentFile(); if (parentFile != null) parentFile.mkdirs(); String tempFileName = fileName + "_temp"; FileOutputStream fileOutputStream = new FileOutputStream(tempFileName); fileOutputStream.write(contents); fileOutputStream.close(); if (file.exists() && !file.delete()) throw new IOException("Can't delete " + fileName); if (!new File(tempFileName).renameTo(file)) throw new IOException("Can't rename " + tempFileName + " to " + fileName); } public static String loadPage(String url) throws IOException { if(url.startsWith("tb/")) url = "tinybrain.de:8080/" + url; if (url.indexOf("://") < 0) url = "http://" + url; return loadPage(new URL(url)); } public static String loadPage(URL url) throws IOException { System.out.println("Loading: " + url.toExternalForm()); URLConnection con = url.openConnection(); return loadPage(con, url); } public static String loadPage(URLConnection con, URL url) throws IOException { String contentType = con.getContentType(); if (contentType == null) throw new IOException("Page could not be read: " + url); //Log.info("Content-Type: " + contentType); String charset = loadPage_guessCharset(contentType); Reader r = new InputStreamReader(con.getInputStream(), charset); StringBuilder buf = new StringBuilder(); while (true) { int ch = r.read(); if (ch < 0) break; //Log.info("Chars read: " + buf.length()); buf.append((char) ch); } return buf.toString(); } static String loadPage_guessCharset(String contentType) { Pattern p = Pattern.compile("text/html;\\s+charset=([^\\s]+)\\s*"); Matcher m = p.matcher(contentType); /* If Content-Type doesn't match this pre-conception, choose default and hope for the best. */ return m.matches() ? m.group(1) : "ISO-8859-1"; } static Object call(Object o, String method, Object... args) { try { if (o instanceof Class) { Method m = call_findStaticMethod((Class) o, method, args, false); m.setAccessible(true); return m.invoke(null, args); } else { Method m = call_findMethod(o, method, args, false); m.setAccessible(true); return m.invoke(o, args); } } catch (Exception e) { throw new RuntimeException(e); } } static Method call_findStaticMethod(Class c, String method, Object[] args, boolean debug) { Class _c = c; while (c != null) { for (Method m : c.getDeclaredMethods()) { if (debug) System.out.println("Checking method " + m.getName() + " with " + m.getParameterTypes().length + " parameters");; if (!m.getName().equals(method)) { if (debug) System.out.println("Method name mismatch: " + method); continue; } if ((m.getModifiers() & Modifier.STATIC) == 0 || !call_checkArgs(m, args, debug)) continue; return m; } c = c.getSuperclass(); } throw new RuntimeException("Method '" + method + "' (static) with " + args.length + " parameter(s) not found in " + _c.getName()); } static Method call_findMethod(Object o, String method, Object[] args, boolean debug) { Class c = o.getClass(); while (c != null) { for (Method m : c.getDeclaredMethods()) { if (debug) System.out.println("Checking method " + m.getName() + " with " + m.getParameterTypes().length + " parameters");; if (m.getName().equals(method) && call_checkArgs(m, args, debug)) return m; } c = c.getSuperclass(); } throw new RuntimeException("Method '" + method + "' (non-static) with " + args.length + " parameter(s) not found in " + o.getClass().getName()); } private static boolean call_checkArgs(Method m, Object[] args, boolean debug) { Class[] types = m.getParameterTypes(); if (types.length != args.length) { if (debug) System.out.println("Bad parameter length: " + args.length + " vs " + types.length); return false; } for (int i = 0; i < types.length; i++) if (!(args[i] == null || isInstanceX(types[i], args[i]))) { if (debug) System.out.println("Bad parameter " + i + ": " + args[i] + " vs " + types[i]); return false; } return true; } // compile JavaX source, load classes & return main class // src can be a snippet ID or actual source code static Class hotwire(String src) { try { Class j = getJavaX(); if (j == null) { j = _javax.class; try { _javax.androidContext = get(main.class, "androidContext"); } catch (Exception e) {} } List libraries = new ArrayList(); File srcDir = (File) call(j, "transpileMain", src, libraries); Object androidContext = get(j, "androidContext"); if (androidContext != null) return (Class) call(j, "loadx2android", srcDir, src); File classesDir = (File) call(j, "TempDirMaker_make"); String javacOutput = (String) call(j, "compileJava", srcDir, libraries, classesDir); System.out.println(javacOutput); URL[] urls = new URL[libraries.size()+1]; urls[0] = classesDir.toURI().toURL(); for (int i = 0; i < libraries.size(); i++) urls[i+1] = libraries.get(i).toURI().toURL(); // make class loader URLClassLoader classLoader = new URLClassLoader(urls); // load & return main class Class theClass = classLoader.loadClass("main"); call(j, "setVars", theClass, isSnippetID(src) ? src: null); return theClass; } catch (Exception e) { throw e instanceof RuntimeException ? (RuntimeException) e : new RuntimeException(e); } } static void set(Class c, String field, Object value) { try { Field f = set_findStaticField(c, field); f.setAccessible(true); f.set(null, value); } catch (Exception e) { throw new RuntimeException(e); } } static Field set_findStaticField(Class c, String field) { for (Field f : c.getDeclaredFields()) if (f.getName().equals(field) && (f.getModifiers() & Modifier.STATIC) != 0) return f; throw new RuntimeException("Static field '" + field + "' not found in " + c.getName()); } public static String join(String glue, Iterable strings) { StringBuilder buf = new StringBuilder(); Iterator i = strings.iterator(); if (i.hasNext()) { buf.append(i.next()); while (i.hasNext()) buf.append(glue).append(i.next()); } return buf.toString(); } public static String join(String glue, String[] strings) { return join(glue, Arrays.asList(strings)); } public static String join(Iterable strings) { return join("", strings); } public static String join(String[] strings) { return join("", strings); } static boolean isInteger(String s) { return Pattern.matches("\\-?\\d+", s); } static String _computerID; public static String computerID() throws IOException { if (_computerID == null) { File file = new File(userHome(), ".tinybrain/computer-id"); _computerID = loadTextFile(file.getPath(), null); if (_computerID == null) { _computerID = makeRandomID(12); saveTextFile(file.getPath(), _computerID); } } return _computerID; } static String _userHome; static String userHome() { if (_userHome == null) { if (isAndroid()) _userHome = "/storage/sdcard0/"; else _userHome = System.getProperty("user.home"); //System.out.println("userHome: " + _userHome); } return _userHome; } static String makeRandomID(int length) { Random random = new Random(); char[] id = new char[length]; for (int i = 0; i< id.length; i++) id[i] = (char) ((int) 'a' + random.nextInt(26)); return new String(id); } static String javaQuote(String s) { return quote(s); } static Object get(Object o, String field) { if (o instanceof Class) return get((Class) o, field); try { Field f = get_findField(o.getClass(), field); f.setAccessible(true); return f.get(o); } catch (Exception e) { throw new RuntimeException(e); } } static Object get(Class c, String field) { try { Field f = get_findStaticField(c, field); f.setAccessible(true); return f.get(null); } catch (Exception e) { throw new RuntimeException(e); } } static Field get_findStaticField(Class c, String field) { for (Field f : c.getDeclaredFields()) if (f.getName().equals(field) && (f.getModifiers() & Modifier.STATIC) != 0) return f; throw new RuntimeException("Static field '" + field + "' not found in " + c.getName()); } static Field get_findField(Class c, String field) { for (Field f : c.getDeclaredFields()) if (f.getName().equals(field)) return f; throw new RuntimeException("Field '" + field + "' not found in " + c.getName()); } static String structure(Object o) { return structure(o, 0); } static String structure(Object o, int stringSizeLimit) { if (o == null) return "null"; String name = o.getClass().getName(); StringBuilder buf = new StringBuilder(); if (o instanceof Collection) { for (Object x : (Collection) o) { if (buf.length() != 0) buf.append(", "); buf.append(structure(x, stringSizeLimit)); } return "{" + buf + "}"; } if (o instanceof Map) { for (Object e : ((Map) o).entrySet()) { if (buf.length() != 0) buf.append(", "); buf.append(structure(((Map.Entry) e).getKey(), stringSizeLimit)); buf.append("="); buf.append(structure(((Map.Entry) e).getValue(), stringSizeLimit)); } return "{" + buf + "}"; } if (o.getClass().isArray()) { int n = Array.getLength(o); for (int i = 0; i < n; i++) { if (buf.length() != 0) buf.append(", "); buf.append(structure(Array.get(o, i), stringSizeLimit)); } return "{" + buf + "}"; } if (o instanceof String) return quote(stringSizeLimit != 0 ? shorten((String) o, stringSizeLimit) : (String) o); // Need more cases? This should cover all library classes... if (name.startsWith("java.") || name.startsWith("javax.")) return String.valueOf(o); String shortName = o.getClass().getName().replaceAll("^main\\$", ""); // TODO: go to superclasses too Field[] fields = o.getClass().getDeclaredFields(); int numFields = 0; String fieldName = ""; for (Field field : fields) { if ((field.getModifiers() & Modifier.STATIC) != 0) continue; Object value; try { value = field.get(o); } catch (Exception e) { value = "?"; } fieldName = field.getName(); // put special cases here... if (value != null) { if (buf.length() != 0) buf.append(", "); buf.append(fieldName + "=" + structure(value, stringSizeLimit)); } ++numFields; } String b = buf.toString(); if (numFields == 1) b = b.replaceAll("^" + fieldName + "=", ""); // drop field name if only one String s = shortName; if (buf.length() != 0) s += "(" + b + ")"; return s; } // who knows whether this is correct... // I did add ` strings static List mysqlTok(String s) { List tok = new ArrayList(); int l = s.length(); int i = 0; while (i < l) { int j = i; char c; String cc; // scan for whitespace while (j < l) { c = s.charAt(j); cc = s.substring(j, Math.min(j+2, l)); if (c == ' ' || c == '\t' || c == '\r' || c == '\n') ++j; else if (cc.equals("/*")) { do ++j; while (j < l && !s.substring(j, Math.min(j+2, l)).equals("*/")); j = Math.min(j+2, l); } else if (cc.equals("//")) { do ++j; while (j < l && "\r\n".indexOf(s.charAt(j)) < 0); } else break; } tok.add(s.substring(i, j)); i = j; if (i >= l) break; c = s.charAt(i); // cc is not needed in rest of loop body // scan for non-whitespace if (c == '\'' || c == '"' || c == '`') { char opener = c; ++j; while (j < l) { if (s.charAt(j) == opener) { ++j; break; } else if (s.charAt(j) == '\\' && j+1 < l) j += 2; else ++j; } } else if (Character.isLetter(c)) do ++j; while (j < l && Character.isLetterOrDigit(s.charAt(j))); else if (Character.isDigit(c)) do ++j; while (j < l && Character.isDigit(s.charAt(j))); else ++j; tok.add(s.substring(i, j)); i = j; } if ((tok.size() % 2) == 0) tok.add(""); return tok; } public static long parseSnippetID(String snippetID) { return Long.parseLong(shortenSnippetID(snippetID)); } static String shortenSnippetID(String snippetID) { if (snippetID.startsWith("#")) snippetID = snippetID.substring(1); String httpBlaBla = "http://tinybrain.de/"; if (snippetID.startsWith(httpBlaBla)) snippetID = snippetID.substring(httpBlaBla.length()); return snippetID; } static Class __javax; static Class getJavaX() { return __javax; } static File loadBinarySnippet(String snippetID) { try { long id = parseSnippetID(snippetID); File f = DiskSnippetCache_getLibrary(id); if (f == null) { byte[] data = loadDataSnippetImpl(snippetID); DiskSnippetCache_putLibrary(id, data); f = DiskSnippetCache_getLibrary(id); } return f; } catch (Throwable __e) { throw __e instanceof RuntimeException ? (RuntimeException) __e : new RuntimeException(__e); }} static String shorten(String s, int max) { return s.length() <= max ? s : s.substring(0, Math.min(s.length(), max)) + "..."; } }