* The basic algorithm is described in:
* "An O(ND) Difference Algorithm and its Variations", Eugene Myers, * Algorithmica Vol. 1 No. 2, 1986, p 251. *

* This class outputs different results from GNU diff 1.15 on some * inputs. Our results are actually better (smaller change list, smaller * total size of changes), but it would be nice to know why. Perhaps * there is a memory overwrite bug in GNU diff 1.15. * * @author Stuart D. Gathman, translated from GNU diff 1.15 * Copyright (C) 2000 Business Management Systems, Inc. *

* This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 1, or (at your option) * any later version. *

* This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. *

* You should have received a copy of the * GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */ static class EGDiff { /** * Prepare to find differences between two arrays. Each element of * the arrays is translated to an "equivalence number" based on * the result of equals. The original Object arrays * are no longer needed for computing the differences. They will * be needed again later to print the results of the comparison as * an edit script, if desired. */ public EGDiff(Object[] a, Object[] b) { Hashtable h = new Hashtable(a.length + b.length); filevec[0] = new file_data(a, h); filevec[1] = new file_data(b, h); } /** * 1 more than the maximum equivalence value used for this or its * sibling file. */ private int equiv_max = 1; /** * When set to true, the comparison uses a heuristic to speed it up. * With this heuristic, for files with a constant small density * of changes, the algorithm is linear in the file size. */ public boolean heuristic = false; /** * When set to true, the algorithm returns a guarranteed minimal * set of changes. This makes things slower, sometimes much slower. */ public boolean no_discards = false; private int[] xvec, yvec; /* Vectors being compared. */ private int[] fdiag; /* Vector, indexed by diagonal, containing the X coordinate of the point furthest along the given diagonal in the forward search of the edit matrix. */ private int[] bdiag; /* Vector, indexed by diagonal, containing the X coordinate of the point furthest along the given diagonal in the backward search of the edit matrix. */ private int fdiagoff, bdiagoff; private final file_data[] filevec = new file_data[2]; private int cost; /** * Find the midpoint of the shortest edit script for a specified * portion of the two files. *

* We scan from the beginnings of the files, and simultaneously from the ends, * doing a breadth-first search through the space of edit-sequence. * When the two searches meet, we have found the midpoint of the shortest * edit sequence. *

* The value returned is the number of the diagonal on which the midpoint lies. * The diagonal number equals the number of inserted lines minus the number * of deleted lines (counting only lines before the midpoint). * The edit cost is stored into COST; this is the total number of * lines inserted or deleted (counting only lines before the midpoint). *

* This function assumes that the first lines of the specified portions * of the two files do not match, and likewise that the last lines do not * match. The caller must trim matching lines from the beginning and end * of the portions it is going to specify. *

* Note that if we return the "wrong" diagonal value, or if * the value of bdiag at that diagonal is "wrong", * the worst this can do is cause suboptimal diff output. * It cannot cause incorrect diff output. */ private int diag(int xoff, int xlim, int yoff, int ylim) { final int[] fd = fdiag; // Give the compiler a chance. final int[] bd = bdiag; // Additional help for the compiler. final int[] xv = xvec; // Still more help for the compiler. final int[] yv = yvec; // And more and more . . . final int dmin = xoff - ylim; // Minimum valid diagonal. final int dmax = xlim - yoff; // Maximum valid diagonal. final int fmid = xoff - yoff; // Center diagonal of top-down search. final int bmid = xlim - ylim; // Center diagonal of bottom-up search. int fmin = fmid, fmax = fmid; // Limits of top-down search. int bmin = bmid, bmax = bmid; // Limits of bottom-up search. /* True if southeast corner is on an odd diagonal with respect to the northwest. */ final boolean odd = (fmid - bmid & 1) != 0; fd[fdiagoff + fmid] = xoff; bd[bdiagoff + bmid] = xlim; for (int c = 1; ; ++c) { int d; /* Active diagonal. */ boolean big_snake = false; /* Extend the top-down search by an edit step in each diagonal. */ if (fmin > dmin) fd[fdiagoff + --fmin - 1] = -1; else ++fmin; if (fmax < dmax) fd[fdiagoff + ++fmax + 1] = -1; else --fmax; for (d = fmax; d >= fmin; d -= 2) { int x, y, oldx, tlo = fd[fdiagoff + d - 1], thi = fd[fdiagoff + d + 1]; if (tlo >= thi) x = tlo + 1; else x = thi; oldx = x; y = x - d; while (x < xlim && y < ylim && xv[x] == yv[y]) { ++x; ++y; } if (x - oldx > 20) big_snake = true; fd[fdiagoff + d] = x; if (odd && bmin <= d && d <= bmax && bd[bdiagoff + d] <= fd[fdiagoff + d]) { cost = 2 * c - 1; return d; } } /* Similar extend the bottom-up search. */ if (bmin > dmin) bd[bdiagoff + --bmin - 1] = Integer.MAX_VALUE; else ++bmin; if (bmax < dmax) bd[bdiagoff + ++bmax + 1] = Integer.MAX_VALUE; else --bmax; for (d = bmax; d >= bmin; d -= 2) { int x, y, oldx, tlo = bd[bdiagoff + d - 1], thi = bd[bdiagoff + d + 1]; if (tlo < thi) x = tlo; else x = thi - 1; oldx = x; y = x - d; while (x > xoff && y > yoff && xv[x - 1] == yv[y - 1]) { --x; --y; } if (oldx - x > 20) big_snake = true; bd[bdiagoff + d] = x; if (!odd && fmin <= d && d <= fmax && bd[bdiagoff + d] <= fd[fdiagoff + d]) { cost = 2 * c; return d; } } /* Heuristic: check occasionally for a diagonal that has made lots of progress compared with the edit distance. If we have any such, find the one that has made the most progress and return it as if it had succeeded. With this heuristic, for files with a constant small density of changes, the algorithm is linear in the file size. */ if (c > 200 && big_snake && heuristic) { int best = 0; int bestpos = -1; for (d = fmax; d >= fmin; d -= 2) { int dd = d - fmid; if ((fd[fdiagoff + d] - xoff) * 2 - dd > 12 * (c + (dd > 0 ? dd : -dd))) { if (fd[fdiagoff + d] * 2 - dd > best && fd[fdiagoff + d] - xoff > 20 && fd[fdiagoff + d] - d - yoff > 20) { int k; int x = fd[fdiagoff + d]; /* We have a good enough best diagonal; now insist that it end with a significant snake. */ for (k = 1; k <= 20; k++) if (xvec[x - k] != yvec[x - d - k]) break; if (k == 21) { best = fd[fdiagoff + d] * 2 - dd; bestpos = d; } } } } if (best > 0) { cost = 2 * c - 1; return bestpos; } best = 0; for (d = bmax; d >= bmin; d -= 2) { int dd = d - bmid; if ((xlim - bd[bdiagoff + d]) * 2 + dd > 12 * (c + (dd > 0 ? dd : -dd))) { if ((xlim - bd[bdiagoff + d]) * 2 + dd > best && xlim - bd[bdiagoff + d] > 20 && ylim - (bd[bdiagoff + d] - d) > 20) { /* We have a good enough best diagonal; now insist that it end with a significant snake. */ int k; int x = bd[bdiagoff + d]; for (k = 0; k < 20; k++) if (xvec[x + k] != yvec[x - d + k]) break; if (k == 20) { best = (xlim - bd[bdiagoff + d]) * 2 + dd; bestpos = d; } } } } if (best > 0) { cost = 2 * c - 1; return bestpos; } } } } /** * Compare in detail contiguous subsequences of the two files * which are known, as a whole, to match each other. *

* The results are recorded in the vectors filevec[N].changed_flag, by * storing a 1 in the element for each line that is an insertion or deletion. *

* The subsequence of file 0 is [XOFF, XLIM) and likewise for file 1. *

* Note that XLIM, YLIM are exclusive bounds. * All line numbers are origin-0 and discarded lines are not counted. */ private void compareseq(int xoff, int xlim, int yoff, int ylim) { /* Slide down the bottom initial diagonal. */ while (xoff < xlim && yoff < ylim && xvec[xoff] == yvec[yoff]) { ++xoff; ++yoff; } /* Slide up the top initial diagonal. */ while (xlim > xoff && ylim > yoff && xvec[xlim - 1] == yvec[ylim - 1]) { --xlim; --ylim; } /* Handle simple cases. */ if (xoff == xlim) while (yoff < ylim) filevec[1].changed_flag[1 + filevec[1].realindexes[yoff++]] = true; else if (yoff == ylim) while (xoff < xlim) filevec[0].changed_flag[1 + filevec[0].realindexes[xoff++]] = true; else { /* Find a point of correspondence in the middle of the files. */ int d = diag(xoff, xlim, yoff, ylim); int c = cost; int b = bdiag[bdiagoff + d]; if (c == 1) { /* This should be impossible, because it implies that one of the two subsequences is empty, and that case was handled above without calling `diag'. Let's verify that this is true. */ throw new IllegalArgumentException("Empty subsequence"); } else { /* Use that point to split this problem into two subproblems. */ compareseq(xoff, b, yoff, b - d); /* This used to use f instead of b, but that is incorrect! It is not necessarily the case that diagonal d has a snake from b to f. */ compareseq(b, xlim, b - d, ylim); } } } /** * Discard lines from one file that have no matches in the other file. */ private void discard_confusing_lines() { filevec[0].discard_confusing_lines(filevec[1]); filevec[1].discard_confusing_lines(filevec[0]); } private boolean inhibit = false; /** * Adjust inserts/deletes of blank lines to join changes * as much as possible. */ private void shift_boundaries() { if (inhibit) return; filevec[0].shift_boundaries(filevec[1]); filevec[1].shift_boundaries(filevec[0]); } public interface ScriptBuilder { /** * Scan the tables of which lines are inserted and deleted, * producing an edit script. * * @param changed0 true for lines in first file which do not match 2nd * @param len0 number of lines in first file * @param changed1 true for lines in 2nd file which do not match 1st * @param len1 number of lines in 2nd file * @return a linked list of changes - or null */ public change build_script(boolean[] changed0, int len0, boolean[] changed1, int len1); } /** * Scan the tables of which lines are inserted and deleted, * producing an edit script in reverse order. */ static class ReverseScript implements ScriptBuilder { public change build_script(final boolean[] changed0, int len0, final boolean[] changed1, int len1) { change script = null; int i0 = 0, i1 = 0; while (i0 < len0 || i1 < len1) { if (changed0[1 + i0] || changed1[1 + i1]) { int line0 = i0, line1 = i1; /* Find # lines changed here in each file. */ while (changed0[1 + i0]) ++i0; while (changed1[1 + i1]) ++i1; /* Record this change. */ script = new change(line0, line1, i0 - line0, i1 - line1, script); } /* We have reached lines in the two files that match each other. */ i0++; i1++; } return script; } } static class ForwardScript implements ScriptBuilder { /** * Scan the tables of which lines are inserted and deleted, * producing an edit script in forward order. */ public change build_script(final boolean[] changed0, int len0, final boolean[] changed1, int len1) { change script = null; int i0 = len0, i1 = len1; while (i0 >= 0 || i1 >= 0) { if (changed0[i0] || changed1[i1]) { int line0 = i0, line1 = i1; /* Find # lines changed here in each file. */ while (changed0[i0]) --i0; while (changed1[i1]) --i1; /* Record this change. */ script = new change(i0, i1, line0 - i0, line1 - i1, script); } /* We have reached lines in the two files that match each other. */ i0--; i1--; } return script; } } /** * Standard ScriptBuilders. */ public final static ScriptBuilder forwardScript = new ForwardScript(), reverseScript = new ReverseScript(); /* Report the differences of two files. DEPTH is the current directory depth. */ public final change diff_2(final boolean reverse) { return diff(reverse ? reverseScript : forwardScript); } /** * Get the results of comparison as an edit script. The script * is described by a list of changes. The standard ScriptBuilder * implementations provide for forward and reverse edit scripts. * Alternate implementations could, for instance, list common elements * instead of differences. * * @param bld an object to build the script from change flags * @return the head of a list of changes */ public change diff(final ScriptBuilder bld) { /* Some lines are obviously insertions or deletions because they don't match anything. Detect them now, and avoid even thinking about them in the main comparison algorithm. */ discard_confusing_lines(); /* Now do the main comparison algorithm, considering just the undiscarded lines. */ xvec = filevec[0].undiscarded; yvec = filevec[1].undiscarded; int diags = filevec[0].nondiscarded_lines + filevec[1].nondiscarded_lines + 3; fdiag = new int[diags]; fdiagoff = filevec[1].nondiscarded_lines + 1; bdiag = new int[diags]; bdiagoff = filevec[1].nondiscarded_lines + 1; compareseq(0, filevec[0].nondiscarded_lines, 0, filevec[1].nondiscarded_lines); fdiag = null; bdiag = null; /* Modify the results slightly to make them prettier in cases where that can validly be done. */ shift_boundaries(); /* Get the results of comparison in the form of a chain of `struct change's -- an edit script. */ return bld.build_script(filevec[0].changed_flag, filevec[0].buffered_lines, filevec[1].changed_flag, filevec[1].buffered_lines); } /** * The result of comparison is an "edit script": a chain of change objects. * Each change represents one place where some lines are deleted * and some are inserted. *

* LINE0 and LINE1 are the first affected lines in the two files (origin 0). * DELETED is the number of lines deleted here from file 0. * INSERTED is the number of lines inserted here in file 1. *

* If DELETED is 0 then LINE0 is the number of the line before * which the insertion was done; vice versa for INSERTED and LINE1. */ public static class change { /** * Previous or next edit command. */ public change link; /** * # lines of file 1 changed here. */ public final int inserted; /** * # lines of file 0 changed here. */ public final int deleted; /** * Line number of 1st deleted line. */ public final int line0; /** * Line number of 1st inserted line. */ public final int line1; /** * Cons an additional entry onto the front of an edit script OLD. * LINE0 and LINE1 are the first affected lines in the two files (origin 0). * DELETED is the number of lines deleted here from file 0. * INSERTED is the number of lines inserted here in file 1. *

* If DELETED is 0 then LINE0 is the number of the line before * which the insertion was done; vice versa for INSERTED and LINE1. */ public change(int line0, int line1, int deleted, int inserted, change old) { this.line0 = line0; this.line1 = line1; this.inserted = inserted; this.deleted = deleted; this.link = old; //System.err.println(line0+","+line1+","+inserted+","+deleted); } } /** * Data on one input file being compared. */ class file_data { /** * Allocate changed array for the results of comparison. */ void clear() { /* Allocate a flag for each line of each file, saying whether that line is an insertion or deletion. Allocate an extra element, always zero, at each end of each vector. */ changed_flag = new boolean[buffered_lines + 2]; } /** * Return equiv_count[I] as the number of lines in this file * that fall in equivalence class I. * * @return the array of equivalence class counts. */ int[] equivCount() { int[] equiv_count = new int[equiv_max]; for (int i = 0; i < buffered_lines; ++i) ++equiv_count[equivs[i]]; return equiv_count; } /** * Discard lines that have no matches in another file. *

* A line which is discarded will not be considered by the actual * comparison algorithm; it will be as if that line were not in the file. * The file's `realindexes' table maps virtual line numbers * (which don't count the discarded lines) into real line numbers; * this is how the actual comparison algorithm produces results * that are comprehensible when the discarded lines are counted. *

* When we discard a line, we also mark it as a deletion or insertion * so that it will be printed in the output. * * @param f the other file */ void discard_confusing_lines(file_data f) { clear(); /* Set up table of which lines are going to be discarded. */ final byte[] discarded = discardable(f.equivCount()); /* Don't really discard the provisional lines except when they occur in a run of discardables, with nonprovisionals at the beginning and end. */ filterDiscards(discarded); /* Actually discard the lines. */ discard(discarded); } /** * Mark to be discarded each line that matches no line of another file. * If a line matches many lines, mark it as provisionally discardable. * * @param counts The count of each equivalence number for the other file. * @return 0=nondiscardable, 1=discardable or 2=provisionally discardable * for each line */ private byte[] discardable(final int[] counts) { final int end = buffered_lines; final byte[] discards = new byte[end]; final int[] equivs = this.equivs; int many = 5; int tem = end / 64; /* Multiply MANY by approximate square root of number of lines. That is the threshold for provisionally discardable lines. */ while ((tem = tem >> 2) > 0) many *= 2; for (int i = 0; i < end; i++) { int nmatch; if (equivs[i] == 0) continue; nmatch = counts[equivs[i]]; if (nmatch == 0) discards[i] = 1; else if (nmatch > many) discards[i] = 2; } return discards; } /** * Don't really discard the provisional lines except when they occur * in a run of discardables, with nonprovisionals at the beginning * and end. */ private void filterDiscards(final byte[] discards) { final int end = buffered_lines; for (int i = 0; i < end; i++) { /* Cancel provisional discards not in middle of run of discards. */ if (discards[i] == 2) discards[i] = 0; else if (discards[i] != 0) { /* We have found a nonprovisional discard. */ int j; int length; int provisional = 0; /* Find end of this run of discardable lines. Count how many are provisionally discardable. */ for (j = i; j < end; j++) { if (discards[j] == 0) break; if (discards[j] == 2) ++provisional; } /* Cancel provisional discards at end, and shrink the run. */ while (j > i && discards[j - 1] == 2) { discards[--j] = 0; --provisional; } /* Now we have the length of a run of discardable lines whose first and last are not provisional. */ length = j - i; /* If 1/4 of the lines in the run are provisional, cancel discarding of all provisional lines in the run. */ if (provisional * 4 > length) { while (j > i) if (discards[--j] == 2) discards[j] = 0; } else { int consec; int minimum = 1; int tem = length / 4; /* MINIMUM is approximate square root of LENGTH/4. A subrun of two or more provisionals can stand when LENGTH is at least 16. A subrun of 4 or more can stand when LENGTH >= 64. */ while ((tem = tem >> 2) > 0) minimum *= 2; minimum++; /* Cancel any subrun of MINIMUM or more provisionals within the larger run. */ for (j = 0, consec = 0; j < length; j++) if (discards[i + j] != 2) consec = 0; else if (minimum == ++consec) /* Back up to start of subrun, to cancel it all. */ j -= consec; else if (minimum < consec) discards[i + j] = 0; /* Scan from beginning of run until we find 3 or more nonprovisionals in a row or until the first nonprovisional at least 8 lines in. Until that point, cancel any provisionals. */ for (j = 0, consec = 0; j < length; j++) { if (j >= 8 && discards[i + j] == 1) break; if (discards[i + j] == 2) { consec = 0; discards[i + j] = 0; } else if (discards[i + j] == 0) consec = 0; else consec++; if (consec == 3) break; } /* I advances to the last line of the run. */ i += length - 1; /* Same thing, from end. */ for (j = 0, consec = 0; j < length; j++) { if (j >= 8 && discards[i - j] == 1) break; if (discards[i - j] == 2) { consec = 0; discards[i - j] = 0; } else if (discards[i - j] == 0) consec = 0; else consec++; if (consec == 3) break; } } } } } /** * Actually discard the lines. * * @param discards flags lines to be discarded */ private void discard(final byte[] discards) { final int end = buffered_lines; int j = 0; for (int i = 0; i < end; ++i) if (no_discards || discards[i] == 0) { undiscarded[j] = equivs[i]; realindexes[j++] = i; } else changed_flag[1 + i] = true; nondiscarded_lines = j; } file_data(Object[] data, Hashtable h) { buffered_lines = data.length; equivs = new int[buffered_lines]; undiscarded = new int[buffered_lines]; realindexes = new int[buffered_lines]; for (int i = 0; i < data.length; ++i) { Integer ir = (Integer) h.get(data[i]); if (ir == null) h.put(data[i], new Integer(equivs[i] = equiv_max++)); else equivs[i] = ir.intValue(); } } /** * Adjust inserts/deletes of blank lines to join changes * as much as possible. *

* We do something when a run of changed lines include a blank * line at one end and have an excluded blank line at the other. * We are free to choose which blank line is included. * `compareseq' always chooses the one at the beginning, * but usually it is cleaner to consider the following blank line * to be the "change". The only exception is if the preceding blank line * would join this change to other changes. * * @param f the file being compared against */ void shift_boundaries(file_data f) { final boolean[] changed = changed_flag; final boolean[] other_changed = f.changed_flag; int i = 0; int j = 0; int i_end = buffered_lines; int preceding = -1; int other_preceding = -1; for (; ;) { int start, end, other_start; /* Scan forwards to find beginning of another run of changes. Also keep track of the corresponding point in the other file. */ while (i < i_end && !changed[1 + i]) { while (other_changed[1 + j++]) /* Non-corresponding lines in the other file will count as the preceding batch of changes. */ other_preceding = j; i++; } if (i == i_end) break; start = i; other_start = j; for (; ;) { /* Now find the end of this run of changes. */ while (i < i_end && changed[1 + i]) i++; end = i; /* If the first changed line matches the following unchanged one, and this run does not follow right after a previous run, and there are no lines deleted from the other file here, then classify the first changed line as unchanged and the following line as changed in its place. */ /* You might ask, how could this run follow right after another? Only because the previous run was shifted here. */ if (end != i_end && equivs[start] == equivs[end] && !other_changed[1 + j] && end != i_end && !((preceding >= 0 && start == preceding) || (other_preceding >= 0 && other_start == other_preceding))) { changed[1 + end] = true; changed[1 + start++] = false; ++i; /* Since one line-that-matches is now before this run instead of after, we must advance in the other file to keep in synch. */ ++j; } else break; } preceding = i; other_preceding = j; } } /** * Number of elements (lines) in this file. */ final int buffered_lines; /** * Vector, indexed by line number, containing an equivalence code for * each line. It is this vector that is actually compared with that * of another file to generate differences. */ private final int[] equivs; /** * Vector, like the previous one except that * the elements for discarded lines have been squeezed out. */ final int[] undiscarded; /** * Vector mapping virtual line numbers (not counting discarded lines) * to real ones (counting those lines). Both are origin-0. */ final int[] realindexes; /** * Total number of nondiscarded lines. */ int nondiscarded_lines; /** * Array, indexed by real origin-1 line number, * containing true for a line that is an insertion or a deletion. * The results of comparison are stored here. */ boolean[] changed_flag; } } static class BlockDiff { public CopyBlock asCopyBlock() { return null; } public NewBlock asNewBlock () { return null; } } static class CopyBlock extends BlockDiff { int firstLine, lines; CopyBlock(int firstLine, int lines) { this.firstLine = firstLine; this.lines = lines; } public CopyBlock asCopyBlock() { return this; } public int getFirstLine() { return firstLine; } public int getLines() { return lines; } } static class NewBlock extends BlockDiff { int originalStart; List contents; NewBlock(int originalStart, List contents) { this.originalStart = originalStart; this.contents = contents; } public NewBlock asNewBlock () { return this; } public int getOriginalStart() { return originalStart; } public List getContents() { return contents; } } static class ExplodedLine { int type; String left, right; int leftIndex, rightIndex; ExplodedLine(int type, String left, String right, int leftIndex, int rightIndex) { this.type = type; this.left = left; this.right = right; this.leftIndex = leftIndex; this.rightIndex = rightIndex; } public int getType() { return type; } public String getLeft() { return left; } public String getRight() { return right; } public int getLeftIndex() { return leftIndex; } public int getRightIndex() { return rightIndex; } } static class BlockDiffer { public static final int IDENTICAL = 0; public static final int DIFFERENT = 1; public static final int LEFT_ONLY = 2; public static final int RIGHT_ONLY = 3; private static void printChange(EGDiff.change change) { if (change != null) { System.out.println("line0="+change.line0+", line1="+change.line1 +", inserted="+change.inserted+", deleted="+change.deleted); printChange(change.link); } } /** Generates the text content of a Unified-format context diff between 2 files * (NB the 'files-changed' header must be added separately). */ public static List generateUniDiff(List fileA, List fileB, int contextSize) { EGDiff diff = new EGDiff(fileA.toArray(), fileB.toArray()); EGDiff.change change = diff.diff_2(false); if (change != null) { int inserted, deleted; List hunkLines = new ArrayList(); int cumulExtraLinesBwrtA = 0; // Each hunk is generated with a header do { int line0 = change.line0, line1 = change.line1; int changeStart = ((line1 < line0) ? line1 : line0); int contextStart = ((changeStart > contextSize) ? changeStart - contextSize : 0); int headerPosn = hunkLines.size(); // Provide the first lines of context for (int i = contextStart; i < changeStart; i++) //System.out.println(" " + fileA.get(i)); hunkLines.add(" " + fileA.get(i)); boolean hunkFinish = false; // Step through each change giving the change lines and following context do { inserted = change.inserted; deleted = change.deleted; line0 = change.line0; line1 = change.line1; if (line1 < line0) // An insert comes earlier while (inserted-- > 0) hunkLines.add("+" + fileB.get(line1++)); while (deleted-- > 0) hunkLines.add("-" + fileA.get(line0++)); while (inserted-- > 0) hunkLines.add("+" + fileB.get(line1++)); // Lines following are trailing context, identical in fileA and fileB // The next change may overlap the context, so check and if so, form one hunk EGDiff.change nextChange = change.link; int nextChangeStart = fileA.size(); if (nextChange != null) nextChangeStart = ((nextChange.line1 < nextChange.line0) ? nextChange.line1 : nextChange.line0); if (nextChangeStart - line0 > contextSize * 2) { // A separate hunk nextChangeStart = line0 + contextSize; hunkFinish = true; } if (nextChangeStart > fileA.size()) nextChangeStart = fileA.size(); // Limit to file size while (line0 < nextChangeStart) { hunkLines.add(" " + fileA.get(line0++)); line1++; // Keep in sync with trailing context } change = change.link; } while (!hunkFinish && change != null); int hunkStartB = contextStart + cumulExtraLinesBwrtA; int hunkTotA = line0 - contextStart; int hunkTotB = line1 - hunkStartB; hunkLines.add(headerPosn, "@@ -" + (contextStart + 1) + ',' + hunkTotA + " +" + (hunkStartB + 1) + ',' + hunkTotB + " @@"); cumulExtraLinesBwrtA += hunkTotB - hunkTotA; } while (change != null); return hunkLines; } return null; } /* For testing: private static void printUniDiff(List fileA, List fileB, int contextSize) { List uniDiff = generateUniDiff(fileA, fileB, contextSize); if (uniDiff != null) for (int j = 0; j < uniDiff.size(); j++) System.out.println(uniDiff.get(j)); } */ public static List diffLines(List lines, List reference) { List diffs = new ArrayList(); EGDiff diff = new EGDiff(reference.toArray(), lines.toArray()); EGDiff.change change = diff.diff_2(false); //printChange(change); //printUniDiff(reference, lines, 3); int l0 = 0, l1 = 0; while (change != null) { if (change.line0 > l0 && change.line1 > l1) diffs.add(new CopyBlock(l0, change.line0-l0)); if (change.inserted != 0) diffs.add(new NewBlock(change.line1, lines.subList(change.line1, change.line1+change.inserted))); l0 = change.line0 + change.deleted; l1 = change.line1 + change.inserted; change = change.link; } if (l0 < reference.size()) diffs.add(new CopyBlock(l0, reference.size()-l0)); return diffs; } /** fills files with empty lines to align matching blocks * * @param file1 first file * @param file2 second file * @return an array with two lists */ public static List explode(List file1, List file2) { List lines = new ArrayList(); List diffs = BlockDiffer.diffLines(file2, file1); int lastLineCopied = 0, rightOnlyStart = -1, rightPosition = 0; for (int i = 0; i < diffs.size(); i++) { BlockDiff diff = diffs.get(i); if (diff instanceof CopyBlock) { CopyBlock copyBlock = (CopyBlock) diff; if (lastLineCopied < copyBlock.getFirstLine()) { if (rightOnlyStart >= 0) { int overlap = Math.min(lines.size()-rightOnlyStart, copyBlock.getFirstLine()-lastLineCopied); //lines.subList(rightOnlyStart, rightOnlyStart+overlap).clear(); convertRightOnlyToDifferent(lines, rightOnlyStart, overlap, file1, lastLineCopied); lastLineCopied += overlap; } addBlock(lines, LEFT_ONLY, file1, lastLineCopied, copyBlock.getFirstLine(), lastLineCopied, -1); } addBlock(lines, IDENTICAL, file1, copyBlock.getFirstLine(), copyBlock.getFirstLine()+copyBlock.getLines(), copyBlock.getFirstLine(), rightPosition); rightPosition += copyBlock.getLines(); lastLineCopied = copyBlock.getFirstLine()+copyBlock.getLines(); rightOnlyStart = -1; } else if (diff instanceof NewBlock) { NewBlock newBlock = (NewBlock) diff; /*if (nextDiff instanceof BlockDiffer.CopyBlock) { BlockDiffer.CopyBlock copyBlock = (BlockDiffer.CopyBlock) nextDiff; copyBlock.getFirstLine()-lastLineCopied*/ rightOnlyStart = lines.size(); addBlock(lines, RIGHT_ONLY, newBlock.getContents(), 0, newBlock.getContents().size(), -1, rightPosition); rightPosition += newBlock.getContents().size(); } } if (rightOnlyStart >= 0) { int overlap = Math.min(lines.size()-rightOnlyStart, file1.size()-lastLineCopied); //lines.subList(rightOnlyStart, rightOnlyStart+overlap).clear(); convertRightOnlyToDifferent(lines, rightOnlyStart, overlap, file1, lastLineCopied); lastLineCopied += overlap; } addBlock(lines, LEFT_ONLY, file1, lastLineCopied, file1.size(), lastLineCopied, -1); return lines; } private static void convertRightOnlyToDifferent(List lines, int start, int numLines, List leftLines, int leftStart) { for (int i = 0; i < numLines; i++) { ExplodedLine line = lines.get(start+i); lines.set(start+i, new ExplodedLine(DIFFERENT, leftLines.get(i+leftStart), line.getRight(), i+leftStart, line.getRightIndex())); } } private static void addBlock(List lines, int type, List srcLines, int start, int end, int leftStart, int rightStart) { for (int i = start; i < end; i++) lines.add(new ExplodedLine(type, type == RIGHT_ONLY ? "" : srcLines.get(i), type == LEFT_ONLY ? "" : srcLines.get(i), type == RIGHT_ONLY ? -1 : i - start + leftStart, type == LEFT_ONLY ? -1 : i - start + rightStart)); } public static List condense(List lines) { List result = new ArrayList(); for (Iterator i = lines.iterator(); i.hasNext();) { ExplodedLine line = i.next(); if (line.getType() == IDENTICAL) { if (result.isEmpty() || result.get(result.size()-1).getType() != IDENTICAL) result.add(new ExplodedLine(IDENTICAL, "[...]", "[...]", -1, -1)); } else result.add(line); } return result; } } static abstract class DialogIO { String line; boolean eos, loud, noClose; Lock lock = lock(); abstract String readLineImpl(); abstract boolean isStillConnected(); abstract void sendLine(String line); abstract boolean isLocalConnection(); abstract Socket getSocket(); abstract void close(); int getPort() { Socket s = getSocket(); return s == null ? 0 : s.getPort(); } boolean helloRead; int shortenOutputTo = 500; String readLineNoBlock() { String l = line; line = null; return l; } boolean waitForLine() { try { ping(); if (line != null) return true; //print("Readline"); line = readLineImpl(); //print("Readline done: " + line); if (line == null) eos = true; return line != null; } catch (Exception __e) { throw rethrow(__e); } } String readLine() { waitForLine(); helloRead = true; return readLineNoBlock(); } String ask(String s, Object... args) { if (loud) return askLoudly(s, args); if (!helloRead) readLine(); if (args.length != 0) s = format3(s, args); sendLine(s); return readLine(); } String askLoudly(String s, Object... args) { if (!helloRead) readLine(); if (args.length != 0) s = format3(s, args); print("> " + shorten(s, shortenOutputTo)); sendLine(s); String answer = readLine(); print("< " + shorten(answer, shortenOutputTo)); return answer; } void pushback(String l) { if (line != null) throw fail(); line = l; helloRead = false; } } static abstract class DialogHandler { abstract void run(DialogIO io); } static final class IndexedList2 extends AbstractList { ArrayList l = new ArrayList(); MultiSet index = new MultiSet(false); // HashMap static boolean debug; static int instances; IndexedList2() { ++instances; } IndexedList2(List x) { this(); l.ensureCapacity(l(x)); addAll(x); } // required immutable list methods @Override public A get(int i) { return l.get(i); } @Override public int size() { return l.size(); } // required mutable list methods @Override public A set(int i, A a) { A prev = l.get(i); if (prev != a) { index.remove(prev); index.add(a); } l.set(i, a); return prev; } @Override public void add(int i, A a) { index.add(a); l.add(i, a); } @Override public A remove(int i) { A a = l.get(i); index.remove(a); l.remove(i); return a; } // speed up methods @Override protected void removeRange(int fromIndex, int toIndex) { for (int i = fromIndex; i < toIndex; i++) unindex(i); l.subList(fromIndex, toIndex).clear(); } @Override public int indexOf(Object a) { if (!contains(a)) return -1; return l.indexOf(a); } @Override public boolean contains(Object a) { boolean b = index.contains((A) a); if (debug) print("IndexedList2.contains " + a + " => " + b); return b; } @Override public void clear() { index.clear(); l.clear(); } @Override public boolean addAll(int i, Collection c) { index.addAll((Collection) c); return l.addAll(i, c); } // indexing methods void unindex(int i) { index.remove(l.get(i)); } void index(int i) { index.add(l.get(i)); } // static methods static IndexedList2 ensureIndexed(List l) { return l instanceof IndexedList2 ? (IndexedList2) l : new IndexedList2(l); } } // uses HashMap by default static class MultiSet { Map map = new HashMap(); MultiSet(boolean useTreeMap) { if (useTreeMap) map = new TreeMap(); } MultiSet() {} MultiSet(Iterable c) { addAll(c); } MultiSet(MultiSet ms) { synchronized(ms) { for (A a : ms.keySet()) add(a, ms.get(a)); }} synchronized void add(A key) { add(key, 1); } synchronized void addAll(Iterable c) { if (c != null) for (A a : c) add(a); } synchronized void addAll(MultiSet ms) { for (A a : ms.keySet()) add(a, ms.get(a)); } synchronized void add(A key, int count) { if (count <= 0) return; if (map.containsKey(key)) map.put(key, map.get(key)+count); else map.put(key, count); } synchronized int get(A key) { Integer i = map.get(key); return i != null ? i : 0; //ret key != null && map.containsKey(key) ? map.get(key) : 0; } synchronized boolean contains(A key) { return map.containsKey(key); } synchronized void remove(A key) { Integer i = map.get(key); if (i != null && i > 1) map.put(key, i - 1); else map.remove(key); } synchronized List topTen() { return getTopTen(); } synchronized List getTopTen() { return getTopTen(10); } synchronized List getTopTen(int maxSize) { List list = getSortedListDescending(); return list.size() > maxSize ? list.subList(0, maxSize) : list; } synchronized List highestFirst() { return getSortedListDescending(); } synchronized List lowestFirst() { return reversedList(getSortedListDescending()); } synchronized List getSortedListDescending() { List list = new ArrayList(map.keySet()); Collections.sort(list, new Comparator() { public int compare(A a, A b) { return map.get(b).compareTo(map.get(a)); } }); return list; } synchronized int getNumberOfUniqueElements() { return map.size(); } synchronized int uniqueSize() { return map.size(); } synchronized Set asSet() { return map.keySet(); } synchronized NavigableSet navigableSet() { return navigableKeys((NavigableMap) map); } synchronized Set keySet() { return map.keySet(); } synchronized A getMostPopularEntry() { int max = 0; A a = null; for (Map.Entry entry : map.entrySet()) { if (entry.getValue() > max) { max = entry.getValue(); a = entry.getKey(); } } return a; } synchronized void removeAll(A key) { map.remove(key); } synchronized int size() { int size = 0; for (int i : map.values()) size += i; return size; } synchronized MultiSet mergeWith(MultiSet set) { MultiSet result = new MultiSet(); for (A a : set.asSet()) { result.add(a, set.get(a)); } return result; } synchronized boolean isEmpty() { return map.isEmpty(); } synchronized public String toString() { // hmm. sync this? return str(map); } synchronized void clear() { map.clear(); } synchronized Map asMap() { return cloneMap(map); } } static ThreadLocal DynamicObject_loading = new ThreadLocal(); static class DynamicObject { String className; // just the name, without the "main$" LinkedHashMap fieldValues = new LinkedHashMap(); DynamicObject() {} // className = just the name, without the "main$" DynamicObject(String className) { this.className = className;} } static class CompilerBot { static boolean verbose; static File compileSnippet(String snippetID) { return compileSnippet(snippetID, ""); } static Pair compileSnippet2(String snippetID) { return compileSnippet2(snippetID, ""); } // returns jar path static File compileSnippet(String snippetID, String javaTarget) { return compileSnippet2(snippetID, javaTarget).a; } // returns jar path, Java source static Pair compileSnippet2(String snippetID, String javaTarget) { String transpiledSrc = getServerTranspiled2(snippetID); int i = transpiledSrc.indexOf('\n'); String libs = transpiledSrc.substring(0, Math.max(0, i)); if (verbose) print("Compiling snippet: " + snippetID + ". Libs: " + libs); transpiledSrc = transpiledSrc.substring(i+1); return pair(compile(transpiledSrc, libs, javaTarget, snippetID), transpiledSrc); } static File compile(String src) { return compile(src, ""); } static File compile(String src, String libs) { return compile(src, libs, null); } static File compile(String src, String dehlibs, String javaTarget) { return compile(src, dehlibs, javaTarget, null); } static File compile(String src, String dehlibs, String javaTarget, String progID) { if (verbose) print("Compiling " + l(src) + " chars"); // Note: This is different from the calculation in x30 // (might lead to programs being compiled twice) String md5 = md5(dehlibs + "\n" + src + "\n" + progID); File jar = getJarFile(md5); if (jar == null || jar.length() <= 22) { // have to compile List mainClass = findMainClass(javaTok(src)); boolean canRename = mainClass != null && useDummyMainClasses() && isSnippetID(progID) && !tok_classHasModifier(mainClass, "public"); if (verbose) print("useRenaming: " + useDummyMainClasses() + ", canRename: " + canRename + ", progID: " + progID); javaCompileToJar_optionalRename(src, dehlibs, jar, canRename ? progID : null, progID); } else { if (verbose) print("Getting classes from cache (" + jar.getAbsolutePath() + ", " + jar.length() + " bytes)"); touchFile(jar); // so we can find the unused ones easier } return jar; } static File getJarFile(String md5) { assertTrue(isMD5(md5)); return new File(getCacheProgramDir("#1002203"), md5 + ".jar"); } } static boolean isTrue(Object o) { if (o instanceof Boolean) return ((Boolean) o).booleanValue(); if (o == null) return false; if (o instanceof ThreadLocal) return isTrue(((ThreadLocal) o).get()); throw fail(getClassName(o)); } static String unquote(String s) { if (s == null) return null; if (s.startsWith("[")) { int i = 1; while (i < s.length() && s.charAt(i) == '=') ++i; if (i < s.length() && s.charAt(i) == '[') { String m = s.substring(1, i); if (s.endsWith("]" + m + "]")) return s.substring(i+1, s.length()-i-1); } } if ((s.startsWith("\"") || s.startsWith("\'")) && s.length() > 1) { int l = s.endsWith(substring(s, 0, 1)) ? s.length()-1 : s.length(); StringBuilder sb = new StringBuilder(l-1); for (int i = 1; i < l; i++) { char ch = s.charAt(i); if (ch == '\\') { char nextChar = (i == l - 1) ? '\\' : s.charAt(i + 1); // Octal escape? if (nextChar >= '0' && nextChar <= '7') { String code = "" + nextChar; i++; if ((i < l - 1) && s.charAt(i + 1) >= '0' && s.charAt(i + 1) <= '7') { code += s.charAt(i + 1); i++; if ((i < l - 1) && s.charAt(i + 1) >= '0' && s.charAt(i + 1) <= '7') { code += s.charAt(i + 1); i++; } } sb.append((char) Integer.parseInt(code, 8)); continue; } switch (nextChar) { case '\\': ch = '\\'; break; case 'b': ch = '\b'; break; case 'f': ch = '\f'; break; case 'n': ch = '\n'; break; case 'r': ch = '\r'; break; case 't': ch = '\t'; break; case '\"': ch = '\"'; break; case '\'': ch = '\''; break; // Hex Unicode: u???? case 'u': if (i >= l - 5) { ch = 'u'; break; } int code = Integer.parseInt( "" + s.charAt(i + 2) + s.charAt(i + 3) + s.charAt(i + 4) + s.charAt(i + 5), 16); sb.append(Character.toChars(code)); i += 5; continue; default: ch = nextChar; // added by Stefan } i++; } sb.append(ch); } return sb.toString(); } return s; // not quoted - return original } static String transpilingSnippetID; static int varCount; static Map snippetCache = new HashMap(); static boolean useIndexedList = true, opt_javaTok = true; static boolean cacheStdFunctions = true, cacheStdClasses = true; static HashMap cachedIncludes = new HashMap(); static ExecutorService executor; static List lclasses; static long startTime, lastPrint; // These variables have to be cleared manually for each transpilation static HashSet included = new HashSet(); static Set definitions = ciSet(); static HashMap rewrites = new HashMap(); static HashSet shouldNotIncludeFunction = new HashSet(), shouldNotIncludeClass = new HashSet(); static HashSet doNotIncludeFunction = new HashSet(); static HashSet addedFunctions = new HashSet(); static HashSet addedClasses = new HashSet(); static HashSet hardFunctionReferences = new HashSet(); static HashSet mapLikeFunctions = new HashSet(); static HashSet mapMethodLikeFunctions = new HashSet(); static HashSet nuLikeFunctions = new HashSet(); static Map extraStandardFunctions; static boolean quickmainDone1, quickmainDone2; static TreeSet libs = new TreeSet(); static String mainBaseClass, mainPackage, mainClassName; static boolean localStuffOnly; // for transpiling a fragment static boolean allowMetaCode = false; // run any embedded meta code static List metaPostBlocks; static boolean dontPrintSource; static class CachedInclude { String javax; Future java; String realJava; String java() { return realJava != null ? realJava : getFuture(java); } Future javaFuture() { return realJava != null ? nowFuture(realJava) : java; } void clean() { if (java != null) { realJava = getFuture(java); java = null; } } } public static void main(final String[] args) throws Exception { startTime = lastPrint = sysNow(); try { vmKeepWithProgramMD5_get("cachedIncludes"); } catch (Throwable __e) { _handleException(__e); } executor = Executors.newFixedThreadPool(numberOfCores()); transpilingSnippetID = or(getThreadLocal((ThreadLocal) getOpt(javax(), "transpilingSnippetID")), transpilingSnippetID); print("transpilingSnippetID", transpilingSnippetID); Object oldPrint = or(print_byThread().get(), "print_raw"); AutoCloseable __17 = tempInterceptPrint(new F1() { Boolean get(String s) { long now = sysNow(); long time = now-lastPrint; // -startTime; lastPrint = now; callF(oldPrint, "[" + formatInt(time/1000, 2) + ":" + formatInt(time % 1000, 3) + "] " + s); return false; } }); try { try { _main(); } finally { interceptPrintInThisThread(oldPrint); if (executor != null) executor.shutdown(); executor = null; localStuffOnly = false; } } finally { _close(__17); }} static void _main() { try { if (sameSnippetID(programID(), defaultJavaXTranslatorID())) setDefaultJavaXTranslatorID("#7"); //reTok_modify_check = true; //if (useIndexedList) findCodeTokens_debug = true; javaTok_opt = opt_javaTok; findCodeTokens_indexed = findCodeTokens_unindexed = 0; findCodeTokens_bails = findCodeTokens_nonbails = 0; javaTok_n = javaTok_elements = 0; String in = loadMainJava(); print("759 STARTING " + identityHashCode(main.class)); included.clear(); definitions.clear(); rewrites.clear(); definitions.add("SymbolAsString"); shouldNotIncludeFunction.clear(); shouldNotIncludeClass.clear(); doNotIncludeFunction.clear(); addedFunctions.clear(); addedClasses.clear(); hardFunctionReferences.clear(); mapLikeFunctions = cloneHashSet(tok_mapLikeFunctions()); mapMethodLikeFunctions = cloneHashSet(tok_mapMethodLikeFunctions()); nuLikeFunctions.clear(); extraStandardFunctions = new HashMap(); libs.clear(); mainBaseClass = mainPackage = mainClassName = null; varCount = 0; quickmainDone1 = quickmainDone2 = false; metaPostBlocks = new ArrayList(); dontPrintSource = false; //L ts = findTranslators(toLines(join(tok))); //print("Translators in source at start: " + structure(ts)); // "duplicate" statement - unused /*L lines = toLines(in); call(getJavaX(), "findTranslators", lines); in = fromLines(lines); new Matches m; if (match("duplicate *", in, m)) { // actual copying - unused // tok = jtok(loadSnippet(m.get(0))); // reference by include() in = "m { p { callMain(include(" + quote(m.get(0)) + ")); } }"; }*/ List tok = jtok(in); try { tok_definitions(tok); // add m { } if (!localStuffOnly && !hasCodeTokens(tok, "m", "{") && !hasCodeTokens(tok, "main", "{") && !hasCodeTokens(tok, "class", "main")) { //tok = jtok(moveImportsUp("m {\n" + in + "\n}")); if (l(tok) == 1) tok = singlePlusList(first(tok), dropFirst(javaTok("m {}"))); else { replaceTokens_reTok(tok, 1, 2, "m {\n\n" + unnull(get(tok, 1))); replaceTokens_reTok(tok, l(tok)-2, l(tok)-1, unnull(get(tok, l(tok)-2)) + "}"); } tok_moveImportsUp(tok); } // standard translate //ts = findTranslators(toLines(join(tok))); //print("Translators in source: " + structure(ts)); if (tok_hasTranslators(tok)) tok = jtok(defaultTranslate(join(tok))); //print("end of default translate"); //print(join(tok)); //tok_autoCloseBrackets(tok); tok_compactModules(tok); tok = tok_processIncludes(tok); // before standard functions if (processConceptsDot(tok)) tok = tok_processIncludes(tok); tok = localStuff1(tok); if (!localStuffOnly) { int safety = 0; boolean same; do { // BIG LOOP String before = join(tok); // XXX tok = localStuff1(tok); // shortened method declarations BEFORE standardFunctions jreplace(tok, "svoid", "static void"); jreplace(tok, "void {", "$1 $2() {"); jreplace(tok, "String {", "$1 $2() {"); jreplace(tok, "Object {", "$1 $2() {"); jreplace(tok, "List {", "$1 $2() {"); // do the non-local stuff (flags and rewrites) tok_mainClassNameAndPackage(tok); tok_definitions(tok); tok_ifndef(tok); tok_ifdef(tok); defineMapLikes(tok); if (tok_applyMapLikeFunctions(tok, mapLikeFunctions)) functionReferences(tok); defineMapMethodLikes(tok); tok_applyMapMethodLikeFunctions(tok, mapMethodLikeFunctions); defineNuLikes(tok); tok_applyNuLikeFunctions(tok, nuLikeFunctions); tok_replaceWith(tok); tok_findRewrites(tok); tok_processRewrites(tok); try { if (safety == 0) tok = quickmain(tok); } catch (Throwable e) { printSources(tok); rethrow(e); } tok_collectMetaPostBlocks(tok, metaPostBlocks); // Hack to allow DynModule to reimplement _registerThread /*if (tok.contains("DynModule") && !addedClasses.contains("DynModule")) addStandardClasses_v2(tok);*/ defineExtraSF(tok); tok = standardFunctions(tok); tok = stdstuff(tok); // all the keywords, standard String diff; long startTime = now(); //diff = unidiff(before, join(tok)); //print("unidiff: " + (now()-startTime) + " ms"); //same = eq(diff, ""); same = tok_sameTest(tok, before); if (!same) { print("Not same " + safety + "."); //print(indent(2, diff)); } if (safety++ >= 10) { //print(unidiff(before, join(tok))); printSources(tok); throw fail("safety 10 error!"); } } while (!same); // END OF BIG LOOP print("Post."); if (mainBaseClass != null) { jreplace1(tok, "class main", "class main extends " + mainBaseClass); mainBaseClass = null; } print("moveImportsUp"); tok_moveImportsUp(tok); print("Indexing"); tok = indexedList2(tok); // POST-PROCESSING after stdstuff loop if (transpilingSnippetID != null) jreplace_dyn(tok, "class whatever", new F2, Integer, String>() { String get(List tok, Integer cIndex) { try { try { return "class " + stringToLegalIdentifier(getSnippetTitle(transpilingSnippetID)); } catch (Throwable __e) { _handleException(__e); } return "class Whatever"; } catch (Exception __e) { throw rethrow(__e); } } public String toString() { return "pcall { ret \"class \" + stringToLegalIdentifier(getSnippetTitle(transpilingSni..."; }}); //print("Type"); // Type to Type print("extendClasses"); tok = extendClasses(tok); print("libs"); libs(tok); print("sourceCodeLine"); sourceCodeLine(tok); tok_overridableFunctionDefs(tok, null); // escaping for lambdas //jreplace(tok, "-=>", "->"); // Stuff that depends on the list of inner classes (haveClasses) HashSet haveClasses = haveClasses_actual(tok); print("innerClassesVar"); innerClassesVar(tok, haveClasses); fillVar_transpilationDate(tok); haveClasses_addImported(tok, haveClasses); print("ifclass"); tok_ifclass(tok, haveClasses); print("slashCasts"); slashCasts(tok, haveClasses); print("newWithoutNew"); newWithoutNew(tok, haveClasses); if (!assumeTriple) { print("Triple"); if (tok.contains("Triple") && !haveClasses.contains("Triple")) { jreplace(tok, "Triple", "T3"); haveClasses.remove("Triple"); haveClasses.add("T3"); slashCasts(tok, lithashset("T3")); tok_quickInstanceOf(tok, lithashset("T3")); } expandTriple(tok); } if (hasDef("SymbolAsString")) jreplace(tok, "Symbol", "String"); print("classReferences"); expandClassReferences_lazy(tok, haveClasses); if (allowMetaCode || containsIC(definitions, "allowMetaCode")) runMetaPostBlocks(tok); // Error-checking print("Error-checking"); Set functions = new HashSet(findFunctions(tok)); for (String f : hardFunctionReferences) if (!functions.contains(f)) throw fail("Function " + f + " requested, but not supplied"); print("autoImports"); tok = autoImports(tok); // faster to do it at the end print("definitions=" + sfu(definitions)); if (containsOneOfIC(definitions, "allpublic", "reparse", "PublicExceptTopClass")) { // Fire up the Java parser & pretty printer! print(containsIC(definitions, "allpublic")? "Making all public." : "Reparsing."); //try { String src = join(tok); try { if (containsIC(definitions, "PublicExceptTopClass")) src = javaParser_makeAllPublic(src, "notTopLevelClassDecl" , true); else if (containsIC(definitions, "keepComments")) src = javaParser_makeAllPublic_keepComments(src); else if (containsIC(definitions, "allpublic")) src = javaParser_makeAllPublic(src); else src = javaParser_reparse_keepComments(src); } catch (Throwable e) { extractAndPrintJavaParseError(src, e); dontPrintSource = true; throw e; } tok = jtok(src); /*} catch e { S src = join(tok); if (!dontPrintSource) print(src); print(f2s(saveProgramTextFile("error.java", src))); throw rethrow(e); }*/ } // Do this after JavaParser (because it doesn't like package after class) if (mainPackage != null) { print("mainPackage"); tokPrepend(tok, 1, "package " + mainPackage + ";\n"); reTok(tok, 1, 2); } if (mainClassName != null) { print("mainClassName"); jreplace(tok, "class main", "class " + mainClassName); jreplace(tok, "main.class", mainClassName + ".class"); //tokPrepend(tok, 1, "class main {}\n"); // so main.class is generated and compiler sanity checks succeed. we can later skip it in the JavaXClassLoader } if (nempty(libs)) { print("Adding libs: " + libs); tok.add(concatMap_strings(new F1() { String get(String s) { try { return "\n!" + s; } catch (Exception __e) { throw rethrow(__e); } } public String toString() { return "\"\\n!\" + s"; }}, libs)); } } // if (!localStuffOnly) } catch (Throwable e) { String src = join(tok); if (!dontPrintSource) print(src); print(f2s(saveProgramTextFile("error.java", src))); throw rethrow(e); } /*if (useIndexedList) print("Indexed/unindexed lookups: " + findCodeTokens_indexed + "/" + findCodeTokens_unindexed + ", lists made: " + IndexedList2.instances); print("findCodeToken bails: " + findCodeTokens_bails + "/" + findCodeTokens_nonbails); print("javaToks: " + javaTok_n + "/" + javaTok_elements);*/ print("Saving."); // for dexcompile.php if (mainClassName != null) tokPrepend(tok, 0, "//FILENAME: " + (mainPackage != null ? mainPackage.replace(".", "/") + "/" : "") + mainClassName + ".java\n"); if (mainJava != null) mainJava = join(tok); else if (tok.contains("package")) splitJavaFiles(tok); else saveMainJava(tok); } catch (Exception __e) { throw rethrow(__e); } } static List localStuff1(List tok) { int safety = 0, i; boolean same; tok = indexedList2(tok); do { String before = join(tok); earlyStuff(tok); conceptDeclarations(tok); tok_recordDecls(tok); tok = multilineStrings(tok); tok_singleQuoteIdentifiersToStringConstants(tok); inStringEvals(tok); tok_listComprehensions(tok); tok_doubleFor_v2(tok); tok_forUnnull(tok); tok_ifCast(tok); forPing(tok); tok_directSnippetRefs(tok); quicknu(tok); //tok_juxtaposeCalls(tok); expandVarCopies(tok); jreplace(tok, "LS", "L"); jreplace(tok, "do ping {", "do { ping();"); replaceKeywordBlock(tok, "swing", "{ swing(r {", "}); }"); replaceKeywordBlock(tok, "afterwards", "temp tempAfterwards(r {", "});"); replaceKeywordBlock(tok, "tokcondition", "new TokCondition { bool get(final L tok, final int i) {", "}}"); jreplace(tok, "synced ", "synchronized $2"); replaceKeywordBlock(tok, "answer", "static S answer(S s) {\nfinal new Matches m;\n", "\nret null;\n}"); replaceKeywordBlock(tok, "static-pcall", "static { pcall {", "}}"); replaceKeywordBlock(tok, "loading", "{ JWindow _loading_window = showLoadingAnimation(); try {", "} finally { disposeWindow(_loading_window); }}"); replaceKeywordPlusQuotedBlock(tok, "loading", new Object() { String[] get(List tok, int i) { String text = tok.get(i+2); return new String[] { "{ JWindow _loading_window = showLoadingAnimation(" + text + "); try {", "} finally { disposeWindow(_loading_window); }}" }; }}); while ((i = jfind(tok, "visualize as")) >= 0) { int j = tok_findEndOfStatement(tok, i); // expression, rather tok.set(i+2, "{ ret"); tok.set(j-1, "; }"); reTok(tok, i, j); } jreplace(tok, "visualize {", "JComponent visualize() {", tokCondition_beginningOfMethodDeclaration()); jreplace(tok, "visualize2 {", "JComponent visualize2() {", tokCondition_beginningOfMethodDeclaration()); jreplace(tok, "start {", "void start() { super.start();", tokCondition_beginningOfMethodDeclaration()); replaceKeywordBlock(tok, "html", "static O html(S uri, fMap params) ctex " + "{\n", "}"); replaceKeywordBlock(tok, "afterVisualize", "visualize { JComponent _c = super.visualize();", "ret _c; }"); replaceKeywordBlock(tok, "enhanceFrame", "void enhanceFrame(Container f) { super.enhanceFrame(f);", "}"); if (assumeTriple) { jreplace(tok, "Triple", "T3"); expandTriple(tok); } tok_shortFinals(tok); tok_moduleClassDecls(tok); jreplace(tok, "static sync", "static synchronized"); jreplace(tok, "sclass", "static class"); jreplace(tok, "srecord", "static record"); jreplace(tok, "record noeq", "noeq record"); jreplace(tok, "asclass", "abstract static class"); jreplace(tok, "sinterface", "static interface"); jreplace(tok, "ssynchronized", "static synchronized"); jreplace(tok, "ssvoid", "static synchronized void"); jreplace(tok, "sbool", "static bool"); jreplace(tok, "fbool", "final bool"); jreplace(tok, "sint", "static int"); jreplace(tok, "snew", "static new"); jreplace(tok, "sv ", "static void $2"); jreplace(tok, "pvoid", "public void"); // "sS" => static S jreplace(tok, "sS", "static S"); // "sO" => static O jreplace(tok, "sO", "static O"); // "sL" => static L jreplace(tok, "sL", "static L"); // "toString {" => "public S toString() {" jreplace(tok, "toString {", "public S toString() {"); jreplace(tok, "Int", "Integer"); jreplace(tok, "Bool", "Boolean"); jreplace(tok, "BigInt", "BigInteger"); jreplace(tok, "Char", "Character"); jreplace(tok, "Sym", "Symbol"); jreplace(tok, "SymSym", "SymbolSymbol"); jreplace(tok, "SS", "Map"); jreplace(tok, "SymbolSymbol", "Map"); jreplace(tok, "PairS", "Pair"); jreplace(tok, "LPairS", "L>"); jreplace(tok, "T3S", "T3"); jreplace(tok, "ItIt", "IterableIterator"); jreplace(tok, "class > {", "class $2 extends $4 {"); jreplace(tok, "class > <> {", "class $2 extends $4 $5 $6 $7 {"); // "on fail {" => "catch (Throwable _e) { ... rethrow(_e); }" replaceKeywordBlock(tok, "on fail", "catch (Throwable _e) {", "\nthrow rethrow(_e); }"); // "catch {" => "catch (Throwable _e) {" jreplace(tok, "catch {", "catch (Throwable _e) {"); // "catch print e {" => "catch e { _handleException(e); " jreplace(tok, "catch print {", "catch $3 { _handleException($3);"); // "catch print short e {" => "catch e { printExceptionShort(e); " jreplace(tok, "catch print short {", "catch $4 { printExceptionShort($4);"); // "catch X e {" => "catch (X e) {" jreplace(tok, "catch {", "catch ($2 $3) {"); // "catch e {" => "catch (Throwable e) {" (if e is lowercase) jreplace(tok, "catch {", "catch (Throwable $2) {", new TokCondition() { boolean get(final List tok, final int i) { String word = tok.get(i+3); return startsWithLowerCaseOrUnderscore(word); }}); jreplace(tok, "+ +", "+", new TokCondition() { boolean get(final List tok, final int i) { //printStructure("++: ", subList(tok, i-1, i+6)); if (empty(_get(tok, i+2))) return false; // no space between the pluses if (empty(_get(tok, i)) && eq("+", _get(tok, i-1))) return false; // an actual "++" at the left if (empty(_get(tok, i+4)) && eq("+", _get(tok, i+5))) return false; // an actual "++" at the right //print("doing it"); return true; }}); // some crazy fancy syntax jreplace(tok, "set ;", "$2 = true;"); // single underscore (not allowed in Java anymore) to double underscore jreplace(tok, "_", "__"); // [stdEq] -> implementation of equals() and hashCode() jreplace(tok, "[stdEq]", "public bool equals(O o) { ret stdEq2(this, o); }\n" + "public int hashCode() { ret stdHash2(this); }"); // [concepts] "concept.field!" for dereferencing references jreplace(tok, "*!", "$1.get()", new TokCondition() { boolean get(final List tok, final int i) { String l = tok.get(i+1); if (!(isIdentifier(l) || eq(l, ")"))) return false; if (tok.get(i+2).contains("\n")) return false; // no line break between and ! if (nempty(tok.get(i+4))) return true; // space after = ok String t = _get(tok, i+5); if (t == null) return false; if (isIdentifier(t) || eqOneOf(t, "=", "(")) return false; return true; }}); // [concepts] "field := value" for defining fields e.g. in "uniq" while ((i = jfind(tok, " :=")) >= 0) { tok.set(i, quote(tok.get(i))); tok.set(i+2, ","); tok.set(i+4, ""); reTok(tok, i, i+5); } // "quoted" := value while ((i = jfind(tok, " :=")) >= 0) { tok.set(i, tok.get(i)); tok.set(i+2, ","); tok.set(i+4, ""); reTok(tok, i, i+5); } jreplace(tok, "for ( )", "for ($3 $4 : list($3))"); jreplace(tok, "for (final )", "for (final $4 $5 : list($4))"); // "continue unless" while ((i = jfind(tok, "continue unless")) >= 0) { int j = scanOverExpression(tok, getBracketMap(tok), i+4, ";"); replaceTokens(tok, i, i+4, "{ if (!("); tok.set(j, ")) continue; }"); reTok(tok, i, j+1); } // "continue if" while ((i = jfind(tok, "continue if")) >= 0) { int j = scanOverExpression(tok, getBracketMap(tok), i+4, ";"); replaceTokens(tok, i, i+4, "{ if ("); tok.set(j, ") continue; }"); reTok(tok, i, j+1); } // "return if" while ((i = jfind(tok, "return if")) >= 0) { int j = scanOverExpression(tok, getBracketMap(tok), i+4, ";"); replaceTokens(tok, i, i+4, "{ if ("); tok.set(j, ") return; }"); reTok(tok, i, j+1); } // "return unless" while ((i = jfind(tok, "return unless")) >= 0) { int j = scanOverExpression(tok, getBracketMap(tok), i+4, ";"); replaceTokens(tok, i, i+4, "{ if (!("); tok.set(j, ")) return; }"); reTok(tok, i, j+1); } // "return if" while ((i = jfind(tok, "return if")) >= 0) { int j = scanOverExpression(tok, getBracketMap(tok), i+4, ";"); tok.set(j, ") return " + tok.get(i+2) + "; }"); replaceTokens(tok, i, i+6, "{ if ("); reTok(tok, i, j+1); } // "return with " / "continue with " / "break with " while ((i = jfind(tok, " with", new TokCondition() { boolean get(final List tok, final int i) { return eqOneOf(tok.get(i+1), "return", "continue", "break"); }})) >= 0) { int j = scanOverExpression(tok, getBracketMap(tok), i+4, ";"); tok.set(j, "; " + tok.get(i) + "; }"); replaceTokens(tok, i, i+3, "{"); reTok(tok, i, j+1); } // return "bla" with while ((i = jfindOneOf(tok, "return with", "return with")) >= 0) { String result = tok.get(i+2); int j = scanOverExpression(tok, getBracketMap(tok), i+6, ";"); replaceTokens(tok, i, i+5, "{"); tok.set(j, "; return " + result + "; }"); reTok(tok, i, j+1); } tok_debugStatements(tok); // while not null () while ((i = jfind(tok, "while not null (")) >= 0) { int closingBracket = findEndOfBracketPart(tok, i+6)-1; replaceTokens(tok, i+2, i+6, "("); tok.set(closingBracket, ") != null)"); reTok(tok, i, closingBracket+1); } // Replace $1 with m.unq(0) etc. - caveat: this blocks identifiers $1, $2, ... for (i = 1; i < l(tok); i += 2) { String s = tok.get(i); if (s.startsWith("$")) { s = substring(s, 1); if (isInteger(s)) { tok.set(i, "m.unq(" + (parseInt(s)-1) + ")"); reTok(tok, i); } } } // instanceof trickery jreplace(tok, "is a ", "instanceof $3"); jreplace(tok, "! instanceof .", "!($2 instanceof $4.$6)"); jreplace(tok, "! instanceof ", "!($2 instanceof $4)"); jreplace(tok, " !instanceof ", "!($1 instanceof $4)"); // map func1 func2 func3(...) => mapFGH(f func1, f func2, f func3, ...) jreplace(tok, "map (", "mapFGH(f $2, f $3, f $4,"); // map func1 func2(...) => mapFG(f func1, f func2, ...) jreplace(tok, "map (", "mapFG(f $2, f $3,"); // "ref->bla" for dereferencing Concept.Ref or ThreadLocal or other // For lambdas, use SPACES on the left or right of the arrow! //jreplace(tok, " ->", "$1.get()."); jreplace(tok, "->", ".get().", new TokCondition() { boolean get(final List tok, final int i) { return empty(tok.get(i)) // no space on left of arrow && empty(tok.get(i+2)) // no space inside of arrow && empty(tok.get(i+4)) // no space on right of arrow && !eq(_get(tok, i-1), "-"); // i-->0; }}); // shortened subconcept declaration (before star constructors!) shortenedSubconcepts(tok); // "case" as a variable name ( => _case) caseAsVariableName(tok); // "do" as a function name ( => dO) tok_doAsMethodName(tok); // "continue" as a function name ( => _continue) continueAsFunctionName(tok); tok_extend(tok); jreplace(tok, "pn {", "p-noconsole {"); // Do these BEFORE awt replacement! ("p-awt" contains "awt" token) replaceKeywordBlock(tok, "r-awt", "r { awt {", "}}"); if (hasCodeTokens(tok, "p", "-")) tok_p_old(tok); replaceKeywordBlock(tok, "awt-messagebox", "awt { pcall-messagebox {", "}}"); replaceKeywordBlock(tok, "awt", "swingLater(r {", "});"); unswing(tok); lockBlocks(tok); tempBlocks(tok); tok_switchTo(tok); // trim x; jreplace(tok, "trim ;", "$2 = trim($2);"); // iterate with index jreplace (tok, "for over :", "for (int $2 = 0; $2 < l($4); $2++)"); jreplace (tok, "for backwards over :", "for (int $2 = l($5)-1; $2 >= 0; $2--)"); jreplace (tok, "for , over : {", "for (int $2 = 0; $2 < l($7); $2++) { $4 $5 = $7.get($2);"); jreplace (tok, "for , backwards over : {", "for (int $2 = l($8)-1; $2 >= 0; $2--) { $4 $5 = $8.get($2);"); jreplace (tok, "for to :", "for (int $2 = 0; $2 < $4; $2++)"); jreplace (tok, "for to :", "for (int $2 = 0; $2 < $4; $2++)"); tok = expandShortTypes(tok); tok_equalsCast(tok); replaceKeywordBlock(tok, "r-thread-messagebox", "r-thread { pcall-messagebox {", "}}"); replaceKeywordBlock(tok, "thread-messagebox", "thread { pcall-messagebox {", "}}"); jreplace(tok, "rThread {", "r-thread {"); replaceKeywordBlock(tok, "r-thread", "runnableThread(r {", "})"); rNamedThread(tok); // only works in the scope of a DynModule jreplace(tok, "rEnter {", "r { temp enter(); "); replaceKeywordBlock(tok, "r-messagebox", "r { pcall-messagebox {", "}}"); jreplace(tok, "r + r ", "r { $2(); $5(); }"); // runnable and r - now also with automatic toString if enabled for (String keyword : ll("runnable", "r")) { while ((i = jfind(tok, keyword + " {")) >= 0) { int idx = findCodeTokens(tok, i, false, "{"); int j = findEndOfBracketPart(tok, idx); List contents = subList(tok, idx+1, j-1); replaceTokens(tok, i, j+1, "new Runnable {" + " public void run() ctex { " + tok_addSemicolon(contents) + "\n}" + (autoQuine ? tok_autoQuineFunc(contents) : "") + "}"); reTok(tok, i, j+1); } while ((i = jfind(tok, keyword + " {")) >= 0) { int idx = findCodeTokens(tok, i, false, "{"); int j = findEndOfBracketPart(tok, idx); List contents = subList(tok, idx+1, j-1); replaceTokens(tok, i, j+1, "new Runnable {" + " public void run() ctex { " + tok_addSemicolon(contents) + "\n}" + " toString { ret " + tok.get(i+2) + "; }" + (autoQuine ? tok_autoQuineFunc(contents, "_shortenedSourceCode") : "") + "}"); reTok(tok, i, j+1); } } replaceKeywordBlock(tok, "expectException", "{ bool __ok = false; try {", "} catch { __ok = true; } assertTrue(\"expected exception\", __ok); }"); while ((i = tok.indexOf("tex")) >= 0) { tok.set(i, "throws Exception"); tok = jtok(tok); } // shorter & smarter whiles jreplace(tok, "while true", "while (true)"); jreplace(tok, "while licensed", "while (licensed())"); jreplace(tok, "repeat {", "while (licensed()) {"); tok_repeatWithSleep(tok); // null; => return null; etc. Object cond = new TokCondition() { boolean get(final List tok, final int i) { return tok_tokenBeforeLonelyReturnValue(tok, i-1); }}; jreplace(tok, "null;", "return null;", cond); jreplace(tok, "false;", "return false;", cond); jreplace(tok, "true;", "return true;", cond); jreplace(tok, "this;", "return this;", cond); // ok => ret "OK" with jreplace(tok, "ok ", "return \"OK\" with $2"); replaceKeywordBlock(tok, "ok", "{", " return \"OK\"; }"); // "myFunction;" instead of "myFunction();" - quite rough // (isolated identifier as function call) cond = new TokCondition() { boolean get(List tok, int i) { String word = tok.get(i+3); //print("single word: " + word); return !eqOneOf(word, "break", "continue", "return", "else", "endifdef", "endif"); } }; for (String pre : litlist("}", ";")) jreplace(tok, pre + " ;", "$1 $2();", cond); // shorter match syntax for answer methods jreplace(tok, "if || ", "if (matchOneOf(s, m, $2, $5))"); // "...bla..." jreplace(tok, "if ", "if (find3plusRestsX($2, s, m))", new TokCondition() { boolean get(final List tok, final int i) { return startsAndEndsWith(unquote(tok.get(i+3)), "..."); }}); // "bla..." jreplace(tok, "if ", "if (matchStartX($2, s, m))", new TokCondition() { boolean get(final List tok, final int i) { return unquote(tok.get(i+3)).endsWith("..."); }}); // "bla * bla | blubb * blubb" jreplace_dyn(tok, "if ", new F2, Integer, String>() { String get(List tok, Integer cIdx) { try { String s = unquote(tok.get(cIdx+2)); //print("multimatch: " + quote(s)); List l = new ArrayList(); for (String pat : splitAtJavaToken(s, "|")) { //print("multimatch part: " + quote(pat)); if (javaTok(pat).contains("*")) l.add("match(" + quote(trim(pat)) + ", s, m)"); else l.add("match(" + quote(trim(pat)) + ", s)"); } return "if (" + join(" || ", l) + ")"; } catch (Exception __e) { throw rethrow(__e); } } public String toString() { return "S s = unquote(tok.get(cIdx+2));\r\n //print(\"multimatch: \" + quote(s));\r\n ..."; }}, new TokCondition() { boolean get(final List tok, final int i) { return javaTokC(unquote(tok.get(i+3))).contains("|"); }}); // "bla" jreplace(tok, "if ", "if (match($2, s))", new TokCondition() { boolean get(final List tok, final int i) { return !javaTokC(unquote(tok.get(i+3))).contains("*"); }}); // "bla * bla" jreplace(tok, "if ", "if (match($2, s, m))"); jreplace(tok, "if match ", "if (match($3, s, m))"); // extra commas ("litlist(1, 2,)") jreplace(tok, ",)", ")"); // additional translations (if necessary) jreplace(tok, "pcall ping {", "pcall { ping();"); replaceKeywordBlock(tok, ") pcall", ") { pcall {", "}}"); replaceKeywordBlock(tok, "pcall", "try {", "} catch (Throwable __e) { _handleException(__e); }"); replaceKeywordBlock(tok, "pcall-short", "try {", "} catch (Throwable __e) { print(exceptionToStringShort(__e)); }"); replaceKeywordBlock(tok, "pcall-silent", "try {", "} catch (Throwable __e) { silentException(__e); }"); replaceKeywordBlock(tok, "pcall-messagebox", "try {", "} catch __e { messageBox(__e); }"); replaceKeywordBlock(tok, "pcall-infobox", "try {", "} catch __e { infoBox(__e); }"); tok = dialogHandler(tok); replaceKeywordBlock(tok, "exceptionToUser", "try {", "} catch (Throwable __e) { ret exceptionToUser(__e); }"); if (hasCodeTokens(tok, "twice", "{")) replaceKeywordBlock(tok, "twice", "for (int __twice = 0; __twice < 2; __twice++) {", "}"); while ((i = findCodeTokens(tok, "repeat", "*", "{")) >= 0) { String v = makeVar("repeat"); tok.set(i, "for (int " + v + " = 0; " + v + " < " + tok.get(i+2) + "; " + v + "++)"); tok.set(i+2, ""); reTok(tok, i, i+3); } while ((i = findCodeTokens(tok, "bench", "*", "{")) >= 0) { int j = findEndOfBracketPart(tok, i+4)-1; String time = makeVar("time"); String v = makeVar("bench"); String n = tok.get(i+2); tok.set(i, "{ long " + time + " = sysNow(); for (int " + v + " = 0; " + v + " < " + n + "; " + v + "++)"); tok.set(i+2, ""); tok.set(j, "} printBenchResult(sysNow()-" + time + ", " + n + "); }"); reTok(tok, i, j+1); } replaceKeywordBlockDyn(tok, "time", new Object() { String[] get() { String var = makeVar("startTime"); return new String[] { "{ long " + var + " = sysNow(); try { ", "} finally { " + var + " = sysNow()-" + var + "; saveTiming(" + var + "); } }"}; }}); // version without { } replaceKeywordBlockDyn(tok, "time2", new Object() { String[] get() { String var = makeVar("startTime"); return new String[] { "long " + var + " = sysNow(); ", " " + var + " = sysNow()-" + var + "; saveTiming(" + var + "); "}; }}); // time "bla" { replaceKeywordPlusQuotedBlock(tok, "time", new Object() { String[] get(List tok, int i) { String var = makeVar("startTime"); return new String[] { "long " + var + " = sysNow(); ", " done2_always(" + tok.get(i+2) + ", " + var + "); "}; }}); if (hasCodeTokens(tok, "assertFail", "{")) { String var = makeVar("oops"); replaceKeywordBlock(tok, "assertFail", "boolean " + var + " = false; try {", "\n" + var + " = true; } catch (Exception e) { /* ok */ } assertFalse(" + var + ");"); } replaceKeywordBlock(tok, "yo", "try {", "} catch (Exception " + makeVar("e") + ") { ret false; }", new TokCondition() { boolean get(final List tok, final int i) { return neqOneOf(_get(tok, i-1), "svoid", "void"); }}); replaceKeywordBlock(tok, "awtIfNecessary", "swingNowOrLater(r " + "{", "});"); ctex(tok); replaceKeywordBlock(tok, "actionListener", "new java.awt.event.ActionListener() { " + "public void actionPerformed(java.awt.event.ActionEvent _evt) { pcall-messagebox {", "}}}"); for (String keyword : ll("autocloseable", "autoCloseable")) /*replaceKeywordBlock(tok, keyword, "new AutoCloseable() { public void close() throws Exception {", "}}");*/ replaceKeywordBlock_dyn2_legacy(tok, keyword, new Object() { String[] get(List tok, int iOpening, int iClosing) { List contents = subList(tok, iOpening+1, iClosing); return new String[] { "new AutoCloseable() { toString { ret " + quote(shorten(maxQuineLength, trimJoin(contents))) + "; } public void close() throws Exception {", "}}" }; } }); namedThreads(tok); threads(tok); // try answer (string, test with nempty) while ((i = findCodeTokens(tok, "try", "answer")) >= 0) { int j = findEndOfStatement(tok, i); String v = makeVar("a"); tok.set(i, "{ S " + v); tok.set(i+2, "="); tok.set(j-1, "; if (!empty(" + v + ")) ret " + v + "; }"); reTok(tok, i, j); } // try bool[ean] (try answer with Bool type) while ((i = findCodeTokens(tok, "try", "boolean")) >= 0) { int j = findEndOfStatement(tok, i); String v = makeVar("a"); tok.set(i, "{ Bool " + v); tok.set(i+2, "="); tok.set(j-1, "; if (" + v + " != null) ret " + v + "; }"); reTok(tok, i, j); } // return optional (return if not null) while ((i = jfind(tok, "return optional =")) >= 0) { int j = findEndOfStatement(tok, i); String v = tok.get(i+4); clearTokens(tok, i+2, i+4); tok.set(i, "{"); tok.set(j-1, "; if (" + v + " != null) ret " + v + "; }"); reTok(tok, i, j); } // try object (return if not null) while ((i = jfind(tok, "try object")) >= 0) { int j = findEndOfStatement(tok, i); clearTokens(tok, i, i+3); boolean isDecl = isIdentifier(get(tok, i+4)) && isIdentifier(get(tok, i+6)) && eqGet(tok, i+8, "="); if (isDecl) { String v = get(tok, i+6); tok.set(i, "{"); tok.set(j-1, "; if (" + v + " != null) ret " + v + "; }"); } else { String v = makeVar(); tok.set(i, "{ O " + v + "="); tok.set(j-1, "; if (" + v + " != null) ret " + v + "; }"); } reTok(tok, i, j); } // debug print ...; => if (debug) print(...); while ((i = jfind(tok, "debug print")) >= 0) { int j = findEndOfStatement(tok, i); replaceTokens(tok, i, i+4, "if (debug) print("); tok.set(j-1, ");"); reTok(tok, i, j); } functionReferences(tok); tok_expandLPair(tok); quicknew2(tok); tok_unpair(tok); tok_cachedFunctions(tok); tok_simplyCachedFunctions(tok); throwFailEtc(tok); tok_typeAA(tok, pairClasses); // do this after expanding sclass etc. tok = expandStarConstructors(tok); tok_kiloConstants(tok); //tok_colonMessages(tok); while ((i = jfind(tok, "shit:")) >= 0) { int j = tok_findEndOfStatement(tok, i); tok.set(i+2, "("); tok.set(j-1, ");"); reTok(tok, i, j); } jreplace(tok, "shit(", "ret with print("); // references to objects in other realm (replace with Object) jreplace(tok, "virtual ", "Object"); tok_autoLongConstants(tok); // common misordering of type arguments jreplace(tok, "boolean ", " boolean"); tok_unimplementedMethods(tok); tok_switchableFields(tok); tok_shortVisualize(tok); tok_whileGreaterThan(tok); // end of local stuff same = tok_sameTest(tok, before); /*if (!same) print("local not same " + safety + " (" + l(tok) + " tokens)");*/ if (safety++ >= 10) { printSources(tok); throw fail("safety 10 error!"); } } while (!same); return tok; } static List reTok_include(List tok, int i, int j) { return reTok_modify(tok, i, j, "localStuff1"); } static List includeInMainLoaded_reTok(List tok, int i, int j) { return reTok_include(tok, i, j); } static List stdstuff(List tok) { //if (++level >= 10) fail("woot? 10"); print("stdstuff!"); int i; List ts = new ArrayList(); tok_findTranslators(tok, ts); if (nempty(ts)) print("DROPPING TRANSLATORS: " + structure(ts)); print("quickmain"); tok = quickmain(tok); print("includes"); tok = tok_processIncludes(tok); print("conceptsDot"); if (processConceptsDot(tok)) tok = tok_processIncludes(tok); //print('starConstructors); tok = expandStarConstructors(tok); // drop Java 8 annotations since we're compiling for Java 7 jreplace(tok, "@Nullable", ""); // STANDARD CLASSES & INTERFACES print("standard classes"); final Set haveClasses = addStandardClasses_v2(tok); tok_quickInstanceOf(tok, haveClasses); // concept-related stuff // auto-import concepts boolean _a = tok_hasClassRef2(tok, /*"extends",*/ "Concept") || tok_hasClassRef2(tok, "Concepts"), _b = !haveClasses.contains("Concept"); //print("auto-import: " + _a + ", " + _b); if (_a && _b) { print("Auto-including concepts."); if (shouldNotIncludeClass.contains("Concepts")) { print(join(tok)); throw fail("Unwanted concepts import"); } printStruct(haveClasses); tok = includeInMainLoaded(tok, "concepts."); reTok(tok, l(tok)-1, l(tok)); //processConceptsDot(tok); } return tok; } // end of stdStuff! static List multilineStrings(List tok) { for (int i = 1; i < tok.size(); i += 2) { String t = tok.get(i); if (isQuoted(t)) if (t.startsWith("[") || t.contains("\r") || t.contains("\n")) tok.set(i, quote(unquote(t))); } return tok; } static void inStringEvals(List tok) { boolean change = false; for (int i = 1; i < tok.size(); i += 2) { String t = tok.get(i); if (!isQuoted(t)) continue; if (t.contains("\\*") && !t.contains("\\\\")) { // << rough tok.set(i, inStringEval(t)); change = true; } } if (change) reTok(tok); } static String inStringEval(String t) { t = dropPrefix("\"", dropSuffix("\"", t)); List l = new ArrayList(); int idx; while ((idx = t.indexOf("\\*")) >= 0) { int j = indexOf(t, idx, "*/"); if (j < 0) break; if (idx > 0) l.add("\"" + substring(t, 0, idx) + "\""); l.add("(" + trim(substring(t, idx+2, j)) + ")"); t = substring(t, j+2); } if (nempty(t)) l.add("\"" + t + "\""); return "(" + join(" + ", l) + ")"; } static List quickmain(List tok) { if (quickmainDone1 && quickmainDone2) return tok; int i = findCodeTokens(tok, "main", "{"); if (i < 0) i = findCodeTokens(tok, "m", "{"); if (i >= 0 && !(i-2 > 0 && tok.get(i-2).equals("class"))) { tokSet(tok, i, "class main"); quickmainDone1 = true; } i = findCodeTokens(tok, "psvm", "{"); if (i < 0) i = findCodeTokens(tok, "p", "{"); if (i >= 0) { int idx = i+2; int j = findEndOfBracketPart(tok, idx)-1; List contents = subList(tok, idx+1, j); //print("contents: " + sfu(contents)); tok.set(i, "public static void main(final String[] args) throws Exception"); replaceTokens(tok, idx+1, j, tok_addSemicolon(contents)); reTok(tok, i, j); quickmainDone2 = true; } return tok; } static String makeVar(String name) { return "_" + name + "_" + varCount++; } static String makeVar() { return makeVar(""); } static List rtq(List tok, String id) { return runTranslatorQuick(tok, id); } static List expandShortTypes(List tok) { // replace with for (int i = 1; i+4 < tok.size(); i += 2) if (tok.get(i).equals("<") && litlist(">", ",").contains(tok.get(i+4))) { String type = tok.get(i+2); if (type.equals("int")) type = "Integer"; else if (type.equals("long")) type = "Long"; tok.set(i+2, type); } // O = Object, S = String, ret = return for (int i = 1; i < tok.size(); i += 2) { String t = tok.get(i); if (t.equals("O")) t = "Object"; if (t.equals("S")) t = "String"; else if (t.equals("L")) t = "List"; //else if (t.equals("F")) t = "Function"; else if (t.equals("ret") && neqOneOf(get(tok, i+2), "=", ")", ".")) t = "return"; else if (t.equals("bool") && i+2 < tok.size() && neq(tok.get(i+2), "(")) t = "boolean"; // bool -> boolean if it's not a function name tok.set(i, t); } jreplace(tok, "LL< >", "L>"); jreplace(tok, "LL", "L>"); jreplace(tok, "LL", "L"); jreplace(tok, "Clusters< >", "Map<$3, Collection<$3>>"); return tok; } static List autoImports(List tok) { HashSet imports = new HashSet(tok_findImports(tok)); StringBuilder buf = new StringBuilder(); for (String c : standardImports) if (!(imports.contains(c))) buf.append("import " + c + ";\n"); if (buf.length() == 0) return tok; tok.set(0, buf+tok.get(0)); return reTok(tok, 0, 1); } static String[] standardImports = { "java.util.*", "java.util.zip.*", "java.util.List", "java.util.regex.*", "java.util.concurrent.*", "java.util.concurrent.atomic.*", "java.util.concurrent.locks.*", "javax.swing.*", "javax.swing.event.*", "javax.swing.text.*", "javax.swing.table.*", "java.io.*", "java.net.*", "java.lang.reflect.*", "java.lang.ref.*", "java.lang.management.*", "java.security.*", "java.security.spec.*", "java.awt.*", "java.awt.event.*", "java.awt.image.*", "javax.imageio.*", "java.math.*" }; static List expandStarConstructors(List tok) { mainLoop: for (int i = 3; i+6 < tok.size(); i += 2) { String t = tok.get(i), l = tok.get(i-2); if (!t.equals("*")) continue; if (!tok.get(i+2).equals("(")) continue; if (!eqOneOf(l, "}", "public", "private", "protected", ";", "{", "endif") && neq(get(tok, i-4), "ifclass")) // is this correct...?? continue; // ok, it seems like a constructor declaration. // Now find class name by going backwards. int j = i, level = 1; while (j > 0 && level > 0) { t = tok.get(j); if (t.equals("}")) ++level; if (t.equals("{")) --level; j -= 2; } // search for class name while (j > 0) { t = tok.get(j); if (t.equals("class")) { String className = tok.get(j+2); tok.set(i, className); // exchange constructor name! // now for the parameters. // Syntax: *(Learner *learner) { // We will surely add type inference here in time... :) j = i+2; while (!tok.get(j).equals("{")) j += 2; int block = j+1; for (int k = i+2; k < block-1; k += 2) if (tok.get(k).equals("*")) { tok.remove(k); tok.remove(k); block -= 2; String name = tok.get(k); tok.addAll(block, Arrays.asList(new String[] { "\n ", "this", "", ".", "", name, " ", "=", " ", name, "", ";" })); } continue mainLoop; } j -= 2; } } return tok; } static List tok_processIncludes(List tok) { int safety = 0; while (hasCodeTokens(tok, "!", "include") && ++safety < 100) tok = tok_processIncludesSingle(tok); //tok_autoCloseBrackets(tok); return tok; } static List tok_processIncludesSingle(List tok) { int i; while ((i = jfind(tok, "!include #")) >= 0) { String id = tok.get(i+6); included.add(parseLong(id)); replaceTokens(tok, i, i+8, "\n" + cacheGet(id) + "\n"); reTok_include(tok, i, i+8); } while ((i = jfind(tok, "!include once #")) >= 0) { String id = tok.get(i+8); boolean isNew = included.add(parseLong(id)); replaceTokens(tok, i, i+10, isNew ? "\n" + cacheGet(id) + "\n" : ""); reTok_include(tok, i, i+10); } return tok; } static void ctex(List tok) { replaceKeywordBlock(tok, "ctex", "{ try {", "} catch (Exception __e) { throw rethrow(__e); } }"); for (String keyword : ll("null on exception", "null on error")) replaceKeywordBlock(tok, keyword, "{ try {", "} catch (Throwable __e) { return null; } }"); replaceKeywordBlock(tok, "false on exception", "{ try {", "} catch (Throwable __e) { return false; } }"); } static List dialogHandler(List tok) { return replaceKeywordBlock(tok, "dialogHandler", "new DialogHandler() {\n" + "public void run(final DialogIO io) {", "}}"); } static void quicknew2(List tok) { jreplace(tok, "new ;", "$2 $3 = new $2;"); jreplace(tok, "new <> ;", "$2<$4> $6 = new $2;"); jreplace(tok, "new . ;", "$2.$4 $5 = new $2.$4();"); jreplace(tok, "new <> , ;", "$2<$4> $6 = new $2, $8 = new $2;"); jreplace(tok, "new <,> ;", "$2<$4,$6> $8 = new $2;"); jreplace(tok, "new <<>> ;", "$2<$4<$6>> $9 = new $2;"); jreplace(tok, "new <[]> ;", "$2 $3 $4 $5 $6 $7 $8 = new $2;"); jreplace(tok, "new < <>, > ;", "$2 $3 $4 $5 $6 $7 $8 $9 $10 $11 = new $2;"); jreplace(tok, "new < <,> > ;", "$2 $3 $4 $5 $6 $7 $8 $9 $10 $11 = new $2;"); jreplace(tok, "new < , <> > ;", "$2 $3 $4 $5 $6 $7 $8 $9 $10 $11 = new $2;"); jreplace(tok, "new < <,,> > ;", "$2 $3 $4 $5 $6 $7 $8 $9 $10 $11 $12 $13 = new $2;"); jreplace(tok, "new < <,>, > ;", "$2 $3 $4 $5 $6 $7 $8 $9 $10 $11 $12 $13 = new $2;"); jreplace(tok, "new < , <,>> ;", "$2 $3 $4 $5 $6 $7 $8 $9 $10 $11 $12 $13 = new $2;"); jreplace(tok, "new < < , <,>>> ;", "$2 $3 $4 $5 $6 $7 $8 $9 $10 $11 $12 $13 $14 $15 $16 = new $2;"); jreplace(tok, "new < < ., <,>>> ;", "$2 $3 $4 $5 $6 $7 $8 $9 $10 $11 $12 $13 $14 $15 $16 $17 $18 = new $2;"); jreplace(tok, "new < < <>, <>>> ;", "$2 $3 $4 $5 $6 $7 $8 $9 $10 $11 $12 $13 $14 $15 $16 $17 = new $2;"); // [abandoned, confusing, looks like a function definition] with arguments - new A a(...); => A a = new A(...); //jreplace(tok, "new (", "$2 $3 = new $2("); jreplace(tok, "for args " + "{", "for (int i = 0; i < args.length; i++) { final String arg = args[i];"); // Constructor calls without parentheses // So you can say something like: predictors.add(new P1); jreplace1(tok, "new ", "new $2()", new TokCondition() { boolean get(final List tok, final int i) { return eqOneOf(_get(tok, i+5), "{", ",", ")", ";", ":"); }}); jreplace1(tok, "new < >", "new $2<$4>()", new TokCondition() { boolean get(final List tok, final int i) { return eqOneOf(_get(tok, i+11), "{", ",", ")", ";", ":"); }}); jreplace(tok, "new List(", "new ArrayList("); jreplace(tok, "new Map(", "new HashMap("); jreplace(tok, "new Set(", "new HashSet("); jreplace(tok, "new (Hash)Set", "new HashSet"); // rough jreplace(tok, "new (Tree)Set", "new TreeSet"); jreplace(tok, "new (Hash)Map", "new HashMap"); jreplace(tok, "new (Tree)Map", "new TreeMap"); jreplace(tok, "\\*[] ;", "$2[] $6 = new $2[$4];"); // X x = new(...) => X x = new X(...) // X x = new {...} => X x = new X {...} // X x = new => X x = new jreplace(tok, " = new", "$1 $2 = new $1", new TokCondition() { boolean get(final List tok, final int i) { return eqOneOf(_get(tok, i+9), "(", ";", ",", ")", "{"); }}); jreplace(tok, " = new \\*", "$1 $2 = new $1"); jreplace(tok, "\\* = new ", "$5 $2 = new $5"); // TODO: use backward type scanning jreplace(tok, "<> = new", "$1 $2 $3 $4 $5 = new $1", new TokCondition() { boolean get(final List tok, final int i) { return eqOneOf(_get(tok, i+9+3*2), "(", ";", ",", ")"); }}); jreplace(tok, "<,> = new", "$1 $2 $3 $4 $5 $6 $7 = new $1", new TokCondition() { boolean get(final List tok, final int i) { return eqOneOf(_get(tok, i+9+5*2), "(", ";", ",", ")"); }}); jreplace(tok, "<<>> = new", "$1 $2 $3 $4 $5 $6 $7 $8 = new $1", new TokCondition() { boolean get(final List tok, final int i) { return eqOneOf(_get(tok, i+9+6*2), "(", ";", ",", ")"); }}); } static List extendClasses(List tok) { int i; while ((i = jfind(tok, "extend {")) >= 0) { String className = tok.get(i+2); int idx = findCodeTokens(tok, i, false, "{"); int j = findEndOfBracketPart(tok, idx+2); String content = join(subList(tok, idx+1, j-1)); List c = findInnerClassOfMain(tok, className); print("Extending class " + className); clearTokens(tok.subList(i, j+1)); if (c == null) { print("Warning: Can't extend class " + className + ", not found"); continue; } int startOfClass = indexOfSubList(tok, c); // magicIndexOfSubList is broken int endOfClass = startOfClass + l(c)-1; //print("Extending class " + className + " ==> " + join(subList(tok, startOfClass, endOfClass))); while (neq(tok.get(endOfClass), "}")) --endOfClass; //print("Extending class " + className + " ==> " + join(subList(tok, startOfClass, endOfClass))); tok.set(endOfClass, content + "\n" + tok.get(endOfClass)); doubleReTok(tok, i, j+1, endOfClass, endOfClass+1); } return tok; } // for ping / while ping static void forPing(List tok) { int i; for (String keyword : ll("for", "while")) while ((i = jfind(tok, keyword + " ping (")) >= 0) { int bracketEnd = findEndOfBracketPart(tok, i+4)-1; int iStatement = bracketEnd+2; int iEnd = findEndOfStatement(tok, iStatement); tok.set(i+2, ""); // turn into block if (!eq(get(tok, iStatement), "{")) { tok.set(iStatement, "{ " + tok.get(iStatement)); tok.set(iEnd-1, tok.get(iEnd-1) + " }"); } // add ping tok.set(iStatement, "{ ping(); " + dropPrefixTrim("{", tok.get(iStatement))); reTok(tok, i+2, iEnd); } } // lib 123 => !123 static void libs(List tok) { int i; while ((i = jfind(tok, "lib ")) >= 0) { String id = tok.get(i+2); print("lib " + id); if (!libs.contains(id)) { libs.add(id); clearAllTokens(tok, i, i+3); /*tok.set(i, "!"); tok.set(i+1, "");*/ } else { print("...ignoring (duplicate)"); clearAllTokens(tok, i, i+3); reTok(tok, i, i+3); } } print("libs found: " + libs); } // sourceCodeLine() => 1234 static void sourceCodeLine(List tok) { int i ; while ((i = jfind(tok, "sourceCodeLine()")) >= 0) { replaceTokens(tok, i, i+5, str(countChar(join(subList(tok, 0, i)), '\n')+1)); reTok(tok, i, i+5); } } // done before any other processing static void earlyStuff(List tok) { int i; tok_scopes(tok); tok_autosemi(tok); tok_autoCloseBrackets(tok); jreplace(tok, "°", "()"); // Note: this makes the word "quine" a special operator // (unusable as a function name) while ((i = jfind(tok, "quine(")) >= 0) { int idx = findCodeTokens(tok, i, false, "("); int j = findEndOfBracketPart(tok, idx+2); tok.set(i, "new Quine"); tok.set(idx, "(" + quote(join(subList(tok, idx+1, j-1))) + ", "); reTok(tok, i, idx+1); } // do this before func & voidfunc because otherwise they swallow it jreplace(tok, "enter {", "{ temp enter();"); // func keyword for lambdas - now automatically quines toString() if enabled // do func & voidfunc early to preserve original code as toString while ((i = jfind(tok, "func(")) >= 0) { int argsFrom = i+4, argsTo = findCodeTokens(tok, i, false, ")"); int idx = findCodeTokens(tok, argsTo, false, "{"); int j = findEndOfBracketPart(tok, idx); List contents = subList(tok, idx+1, j-1); String returnType = "O"; if (eq(tok.get(argsTo+2), "-") && eq(tok.get(argsTo+4), ">")) returnType = tok_toNonPrimitiveTypes(join(subList(tok, argsTo+6, idx-1))); String toString = autoQuine ? " public S toString() { ret " + quote(shorten(maxQuineLength, trimJoin(contents))) + "; }" : ""; List args = cloneSubList(tok, argsFrom-1, argsTo); tok_shortFinals(args); tok_toNonPrimitiveTypes(args); List types = tok_typesOfParams(args); Object type = "O"; if (l(types) <= 3) type = "F" + l(types) + "<" + joinWithComma(types) + ", " + returnType + ">"; String body = tok_addReturn(contents); replaceTokens_reTok(tok, i, j, "new " + type + "() { " + returnType + " get(" + trimJoin(args) + ") ctex { " + body + " }\n" + toString + "}"); } while ((i = jfind(tok, "voidfunc(")) >= 0) { int argsFrom = i+4, argsTo = findCodeTokens(tok, i, false, ")"); int idx = findCodeTokens(tok, argsTo, false, "{"); int j = findEndOfBracketPart(tok, idx); List contents = subList(tok, idx+1, j-1); List args = cloneSubList(tok, argsFrom-1, argsTo); tok_shortFinals(args); tok_toNonPrimitiveTypes(args); List types = tok_typesOfParams(args); Object type = "O"; if (l(types) <= 4) type = "VF" + l(types) + "<" + joinWithComma(types) + ">"; replaceTokens(tok, i, j, "new " + type + "() { public void get(" + trimJoin(args) + ") ctex { " + tok_addSemicolon(contents) + " }\n" + (autoQuine ? " public S toString() { ret " + quote(shorten(maxQuineLength, trim(join(contents)))) + "; }" : "") + "}"); reTok(tok, i, j); } jreplace(tok, "func {", "func -> O {"); jreplace(tok, "f {", "f -> O {"); // swing -> S { ... } => swing(func -> S { ... }) while ((i = jfind(tok, "swing ->")) >= 0) { int bracket = findCodeTokens(tok, i, false, "{"); int j = findEndOfBracketPart(tok, bracket); tok.set(i, "swing(func"); tok.set(j-1, "})"); reTok(tok, i, j); } while ((i = jfind(tok, "void q {")) >= 0) { int bracket = findCodeTokens(tok, i, false, "{"); int j = findEndOfBracketPart(tok, bracket); clearTokens(tok, i+4, i+6); tok.set(bracket, "{ dm_q(r {"); tok.set(j-1, "}); }"); reTok(tok, i+4, j); } while ((i = jfind(tok, ") q {")) >= 0) { int opening = findBeginningOfBracketPart(tok, i); String fname = assertIdentifier(get(tok, opening-2)); int bracket = findCodeTokens(tok, i, false, "{"); int j = findEndOfBracketPart(tok, bracket); clearTokens(tok, i+2, i+4); tok.set(bracket, "{ dm_q(r {"); tok.set(j-1, "}); }"); reTok(tok, i+4, j); tok_makeArgumentsFinal(tok, opening, i); } for (String keyword : ll("f", "func")) { while ((i = jfind(tok, keyword + " ->", new TokCondition() { boolean get(final List tok, final int i) { return isIdentifier(_get(tok, i+7)); // avoid lambda declaration like: f -> { ... } }})) >= 0) { // I think there is a bug here for something like func -> x { new x { } } int idx = findCodeTokens(tok, i, false, "{"); int j = findEndOfBracketPart(tok, idx); String returnType = tok_toNonPrimitiveTypes(join(subList(tok, i+6, idx-1))); List contents = subList(tok, idx+1, j-1); replaceTokens(tok, i, j, "new F0<" + returnType + ">() { " + returnType + " get() ctex { " + tok_addReturn(contents) + " }\n" + (autoQuine ? " public S toString() { ret " + quote(shorten(maxQuineLength, trim(join(contents)))) + "; }" : "") + "}"); reTok(tok, i, j); } } } static void throwFailEtc(List tok) { boolean anyChange = false; for (int i = 1; i+4 < l(tok); i += 2) if (eqOneOf(get(tok, i+2), "fail", "todo") && eq(get(tok, i+4), "(") && !eqOneOf(get(tok, i), "throw", "RuntimeException", "return", "f", "function")) { tok.set(i+2, "throw " + tok.get(i+2)); anyChange = true; } if (anyChange) reTok(tok); } static void namedThreads(List tok) { for (int i = 0; i < 100; i++) { int idx = findCodeTokens(tok, "thread", "", "{"); if (idx < 0) idx = findCodeTokens(tok, "thread", "", "{"); if (idx < 0) break; int j = findEndOfBracketPart(tok, idx+4); String tName = tok.get(idx+2); String pre = "startThread(" + tName + ", r { "; String post = "})"; if (!tok_tokenLeftOfExpression(tok, idx-2)) post += ";"; tok.set(idx, pre); tok.set(idx+2, ""); tok.set(idx+4, ""); tok.set(j-1, post); //print(join(subList(tok, idx, j))); reTok(tok, idx, j); } } static void rNamedThread(List tok) { for (int i = 0; i < 100; i++) { int idx = findCodeTokens(tok, "r", "-", "thread", "", "{"); if (idx < 0) idx = findCodeTokens(tok, "r", "-", "thread", "", "{"); if (idx < 0) break; int j = findEndOfBracketPart(tok, idx+8); String tName = tok.get(idx+6); String pre = "r { thread " + tName + " {"; String post = "}}"; replaceTokens(tok, idx, idx+9, pre); tok.set(j-1, post); reTok(tok, idx, j); } } static void threads(List tok) { replaceKeywordBlock(tok, "daemon", "startDaemonThread(r {", "});"); //replaceKeywordBlock(tok, "thread", "startThread(r {", "});"); replaceKeywordBlock_dyn2_legacy(tok, "thread", new Object() { String[] get(List tok, int iOpening, int iClosing) { return new String[] {"startThread(r {", "})" + (tok_tokenLeftOfExpression(tok, iOpening-4) ? "" : ";")}; } }); } static Map sf; // standard standard functions (haha) static Boolean _761ok; static List standardFunctions(List tok) { if (sf == null) { String _761 = cacheGet("#761"); if (_761ok == null) _761ok = isBracketHygienic(_761); assertTrue("Whoa! #761 truncated?", _761ok); List standardFunctions = concatLists( (List) loadVariableDefinition(_761, "standardFunctions"), (List) loadVariableDefinition(cacheGet("#1006654"), "standardFunctions")); sf = new HashMap(); for (String x : standardFunctions) { String[] f = x.split("/"); sf.put(f[1], f[0]); } } Map sfMap = combinedMap(extraStandardFunctions, sf); for (int i = 0; ; i++) { Set defd = new HashSet(findFunctions(tok)); int j; while ((j = jfind(tok, "should not include function *.")) >= 0) { String fname = tok.get(j+8); shouldNotIncludeFunction.add(fname); clearAllTokens(tok.subList(j, j+12)); } while ((j = jfind(tok, "do not include function *.")) >= 0) { String fname = tok.get(j+8); doNotIncludeFunction.add(fname); clearAllTokens(tok.subList(j, j+12)); } doNotIncludeFunction.addAll(tok_importedStaticFunctionNames(tok)); // changes tok Set invocations = findFunctionInvocations(tok, sfMap, hardFunctionReferences, defd, true); /*if (invocations.contains("str")) print("==STR==" + join(tok) + "==STR==");*/ if (!cic(definitions, "leanMode")) invocations.addAll(functionsToAlwaysInclude); //print("Functions invoked: " + structure(invocations)); List needed = diff(invocations, defd); for (String f : doNotIncludeFunction) needed.remove(f); if (needed.isEmpty()) break; print("Adding functions: " + join(" " , needed)); HashSet neededSet = new HashSet(needed); Collection bad = setIntersection(neededSet, shouldNotIncludeFunction); if (nempty(bad)) { String msg = "INCLUDING BAD FUNCTIONS: " + sfu(bad); print(msg); print(join(tok)); throw fail(msg); } List added = new ArrayList(); List < Future < String > > parts = new ArrayList(); List preload = new ArrayList(); for (String x : needed) if (sfMap.containsKey(x)) preload.add(sfMap.get(x)); print("Preloading: " + preload); cachePreload(preload); for (String x : cloneList(needed)) { if (defd.contains(x)) continue; String id = sfMap.get(x); if (id == null) { print("Standard function " + x + " not found."); needed.remove(x); continue; } print("Adding function: " + x + " (" + id + ")"); String function = cacheGet(id); //if (("\n" + function).contains("\n!")) print("Warning: " + id + " contains translators."); if (cacheStdFunctions) { long _id = psI(id); CachedInclude ci = cachedIncludes.get(_id); if (ci == null) { cachedIncludes.put(_id, ci = new CachedInclude()); //println("Caching include " + _id + ", l=" + l(cachedIncludes)); } function += "\n"; final String _function = function; if (neq(ci.javax, function)) { print("Compiling function: " + x); ci.javax = function; ci.java = executor.submit(new Callable() { public String call() { return join(localStuff1(jtok(_function))); } }); ci.realJava = null; } parts.add(ci.javaFuture()); } else parts.add(nowFuture(function + "\n")); added.add(x); Collection newFunctions = new HashSet(findFunctionDefs(javaTok(function))); defd.addAll(newFunctions); for (String f : newFunctions) if (!addedFunctions.add(f)) { printSources(tok); throw fail("Trying to add function " + f + " again - main class syntax broken!"); } } String text = join(getAllFutures(parts)); if (cacheStdFunctions) tok = includeInMainLoaded_stdReTok(tok, text); else tok = includeInMainLoaded(tok, text); // defd = new HashSet(findFunctions(tok)); //print("Functions added: " + joinWithSpace(added)); for (String x : needed) if (!defd.contains(x)) { print(join(tok)); throw fail("Function not defined properly: " + x); } //print("Iteration " + (i+2)); tok_definitions(tok); tok_ifndef(tok); tok_ifdef(tok); if (i >= 1000) throw fail("Too many iterations"); } return tok; } static List findFunctions(List tok) { //ret findFunctionDefinitions(join(findMainClass(tok))); return findFunctionDefs(findMainClass(tok)); } static String cacheGet(String snippetID) { snippetID = formatSnippetID(snippetID); String text = snippetCache.get(snippetID); if (text == null) { snippetCache.put(snippetID, text = loadSnippet(snippetID)); if (hasUnclosedStringLiterals(text)) throw fail("Unclosed string literals in " + snippetID); } return text; } static void cachePreload(Collection ids) { List needed = new ArrayList(); for (String id : ids) if (!snippetCache.containsKey(formatSnippetID(id))) needed.add(formatSnippetID(id)); if (l(needed) > 1) { List texts = loadSnippets(needed); for (int i = 0; i < l(needed); i++) if (texts.get(i) != null) snippetCache.put(needed.get(i), texts.get(i)); } } static List jtok(List tok) { return jtok(join(tok)); } static List jtok(String s) { List l = javaTok(s); return useIndexedList ? new IndexedList2(l) : l; } static HashSet haveClasses_actual(List tok) { HashSet have = new HashSet(); for (List c : innerClassesOfMain(tok)) have.add(getClassDeclarationName(c)); return have; } static HashSet haveClasses_addImported(List tok, HashSet have) { have.addAll(tok_importedClassNames(tok)); have.addAll(usualJavaClassNames()); // for S => S.class return have; } // works on Java level (no "sclass" etc) // returns list of classes we have (useful for other processing) static Set addStandardClasses_v2(List tok) { if (lclasses == null) { String sc = cacheGet("#1003674"); lclasses = new ArrayList(); for (String line : tlft_j(sc)) { int idx = line.indexOf('/'); lclasses.addAll(ll(line.substring(0, idx), line.substring(idx+1))); } } List definitions = lclasses; for (int safety = 0; safety < 10; safety++) { HashSet have = haveClasses_actual(tok); have.addAll(tok_importedClassNames(tok)); int j; while ((j = jfind(tok, "should not include class *.")) >= 0) { String cname = tok.get(j+8); shouldNotIncludeClass.add(cname); clearAllTokens(tok.subList(j, j+12)); } Map need = new HashMap(); Set idx = tokenIndexWithoutIfclass_forStdClasses(tok); while ((j = jfind(tok, "please include class *.")) >= 0) { String cname = tok.get(j+6); idx.add(cname); clearAllTokens(tok.subList(j, j+10)); } for (int i = 0; i+1 < l(definitions); i += 2) { String className = definitions.get(i); if (idx.contains(className) && !have.contains(className)) need.put(className, definitions.get(i+1)); } if (hasDef("SymbolAsString")) { print("Have SymbolAsString."); if (need.containsKey("Symbol")) { print("Have Symbol."); need.remove("Symbol"); } } else print("No SymbolAsString."); if (empty(need)) return have; for (String className : keys(need)) if (shouldNotIncludeClass.contains(className)) { String msg = "INCLUDING BAD CLASS: " + className; print(msg); print(join(tok)); throw fail(msg); } cachePreload(values(need)); StringBuilder buf = new StringBuilder(); print("Adding classes: " + joinWithSpace(keys(need))); for (String className : keys(need)) { if (have.contains(className)) continue; // intermittent add String snippetID = need.get(className); //print("Adding class " + className + " / " + snippetID); snippetID = fsI(snippetID); String text = cacheGet(snippetID); assertTrue(cacheStdClasses); long _id = psI(snippetID); CachedInclude ci = cachedIncludes.get(_id); if (ci == null) cachedIncludes.put(_id, ci = new CachedInclude()); if (neq(ci.javax, text)) { print("Compiling class: " + className); ci.javax = text; ci.java = null; ci.realJava = join(localStuff1(jtok(text))); } buf.append(ci.java()); List ct = javaTok(ci.java()); for (List c : allClasses(ct)) { String name = getClassDeclarationName(c); have.add(name); } if (!have.contains(className)) throw fail("Wrongly defined class: " + className + " / " + snippetID); if (!addedClasses.add(className)) { printSources(tok); throw fail("Trying to add class " + className + " again - main class syntax broken!"); } } // for className tok = includeInMainLoaded_stdReTok(tok, str(buf)); } throw fail("safety 10"); } static Set expandableClassNames = lithashset("BigInteger"); // no reTok - leaves tok dirty // magically append ".class" to class name references static boolean expandClassReferences_lazy(List tok, Set classNames) { boolean change = false; for (int i = 3; i+2 < l(tok); i += 2) { String t = tok.get(i); // skip implements/extends/throws lists if (eqOneOf(t, "implements", "extends", "throws")) { i = tok_endOfImplementsList(tok, i); continue; } if (classNames.contains(t) || expandableClassNames.contains(t)) { String s = tok.get(i-2); t = tok.get(i+2); // TODO: This whole logic ain't very good // (Hard to distinguish between "Int.class" as an argument // and "Int" as a type parameter.) if (eqOneOf(s, "instanceof", "new", ".", "<", ">", "/", "nu")) continue; if (isIdentifier(s) || isInteger(s)) continue; if (eq(t, ",") && isIdentifier(get(tok, i+4)) && eqGet(tok, i+6, ">")) continue; // e.g. T3, S, S> if (eq(s, ",") && isIdentifier(get(tok, i-4)) && eqGet(tok, i-6, "<")) continue; // e.g. T3 if (eq(s, ",") && eqOneOf(_get(tok, i-6), "implements", "throws")) continue; // check for cast if (eq(s, "(") && eq(t, ")") && i >= 5) { if (!eqOneOf(get(tok, i+4), "{", ";")) { String x = tok.get(i-4); if (!isIdentifier(x)) continue; if (eqOneOf(x, "ret", "return")) continue; } } if (eqOneOf(t, ",", ")", ";", ":")) { tok.set(i, tok.get(i) + ".class"); change = true; } } } return change; } static void expandClassReferences(List tok, Set classNames) { if (expandClassReferences_lazy(tok, classNames)) reTok(tok); } // "/" => "((ClassName) )" static void slashCasts(List tok, final Set classNames) { /*jreplace(tok, "/", "(($3) $1)", tokcondition { ret classNames.contains(tok.get(i+5)); });*/ int n = l(tok)-4; for (int i = 1; i < n; i += 2) if (tok.get(i+2).equals("/") && isIdentifier(tok.get(i)) && classNames.contains(tok.get(i+4))) replaceTokens_reTok(tok, i, i+5, "((" + tok.get(i+4) + ") " + tok.get(i) + ")"); } // "(...)" => "new (...)" // doesn't work at beginning of statement as we can't easily // distinguish it from a constructor declaration. static void newWithoutNew(List tok, final Set classNames) { TokCondition cond = newWithoutNew_condition(classNames); jreplace(tok, "(", "new $1(", cond); // just one case with type arg for now jreplace(tok, "<>(", "new $1<$3>(", cond); } static TokCondition newWithoutNew_condition(final Set classNames) { return new TokCondition() { boolean get(final List tok, final int i) { if (!classNames.contains(tok.get(i+1))) return false; boolean ok = neqOneOf(_get(tok, i-1), "new", ";", "}", "{", "public", "protected", "private", "."); //print("newWithoutNew: checking " + struct(subList(tok, i-1, i+2)) + " => " + ok); return ok; }}; } // +var => "var", +var static void expandVarCopies(List tok) { boolean change = false; for (int i = 3; i+2 < l(tok); i += 2) { if (!eq(tok.get(i), "+")) continue; if (!eqOneOf(tok.get(i-2), "(", ",", "{")) continue; String s = tok.get(i+2); if (!isIdentifier(s)) continue; tok.set(i, quote(s) + ", "); change = true; } if (change) reTok(tok); } static boolean processConceptsDot(List tok) { boolean anyChange = false, change; do { change = false; for (int i : jfindAll(tok, "concepts.")) if (contains(get(tok, i+3), "\n")) { replaceTokens(tok, i, i+3, "!" + "include once #1004863 // Dynamic Concepts"); reTok(tok, i, i+3); change = anyChange = true; break; } } while (change); return anyChange; } static void caseAsVariableName(List tok) { if (!tok.contains("case")) return; for (int i = 1; i+2 < l(tok); i += 2) { String t = tok.get(i+2); if (tok.get(i).equals("case") && !(t.startsWith("'") || isInteger(t) || isIdentifier(t))) tok.set(i, "_case"); } } static void continueAsFunctionName(List tok) { jreplace(tok, "continue(", "_continue("); } // f bla => "bla" - and "please include function bla." static void functionReferences(List tok) { int i; jreplace_dyn(tok, "f-thread ", new F2, Integer, Object>() { Object get(List tok, Integer cIdx) { try { return "dynamicCallableMC_thread(" + quote(tok.get(cIdx+6)) + ")"; } catch (Exception __e) { throw rethrow(__e); } } public String toString() { return "\"dynamicCallableMC_thread(\" + quote(tok.get(cIdx+6)) + \")\""; }}); String keyword = "f"; while ((i = jfind(tok, keyword + " ", new TokCondition() { boolean get(final List tok, final int i) { return !eq(tok.get(i+3), "instanceof"); }})) >= 0) { String f = tok.get(i+2); clearTokens(tok, i, i+2); tok.set(i+2, quote(f)); reTok(tok, i, i+2); tok.set(l(tok)-1, last(tok) + "\nplease include function " + f + "."); reTok(tok, l(tok)-1, l(tok)); } // r fname => r { fname() } // rThread fname => rThread { fname() } while ((i = jfindOneOf_cond(tok, new TokCondition() { boolean get(final List tok, final int i) { return !eq(tok.get(i+3), "instanceof"); }}, "r ", "rThread ", "rEnter ")) >= 0) { String f = tok.get(i+2); replaceTokens(tok, i, i+3, tok.get(i) + " { " + f + "(); }"); reTok(tok, i, i+3); } // dm_q fname => r_dm_q(r fname) jreplace(tok, "dm_q ", "r_dm_q(r $2)"); // vf fname => voidfunc(S s) { fname(s) } jreplace(tok, "vf<> ", "voidfunc($3 a) { $5(a) }"); // construct Entry => func(S s) -> Entry { new Entry(s) } jreplace(tok, "construct<> ", "func($3 a) -> $5 { new $5(a) }"); // f fname => func -> S { fname() } jreplace(tok, "f<> ", "func -> $3 { $5() }"); // f fname => func(S x) -> S { fname(x) } jreplace(tok, "f<, > ", "func($3 x) -> $5 { $7(x) }"); // f> fname => func(S x) -> L { fname(x) } jreplace(tok, "f<, <>> ", "func($3 x) -> $5 $6 $7 $8 { $10(x) }"); // f, S> fname => func(S x, L y) -> S { fname(x, y) } jreplace(tok, "f<, <>, > ", "func($3 x, $5 $6 $7 $8 y) -> $10 { $12(x, y) }"); } static void quicknu(List tok) { jreplace(tok, "nu (", "nu($2.class, "); jreplace(tok, "nu ", "new $2"); } // fill variable innerClasses_list static void innerClassesVar(List tok, Set have) { if (!tok.contains("myInnerClasses_list")) return; int i = jfind(tok, ">myInnerClasses_list;"); if (i < 0) return; tok.set(i+4, "=litlist(\n" + joinQuoted(", ", have) + ");"); reTok(tok, i+4, i+5); } // fill variable innerClasses_list static void fillVar_transpilationDate(List tok) { if (!tok.contains("myTranspilationDate_value")) return; int i = jfind(tok, "long myTranspilationDate_value;"); if (i < 0) return; tok.set(i+4, " = " + now() + "L;"); reTok(tok, i+4, i+5); } // process ifclass x ... endif blocks static void tok_ifclass(List tok, Set have) { if (!tok.contains("ifclass")) return; int i; while ((i = rjfind(tok, "ifclass ")) >= 0) { int j = jfind(tok, i+4, "endif"); if (j < 0) j = l(tok)-1; String name = tok.get(i+2); boolean has = have.contains(name); //print("ifclass " + name + " => " + has); clearTokens(tok, i, i+3); clearTokens(tok, j, j+1); if (!has) clearTokens(tok, i+3, j); reTok(tok, i, j+1); } } // set flag *. static void tok_definitions(List tok) { int i; while ((i = jfind(tok, "set flag .")) >= 0) { String fname = tok.get(i+4); print("Setting flag " + fname); definitions.add(fname); clearAllTokens(tok.subList(i, i+8)); } while ((i = jfind(tok, "unset flag .")) >= 0) { String fname = tok.get(i+4); print("Unsetting flag " + fname); definitions.remove(fname); clearAllTokens(tok.subList(i, i+8)); } } static void tok_findRewrites(List tok) { int i; while ((i = jfind(tok, "rewrite =")) >= 0) { String token = tok.get(i+2); int repStart = i+6; int repEnd = smartIndexOf(tok, repStart, "."); String replacement = joinSubList(tok, repStart, repEnd-1); clearTokens(tok, i, repEnd+1); rewrites.put(token, replacement); print("Have rewrite: " + token + " => " + replacement); } } static void tok_processRewrites(List tok) { for (String token : keys(rewrites)) jreplace(tok, token, rewrites.get(token)); } // extend *. (set base class of main class) static void tok_extend(List tok) { int i; while ((i = jfind(tok, "extend .")) >= 0) { mainBaseClass = tok.get(i+2); clearAllTokens(tok, i, i+7); } } // process ifndef x ... endifndef blocks static void tok_ifndef(List tok) { if (!tok.contains("ifndef")) return; int i; while ((i = rjfind(tok, "ifndef ")) >= 0) { int j = jfind(tok, i+4, "endifndef"); if (j < 0) j = l(tok)-1; String fname = tok.get(i+2); boolean has = !definitions.contains(fname); //print("ifndef " + fname + " => " + has); clearTokens(tok, i, i+3); clearTokens(tok, j, j+1); if (!has) clearTokens(tok, i+3, j); reTok(tok, i, j+1); } } // process ifdef x ... endifdef blocks static void tok_ifdef(List tok) { if (!tok.contains("ifdef")) return; int i; while ((i = rjfind(tok, "ifdef ")) >= 0) { int j = jfind(tok, i+4, "endifdef"); if (j < 0) j = l(tok)-1; String fname = tok.get(i+2); boolean has = definitions.contains(fname); //print("ifdef " + fname + " => " + has); clearTokens(tok, i, i+3); clearTokens(tok, j, j+1); if (!has) clearTokens(tok, i+3, j); reTok(tok, i, j+1); } } static void conceptDeclarations(List tok) { for (String kw : ll("concept", "sconcept")) { Object cond = new TokCondition() { boolean get(final List tok, final int i) { tok_addFieldOrder(tok, i+1); return true; }}; boolean re = false; if (jreplace(tok, kw + " {", "static class $2 extends Concept {", cond)) re = true; if (jreplace(tok, kw + " implements", "static class $2 extends Concept implements", cond)) re = true; if (jreplace(tok, kw + " ", "static class $2", cond)) re = true; if (re) reTok(tok); } } static void shortenedSubconcepts(List tok) { jreplace(tok, " > {", "concept $3 extends $1 {", new TokCondition() { boolean get(final List tok, final int i) { boolean b = (i == 0 || tok.get(i).contains("\n")) || eq(_get(tok, i-1), "abstract"); // only at beginning of line or after "abstract" //print("subconcept " + b + ": " + structure(subList(tok, i-1, i+5))); return b; }}); } // -slightly experimental // -do calculation in another thread, then return to AWT thread // -must be placed in a block // -transforms rest of block static void unswing(List tok) { int i; while ((i = jfind(tok, "unswing {")) >= 0) { int idx = i+2; int closingBracket = findEndOfBracketPart(tok, idx)-1; int endOfOuterBlock = findEndOfBracketPart(tok, closingBracket)-1; tok.set(i, "thread"); tok.set(closingBracket, " awt {"); tok.set(endOfOuterBlock, "}}}"); reTok(tok, closingBracket-1, endOfOuterBlock+1); } } // -Syntax: lock theLock; // -lock a lock, unlock at end of current block with finally static void lockBlocks(List tok) { int i; while ((i = jfind(tok, "lock ", new TokCondition() { boolean get(final List tok, final int i) { return neq(tok.get(i+3), "instanceof"); }})) >= 0) { int semicolon = findEndOfStatement(tok, i)-1; String var = makeVar(); int endOfOuterBlock = findEndOfBracketPart(tok, semicolon)-1; replaceTokens(tok, i, semicolon+1, "Lock " + var + " = " + joinSubList(tok, i+2, semicolon-1) + "; lock(" + var + "); try {"); tok.set(endOfOuterBlock, "} finally { unlock(" + var + "); } }"); reTok(tok, i, endOfOuterBlock+1); } } // -Syntax: temp Bla bla = bla(); // -expands to try(Bla bla = bla()) { ... } with rest of block inside static void tempBlocks(List tok) { int i; while ((i = jfind(tok, "temp ")) >= 0) { int semicolon = findEndOfStatement(tok, i)-1; int endOfOuterBlock = findEndOfBracketPart(tok, semicolon)-1; List sub = subList(tok, i-1, semicolon); int eq = subList(sub, 0, smartIndexOf(sub, "{")).indexOf("="); String var; if (eq >= 0) var = sub.get(eq-2); else { // no var name, e.g. temp newThoughtSpace(); var = makeVar(); tok.set(i+2, "AutoCloseable " + var + " = " + tok.get(i+2)); } //tok.set(i, "try ("); //tok.set(semicolon, ") {"; //tok.set(endOfOuterBlock, "}}"); tok.set(i, ""); tok.set(semicolon, "; try {"); tok.set(endOfOuterBlock, "} finally { _close(" + var + "); }}"); reTok(tok, i, endOfOuterBlock+1); } } static void cleanMeUp() { for (CachedInclude ci : values(cachedIncludes)) ci.clean(); vmKeepWithProgramMD5_save("cachedIncludes"); } static void printSources(List tok) { print("----"); print(join(tok)); print("----"); } static void tok_quickInstanceOf(List tok, final Set haveClasses) { // "x << X" or "x >> X" => "x instanceof X" for (String op : ll("<<", ">>")) jreplace(tok, " " + op + " ", "$1 instanceof $4", new TokCondition() { boolean get(final List tok, final int i) { return haveClasses.contains(tok.get(i+7)) && !eqOneOf(tok.get(i-1), "<", "extends", "implements"); }}); } static boolean hasDef(String s) { return definitions.contains(s); } static void expandTriple(List tok) { jreplace(tok, "T3< >", "T3<$3, $3, $3>"); jreplace(tok, "T3< <> >", "T3<$3<$5>, $3<$5>, $3<$5>>"); } static void tok_shortFinals(List tok) { jreplace(tok, "fS", "final S"); jreplace(tok, "fO", "final O"); jreplace(tok, "fL", "final L"); jreplace(tok, "fMap", "final Map"); jreplace(tok, "fRunnable", "final Runnable"); jreplace(tok, "f int", "final int"); } static void tok_mainClassNameAndPackage(List tok) { int i; if ((i = jfind(tok, "mainClassName ")) >= 0) { mainClassName = tok.get(i+2); clearTokens(tok, i, i+3); reTok(tok, i, i+3); } if ((i = jfind(tok, "mainPackage ")) >= 0) { mainPackage = tok.get(i+2); clearTokens(tok, i, i+3); reTok(tok, i, i+3); } } static void defineMapLikes(List tok) { int i; while ((i = jfind(tok, "mapLike ")) >= 0) { mapLikeFunctions.add(tok.get(i+2)); clearTokens_reTok(tok, i, i+2); } } static void defineMapMethodLikes(List tok) { int i; while ((i = jfind(tok, "mapMethodLike ")) >= 0) { mapMethodLikeFunctions.add(tok.get(i+2)); clearTokens_reTok(tok, i, i+2); } } // functions that work like "nu" syntactically (accept a class as "super-first" parameter [left of the bracket]) static void defineNuLikes(List tok) { int i; while ((i = jfind(tok, "nuLike ")) >= 0) { nuLikeFunctions.add(tok.get(i+2)); clearTokens_reTok(tok, i, i+2); } } static void defineExtraSF(List tok) { int i; IntRange reTok = null; while ((i = jfind(tok, "function is in *.")) >= 0) { extraStandardFunctions.put(tok.get(i+2), fsI(unquote(tok.get(i+8)))); clearTokens(tok, i, i+12); reTok = combineIntRanges(reTok, intRange(i, i+12)); } reTok(tok, reTok); } static boolean tok_applyMapLikeFunctions(List tok, final Set mapLikeFunctions) { // map funcname(...) => map(f funcname, ...) // filter funcname(...) => filter(f funcname, ...) // ... return jreplace(tok, " (", "$1(f $2,", new F2, Integer, Boolean>() { Boolean get(List tok, Integer i) { try { return contains(mapLikeFunctions, tok.get(i+1)); } catch (Exception __e) { throw rethrow(__e); } } public String toString() { return "contains(mapLikeFunctions, tok.get(i+1))"; }}); } static boolean tok_applyMapMethodLikeFunctions(List tok, final Set mapMethodLikeFunctions) { // mapMethod funcname(...) => mapMethod('funcname, ...) // collect funcname(...) => collect('funcname, ...) // ... return jreplace_dyn(tok, " (", new F2, Integer, String>() { String get(List tok, Integer cIdx) { try { return tok.get(cIdx) + "(" + quote(tok.get(cIdx+2)) + ","; } catch (Exception __e) { throw rethrow(__e); } } public String toString() { return "tok.get(cIdx) + \"(\" + quote(tok.get(cIdx+2)) + \",\""; }}, new F2, Integer, Boolean>() { Boolean get(List tok, Integer i) { try { return contains(mapMethodLikeFunctions, tok.get(i+1)); } catch (Exception __e) { throw rethrow(__e); } } public String toString() { return "contains(mapMethodLikeFunctions, tok.get(i+1))"; }}); } static void runMetaPostBlocks(List tok) { for (String code : unnull(metaPostBlocks)) { ping(); call(hotwireCached(standardFunctionSnippet(assertIdentifier(code))), code, tok); } //callF(codeToFunctionOnArbitraryType(code, "LS", L, "tok"), tok); } static boolean tok_applyNuLikeFunctions(List tok, final Set nuLikeFunctions) { // nu ClassName(...) => nu(ClassName, ...) // ... return jreplace_dyn(tok, " (", new F2, Integer, String>() { String get(List tok, Integer cIdx) { try { return tok.get(cIdx) + "(" + tok.get(cIdx+2) + ".class,"; } catch (Exception __e) { throw rethrow(__e); } } public String toString() { return "tok.get(cIdx) + \"(\" + tok.get(cIdx+2) + \".class,\""; }}, new F2, Integer, Boolean>() { Boolean get(List tok, Integer i) { try { return contains(nuLikeFunctions, tok.get(i+1)); } catch (Exception __e) { throw rethrow(__e); } } public String toString() { return "contains(nuLikeFunctions, tok.get(i+1))"; }}); } static List ll(A... a) { ArrayList l = new ArrayList(a.length); for (A x : a) l.add(x); return l; } static HashSet lithashset(A... items) { HashSet set = new HashSet(); for (A a : items) set.add(a); return set; } // get purpose 1: access a list/array/map (safer version of x.get(y)) static A get(List l, int idx) { return l != null && idx >= 0 && idx < l(l) ? l.get(idx) : null; } // seems to conflict with other signatures /*static B get(Map map, A key) { ret map != null ? map.get(key) : null; }*/ static A get(A[] l, int idx) { return idx >= 0 && idx < l(l) ? l[idx] : null; } // default to false static boolean get(boolean[] l, int idx) { return idx >= 0 && idx < l(l) ? l[idx] : false; } // get purpose 2: access a field by reflection or a map static Object get(Object o, String field) { try { if (o == null) return null; if (o instanceof Class) return get((Class) o, field); if (o instanceof Map) return ((Map) o).get(field); Field f = getOpt_findField(o.getClass(), field); if (f != null) { f.setAccessible(true); return f.get(o); } if (o instanceof DynamicObject) return ((DynamicObject) o).fieldValues.get(field); } catch (Exception e) { throw asRuntimeException(e); } throw new RuntimeException("Field '" + field + "' not found in " + o.getClass().getName()); } static Object get_raw(Object o, String field) { try { Field f = get_findField(o.getClass(), field); f.setAccessible(true); return f.get(o); } catch (Exception e) { throw new RuntimeException(e); } } static Object get(Class c, String field) { try { Field f = get_findStaticField(c, field); f.setAccessible(true); return f.get(null); } catch (Exception e) { throw new RuntimeException(e); } } static Field get_findStaticField(Class c, String field) { Class _c = c; do { for (Field f : _c.getDeclaredFields()) if (f.getName().equals(field) && (f.getModifiers() & java.lang.reflect.Modifier.STATIC) != 0) return f; _c = _c.getSuperclass(); } while (_c != null); throw new RuntimeException("Static field '" + field + "' not found in " + c.getName()); } static Field get_findField(Class c, String field) { Class _c = c; do { for (Field f : _c.getDeclaredFields()) if (f.getName().equals(field)) return f; _c = _c.getSuperclass(); } while (_c != null); throw new RuntimeException("Field '" + field + "' not found in " + c.getName()); } static Object get(String field, Object o) { return get(o, field); } static String formatSnippetID(String id) { return "#" + parseSnippetID(id); } static String formatSnippetID(long id) { return "#" + id; } static String fsi(String id) { return formatSnippetID(id); } static String joinWithComma(Collection c) { return join(", ", c); } static String joinWithComma(String... c) { return join(", ", c); } static String joinWithComma(Pair p) { return p == null ? "" : joinWithComma(str(p.a), str(p.b)); } static List quoteAll(Collection l) { List x = new ArrayList(); for (String s : l) x.add(quote(s)); return x; } static ArrayList asList(A[] a) { return a == null ? new ArrayList() : new ArrayList(Arrays.asList(a)); } static ArrayList asList(int[] a) { if (a == null) return null; ArrayList l = emptyList(a.length); for (int i : a) l.add(i); return l; } static ArrayList asList(float[] a) { if (a == null) return null; ArrayList l = emptyList(a.length); for (float i : a) l.add(i); return l; } static ArrayList asList(Iterable s) { if (s instanceof ArrayList) return (ArrayList) s; ArrayList l = new ArrayList(); if (s != null) for (A a : s) l.add(a); return l; } static ArrayList asList(Producer p) { ArrayList l = new ArrayList(); A a; if (p != null) while ((a = p.next()) != null) l.add(a); return l; } static ArrayList asList(Enumeration e) { ArrayList l = new ArrayList(); if (e != null) while (e.hasMoreElements()) l.add(e.nextElement()); return l; } static void change() { //mainConcepts.allChanged(); // safe version for now cause function is sometimes included unnecessarily (e.g. by EGDiff) callOpt(getOptMC("mainConcepts"), "allChanged"); } static List diff(Collection a, Collection b) { Set set = asSet(b); List l = new ArrayList(); for (String s : a) if (!set.contains(s)) l.add(s); return l; } static void clear(Collection c) { if (c != null) c.clear(); } static void lock(Lock lock) { try { ping(); if (lock == null) return; try { lock.lockInterruptibly(); } catch (InterruptedException e) { print("Locking interrupted! I probably deadlocked, oops."); printStackTrace(e); rethrow(e); } ping(); } catch (Exception __e) { throw rethrow(__e); } } static void lock(Lock lock, String msg) { print("Locking: " + msg); lock(lock); } static void lock(Lock lock, String msg, long timeout) { print("Locking: " + msg); lockOrFail(lock, timeout); } static ReentrantLock lock() { return fairLock(); } //sbool ping_actions_shareable = true; static volatile boolean ping_pauseAll; static int ping_sleep = 100; // poll pauseAll flag every 100 static volatile boolean ping_anyActions; static Map ping_actions = newWeakHashMap(); static ThreadLocal ping_isCleanUpThread = new ThreadLocal(); // always returns true static boolean ping() { if (ping_pauseAll || ping_anyActions ) ping_impl(true /* XXX */); //ifndef LeanMode ping_impl(); endifndef return true; } // returns true when it slept static boolean ping_impl(boolean okInCleanUp) { try { if (ping_pauseAll && !isAWTThread()) { do Thread.sleep(ping_sleep); while (ping_pauseAll); return true; } if (ping_anyActions) { // don't allow sharing ping_actions if (!okInCleanUp && !isTrue(ping_isCleanUpThread.get())) failIfUnlicensed(); Object action = null; synchronized(ping_actions) { if (!ping_actions.isEmpty()) { action = ping_actions.get(currentThread()); if (action instanceof Runnable) ping_actions.remove(currentThread()); if (ping_actions.isEmpty()) ping_anyActions = false; } } if (action instanceof Runnable) ((Runnable) action).run(); else if (eq(action, "cancelled")) throw fail("Thread cancelled."); } return false; } catch (Exception __e) { throw rethrow(__e); } } static RuntimeException rethrow(Throwable t) { if (t instanceof Error) _handleError((Error) t); throw t instanceof RuntimeException ? (RuntimeException) t : new RuntimeException(t); } static RuntimeException rethrow(String msg, Throwable t) { throw new RuntimeException(msg, t); } static volatile boolean readLine_noReadLine; static String readLine_lastInput; static String readLine_prefix = "[] "; static String readLine() { if (readLine_noReadLine) return null; String s = readLineHidden(); if (s != null) { readLine_lastInput = s; print(readLine_prefix + s); } return s; } static String format3(String pat, Object... args) { if (args.length == 0) return pat; List tok = javaTokPlusPeriod(pat); int argidx = 0; for (int i = 1; i < tok.size(); i += 2) if (tok.get(i).equals("*")) tok.set(i, format3_formatArg(argidx < args.length ? args[argidx++] : "null")); return join(tok); } static String format3_formatArg(Object arg) { if (arg == null) return "null"; if (arg instanceof String) { String s = (String) arg; return isIdentifier(s) || isNonNegativeInteger(s) ? s : quote(s); } if (arg instanceof Integer || arg instanceof Long) return String.valueOf(arg); return quote(structure(arg)); } static volatile StringBuffer local_log = new StringBuffer(); // not redirected static volatile Appendable print_log = local_log; // might be redirected, e.g. to main bot // in bytes - will cut to half that static volatile int print_log_max = 1024*1024; static volatile int local_log_max = 100*1024; static boolean print_silent; // total mute if set static Object print_byThread_lock = new Object(); static volatile ThreadLocal