srecord noeq LCMerger_v3(LineCompReader lc1) { LineCompReader lc2; LineCompReader lcOut; // actually the same as lc1 now LCSortedPairIndex pairIndex; Map literalIndex; new IntBuffer lc2map; int newLiterals; // number of literals added compared to lc1 int newPairs; int nOriginalLiterals, nOriginalPairs; int lc2PairCount; *(LineCompReader *lc1, LineCompReader *lc2) {} *(File ubcFile) { lcOut = lc1 = LineCompReader(ubcFile); } void add(File ubcFile) { //if (lc2 != null) fail("Can only run once for now"); lc2 = new LineCompReader; lc2PairCount = 0; lcOut = lc1; nOriginalLiterals = l(lc1.literals); nOriginalPairs = l(lc1.pairs); newLiterals = newPairs = 0; printWithTime("Making literalIndex"); literalIndex = indexList(lc1.literals); lc2.onPair = p -> { if (lc2PairCount == 0) { initPhase2(); } ++lc2PairCount; long pAdjusted = twoIntsToLong(adjust2(firstIntFromLong(p)), adjust2(secondIntFromLong(p))); int iPair = pairIndex.get(pAdjusted); if (iPair < 0) { // new pair iPair = l(lc1.pairs); lc1.pairs.add(pAdjusted); ++newPairs; } lc2map.add(l(lcOut.literals)+iPair); }; lc2.load(ubcFile); // copy files printWithTime("Merging files"); lc1.versions = (LinkedHashMap) mapValues(lc1.versions, enc -> lmap adjust1(enc)); for (S name, L encoding : lc2.versions) { if (lc1.versions.containsKey(name)) continue with print("Warning: Duplicate file name " + name); lc1.versions.put(name, lmap adjust2(encoding)); } printVars_str(+newLiterals, +newPairs, +lc2PairCount); print("Synergy factor: " + doubleRatio(lc2PairCount-newPairs, lc2PairCount)); } // convert symbols from lc1 int adjust1(int i) { if (i >= nOriginalLiterals) if (i >= nOriginalLiterals+nOriginalPairs) ret i+newLiterals+newPairs; else ret i+newLiterals; ret i; } int adjust2(int i) { ret lc2map.get(i); } void initPhase2 { // add lc2.literals to lc1.literals lc2map = new IntBuffer(l(lc2.literals) + l(lc2.pairs)); printWithTime("Merging literals"); for (int i = 0; i < l(lc2.literals); i++) { S c = lc2.literals.get(i); Int iLit = literalIndex.get(c); if (iLit == null) { iLit = addAndReturnIndex(lc1.literals, c); ++newLiterals; literalIndex.put(c, iLit); } lc2map.add(iLit); } // merge pairs printWithTime("Adjusting lc1 pairs"); for i to nOriginalPairs: { long p = lc1.pairs.get(i); lc1.pairs.set(i, twoIntsToLong(adjust1(firstIntFromLong(p)), adjust1(secondIntFromLong(p)))); } // Make index after adjustment printWithTime("Making pairIndex for " + nPairs(l(lc1.pairs))); pairIndex = new LCSortedPairIndex(lc1.pairs.toArray(), pairIndex); printWithTime("Merging pairs"); } }