Warning: session_start(): open(/var/lib/php/sessions/sess_rg03mmf7a6ejnfr22slkl3lrl8, O_RDWR) failed: No space left on device (28) in /var/www/tb-usercake/models/config.php on line 51
Warning: session_start(): Failed to read session data: files (path: /var/lib/php/sessions) in /var/www/tb-usercake/models/config.php on line 51
sclass LineCompCompressor {
int safety = 0;
replace Encodings with Map>.
abstract sclass Chunk {
abstract S text(L chunks);
}
srecord CPair(int i1, int i2) > Chunk {
CPair(IntPair p) { i1 = p.a; i2 = p.b; }
S text(L chunks) {
ret linesLL_rtrim(chunks.get(i1).text(chunks), chunks.get(i2).text(chunks));
}
}
srecord CPrim(S s) > Chunk {
S text(L chunks) { ret s; }
}
bool verbose = false, verboseCompressionSteps = false;
bool sortLines = true;
bool verify = true;
Map textIDToLines;
LS allUniqueLines;
new L chunks;
int primChunks;
Map lineIndex;
new Map linePairIndex;
Encodings finalEncodings;
// key = version ID, values = text
*(SS texts) {
textIDToLines = mapValuesToLinkedHashMap myToLines(texts);
}
LS myToLines(S s) { ret toLines_nOnly_reversible(s); }
S myFromLines(LS l) { ret fromLines_rtrim(l); }
run {
allUniqueLines = uniquify(concatLists(values(textIDToLines)));
if (verboseCompressionSteps) print("Have " + n2(allUniqueLines, "unique line"));
if (sortLines) sortInPlace(allUniqueLines);
if (verboseCompressionSteps) print("Sorted " + nLines(allUniqueLines));
for (S line : allUniqueLines)
chunks.add(new CPrim(line));
primChunks = l(chunks);
lineIndex = listIndex(collect s(chunks));
// simple encoding (only direct line references)
Encodings simpleEncodings = mapValues(textIDToLines,
(IF1>) (lines -> map(lines, line -> lineIndex.get(line))));
//printAndCheckEncodings(simpleEncodings);
if (verboseCompressionSteps) print("Have simple encodings");
finalEncodings = compressPairs(simpleEncodings);
if (verbose || verify) printAndCheckEncodings(finalEncodings);
}
void saveAsTextFile(File f) {
S out = exportEncoding(finalEncodings);
saveTextFile(f, out);
if (verify) checkDecompression(f, textIDToLines);
}
void checkDecompression(File file, Map textIDToLines) {
temp BufferedReader reader = bufferedUtf8Reader(file);
LineCompReader lcr = new(reader);
assertEquals(keysList(textIDToLines), asList(lcr.versions()));
for (S version : keys(textIDToLines))
assertEquals(lcr.textForVersion(version), lines_rtrim(textIDToLines.get(version)));
if (verbose) print("Decompression OK for " + nVersions(textIDToLines));
}
S asText() { ret exportEncoding(finalEncodings); }
S exportEncoding(Encodings encodings) {
new LS buf;
buf.add("LINECOMP " + primChunks); // magic signature
for (Chunk c : chunks) {
if (c cast CPair)
buf.add(c.i1 + " " + c.i2);
else
buf.add(((CPrim) c).s);
}
for (S id, L l : encodings)
buf.add(id + "=" + joinWithSpace(l));
ret lines_rtrim(buf);
}
// new fast version of magic compression function
Encodings compressPairs(Encodings encodings) {
// get initial pair counts
new LineComp_PairCounts pairCounts;
for (L l : values(encodings))
pairCounts.addAll(overlappingIntPairs(l));
// Convert to LinkedList for more efficient modification
Map> encodings2 = /*mapValues toLinkedList*/(encodings);
IntPair toCompress;
// Compress only most popular pair in one step
//int lastDups = Int.MAX_VALUE;
while ping ((toCompress = pairCounts.mostPopularDuplicate()) != null) {
if (safety > 0 && --safety <= 0) fail("safety");
int count = pairCounts.getCount(toCompress), idx = makeCPair(toCompress);
int dups = pairCounts.numberOfDuplicates();
/*if (lastDups == dups)
fail("Number of duplicates not decreasing");
lastDups = dups;*/
if (verboseCompressionSteps) print("Compressing pair " + toCompress + " (count=" + count + ") -> " + idx + ", " + (dups-1) + " remaining");
for (L l : values(encodings2))
compressPair(pairCounts, l, toCompress, idx);
}
// reconvert to normal list
ret mapValues toArrayList(encodings2);
}
// replace replacing (pair) with replaceWith in l
void compressPair(LineComp_PairCounts pairCounts, L l, IntPair replacing, int replaceWith) {
lineComp_replaceSublistWithUpdatingPairCount(l, intPairToList(replacing), ll(replaceWith), pairCounts);
}
int makeCPair(IntPair p) {
int idx = addAndReturnIndex(chunks, new CPair(p));
ret idx;
}
void printAndCheckEncodings(Encodings encodings) {
for (S id, L encoded : encodings) {
if (verbose) print(id + ": " + joinWithSpace(encoded));
assertEquals(myFromLines(textIDToLines.get(id)), decode(encoded));
}
}
S decode(L encoded) {
ret myFromLines(lambdaMap chunkText(encoded));
}
S chunkText(int idx) {
ret chunks.get(idx).text(chunks);
}
}