!7

cprint CompressorSpike {
  switchable S snippetID = #1020763;
  LinkedHashMap<S> versions;
  
  start-thread {
    print("Snippet ID: " + snippetID);
    dm_onFieldChange snippetID(r { setField(versions := null); dm_reload() });
    if (versions == null) {
      versions = mapToLinkedHashMap(reversed(textChangesOfSnippet(snippetID)), sv -> pair(str(sv.versionID), sv.previousValue));
      versions.put("latest", loadSnippet(snippetID));
      versions = mapValuesToLinkedHashMap toLinesAndBack(versions); // canonicalize line breaks
      change();
    }
    print("Have " + nVersions(versions) + " with a total of " + nLines(totalLineCount(values(versions))));
    
    new Compressor().run();
  }
  
  abstract sclass Chunk {
    abstract S text(L<Chunk> chunks);
  }
  
  srecord CPair(int i1, int i2) > Chunk {
    CPair(Pair<Int> p) { i1 = p.a; i2 = p.b; }
    
    S text(L<Chunk> chunks) {
      ret linesLL_rtrim(chunks.get(i1).text(chunks), chunks.get(i2).text(chunks));
    }
  }
  
  srecord CPrim(S s) > Chunk {
    S text(L<Chunk> chunks) { ret s; }
  }
  
  class Compressor {
    replace Encodings with Map<S, L<Int>>.
    
    bool sortLines = true;
    Map<S, LS> textIDToLines = mapValuesToLinkedHashMap lines(versions);
    LS allUniqueLines;
    new L<Chunk> chunks;
    int primChunks;
    Map<S, Int> lineIndex;
    new Map<PairS, Int> linePairIndex;
    Encodings finalEncodings;
    
    run {
      allUniqueLines = uniquify(concatLists(values(textIDToLines)));
      if (sortLines) sortInPlace(allUniqueLines);
      for (S line : allUniqueLines)
        chunks.add(new CPrim(line));
      primChunks = l(chunks);
      lineIndex = listIndex(collect s(chunks));
      
      // simple encoding (only direct line references)
      Encodings simpleEncodings = mapValues(textIDToLines,
        (IF1<LS, L<Int>>) (lines -> map(lines, line -> lineIndex.get(line))));
      //printAndCheckEncodings(simpleEncodings);
      
      Encodings advancedEncodings = simpleEncodings;
      while licensed {
        Encodings e = compressPairs(advancedEncodings);
        if (e == advancedEncodings) break;
        advancedEncodings = e;
      }

      finalEncodings = advancedEncodings;
      printAndCheckEncodings(finalEncodings);
      
      S out = exportEncoding(finalEncodings);
      printWithPrecedingNL(out);
      File file = saveTextFile_infoBox(javaxDataDir("Compressed Snippet Version History/versions-of-" + psI(snippetID) + ".linecomp"), out);
      print(renderFileInfo(gzipFile(file)));
      
      // Make .zip and .tgz for comparison
      File zipFile = replaceFileExtension(file, ".zip");
      {
        temp ZipOutputStream zipOut = zipOutputStream(zipFile);
        for (S id, text : versions)
          zip_addTextFile(zipOut, id, text);
      }
      printFileInfo(zipFile);
      
      File tgzFile = replaceFileExtension(file, ".tgz");
      zip2tgz(zipFile, tgzFile);
      printFileInfo(tgzFile);
      
      checkDecompression(file, textIDToLines);
    }
    
    void checkDecompression(File file, Map<S, LS> textIDToLines) {
      temp BufferedReader reader = bufferedUtf8Reader(file);
      LineCompReader lcr = new(reader);
      assertEquals(keysList(textIDToLines), asList(lcr.versions()));
      for (S version : keys(textIDToLines))
        assertEquals(lcr.textForVersion(version), lines_rtrim(textIDToLines.get(version)));
      print("Decompression OK for " + nVersions(textIDToLines));
    }
    
    S exportEncoding(Encodings encodings) {
      new LS buf;
      buf.add("LINECOMP " + primChunks); // magic signature
      for (Chunk c : chunks) {
        if (c cast CPair)
          buf.add(c.i1 + " " + c.i2);
        else
          buf.add(((CPrim) c).s);
      }
      for (S id, L<Int> l : encodings)
        buf.add(id + "=" + joinWithSpace(l));
      ret lines_rtrim(buf);
    }
    
    Encodings compressPairs(Encodings encodings) {
      new MultiSet<Pair<Int>> pairCounts;
      
      for (L<Int> l : values(encodings)) {
        Pair<Int> lastPair = null;
        for (Pair<Int> pair : overlappingPairs(l)) {
          if (neq(pair, lastPair)) {
            lastPair = pair;
            pairCounts.add(pair);
          }
        }
      }
      
      //print("Pair counts: " + pairCounts);
      Pair<Int> toCompress = msMostPopularDuplicate(pairCounts);

      // Compress only most popular pair
      if (toCompress == null) ret encodings; // Nothing to do
      int idx = makeCPair(toCompress);
      print("Made pair: " + toCompress + " -> " + idx + ", " + (msNumberOfDuplicates(pairCounts)-1) + " remaining");
      ret mapValues(encodings, (IF1<L<Int>>) encoded ->
        replaceSublist(encoded, pairToList(toCompress), ll(idx)));
    }
    
    int makeCPair(Pair<Int> p) {
      int idx = addAndReturnIndex(chunks, new CPair(p));
      ret idx;
    }
    
    void printAndCheckEncodings(Encodings encodings) {
      for (S id, L<Int> encoded : encodings) {
        print(id + ": " + joinWithSpace(encoded));
        assertEquals(lines(textIDToLines.get(id)), decode(encoded));
      }
    }
    
    S decode(L<Int> encoded) {
      ret lines(lambdaMap chunkText(encoded));
    }
    
    S chunkText(int idx) {
      ret chunks.get(idx).text(chunks);
    }
  }
}