!7

cprint CompressorSpike {
  switchable S snippetID = #1020763;
  LinkedHashMap<S> versions;
  
  start-thread {
    print("Snippet ID: " + snippetID);
    dm_onFieldChange snippetID(r { setField(versions := null); dm_reload() });
    if (versions == null) {
      versions = mapToLinkedHashMap(textChangesOfSnippet(snippetID), sv -> pair(str(sv.versionID), sv.previousValue));
      versions.put("latest", loadSnippet(snippetID));
      versions = mapValuesToLinkedHashMap toLinesAndBack(versions); // canonicalize line breaks
      change();
    }
    print("Have " + nVersions(versions) + " with a total of " + nLines(totalLineCount(values(versions))));
    
    new Compressor().run();
  }
  
  abstract sclass Chunk {
    abstract S text(L<Chunk> chunks);
  }
  
  srecord CPair(int i1, int i2) > Chunk {
    CPair(Pair<Int> p) { i1 = p.a; i2 = p.b; }
    
    S text(L<Chunk> chunks) {
      ret linesLL(chunks.get(i1).text(chunks), chunks.get(i2).text(chunks));
    }
  }
  
  srecord CPrim(S s) > Chunk {
    S text(L<Chunk> chunks) { ret s; }
  }
  
  class Compressor {
    Map<S, LS> textIDToLines = mapValuesToLinkedHashMap lines(versions);
    LS allUniqueLines;
    new L<Chunk> chunks;
    Map<S, Int> lineIndex;
    new Map<PairS, Int> linePairIndex;
    
    replace Encodings with Map<S, L<Int>>.
    
    run {
      allUniqueLines = uniquify(concatLists(values(textIDToLines)));
      for (S line : allUniqueLines)
        chunks.add(new CPrim(line));
      lineIndex = listIndex(collect s(chunks));
      
      // simple encoding (only direct line references)
      Encodings simpleEncodings = mapValues(textIDToLines,
        (IF1<LS, L<Int>>) (lines -> map(lines, line -> lineIndex.get(line))));
      printAndCheckEncodings(simpleEncodings);
      
      Encodings advancedEncodings = compressPairs(simpleEncodings);

      printAndCheckEncodings(advancedEncodings);
    }
    
    Encodings compressPairs(Encodings encodings) {
      new MultiSet<Pair<Int>> pairCounts;
      
      for (L<Int> l : values(encodings)) {
        Pair<Int> lastPair = null;
        for (Pair<Int> pair : overlappingPairs(l)) {
          if (neq(pair, lastPair)) {
            lastPair = pair;
            pairCounts.add(pair);
          }
        }
      }
      
      //print("Pair counts: " + pairCounts);
      L<Pair<Int>> duplicates = multiSetDuplicatesByPopularity(pairCounts);
      print("Pairs: " + duplicates);
      
      // Compress only most popular pair
      if (empty(duplicates)) ret encodings; // Nothing to do
      Pair<Int> toCompress = first(duplicates);
      int idx = makeCPair(toCompress);
      ret mapValues(encodings, encoded -> replaceSublist(encoded, pairToList(toCompress), ll(idx)));
    }
    
    int makeCPair(Pair<Int> p) {
      int idx = addAndReturnIndex(chunks, new CPair(p));
      print("Made pair: " + p + " -> " + idx);
      ret idx;
    }
    
    void printAndCheckEncodings(Encodings encodings) {
      for (S id, L<Int> encoded : encodings) {
        print(id + ": " + joinWithSpace(encoded));
        assertEquals(lines(textIDToLines.get(id)), decode(encoded));
      }
    }
    
    S decode(L<Int> encoded) {
      ret lines(lambdaMap chunkText(encoded));
    }
    
    S chunkText(int idx) {
      ret chunks.get(idx).text(chunks);
    }
  }
}