sclass LineCompCompressor {

  replace Encodings with Map<S, L<Int>>.

  abstract sclass Chunk {

    abstract S text(L<Chunk> chunks);

  }

  srecord CPair(int i1, int i2) > Chunk {

    CPair(IntPair p) { i1 = p.a; i2 = p.b; }

    S text(L<Chunk> chunks) {

      ret linesLL_rtrim(chunks.get(i1).text(chunks), chunks.get(i2).text(chunks));

    }

  }

  srecord CPrim(S s) > Chunk {

    S text(L<Chunk> chunks) { ret s; }

  }

  bool verbose = false;

  bool sortLines = true;

  bool verify = true;

  Map<S, LS> textIDToLines;

  LS allUniqueLines;

  new L<Chunk> chunks;

  int primChunks;

  Map<S, Int> lineIndex;

  new Map<PairS, Int> linePairIndex;

  Encodings finalEncodings;

  // key = version ID, values = text

  *(SS texts) {

    textIDToLines = mapValuesToLinkedHashMap lines(texts);

  }

  run {

    allUniqueLines = uniquify(concatLists(values(textIDToLines)));

    if (sortLines) sortInPlace(allUniqueLines);

    for (S line : allUniqueLines)

      chunks.add(new CPrim(line));

    primChunks = l(chunks);

    lineIndex = listIndex(collect s(chunks));

    // simple encoding (only direct line references)

    Encodings simpleEncodings = mapValues(textIDToLines,

      (IF1<LS, L<Int>>) (lines -> map(lines, line -> lineIndex.get(line))));

    //printAndCheckEncodings(simpleEncodings);

    finalEncodings = repeatUntilSame compressPairs(simpleEncodings);

    if (verbose || verify) printAndCheckEncodings(finalEncodings);

  }

  void saveAsTextFile(File f) {

    S out = exportEncoding(finalEncodings);

    saveTextFile(f, out);

    if (verify) checkDecompression(f, textIDToLines);

  }

  void checkDecompression(File file, Map<S, LS> textIDToLines) {

    temp BufferedReader reader = bufferedUtf8Reader(file);

    LineCompReader lcr = new(reader);

    assertEquals(keysList(textIDToLines), asList(lcr.versions()));

    for (S version : keys(textIDToLines))

      assertEquals(lcr.textForVersion(version), lines_rtrim(textIDToLines.get(version)));

    if (verbose) print("Decompression OK for " + nVersions(textIDToLines));

  }

  S asText() { ret exportEncoding(finalEncodings); }

  S exportEncoding(Encodings encodings) {

    new LS buf;

    buf.add("LINECOMP " + primChunks); // magic signature

    for (Chunk c : chunks) {

      if (c cast CPair)

        buf.add(c.i1 + " " + c.i2);

      else

        buf.add(((CPrim) c).s);

    }

    for (S id, L<Int> l : encodings)

      buf.add(id + "=" + joinWithSpace(l));

    ret lines_rtrim(buf);

  }

  Encodings compressPairs(Encodings encodings) {

    new MultiSet<IntPair> pairCounts;

    for (L<Int> l : values(encodings)) {

      IntPair lastPair = null;

      for (IntPair pair : overlappingIntPairs(l)) {

        if (neq(pair, lastPair)) {

          lastPair = pair;

          pairCounts.add(pair);

        }

      }

    }

    //print("Pair counts: " + pairCounts);

    IntPair toCompress = msMostPopularDuplicate(pairCounts);

    // Compress only most popular pair

    if (toCompress == null) ret encodings; // Nothing to do

    int idx = makeCPair(toCompress);

    print("Made pair: " + toCompress + " -> " + idx + ", " + (msNumberOfDuplicates(pairCounts)-1) + " remaining");

    ret mapValues(encodings, (IF1<L<Int>>) encoded ->

      replaceSublist(encoded, ll(toCompress.a, toCompress.b), ll(idx)));

  }

  int makeCPair(IntPair p) {

    int idx = addAndReturnIndex(chunks, new CPair(p));

    ret idx;

  }

  void printAndCheckEncodings(Encodings encodings) {

    for (S id, L<Int> encoded : encodings) {

      if (verbose) print(id + ": " + joinWithSpace(encoded));

      assertEquals(lines(textIDToLines.get(id)), decode(encoded));

    }

  }

  S decode(L<Int> encoded) {

    ret lines(lambdaMap chunkText(encoded));

  }

  S chunkText(int idx) {

    ret chunks.get(idx).text(chunks);

  }

}