sclass LineCompCompressor {

  int safety = 0;

  replace Encodings with Map<S, L<Int>>.

  abstract sclass Chunk {

    abstract S text(L<Chunk> chunks);

  }

  srecord CPair(int i1, int i2) > Chunk {

    CPair(Pair<Int> p) { i1 = p.a; i2 = p.b; }

    S text(L<Chunk> chunks) {

      ret linesLL_rtrim(chunks.get(i1).text(chunks), chunks.get(i2).text(chunks));

    }

  }

  srecord CPrim(S s) > Chunk {

    S text(L<Chunk> chunks) { ret s; }

  }

  bool verbose = false, verboseCompressionSteps = false;

  bool sortLines = true;

  bool verify = true;

  Map<S, LS> textIDToLines;

  LS allUniqueLines;

  new L<Chunk> chunks;

  int primChunks;

  Map<S, Int> lineIndex;

  new Map<PairS, Int> linePairIndex;

  Encodings finalEncodings;

  // key = version ID, values = text

  *(SS texts) {

    textIDToLines = mapValuesToLinkedHashMap myToLines(texts);

  }

  LS myToLines(S s) { ret toLines_nOnly_reversible(s); }

  S myFromLines(LS l) { ret fromLines_rtrim(l); }

  run {

    LS allLines = concatLists(values(textIDToLines));

    if (verboseCompressionSteps) print("Uniquifying " + nLines(allLines));

    allUniqueLines = uniquify(allLines);

    if (verboseCompressionSteps) print("Have " + n2(allUniqueLines, "unique line"));

    allLines = null; // allow me to forget

    if (sortLines) sortInPlace(allUniqueLines);

    if (verboseCompressionSteps) print("Sorted " + nLines(allUniqueLines));

    for (S line : allUniqueLines)

      chunks.add(new CPrim(line));

    primChunks = l(chunks);

    lineIndex = listIndex(collect s(chunks));

    // simple encoding (only direct line references)

    Encodings simpleEncodings = mapValues(textIDToLines,

      (IF1<LS, L<Int>>) (lines -> map(lines, line -> lineIndex.get(line))));

    //printAndCheckEncodings(simpleEncodings);

    if (verboseCompressionSteps) print("Have simple encodings");

    finalEncodings = compressPairs(simpleEncodings);

    if (verbose || verify) printAndCheckEncodings(finalEncodings);

  }

  void saveAsTextFile(File f) {

    S out = exportEncoding(finalEncodings);

    saveTextFile(f, out);

    if (verify) checkDecompression(f, textIDToLines);

  }

  void checkDecompression(File file, Map<S, LS> textIDToLines) {

    temp BufferedReader reader = bufferedUtf8Reader(file);

    LineCompReader lcr = new(reader);

    assertEquals(keysList(textIDToLines), asList(lcr.versions()));

    for (S version : keys(textIDToLines))

      assertEquals(lcr.textForVersion(version), lines_rtrim(textIDToLines.get(version)));

    if (verbose) print("Decompression OK for " + nVersions(textIDToLines));

  }

  S asText() { ret exportEncoding(finalEncodings); }

  S exportEncoding(Encodings encodings) {

    new LS buf;

    buf.add("LINECOMP " + primChunks); // magic signature

    for (Chunk c : chunks) {

      if (c cast CPair)

        buf.add(c.i1 + " " + c.i2);

      else

        buf.add(((CPrim) c).s);

    }

    for (S id, L<Int> l : encodings)

      buf.add(id + "=" + joinWithSpace(l));

    ret lines_rtrim(buf);

  }

  // new fast version of magic compression function

  Encodings compressPairs(Encodings encodings) {

    // get initial pair counts

    new LineComp_PairCounts pairCounts;

    for (L<Int> l : values(encodings))

      pairCounts.addAll(overlappingPairs(l));

    // Convert to LinkedList for more efficient modification

    Map<S, L<Int>> encodings2 = /*mapValues toLinkedList*/(encodings);

    Pair<Int> toCompress;

    // Compress only most popular pair in one step

    //int lastDups = Int.MAX_VALUE;

    while ping ((toCompress = pairCounts.mostPopularDuplicate()) != null) {

      if (safety > 0 && --safety <= 0) fail("safety");

      int count = pairCounts.getCount(toCompress), idx = makeCPair(toCompress);

      int dups = pairCounts.numberOfDuplicates();

      /*if (lastDups == dups)

        fail("Number of duplicates not decreasing");

      lastDups = dups;*/

      if (verboseCompressionSteps) print("Compressing pair " + toCompress + " (count=" + count + ") -> " + idx + ", " + (dups-1) + " remaining");

      for (L<Int> l : values(encodings2))

        compressPair(pairCounts, l, toCompress, idx);

    }

    // reconvert to normal list

    ret mapValues toArrayList(encodings2);

  }

  // replace replacing (pair) with replaceWith in l

  void compressPair(LineComp_PairCounts pairCounts, L<Int> l, Pair<Int> replacing, int replaceWith) {

    lineComp_replaceSublistWithUpdatingPairCount(l, pairToList(replacing), ll(replaceWith), pairCounts);

  }

  int makeCPair(Pair<Int> p) {

    int idx = addAndReturnIndex(chunks, new CPair(p));

    ret idx;

  }

  void printAndCheckEncodings(Encodings encodings) {

    for (S id, L<Int> encoded : encodings) {

      if (verbose) print(id + ": " + joinWithSpace(encoded));

      assertEquals(myFromLines(textIDToLines.get(id)), decode(encoded));

    }

  }

  S decode(L<Int> encoded) {

    ret myFromLines(lambdaMap chunkText(encoded));

  }

  S chunkText(int idx) {

    ret chunks.get(idx).text(chunks);

  }

}