sclass LineCompReader {
  new LS literals;
  IVF1<Long> onPair; // if not null, don't save pairs, but stream them to onPair instead
  int[] literalOffsets; // where they start in file
  new LongBuffer pairs;
  new LinkedHashMap<S, L<Int>> versions;
  bool byteMode;
  
  // internal, optional
  CountingInputStream countingInputStream;
  long fileSize;
  
  *() {}
  
  // takes text or gzipped input file
  *(File f) { load(f); }
  
  *(InputStream in) { load(rawByteReader(in, 128*1024)); }
  *(BufferedReader reader) { load(reader); }
  
  void load(File f) {
    fileSize = fileSize(f);
    countingInputStream = new CountingInputStream(bufferedFileInputStream(f));
    temp BufferedReader reader = isGZipFile(f)
      ? rawByteReader(gzipInputStream(countingInputStream))
      : rawByteReader(countingInputStream);
      //rawByteReader_possiblyGZipped(f);
    load(reader);
  }
  
  void load(BufferedReader reader) ctex {
    new StringBuilder lineBuf;
    S s = readLineIgnoreCR(reader, lineBuf);
    int ofs = l(s)+1;
    new Matches m;
    if (startsWith(s, "BYTECOMP ", m)) set byteMode;
    else if (!startsWith(s, "LINECOMP ", m))
      fail("Not a LINECOMP file");
    int nLiterals = parseInt(m.rest());
    new IntBuffer offsets;
    for i to nLiterals: {
      S line = readLineIgnoreCR(reader, lineBuf);
      assertNotNull(line);
      literals.add(byteMode ? str(charFromHex(line)) : line);
      offsets.add(ofs);
      ofs += l(line)+1;
    }
    offsets.add(ofs);
    literalOffsets = offsets.toArray();
    int n = 0;
    while licensed {
      s = readLineIgnoreCR(reader, lineBuf);
      if (s == null || contains(s, "=")) break;
      try {
        int iSpace = s.indexOf(' ');
        long pair = twoIntsToLong(
          Int.parseInt(s, 0, iSpace, 10),
          Int.parseInt(s, iSpace+1, l(s), 10));
        if (onPair != null) onPair.get(pair);
        else pairs.add(pair);
        if (((++n) % oneMillion()) == 0) {
          S percentage = "";
          if (fileSize != 0 && countingInputStream != null)
            percentage = " (" + intPercentRatio(countingInputStream.getFilePointer(), fileSize) + "%)";
          print(nPairs(n) + " read" + percentage);
        }
      } on fail {
        print("On line " + (nLiterals + l(pairs)));
      }
    }
    pairs.trimToSize();
    while (contains(s, "=")) {
      int i = indexOf(s, '=');
      versions.put(takeFirst(s, i), compactIntList(parseInts(splitAtSpace(substring(s, i+1)))));
      s = readLineIgnoreCR(reader, lineBuf);
    }
  }
  
  Set<S> versions() { ret keys(versions); }
  
  S getText(S version) { ret textForVersion(version); }
  S textForVersion(S version) {
    L<Int> encoded = versions.get(version);
    if (encoded == null) null;
    new LS buf;
    for (int idx : encoded)
      decode(idx, buf);
    ret myFromLines(buf);
  }
  
  // name of first (or only) file
  S firstFile() { ret first(versions()); }
  
  // text for first (or only) file
  S text() { ret getText(firstFile()); }
  
  L<Int> encoding() { ret versions.get(firstFile()); }
  
  S myFromLines(LS l) {
    ret byteMode
      ? join(l)
      : fromLines_rtrim(l);
  }
  
  void decode(int idx, LS buf) {
    if (idx < l(literals))
      buf.add(literals.get(idx));
    else {
      long p = pairs.get(idx-l(literals));
      decode(firstIntFromLong(p), buf);
      decode(secondIntFromLong(p), buf);
    }
  }
  
  // That was it! The rest of this file is just for calculating some stats.
  
  new Map<Int> lineCountsForPairs;
  new Map<Int, Long> byteCountsForPairs;

  int lineCountForPointer(int idx) {
    ret idx < l(literals) ? 1 : lineCountForPair(idx);
  }
  
  long byteCountForPointer(int idx) {
    ret idx < l(literals) ? l(literals.get(idx))+1 : byteCountForPair(idx);
  }
  
  int lineCountForPair(int idx) {
    Int c = lineCountsForPairs.get(idx);
    if (c == null) {
      long p = pairs.get(idx-l(literals));
      c = lineCountForPointer(firstIntFromLong(p)) + lineCountForPointer(secondIntFromLong(p));
      lineCountsForPairs.put(idx, c);
    }
    ret c;
  }
  
  long byteCountForPair(int idx) {
    Long c = byteCountsForPairs.get(idx);
    if (c == null) {
      long p = pairs.get(idx-l(literals));
      c = byteCountForPointer(firstIntFromLong(p)) + byteCountForPointer(secondIntFromLong(p));
      byteCountsForPairs.put(idx, c);
    }
    ret c;
  }
  
  int lineCountForVersion(S version) {
    L<Int> encoded = versions.get(version);
    if (encoded == null) ret 0;
    int n = 0;
    for (int i : encoded) n += lineCountForPointer(i);
    ret n;
  }
  
  long byteCountForVersion(S version) {
    L<Int> encoded = versions.get(version);
    if (encoded == null) ret 0;
    long n = 0;
    for (int i : encoded) n += byteCountForPointer(i);
    ret max(0, n-1);
  }
  
  long totalByteCount() {
    ret longSum(lambdaMap byteCountForVersion(versions()));
  }
  
  // now we can also save again
  
  void save(PrintWriter out) {
    out.println((byteMode ? "BYTECOMP " : "LINECOMP ") + l(literals));
    for (S s : literals)
      out.println(byteMode ? charToHex(first(s)) : s);
    for (long p : pairs)
      out.println(firstIntFromLong(p) + " " + secondIntFromLong(p));
    for (S id, L<Int> l : versions)
      out.println(id + "=" + joinWithSpace(l));
  }  
}