!747

m {
  static S corpusID = "#1001010";
  static int numSnippets = 3000;
  static boolean showGUI = true;
  static int maxCharsGUI = 500000;
  static boolean allTokens = true;
  
  static Collector collector;
  static L<F> files;
  static Set<int> predicted;
  
  // a file to learn from
  static class F {
    String id, name;
    L<S> tok;
  }
  
  // a predictor
  static abstract class P {
    abstract S read(S file, L<S> tok);
    abstract P derive(); // clone & reset counter for actual use
  }

  static class Chain extends P {
    new L<P> list;
    
    *() {}
    *(L<P> *list) {}
    *(P... a) { list = asList(a); }
    
    void add(P p) { list.add(p); }
    
    S read(S file, L<S> tok) {
      for (P p : list) {
        S s = p.read(file, tok);
        if (s != null) return s;
      }
      return null;
    }
    
    P derive() {
      new Chain c;
      for (P p : list)
        c.add(p.derive());
      return c;
    }
  }
    
  static class Tuples extends P {
    Map<L<S>,S> map = new HashMap<L<S>,S>();
    int n, seen;
    S file;

    *(int *n) {
    }
    
    S read(S file, L<S> tok) {
      if (!eq(file, this.file)) {
        seen = 0;
        this.file = file;
      }
      
      while (tok.size() > seen) {
        ++seen;
        if (seen > n)
          map.put(new ArrayList<S>(tok.subList(seen-n-1, seen-1)), tok.get(seen-1));
      }
      
      if (tok.size() >= n)
        return map.get(new ArrayList<S>(tok.subList(tok.size()-n, tok.size())));
        
      return null;
    }
    
    // slow...
    P oldDerive() {
      Tuples t = new Tuples(n);
      t.map.putAll(map);
      // t.seen == 0 which is ok
      return t;
    }
    
    // fast!
    P derive() {
      Tuples t = new Tuples(n);
      t.map = new DerivedHashMap<L<S>,S>(map);
      return t;
    }
  }
  
  static class DerivedHashMap<A, B> extends AbstractMap<A, B> {
    Map<A, B> base;
    new HashMap<A, B> additions;
    
    *(Map<A, B> *base) {}
    
    public B get(Object key) {
      B b = additions.get(key);
      if (b != null) return b;
      return base.get(key);
    }
    
    public B put(A key, B value) {
      return additions.put(key, value);
    }
    
    public Set<Map.Entry<A,B>> entrySet() {
      throw fail();
    }
  }
  
  // TODO: Put NewX back in
  
  p {
    files = makeCorpus();
    print("Files in corpus: " + files.size());
    
    print("Learning...");
    collector = new Collector;
    test(new Tuples(1));
    //test(new Chain(new Tuples(4), new Tuples(3), new Tuples(2), new Tuples(1)));

    print("Learning done.");
    /*if (collector.winner != null && showGUI) {
      predicted = collector.predicted;
      showColoredText();
    }*/
  }
  
  // train & evaluate a predictor
  static void test(P p) {
    //predicted = new TreeSet<int>();
    int points = 0, total = 0, lastPercent = 0;
    for (int ii = 0; ii < files.size(); ii++) {
      F f = files.get(ii);
      
      new L<S> history;
      for (int i = allTokens ? 0 : 1; i < f.tok.size(); i += allTokens ? 1 : 2) {
        S t = f.tok.get(i);
        S x = p.read(f.name, history);
        boolean correct = t.equals(x);
        total += t.length();
        if (correct) {
          //predicted.add(i);
          points += t.length();
        }
        history.add(t);
      }
      
      int percent = roundUpTo(10, (int) (ii*100L/files.size()));
      if (percent > lastPercent) {
        print("Learning " + percent + "% done.");
        lastPercent = percent;
      }
    }
    double score = points*100.0/total;
    collector.add(p, score);
  }
  
  !include #1000989 // SnippetDB
  
  static L<F> makeCorpus() {
    S name = getSnippetTitle(corpusID);
    if (name.toLowerCase().indexOf(".zip") >= 0)
      return makeCorpus_zip();
    else
      return makeCorpus_mysqldump();
  }
  
  static L<F> makeCorpus_zip() ctex {
    new L<F> files;
    ZipFile zipFile = new ZipFile(loadLibrary(corpusID));
    Enumeration entries = zipFile.entries();

    while (entries.hasMoreElements()) {
      ZipEntry entry = (ZipEntry) entries.nextElement(); 
      //System.out.println("File found: " + entry.getName());

      InputStream fin = zipFile.getInputStream(entry);
      // TODO: try to skip binary files?
      
      InputStreamReader reader = new InputStreamReader(fin, "UTF-8");
      new StringBuilder builder;
      BufferedReader bufferedReader = new BufferedReader(reader);
      String line;
      while ((line = bufferedReader.readLine()) != null)
        builder.append(line).append('\n');
      fin.close();
      S text = builder.toString();
      
      new F f;
      f.name = entry.getName();
      f.tok = internAll(javaTok(text));
      files.add(f);
    }
    
    zipFile.close();
    return files;
  }
  
  static L<F> makeCorpus_mysqldump() {
    new L<F> files;
    SnippetDB db = new SnippetDB(corpusID);
    List<List<S>> rows = db.rowsOrderedBy("sn_created");
    for (int i = 0; i < Math.min(rows.size(), numSnippets); i++) {
      new F f;
      f.id = db.getField(rows.get(i), "sn_id");
      f.name = db.getField(rows.get(i), "sn_title");
      S text = db.getField(rows.get(i), "sn_text");
      f.tok = internAll(javaTok(text));
      files.add(f);
      ++i;
    }
    return files;
  }

  static class Collector {
    P winner;
    double bestScore = -1;
    Set<int> predicted;

    void add(P p, double score) {
      if (winner == null || score > bestScore) {
        winner = p;
        bestScore = score;
        //S name = shorten(structure(p), 100);
        S name = p.getClass().getName();
        print("New best score: " + formatDouble(score, 2) + "% (" + name + ")");
        this.predicted = main.predicted;
      }
    }
  }
}

Began life as a copy of #1001011