!747

abstract class P {

  abstract void write(S tok);

  abstract S read();

}

!include #1000774 // CircularFifoBuffer

m {

  static int numSnippets = 2000;

  static boolean showGUI = true;

  static int maxCharsGUI = 500000;

  static Collector collector;

  static L<S> tok;

  static Set<int> predicted;

  static class Chain extends P {

    new L<P> list;

    *() {}

    *(L<P> *list) {}

    *(P... a) { list = asList(a); }

    void add(P p) { list.add(p); }

    void write(S tok) {

      for (P p : list)

        p.write(tok);

    }

    S read() {

      for (P p : list) {

        S s = p.read();

        if (s != null) return s;

      }

      return null;

    }

  }

  static class Pairs extends P {

    new Map<S,S> map;

    S last;

    void write(S tok) {

      if (last != null)

        map.put(last, tok);

      last = tok;

    }

    S read() {

      return last == null ? null : map.get(last);

    }

  }

  static class Tuples extends P {

    Map<L<S>,S> map = new HashMap<L<S>,S>();

    int n;

    CircularFifoBuffer<S> buf;

    *(int *n) {

      buf = new CircularFifoBuffer<S>(n);

    }

    void write(S tok) {

      if (buf.size() == n)

        map.put(new ArrayList<S>(buf.getBackingStore()), tok);

      buf.add(tok);

    }

    S read() {

      if (buf.size() == n)

        return map.get(new ArrayList<S>(buf.getBackingStore()));

      return null;

    }

  }

  // Bla x = new Bla

  static class NewX extends P {

    CircularFifoBuffer<S> buf = new CircularFifoBuffer<S>(4);

    void write(S tok) { buf.add(tok); }

    S read() {

      if (buf.size() == 4) {

        L<S> l = buf.asList();

        if (l.get(2).equals("=") && l.get(3).equals("new"))

          return l.get(0);

      }

      return null;

    }

  }

  p {

    tok = makeCorpusJavaTok(numSnippets);

    print("Tokens in corpus: " + tok.size());

    collector = new Collector;

    test(new Pairs);

    test(new Tuples(2));

    test(new Tuples(3));

    test(new Tuples(4));

    test(new Chain(new Tuples(2), new Pairs));

    test(new Chain(new Tuples(4), new Tuples(3), new Tuples(2), new Pairs));

    test(new Chain(new NewX, new Tuples(4), new Tuples(3), new Tuples(2), new Pairs));

    if (collector.winner != null && showGUI) {

      predicted = collector.predicted;

      showColoredText();

    }

  }

  // test a predictor

  static void test(P p) {

    predicted = new TreeSet<int>();

    int points = 0, total = 0;

    for (int i = 1; i < tok.size(); i += 2) {

      S t = tok.get(i);

      S x = p.read();

      boolean correct = t.equals(x);

      total += t.length();

      if (correct) {

        predicted.add(i);

        points += t.length();

      }

      p.write(t);

    }

    double score = points*100.0/total;

    collector.add(p, score);

  }

  static void showColoredText() ctex {

    JFrame jf = new JFrame("Predicted = green");

    Container cp = jf.getContentPane();

    JTextPane pane = new JTextPane();

    //pane.setFont(loadFont("#1000993", 24));

    Document doc = pane.getStyledDocument();

    int i = tok.size(), len = 0;

    while (len <= maxCharsGUI && i > 0) {

      --i;

      len += tok.get(i).length();

    }

    for (; i < tok.size(); i++) {

      if (tok.get(i).length() == 0) continue;

      boolean green = predicted.contains(i);

      SimpleAttributeSet set = new SimpleAttributeSet();

      StyleConstants.setForeground(set, green ? Color.green : Color.gray);

      doc.insertString(doc.getLength(), tok.get(i), set);

    }

    JScrollPane scrollPane = new JScrollPane(pane);

    cp.add(scrollPane, BorderLayout.CENTER);

    jf.setBounds(100, 100, 600, 600);

    jf.setVisible(true);

  }

  !include #1000989 // SnippetDB

  static L<S> makeCorpusJavaTok(int numSnippets) {

    SnippetDB db = new SnippetDB("#1000673");

    List<List<S>> rows = db.rowsOrderedBy("sn_created");

    new L<S> tok;

    for (int i = 0; i < Math.min(rows.size(), numSnippets); i++) {

      new StringBuilder buf;

      S id = db.getField(rows.get(i), "sn_id");

      S title = db.getField(rows.get(i), "sn_title");

      S text = db.getField(rows.get(i), "sn_text");

      buf.append("\n== ID: " + id);

      buf.append("\n== Title: " + title);

      buf.append("\n==\n");

      buf.append(text).append("\n");

      if (tok.size() != 0) tok.remove(tok.size()-1);

      tok.addAll(javaTok(buf.toString()));

      ++i;

    }

    return tok;

  }

  static class Collector {

    P winner;

    double bestScore = -1;

    Set<int> predicted;

    void add(P p, double score) {

      if (winner == null || score > bestScore) {

        winner = p;

        bestScore = score;

        print("New best score: " + formatDouble(score, 2) + "% (" + shorten(structure(p), 100) + ")");

        this.predicted = main.predicted;

      }

    }

  }

}