!7

p {

  ILister lister = persistentListerReadOnly(chatTheoryModule());

  L<S> msgs = lister.getList('globalIDList);

  Map<S, Web> webs = lister.getHashMap('theoryForLineMap);

  print("Have " + n(countNotNull(msgs), "msg") + ", " + n(webs, "web"));

  //print("Missing msgs: " + elementsNotInMap(notNullOnly(msgs), webs));

  //print("Extra webs: " + listMinusSet(keys(webs), msgs));

  new L<Pair<WebNode>> nodePairs;

  for (int i = 1; i+1 < l(msgs); i++) {

    S id1 = msgs.get(i), id2 = msgs.get(i+1);

    if (id1 == null || id2 == null) continue;

    Web a = webs.get(id1);

    Web b = webs.get(id2);

    if (a == null || b == null) {

      print("Missing webs at " + i);

      continue;

    }

    //print("Have " + web_firstNode(a));

    nodePairs.add(pair(web_firstNode(a), web_firstNode(b)));

  }

  print("Have " + n(nodePairs, "node pair"));

  // Analysis

  new MultiMap<S, WebNode> index; // left text to right node

  for (Pair<WebNode> p : nodePairs) {

    L<S> texts = web_texts(p.a);

    for (S text : texts)

      index.setPut(text, p.b);

  }

  print("Index size: " + index.keysSize() + " / " + index.size());

  MultiSet<S> ms = multiMapAsMultiSet(index);

  for (S key : ms.highestFirst()) {

    L<WebNode> nodes = index.get(key);

    if (l(nodes) < 2) continue;

    print(l(nodes) + " - testing key " + key);

    L<S> common = web_commonTexts(nodes);

    LL<S> l = map web_texts(nodes);

    if (nempty(common)) {

      print("  Always: " + common);

      for (L x : l ) x.removeAll(common);

    }

    //Pair<S, Int> p = web_mostCommonTextWithCount(nodes);

    Pair<S, Int> p = mostCommonWithCount(l);

    if (p != null && p.b > 1) print("  Often: " + p.a + " (" + p.b + ")");

  }

}