<br />
<b>Warning</b>:  session_start(): open(/var/lib/php/sessions/sess_p0i8vkndi6m45jgae3t9gin9fv, O_RDWR) failed: No space left on device (28) in <b>/var/www/tb-usercake/models/config.php</b> on line <b>51</b><br />
<br />
<b>Warning</b>:  session_start(): Failed to read session data: files (path: /var/lib/php/sessions) in <b>/var/www/tb-usercake/models/config.php</b> on line <b>51</b><br />
!7

cmodule TheoryMaker > DynConvo {
  /*
  1. measurable features (fields of object)
  2. higher level features (words the user throws in)
  3. make theories (random statements)
  4. check theories
  
  1. show a random line
  2. user types keyword
  3. assign keyword to line

  (not yet used)
  line $a has keyword $x & line $a's text is $bla
    & line $y has text $bla => line $b might have keyword $x

  Basic theory making
  -------------------
  
  For any label X:
    test theory (for every M: M has label X)
    test theory (for every M: M doesn't have label X)
  
  For any feature F:
    for every seen value V of F:
      for every label X:
        test theory (for every M: msg M's feature F has value V => msg has label x))
        test theory (for every M: msg M's feature F has value V => msg doesn't have label x))

  */
  
  srecord Theory(BasicLogicRule statement) {
    PosNeg<Msg> examples;
    toString { ret str(statement.lhs instanceof MPTrue ? "Every message is " + statement.rhs : statement); }
  }

  // propositions about a message. check returns null if unknown
  asclass MsgProp { abstract Bool check(Msg msg); }

  srecord MPTrue() > MsgProp {
    Bool check(Msg msg) { true; }
    toString { ret "always"; }
  }
  
  record HasLabel(S label) > MsgProp {
    Bool check(Msg msg) { ret msg2label.get(pair(msg, label)); }
    toString { ret label; }
  }
  
  record DoesntHaveLabel(S label) > MsgProp {
    Bool check(Msg msg) { ret not(msg2label.get(pair(msg, label))); }
    toString { ret "not " + label; }
  }

  record FeatureValueIs(S feature, O value) > MsgProp {
    Bool check(Msg msg) { ret eq(msg2features.get(msg).get(feature), value); }
    toString { ret feature + "=" + value; }
  }

  class Label {
    S name;

    *() {}
    *(S *name) {}
    
    TreeSetWithDuplicates<Theory> bestTheories = new(reverseComparatorFromCalculatedField theoryScore());
  }
  
  L<Msg> msgs; // full dialog
  L<Msg> shownMsgs;
  transient Map<Msg, Map<S, O>> msg2features = AutoMap<>(lambda1 calcMsgFeatures);
  new LinkedHashSet<Theory> theories;
  S featuresText, trainedLabelsText, allLabelsText, theoriesText;
  transient JTable theoryTable, theoryByLabelTable, labelsTable;
  transient new Map<S, Label> labelsByName;

  new Set<S> allLabels;
  transient new L<IVF1<S>> onNewLabel;
  new Map<Pair<Msg, S>, Bool> msg2label;
  transient new Map<S, IF1<Msg, O>> featureExtractors;
  
  start-thread {
    // legacy + after deletion cleaning
    setField(msg2label := mapKeys(p -> transformPairB cleanLabel(p), msg2label));
    setField(allLabels := asTreeSet(pairsBOfKeys(msg2label)));
    updateLabelsByName();
    
    onNewLabel.add(lbl -> change());

    makeTheoriesAboutLabels();
    makeTheoriesAboutFeaturesAndLabels();
    
    for (S field : fields(Msg))
      featureExtractors.put(field, o -> getOpt(o, field));

    callFAllOnAll(onNewLabel, allLabels);
    
    msg2labelUpdated();
    dm_watchCollectionFieldAndNow allLabels(r allLabelsUpdated);
    if (empty(msgs))
      setField(msgs := mainCruddieLog());
    showRandomMsg();
  }

  void makeTheoriesAboutLabels {
    // For any label X:
    onNewLabel.add(lbl -> {
      // test theory (for every M: M has label X)
      addTheory(new Theory(BasicLogicRule(new MPTrue, new HasLabel(lbl))));
      // test theory (for every M: M doesn't have label X)
      addTheory(new Theory(BasicLogicRule(new MPTrue, new DoesntHaveLabel(lbl))));
    });
  }

  void makeTheoriesAboutFeaturesAndLabels {
    // for every label X:
    onNewLabel.add(lbl -> {
      // For any feature F:
      for (S feature : keys(featureExtractors))
        // for every seen value V of F:
        for (O value : possibleValuesOfFeature(feature))
          for (O rhs : ll(new HasLabel(lbl), new DoesntHaveLabel(lbl)))
            // test theory (for every M: msg M's feature F has value V => msg has/doesn't have label x))
            addTheory(new Theory(BasicLogicRule(
              new FeatureValueIs(feature, value), rhs)));
    });
  }

  L possibleValuesOfFeature(S feature) {
    if (isBoolField(Msg, feature))
      ret ll(false, true);
    ret ll();
  }

  Map<S, O> calcMsgFeatures(Msg msg) {
    ret mapValues_pcall(featureExtractors, ex -> ex.get(msg));
  }

  void showMsgs(L<Msg> l) {
    setField(shownMsgs := l);
    setMsgs(l);
    if (l(shownMsgs) == 1)
      setField(featuresText := formatColonProperties_quoteStringValues(
        msg2features.get(first(shownMsgs))));
    else setField(featuresText := "");
  }

  void showRandomMsg {
    showMsgs(randomElementAsList(msgs));
  }

  @Override
  void sendInput2(S s) {
    // treat input as a label
    if (l(shownMsgs) == 1) {
      Msg shown = first(shownMsgs);
      new Matches m;
      if "not ..." {
        S label = cleanLabel(m.rest());
        mapPutVerbose(+msg2label, pair(shown, label), false);
        msg2labelUpdated();
      } else {
        S label = cleanLabel(s);
        mapPutVerbose(+msg2label, pair(shown, label), true);
        msg2labelUpdated();
      }
      change();
    }
  }

  Map<S, Bool> getMsgLabels(Msg msg) {
    ret pairKeysToMapOfMap(msg2label).get(msg);
  }

  void msg2labelUpdated() {
    callFAllOnAll(onNewLabel, addAll_returnNew(allLabels, pairsBOfKeys(msg2label)));
    setField(trainedLabelsText :=
      formatProperties_reversed(" => ", mapKeysAndValues(
        msg -> msg.text + " [" + (msg.fromUser ? "User" : "Bot") + "]",
        map -> joinWithComma(concatLists(
          keysWithValue(true, map),
          prependAll("not ", keysWithValue(false, map)))),
        pairKeysToMapOfMap(msg2label))));
  }

  void allLabelsUpdated() {
    setField(allLabelsText := lines(allLabels));
  }

  JComponent mainPart() {
    ret jhsplit(jvsplit(
        jCenteredSection("Focused Message", super.mainPart()),
        jCenteredSection("Message Analysis", dm_textArea featuresText())),
      jtabs(
        "Theories by label", with(r updateTheoryByLabelTable, theoryByLabelTable = sexyTable()),
        "Labels & status", with(r updateLabelsTable, labelsTable = sexyTable()),
        "Theories", with(r updateTheoryTable, theoryTable = sexyTable()),
        "Labels", dm_textArea allLabelsText(),
        "Assigned Labels", dm_textArea trainedLabelsText()));
  }

  void updateTheoriesText {
    setField(theoriesText := lines(theories));
  }

  void updateTheoryTable {
    L<Theory> sorted = sortedByCalculatedFieldDesc(theories, t ->
      t.examples == null ? null : t.examples.score());
    dataToTable_uneditable(theoryTable, map(sorted, t -> litorderedmap(
      "Score" := renderTheoryScore(t),
      "Theory" := str(t))));
  }

  Map<S, Theory> labelsToBestTheoryMap() {
    Map<S, L<Theory>> map = multiMapToMap(multiMapIndex targetLabelOfTheory(theories));
    ret mapValues(map,
      theories -> highestBy theoryScore(theories));
  }

  void updateTheoryByLabelTable {
    dataToTable_uneditable(theoryByLabelTable, map(labelsToBestTheoryMap(), (label, theory) -> litorderedmap(
      "Label" := label,
      "Best Score" := renderTheoryScore(theory),
      "Best Theory" := str(theory))));
  }

  void updateLabelsTable {
    dataToTable_uneditable(labelsTable, mapValuesToList(labelsByName, label -> {
      Theory bestTheory = first(label.bestTheories);
      ret litorderedmap(
        "Label" := label.name,
        "Best Score" := renderTheoryScore(bestTheory),
        "Best Theory" := str(bestTheory));
    }));
  }
  
  S renderTheoryScore(Theory t) {
    //ret renderPosNegCounts(t.examples);
    ret t == null ? "" : renderPosNegScoreAndCount(t.examples);
  }

  int theoryScore(Theory t) {
    ret t.examples.score();
  }

  void theoriesChanged {
    updateTheoriesText();
    updateTheoryTable();
    updateTheoryByLabelTable();
    updateLabelsTable();
    change();
  }

  visual
    withCenteredButtons(super,
      "Show random msg", rThreadEnter showRandomMsg,
      jPopDownButton_noText(flattenObjectArray(
        "Check theories", rThreadEnter checkAllTheories,
        "Clear theories", rThreadEnter clearTheories,
        dm_importAndExportAllDataMenuItems())));

  void addTheory(Theory theory) {
    if (theories.add(theory)) {
      print("New theory: " + theory);
      addTheoryToCollectors(theory);
      theoriesChanged();
    }
  }

  void clearTheories { theories.clear(); theoriesChanged(); }

  Bool checkMsgProp(MsgProp prop, Msg msg) {
    ret prop.check(msg);
  }

  Bool testTheoryOnMsg(Theory theory, Msg msg) {
    Bool lhs = checkMsgProp((MsgProp) theory.statement.lhs, msg);
    Bool rhs = checkMsgProp((MsgProp) theory.statement.rhs, msg);
    if (lhs == null || rhs == null) null;
    ret isTrue(rhs) || isFalse(lhs);
  }

  void checkAllTheories {
    for (Theory theory : theories)
      checkTheory(theory);
    theoriesChanged();
  }

  // doesn't trigger updates
  void checkTheory(Theory theory) {
    new PosNeg<Msg> pn;
    for (Msg msg : msgs)
      pn.add(msg, testTheoryOnMsg(theory, msg));
    if (!eq(theory.examples, pn)) {
      removeTheoryFromCollectors(theory);
      theory.examples = pn;
      addTheoryToCollectors(theory);
    }
  }

  S cleanLabel(S label) { ret upper(label); }
  
  S targetLabelOfTheory(Theory theory) {
    O o = theory.statement.rhs;
    if (o cast HasLabel) ret o.label;
    if (o cast DoesntHaveLabel) ret o.label;
    null;
  }

  void addTheoryToCollectors(Theory theory) {
    S lbl = targetLabelOfTheory(theory);
    if (lbl != null)
      labelByName(lbl).bestTheories.add(theory);
  }

  void removeTheoryFromCollectors(Theory theory) {
    S lbl = targetLabelOfTheory(theory);
    if (lbl != null)
      labelByName(lbl).bestTheories.remove(theory);
  }

  Label labelByName(S name) {
    ret getOrCreate(labelsByName, name, () -> new Label(name));
  }

  void updateLabelsByName() {
    for (S lbl : allLabels)
      labelByName(lbl);
    for (Theory t : theories)
      addTheoryToCollectors(t);
  }
}