!7

sclass Interpretable {
  S text;
  S ruleID;
  
  *() {}
  *(S *text, S *ruleID) {}
}

sclass Entry {
  long msgID;
  S originalLine;
  S interpretableText;
  S ruleID;
  //S matchedRuleStruct;
  
  sS _fieldOrder = "msgID originalLine interpretableText";
}

set flag DynModule.

cmodule InterpretableTest > DynObjectTable<Entry> {
  start {
    addCountToName();
    thread { scan(); }
  }
  
  visualize {
    ret withCenteredButtons(super.visualize(), "Scan", rThread scan);
  }
  
  void scan enter {
    new L<Entry> list;
    new Flag interrupted;
    temp AutoCloseable action = dm_currentAction("Scanning chat", rRaiseFlag(interrupted));
    F0<GazelleEvalContext> contextMaker = new GazelleContextCache_v2().fill();
    L<GazelleLine> lines = dm_discord_allLines();
    int i = 0;
    for (GazelleLine line : reversed(lines)) {
      ++i;
      if (!licensed() || interrupted!) break;
      Collection<Interpretable> l = getInterpretables(line, +contextMaker);
      for (Interpretable intp : l)
        list.add(nu Entry(originalLine := line.text, msgID := line.msgID, interpretableText := intp.text, ruleID := intp.ruleID));
      call(action, 'setText, "Found " + n2(list, "interpretable") + ", scanned " + i + "/" + n2(lines, "line"));
    }
    setList(list);
  }
  
  Collection<Interpretable> getInterpretables(GazelleLine line, O... _) {
    Set<S> texts = litCISet(line.text);
    new L<Interpretable> interpretables;
    RuleEngine2 engine = ((F0<GazelleEvalContext>) optPar contextMaker(_)).get().engine;
    
    for (RuleEngine2.SimplifyWithRule rule : engine.splitterRules()) pcall {
      for (S out : gazelle_executeSplitterRule(engine, rule, line.text))
        if (texts.add(out))
          interpretables.add(new Interpretable(out, rule.globalID));
    }
    
    for (GazelleTree t : dm_gazelle_reasonAboutChatInput_v2(null, line.text, 
      paramsPlus(_, skipSplitters := true, acceptablePurposes := litset('preprocess))))
        if (texts.add(t.line))
          interpretables.add(new Interpretable(t.line, t.ruleID()));
      
    ret interpretables;
  }
}

Began life as a copy of #1022170