!7

/*sclass Interpretable {
  S text;
  RuleEngine_MatchedRule2 matched;
}*/

sclass Entry {
  long msgID;
  S originalLine;
  S interpretableText;
  S matchedRuleStruct;
  
  sS _fieldOrder = "msgID originalLine interpretableText";
}

set flag DynModule.

cmodule InterpretableTest > DynObjectTable<Entry> {
  start {
    addCountToName();
    thread { scan(); }
  }
  
  visualize {
    ret withCenteredButtons(super.visualize(), "Scan", rThread scan);
  }
  
  void scan enter {
    new L<Entry> list;
    new Flag interrupted;
    temp AutoCloseable action = dm_currentAction("Scanning chat", rRaiseFlag(interrupted));
    F0<GazelleEvalContext> contextMaker = new GazelleContextCache_v2().fill();
    L<GazelleLine> lines = dm_discord_allLines();
    int i = 0;
    for (GazelleLine line : lines) {
      ++i;
      if (!licensed() || interrupted!) break;
      Collection<S> l = getInterpretables(line, +contextMaker);
      for (S interpretableText : l)
        list.add(nu Entry(originalLine := line.text, msgID := line.msgID, +interpretableText));
      call(action, 'setText, "Found " + n2(list, "interpretable") + ", scanned " + i + "/" + n2(lines, "line"));
    }
    setList(list);
  }
  
  Collection<S> getInterpretables(GazelleLine line, O... _) {
    Set<S> interpretables = linkedCISet(); // fancy new ordered case-insensitive set
    RuleEngine2 engine = ((F0<GazelleEvalContext>) optPar contextMaker(_)).get().engine;
    for (RuleEngine2.SimplifyWithRule rule : engine.splitterRules()) pcall {
      _addAll(interpretables, gazelle_executeSplitterRule(engine, rule, line.text));
    }
    
    _addAll(interpretables, collect line(
      dm_gazelle_reasonAboutChatInput_v2(null, line.text, 
        paramsPlus(_, skipSplitters := true, acceptablePurposes := litset('preprocess)))));
        
    interpretables.remove(line.text);
    ret interpretables;
  }
}