!7

sclass Interpretable {
  S text;
  S ruleID;
  
  *() {}
  *(S *text, S *ruleID) {}
}

sclass Entry {
  long msgID;
  S originalLine;
  S interpretableText;
  S ruleID;
  //S matchedRuleStruct;
  
  sS _fieldOrder = "msgID originalLine interpretableText";
}

set flag DynModule.

cmodule InterpretableTest > DynObjectTable<Entry> {
  transient int maxInterpretablesPerLine = 20;
  
  start {
    addCountToName();
    thread { scan(); }
  }
  
  visualize {
    ret withCenteredButtons(super.visualize(), "Scan", rThread scan);
  }
  
  void scan enter {
    new L<Entry> list;
    new Flag interrupted;
    temp AutoCloseable action = dm_currentAction("Scanning chat", rRaiseFlag(interrupted));
    F0<GazelleEvalContext> contextMaker = new GazelleContextCache_v2().fill();
    L<GazelleLine> lines = dm_discord_allLines();
    int i = 0;
    for (GazelleLine line : reversed(lines)) {
      ++i;
      if (!licensed() || interrupted!) break;
      Collection<Interpretable> l = getInterpretables(line, +contextMaker);
      for (Interpretable intp : l)
        list.add(nu Entry(originalLine := line.text, msgID := line.msgID, interpretableText := intp.text, ruleID := intp.ruleID));
      call(action, 'setText, "Found " + n2(list, "interpretable") + ", scanned " + i + "/" + n2(lines, "line"));
    }
    setList(list);
  }
  
  Collection<Interpretable> getInterpretables(GazelleLine line, O... _) {
    Set<S> texts = litCISet(line.text);
    new L<Interpretable> interpretables;
    Collector<Interpretable> collector = limitedCollector(interpretables, maxInterpretablesPerLine);
    RuleEngine2 engine = ((F0<GazelleEvalContext>) optPar contextMaker(_)).get().engine;

    int nInterpreted = -1;
    
    bigLoop: while (!collector.full() && nInterpreted < l(interpretables)) {
      Interpretable workingOn = get(interpretables, nInterpreted);
      S input = nInterpreted < 0 ? line.text : workingOn.text;
      S ruleBase = workingOn == null ? "" : workingOn.ruleID + "+";
      ++nInterpreted;
      
      for (RuleEngine2.SimplifyWithRule rule : engine.splitterRules()) pcall {
        for (S out : gazelle_executeSplitterRule(engine, rule, input))
          if (texts.add(out))
            if (collector.add(new Interpretable(out, ruleBase + rule.globalID))) break bigLoop;
      }
      
      for (GazelleTree t : dm_gazelle_reasonAboutChatInput_v2(null, input, 
        paramsPlus(_, skipSplitters := true, acceptablePurposes := litset('preprocess))))
          if (texts.add(t.line))
            if (collector.add(new Interpretable(t.line, ruleBase + t.ruleID()))) break bigLoop;
    }
        
    ret interpretables;
  }
}

Began life as a copy of #1022179