!7

/*sclass Interpretable {

  S text;

  RuleEngine_MatchedRule2 matched;

}*/

sclass Entry {

  long msgID;

  S originalLine;

  S interpretableText;

  S matchedRuleStruct;

  sS _fieldOrder = "msgID originalLine interpretableText";

}

set flag DynModule.

cmodule InterpretableTest > DynObjectTable<Entry> {

  start {

    addCountToName();

    thread { scan(); }

  }

  visualize {

    ret withCenteredButtons(super.visualize(), "Scan", rThread scan);

  }

  void scan enter {

    new L<Entry> list;

    new Flag interrupted;

    temp AutoCloseable action = dm_currentAction("Scanning chat", rRaiseFlag(interrupted));

    F0<GazelleEvalContext> contextMaker = new GazelleContextCache_v2().fill();

    L<GazelleLine> lines = dm_discord_allLines();

    int i = 0;

    for (GazelleLine line : lines) {

      ++i;

      if (!licensed() || interrupted!) break;

      Collection<S> l = getInterpretables(line, +contextMaker);

      for (S interpretableText : l)

        list.add(nu Entry(originalLine := line.text, msgID := line.msgID, +interpretableText));

      call(action, 'setText, "Found " + n2(list, "interpretable") + ", scanned " + i + "/" + n2(lines, "line"));

    }

    setList(list);

  }

  Collection<S> getInterpretables(GazelleLine line, O... _) {

    Set<S> interpretables = linkedCISet(); // fancy new ordered case-insensitive set

    RuleEngine2 engine = ((F0<GazelleEvalContext>) optPar contextMaker(_)).get().engine;

    for (RuleEngine2.SimplifyWithRule rule : engine.splitterRules()) pcall {

      _addAll(interpretables, gazelle_executeSplitterRule(engine, rule, line.text));

    }

    _addAll(interpretables, collect line(

      dm_gazelle_reasonAboutChatInput_v2(null, line.text, 

        paramsPlus(_, skipSplitters := true, acceptablePurposes := litset('preprocess)))));

    interpretables.remove(line.text);

    ret interpretables;

  }

}