!7

sclass Interpretable {

  S text;

  S ruleID;

  *() {}

  *(S *text, S *ruleID) {}

}

sclass Entry {

  long msgID;

  S originalLine;

  S interpretableText;

  S ruleID;

  //S matchedRuleStruct;

  sS _fieldOrder = "msgID originalLine interpretableText";

}

set flag DynModule.

cmodule InterpretableTest > DynObjectTable<Entry> {

  start {

    addCountToName();

    thread { scan(); }

  }

  visualize {

    ret withCenteredButtons(super.visualize(), "Scan", rThread scan);

  }

  void scan enter {

    new L<Entry> list;

    new Flag interrupted;

    temp AutoCloseable action = dm_currentAction("Scanning chat", rRaiseFlag(interrupted));

    F0<GazelleEvalContext> contextMaker = new GazelleContextCache_v2().fill();

    L<GazelleLine> lines = dm_discord_allLines();

    int i = 0;

    for (GazelleLine line : reversed(lines)) {

      ++i;

      if (!licensed() || interrupted!) break;

      Collection<Interpretable> l = getInterpretables(line, +contextMaker);

      for (Interpretable intp : l)

        list.add(nu Entry(originalLine := line.text, msgID := line.msgID, interpretableText := intp.text, ruleID := intp.ruleID));

      call(action, 'setText, "Found " + n2(list, "interpretable") + ", scanned " + i + "/" + n2(lines, "line"));

    }

    setList(list);

  }

  Collection<Interpretable> getInterpretables(GazelleLine line, O... _) {

    Set<S> texts = litCISet(line.text);

    new L<Interpretable> interpretables;

    RuleEngine2 engine = ((F0<GazelleEvalContext>) optPar contextMaker(_)).get().engine;

    for (RuleEngine2.SimplifyWithRule rule : engine.splitterRules()) pcall {

      for (S out : gazelle_executeSplitterRule(engine, rule, line.text))

        if (texts.add(out))

          interpretables.add(new Interpretable(out, rule.globalID));

    }

    for (GazelleTree t : dm_gazelle_reasonAboutChatInput_v2(null, line.text, 

      paramsPlus(_, skipSplitters := true, acceptablePurposes := litset('preprocess))))

        if (texts.add(t.line))

          interpretables.add(new Interpretable(t.line, t.ruleID()));

    ret interpretables;

  }

}