!7

sclass Interpretable {

  S text;

  S ruleID;

  *() {}

  *(S *text, S *ruleID) {}

}

sclass Entry {

  long msgID;

  S originalLine;

  S interpretableText;

  S ruleID;

  //S matchedRuleStruct;

  sS _fieldOrder = "msgID originalLine interpretableText";

}

set flag DynModule.

cmodule InterpretableTest > DynObjectTable<Entry> {

  transient int maxInterpretablesPerLine = 20;

  start {

    addCountToName();

    thread { scan(); }

  }

  visualize {

    ret withCenteredButtons(super.visualize(), "Scan", rThread scan);

  }

  void scan enter {

    new L<Entry> list;

    new Flag interrupted;

    temp AutoCloseable action = dm_currentAction("Scanning chat", rRaiseFlag(interrupted));

    F0<GazelleEvalContext> contextMaker = new GazelleContextCache_v2().fill();

    L<GazelleLine> lines = dm_discord_allLines();

    int i = 0;

    for (GazelleLine line : reversed(lines)) {

      ++i;

      if (!licensed() || interrupted!) break;

      Collection<Interpretable> l = getInterpretables(line, +contextMaker);

      for (Interpretable intp : l)

        list.add(nu Entry(originalLine := line.text, msgID := line.msgID, interpretableText := intp.text, ruleID := intp.ruleID));

      call(action, 'setText, "Found " + n2(list, "interpretable") + ", scanned " + i + "/" + n2(lines, "line"));

    }

    setList(list);

  }

  Collection<Interpretable> getInterpretables(GazelleLine line, O... _) {

    Set<S> texts = litCISet(line.text);

    new L<Interpretable> interpretables;

    Collector<Interpretable> collector = limitedCollector(interpretables, maxInterpretablesPerLine);

    RuleEngine2 engine = ((F0<GazelleEvalContext>) optPar contextMaker(_)).get().engine;

    int nInterpreted = -1;

    bigLoop: while (!collector.full() && nInterpreted < l(interpretables)) {

      Interpretable workingOn = get(interpretables, nInterpreted);

      S input = nInterpreted < 0 ? line.text : workingOn.text;

      S ruleBase = workingOn == null ? "" : workingOn.ruleID + "+";

      ++nInterpreted;

      for (RuleEngine2.SimplifyWithRule rule : engine.splitterRules()) pcall {

        for (S out : gazelle_executeSplitterRule(engine, rule, input))

          if (texts.add(out))

            if (collector.add(new Interpretable(out, ruleBase + rule.globalID))) break bigLoop;

      }

      for (GazelleTree t : dm_gazelle_reasonAboutChatInput_v2(null, input, 

        paramsPlus(_, skipSplitters := true, acceptablePurposes := litset('preprocess))))

          if (texts.add(t.line))

            if (collector.add(new Interpretable(t.line, ruleBase + t.ruleID()))) break bigLoop;

    }

    ret interpretables;

  }

}