!7

concept Entry {
  long msgID;
  S line;
  S preprocessed;
  S ruleIDs; // rule(s) used for preprocessing

  sS _fieldOrder = "msgID line preprocessed";
}

set flag DynModule.

cmodule GPreprocessedLines > DynCRUD<Entry> {
  transient int maxInterpretablesPerLine = 20;
  
  start {
    dbIndexing(Entry, 'msgID);
    addCountToName();
  }
  
  visualize {
    ret withCenteredButtons(super.visualize(), "Full Scan", rThread fullScan);
  }
  
  void fullScan enter {
    new L<Entry> list;
    new Flag interrupted;
    temp AutoCloseable action = dm_currentAction("Scanning chat", rRaiseFlag(interrupted));
    F0<GazelleEvalContext> contextMaker = new GazelleContextCache_v2().fill();
    L<GazelleLine> lines = dm_discord_allLines();
    for (GazelleLine line : iterateOverListSendingPercentage(voidfunc(O percent) { callOpt(action, 'setText, percent + "% - Scanning chat") }, reversed(lines))) {
      if (!licensed() || interrupted!) break;
      scanLine(line, +contextMaker);
    }
  }
  
  void scanLine(GazelleLine line, O... _) {
    new L<Entry> list;
    Collection<GInterpretable> l = takeFirst(maxInterpretablesPerLine, gazelle_preprocess(line.text, _));
    for (GInterpretable intp : l) {
      if (eq(intp.text, line.text)) continue;
      list.add(unlistedWithValues Entry(
        msgID := line.msgID,
        line := line.text,
        preprocessed := intp.text,
        ruleIDs := intp.ruleID));
    }
    replaceItemsForMessage(line.msgID, list);
  }
  
  void replaceItemsForMessage(long msgID, L<Entry> items) {
    deleteConceptsWhere Entry(+msgID);
    registerAll(items);
  }
}

Began life as a copy of #1022191