!7

concept Entry {

  long msgID;

  S line;

  S preprocessed;

  S ruleIDs; // rule(s) used for preprocessing

  sS _fieldOrder = "msgID line preprocessed";

}

set flag DynModule.

cmodule GPreprocessedLines > DynCRUD<Entry> {

  transient int maxInterpretablesPerLine = 20;

  start {

    dbIndexing(Entry, 'msgID);

    addCountToName();

  }

  visualize {

    ret withCenteredButtons(super.visualize(), "Full Scan", rThread fullScan);

  }

  void fullScan enter {

    new L<Entry> list;

    new Flag interrupted;

    temp AutoCloseable action = dm_currentAction("Scanning chat", rRaiseFlag(interrupted));

    F0<GazelleEvalContext> contextMaker = new GazelleContextCache_v2().fill();

    L<GazelleLine> lines = dm_discord_allLines();

    for (GazelleLine line : iterateOverListSendingPercentage(voidfunc(O percent) { callOpt(action, 'setText, percent + "% - Scanning chat") }, reversed(lines))) {

      if (!licensed() || interrupted!) break;

      scanLine(line, +contextMaker);

    }

  }

  void scanLine(GazelleLine line, O... _) {

    new L<Entry> list;

    Collection<GInterpretable> l = takeFirst(maxInterpretablesPerLine, gazelle_preprocess(line.text, _));

    for (GInterpretable intp : l) {

      if (eq(intp.text, line.text)) continue;

      list.add(unlistedWithValues Entry(

        msgID := line.msgID,

        line := line.text,

        preprocessed := intp.text,

        ruleIDs := intp.ruleID));

    }

    replaceItemsForMessage(line.msgID, list);

  }

  void replaceItemsForMessage(long msgID, L<Entry> items) {

    deleteConceptsWhere Entry(+msgID);

    registerAll(items);

  }

}