!752

p {
  typeWriterConsole();
  L<S> paragraphs = rawTextFromODT(userHome("Documents/super-state.odt"));
  printNumberedParagraphs(paragraphs);
  L<S> structures = map(paragraphs, "structureNoQuote");
  MultiSet<S> ms = asMultiSet(structures);
  new L<S> unique;
  for (S s : ms.lowestFirst()) {
    int n = ms.get(s);
    unique.add(n == 1 ? s : "[" + n + "]\n" + s);
  }
  printAsciiHeading("UNIQUE STRUCTURES");
  printNumberedParagraphs(unique);
  print("\n" + nl(unique, "unique structure")
    + " out of " + nl(paragraphs, "paragraph"));
  editText(getPrintLog());
}

static L<S> structure_pre(S s) {
  s = normalizeQuotes(s);
  s = replaceLineBreaksWith(s, " # ");
  ret replaceAll(codeTokens(nlTok(s)), "#", "\n");
}

// probably not so useful, keeps everything in place except white space
static S structure0(S s) {
  ret join(" ", structure_pre(s));
}

// replaces all quoted stuff with "*"
static S structureNoQuote(S s) {
  L<S> tok = structure_pre(s);
  tok = map(tok, func(S t) { isQuoted(t) ? "*" : t });
  ret join(" ", tok).replaceAll(" ?\n ?", "\n");
}