// also translates
sclass RecursiveProbabilisticParser1 {
  // example rule:
  // "Ein/eine/der/die/das <ding> führt (nach/in/hin zu/ins) <ort>" => satz => $1 leads to $2

  srecord Production(LS tok, S outClass, S rewritten) {}

  new L<Production> productions;
  Set<S> fillerClasses = litciset("<filler>", "<füller>");
  S sentenceClass = "sentence";
  SS simpleTranslations = ciMap();
  LS sentences;
  LS translations, translations2;
  bool useRoundBrackets; // instead of nested quotes

  L<Production> otherProductions;

  int recursionLevels = 10;

  void loadRules(S rules) {
    for (LS l : lambdaMap splitAtDoubleArrow(tlft(rules))) {
      continue unless isBetween(l(l), 2, 3);
      S lhs = first(l), rhs = second(l), rewritten = third(l);
      //continue unless isQuoted(lhs);
      LS tok = javaTokWithAllPlusAngleBrackets(replaceWithNothing("/...", tok_unquote(lhs)));
      tok = tok_combineSpacelessSlashCombos(tok);
      tok = mapCodeTokens tok_deRoundBracket(tok);
      //printStruct(codeTokens(tok));
      //print(patternToRule(new ProbabilisticParser1, tok));
      productions.add(new Production(tok, deAngleBracket(rhs), rewritten));
    }
  }

  // returns list of translated sentences
  LS parse(S text) {
    setField(sentences := sentences(text));
    
    new LS translations;
    for (S sentence : sentences) {
      print(sentence);
      ProbabilisticParser1.State s = parserForInputAndProductions(javaTok(sentence),
        objectsWhereIC(productions, outClass := sentenceClass)).bestDoneState();
      print("State", s);
      S translated = getTranslation(s);
      addIfNempty(translations, translated);
      //printIfNempty("  => ", translated);
    }

    setField(+translations);

    setField(otherProductions := objectsWhereNotIC(productions, outClass := sentenceClass));

    LS translations2 = map(translations, line -> translateQuotedParts(line, recursionLevels));
    //translations2 = lambdaMap recursiveUnquoteStartingAtLevel2(translations2);
    translations2 = map(line -> join(translatePhrases(javaTokNoQuotes(line), simpleTranslations)), translations2);
    if (useRoundBrackets) translations2 = lambdaMap recursiveQuotesToRoundBrackets(translations2);
    setField(+translations2);
    //pnl(translations2);
    ret translations2;
  }

  LS topLevelPatterns() {
    ret map(translations, s -> jreplace(s, "<quoted>", "*"));
  }

  // also parses
  ProbabilisticParser1 parserForInputAndProductions(LS tok, Cl<Production> productions) {
    new ProbabilisticParser1 parser;
    //parser.verbose = parser.pm.verbose = true;
    //parser.pm.cutoffPercentage = 10;
    for (Production p : productions)
      parser.addState(tok, patternToRule(parser, p.tok)).userObject = p;
    parser.pm.think();
    ret parser;
  }

  BasicLogicRule patternToRule(ProbabilisticParser1 parser, LS tok) {
    ret BasicLogicRule(
      makeAnd(listPlus(
        mapWithIndex(tok, (i, t) -> even(i) || contains(fillerClasses, t)
          ? parser.new Filler
          : isAngleBracketed(t) ? set(parser.new Any(t), minTokensToConsume := 1)
          : containsSlash(t) ? parser.new ConsumeOneOfTokens(asCISet(splitAtSlash(t)))
          : parser.new ConsumeToken(t)),
        parser.new EndOfInput)),
      formatFrag("parsed " + join(tok)));
  }

  S getTranslation(ProbabilisticParser1.State state) {
    if (state == null) null;
    Production prod = cast state.userObject;
    Matches m = state.parser().stateToMatches(state);
    print(+m);
    ret expandDollarRefsToMatches_alwaysQuote(prod.rewritten, m);
  }

  S translateQuotedParts(S line, int recursionLimit) {
    LS tok = javaTok(line);
    for (int i : indicesOfQuoted(tok)) {
      S x = unquote(tok.get(i));
      ProbabilisticParser1.State s = parserForInputAndProductions(javaTok(x),
        otherProductions).bestDoneState();
      //printVars(+x, +s);
      if (s != null && s.probability >= 80) {
        S y = getTranslation(s);
        printVars_str(+x, +y, +s);
        //printVars(+x, +y, probability := s.probability);
        if (recursionLimit > 0) {
          S yy = y;
          y = translateQuotedParts(y, recursionLimit-1);
          //if (neq(y, yy)) printVars(+yy, +y);
        }
        tok.set(i, quote(y));
      }
    }
    ret join(tok);
  }

  event change; !include #1027843 // setField
}