// parses & translates

// main probabilistic functions: patternToRule + tokenToAction

sclass RecursiveProbabilisticParser1 {

  // example rule:

  // "Ein/eine/der/die/das <ding> führt (nach/in/hin zu/ins) <ort>" => satz => $1 leads to $2

  BasicLogicRule patternToRule(ProbabilisticParser1 parser, LS tok) {

    ret BasicLogicRule(makeAnd(listPlus(mapWithIndex(tok, (i, t) -> tokenToAction(parser, i, t)),

      parser.new EndOfInput)), formatFrag("parsed " + join(tok)));

  }

  swappable ProbabilisticParser1.Action tokenToAction(ProbabilisticParser1 parser, int i, S t) {

    ret

      even(i) || contains(fillerClasses, t)

        ? parser.new Filler

      : isAngleBracketed(t) || starsAreWildcards && eq(t, "*")

          ? set(parser.new Any(t), minTokensToConsume := minTokensForAny)

      : containsSlash(t)

        ? parser.new ConsumeOneOfTokens(asCISet(splitAtSlash_keepAll(t)))

      : eq(t, "?")

        ? set(parser.new ConsumeToken("?"), emptyProbability := 95.0)

      : parser.new ConsumeToken(t);

  }

  srecord Production(LS tok, S outClass, S rewritten) {}

  new L<Production> productions;

  Set<S> fillerClasses = litciset("<filler>", "<füller>");

  S sentenceClass = "sentence";

  SS simpleTranslations = ciMap();

  LS sentences;

  LS translations, translations2;

  bool requireQuotedProductions = true;

  bool useRoundBrackets; // instead of nested quotes

  bool printSubtranslations;

  bool productionsStartWithClass; // swap first & second entry in production definitions

  bool starsAreWildcards;

  int minTokensForAny = 1; // set to 0 to allow Any to consume 0 tokens

  bool verbose;

  L<Production> otherProductions;

  int recursionLevels = 10;

  // rules can contain Java-style comments

  void loadRules(S rules) {

    for (LS l : lambdaMap splitAtDoubleArrow(tlft_j(rules))) {

      continue unless isBetween(l(l), 2, 3);

      S lhs = first(l), rhs = second(l), rewritten = third(l);

      if (productionsStartWithClass) swap S lhs, rhs;

      if (requireQuotedProductions && !isQuoted(lhs)) {

        if (verbose) print("Production LHS not quoted", l);

        continue;

      }

      LS tok = javaTokWithAllPlusAngleBrackets(replaceWithNothing("/...", tok_unquote(lhs)));

      tok = tok_combineSpacelessSlashCombos(tok);

      tok = mapCodeTokens tok_deRoundBracket(tok);

      //printStruct(codeTokens(tok));

      //print(patternToRule(new ProbabilisticParser1, tok));

      productions.add(new Production(tok, deAngleBracket(rhs), rewritten));

    }

  }

  // returns list of translated sentences

  LS parse(S text, S sentenceClass) {

    this.sentenceClass = sentenceClass;

    ret parse(text);

  }

  // returns list of translated sentences. Can be called multiple times

  LS parse(S text) {

    setField(sentences := splitIntoSentences(text));

    new LS translations;

    for (S sentence : sentences) {

      L<Production> prods = objectsWhereIC(productions, outClass := sentenceClass);

      print("Trying " + nProductions(prods) + " on sentence", sentence);

      ProbabilisticParser1.State s = parserForInputAndProductions(javaTok(sentence),

        prods).bestDoneState();

      print("State", s);

      S translated = getTranslation(s);

      addIfNempty(translations, translated);

      //printIfNempty("  => ", translated);

    }

    setField(+translations);

    setField(otherProductions := objectsWhereNotIC(productions, outClass := sentenceClass));

    LS translations2 = map(translations, line -> translateQuotedParts(line, recursionLevels));

    //translations2 = lambdaMap recursiveUnquoteStartingAtLevel2(translations2);

    translations2 = map(line -> join(translatePhrases(javaTokNoQuotes(line), simpleTranslations)), translations2);

    if (useRoundBrackets) translations2 = lambdaMap recursiveQuotesToRoundBrackets(translations2);

    setField(+translations2);

    //pnl(translations2);

    ret translations2;

  }

  LS topLevelPatterns() {

    ret map(translations, s -> jreplace(s, "<quoted>", "*"));

  }

  // also parses

  ProbabilisticParser1 parserForInputAndProductions(LS tok, Cl<Production> productions) {

    new ProbabilisticParser1 parser;

    parser.verbose = parser.pm.verbose = verbose;

    //parser.pm.cutoffPercentage = 10;

    for (Production p : productions)

      parser.addState(tok, patternToRule(parser, p.tok)).userObject = p;

    parser.pm.think();

    ret parser;

  }

  S getTranslation(ProbabilisticParser1.State state) {

    if (state == null) null;

    Production prod = cast state.userObject;

    Matches m = state.parser().stateToMatches(state);

    //print(+m);

    ret expandDollarRefsToMatches_alwaysQuote(prod.rewritten, m);

  }

  S translateQuotedParts(S line, int recursionLimit) {

    LS tok = javaTok(line);

    for (int i : indicesOfQuoted(tok)) {

      S x = unquote(tok.get(i));

      ProbabilisticParser1.State s = parserForInputAndProductions(javaTok(x),

        otherProductions).bestDoneState();

      //printVars(+x, +s);

      if (s != null && s.probability >= 80) {

        S y = getTranslation(s);

        if (printSubtranslations) printVars_str(+x, +y, +s);

        //printVars(+x, +y, probability := s.probability);

        if (recursionLimit > 0) {

          S yy = y;

          y = translateQuotedParts(y, recursionLimit-1);

          //if (neq(y, yy)) printVars(+yy, +y);

        }

        tok.set(i, quote(y));

      }

    }

    ret join(tok);

  }

  event change; !include #1027843 // setField

  swappable LS splitIntoSentences(S s) {

    ret sentences_dropExclam(s);

  }

}