Warning: session_start(): open(/var/lib/php/sessions/sess_eh8fgkig086v566c93u26kkil9, O_RDWR) failed: No space left on device (28) in /var/www/tb-usercake/models/config.php on line 51
Warning: session_start(): Failed to read session data: files (path: /var/lib/php/sessions) in /var/www/tb-usercake/models/config.php on line 51
!7
static Guesser best;
static double bestScore;
concept Sentence {
S text;
SS data;
}
sclass Example {
L tok;
int start, end;
*() {}
*(L *tok, IntRange subjectTokens) {
start = subjectTokens.start;
end = subjectTokens.end;
}
toString {
ret quote(joinWithSpaces(tok)) + " => " + joinWithSpaces(subList(tok, start, end));
}
}
abstract sclass GuesserBase {
void learn(L material) {
for (Example e : material)
learn(e);
}
void learn(Example e) {}
}
abstract sclass Guesser extends GuesserBase {
abstract IntRange getSubjectTokens(L tok);
}
Guesser > GLengthOfSubject {
new MultiSet pos; // words to end on
new MultiSet neg; // words not to end on
IntRange getSubjectTokens(L tok) {
ret getSubjectTokens(tok, 0);
}
IntRange getSubjectTokens(L tok, int startAt) {
int i = startAt;
while (i < l(tok)) {
S t = lower(tok.get(i));
if (pos.get(t) <= neg.get(t)) // also stop if unknown word
break;
++i;
}
ret intRange(startAt, min(l(tok), i+1));
}
void learn(Example e) {
L subjectTokens = allToLower(subList(e.tok, e.start, e.end));
for (S word : dropLast(subjectTokens))
pos.add(word);
addIfNotNull(neg, last(subjectTokens));
}
}
Guesser > GSkip1 { // returns first word or second word
new MultiSet pos; // words to skip
new MultiSet neg; // words not to skip
void learn(Example e) {
(e.start > 0 ? pos : neg).add(lower(first(e.tok));
}
IntRange getSubjectTokens(L tok) {
S t = lower(first(tok));
ret intRangeFromStartAndLength(pos.get(t) > neg.get(t) ? 1 : 0, 1);
}
}
Guesser > GSkip2 { // can skip multiple words
new MultiSet pos; // words to skip
new MultiSet neg; // words not to skip
void learn(Example e) {
(e.start > 0 ? pos : neg).add(lower(first(e.tok));
}
IntRange getSubjectTokens(L tok) {
int i = 0;
while (i < l(tok)) {
S t = lower(tok.get(i));
if (pos.get(t) <= neg.get(t)) // also stop if unknown word
break;
++i;
}
ret intRangeFromStartAndLength(i, i+1);
}
}
Guesser > GCombine {
Guesser a;
new GLengthOfSubject b;
*() {}
*(Guesser *a) {}
IntRange getSubjectTokens(L tok) {
IntRange r = a.getSubjectTokens(tok);
int skip = r == null ? 0 : r.start;
ret b.getSubjectTokens(tok, skip);
}
void learn(L material) {
a.learn(material);
b.learn(material);
}
}
p {
loadConceptsFrom(#1008607);
L material = learningMaterial();
//pnlStruct(material);
// This yields the empty learner
Pair p = bestLearner(material,
//ll(new GSkip1),
ll(new GCombine(new GSkip1), new GCombine(new GSkip2)),
50, 3, true);
// Now we train it with all data for in-program use
p.a.learn(material);
// Print and store
print("Best learner: " + formatDouble(p.b, 1) + "% - " + struct(p.a));
best = p.a;
bestScore = p.b;
}
sbool printDetails, printSuccesses;
static double checkGuesser(L testMaterial, Guesser g) {
print();
int score = 0, n = 0;
for (Example e : testMaterial) {
IntRange r = cast pcall(g, "getSubjectTokens", e.tok);
bool ok = eq(IntRange(e.start, e.end), r);
if (ok) ++score;
++n;
if (printDetails || ok && printSuccesses)
if (ok)
print("OK " + e);
else
print("FAIL " + (r == null ? "-" : joinWithSpaces(subList(e.tok, r.start, r.end))) + " for " + e);
}
printScore(shortClassName(g), score, n);
ret ratioToPercent(score, n);
}
static double checkGuesserAfterRandomizedPartialLearn(L testMaterial, Guesser g, double percentToLearn, bool hardMode) {
Pair> p = getRandomPercent2(testMaterial, percentToLearn);
g.learn(p.a);
ret checkGuesser(hardMode ? p.b : testMaterial, g);
}
// best learner with randomized x% training material
// returns guesser, percentage solved
// hardMode = only count scores on untrained examples
static Pair bestLearner(final L material, L guessers, final double percent, int repetitions, final bool hardMode) {
new Best best;
for (final Guesser g : guessers)
best.put(g, repeatAndAdd_double(repetitions, func {
checkGuesserAfterRandomizedPartialLearn(material, cloneObject(g), percent, hardMode)
})/repetitions);
ret best.pair();
}
static L learningMaterial() {
L out = new L;
for (Sentence s) {
S action = data.get("subject");
if (action == null) continue;
IntRange r = ai_parseSubjectAction(action);
if (r != null) {
L tok = nlTok5(s.text);
r = charRangeToTokenRange(tok, r);
r = tokenRangeToCodeTokens(r);
tok = codeTokens(tok);
out.add(Example(tok, r));
}
}
ret out;
}
// to be called from applications - works on character level
// modifies data
static void callGuesser(Guesser g, S sentence, SS data) {
L tok = nlTok5(sentence);
IntRange r = g.getSubjectTokens(codeTokens(tok));
if (r == null) ret;
r = tokenRangeFromCodeTokens(r);
data.put("subject", ai_renderAction(tokenRangeToCharRange(tok, r)));
}