!7

concept Slurpable {

  S videoID;

  S label;

  bool slurped;

}

cmodule YouTubeSurfer > DynPrintLogAndEnabled {

  switchable double interval = 10.0;

  // TODO: test case for persisting concept references

  // (set dynamicObjectIsLoading in module while destructuring)

  transient Slurpable lastSlurped;

  start-thread {

    dbIndexing(Slurpable, 'videoID, Slurpable, 'slurped);

    if (noConcepts(Slurpable)) addVideoID("GKvMhYCJA-c");

    doEvery(interval, rEnter { if (enabled) slurpOne() });

    printStats();

    print("Fetch interval: " + interval + " s");

  }

  Slurpable addVideoID(S videoID) {

    ret uniq Slurpable(+videoID);

  }

  Cl<Slurpable> urlsToSlurp() {

    ret conceptsWhere(Slurpable, slurped := false);

  }

  void slurpOne {

    Slurpable s = random(urlsToSlurp()), ret if null;

    print(quote(s.label));

    new Var<S> html;

    for (PairS p : videoLinksFromYouTubePage(s.videoID, +html))

      cset(addVideoID(p.a), label := p.b);

    print(quote(htmlExtractTitle(html!)));

    cset(s, slurped := true);

    setField(lastSlurped := cloneToUnlisted(s));

    printStats();

  }

  void printStats { print(stats()); }

  S stats() {

    ret "URLs: " + countConcepts(Slurpable) + ", slurpable: " + l(urlsToSlurp()) + ". Last slurped: " + getString label(lastSlurped);

  }

}