!7

concept Slurpable {
  S videoID;
  S label;
  bool slurped;
}

cmodule YouTubeSurfer > DynPrintLogAndEnabled {
  switchable double interval = 10.0;
  // TODO: test case for persisting concept references
  // (set dynamicObjectIsLoading in module while destructuring)
  transient Slurpable lastSlurped;

  start-thread {
    dbIndexing(Slurpable, 'videoID, Slurpable, 'slurped);
    if (noConcepts(Slurpable)) addVideoID("GKvMhYCJA-c");
    doEvery(interval, rEnter { if (enabled) slurpOne() });
    printStats();
    print("Fetch interval: " + interval + " s");
  }
  
  Slurpable addVideoID(S videoID) {
    ret uniq Slurpable(+videoID);
  }
  
  Cl<Slurpable> urlsToSlurp() {
    ret conceptsWhere(Slurpable, slurped := false);
  }
  
  void slurpOne {
    Slurpable s = random(urlsToSlurp()), ret if null;
    print(quote(s.label));
    new Var<S> html;
    for (PairS p : videoLinksFromYouTubePage(s.videoID, +html))
      cset(addVideoID(p.a), label := p.b);
    print(quote(htmlExtractTitle(html!)));
    cset(s, slurped := true);
    setField(lastSlurped := cloneToUnlisted(s));
    printStats();
  }
  
  void printStats { print(stats()); }
  S stats() {
    ret "URLs: " + countConcepts(Slurpable) + ", slurpable: " + l(urlsToSlurp()) + ". Last slurped: " + getString label(lastSlurped);
  }
}