!7

static double updateInterval = 60;

sbool infoBoxes, fullUpdateAtStart = true;

sbool snippetsDB_inited;

sbool autoDownloadText = true;

!include once #1020401 // CSnippet

static Thread downloadThread;

p {

  //restartWith384MBHeap(); // for xfullgrab

  autoRestart10Mins();

  snippetsDB_init();

}

svoid snippetsDB_init() {

  lock programLock();

  if (snippetsDB_inited) ret;

  set snippetsDB_inited;

  db_mainConcepts().autoSaveInterval = 1000*60; // every minute

  db_mainConcepts().noXFullGrab = true;

  dbIndexing(CSnippet, 'snippetID);

  botWithCommandList("Snippets.");

  int n = countConcepts(CSnippet);

  print("Have " + n + " snippet concepts");

  int nUnique = countUnique(collect snippetID(list(CSnippet)));

  print("Have " + nUnique + " snippet IDs");

  if (fullUpdateAtStart || countConcepts(CSnippet) == 0) fullDownload();

  doEvery(0, updateInterval, r update);

}

svoid fullDownload { fullDownload(autoDownloadText); }

svoid fullDownload(bool withText) {

  L<Snippet> l = listAllSnippets_fromHourlyCache();

  printWithTime(n(l, "snippet") + " on server");

  slurp(l);

  if (withText) downloadSomeText();

}

static int slurp(L<Snippet> l) {

  lock dbLock();

  int changedSnippets = 0;

  for (Snippet s : l) {

    CSnippet cs = uniq(CSnippet, snippetID := parseSnippetID(s.id));

    int changes = 0;

    changes += cset(cs, title := s.title, type := s.type, isPublic := s.isPublic);

    if (neq(cs.md5, s.md5)) {

      changes += cset(cs, md5 := s.md5);

      if (cs.hasText()) { cs.setText(null); ++changes; }

    }

    if (changes > 0) ++changedSnippets;

  }

  S text = n2(changedSnippets, "changed snippet");

  if (infoBoxes && changedSnippets != 0) infoBox(text);

  printWithTime(text);

  ret changedSnippets;

}

// with text if autoDownloadText is true

svoid update {

  L<Snippet> l = listRecentlyChangedSnippets();

  printWithTime(n(l, "snippet") + " on server");

  int changedSnippets = slurp(l);

  if (changedSnippets >= l(l))

    fullDownload(autoDownloadText);

  else if (autoDownloadText)

    downloadSomeText();

}

answer {

  lock dbLock();

  if "full download" {

    fullDownload();

    ret "OK, have " + n(countConcepts(CSnippet), "snippet");

  }

  if "update" {

    update();

    ret "OK";

  }

  if "download some text" {

    if (downloadThread != null) ret "Already downloading";

    downloadSomeText();

    ret "Yo";

  }

  if "stop downloading" {

    cancelThread(downloadThread);

    downloadThread = null;

    ret "OK";

  }

  if "get snippet text *" {

    CSnippet c = findConcept(CSnippet, snippetID := parseSnippetID($1));

    if (c == null) ret "not found";

    downloadText(c);

    ret "OK " + quote(c.text());

  }

  if "count" ret n2(countConcepts(CSnippet), "snippet");

  if "all snippet ids"

    ret struct(map fsI(sorted(collectStrings snippetID(list(CSnippet)))));

  if "kill" { cleanKillIn1(); ret "OK"; }

}

svoid downloadText(CSnippet s) {

  if (s.hasText()) ret;

  if (eq(s.md5, emptyMD5())) ret with s.setText("");

  S text;

  try {

    text = loadSnippet(s.snippetID);

  } catch e {

    if (cic(e.getMessage(), "not found"))

      text = "";

    else

      throw rethrow(e);

  }

  s.setText(text);

}

svoid downloadSomeText {

  lock dbLock();

  if (downloadThread != null) ret;

  downloadThread = startThread(r {

    try {

      int n = 0; for (CSnippet s) if (s.hasText()) ++n;

      printWithTime("Have text for " + n + "/" + n(countConcepts(CSnippet), "concept");

      new ConcurrentEvaluator e;

      e.coresToUse = 10;

      for (final CSnippet s) {

        ping();

        if (!s.hasText()) e.add(r {

          ping();

          downloadText(s);

          /*++n;

          if ((n % 100) == 0)

            printWithTime("Have text for " + n + "/" + n(countConcepts(CSnippet), "concept");*/

        });

      }

      e.startWaitCleanUp();

    } finally {

      downloadThread = null;

    }

  });

}

sS getSnippetText(S snippetID) {

  CSnippet c = findConcept(CSnippet, snippetID := parseSnippetID(snippetID));

  if (c == null) null;

  downloadText(c);

  ret c.text();

}