!7 static double updateInterval = 60; sbool infoBoxes, fullUpdateAtStart = true; sbool snippetsDB_inited; sbool autoDownloadText = true; !include once #1020401 // CSnippet static Thread downloadThread; p { //restartWith384MBHeap(); // for xfullgrab autoRestart10Mins(); snippetsDB_init(); } svoid snippetsDB_init() { lock programLock(); if (snippetsDB_inited) ret; set snippetsDB_inited; db_mainConcepts().autoSaveInterval = 1000*60; // every minute db_mainConcepts().noXFullGrab = true; dbIndexing(CSnippet, 'snippetID); botWithCommandList("Snippets."); int n = countConcepts(CSnippet); print("Have " + n + " snippet concepts"); int nUnique = countUnique(collect snippetID(list(CSnippet))); print("Have " + nUnique + " snippet IDs"); if (fullUpdateAtStart || countConcepts(CSnippet) == 0) fullDownload(); doEvery(0, updateInterval, r update); } svoid fullDownload { fullDownload(autoDownloadText); } svoid fullDownload(bool withText) { L l = listAllSnippets_fromHourlyCache(); printWithTime(n(l, "snippet") + " on server"); slurp(l); if (withText) downloadSomeText(); } static int slurp(L l) { lock dbLock(); int changedSnippets = 0; for (Snippet s : l) { CSnippet cs = uniq(CSnippet, snippetID := parseSnippetID(s.id)); int changes = 0; changes += cset(cs, title := s.title, type := s.type, isPublic := s.isPublic); if (neq(cs.md5, s.md5)) { changes += cset(cs, md5 := s.md5); if (cs.hasText()) { cs.setText(null); ++changes; } } if (changes > 0) ++changedSnippets; } S text = n2(changedSnippets, "changed snippet"); if (infoBoxes && changedSnippets != 0) infoBox(text); printWithTime(text); ret changedSnippets; } // with text if autoDownloadText is true svoid update { L l = listRecentlyChangedSnippets(); printWithTime(n(l, "snippet") + " on server"); int changedSnippets = slurp(l); if (changedSnippets >= l(l)) fullDownload(autoDownloadText); else if (autoDownloadText) downloadSomeText(); } answer { lock dbLock(); if "full download" { fullDownload(); ret "OK, have " + n(countConcepts(CSnippet), "snippet"); } if "update" { update(); ret "OK"; } if "download some text" { if (downloadThread != null) ret "Already downloading"; downloadSomeText(); ret "Yo"; } if "stop downloading" { cancelThread(downloadThread); downloadThread = null; ret "OK"; } if "get snippet text *" { CSnippet c = findConcept(CSnippet, snippetID := parseSnippetID($1)); if (c == null) ret "not found"; downloadText(c); ret "OK " + quote(c.text()); } if "count" ret n2(countConcepts(CSnippet), "snippet"); if "all snippet ids" ret struct(map fsI(sorted(collectStrings snippetID(list(CSnippet))))); if "kill" { cleanKillIn1(); ret "OK"; } } svoid downloadText(CSnippet s) { if (s.hasText()) ret; if (eq(s.md5, emptyMD5())) ret with s.setText(""); S text; try { text = loadSnippet(s.snippetID); } catch e { if (cic(e.getMessage(), "not found")) text = ""; else throw rethrow(e); } s.setText(text); } svoid downloadSomeText { lock dbLock(); if (downloadThread != null) ret; downloadThread = startThread(r { try { int n = 0; for (CSnippet s) if (s.hasText()) ++n; printWithTime("Have text for " + n + "/" + n(countConcepts(CSnippet), "concept"); new ConcurrentEvaluator e; e.coresToUse = 10; for (final CSnippet s) { ping(); if (!s.hasText()) e.add(r { ping(); downloadText(s); /*++n; if ((n % 100) == 0) printWithTime("Have text for " + n + "/" + n(countConcepts(CSnippet), "concept");*/ }); } e.startWaitCleanUp(); } finally { downloadThread = null; } }); } sS getSnippetText(S snippetID) { CSnippet c = findConcept(CSnippet, snippetID := parseSnippetID(snippetID)); if (c == null) null; downloadText(c); ret c.text(); }