!7

static double updateInterval = 60;

concept CSnippet {
  long id;
  S type;
  S title, md5, text;
}

static Thread downloadThread;

p-subst {
  autoRestart10Mins();
  mainConcepts.autoSaveInterval = 1000*60; // every minute
  dbIndexing(CSnippet, 'id);
  botWithCommandList("Snippets.");
  doEvery(0, updateInterval, f update);
}

svoid fullDownload {
  L<Snippet> l = listAllSnippets();
  printWithTime(n(l, "snippet") + " on server");
  slurp(l);
}

static int slurp(L<Snippet> l) {
  lock dbLock();
  int changes = 0;
  for (Snippet s : l) {
    CSnippet cs = uniq(CSnippet, id := parseSnippetID(s.id));
    changes += cset(cs, title := s.title, type := s.type);
    if (neq(cs.md5, s.md5))
      changes += cset(cs, md5 := s.md5, text := null);
  }
  printWithTime(n(changes, "change"));
  ret changes;
}

// with text
svoid update {
  L<Snippet> l = listRecentlyChangedSnippets();
  printWithTime(n(l, "snippet") + " on server");
  slurp(l);
  downloadSomeTextInParallel();
}

answer {
  lock dbLock();
  if "full download" {
    fullDownload();
    ret "OK, have " + n(countConcepts(CSnippet), "snippet");
  }
  
  if "update" {
    update();
    ret "OK";
  }
  
  if "download some text" {
    if (downloadThread != null) ret "Already downloading";
    downloadSomeTextInParallel();
    ret "Yo";
  }
  
  if "download some text serially" {
    if (downloadThread != null) ret "Already downloading";
    downloadSomeText();
    ret "Yo";
  }
  
  if "stop downloading" {
    cancelThread(downloadThread);
    downloadThread = null;
    ret "OK";
  }
  
  if "get snippet text *" {
    CSnippet c = findConcept(CSnippet, id := parseSnippetID($1));
    if (c == null) ret "not found";
    downloadText(c);
    ret "OK " + quote(c.text);
  }
}

svoid downloadText(CSnippet s) {
  if (s.text != null) ret;
  cset(s, text := eq(s.md5, emptyMD5()) ? "" : loadSnippet(s.id));
}

svoid downloadSomeText {
  lock dbLock();
  if (downloadThread != null) ret;
  downloadThread = startThread(r {
    try {
      int n = 0; for (CSnippet s) if (s.text != null) ++n;
      printWithTime("Have text for " + n + "/" + n(countConcepts(CSnippet), "concept");
      for (final CSnippet s) {
        ping();
        if (s.text == null) {
          downloadText(s);
          ++n;
          if ((n % 100) == 0)
            printWithTime("Have text for " + n + "/" + n(countConcepts(CSnippet), "concept");
        }
      }
    } finally {
      downloadThread = null;
    }
  });
}

svoid downloadSomeTextInParallel {
  lock dbLock();
  if (downloadThread != null) ret;
  downloadThread = startThread(r {
    try {
      int n = 0; for (CSnippet s) if (s.text != null) ++n;
      printWithTime("Have text for " + n + "/" + n(countConcepts(CSnippet), "concept");
      new ConcurrentEvaluator e;
      e.coresToUse = 10;
      for (final CSnippet s) {
        ping();
        if (s.text == null) e.add(r {
          ping();
          downloadText(s);
          /*++n;
          if ((n % 100) == 0)
            printWithTime("Have text for " + n + "/" + n(countConcepts(CSnippet), "concept");*/
        });
      }
      try {
        e.start();
        while (!e.done()) sleepSeconds(1);
      } finally {
        e.cancel();
      }
    } finally {
      downloadThread = null;
    }
  });
}