Not logged in.  Login/Logout/Register | List snippets | | Create snippet | Upload image | Upload data

155
LINES

< > BotCompany Repo | #1016871 - Local Snippets DB v2 [with text on disk, OK]

JavaX source code (desktop) [tags: use-pretranspiled] - run with: x30.jar

Libraryless. Click here for Pure Java version (16278L/119K).

!7

static double updateInterval = 60;
sbool infoBoxes, fullUpdateAtStart = true;
sbool snippetsDB_inited;
sbool autoDownloadText = true;

!include once #1020401 // CSnippet

static Thread downloadThread;

p {
  //restartWith384MBHeap(); // for xfullgrab
  autoRestart10Mins();
  snippetsDB_init();
}

svoid snippetsDB_init() {
  lock programLock();
  if (snippetsDB_inited) ret;
  set snippetsDB_inited;
  db_mainConcepts().autoSaveInterval = 1000*60; // every minute
  db_mainConcepts().noXFullGrab = true;
  dbIndexing(CSnippet, 'snippetID);
  botWithCommandList("Snippets.");
  if (fullUpdateAtStart || countConcepts(CSnippet) == 0) fullDownload();
  doEvery(0, updateInterval, f update);
}

svoid fullDownload { fullDownload(autoDownloadText); }

svoid fullDownload(bool withText) {
  L<Snippet> l = listAllSnippets_fromHourlyCache();
  printWithTime(n(l, "snippet") + " on server");
  slurp(l);
  if (withText) downloadSomeText();
}

static int slurp(L<Snippet> l) {
  lock dbLock();
  int changedSnippets = 0;
  for (Snippet s : l) {
    CSnippet cs = uniq(CSnippet, snippetID := parseSnippetID(s.id));
    int changes = 0;
    changes += cset(cs, title := s.title, type := s.type, isPublic := s.isPublic);
    if (neq(cs.md5, s.md5)) {
      changes += cset(cs, md5 := s.md5);
      if (cs.hasText()) { cs.setText(null); ++changes; }
    }
    if (changes > 0) ++changedSnippets;
  }
  S text = n2(changedSnippets, "changed snippet");
  
  if (infoBoxes && changedSnippets != 0) infoBox(text);
  printWithTime(text);
  ret changedSnippets;
}

// with text if autoDownloadText is true
svoid update {
  L<Snippet> l = listRecentlyChangedSnippets();
  printWithTime(n(l, "snippet") + " on server");
  int changedSnippets = slurp(l);
  if (changedSnippets >= l(l))
    fullDownload(autoDownloadText);
  else if (autoDownloadText)
    downloadSomeText();
}

answer {
  lock dbLock();
  if "full download" {
    fullDownload();
    ret "OK, have " + n(countConcepts(CSnippet), "snippet");
  }
  
  if "update" {
    update();
    ret "OK";
  }
  
  if "download some text" {
    if (downloadThread != null) ret "Already downloading";
    downloadSomeText();
    ret "Yo";
  }
  
  if "stop downloading" {
    cancelThread(downloadThread);
    downloadThread = null;
    ret "OK";
  }
  
  if "get snippet text *" {
    CSnippet c = findConcept(CSnippet, snippetID := parseSnippetID($1));
    if (c == null) ret "not found";
    downloadText(c);
    ret "OK " + quote(c.text());
  }
  
  if "count" ret n2(countConcepts(CSnippet), "snippet");
  
  if "all snippet ids"
    ret struct(map fsI(sorted(collect(list(CSnippet), 'snippetID))));

  if "kill" { cleanKillIn1(); ret "OK"; }
}

svoid downloadText(CSnippet s) {
  if (s.hasText()) ret;
  if (eq(s.md5, emptyMD5())) ret with s.setText("");
  S text;
  try {
    text = loadSnippet(s.snippetID);
  } catch e {
    if (cic(e.getMessage(), "not found"))
      text = "";
    else
      throw rethrow(e);
  }
  s.setText(text);
}

svoid downloadSomeText {
  lock dbLock();
  if (downloadThread != null) ret;
  downloadThread = startThread(r {
    try {
      int n = 0; for (CSnippet s) if (s.hasText()) ++n;
      printWithTime("Have text for " + n + "/" + n(countConcepts(CSnippet), "concept");
      new ConcurrentEvaluator e;
      e.coresToUse = 10;
      for (final CSnippet s) {
        ping();
        if (!s.hasText()) e.add(r {
          ping();
          downloadText(s);
          /*++n;
          if ((n % 100) == 0)
            printWithTime("Have text for " + n + "/" + n(countConcepts(CSnippet), "concept");*/
        });
      }
      e.startWaitCleanUp();
    } finally {
      downloadThread = null;
    }
  });
}

sS getSnippetText(S snippetID) {
  CSnippet c = findConcept(CSnippet, snippetID := parseSnippetID(snippetID));
  if (c == null) null;
  downloadText(c);
  ret c.text();
}

Author comment

Began life as a copy of #1009918

download  show line numbers  debug dex   

Travelled to 15 computer(s): aoiabmzegqzx, bhatertpkbcr, cbybwowwnfue, cfunsshuasjs, gwrvuhgaqvyk, hpgrupgrauku, irmadwmeruwu, ishqpsrjomds, lpdgvwnxivlt, mqqgnosmbjvj, pyentgdyhuwx, pzhvpgtvlbxg, tslmcundralx, tvejysmllsmz, xrpafgyirdlv

No comments. add comment

Snippet ID: #1016871
Snippet name: Local Snippets DB v2 [with text on disk, OK]
Eternal ID of this version: #1016871/19
Text MD5: 62a837ad1893fb7fadec546fe93d29ea
Transpilation MD5: da0e2359379f73a833c8af2cddd398f2
Author: stefan
Category: javax
Type: JavaX source code (desktop)
Public (visible to everyone): Yes
Archived (hidden from active list): No
Created/modified: 2020-07-08 22:42:56
Source code size: 4010 bytes / 155 lines
Pitched / IR pitched: No / No
Views / Downloads: 243 / 1056
Version history: 18 change(s)
Referenced in: [show references]

Formerly at http://tinybrain.de/1016871 & http://1016871.tinybrain.de