Not logged in.  Login/Logout/Register | List snippets | | Create snippet | Upload image | Upload data

160
LINES

< > BotCompany Repo | #1016871 // Local Snippets DB v2 [with text on disk, OK]

JavaX source code (desktop) [tags: use-pretranspiled] - run with: x30.jar

Download Jar. Transpiled version (21096L) is out of date.

1  
!7
2  
3  
static double updateInterval = 60;
4  
sbool infoBoxes, fullUpdateAtStart = true;
5  
sbool snippetsDB_inited;
6  
sbool autoDownloadText = true;
7  
8  
!include once #1020401 // CSnippet
9  
10  
static Thread downloadThread;
11  
12  
p {
13  
  //restartWith384MBHeap(); // for xfullgrab
14  
  autoRestart10Mins();
15  
  snippetsDB_init();
16  
}
17  
18  
svoid snippetsDB_init() {
19  
  lock programLock();
20  
  if (snippetsDB_inited) ret;
21  
  set snippetsDB_inited;
22  
  db_mainConcepts().autoSaveInterval = 1000*60; // every minute
23  
  db_mainConcepts().noXFullGrab = true;
24  
  dbIndexing(CSnippet, 'snippetID);
25  
  botWithCommandList("Snippets.");
26  
  int n = countConcepts(CSnippet);
27  
  print("Have " + n + " snippet concepts");
28  
  int nUnique = countUnique(collect snippetID(list(CSnippet)));
29  
  print("Have " + nUnique + " snippet IDs");
30  
  
31  
  if (fullUpdateAtStart || countConcepts(CSnippet) == 0) fullDownload();
32  
  doEvery(0, updateInterval, r update);
33  
}
34  
35  
svoid fullDownload { fullDownload(autoDownloadText); }
36  
37  
svoid fullDownload(bool withText) {
38  
  L<Snippet> l = listAllSnippets_fromHourlyCache();
39  
  printWithTime(n(l, "snippet") + " on server");
40  
  slurp(l);
41  
  if (withText) downloadSomeText();
42  
}
43  
44  
static int slurp(L<Snippet> l) {
45  
  lock dbLock();
46  
  int changedSnippets = 0;
47  
  for (Snippet s : l) {
48  
    CSnippet cs = uniq(CSnippet, snippetID := parseSnippetID(s.id));
49  
    int changes = 0;
50  
    changes += cset(cs, title := s.title, type := s.type, isPublic := s.isPublic);
51  
    if (neq(cs.md5, s.md5)) {
52  
      changes += cset(cs, md5 := s.md5);
53  
      if (cs.hasText()) { cs.setText(null); ++changes; }
54  
    }
55  
    if (changes > 0) ++changedSnippets;
56  
  }
57  
  S text = n2(changedSnippets, "changed snippet");
58  
  
59  
  if (infoBoxes && changedSnippets != 0) infoBox(text);
60  
  printWithTime(text);
61  
  ret changedSnippets;
62  
}
63  
64  
// with text if autoDownloadText is true
65  
svoid update {
66  
  L<Snippet> l = listRecentlyChangedSnippets();
67  
  printWithTime(n(l, "snippet") + " on server");
68  
  int changedSnippets = slurp(l);
69  
  if (changedSnippets >= l(l))
70  
    fullDownload(autoDownloadText);
71  
  else if (autoDownloadText)
72  
    downloadSomeText();
73  
}
74  
75  
answer {
76  
  lock dbLock();
77  
  if "full download" {
78  
    fullDownload();
79  
    ret "OK, have " + n(countConcepts(CSnippet), "snippet");
80  
  }
81  
  
82  
  if "update" {
83  
    update();
84  
    ret "OK";
85  
  }
86  
  
87  
  if "download some text" {
88  
    if (downloadThread != null) ret "Already downloading";
89  
    downloadSomeText();
90  
    ret "Yo";
91  
  }
92  
  
93  
  if "stop downloading" {
94  
    cancelThread(downloadThread);
95  
    downloadThread = null;
96  
    ret "OK";
97  
  }
98  
  
99  
  if "get snippet text *" {
100  
    CSnippet c = findConcept(CSnippet, snippetID := parseSnippetID($1));
101  
    if (c == null) ret "not found";
102  
    downloadText(c);
103  
    ret "OK " + quote(c.text());
104  
  }
105  
  
106  
  if "count" ret n2(countConcepts(CSnippet), "snippet");
107  
  
108  
  if "all snippet ids"
109  
    ret struct(map fsI(sorted(collectStrings snippetID(list(CSnippet)))));
110  
111  
  if "kill" { cleanKillIn1(); ret "OK"; }
112  
}
113  
114  
svoid downloadText(CSnippet s) {
115  
  if (s.hasText()) ret;
116  
  if (eq(s.md5, emptyMD5())) ret with s.setText("");
117  
  S text;
118  
  try {
119  
    text = loadSnippet(s.snippetID);
120  
  } catch e {
121  
    if (cic(e.getMessage(), "not found"))
122  
      text = "";
123  
    else
124  
      throw rethrow(e);
125  
  }
126  
  s.setText(text);
127  
}
128  
129  
svoid downloadSomeText {
130  
  lock dbLock();
131  
  if (downloadThread != null) ret;
132  
  downloadThread = startThread(r {
133  
    try {
134  
      int n = 0; for (CSnippet s) if (s.hasText()) ++n;
135  
      printWithTime("Have text for " + n + "/" + n(countConcepts(CSnippet), "concept");
136  
      new ConcurrentEvaluator e;
137  
      e.coresToUse = 10;
138  
      for (final CSnippet s) {
139  
        ping();
140  
        if (!s.hasText()) e.add(r {
141  
          ping();
142  
          downloadText(s);
143  
          /*++n;
144  
          if ((n % 100) == 0)
145  
            printWithTime("Have text for " + n + "/" + n(countConcepts(CSnippet), "concept");*/
146  
        });
147  
      }
148  
      e.startWaitCleanUp();
149  
    } finally {
150  
      downloadThread = null;
151  
    }
152  
  });
153  
}
154  
155  
sS getSnippetText(S snippetID) {
156  
  CSnippet c = findConcept(CSnippet, snippetID := parseSnippetID(snippetID));
157  
  if (c == null) null;
158  
  downloadText(c);
159  
  ret c.text();
160  
}

Author comment

Began life as a copy of #1009918

download  show line numbers  debug dex  old transpilations   

Travelled to 16 computer(s): aoiabmzegqzx, bhatertpkbcr, cbybwowwnfue, cfunsshuasjs, gwrvuhgaqvyk, hpgrupgrauku, irmadwmeruwu, ishqpsrjomds, lpdgvwnxivlt, mqqgnosmbjvj, pyentgdyhuwx, pzhvpgtvlbxg, tslmcundralx, tvejysmllsmz, vouqrxazstgt, xrpafgyirdlv

No comments. add comment

Snippet ID: #1016871
Snippet name: Local Snippets DB v2 [with text on disk, OK]
Eternal ID of this version: #1016871/23
Text MD5: 704a758c13a1034109bacf93c39b9d9d
Author: stefan
Category: javax
Type: JavaX source code (desktop)
Public (visible to everyone): Yes
Archived (hidden from active list): No
Created/modified: 2022-03-12 23:44:19
Source code size: 4211 bytes / 160 lines
Pitched / IR pitched: No / No
Views / Downloads: 580 / 2034
Version history: 22 change(s)
Referenced in: [show references]