1 | please include function webs_readTripleFileAsTriples. |
2 | |
3 | static L<TripleWeb> webs_readTripleFileAsTriples_parallel2(File f) ctex { |
4 | if (!f.exists()) ret ll(); |
5 | time "Getting names" { |
6 | Iterator<S> it = linesFromFile(f); |
7 | final new L<Symbol> names; |
8 | while (it.hasNext()) { |
9 | S s = trim(it.next()); |
10 | if (empty(s)) break; |
11 | names.add(symbol(unquote(s))); |
12 | } |
13 | } |
14 | |
15 | if (webs_readTripleFileAsTriples_debug) |
16 | print("Have " + l(names) + " names"); |
17 | |
18 | final L<Pair<L<TripleWeb>, Int>> triples = new L; |
19 | |
20 | int poolSize = numberOfCores(), queueSize = 10000; |
21 | int blockSize = 1000; |
22 | NotifyingBlockingThreadPoolExecutor e = new(poolSize, queueSize, 15, TimeUnit.SECONDS); |
23 | print("Using " + poolSize + " cores, block size: " + blockSize + "."); |
24 | |
25 | try { |
26 | int i = 0; |
27 | L<S> block = emptyListWithCapacity(blockSize); |
28 | while licensed { |
29 | ++i; |
30 | final Int _i = i; |
31 | bool hasNext = it.hasNext(); |
32 | if ((!hasNext || l(block) >= blockSize) && nempty(block)) { |
33 | final L<S> _block = cloneList(block); |
34 | e.execute(r { pcall { |
35 | L<TripleWeb> l = emptyListWithCapacity(_block); |
36 | for (S s : _block) { |
37 | TripleWeb w = webs_readTripleFileAsTriples_line(s, names); |
38 | if (w != null) |
39 | l.add(w); |
40 | } |
41 | synchronized(triples) { |
42 | triples.add(pair(l, _i)); |
43 | } |
44 | }}); |
45 | block.clear(); |
46 | } |
47 | if (hasNext) |
48 | block.add(it.next()); |
49 | else |
50 | break; |
51 | } |
52 | e.shutdown(); |
53 | e.awaitTermination(1, TimeUnit.DAYS); |
54 | } finally { |
55 | e.shutdown(); |
56 | } |
57 | |
58 | sortBySecondOfPairs_inPlace(triples); |
59 | int n = 0; |
60 | for (Pair<L<TripleWeb>, Int> p : triples) |
61 | n += l(p.a); |
62 | L<TripleWeb> l = emptyListWithCapacity(n); |
63 | for (Pair<L<TripleWeb>, Int> p : triples) |
64 | l.addAll(p.a); |
65 | ret l; |
66 | } |
Began life as a copy of #1012453
download show line numbers debug dex old transpilations
Travelled to 13 computer(s): aoiabmzegqzx, bhatertpkbcr, cbybwowwnfue, cfunsshuasjs, gwrvuhgaqvyk, ishqpsrjomds, lpdgvwnxivlt, mqqgnosmbjvj, pyentgdyhuwx, pzhvpgtvlbxg, tslmcundralx, tvejysmllsmz, vouqrxazstgt
No comments. add comment
Snippet ID: | #1012454 |
Snippet name: | webs_readTripleFileAsTriples_parallel2 |
Eternal ID of this version: | #1012454/10 |
Text MD5: | b6d35ec8cdf426977c05a0a62697eae9 |
Author: | stefan |
Category: | javax / a.i. |
Type: | JavaX fragment (include) |
Public (visible to everyone): | Yes |
Archived (hidden from active list): | No |
Created/modified: | 2017-12-31 21:00:22 |
Source code size: | 1906 bytes / 66 lines |
Pitched / IR pitched: | No / No |
Views / Downloads: | 407 / 427 |
Version history: | 9 change(s) |
Referenced in: | [show references] |