Uses 911K of libraries. Click here for Pure Java version (15313L/81K).
1 | !7 |
2 | |
3 | concept Page {
|
4 | S url; |
5 | S mimeType; |
6 | long loadedWhen, loadTime; |
7 | bool shouldLoad; |
8 | |
9 | File contentsFile() {
|
10 | ret javaxCachesDir("Loaded Web Pages/" + uniqueFileNameUsingMD5_80_v2(url));
|
11 | } |
12 | |
13 | long contentSize() { ret l(contentsFile()); }
|
14 | } |
15 | |
16 | cmodule MultiWebpageLoader > DynCRUD<Page> {
|
17 | transient Q loaderQ; |
18 | |
19 | start {
|
20 | loaderQ = dm_startQ("Loader Queue");
|
21 | indexConceptField(Page, 'url); |
22 | crud.renderer = func(Page page) -> Map {
|
23 | litorderedmap( |
24 | "URL" := page.url, |
25 | "Content size" := n2(page.contentSize()), |
26 | "Loaded" := formatDateWithSeconds(page.loadedWhen), |
27 | "Load time" := formatDouble(msToSeconds(page.loadTime), 1) + " s", |
28 | "Content type" := page.mimeType, |
29 | "Should load" := page.shouldLoad) |
30 | }; |
31 | } |
32 | |
33 | afterVisualize {
|
34 | addButton("Add URL...", rThreadEnter {
|
35 | inputText("URL to load", voidfunc(S url) enter {
|
36 | addURLIfNotKnown(url); |
37 | }); |
38 | }); |
39 | tablePopupMenuItemsThreaded_top(table(), |
40 | "Load now", rEnter {
|
41 | loaderQ.add(r { loadAPage(selected()) })
|
42 | }, |
43 | "Show source", rEnter {
|
44 | Page page = selected(); |
45 | dm_showText(page.url + " [Source]", loadTextFile(page.contentsFile())); |
46 | }); |
47 | } |
48 | |
49 | void loadAPage(Page page) {
|
50 | if (page == null) ret; |
51 | long loadedWhen = now(); |
52 | cset(page, shouldLoad := false, +loadedWhen, mimeType := null); |
53 | S mimeType; |
54 | try {
|
55 | byte[] content = loadBinaryPage(page.url); |
56 | saveFile(page.contentsFile(), content); |
57 | mimeType = or2(first(lookupPossiblyIgnoreCase(loadBinaryPage_responseHeaders!, "Content-Type")), "?"); |
58 | } catch print e {
|
59 | deleteFile(page.contentsFile()); |
60 | mimeType = "ERROR: " + exceptionToStringShort(e); |
61 | } |
62 | long loadTime = now()-loadedWhen; |
63 | cset(page, +loadTime, +mimeType); |
64 | } |
65 | |
66 | void possiblyLoad(Page page) {
|
67 | if (page != null && page.shouldLoad) |
68 | loaderQ.add(r { if (!page.isDeleted() && page.shouldLoad) loadAPage(page) });
|
69 | } |
70 | |
71 | // API |
72 | |
73 | void addURLIfNotKnown(S url) {
|
74 | possiblyLoad(csetAndReturn(uniq_returnIfNew Page(+url), shouldLoad := true)); |
75 | } |
76 | } |
download show line numbers debug dex old transpilations
Travelled to 8 computer(s): bhatertpkbcr, iveijnkanddl, mqqgnosmbjvj, pyentgdyhuwx, pzhvpgtvlbxg, tvejysmllsmz, vouqrxazstgt, xrpafgyirdlv
No comments. add comment
| Snippet ID: | #1028050 |
| Snippet name: | Multi Webpage Loader [basically works] |
| Eternal ID of this version: | #1028050/23 |
| Text MD5: | 10bd637001462c67f79beb5b118a4672 |
| Transpilation MD5: | d9fb91ae272e8a1b8fa566bd45615dad |
| Author: | stefan |
| Category: | javax |
| Type: | JavaX source code (Dynamic Module) |
| Public (visible to everyone): | Yes |
| Archived (hidden from active list): | No |
| Created/modified: | 2020-05-14 11:18:06 |
| Source code size: | 2205 bytes / 76 lines |
| Pitched / IR pitched: | No / No |
| Views / Downloads: | 182 / 528 |
| Version history: | 22 change(s) |
| Referenced in: | [show references] |