Not logged in.  Login/Logout/Register | List snippets | | Create snippet | Upload image | Upload data

76
LINES

< > BotCompany Repo | #1028050 // Multi Webpage Loader [basically works]

JavaX source code (Dynamic Module) [tags: use-pretranspiled] - run with: Stefan's OS

Uses 911K of libraries. Click here for Pure Java version (15313L/81K).

!7

concept Page {
  S url;
  S mimeType;
  long loadedWhen, loadTime;
  bool shouldLoad;
  
  File contentsFile() {
    ret javaxCachesDir("Loaded Web Pages/" + uniqueFileNameUsingMD5_80_v2(url));
  }
  
  long contentSize() { ret l(contentsFile()); }
}

cmodule MultiWebpageLoader > DynCRUD<Page> {
  transient Q loaderQ;
  
  start {
    loaderQ = dm_startQ("Loader Queue");
    indexConceptField(Page, 'url);
    crud.renderer = func(Page page) -> Map {
      litorderedmap(
        "URL" := page.url,
        "Content size" := n2(page.contentSize()),
        "Loaded" := formatDateWithSeconds(page.loadedWhen),
        "Load time" := formatDouble(msToSeconds(page.loadTime), 1) + " s",
        "Content type" := page.mimeType,
        "Should load" := page.shouldLoad)
    };
  }
  
  afterVisualize {
    addButton("Add URL...", rThreadEnter {
      inputText("URL to load", voidfunc(S url) enter {
        addURLIfNotKnown(url);
      });
    });
    tablePopupMenuItemsThreaded_top(table(),
      "Load now", rEnter {
        loaderQ.add(r { loadAPage(selected()) })
      },
      "Show source", rEnter {
        Page page = selected();
        dm_showText(page.url + " [Source]", loadTextFile(page.contentsFile()));
      });
  }
  
  void loadAPage(Page page) {
    if (page == null) ret;
    long loadedWhen = now();
    cset(page, shouldLoad := false, +loadedWhen, mimeType := null);
    S mimeType;
    try {
      byte[] content = loadBinaryPage(page.url);
      saveFile(page.contentsFile(), content);
      mimeType = or2(first(lookupPossiblyIgnoreCase(loadBinaryPage_responseHeaders!, "Content-Type")), "?");
    } catch print e {
      deleteFile(page.contentsFile());
      mimeType = "ERROR: " + exceptionToStringShort(e);
    }
    long loadTime = now()-loadedWhen;
    cset(page, +loadTime, +mimeType);
  }
  
  void possiblyLoad(Page page) {
    if (page != null && page.shouldLoad)
      loaderQ.add(r { if (!page.isDeleted() && page.shouldLoad) loadAPage(page) });
  }
  
  // API
  
  void addURLIfNotKnown(S url) {
    possiblyLoad(csetAndReturn(uniq_returnIfNew Page(+url), shouldLoad := true));
  }
}

download  show line numbers  debug dex  old transpilations   

Travelled to 8 computer(s): bhatertpkbcr, iveijnkanddl, mqqgnosmbjvj, pyentgdyhuwx, pzhvpgtvlbxg, tvejysmllsmz, vouqrxazstgt, xrpafgyirdlv

No comments. add comment

-
Snippet ID: #1028050
Snippet name: Multi Webpage Loader [basically works]
Eternal ID of this version: #1028050/23
Text MD5: 10bd637001462c67f79beb5b118a4672
Transpilation MD5: d9fb91ae272e8a1b8fa566bd45615dad
Author: stefan
Category: javax
Type: JavaX source code (Dynamic Module)
Public (visible to everyone): Yes
Archived (hidden from active list): No
Created/modified: 2020-05-14 11:18:06
Source code size: 2205 bytes / 76 lines
Pitched / IR pitched: No / No
Views / Downloads: 178 / 523
Version history: 22 change(s)
Referenced in: