!7

concept Page {
  S url;
  S mimeType;
  long loadedWhen, loadTime;
  bool shouldLoad;
  
  File contentsFile() {
    ret javaxCachesDir("Loaded Web Pages/" + uniqueFileNameUsingMD5_80_v2(url));
  }
  
  long contentSize() { ret l(contentsFile()); }
}

cmodule MultiWebpageLoader > DynCRUD<Page> {
  transient Q loaderQ;
  
  start {
    loaderQ = dm_startQ("Loader Queue");
    indexConceptField(Page, 'url);
    crud.renderer = func(Page page) -> Map {
      litorderedmap(
        "URL" := page.url,
        "Content size" := n2(page.contentSize()),
        "Loaded" := formatDateWithSeconds(page.loadedWhen),
        "Load time" := formatDouble(msToSeconds(page.loadTime), 1) + " s",
        "Content type" := page.mimeType,
        "Should load" := page.shouldLoad)
    };
  }
  
  afterVisualize {
    addButton("Add URL...", rThreadEnter {
      inputText("URL to load", voidfunc(S url) enter {
        addURLIfNotKnown(url);
      });
    });
    tablePopupMenuItemsThreaded_top(table(),
      "Load now", rEnter {
        loaderQ.add(r { loadAPage(selected()) })
      },
      "Show source", rEnter {
        Page page = selected();
        dm_showText(page.url + " [Source]", loadTextFile(page.contentsFile()));
      });
  }
  
  void loadAPage(Page page) {
    if (page == null) ret;
    long loadedWhen = now();
    cset(page, shouldLoad := false, +loadedWhen, mimeType := null);
    S mimeType;
    try {
      byte[] content = loadBinaryPage(page.url);
      saveFile(page.contentsFile(), content);
      mimeType = or2(first(lookupPossiblyIgnoreCase(loadBinaryPage_responseHeaders!, "Content-Type")), "?");
    } catch print e {
      deleteFile(page.contentsFile());
      mimeType = "ERROR: " + exceptionToStringShort(e);
    }
    long loadTime = now()-loadedWhen;
    cset(page, +loadTime, +mimeType);
  }
  
  void possiblyLoad(Page page) {
    if (page != null && page.shouldLoad)
      loaderQ.add(r { if (!page.isDeleted() && page.shouldLoad) loadAPage(page) });
  }
  
  // API
  
  void addURLIfNotKnown(S url) {
    possiblyLoad(csetAndReturn(uniq_returnIfNew Page(+url), shouldLoad := true));
  }
}