!7

concept Page {
  S url;
  S mimeType;
  long loadedWhen, loadTime;
  bool shouldLoad;
  
  File contentsFile() {
    ret javaxCachesDir("Loaded Web Pages/" + uniqueFileNameUsingMD5_80_v2(url));
  }
  
  long contentSize() { ret l(contentsFile()); }
}

cmodule MultiWebpageLoader > DynCRUD<Page> {
  start {
    indexConceptField(Page, 'url);
  }
  
  afterVisualize {
    addButton("Add URL...", rThreadEnter {
      inputText("URL to load", voidfunc(S url) enter {
        addURLIfNotKnown(url);
      });
    });
  }
  
  void loadAPage(Page page) {
    long loadedWhen = now();
    cset(page, shouldLoad := false, +loadedWhen, mimeType := null);
    S mimeType;
    try {
      byte[] content = loadBinaryPage(page.url);
      saveFile(page.contentsFile(), content);
      mimeType = or2(first(lookupPossiblyIgnoreCase(loadBinaryPage_responseHeaders!, "Content-Type")), "?");
    } catch print e {
      mimeType = "ERROR: " + exceptionToStringShort(e);
    }
    long loadTime = now()-loadedWhen;
    cset(page, +loadTime, +mimeType);
  }
  
  // API
  
  void addURLIfNotKnown(S url) {
    cset(uniq_returnIfNew Page(+url), shouldLoad := true);
  }
}