static new ThreadLocal<S> loadPage_charset;
static boolean loadPage_allowGzip = true, loadPage_debug;
static boolean loadPage_anonymous; // don't send computer ID
static int loadPage_verboseness = 100000;
static int loadPage_retries = 1; //60; // seconds
static new ThreadLocal<Bool> loadPage_silent;
static volatile int loadPage_forcedTimeout; // ms
static new ThreadLocal<Int> loadPage_forcedTimeout_byThread; // ms
static ThreadLocal<Map<S, L<S>>> loadPage_responseHeaders = new ThreadLocal;
static ThreadLocal<SS> loadPage_extraHeaders = new ThreadLocal;

public static String loadPageSilently(String url) ctex {
  return loadPageSilently(new URL(loadPage_preprocess(url)));
}

public static String loadPageSilently(URL url) ctex {
  if (url.getProtocol().equals("https"))
    disableCertificateValidation();
    
  IOException e = null;
  for (int tries = 0; tries < loadPage_retries; tries++)
    try {
      URLConnection con = loadPage_openConnection(url);
      ret loadPage(con, url);
    } catch (IOException _e) {
      e = _e;
      if (loadPageThroughProxy_enabled) {
        print("Trying proxy because of: " + e);
        try {
          ret loadPageThroughProxy(str(url));
        } catch (Throwable e2) {
          print("  " + exceptionToStringShort(e2));
        }
      } else if (loadPage_debug)
        print(e);
      if (tries < loadPage_retries-1) sleepSeconds(1);
    }
  throw e;
}

static String loadPage_preprocess(S url) {  
  if (url.startsWith("tb/")) // don't think we use this anymore
    url = tb_mainServer() + "/" + url;
  if (url.indexOf("://") < 0)
    url = "http://" + url;
  return url;
}

static S loadPage(S url) ctex {
  url = loadPage_preprocess(url);
  if (!isTrue(loadPage_silent!))
    printWithTime("Loading: " + hideCredentials(url));
  ret loadPageSilently(new URL(url));
}

static S loadPage(URL url) {
  ret loadPage(url.toExternalForm());
}

static S loadPage(URLConnection con, URL url) throws IOException {
  ret loadPage(con, url, true);
}

static String loadPage(URLConnection con, URL url, bool addHeaders) throws IOException {
  SS extraHeaders = getAndClearThreadLocal(loadPage_extraHeaders);
  if (addHeaders) try {
    if (!loadPage_anonymous)
      setHeaders(con);
    if (loadPage_allowGzip)
      con.setRequestProperty("Accept-Encoding", "gzip");
    con.setRequestProperty("X-No-Cookies", "1");
    for (S key : keys(extraHeaders))
      con.setRequestProperty(key, extraHeaders.get(key));
  } catch (Throwable e) {} // fails if within doPost
  loadPage_responseHeaders.set(con.getHeaderFields());
  String contentType = con.getContentType();
  if (contentType == null) {
    //printStruct("Headers: ", con.getHeaderFields());
    throw new IOException("Page could not be read: " + url);
  }
  //print("Content-Type: " + contentType);
  String charset = loadPage_charset == null ? null : loadPage_charset.get();
  if (charset == null) charset = loadPage_guessCharset(contentType);
  
  InputStream in = con.getInputStream();
  try {
    if ("gzip".equals(con.getContentEncoding())) {
      if (loadPage_debug)
        print("loadPage: Using gzip.");
      in = newGZIPInputStream(in);
    }
    Reader r = new InputStreamReader(in, charset);
    
    StringBuilder buf = new StringBuilder();
    int n = 0;
    while (true) {
      int ch = r.read();
      if (ch < 0)
        break;
      buf.append((char) ch);
      ++n;
      if ((n % loadPage_verboseness) == 0) print("  " + n + " chars read");
    }
    return buf.toString();
  } finally { in.close(); }
}

static String loadPage_guessCharset(String contentType) {
  Pattern p = Pattern.compile("text/[a-z]+;\\s*charset=([^\\s]+)\\s*");
  Matcher m = p.matcher(contentType);
  S match = m.matches() ? m.group(1) : null;
  if (loadPage_debug)
    print("loadPage: contentType=" + contentType + ", match: " + match);
  /* If Content-Type doesn't match this pre-conception, choose default and hope for the best. */
  //return or(match, "ISO-8859-1");
  return or(match, "UTF-8");
}

static URLConnection loadPage_openConnection(URL url) {
  URLConnection con = openConnection(url);
  int timeout = toInt(loadPage_forcedTimeout_byThread!);
  if (timeout == 0) timeout = loadPage_forcedTimeout;
  if (timeout != 0)
    setURLConnectionTimeouts(con, loadPage_forcedTimeout);
  ret con;
}