!7

sbool smartBotToo = false;
sS url = "http://localhost:8080/1010504/raw/number-of-threads";
sS mainURL = "http://localhost";
sS smartBotURL = "http://localhost:4678/number-of-threads";
static double timeout = 60.0, delay = 30.0;
static int consecutiveFailsThreshold = /*2*/ 4; // times delay
static int consecutiveFailsThresholdWhenLoading = 20; // not used

static int consecutiveFails;
static volatile S status;
sbool playMode;

p {
  bot("New Eleu Watch Dog.");
  
  if (cic(args, "playMode")) set playMode;
  
  if (smartBotToo) thread "Smart Bot Loop" { smartBotLoop(); }
  
  printWithDateAndTimeInThisThread();
  repeat with sleep delay {
    bool ok = false, loading = false;
    pcall {
      long time = sysNow();
      
      S mainData = loadPageWithTimeout(mainURL, timeout/2);
      if (!startsWith(mainData, "<") || swic(mainData, "ERROR")) {
        print("Error! " + mainData);
      } else {
        S s = loadPageWithTimeout(url, timeout/2);
        if (startsWith(s, "LOADING")) loading = true; // TODO: loading is now code 500
        if (isInteger(s)) {
          ok = true;
          consecutiveFails = 0;
          print(status = "Server OK - " + n2(parseLong(s), "thread", "threads") + ", " + (sysNow()-time) + " ms");
        }
      }
    }
    
    if (!ok) {
      ++consecutiveFails;
      print(status = "Server fail #" + consecutiveFails + " of " + actualThreshold(loading) + (loading ? " (loading)" : ""));
      pcall { serverFailActivity(loading); }
    }
  }
}

static int actualThreshold(bool loading) {
  ret loading ? consecutiveFailsThresholdWhenLoading : consecutiveFailsThreshold;
}

svoid serverFailActivity(bool loading) {
  if (consecutiveFails == actualThreshold(loading)) {
    consecutiveFails = 0;
    if (playMode) ret with print("WOULD RESTART ELEU.");
    print("RESTARTING ELEU.");
    hardKillProgram(#1013896);
    S eleuOptions = loadTextFileTrim(javaxDataDir("eleu.options"));
    directNohupJavax(joinNemptiesWithSpace(eleuOptions, str(psI(#1013896))));
  }
}

answer {
  if "status" ret status;
}

svoid smartBotLoop {
  int consecutiveFails = 0;
  S status = "";
  printWithDateAndTimeInThisThread();
  repeat with sleep delay {
    bool ok = false, loading = false;
    pcall {
      long time = sysNow();
      S s = loadPageWithTimeout(smartBotURL, timeout);
      if (startsWith(s, "LOADING")) loading = true;
      if (isInteger(s)) {
        ok = true;
        consecutiveFails = 0;
        print(status = "Smart Bot OK - " + n2(parseLong(s), "thread", "threads") + ", " + (sysNow()-time) + " ms");
      }
    }
    
    if (!ok) {
      ++consecutiveFails;
      print(status = "Smart Bot fail #" + consecutiveFails + " of " + actualThreshold(loading) + (loading ? " (loading)" : ""));
      pcall {
        if (smartBotFailActivity(consecutiveFails, loading))
          consecutiveFails = 0;
      }
    }
  }
}

sbool smartBotFailActivity(int consecutiveFails, bool loading) {
  if (consecutiveFails >= actualThreshold(loading)) {
    print("RESTARTING SMART BOT.");
    hardKillProgram(#1010745);
    nohupJavaxHeadless(#1010745);
    true;
  }
  false;
}

Began life as a copy of #1014075