!7

sS iconID = /*#1101425*/#1101427;
static double delay = 10.0, timeout = 10.0; // change for fast computers
static int consecutiveFailsThreshold = 3; // TODO: change if loading
sS osBotName = "Stefan's OS.";

sbool v6;
static int consecutiveFails;
static volatile S status;
static TrayIcon trayIcon;

svoid activateOS {
  sendOpt("Stefan'S OS", "activate frames");
}

sS osProgramID;

p {
  // TODO
  osProgramID = or(get(args, 0), #1016005);
  S homeDir = get(args, 0);
  
  if (isAbsolutePath(homeDir)) _userHome = args[0];
  print("Home: " + _userHome);
  
  // TODO: put "Restart OS Now" in consoleMemoryView()'s popup menu

  bot(stefansOS_watchDogBotName());
  trayIcon = installTrayIcon(iconID, dropSuffix(".", stefansOS_watchDogBotName()),
    r showConsole,
    "Restart OS Now", r restartOSNow,
    "Show Watch Dog Window", r showConsole,
    "Hide Watch Dog Window", r hideConsole,
    "Exit Watch Dog", rThread cleanKill);
  consoleIcon(iconID);
  //printWithDateAndTimeInThisThread();
  doEvery(delay, r {
    bool ok = false;
    pcall {
      long time = sysNow();
      S s = sendOptWithTimeout(timeout, osBotName, "swing latency");
      if (isInteger(s)) {
        hideConsole();
        ok = true;
        consecutiveFails = 0;
        setStatus("Stefan's OS OK - Swing latency " + s + " ms, " + (sysNow()-time) + " ms");
        //v6 = isProgramRunning(#1016478);
        S progID = evalWithTimeoutOrNull(timeout, func -> S {
          send("Stefan's OS", "program id")
        });
        if (isSnippetID(progID)) osProgramID = or(progID, osProgramID);
      } else
        showConsole();
      consoleStatus(ok ? "OK" : "FAIL");
    }
    
    if (!ok) {
      ++consecutiveFails;
      printAndProgramLog(status = "Stefan's OS fail #" + consecutiveFails + " of " + consecutiveFailsThreshold);
      S stackTraces = sendOptWithTimeout(timeout, osBotName, "stack traces");
      if (nempty(stackTraces)) {
        stackTraces = unquote(stackTraces);
        print(stackTraces);
        saveTextFile(newFile(stefansOS_watchDogStackTracesDir(), "stack-trace-" +  ymd_minus_hms() + ".txt"), stackTraces);
      }
      pcall { osFailActivity(); }
    }
  });
  hideConsole();
  
  quickRestartOnSocketLoss();
  sleep();
}

svoid osFailActivity {
  if (consecutiveFails >= consecutiveFailsThreshold)
    restartOSNow();
}

svoid restartOSNow {
  printAndProgramLog("RESTARTING STEFAN'S OS " + (v6 ? "v6" : "v5") + ".");
  consecutiveFails = 0;
  hardKillStefansOS();
  nohupJavax(osProgramID);
}

answer {
  if "status" ret status;
}

// TODO
svoid quickRestartOnSocketLoss {
  /*int port = getVMPortForBot(osBotName);
  if (port != 0) talkTo(port);*/
}

svoid setStatus(S status) {
  printAndProgramLog(main.status = status);
  setTrayIconToolTip(trayIcon, status);
}

Began life as a copy of #1012699