1 files changed, 52 insertions, 36 deletions

diff --git a/src/server/ChildProcessUtilities/daemon/session.ts b/src/server/ChildProcessUtilities/daemon/session.ts
index 1a3ce61af..833e90581 100644
--- a/src/server/ChildProcessUtilities/daemon/session.ts
+++ b/src/server/ChildProcessUtilities/daemon/session.ts
@@ -6,9 +6,9 @@ import { MailOptions } from "nodemailer/lib/json-transport";
 import { writeFileSync, existsSync, mkdirSync } from "fs";
 import { resolve } from 'path';
 import { ChildProcess, exec } from "child_process";
-// import * as killport from "kill-port";
+const killport = require("kill-port");
 
-const identifier = yellow("__daemon__:");
+const identifier = yellow("__session_manager__:");
 
 process.on('SIGINT', () => current_backup?.kill("SIGTERM"));
 
@@ -39,21 +39,23 @@ if (!["win32", "darwin"].includes(process.platform)) {
 
 const onWindows = process.platform === "win32";
 const LOCATION = "http://localhost";
+const heartbeat = `${LOCATION}:1050/serverHeartbeat`;
 const recipient = "samuel_wilkins@brown.edu";
 const frequency = 10;
 const { pid } = process;
 let restarting = false;
+let count = 0;
 
-function startCommand() {
+function startServerCommand() {
     if (onWindows) {
         return '"C:\\Program Files\\Git\\git-bash.exe" -c "npm run start-release"';
     }
     return `osascript -e 'tell app "Terminal"\ndo script "cd ${pathFromRoot()} && npm run start-release"\nend tell'`;
 }
 
-identifiedLog("Initializing daemon...");
+identifiedLog("Initializing session...");
 
-writeLocalPidLog("daemon", pid);
+writeLocalPidLog("session_manager", pid);
 
 function writeLocalPidLog(filename: string, contents: any) {
     const path = `./logs/current_${filename}_pid.log`;
@@ -65,34 +67,39 @@ function timestamp() {
     return `@ ${new Date().toISOString()}`;
 }
 
+async function clear_ports(...targets: number[]) {
+    return Promise.all(targets.map(port => {
+        const task = killport(port, 'tcp');
+        return task.catch((error: any) => identifiedLog(red(error)));
+    }));
+}
+
 let current_backup: ChildProcess | undefined = undefined;
 
-async function listen() {
-    identifiedLog(yellow(`Beginning to poll server heartbeat every ${frequency} seconds...\n`));
-    if (!LOCATION) {
-        identifiedLog(red("No location specified for persistence daemon. Please include as a command line environment variable or in a .env file."));
-        process.exit(0);
-    }
-    const heartbeat = `${LOCATION}:1050/serverHeartbeat`;
-    // if this is on our remote server, the server must be run in release mode
-    // const suffix = LOCATION.includes("localhost") ? "" : "-release";
-    setInterval(async () => {
-        let error: any;
-        try {
-            await request.get(heartbeat);
-            if (restarting) {
-                addLogEntry("Backup server successfully restarted", green);
-            }
-            restarting = false;
-        } catch (e) {
-            error = e;
-        } finally {
-            if (error) {
-                if (!restarting) {
-                    restarting = true;
+async function checkHeartbeat() {
+    let error: any;
+    try {
+        await request.get(heartbeat);
+        if (restarting) {
+            addLogEntry(`Backup server successfully ${count ? "restarted" : "started"}`, green);
+            count++;
+        }
+        restarting = false;
+    } catch (e) {
+        error = e;
+    } finally {
+        if (error) {
+            if (!restarting) {
+                restarting = true;
+                if (count) {
+                    console.log();
                     addLogEntry("Detected a server crash", red);
-                    current_backup?.kill();
-                    // await killport(1050, 'tcp');
+                    current_backup?.kill("SIGTERM");
+
+                    identifiedLog(yellow("Cleaning up previous connections..."));
+                    await clear_ports(1050, 4321);
+                    identifiedLog(yellow("Finished attempting to clear all ports. Any failures will be printed in red immediately above."));
+
                     await log_execution({
                         startMessage: identifier + " Sending crash notification email",
                         endMessage: ({ error, result }) => {
@@ -102,15 +109,24 @@ async function listen() {
                         action: async () => notify(error || "Hmm, no error to report..."),
                         color: cyan
                     });
+
                     identifiedLog(green("Initiating server restart..."));
-                    current_backup = exec(startCommand(), err => identifiedLog(err?.message || "Previous server process exited."));
-                    writeLocalPidLog("server", `${(current_backup?.pid ?? -2) + 1} created ${timestamp()}`);
-                } else {
-                    identifiedLog(yellow(`Callback ignored because restarting already initiated ${timestamp()}`));
                 }
+                current_backup = exec(startServerCommand(), err => identifiedLog(err?.message || count ? "Previous server process exited." : "Spawned initial server."));
+                writeLocalPidLog("server", `${(current_backup?.pid ?? -2) + 1} created ${timestamp()}`);
             }
         }
-    }, 1000 * 10);
+    }
+}
+
+async function startListening() {
+    identifiedLog(yellow(`After initialization, will poll server heartbeat every ${frequency} seconds...\n`));
+    if (!LOCATION) {
+        identifiedLog(red("No location specified for session manager. Please include as a command line environment variable or in a .env file."));
+        process.exit(0);
+    }
+    await checkHeartbeat();
+    setInterval(checkHeartbeat, 1000 * frequency);
 }
 
 function emailText(error: any) {
@@ -141,4 +157,4 @@ async function notify(error: any) {
     });
 }
 
-listen();
\ No newline at end of file
+startListening();
\ No newline at end of file