stable, clustered session manager

4 files changed, 175 insertions, 38 deletions

diff --git a/log.txt b/log.txt
index 6476c26ec..09066c190 100644
--- a/log.txt
+++ b/log.txt
@@ -1,5 +1,68 @@
-
-> dash@1.0.0 start C:\Users\dash\Documents\Dash-Web
-> ts-node-dev -- src/server/index.ts
-
-Using ts-node version 7.0.1, typescript version 3.4.1
+worker is initializing...
+server is now successfully listening...
+Crash event detected @ Thu, 02 Jan 2020 23:30:23 GMT
+I'm a fake!
+worker is initializing...
+server is now successfully listening...
+Crash event detected @ Thu, 02 Jan 2020 23:32:07 GMT
+I'm a fake!
+worker is initializing...
+server is now successfully listening...
+worker is initializing...
+server is now successfully listening...
+Crash event detected @ Thu, 02 Jan 2020 23:35:45 GMT
+I'm a fake!
+worker is initializing...
+server is now successfully listening...
+Crash event detected @ Thu, 02 Jan 2020 23:37:30 GMT
+I'm a fake!
+worker is initializing...
+server is now successfully listening...
+worker is initializing...
+server is now successfully listening...
+Crash event detected @ Thu, 02 Jan 2020 23:40:41 GMT
+I'm a fake!
+worker is initializing...
+server is now successfully listening...
+worker is initializing...
+server is now successfully listening...
+Crash event detected @ Thu, 02 Jan 2020 23:43:53 GMT
+I'm a fake!
+worker is initializing...
+server is now successfully listening...
+Crash event detected @ Thu, 02 Jan 2020 23:45:37 GMT
+I'm a fake!
+worker is initializing...
+server is now successfully listening...
+Crash event detected @ Thu, 02 Jan 2020 23:47:22 GMT
+I'm a fake!
+worker is initializing...
+worker is initializing...
+server is now successfully listening...
+Crash event detected @ Thu, 02 Jan 2020 23:49:36 GMT
+I'm a fake!
+worker is initializing...
+server is now successfully listening...
+Crash event detected @ Thu, 02 Jan 2020 23:51:22 GMT
+I'm a fake!
+worker is initializing...
+worker is initializing...
+worker is initializing...
+server is now successfully listening...
+Crash event detected @ Thu, 02 Jan 2020 23:54:35 GMT
+I'm a fake!
+worker is initializing...
+server is now successfully listening...
+worker is initializing...
+server is now successfully listening...
+worker is initializing...
+server is now successfully listening...
+worker is initializing...
+server is now successfully listening...
+worker is initializing...
+worker is initializing...
+worker is initializing...
+worker is initializing...
+worker is initializing...
+worker is initializing...
+worker is initializing...
diff --git a/src/server/Websocket/Websocket.ts b/src/server/Websocket/Websocket.ts
index e1e157fc4..0b58ca344 100644
--- a/src/server/Websocket/Websocket.ts
+++ b/src/server/Websocket/Websocket.ts
@@ -13,6 +13,7 @@ import { green } from "colors";
 
 export namespace WebSocket {
 
+    export let _socket: Socket;
     const clients: { [key: string]: Client } = {};
     export const socketMap = new Map<SocketIO.Socket, string>();
     export let disconnect: Function;
@@ -28,6 +29,8 @@ export namespace WebSocket {
     export function initialize(socketPort: number, isRelease: boolean) {
         const endpoint = io();
         endpoint.on("connection", function (socket: Socket) {
+            _socket = socket;
+
             socket.use((_packet, next) => {
                 const userEmail = socketMap.get(socket);
                 if (userEmail) {
diff --git a/src/server/index.ts b/src/server/index.ts
index 83413c23c..c26e0ec19 100644
--- a/src/server/index.ts
+++ b/src/server/index.ts
@@ -21,17 +21,11 @@ import UploadManager from "./ApiManagers/UploadManager";
 import { log_execution } from "./ActionUtilities";
 import GeneralGoogleManager from "./ApiManagers/GeneralGoogleManager";
 import GooglePhotosManager from "./ApiManagers/GooglePhotosManager";
-import { yellow, red, cyan } from "colors";
 import { disconnect } from "../server/Initialization";
 import { Logger } from "./ProcessFactory";
-import { isMaster, on, fork, workers } from "cluster";
-import { identifier } from "./session_manager/config";
-import InputManager from "./session_manager/input_manager";
-import { execSync } from "child_process";
-import { CrashEmail } from "./session_manager/crash_email";
-const killport = require("kill-port");
+import { yellow } from "colors";
+import { Session } from "./session";
 
-export const onWindows = process.platform === "win32";
 export const publicDirectory = path.resolve(__dirname, "public");
 export const filesDirectory = path.resolve(publicDirectory, "files");
 
@@ -144,28 +138,4 @@ async function start() {
     await initializeServer({ serverPort: 1050, routeSetter });
 }
 
-const admin = ["samuel_wilkins@brown.edu"];
-if (isMaster) {
-    fork();
-    on("exit", ({ process: { pid } }, code, signal) => {
-        const prompt = `Server worker with process id ${pid} has died with code ${code}${signal === null ? "" : `, having encountered signal ${signal}`}.\n`;
-        console.log(cyan(prompt));
-        fork();
-    });
-    const { registerCommand } = new InputManager({ identifier });
-    registerCommand("exit", [], () => execSync(onWindows ? "taskkill /f /im node.exe" : "killall -9 node"));
-    registerCommand("restart", [], () => {
-        for (const id in workers) {
-            workers[id]?.kill();
-        }
-        fork();
-    });
-} else {
-    process.on('uncaughtException', async error => {
-        await CrashEmail.dispatch(error, admin);
-        console.error(red(`Crash event detected @ ${new Date().toUTCString()}`));
-        console.error(error.message);
-        process.exit(1);
-    });
-    start();
-}
\ No newline at end of file
+Session.initialize(start);
\ No newline at end of file
diff --git a/src/server/session.ts b/src/server/session.ts
new file mode 100644
index 000000000..53e1be1b1
--- /dev/null
+++ b/src/server/session.ts
@@ -0,0 +1,101 @@
+import { yellow, red, cyan, magenta, green } from "colors";
+import { isMaster, on, fork, setupMaster, Worker } from "cluster";
+import { identifier } from "./session_manager/config";
+import InputManager from "./session_manager/input_manager";
+import { execSync } from "child_process";
+import { CrashEmail } from "./session_manager/crash_email";
+import { get } from "request-promise";
+import { WebSocket } from "./Websocket/Websocket";
+import { Utils } from "../Utils";
+import { MessageStore } from "./Message";
+
+const onWindows = process.platform === "win32";
+const heartbeat = `http://localhost:1050/serverHeartbeat`;
+const admin = ["samuel_wilkins@brown.edu"];
+
+export namespace Session {
+
+    const masterIdentifier = yellow("__master__");
+    const workerIdentifier = magenta("__worker__");
+
+    export async function initialize(work: Function) {
+        let listening = false;
+        let active: Worker;
+        if (isMaster) {
+            process.on("uncaughtException", error => {
+                if (error.message !== "Channel closed") {
+                    console.log(`${masterIdentifier}: ${red(error.message)}`);
+                    if (error.stack) {
+                        console.log(`${masterIdentifier}:\n${red(error.stack)}`);
+                    }
+                }
+            });
+            setupMaster({ silent: true });
+            const spawn = () => {
+                if (active && !active.isDead()) {
+                    active.process.kill();
+                }
+                active = fork();
+                active.on("message", ({ update }) => {
+                    if (update) {
+                        console.log(`${workerIdentifier}: ${update}`);
+                    }
+                });
+            };
+            spawn();
+            on("exit", ({ process: { pid } }, code, signal) => {
+                const prompt = `Server worker with process id ${pid} has exited with code ${code}${signal === null ? "" : `, having encountered signal ${signal}`}.`;
+                console.log(`${masterIdentifier}: ${cyan(prompt)}`);
+                spawn();
+            });
+            const restart = () => {
+                listening = false;
+                const prompt = `Server worker with process id ${active.process.pid} has been manually killed.`;
+                console.log(`${masterIdentifier}: ${cyan(prompt)}`);
+                spawn();
+            };
+            const { registerCommand } = new InputManager({ identifier });
+            registerCommand("exit", [], () => execSync(onWindows ? "taskkill /f /im node.exe" : "killall -9 node"));
+            registerCommand("restart", [], restart);
+        } else {
+            const notifyMaster = (update: string) => process.send?.({ update });
+            notifyMaster(green("initializing..."));
+            const gracefulExit = async (error: Error) => {
+                if (!listening) {
+                    return;
+                }
+                listening = false;
+                await CrashEmail.dispatch(error, admin);
+                const { _socket } = WebSocket;
+                if (_socket) {
+                    Utils.Emit(_socket, MessageStore.ConnectionTerminated, "Manual");
+                }
+                notifyMaster(red(`Crash event detected @ ${new Date().toUTCString()}`));
+                notifyMaster(red(error.message));
+                process.exit(1);
+            };
+            process.on('uncaughtException', gracefulExit);
+            const checkHeartbeat = async () => {
+                await new Promise<void>(resolve => {
+                    setTimeout(async () => {
+                        try {
+                            await get(heartbeat);
+                            if (!listening) {
+                                notifyMaster(green("server is now successfully listening..."));
+                            }
+                            listening = true;
+                        } catch (error) {
+                            await gracefulExit(error);
+                        } finally {
+                            resolve();
+                        }
+                    }, 1000 * 15);
+                });
+                checkHeartbeat();
+            };
+            work();
+            checkHeartbeat();
+        }
+    }
+
+}
\ No newline at end of file