6 files changed, 82 insertions, 46 deletions

diff --git a/src/server/ActionUtilities.ts b/src/server/ActionUtilities.ts
index 2173f4369..9bdc4ed93 100644
--- a/src/server/ActionUtilities.ts
+++ b/src/server/ActionUtilities.ts
@@ -1,12 +1,11 @@
-import * as fs from 'fs';
+import { readFile, writeFile, exists, mkdir, unlink } from 'fs';
 import { ExecOptions } from 'shelljs';
-import { exec, spawn } from 'child_process';
+import { exec } from 'child_process';
 import * as path from 'path';
 import * as rimraf from "rimraf";
 import { yellow, Color } from 'colors';
 
 const projectRoot = path.resolve(__dirname, "../../");
-
 export function pathFromRoot(relative: string) {
     return path.resolve(projectRoot, relative);
 }
@@ -21,24 +20,17 @@ export const command_line = (command: string, fromDirectory?: string) => {
     });
 };
 
-export async function spawn_detached_process(command: string, args?: readonly string[]) {
-    const out = path.resolve(projectRoot, `./logs/${command}-${process.pid}-${new Date().toUTCString()}`);
-    const child_out = fs.openSync(out, 'a');
-    const child_error = fs.openSync(out, 'a');
-    spawn(command, args, { detached: true, stdio: ["ignore", child_out, child_error] }).unref();
-}
-
 export const read_text_file = (relativePath: string) => {
     const target = path.resolve(__dirname, relativePath);
     return new Promise<string>((resolve, reject) => {
-        fs.readFile(target, (err, data) => err ? reject(err) : resolve(data.toString()));
+        readFile(target, (err, data) => err ? reject(err) : resolve(data.toString()));
     });
 };
 
 export const write_text_file = (relativePath: string, contents: any) => {
     const target = path.resolve(__dirname, relativePath);
     return new Promise<void>((resolve, reject) => {
-        fs.writeFile(target, contents, (err) => err ? reject(err) : resolve());
+        writeFile(target, contents, (err) => err ? reject(err) : resolve());
     });
 };
 
@@ -93,10 +85,10 @@ export function msToTime(duration: number) {
 }
 
 export const createIfNotExists = async (path: string) => {
-    if (await new Promise<boolean>(resolve => fs.exists(path, resolve))) {
+    if (await new Promise<boolean>(resolve => exists(path, resolve))) {
         return true;
     }
-    return new Promise<boolean>(resolve => fs.mkdir(path, error => resolve(error === null)));
+    return new Promise<boolean>(resolve => mkdir(path, error => resolve(error === null)));
 };
 
 export async function Prune(rootDirectory: string): Promise<boolean> {
@@ -104,4 +96,4 @@ export async function Prune(rootDirectory: string): Promise<boolean> {
     return error === null;
 }
 
-export const Destroy = (mediaPath: string) => new Promise<boolean>(resolve => fs.unlink(mediaPath, error => resolve(error === null)));
+export const Destroy = (mediaPath: string) => new Promise<boolean>(resolve => unlink(mediaPath, error => resolve(error === null)));
diff --git a/src/server/ProcessManager.ts b/src/server/ProcessManager.ts
index 2237f9e1b..671f0a234 100644
--- a/src/server/ProcessManager.ts
+++ b/src/server/ProcessManager.ts
@@ -1,7 +1,9 @@
-import { writeFileSync, unlinkSync, existsSync, mkdirSync } from "fs";
-import { pathFromRoot, log_execution, spawn_detached_process } from './ActionUtilities';
-import { resolve } from "path";
-import { red, yellow } from "colors";
+import { existsSync, mkdirSync, createWriteStream } from "fs";
+import { pathFromRoot, log_execution } from './ActionUtilities';
+import { red, green } from "colors";
+import rimraf = require("rimraf");
+import { ChildProcess, spawn } from "child_process";
+import { Stream } from "stream";
 
 const daemonPath = pathFromRoot("./src/server/daemon/persistence_daemon.ts");
 
@@ -9,22 +11,33 @@ export namespace ProcessManager {
 
     export async function initialize() {
         const logPath = pathFromRoot("./logs");
-        const filePath = resolve(logPath, "./server_pids.txt");
-        const exists = existsSync(logPath);
-        if (exists) {
-            unlinkSync(filePath);
-        } else {
-            mkdirSync(logPath);
+        if (existsSync(logPath)) {
+            if (!process.env.SPAWNED) {
+                await new Promise<any>(resolve => rimraf(logPath, resolve));
+            }
         }
-        const { pid } = process;
-        if (process.env.SPAWNED === "true") {
-            writeFileSync(filePath, `${pid} created at ${new Date().toUTCString()}\n`);
+        mkdirSync(logPath);
+    }
+
+    function generate_log_name(command: string, args?: readonly string[]) {
+        return pathFromRoot(`./logs/${command}-${args?.length}-${new Date().toUTCString()}.log`);
+    }
+
+    export type Sink = "pipe" | "ipc" | "ignore" | "inherit" | Stream | number | null | undefined;
+
+    export async function spawn_detached(command: string, args?: readonly string[], out?: Sink): Promise<ChildProcess> {
+        if (!out) {
+            const logStream = createWriteStream(generate_log_name(command, args));
+            out = await new Promise<number>(resolve => logStream.on("open", resolve));
         }
+        const child = spawn(command, args, { detached: true, stdio: ["ignore", out, out] });
+        child.unref();
+        return child;
     }
 
     let daemonInitialized = false;
     export async function trySpawnDaemon() {
-        if (!daemonInitialized) {
+        if (!process.env.SPAWNED && !daemonInitialized) {
             daemonInitialized = true;
             await log_execution({
                 startMessage: "\ninitializing persistence daemon",
@@ -32,13 +45,15 @@ export namespace ProcessManager {
                     const success = error === null && result !== undefined;
                     if (!success) {
                         console.log(red("failed to initialize the persistance daemon"));
+                        console.log(error);
                         process.exit(0);
                     }
-                    return "persistence daemon process closed";
+                    return "failsafe daemon process successfully spawned";
                 },
-                action: () => spawn_detached_process("npx ts-node", [daemonPath]),
-                color: yellow
+                action: () => spawn_detached('npx', ['ts-node', daemonPath], process.stdout),
+                color: green
             });
+            console.log();
         }
     }
 
diff --git a/src/server/daemon/current_daemon_pid.txt b/src/server/daemon/current_daemon_pid.txt
new file mode 100644
index 000000000..f3cd0298c
--- /dev/null
+++ b/src/server/daemon/current_daemon_pid.txt
@@ -0,0 +1 @@
+9626
\ No newline at end of file
diff --git a/src/server/daemon/persistence_daemon.ts b/src/server/daemon/persistence_daemon.ts
index 3eb17a9b4..099c7898c 100644
--- a/src/server/daemon/persistence_daemon.ts
+++ b/src/server/daemon/persistence_daemon.ts
@@ -1,22 +1,41 @@
 import * as request from "request-promise";
-import { log_execution, spawn_detached_process } from "../ActionUtilities";
-import { red, yellow, cyan, green } from "colors";
+import { log_execution, pathFromRoot } from "../ActionUtilities";
+import { red, yellow, cyan, green, Color } from "colors";
 import * as nodemailer from "nodemailer";
 import { MailOptions } from "nodemailer/lib/json-transport";
-import { writeFileSync } from "fs";
+import { writeFileSync, appendFileSync, createWriteStream, existsSync } from "fs";
 import { resolve } from 'path';
+import { ChildProcess } from "child_process";
+import { ProcessManager } from "../ProcessManager";
+
+console.log(yellow("Initializing daemon..."));
+
+process.on('SIGINT', () => current_backup?.kill("SIGTERM"));
+
+const crashLogPath = resolve(__dirname, `./session_crashes_${timestamp()}.log`);
+function addLogEntry(message: string, color: Color) {
+    const formatted = color(`${message} ${timestamp()}.`);
+    console.log(formatted);
+    appendFileSync(crashLogPath, `${formatted}\n`);
+}
 
 const LOCATION = "http://localhost";
 const recipient = "samuel_wilkins@brown.edu";
 let restarting = false;
 
-writeFileSync(resolve(__dirname, "./current_pid.txt"), process.pid);
+const frequency = 10;
+const { pid } = process;
+writeFileSync(resolve(__dirname, "./current_daemon_pid.txt"), pid);
+console.log(cyan(`${pid} written to ./current_daemon_pid.txt`));
 
 function timestamp() {
     return `@ ${new Date().toISOString()}`;
 }
 
+let current_backup: ChildProcess | undefined = undefined;
+
 async function listen() {
+    console.log(yellow(`Beginning to poll server heartbeat every ${frequency} seconds...\n`));
     if (!LOCATION) {
         console.log(red("No location specified for persistence daemon. Please include as a command line environment variable or in a .env file."));
         process.exit(0);
@@ -28,34 +47,40 @@ async function listen() {
         let error: any;
         try {
             await request.get(heartbeat);
+            if (restarting) {
+                addLogEntry("Backup server successfully restarted", green);
+            }
+            restarting = false;
         } catch (e) {
             error = e;
         } finally {
             if (error) {
                 if (!restarting) {
                     restarting = true;
-                    console.log(yellow("Detected a server crash!"));
+                    addLogEntry("Detected a server crash", red);
+                    current_backup?.kill();
                     await log_execution({
                         startMessage: "Sending crash notification email",
                         endMessage: ({ error, result }) => {
                             const success = error === null && result === true;
-                            return (success ? `Notification successfully sent to ` : `Failed to notify `) + recipient;
+                            return `${(success ? `Notification successfully sent to` : `Failed to notify`)} ${recipient} ${timestamp()}`;
                         },
                         action: async () => notify(error || "Hmm, no error to report..."),
                         color: cyan
                     });
-                    console.log(await log_execution({
+                    current_backup = await log_execution({
                         startMessage: "Initiating server restart",
-                        endMessage: "Server successfully restarted",
-                        action: () => spawn_detached_process(`npm run start-spawn`),
+                        endMessage: ({ result, error }) => {
+                            const success = error === null && result !== undefined;
+                            return success ? "Child process spawned.." : `An error occurred while attempting to restart the server:\n${error}`;
+                        },
+                        action: () => ProcessManager.spawn_detached('npm', ['run', 'start-spawn']),
                         color: green
-                    }));
-                    restarting = false;
+                    });
+                    writeFileSync(pathFromRoot("./logs/current_server_pid.txt"), `${current_backup?.pid ?? -1} created ${timestamp()}\n`);
                 } else {
                     console.log(yellow(`Callback ignored because restarting already initiated ${timestamp()}`));
                 }
-            } else {
-                console.log(green(`No issues detected ${timestamp()}`));
             }
         }
     }, 1000 * 10);
@@ -85,7 +110,7 @@ async function notify(error: any) {
         text: emailText(error)
     } as MailOptions;
     return new Promise<boolean>(resolve => {
-        smtpTransport.sendMail(mailOptions, (dispatchError: Error | null) => { console.log(dispatchError); resolve(dispatchError === null); });
+        smtpTransport.sendMail(mailOptions, (dispatchError: Error | null) => resolve(dispatchError === null));
     });
 }
 
diff --git a/src/server/daemon/session_crashes_@ 2019-12-11T08:31:56.281Z.log b/src/server/daemon/session_crashes_@ 2019-12-11T08:31:56.281Z.log
new file mode 100644
index 000000000..32b7810ea
--- /dev/null
+++ b/src/server/daemon/session_crashes_@ 2019-12-11T08:31:56.281Z.log
@@ -0,0 +1 @@
+Detected a server crash @ 2019-12-11T08:32:36.317Z
diff --git a/src/server/daemon/session_crashes_@ 2019-12-11T08:43:46.454Z.log b/src/server/daemon/session_crashes_@ 2019-12-11T08:43:46.454Z.log
new file mode 100644
index 000000000..ebb6843c2
--- /dev/null
+++ b/src/server/daemon/session_crashes_@ 2019-12-11T08:43:46.454Z.log
@@ -0,0 +1,2 @@
+Detected a server crash @ 2019-12-11T08:44:26.494Z.
+Backup server successfully restarted @ 2019-12-11T08:45:33.644Z.