Merge branch 'master' of https://github.com/browngraphicslab/Dash-Web

2 files changed, 204 insertions, 0 deletions

diff --git a/src/server/ChildProcessUtilities/ProcessFactory.ts b/src/server/ChildProcessUtilities/ProcessFactory.ts
new file mode 100644
index 000000000..745b1479a
--- /dev/null
+++ b/src/server/ChildProcessUtilities/ProcessFactory.ts
@@ -0,0 +1,67 @@
+import { existsSync, mkdirSync } from "fs";
+import { pathFromRoot, log_execution, fileDescriptorFromStream } from '../ActionUtilities';
+import { red, green } from "colors";
+import rimraf = require("rimraf");
+import { ChildProcess, spawn, StdioOptions } from "child_process";
+import { Stream } from "stream";
+import { resolve } from "path";
+
+export namespace ProcessFactory {
+
+    export type Sink = "pipe" | "ipc" | "ignore" | "inherit" | Stream | number | null | undefined;
+
+    export async function createWorker(command: string, args?: readonly string[], stdio?: StdioOptions | "logfile", detached = true): Promise<ChildProcess> {
+        if (stdio === "logfile") {
+            const log_fd = await Logger.create(command, args);
+            stdio = ["ignore", log_fd, log_fd];
+        }
+        const child = spawn(command, args, { detached, stdio });
+        child.unref();
+        return child;
+    }
+
+    export namespace NamedAgents {
+
+        export async function persistenceDaemon() {
+            await log_execution({
+                startMessage: "\ninitializing persistence daemon",
+                endMessage: ({ result, error }) => {
+                    const success = error === null && result !== undefined;
+                    if (!success) {
+                        console.log(red("failed to initialize the persistance daemon"));
+                        console.log(error);
+                        process.exit(0);
+                    }
+                    return "failsafe daemon process successfully spawned";
+                },
+                action: () => createWorker('npx', ['ts-node', resolve(__dirname, "./daemon/persistence_daemon.ts")], ["ignore", "inherit", "inherit"]),
+                color: green
+            });
+            console.log();
+        }
+    }
+
+}
+
+export namespace Logger {
+
+    const logPath = pathFromRoot("./logs");
+
+    export async function initialize() {
+        if (existsSync(logPath)) {
+            if (!process.env.SPAWNED) {
+                await new Promise<any>(resolve => rimraf(logPath, resolve));
+            }
+        }
+        mkdirSync(logPath);
+    }
+
+    export async function create(command: string, args?: readonly string[]): Promise<number> {
+        return fileDescriptorFromStream(generate_log_path(command, args));
+    }
+
+    function generate_log_path(command: string, args?: readonly string[]) {
+        return pathFromRoot(`./logs/${command}-${args?.length}-${new Date().toUTCString()}.log`);
+    }
+
+}
\ No newline at end of file
diff --git a/src/server/ChildProcessUtilities/daemon/persistence_daemon.ts b/src/server/ChildProcessUtilities/daemon/persistence_daemon.ts
new file mode 100644
index 000000000..888cf38b8
--- /dev/null
+++ b/src/server/ChildProcessUtilities/daemon/persistence_daemon.ts
@@ -0,0 +1,137 @@
+import * as request from "request-promise";
+import { log_execution } from "../../ActionUtilities";
+import { red, yellow, cyan, green, Color } from "colors";
+import * as nodemailer from "nodemailer";
+import { MailOptions } from "nodemailer/lib/json-transport";
+import { writeFileSync, appendFileSync, existsSync, mkdirSync } from "fs";
+import { resolve } from 'path';
+import { ChildProcess } from "child_process";
+import { ProcessFactory } from "../ProcessFactory";
+
+const identifier = yellow("__daemon__:");
+
+process.on('SIGINT', () => current_backup?.kill("SIGTERM"));
+
+const logPath = resolve(__dirname, "./logs");
+const crashPath = resolve(logPath, "./crashes");
+if (!existsSync(logPath)) {
+    mkdirSync(logPath);
+}
+if (!existsSync(crashPath)) {
+    mkdirSync(crashPath);
+}
+
+const crashLogPath = resolve(crashPath, `./session_crashes_${timestamp()}.log`);
+function addLogEntry(message: string, color: Color) {
+    const formatted = color(`${message} ${timestamp()}.`);
+    identifiedLog(formatted);
+    appendFileSync(crashLogPath, `${formatted}\n`);
+}
+
+function identifiedLog(message?: any, ...optionalParams: any[]) {
+    console.log(identifier, message, ...optionalParams);
+}
+
+const LOCATION = "http://localhost";
+const recipient = "samuel_wilkins@brown.edu";
+const frequency = 10;
+const { pid } = process;
+let restarting = false;
+
+identifiedLog("Initializing daemon...");
+
+writeLocalPidLog("daemon", pid);
+
+function writeLocalPidLog(filename: string, contents: any) {
+    const path = `./logs/current_${filename}_pid.log`;
+    identifiedLog(cyan(`${contents} written to ${path}`));
+    writeFileSync(resolve(__dirname, path), `${contents}\n`);
+}
+
+function timestamp() {
+    return `@ ${new Date().toISOString()}`;
+}
+
+let current_backup: ChildProcess | undefined = undefined;
+
+async function listen() {
+    identifiedLog(yellow(`Beginning to poll server heartbeat every ${frequency} seconds...\n`));
+    if (!LOCATION) {
+        identifiedLog(red("No location specified for persistence daemon. Please include as a command line environment variable or in a .env file."));
+        process.exit(0);
+    }
+    const heartbeat = `${LOCATION}:1050/serverHeartbeat`;
+    // if this is on our remote server, the server must be run in release mode
+    // const suffix = LOCATION.includes("localhost") ? "" : "-release";
+    setInterval(async () => {
+        let error: any;
+        try {
+            await request.get(heartbeat);
+            if (restarting) {
+                addLogEntry("Backup server successfully restarted", green);
+            }
+            restarting = false;
+        } catch (e) {
+            error = e;
+        } finally {
+            if (error) {
+                if (!restarting) {
+                    restarting = true;
+                    addLogEntry("Detected a server crash", red);
+                    current_backup?.kill();
+                    await log_execution({
+                        startMessage: identifier + " Sending crash notification email",
+                        endMessage: ({ error, result }) => {
+                            const success = error === null && result === true;
+                            return identifier + ` ${(success ? `Notification successfully sent to` : `Failed to notify`)} ${recipient} ${timestamp()}`;
+                        },
+                        action: async () => notify(error || "Hmm, no error to report..."),
+                        color: cyan
+                    });
+                    current_backup = await log_execution({
+                        startMessage: identifier + " Initiating server restart",
+                        endMessage: ({ result, error }) => {
+                            const success = error === null && result !== undefined;
+                            return identifier + success ? " Child process spawned..." : ` An error occurred while attempting to restart the server:\n${error}`;
+                        },
+                        action: () => ProcessFactory.createWorker('npm', ['run', 'start-spawn'], "inherit"),
+                        color: green
+                    });
+                    writeLocalPidLog("server", `${(current_backup?.pid ?? -2) + 1} created ${timestamp()}`);
+                } else {
+                    identifiedLog(yellow(`Callback ignored because restarting already initiated ${timestamp()}`));
+                }
+            }
+        }
+    }, 1000 * 10);
+}
+
+function emailText(error: any) {
+    return [
+        `Hey ${recipient.split("@")[0]},`,
+        "You, as a Dash Administrator, are being notified of a server crash event. Here's what we know:",
+        `Location: ${LOCATION}\nError: ${error}`,
+        "The server should already be restarting itself, but if you're concerned, use the Remote Desktop Connection to monitor progress."
+    ].join("\n\n");
+}
+
+async function notify(error: any) {
+    const smtpTransport = nodemailer.createTransport({
+        service: 'Gmail',
+        auth: {
+            user: 'brownptcdash@gmail.com',
+            pass: 'browngfx1'
+        }
+    });
+    const mailOptions = {
+        to: recipient,
+        from: 'brownptcdash@gmail.com',
+        subject: 'Dash Server Crash',
+        text: emailText(error)
+    } as MailOptions;
+    return new Promise<boolean>(resolve => {
+        smtpTransport.sendMail(mailOptions, (dispatchError: Error | null) => resolve(dispatchError === null));
+    });
+}
+
+listen();
\ No newline at end of file