8 files changed, 92 insertions, 58 deletions

diff --git a/.gitignore b/.gitignore
index cf4ed308b..38c619c52 100644
--- a/.gitignore
+++ b/.gitignore
@@ -6,3 +6,4 @@ ClientUtils.ts
 solr-8.1.1/server/
 src/server/public/files/
 src/scraping/acm/package-lock.json
+src/server/ChildProcessUtilities/daemon/**/*.log
diff --git a/src/server/ActionUtilities.ts b/src/server/ActionUtilities.ts
index 9bdc4ed93..2e62443c6 100644
--- a/src/server/ActionUtilities.ts
+++ b/src/server/ActionUtilities.ts
@@ -1,4 +1,4 @@
-import { readFile, writeFile, exists, mkdir, unlink } from 'fs';
+import { readFile, writeFile, exists, mkdir, unlink, createWriteStream } from 'fs';
 import { ExecOptions } from 'shelljs';
 import { exec } from 'child_process';
 import * as path from 'path';
@@ -10,6 +10,11 @@ export function pathFromRoot(relative: string) {
     return path.resolve(projectRoot, relative);
 }
 
+export async function fileDescriptorFromStream(path: string) {
+    const logStream = createWriteStream(path);
+    return new Promise<number>(resolve => logStream.on("open", resolve));
+}
+
 export const command_line = (command: string, fromDirectory?: string) => {
     return new Promise<string>((resolve, reject) => {
         const options: ExecOptions = {};
@@ -54,7 +59,7 @@ export async function log_execution<T>({ startMessage, endMessage, action, color
     } catch (e) {
         error = e;
     } finally {
-        log_helper(`${typeof endMessage === "string" ? endMessage : endMessage({ result, error })}.`, resolvedColor);
+        log_helper(typeof endMessage === "string" ? endMessage : endMessage({ result, error }), resolvedColor);
     }
     return result;
 }
diff --git a/src/server/ProcessManager.ts b/src/server/ChildProcessUtilities/ProcessFactory.ts
index 671f0a234..745b1479a 100644
--- a/src/server/ProcessManager.ts
+++ b/src/server/ChildProcessUtilities/ProcessFactory.ts
@@ -1,44 +1,28 @@
-import { existsSync, mkdirSync, createWriteStream } from "fs";
-import { pathFromRoot, log_execution } from './ActionUtilities';
+import { existsSync, mkdirSync } from "fs";
+import { pathFromRoot, log_execution, fileDescriptorFromStream } from '../ActionUtilities';
 import { red, green } from "colors";
 import rimraf = require("rimraf");
-import { ChildProcess, spawn } from "child_process";
+import { ChildProcess, spawn, StdioOptions } from "child_process";
 import { Stream } from "stream";
+import { resolve } from "path";
 
-const daemonPath = pathFromRoot("./src/server/daemon/persistence_daemon.ts");
-
-export namespace ProcessManager {
-
-    export async function initialize() {
-        const logPath = pathFromRoot("./logs");
-        if (existsSync(logPath)) {
-            if (!process.env.SPAWNED) {
-                await new Promise<any>(resolve => rimraf(logPath, resolve));
-            }
-        }
-        mkdirSync(logPath);
-    }
-
-    function generate_log_name(command: string, args?: readonly string[]) {
-        return pathFromRoot(`./logs/${command}-${args?.length}-${new Date().toUTCString()}.log`);
-    }
+export namespace ProcessFactory {
 
     export type Sink = "pipe" | "ipc" | "ignore" | "inherit" | Stream | number | null | undefined;
 
-    export async function spawn_detached(command: string, args?: readonly string[], out?: Sink): Promise<ChildProcess> {
-        if (!out) {
-            const logStream = createWriteStream(generate_log_name(command, args));
-            out = await new Promise<number>(resolve => logStream.on("open", resolve));
+    export async function createWorker(command: string, args?: readonly string[], stdio?: StdioOptions | "logfile", detached = true): Promise<ChildProcess> {
+        if (stdio === "logfile") {
+            const log_fd = await Logger.create(command, args);
+            stdio = ["ignore", log_fd, log_fd];
         }
-        const child = spawn(command, args, { detached: true, stdio: ["ignore", out, out] });
+        const child = spawn(command, args, { detached, stdio });
         child.unref();
         return child;
     }
 
-    let daemonInitialized = false;
-    export async function trySpawnDaemon() {
-        if (!process.env.SPAWNED && !daemonInitialized) {
-            daemonInitialized = true;
+    export namespace NamedAgents {
+
+        export async function persistenceDaemon() {
             await log_execution({
                 startMessage: "\ninitializing persistence daemon",
                 endMessage: ({ result, error }) => {
@@ -50,11 +34,34 @@ export namespace ProcessManager {
                     }
                     return "failsafe daemon process successfully spawned";
                 },
-                action: () => spawn_detached('npx', ['ts-node', daemonPath], process.stdout),
+                action: () => createWorker('npx', ['ts-node', resolve(__dirname, "./daemon/persistence_daemon.ts")], ["ignore", "inherit", "inherit"]),
                 color: green
             });
             console.log();
         }
     }
 
+}
+
+export namespace Logger {
+
+    const logPath = pathFromRoot("./logs");
+
+    export async function initialize() {
+        if (existsSync(logPath)) {
+            if (!process.env.SPAWNED) {
+                await new Promise<any>(resolve => rimraf(logPath, resolve));
+            }
+        }
+        mkdirSync(logPath);
+    }
+
+    export async function create(command: string, args?: readonly string[]): Promise<number> {
+        return fileDescriptorFromStream(generate_log_path(command, args));
+    }
+
+    function generate_log_path(command: string, args?: readonly string[]) {
+        return pathFromRoot(`./logs/${command}-${args?.length}-${new Date().toUTCString()}.log`);
+    }
+
 }
\ No newline at end of file
diff --git a/src/server/daemon/persistence_daemon.ts b/src/server/ChildProcessUtilities/daemon/persistence_daemon.ts
index 099c7898c..888cf38b8 100644
--- a/src/server/daemon/persistence_daemon.ts
+++ b/src/server/ChildProcessUtilities/daemon/persistence_daemon.ts
@@ -1,32 +1,52 @@
 import * as request from "request-promise";
-import { log_execution, pathFromRoot } from "../ActionUtilities";
+import { log_execution } from "../../ActionUtilities";
 import { red, yellow, cyan, green, Color } from "colors";
 import * as nodemailer from "nodemailer";
 import { MailOptions } from "nodemailer/lib/json-transport";
-import { writeFileSync, appendFileSync, createWriteStream, existsSync } from "fs";
+import { writeFileSync, appendFileSync, existsSync, mkdirSync } from "fs";
 import { resolve } from 'path';
 import { ChildProcess } from "child_process";
-import { ProcessManager } from "../ProcessManager";
+import { ProcessFactory } from "../ProcessFactory";
 
-console.log(yellow("Initializing daemon..."));
+const identifier = yellow("__daemon__:");
 
 process.on('SIGINT', () => current_backup?.kill("SIGTERM"));
 
-const crashLogPath = resolve(__dirname, `./session_crashes_${timestamp()}.log`);
+const logPath = resolve(__dirname, "./logs");
+const crashPath = resolve(logPath, "./crashes");
+if (!existsSync(logPath)) {
+    mkdirSync(logPath);
+}
+if (!existsSync(crashPath)) {
+    mkdirSync(crashPath);
+}
+
+const crashLogPath = resolve(crashPath, `./session_crashes_${timestamp()}.log`);
 function addLogEntry(message: string, color: Color) {
     const formatted = color(`${message} ${timestamp()}.`);
-    console.log(formatted);
+    identifiedLog(formatted);
     appendFileSync(crashLogPath, `${formatted}\n`);
 }
 
+function identifiedLog(message?: any, ...optionalParams: any[]) {
+    console.log(identifier, message, ...optionalParams);
+}
+
 const LOCATION = "http://localhost";
 const recipient = "samuel_wilkins@brown.edu";
-let restarting = false;
-
 const frequency = 10;
 const { pid } = process;
-writeFileSync(resolve(__dirname, "./current_daemon_pid.txt"), pid);
-console.log(cyan(`${pid} written to ./current_daemon_pid.txt`));
+let restarting = false;
+
+identifiedLog("Initializing daemon...");
+
+writeLocalPidLog("daemon", pid);
+
+function writeLocalPidLog(filename: string, contents: any) {
+    const path = `./logs/current_${filename}_pid.log`;
+    identifiedLog(cyan(`${contents} written to ${path}`));
+    writeFileSync(resolve(__dirname, path), `${contents}\n`);
+}
 
 function timestamp() {
     return `@ ${new Date().toISOString()}`;
@@ -35,9 +55,9 @@ function timestamp() {
 let current_backup: ChildProcess | undefined = undefined;
 
 async function listen() {
-    console.log(yellow(`Beginning to poll server heartbeat every ${frequency} seconds...\n`));
+    identifiedLog(yellow(`Beginning to poll server heartbeat every ${frequency} seconds...\n`));
     if (!LOCATION) {
-        console.log(red("No location specified for persistence daemon. Please include as a command line environment variable or in a .env file."));
+        identifiedLog(red("No location specified for persistence daemon. Please include as a command line environment variable or in a .env file."));
         process.exit(0);
     }
     const heartbeat = `${LOCATION}:1050/serverHeartbeat`;
@@ -60,26 +80,26 @@ async function listen() {
                     addLogEntry("Detected a server crash", red);
                     current_backup?.kill();
                     await log_execution({
-                        startMessage: "Sending crash notification email",
+                        startMessage: identifier + " Sending crash notification email",
                         endMessage: ({ error, result }) => {
                             const success = error === null && result === true;
-                            return `${(success ? `Notification successfully sent to` : `Failed to notify`)} ${recipient} ${timestamp()}`;
+                            return identifier + ` ${(success ? `Notification successfully sent to` : `Failed to notify`)} ${recipient} ${timestamp()}`;
                         },
                         action: async () => notify(error || "Hmm, no error to report..."),
                         color: cyan
                     });
                     current_backup = await log_execution({
-                        startMessage: "Initiating server restart",
+                        startMessage: identifier + " Initiating server restart",
                         endMessage: ({ result, error }) => {
                             const success = error === null && result !== undefined;
-                            return success ? "Child process spawned.." : `An error occurred while attempting to restart the server:\n${error}`;
+                            return identifier + success ? " Child process spawned..." : ` An error occurred while attempting to restart the server:\n${error}`;
                         },
-                        action: () => ProcessManager.spawn_detached('npm', ['run', 'start-spawn']),
+                        action: () => ProcessFactory.createWorker('npm', ['run', 'start-spawn'], "inherit"),
                         color: green
                     });
-                    writeFileSync(pathFromRoot("./logs/current_server_pid.txt"), `${current_backup?.pid ?? -1} created ${timestamp()}\n`);
+                    writeLocalPidLog("server", `${(current_backup?.pid ?? -2) + 1} created ${timestamp()}`);
                 } else {
-                    console.log(yellow(`Callback ignored because restarting already initiated ${timestamp()}`));
+                    identifiedLog(yellow(`Callback ignored because restarting already initiated ${timestamp()}`));
                 }
             }
         }
diff --git a/src/server/daemon/current_daemon_pid.txt b/src/server/daemon/current_daemon_pid.txt
deleted file mode 100644
index f3cd0298c..000000000
--- a/src/server/daemon/current_daemon_pid.txt
+++ /dev/null
@@ -1 +0,0 @@
-9626
\ No newline at end of file
diff --git a/src/server/daemon/session_crashes_@ 2019-12-11T08:31:56.281Z.log b/src/server/daemon/session_crashes_@ 2019-12-11T08:31:56.281Z.log
deleted file mode 100644
index 32b7810ea..000000000
--- a/src/server/daemon/session_crashes_@ 2019-12-11T08:31:56.281Z.log
+++ /dev/null
@@ -1 +0,0 @@
-Detected a server crash @ 2019-12-11T08:32:36.317Z
diff --git a/src/server/daemon/session_crashes_@ 2019-12-11T08:43:46.454Z.log b/src/server/daemon/session_crashes_@ 2019-12-11T08:43:46.454Z.log
deleted file mode 100644
index ebb6843c2..000000000
--- a/src/server/daemon/session_crashes_@ 2019-12-11T08:43:46.454Z.log
+++ /dev/null
@@ -1,2 +0,0 @@
-Detected a server crash @ 2019-12-11T08:44:26.494Z.
-Backup server successfully restarted @ 2019-12-11T08:45:33.644Z.
diff --git a/src/server/index.ts b/src/server/index.ts
index 795418b31..bebb9b365 100644
--- a/src/server/index.ts
+++ b/src/server/index.ts
@@ -23,7 +23,7 @@ import GeneralGoogleManager from "./ApiManagers/GeneralGoogleManager";
 import GooglePhotosManager from "./ApiManagers/GooglePhotosManager";
 import { yellow, red } from "colors";
 import { disconnect } from "../server/Initialization";
-import { ProcessManager } from "./ProcessManager";
+import { ProcessFactory, Logger } from "./ChildProcessUtilities/ProcessFactory";
 
 export const publicDirectory = path.resolve(__dirname, "public");
 export const filesDirectory = path.resolve(publicDirectory, "files");
@@ -36,7 +36,7 @@ export const ExitHandlers = new Array<() => void>();
  * before clients can access the server should be run or awaited here.
  */
 async function preliminaryFunctions() {
-    await ProcessManager.initialize();
+    await Logger.initialize();
     await GoogleCredentialsLoader.loadCredentials();
     GoogleApiServerUtils.processProjectCredentials();
     await DashUploadUtils.buildFileDirectories();
@@ -121,11 +121,16 @@ function routeSetter({ isRelease, addSupervisedRoute, logRegistrationOutcome }:
         }
     });
 
+    let daemonInitialized = false;
+    const { SPAWNED, RELEASE } = process.env;
     addSupervisedRoute({
         method: Method.GET,
         subscription: "/persist",
         onValidation: ({ res }) => {
-            ProcessManager.trySpawnDaemon();
+            if (RELEASE && !SPAWNED && !daemonInitialized) {
+                daemonInitialized = true;
+                ProcessFactory.NamedAgents.persistenceDaemon();
+            }
             res.redirect("/home");
         }
     });