From ae3603e26adb635380d530b84cb9d6f1284066ef Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Wed, 11 Dec 2019 13:54:28 -0500 Subject: process factory refactor --- src/server/ActionUtilities.ts | 9 +- src/server/ChildProcessUtilities/ProcessFactory.ts | 67 ++++++++++ .../daemon/persistence_daemon.ts | 137 +++++++++++++++++++++ src/server/ProcessManager.ts | 60 --------- src/server/daemon/current_daemon_pid.txt | 1 - src/server/daemon/persistence_daemon.ts | 117 ------------------ .../session_crashes_@ 2019-12-11T08:31:56.281Z.log | 1 - .../session_crashes_@ 2019-12-11T08:43:46.454Z.log | 2 - src/server/index.ts | 11 +- 9 files changed, 219 insertions(+), 186 deletions(-) create mode 100644 src/server/ChildProcessUtilities/ProcessFactory.ts create mode 100644 src/server/ChildProcessUtilities/daemon/persistence_daemon.ts delete mode 100644 src/server/ProcessManager.ts delete mode 100644 src/server/daemon/current_daemon_pid.txt delete mode 100644 src/server/daemon/persistence_daemon.ts delete mode 100644 src/server/daemon/session_crashes_@ 2019-12-11T08:31:56.281Z.log delete mode 100644 src/server/daemon/session_crashes_@ 2019-12-11T08:43:46.454Z.log (limited to 'src') diff --git a/src/server/ActionUtilities.ts b/src/server/ActionUtilities.ts index 9bdc4ed93..2e62443c6 100644 --- a/src/server/ActionUtilities.ts +++ b/src/server/ActionUtilities.ts @@ -1,4 +1,4 @@ -import { readFile, writeFile, exists, mkdir, unlink } from 'fs'; +import { readFile, writeFile, exists, mkdir, unlink, createWriteStream } from 'fs'; import { ExecOptions } from 'shelljs'; import { exec } from 'child_process'; import * as path from 'path'; @@ -10,6 +10,11 @@ export function pathFromRoot(relative: string) { return path.resolve(projectRoot, relative); } +export async function fileDescriptorFromStream(path: string) { + const logStream = createWriteStream(path); + return new Promise(resolve => logStream.on("open", resolve)); +} + export const command_line = (command: string, fromDirectory?: string) => { return new Promise((resolve, reject) => { const options: ExecOptions = {}; @@ -54,7 +59,7 @@ export async function log_execution({ startMessage, endMessage, action, color } catch (e) { error = e; } finally { - log_helper(`${typeof endMessage === "string" ? endMessage : endMessage({ result, error })}.`, resolvedColor); + log_helper(typeof endMessage === "string" ? endMessage : endMessage({ result, error }), resolvedColor); } return result; } diff --git a/src/server/ChildProcessUtilities/ProcessFactory.ts b/src/server/ChildProcessUtilities/ProcessFactory.ts new file mode 100644 index 000000000..745b1479a --- /dev/null +++ b/src/server/ChildProcessUtilities/ProcessFactory.ts @@ -0,0 +1,67 @@ +import { existsSync, mkdirSync } from "fs"; +import { pathFromRoot, log_execution, fileDescriptorFromStream } from '../ActionUtilities'; +import { red, green } from "colors"; +import rimraf = require("rimraf"); +import { ChildProcess, spawn, StdioOptions } from "child_process"; +import { Stream } from "stream"; +import { resolve } from "path"; + +export namespace ProcessFactory { + + export type Sink = "pipe" | "ipc" | "ignore" | "inherit" | Stream | number | null | undefined; + + export async function createWorker(command: string, args?: readonly string[], stdio?: StdioOptions | "logfile", detached = true): Promise { + if (stdio === "logfile") { + const log_fd = await Logger.create(command, args); + stdio = ["ignore", log_fd, log_fd]; + } + const child = spawn(command, args, { detached, stdio }); + child.unref(); + return child; + } + + export namespace NamedAgents { + + export async function persistenceDaemon() { + await log_execution({ + startMessage: "\ninitializing persistence daemon", + endMessage: ({ result, error }) => { + const success = error === null && result !== undefined; + if (!success) { + console.log(red("failed to initialize the persistance daemon")); + console.log(error); + process.exit(0); + } + return "failsafe daemon process successfully spawned"; + }, + action: () => createWorker('npx', ['ts-node', resolve(__dirname, "./daemon/persistence_daemon.ts")], ["ignore", "inherit", "inherit"]), + color: green + }); + console.log(); + } + } + +} + +export namespace Logger { + + const logPath = pathFromRoot("./logs"); + + export async function initialize() { + if (existsSync(logPath)) { + if (!process.env.SPAWNED) { + await new Promise(resolve => rimraf(logPath, resolve)); + } + } + mkdirSync(logPath); + } + + export async function create(command: string, args?: readonly string[]): Promise { + return fileDescriptorFromStream(generate_log_path(command, args)); + } + + function generate_log_path(command: string, args?: readonly string[]) { + return pathFromRoot(`./logs/${command}-${args?.length}-${new Date().toUTCString()}.log`); + } + +} \ No newline at end of file diff --git a/src/server/ChildProcessUtilities/daemon/persistence_daemon.ts b/src/server/ChildProcessUtilities/daemon/persistence_daemon.ts new file mode 100644 index 000000000..888cf38b8 --- /dev/null +++ b/src/server/ChildProcessUtilities/daemon/persistence_daemon.ts @@ -0,0 +1,137 @@ +import * as request from "request-promise"; +import { log_execution } from "../../ActionUtilities"; +import { red, yellow, cyan, green, Color } from "colors"; +import * as nodemailer from "nodemailer"; +import { MailOptions } from "nodemailer/lib/json-transport"; +import { writeFileSync, appendFileSync, existsSync, mkdirSync } from "fs"; +import { resolve } from 'path'; +import { ChildProcess } from "child_process"; +import { ProcessFactory } from "../ProcessFactory"; + +const identifier = yellow("__daemon__:"); + +process.on('SIGINT', () => current_backup?.kill("SIGTERM")); + +const logPath = resolve(__dirname, "./logs"); +const crashPath = resolve(logPath, "./crashes"); +if (!existsSync(logPath)) { + mkdirSync(logPath); +} +if (!existsSync(crashPath)) { + mkdirSync(crashPath); +} + +const crashLogPath = resolve(crashPath, `./session_crashes_${timestamp()}.log`); +function addLogEntry(message: string, color: Color) { + const formatted = color(`${message} ${timestamp()}.`); + identifiedLog(formatted); + appendFileSync(crashLogPath, `${formatted}\n`); +} + +function identifiedLog(message?: any, ...optionalParams: any[]) { + console.log(identifier, message, ...optionalParams); +} + +const LOCATION = "http://localhost"; +const recipient = "samuel_wilkins@brown.edu"; +const frequency = 10; +const { pid } = process; +let restarting = false; + +identifiedLog("Initializing daemon..."); + +writeLocalPidLog("daemon", pid); + +function writeLocalPidLog(filename: string, contents: any) { + const path = `./logs/current_${filename}_pid.log`; + identifiedLog(cyan(`${contents} written to ${path}`)); + writeFileSync(resolve(__dirname, path), `${contents}\n`); +} + +function timestamp() { + return `@ ${new Date().toISOString()}`; +} + +let current_backup: ChildProcess | undefined = undefined; + +async function listen() { + identifiedLog(yellow(`Beginning to poll server heartbeat every ${frequency} seconds...\n`)); + if (!LOCATION) { + identifiedLog(red("No location specified for persistence daemon. Please include as a command line environment variable or in a .env file.")); + process.exit(0); + } + const heartbeat = `${LOCATION}:1050/serverHeartbeat`; + // if this is on our remote server, the server must be run in release mode + // const suffix = LOCATION.includes("localhost") ? "" : "-release"; + setInterval(async () => { + let error: any; + try { + await request.get(heartbeat); + if (restarting) { + addLogEntry("Backup server successfully restarted", green); + } + restarting = false; + } catch (e) { + error = e; + } finally { + if (error) { + if (!restarting) { + restarting = true; + addLogEntry("Detected a server crash", red); + current_backup?.kill(); + await log_execution({ + startMessage: identifier + " Sending crash notification email", + endMessage: ({ error, result }) => { + const success = error === null && result === true; + return identifier + ` ${(success ? `Notification successfully sent to` : `Failed to notify`)} ${recipient} ${timestamp()}`; + }, + action: async () => notify(error || "Hmm, no error to report..."), + color: cyan + }); + current_backup = await log_execution({ + startMessage: identifier + " Initiating server restart", + endMessage: ({ result, error }) => { + const success = error === null && result !== undefined; + return identifier + success ? " Child process spawned..." : ` An error occurred while attempting to restart the server:\n${error}`; + }, + action: () => ProcessFactory.createWorker('npm', ['run', 'start-spawn'], "inherit"), + color: green + }); + writeLocalPidLog("server", `${(current_backup?.pid ?? -2) + 1} created ${timestamp()}`); + } else { + identifiedLog(yellow(`Callback ignored because restarting already initiated ${timestamp()}`)); + } + } + } + }, 1000 * 10); +} + +function emailText(error: any) { + return [ + `Hey ${recipient.split("@")[0]},`, + "You, as a Dash Administrator, are being notified of a server crash event. Here's what we know:", + `Location: ${LOCATION}\nError: ${error}`, + "The server should already be restarting itself, but if you're concerned, use the Remote Desktop Connection to monitor progress." + ].join("\n\n"); +} + +async function notify(error: any) { + const smtpTransport = nodemailer.createTransport({ + service: 'Gmail', + auth: { + user: 'brownptcdash@gmail.com', + pass: 'browngfx1' + } + }); + const mailOptions = { + to: recipient, + from: 'brownptcdash@gmail.com', + subject: 'Dash Server Crash', + text: emailText(error) + } as MailOptions; + return new Promise(resolve => { + smtpTransport.sendMail(mailOptions, (dispatchError: Error | null) => resolve(dispatchError === null)); + }); +} + +listen(); \ No newline at end of file diff --git a/src/server/ProcessManager.ts b/src/server/ProcessManager.ts deleted file mode 100644 index 671f0a234..000000000 --- a/src/server/ProcessManager.ts +++ /dev/null @@ -1,60 +0,0 @@ -import { existsSync, mkdirSync, createWriteStream } from "fs"; -import { pathFromRoot, log_execution } from './ActionUtilities'; -import { red, green } from "colors"; -import rimraf = require("rimraf"); -import { ChildProcess, spawn } from "child_process"; -import { Stream } from "stream"; - -const daemonPath = pathFromRoot("./src/server/daemon/persistence_daemon.ts"); - -export namespace ProcessManager { - - export async function initialize() { - const logPath = pathFromRoot("./logs"); - if (existsSync(logPath)) { - if (!process.env.SPAWNED) { - await new Promise(resolve => rimraf(logPath, resolve)); - } - } - mkdirSync(logPath); - } - - function generate_log_name(command: string, args?: readonly string[]) { - return pathFromRoot(`./logs/${command}-${args?.length}-${new Date().toUTCString()}.log`); - } - - export type Sink = "pipe" | "ipc" | "ignore" | "inherit" | Stream | number | null | undefined; - - export async function spawn_detached(command: string, args?: readonly string[], out?: Sink): Promise { - if (!out) { - const logStream = createWriteStream(generate_log_name(command, args)); - out = await new Promise(resolve => logStream.on("open", resolve)); - } - const child = spawn(command, args, { detached: true, stdio: ["ignore", out, out] }); - child.unref(); - return child; - } - - let daemonInitialized = false; - export async function trySpawnDaemon() { - if (!process.env.SPAWNED && !daemonInitialized) { - daemonInitialized = true; - await log_execution({ - startMessage: "\ninitializing persistence daemon", - endMessage: ({ result, error }) => { - const success = error === null && result !== undefined; - if (!success) { - console.log(red("failed to initialize the persistance daemon")); - console.log(error); - process.exit(0); - } - return "failsafe daemon process successfully spawned"; - }, - action: () => spawn_detached('npx', ['ts-node', daemonPath], process.stdout), - color: green - }); - console.log(); - } - } - -} \ No newline at end of file diff --git a/src/server/daemon/current_daemon_pid.txt b/src/server/daemon/current_daemon_pid.txt deleted file mode 100644 index f3cd0298c..000000000 --- a/src/server/daemon/current_daemon_pid.txt +++ /dev/null @@ -1 +0,0 @@ -9626 \ No newline at end of file diff --git a/src/server/daemon/persistence_daemon.ts b/src/server/daemon/persistence_daemon.ts deleted file mode 100644 index 099c7898c..000000000 --- a/src/server/daemon/persistence_daemon.ts +++ /dev/null @@ -1,117 +0,0 @@ -import * as request from "request-promise"; -import { log_execution, pathFromRoot } from "../ActionUtilities"; -import { red, yellow, cyan, green, Color } from "colors"; -import * as nodemailer from "nodemailer"; -import { MailOptions } from "nodemailer/lib/json-transport"; -import { writeFileSync, appendFileSync, createWriteStream, existsSync } from "fs"; -import { resolve } from 'path'; -import { ChildProcess } from "child_process"; -import { ProcessManager } from "../ProcessManager"; - -console.log(yellow("Initializing daemon...")); - -process.on('SIGINT', () => current_backup?.kill("SIGTERM")); - -const crashLogPath = resolve(__dirname, `./session_crashes_${timestamp()}.log`); -function addLogEntry(message: string, color: Color) { - const formatted = color(`${message} ${timestamp()}.`); - console.log(formatted); - appendFileSync(crashLogPath, `${formatted}\n`); -} - -const LOCATION = "http://localhost"; -const recipient = "samuel_wilkins@brown.edu"; -let restarting = false; - -const frequency = 10; -const { pid } = process; -writeFileSync(resolve(__dirname, "./current_daemon_pid.txt"), pid); -console.log(cyan(`${pid} written to ./current_daemon_pid.txt`)); - -function timestamp() { - return `@ ${new Date().toISOString()}`; -} - -let current_backup: ChildProcess | undefined = undefined; - -async function listen() { - console.log(yellow(`Beginning to poll server heartbeat every ${frequency} seconds...\n`)); - if (!LOCATION) { - console.log(red("No location specified for persistence daemon. Please include as a command line environment variable or in a .env file.")); - process.exit(0); - } - const heartbeat = `${LOCATION}:1050/serverHeartbeat`; - // if this is on our remote server, the server must be run in release mode - // const suffix = LOCATION.includes("localhost") ? "" : "-release"; - setInterval(async () => { - let error: any; - try { - await request.get(heartbeat); - if (restarting) { - addLogEntry("Backup server successfully restarted", green); - } - restarting = false; - } catch (e) { - error = e; - } finally { - if (error) { - if (!restarting) { - restarting = true; - addLogEntry("Detected a server crash", red); - current_backup?.kill(); - await log_execution({ - startMessage: "Sending crash notification email", - endMessage: ({ error, result }) => { - const success = error === null && result === true; - return `${(success ? `Notification successfully sent to` : `Failed to notify`)} ${recipient} ${timestamp()}`; - }, - action: async () => notify(error || "Hmm, no error to report..."), - color: cyan - }); - current_backup = await log_execution({ - startMessage: "Initiating server restart", - endMessage: ({ result, error }) => { - const success = error === null && result !== undefined; - return success ? "Child process spawned.." : `An error occurred while attempting to restart the server:\n${error}`; - }, - action: () => ProcessManager.spawn_detached('npm', ['run', 'start-spawn']), - color: green - }); - writeFileSync(pathFromRoot("./logs/current_server_pid.txt"), `${current_backup?.pid ?? -1} created ${timestamp()}\n`); - } else { - console.log(yellow(`Callback ignored because restarting already initiated ${timestamp()}`)); - } - } - } - }, 1000 * 10); -} - -function emailText(error: any) { - return [ - `Hey ${recipient.split("@")[0]},`, - "You, as a Dash Administrator, are being notified of a server crash event. Here's what we know:", - `Location: ${LOCATION}\nError: ${error}`, - "The server should already be restarting itself, but if you're concerned, use the Remote Desktop Connection to monitor progress." - ].join("\n\n"); -} - -async function notify(error: any) { - const smtpTransport = nodemailer.createTransport({ - service: 'Gmail', - auth: { - user: 'brownptcdash@gmail.com', - pass: 'browngfx1' - } - }); - const mailOptions = { - to: recipient, - from: 'brownptcdash@gmail.com', - subject: 'Dash Server Crash', - text: emailText(error) - } as MailOptions; - return new Promise(resolve => { - smtpTransport.sendMail(mailOptions, (dispatchError: Error | null) => resolve(dispatchError === null)); - }); -} - -listen(); \ No newline at end of file diff --git a/src/server/daemon/session_crashes_@ 2019-12-11T08:31:56.281Z.log b/src/server/daemon/session_crashes_@ 2019-12-11T08:31:56.281Z.log deleted file mode 100644 index 32b7810ea..000000000 --- a/src/server/daemon/session_crashes_@ 2019-12-11T08:31:56.281Z.log +++ /dev/null @@ -1 +0,0 @@ -Detected a server crash @ 2019-12-11T08:32:36.317Z diff --git a/src/server/daemon/session_crashes_@ 2019-12-11T08:43:46.454Z.log b/src/server/daemon/session_crashes_@ 2019-12-11T08:43:46.454Z.log deleted file mode 100644 index ebb6843c2..000000000 --- a/src/server/daemon/session_crashes_@ 2019-12-11T08:43:46.454Z.log +++ /dev/null @@ -1,2 +0,0 @@ -Detected a server crash @ 2019-12-11T08:44:26.494Z. -Backup server successfully restarted @ 2019-12-11T08:45:33.644Z. diff --git a/src/server/index.ts b/src/server/index.ts index 795418b31..bebb9b365 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -23,7 +23,7 @@ import GeneralGoogleManager from "./ApiManagers/GeneralGoogleManager"; import GooglePhotosManager from "./ApiManagers/GooglePhotosManager"; import { yellow, red } from "colors"; import { disconnect } from "../server/Initialization"; -import { ProcessManager } from "./ProcessManager"; +import { ProcessFactory, Logger } from "./ChildProcessUtilities/ProcessFactory"; export const publicDirectory = path.resolve(__dirname, "public"); export const filesDirectory = path.resolve(publicDirectory, "files"); @@ -36,7 +36,7 @@ export const ExitHandlers = new Array<() => void>(); * before clients can access the server should be run or awaited here. */ async function preliminaryFunctions() { - await ProcessManager.initialize(); + await Logger.initialize(); await GoogleCredentialsLoader.loadCredentials(); GoogleApiServerUtils.processProjectCredentials(); await DashUploadUtils.buildFileDirectories(); @@ -121,11 +121,16 @@ function routeSetter({ isRelease, addSupervisedRoute, logRegistrationOutcome }: } }); + let daemonInitialized = false; + const { SPAWNED, RELEASE } = process.env; addSupervisedRoute({ method: Method.GET, subscription: "/persist", onValidation: ({ res }) => { - ProcessManager.trySpawnDaemon(); + if (RELEASE && !SPAWNED && !daemonInitialized) { + daemonInitialized = true; + ProcessFactory.NamedAgents.persistenceDaemon(); + } res.redirect("/home"); } }); -- cgit v1.2.3-70-g09d2