From ad079a088ae9262a4a40a2f0d2a2c5d948140492 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Wed, 11 Dec 2019 03:57:09 -0500 Subject: somewhat functional daemon --- src/server/daemon/current_daemon_pid.txt | 1 + src/server/daemon/persistence_daemon.ts | 53 ++++++++++++++++------ .../session_crashes_@ 2019-12-11T08:31:56.281Z.log | 1 + .../session_crashes_@ 2019-12-11T08:43:46.454Z.log | 2 + 4 files changed, 43 insertions(+), 14 deletions(-) create mode 100644 src/server/daemon/current_daemon_pid.txt create mode 100644 src/server/daemon/session_crashes_@ 2019-12-11T08:31:56.281Z.log create mode 100644 src/server/daemon/session_crashes_@ 2019-12-11T08:43:46.454Z.log (limited to 'src/server/daemon') diff --git a/src/server/daemon/current_daemon_pid.txt b/src/server/daemon/current_daemon_pid.txt new file mode 100644 index 000000000..f3cd0298c --- /dev/null +++ b/src/server/daemon/current_daemon_pid.txt @@ -0,0 +1 @@ +9626 \ No newline at end of file diff --git a/src/server/daemon/persistence_daemon.ts b/src/server/daemon/persistence_daemon.ts index 3eb17a9b4..099c7898c 100644 --- a/src/server/daemon/persistence_daemon.ts +++ b/src/server/daemon/persistence_daemon.ts @@ -1,22 +1,41 @@ import * as request from "request-promise"; -import { log_execution, spawn_detached_process } from "../ActionUtilities"; -import { red, yellow, cyan, green } from "colors"; +import { log_execution, pathFromRoot } from "../ActionUtilities"; +import { red, yellow, cyan, green, Color } from "colors"; import * as nodemailer from "nodemailer"; import { MailOptions } from "nodemailer/lib/json-transport"; -import { writeFileSync } from "fs"; +import { writeFileSync, appendFileSync, createWriteStream, existsSync } from "fs"; import { resolve } from 'path'; +import { ChildProcess } from "child_process"; +import { ProcessManager } from "../ProcessManager"; + +console.log(yellow("Initializing daemon...")); + +process.on('SIGINT', () => current_backup?.kill("SIGTERM")); + +const crashLogPath = resolve(__dirname, `./session_crashes_${timestamp()}.log`); +function addLogEntry(message: string, color: Color) { + const formatted = color(`${message} ${timestamp()}.`); + console.log(formatted); + appendFileSync(crashLogPath, `${formatted}\n`); +} const LOCATION = "http://localhost"; const recipient = "samuel_wilkins@brown.edu"; let restarting = false; -writeFileSync(resolve(__dirname, "./current_pid.txt"), process.pid); +const frequency = 10; +const { pid } = process; +writeFileSync(resolve(__dirname, "./current_daemon_pid.txt"), pid); +console.log(cyan(`${pid} written to ./current_daemon_pid.txt`)); function timestamp() { return `@ ${new Date().toISOString()}`; } +let current_backup: ChildProcess | undefined = undefined; + async function listen() { + console.log(yellow(`Beginning to poll server heartbeat every ${frequency} seconds...\n`)); if (!LOCATION) { console.log(red("No location specified for persistence daemon. Please include as a command line environment variable or in a .env file.")); process.exit(0); @@ -28,34 +47,40 @@ async function listen() { let error: any; try { await request.get(heartbeat); + if (restarting) { + addLogEntry("Backup server successfully restarted", green); + } + restarting = false; } catch (e) { error = e; } finally { if (error) { if (!restarting) { restarting = true; - console.log(yellow("Detected a server crash!")); + addLogEntry("Detected a server crash", red); + current_backup?.kill(); await log_execution({ startMessage: "Sending crash notification email", endMessage: ({ error, result }) => { const success = error === null && result === true; - return (success ? `Notification successfully sent to ` : `Failed to notify `) + recipient; + return `${(success ? `Notification successfully sent to` : `Failed to notify`)} ${recipient} ${timestamp()}`; }, action: async () => notify(error || "Hmm, no error to report..."), color: cyan }); - console.log(await log_execution({ + current_backup = await log_execution({ startMessage: "Initiating server restart", - endMessage: "Server successfully restarted", - action: () => spawn_detached_process(`npm run start-spawn`), + endMessage: ({ result, error }) => { + const success = error === null && result !== undefined; + return success ? "Child process spawned.." : `An error occurred while attempting to restart the server:\n${error}`; + }, + action: () => ProcessManager.spawn_detached('npm', ['run', 'start-spawn']), color: green - })); - restarting = false; + }); + writeFileSync(pathFromRoot("./logs/current_server_pid.txt"), `${current_backup?.pid ?? -1} created ${timestamp()}\n`); } else { console.log(yellow(`Callback ignored because restarting already initiated ${timestamp()}`)); } - } else { - console.log(green(`No issues detected ${timestamp()}`)); } } }, 1000 * 10); @@ -85,7 +110,7 @@ async function notify(error: any) { text: emailText(error) } as MailOptions; return new Promise(resolve => { - smtpTransport.sendMail(mailOptions, (dispatchError: Error | null) => { console.log(dispatchError); resolve(dispatchError === null); }); + smtpTransport.sendMail(mailOptions, (dispatchError: Error | null) => resolve(dispatchError === null)); }); } diff --git a/src/server/daemon/session_crashes_@ 2019-12-11T08:31:56.281Z.log b/src/server/daemon/session_crashes_@ 2019-12-11T08:31:56.281Z.log new file mode 100644 index 000000000..32b7810ea --- /dev/null +++ b/src/server/daemon/session_crashes_@ 2019-12-11T08:31:56.281Z.log @@ -0,0 +1 @@ +Detected a server crash @ 2019-12-11T08:32:36.317Z diff --git a/src/server/daemon/session_crashes_@ 2019-12-11T08:43:46.454Z.log b/src/server/daemon/session_crashes_@ 2019-12-11T08:43:46.454Z.log new file mode 100644 index 000000000..ebb6843c2 --- /dev/null +++ b/src/server/daemon/session_crashes_@ 2019-12-11T08:43:46.454Z.log @@ -0,0 +1,2 @@ +Detected a server crash @ 2019-12-11T08:44:26.494Z. +Backup server successfully restarted @ 2019-12-11T08:45:33.644Z. -- cgit v1.2.3-70-g09d2