From 6665483b80ff6874cf9cc7c9cb3f7e58fcec20ca Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Tue, 10 Dec 2019 23:40:42 -0500 Subject: persistance daemon improvements --- src/server/ActionUtilities.ts | 13 ++++- src/server/ProcessManager.ts | 45 ++++++++++++++++ src/server/daemon/persistence_daemon.ts | 92 +++++++++++++++++++++++++++++++++ src/server/index.ts | 13 ++++- src/server/persistence_daemon.ts | 82 ----------------------------- 5 files changed, 161 insertions(+), 84 deletions(-) create mode 100644 src/server/ProcessManager.ts create mode 100644 src/server/daemon/persistence_daemon.ts delete mode 100644 src/server/persistence_daemon.ts (limited to 'src') diff --git a/src/server/ActionUtilities.ts b/src/server/ActionUtilities.ts index bc978c982..2173f4369 100644 --- a/src/server/ActionUtilities.ts +++ b/src/server/ActionUtilities.ts @@ -1,12 +1,16 @@ import * as fs from 'fs'; import { ExecOptions } from 'shelljs'; -import { exec } from 'child_process'; +import { exec, spawn } from 'child_process'; import * as path from 'path'; import * as rimraf from "rimraf"; import { yellow, Color } from 'colors'; const projectRoot = path.resolve(__dirname, "../../"); +export function pathFromRoot(relative: string) { + return path.resolve(projectRoot, relative); +} + export const command_line = (command: string, fromDirectory?: string) => { return new Promise((resolve, reject) => { const options: ExecOptions = {}; @@ -17,6 +21,13 @@ export const command_line = (command: string, fromDirectory?: string) => { }); }; +export async function spawn_detached_process(command: string, args?: readonly string[]) { + const out = path.resolve(projectRoot, `./logs/${command}-${process.pid}-${new Date().toUTCString()}`); + const child_out = fs.openSync(out, 'a'); + const child_error = fs.openSync(out, 'a'); + spawn(command, args, { detached: true, stdio: ["ignore", child_out, child_error] }).unref(); +} + export const read_text_file = (relativePath: string) => { const target = path.resolve(__dirname, relativePath); return new Promise((resolve, reject) => { diff --git a/src/server/ProcessManager.ts b/src/server/ProcessManager.ts new file mode 100644 index 000000000..2237f9e1b --- /dev/null +++ b/src/server/ProcessManager.ts @@ -0,0 +1,45 @@ +import { writeFileSync, unlinkSync, existsSync, mkdirSync } from "fs"; +import { pathFromRoot, log_execution, spawn_detached_process } from './ActionUtilities'; +import { resolve } from "path"; +import { red, yellow } from "colors"; + +const daemonPath = pathFromRoot("./src/server/daemon/persistence_daemon.ts"); + +export namespace ProcessManager { + + export async function initialize() { + const logPath = pathFromRoot("./logs"); + const filePath = resolve(logPath, "./server_pids.txt"); + const exists = existsSync(logPath); + if (exists) { + unlinkSync(filePath); + } else { + mkdirSync(logPath); + } + const { pid } = process; + if (process.env.SPAWNED === "true") { + writeFileSync(filePath, `${pid} created at ${new Date().toUTCString()}\n`); + } + } + + let daemonInitialized = false; + export async function trySpawnDaemon() { + if (!daemonInitialized) { + daemonInitialized = true; + await log_execution({ + startMessage: "\ninitializing persistence daemon", + endMessage: ({ result, error }) => { + const success = error === null && result !== undefined; + if (!success) { + console.log(red("failed to initialize the persistance daemon")); + process.exit(0); + } + return "persistence daemon process closed"; + }, + action: () => spawn_detached_process("npx ts-node", [daemonPath]), + color: yellow + }); + } + } + +} \ No newline at end of file diff --git a/src/server/daemon/persistence_daemon.ts b/src/server/daemon/persistence_daemon.ts new file mode 100644 index 000000000..3eb17a9b4 --- /dev/null +++ b/src/server/daemon/persistence_daemon.ts @@ -0,0 +1,92 @@ +import * as request from "request-promise"; +import { log_execution, spawn_detached_process } from "../ActionUtilities"; +import { red, yellow, cyan, green } from "colors"; +import * as nodemailer from "nodemailer"; +import { MailOptions } from "nodemailer/lib/json-transport"; +import { writeFileSync } from "fs"; +import { resolve } from 'path'; + +const LOCATION = "http://localhost"; +const recipient = "samuel_wilkins@brown.edu"; +let restarting = false; + +writeFileSync(resolve(__dirname, "./current_pid.txt"), process.pid); + +function timestamp() { + return `@ ${new Date().toISOString()}`; +} + +async function listen() { + if (!LOCATION) { + console.log(red("No location specified for persistence daemon. Please include as a command line environment variable or in a .env file.")); + process.exit(0); + } + const heartbeat = `${LOCATION}:1050/serverHeartbeat`; + // if this is on our remote server, the server must be run in release mode + // const suffix = LOCATION.includes("localhost") ? "" : "-release"; + setInterval(async () => { + let error: any; + try { + await request.get(heartbeat); + } catch (e) { + error = e; + } finally { + if (error) { + if (!restarting) { + restarting = true; + console.log(yellow("Detected a server crash!")); + await log_execution({ + startMessage: "Sending crash notification email", + endMessage: ({ error, result }) => { + const success = error === null && result === true; + return (success ? `Notification successfully sent to ` : `Failed to notify `) + recipient; + }, + action: async () => notify(error || "Hmm, no error to report..."), + color: cyan + }); + console.log(await log_execution({ + startMessage: "Initiating server restart", + endMessage: "Server successfully restarted", + action: () => spawn_detached_process(`npm run start-spawn`), + color: green + })); + restarting = false; + } else { + console.log(yellow(`Callback ignored because restarting already initiated ${timestamp()}`)); + } + } else { + console.log(green(`No issues detected ${timestamp()}`)); + } + } + }, 1000 * 10); +} + +function emailText(error: any) { + return [ + `Hey ${recipient.split("@")[0]},`, + "You, as a Dash Administrator, are being notified of a server crash event. Here's what we know:", + `Location: ${LOCATION}\nError: ${error}`, + "The server should already be restarting itself, but if you're concerned, use the Remote Desktop Connection to monitor progress." + ].join("\n\n"); +} + +async function notify(error: any) { + const smtpTransport = nodemailer.createTransport({ + service: 'Gmail', + auth: { + user: 'brownptcdash@gmail.com', + pass: 'browngfx1' + } + }); + const mailOptions = { + to: recipient, + from: 'brownptcdash@gmail.com', + subject: 'Dash Server Crash', + text: emailText(error) + } as MailOptions; + return new Promise(resolve => { + smtpTransport.sendMail(mailOptions, (dispatchError: Error | null) => { console.log(dispatchError); resolve(dispatchError === null); }); + }); +} + +listen(); \ No newline at end of file diff --git a/src/server/index.ts b/src/server/index.ts index 7671936a2..795418b31 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -18,11 +18,12 @@ import { GoogleCredentialsLoader } from './credentials/CredentialsLoader'; import DeleteManager from "./ApiManagers/DeleteManager"; import PDFManager from "./ApiManagers/PDFManager"; import UploadManager from "./ApiManagers/UploadManager"; -import { log_execution, command_line } from "./ActionUtilities"; +import { log_execution } from "./ActionUtilities"; import GeneralGoogleManager from "./ApiManagers/GeneralGoogleManager"; import GooglePhotosManager from "./ApiManagers/GooglePhotosManager"; import { yellow, red } from "colors"; import { disconnect } from "../server/Initialization"; +import { ProcessManager } from "./ProcessManager"; export const publicDirectory = path.resolve(__dirname, "public"); export const filesDirectory = path.resolve(publicDirectory, "files"); @@ -35,6 +36,7 @@ export const ExitHandlers = new Array<() => void>(); * before clients can access the server should be run or awaited here. */ async function preliminaryFunctions() { + await ProcessManager.initialize(); await GoogleCredentialsLoader.loadCredentials(); GoogleApiServerUtils.processProjectCredentials(); await DashUploadUtils.buildFileDirectories(); @@ -119,6 +121,15 @@ function routeSetter({ isRelease, addSupervisedRoute, logRegistrationOutcome }: } }); + addSupervisedRoute({ + method: Method.GET, + subscription: "/persist", + onValidation: ({ res }) => { + ProcessManager.trySpawnDaemon(); + res.redirect("/home"); + } + }); + logRegistrationOutcome(); // initialize the web socket (bidirectional communication: if a user changes diff --git a/src/server/persistence_daemon.ts b/src/server/persistence_daemon.ts deleted file mode 100644 index 2cb17456c..000000000 --- a/src/server/persistence_daemon.ts +++ /dev/null @@ -1,82 +0,0 @@ -import * as request from "request-promise"; -import { command_line, log_execution } from "./ActionUtilities"; -import { red, yellow, cyan, green } from "colors"; -import * as nodemailer from "nodemailer"; -import { MailOptions } from "nodemailer/lib/json-transport"; -import { Database } from "./database"; - -const LOCATION = "http://localhost"; -const recipient = "samuel_wilkins@brown.edu"; -let restarting = false; - -async function listen() { - if (!LOCATION) { - console.log(red("No location specified for persistence daemon. Please include as a command line environment variable or in a .env file.")); - process.exit(0); - } - const heartbeat = `${LOCATION}:1050/serverHeartbeat`; - // if this is on our remote server, the server must be run in release mode - const suffix = LOCATION.includes("localhost") ? "" : "-release"; - setInterval(async () => { - let response: any; - let error: any; - try { - response = await request.get(heartbeat); - } catch (e) { - error = e; - } finally { - if (!response && !restarting) { - restarting = true; - console.log(yellow("Detected a server crash!")); - await log_execution({ - startMessage: "Sending crash notification email", - endMessage: ({ error, result }) => { - const success = error === null && result === true; - return (success ? `Notification successfully sent to ` : `Failed to notify `) + recipient; - }, - action: async () => notify(error || "Hmm, no error to report..."), - color: cyan - }); - console.log(await log_execution({ - startMessage: "Initiating server restart", - endMessage: "Server successfully restarted", - action: () => command_line(`npm run start${suffix}`), - color: green - })); - restarting = false; - } else { - console.log(green(`No issues detected as of ${new Date().toISOString()}`)); - } - } - }, 1000 * 10); -} - -function emailText(error: any) { - return [ - `Hey ${recipient.split("@")[0]},`, - "You, as a Dash Administrator, are being notified of a server crash event. Here's what we know:", - `Location: ${LOCATION}\nError: ${error}`, - "The server should already be restarting itself, but if you're concerned, use the Remote Desktop Connection to monitor progress." - ].join("\n\n"); -} - -async function notify(error: any) { - const smtpTransport = nodemailer.createTransport({ - service: 'Gmail', - auth: { - user: 'brownptcdash@gmail.com', - pass: 'browngfx1' - } - }); - const mailOptions = { - to: recipient, - from: 'brownptcdash@gmail.com', - subject: 'Dash Server Crash', - text: emailText(error) - } as MailOptions; - return new Promise(resolve => { - smtpTransport.sendMail(mailOptions, (dispatchError: Error | null) => { console.log(dispatchError); resolve(dispatchError === null); }); - }); -} - -listen(); \ No newline at end of file -- cgit v1.2.3-70-g09d2