diff options
Diffstat (limited to 'src')
23 files changed, 574 insertions, 329 deletions
diff --git a/src/client/DocServer.ts b/src/client/DocServer.ts index befe9ea5c..47c63bfb7 100644 --- a/src/client/DocServer.ts +++ b/src/client/DocServer.ts @@ -82,7 +82,7 @@ export namespace DocServer { Utils.AddServerHandler(_socket, MessageStore.UpdateField, respondToUpdate); Utils.AddServerHandler(_socket, MessageStore.DeleteField, respondToDelete); Utils.AddServerHandler(_socket, MessageStore.DeleteFields, respondToDelete); - _socket.on("connection_terminated", () => alert("Your connection to the server has been terminated.")); + Utils.AddServerHandler(_socket, MessageStore.ConnectionTerminated, () => alert("Your connection to the server has been terminated.")); } function errorFunc(): never { diff --git a/src/server/ActionUtilities.ts b/src/server/ActionUtilities.ts index 053576a92..30aed32e6 100644 --- a/src/server/ActionUtilities.ts +++ b/src/server/ActionUtilities.ts @@ -4,6 +4,8 @@ import { exec } from 'child_process'; import * as path from 'path'; import * as rimraf from "rimraf"; import { yellow, Color } from 'colors'; +import * as nodemailer from "nodemailer"; +import { MailOptions } from "nodemailer/lib/json-transport"; const projectRoot = path.resolve(__dirname, "../../"); export function pathFromRoot(relative?: string) { @@ -105,3 +107,37 @@ export async function Prune(rootDirectory: string): Promise<boolean> { } export const Destroy = (mediaPath: string) => new Promise<boolean>(resolve => unlink(mediaPath, error => resolve(error === null))); + +export namespace Email { + + const smtpTransport = nodemailer.createTransport({ + service: 'Gmail', + auth: { + user: 'brownptcdash@gmail.com', + pass: 'browngfx1' + } + }); + + export async function dispatchAll(recipients: string[], subject: string, content: string) { + const failures: string[] = []; + await Promise.all(recipients.map(async (recipient: string) => { + if (!await Email.dispatch(recipient, subject, content)) { + failures.push(recipient); + } + })); + return failures; + } + + export async function dispatch(recipient: string, subject: string, content: string): Promise<boolean> { + const mailOptions = { + to: recipient, + from: 'brownptcdash@gmail.com', + subject, + text: `Hello ${recipient.split("@")[0]},\n\n${content}` + } as MailOptions; + return new Promise<boolean>(resolve => { + smtpTransport.sendMail(mailOptions, (dispatchError: Error | null) => resolve(dispatchError === null)); + }); + } + +}
\ No newline at end of file diff --git a/src/server/ApiManagers/DeleteManager.ts b/src/server/ApiManagers/DeleteManager.ts index 71818c673..88dfa6a64 100644 --- a/src/server/ApiManagers/DeleteManager.ts +++ b/src/server/ApiManagers/DeleteManager.ts @@ -10,7 +10,7 @@ export default class DeleteManager extends ApiManager { register({ method: Method.GET, subscription: "/delete", - onValidation: async ({ res, isRelease }) => { + secureHandler: async ({ res, isRelease }) => { if (isRelease) { return _permission_denied(res, deletionPermissionError); } @@ -22,7 +22,7 @@ export default class DeleteManager extends ApiManager { register({ method: Method.GET, subscription: "/deleteAll", - onValidation: async ({ res, isRelease }) => { + secureHandler: async ({ res, isRelease }) => { if (isRelease) { return _permission_denied(res, deletionPermissionError); } @@ -35,7 +35,7 @@ export default class DeleteManager extends ApiManager { register({ method: Method.GET, subscription: "/deleteWithAux", - onValidation: async ({ res, isRelease }) => { + secureHandler: async ({ res, isRelease }) => { if (isRelease) { return _permission_denied(res, deletionPermissionError); } @@ -47,7 +47,7 @@ export default class DeleteManager extends ApiManager { register({ method: Method.GET, subscription: "/deleteWithGoogleCredentials", - onValidation: async ({ res, isRelease }) => { + secureHandler: async ({ res, isRelease }) => { if (isRelease) { return _permission_denied(res, deletionPermissionError); } diff --git a/src/server/ApiManagers/DownloadManager.ts b/src/server/ApiManagers/DownloadManager.ts index d9808704b..1bb84f374 100644 --- a/src/server/ApiManagers/DownloadManager.ts +++ b/src/server/ApiManagers/DownloadManager.ts @@ -33,7 +33,7 @@ export default class DownloadManager extends ApiManager { register({ method: Method.GET, subscription: new RouteSubscriber("imageHierarchyExport").add('docId'), - onValidation: async ({ req, res }) => { + secureHandler: async ({ req, res }) => { const id = req.params.docId; const hierarchy: Hierarchy = {}; await buildHierarchyRecursive(id, hierarchy); @@ -44,7 +44,7 @@ export default class DownloadManager extends ApiManager { register({ method: Method.GET, subscription: new RouteSubscriber("downloadId").add("docId"), - onValidation: async ({ req, res }) => { + secureHandler: async ({ req, res }) => { return BuildAndDispatchZip(res, async zip => { const { id, docs, files } = await getDocs(req.params.docId); const docString = JSON.stringify({ id, docs }); @@ -59,7 +59,7 @@ export default class DownloadManager extends ApiManager { register({ method: Method.GET, subscription: new RouteSubscriber("serializeDoc").add("docId"), - onValidation: async ({ req, res }) => { + secureHandler: async ({ req, res }) => { const { docs, files } = await getDocs(req.params.docId); res.send({ docs, files: Array.from(files) }); } diff --git a/src/server/ApiManagers/GeneralGoogleManager.ts b/src/server/ApiManagers/GeneralGoogleManager.ts index 3617779d5..a5240edbc 100644 --- a/src/server/ApiManagers/GeneralGoogleManager.ts +++ b/src/server/ApiManagers/GeneralGoogleManager.ts @@ -19,7 +19,7 @@ export default class GeneralGoogleManager extends ApiManager { register({ method: Method.GET, subscription: "/readGoogleAccessToken", - onValidation: async ({ user, res }) => { + secureHandler: async ({ user, res }) => { const token = await GoogleApiServerUtils.retrieveAccessToken(user.id); if (!token) { return res.send(GoogleApiServerUtils.generateAuthenticationUrl()); @@ -31,7 +31,7 @@ export default class GeneralGoogleManager extends ApiManager { register({ method: Method.POST, subscription: "/writeGoogleAccessToken", - onValidation: async ({ user, req, res }) => { + secureHandler: async ({ user, req, res }) => { res.send(await GoogleApiServerUtils.processNewUser(user.id, req.body.authenticationCode)); } }); @@ -39,7 +39,7 @@ export default class GeneralGoogleManager extends ApiManager { register({ method: Method.POST, subscription: new RouteSubscriber("googleDocs").add("sector", "action"), - onValidation: async ({ req, res, user }) => { + secureHandler: async ({ req, res, user }) => { const sector: GoogleApiServerUtils.Service = req.params.sector as GoogleApiServerUtils.Service; const action: GoogleApiServerUtils.Action = req.params.action as GoogleApiServerUtils.Action; const endpoint = await GoogleApiServerUtils.GetEndpoint(GoogleApiServerUtils.Service[sector], user.id); diff --git a/src/server/ApiManagers/GooglePhotosManager.ts b/src/server/ApiManagers/GooglePhotosManager.ts index e2539f120..107542ce2 100644 --- a/src/server/ApiManagers/GooglePhotosManager.ts +++ b/src/server/ApiManagers/GooglePhotosManager.ts @@ -41,7 +41,7 @@ export default class GooglePhotosManager extends ApiManager { register({ method: Method.POST, subscription: "/googlePhotosMediaUpload", - onValidation: async ({ user, req, res }) => { + secureHandler: async ({ user, req, res }) => { const { media } = req.body; const token = await GoogleApiServerUtils.retrieveAccessToken(user.id); if (!token) { @@ -82,7 +82,7 @@ export default class GooglePhotosManager extends ApiManager { register({ method: Method.POST, subscription: "/googlePhotosMediaDownload", - onValidation: async ({ req, res }) => { + secureHandler: async ({ req, res }) => { const contents: { mediaItems: MediaItem[] } = req.body; let failed = 0; if (contents) { diff --git a/src/server/ApiManagers/PDFManager.ts b/src/server/ApiManagers/PDFManager.ts index 7e862631d..0136b758e 100644 --- a/src/server/ApiManagers/PDFManager.ts +++ b/src/server/ApiManagers/PDFManager.ts @@ -17,7 +17,7 @@ export default class PDFManager extends ApiManager { register({ method: Method.GET, subscription: new RouteSubscriber("thumbnail").add("filename"), - onValidation: ({ req, res }) => getOrCreateThumbnail(req.params.filename, res) + secureHandler: ({ req, res }) => getOrCreateThumbnail(req.params.filename, res) }); } diff --git a/src/server/ApiManagers/SearchManager.ts b/src/server/ApiManagers/SearchManager.ts index 37d66666b..c1c908088 100644 --- a/src/server/ApiManagers/SearchManager.ts +++ b/src/server/ApiManagers/SearchManager.ts @@ -8,6 +8,7 @@ import { command_line } from "../ActionUtilities"; import request = require('request-promise'); import { red } from "colors"; import RouteSubscriber from "../RouteSubscriber"; +import { execSync } from "child_process"; export class SearchManager extends ApiManager { @@ -16,7 +17,7 @@ export class SearchManager extends ApiManager { register({ method: Method.GET, subscription: new RouteSubscriber("solr").add("action"), - onValidation: async ({ req, res }) => { + secureHandler: async ({ req, res }) => { const { action } = req.params; if (["start", "stop"].includes(action)) { const status = req.params.action === "start"; @@ -30,7 +31,7 @@ export class SearchManager extends ApiManager { register({ method: Method.GET, subscription: "/textsearch", - onValidation: async ({ req, res }) => { + secureHandler: async ({ req, res }) => { const q = req.query.q; if (q === undefined) { res.send([]); @@ -50,7 +51,7 @@ export class SearchManager extends ApiManager { register({ method: Method.GET, subscription: "/search", - onValidation: async ({ req, res }) => { + secureHandler: async ({ req, res }) => { const solrQuery: any = {}; ["q", "fq", "start", "rows", "hl", "hl.fl"].forEach(key => solrQuery[key] = req.query[key]); if (solrQuery.q === undefined) { @@ -72,10 +73,11 @@ export namespace SolrManager { const args = status ? "start" : "stop -p 8983"; try { console.log(`Solr management: trying to ${args}`); - console.log(await command_line(`solr.cmd ${args}`, "./solr-8.3.1/bin")); + console.log(execSync(`./solr.cmd ${args}`, { cwd: "./solr-8.3.1/bin" })); return true; } catch (e) { console.log(red(`Solr management error: unable to ${args}`)); + console.log(e); return false; } } diff --git a/src/server/ApiManagers/UploadManager.ts b/src/server/ApiManagers/UploadManager.ts index da1f83b75..74f45ae62 100644 --- a/src/server/ApiManagers/UploadManager.ts +++ b/src/server/ApiManagers/UploadManager.ts @@ -41,7 +41,7 @@ export default class UploadManager extends ApiManager { register({ method: Method.POST, subscription: "/upload", - onValidation: async ({ req, res }) => { + secureHandler: async ({ req, res }) => { const form = new formidable.IncomingForm(); form.uploadDir = pathToDirectory(Directory.parsed_files); form.keepExtensions = true; @@ -62,7 +62,7 @@ export default class UploadManager extends ApiManager { register({ method: Method.POST, subscription: "/uploadDoc", - onValidation: ({ req, res }) => { + secureHandler: ({ req, res }) => { const form = new formidable.IncomingForm(); form.keepExtensions = true; // let path = req.body.path; @@ -166,7 +166,7 @@ export default class UploadManager extends ApiManager { register({ method: Method.POST, subscription: "/inspectImage", - onValidation: async ({ req, res }) => { + secureHandler: async ({ req, res }) => { const { source } = req.body; if (typeof source === "string") { const { serverAccessPaths } = await DashUploadUtils.UploadImage(source); @@ -179,7 +179,7 @@ export default class UploadManager extends ApiManager { register({ method: Method.POST, subscription: "/uploadURI", - onValidation: ({ req, res }) => { + secureHandler: ({ req, res }) => { const uri = req.body.uri; const filename = req.body.name; if (!uri || !filename) { diff --git a/src/server/ApiManagers/UserManager.ts b/src/server/ApiManagers/UserManager.ts index 0f7d14320..f2ef22961 100644 --- a/src/server/ApiManagers/UserManager.ts +++ b/src/server/ApiManagers/UserManager.ts @@ -16,7 +16,7 @@ export default class UserManager extends ApiManager { register({ method: Method.GET, subscription: "/getUsers", - onValidation: async ({ res }) => { + secureHandler: async ({ res }) => { const cursor = await Database.Instance.query({}, { email: 1, userDocumentId: 1 }, "users"); const results = await cursor.toArray(); res.send(results.map(user => ({ email: user.email, userDocumentId: user.userDocumentId }))); @@ -26,20 +26,20 @@ export default class UserManager extends ApiManager { register({ method: Method.GET, subscription: "/getUserDocumentId", - onValidation: ({ res, user }) => res.send(user.userDocumentId) + secureHandler: ({ res, user }) => res.send(user.userDocumentId) }); register({ method: Method.GET, subscription: "/getCurrentUser", - onValidation: ({ res, user }) => res.send(JSON.stringify(user)), - onUnauthenticated: ({ res }) => res.send(JSON.stringify({ id: "__guest__", email: "" })) + secureHandler: ({ res, user }) => res.send(JSON.stringify(user)), + publicHandler: ({ res }) => res.send(JSON.stringify({ id: "__guest__", email: "" })) }); register({ method: Method.GET, subscription: "/activity", - onValidation: ({ res }) => { + secureHandler: ({ res }) => { const now = Date.now(); const activeTimes: ActivityUnit[] = []; diff --git a/src/server/ApiManagers/UtilManager.ts b/src/server/ApiManagers/UtilManager.ts index 2f1bd956f..a0d0d0f4b 100644 --- a/src/server/ApiManagers/UtilManager.ts +++ b/src/server/ApiManagers/UtilManager.ts @@ -12,7 +12,7 @@ export default class UtilManager extends ApiManager { register({ method: Method.GET, subscription: new RouteSubscriber("environment").add("key"), - onValidation: ({ req, res }) => { + secureHandler: ({ req, res }) => { const { key } = req.params; const value = process.env[key]; if (!value) { @@ -25,7 +25,7 @@ export default class UtilManager extends ApiManager { register({ method: Method.GET, subscription: "/pull", - onValidation: async ({ res }) => { + secureHandler: async ({ res }) => { return new Promise<void>(resolve => { exec('"C:\\Program Files\\Git\\git-bash.exe" -c "git pull"', err => { if (err) { @@ -42,7 +42,7 @@ export default class UtilManager extends ApiManager { register({ method: Method.GET, subscription: "/buxton", - onValidation: async ({ res }) => { + secureHandler: async ({ res }) => { const cwd = './src/scraping/buxton'; const onResolved = (stdout: string) => { console.log(stdout); res.redirect("/"); }; @@ -56,7 +56,7 @@ export default class UtilManager extends ApiManager { register({ method: Method.GET, subscription: "/version", - onValidation: ({ res }) => { + secureHandler: ({ res }) => { return new Promise<void>(resolve => { exec('"C:\\Program Files\\Git\\bin\\git.exe" rev-parse HEAD', (err, stdout) => { if (err) { diff --git a/src/server/RouteManager.ts b/src/server/RouteManager.ts index 9e84b3687..25259bd88 100644 --- a/src/server/RouteManager.ts +++ b/src/server/RouteManager.ts @@ -14,16 +14,16 @@ export interface CoreArguments { isRelease: boolean; } -export type OnValidation = (core: CoreArguments & { user: DashUserModel }) => any | Promise<any>; -export type OnUnauthenticated = (core: CoreArguments) => any | Promise<any>; -export type OnError = (core: CoreArguments & { error: any }) => any | Promise<any>; +export type SecureHandler = (core: CoreArguments & { user: DashUserModel }) => any | Promise<any>; +export type PublicHandler = (core: CoreArguments) => any | Promise<any>; +export type ErrorHandler = (core: CoreArguments & { error: any }) => any | Promise<any>; export interface RouteInitializer { method: Method; subscription: string | RouteSubscriber | (string | RouteSubscriber)[]; - onValidation: OnValidation; - onUnauthenticated?: OnUnauthenticated; - onError?: OnError; + secureHandler: SecureHandler; + publicHandler?: PublicHandler; + errorHandler?: ErrorHandler; } const registered = new Map<string, Set<Method>>(); @@ -69,7 +69,7 @@ export default class RouteManager { if (malformedCount) { console.log(`please ensure all routes adhere to ^\/$|^\/[A-Za-z]+(\/\:[A-Za-z]+)*$`); } - process.exit(0); + process.exit(1); } else { console.log(green("all server routes have been successfully registered:")); Array.from(registered.keys()).sort().forEach(route => console.log(cyan(route))); @@ -82,7 +82,7 @@ export default class RouteManager { * @param initializer */ addSupervisedRoute = (initializer: RouteInitializer): void => { - const { method, subscription, onValidation, onUnauthenticated, onError } = initializer; + const { method, subscription, secureHandler: onValidation, publicHandler: onUnauthenticated, errorHandler: onError } = initializer; const isRelease = this._isRelease; const supervised = async (req: express.Request, res: express.Response) => { const { user, originalUrl: target } = req; diff --git a/src/server/Session/session.ts b/src/server/Session/session.ts new file mode 100644 index 000000000..cf2231b1f --- /dev/null +++ b/src/server/Session/session.ts @@ -0,0 +1,338 @@ +import { red, cyan, green, yellow, magenta, blue } from "colors"; +import { on, fork, setupMaster, Worker } from "cluster"; +import { execSync } from "child_process"; +import { get } from "request-promise"; +import { Utils } from "../../Utils"; +import Repl, { ReplAction } from "../repl"; +import { readFileSync } from "fs"; +import { validate, ValidationError } from "jsonschema"; +import { configurationSchema } from "./session_config_schema"; + +const onWindows = process.platform === "win32"; + +/** + * This namespace relies on NodeJS's cluster module, which allows a parent (master) process to share + * code with its children (workers). A simple `isMaster` flag indicates who is trying to access + * the code, and thus determines the functionality that actually gets invoked (checked by the caller, not internally). + * + * Think of the master thread as a factory, and the workers as the helpers that actually run the server. + * + * So, when we run `npm start`, given the appropriate check, initializeMaster() is called in the parent process + * This will spawn off its own child process (by default, mirrors the execution path of its parent), + * in which initializeWorker() is invoked. + */ +export namespace Session { + + interface Configuration { + showServerOutput: boolean; + masterIdentifier: string; + workerIdentifier: string; + ports: { [description: string]: number }; + pollingRoute: string; + pollingIntervalSeconds: number; + [key: string]: any; + } + + const defaultConfiguration: Configuration = { + showServerOutput: false, + masterIdentifier: yellow("__monitor__:"), + workerIdentifier: magenta("__server__:"), + ports: { server: 3000 }, + pollingRoute: "/", + pollingIntervalSeconds: 30 + }; + + interface MasterExtensions { + addReplCommand: (basename: string, argPatterns: (RegExp | string)[], action: ReplAction) => void; + addChildMessageHandler: (message: string, handler: ActionHandler) => void; + } + + export interface NotifierHooks { + key?: (key: string) => boolean | Promise<boolean>; + crash?: (error: Error) => boolean | Promise<boolean>; + } + + export interface SessionAction { + message: string; + args: any; + } + + export type ExitHandler = (error: Error) => void | Promise<void>; + export type ActionHandler = (action: SessionAction) => void | Promise<void>; + export interface EmailTemplate { + subject: string; + body: string; + } + + function loadAndValidateConfiguration(): any { + try { + const configuration: Configuration = JSON.parse(readFileSync('./session.config.json', 'utf8')); + const options = { + throwError: true, + allowUnknownAttributes: false + }; + // ensure all necessary and no excess information is specified by the configuration file + validate(configuration, configurationSchema, options); + let formatMaster = true; + let formatWorker = true; + Object.keys(defaultConfiguration).forEach(property => { + if (!configuration[property]) { + if (property === "masterIdentifier") { + formatMaster = false; + } else if (property === "workerIdentifier") { + formatWorker = false; + } + configuration[property] = defaultConfiguration[property]; + } + }); + if (formatMaster) { + configuration.masterIdentifier = yellow(configuration.masterIdentifier + ":"); + } + if (formatWorker) { + configuration.workerIdentifier = magenta(configuration.workerIdentifier + ":"); + } + return configuration; + } catch (error) { + if (error instanceof ValidationError) { + console.log(red("\nSession configuration failed.")); + console.log("The given session.config.json configuration file is invalid."); + console.log(`${error.instance}: ${error.stack}`); + process.exit(0); + } else if (error.code === "ENOENT" && error.path === "./session.config.json") { + console.log(cyan("Loading default session parameters...")); + console.log("Consider including a session.config.json configuration file in your project root for customization."); + return defaultConfiguration; + } else { + console.log(red("\nSession configuration failed.")); + console.log("The following unknown error occurred during configuration."); + console.log(error.stack); + process.exit(0); + } + } + } + + function timestamp() { + return blue(`[${new Date().toUTCString()}]`); + } + + /** + * Validates and reads the configuration file, accordingly builds a child process factory + * and spawns off an initial process that will respawn as predecessors die. + */ + export async function initializeMonitorThread(notifiers?: NotifierHooks): Promise<MasterExtensions> { + let activeWorker: Worker; + const childMessageHandlers: { [message: string]: (action: SessionAction, args: any) => void } = {}; + + // read in configuration .json file only once, in the master thread + // pass down any variables the pertinent to the child processes as environment variables + const { + masterIdentifier, + workerIdentifier, + ports, + pollingRoute, + showServerOutput, + pollingIntervalSeconds + } = loadAndValidateConfiguration(); + + const masterLog = (...optionalParams: any[]) => console.log(timestamp(), masterIdentifier, ...optionalParams); + + // this sends a pseudorandomly generated guid to the configuration's recipients, allowing them alone + // to kill the server via the /kill/:key route + let key: string | undefined; + if (notifiers && notifiers.key) { + key = Utils.GenerateGuid(); + const success = await notifiers.key(key); + const statement = success ? green("distributed session key to recipients") : red("distribution of session key failed"); + masterLog(statement); + } + + // handle exceptions in the master thread - there shouldn't be many of these + // the IPC (inter process communication) channel closed exception can't seem + // to be caught in a try catch, and is inconsequential, so it is ignored + process.on("uncaughtException", ({ message, stack }) => { + if (message !== "Channel closed") { + masterLog(red(message)); + if (stack) { + masterLog(`uncaught exception\n${red(stack)}`); + } + } + }); + + // determines whether or not we see the compilation / initialization / runtime output of each child server process + setupMaster({ silent: !showServerOutput }); + + // attempts to kills the active worker ungracefully + const tryKillActiveWorker = (graceful = false): boolean => { + if (activeWorker && !activeWorker.isDead()) { + if (graceful) { + activeWorker.kill(); + } else { + activeWorker.process.kill(); + } + return true; + } + return false; + }; + + const restart = () => { + // indicate to the worker that we are 'expecting' this restart + activeWorker.send({ setResponsiveness: false }); + tryKillActiveWorker(); + }; + + const setPort = (port: string, value: number, immediateRestart: boolean) => { + if (value > 1023 && value < 65536) { + ports[port] = value; + if (immediateRestart) { + restart(); + } + } else { + masterLog(red(`${port} is an invalid port number`)); + } + }; + + // kills the current active worker and proceeds to spawn a new worker, + // feeding in configuration information as environment variables + const spawn = (): void => { + tryKillActiveWorker(); + activeWorker = fork({ + pollingRoute, + serverPort: ports.server, + socketPort: ports.socket, + pollingIntervalSeconds, + session_key: key + }); + masterLog(`spawned new server worker with process id ${activeWorker.process.pid}`); + // an IPC message handler that executes actions on the master thread when prompted by the active worker + activeWorker.on("message", async ({ lifecycle, action }) => { + if (action) { + const { message, args } = action as SessionAction; + console.log(timestamp(), `${workerIdentifier} action requested (${cyan(message)})`); + switch (message) { + case "kill": + masterLog(red("an authorized user has manually ended the server session")); + tryKillActiveWorker(true); + process.exit(0); + case "notify_crash": + if (notifiers && notifiers.crash) { + const { error } = args; + const success = await notifiers.crash(error); + const statement = success ? green("distributed crash notification to recipients") : red("distribution of crash notification failed"); + masterLog(statement); + } + case "set_port": + const { port, value, immediateRestart } = args; + setPort(port, value, immediateRestart); + default: + const handler = childMessageHandlers[message]; + if (handler) { + handler(action, args); + } + } + } else if (lifecycle) { + console.log(timestamp(), `${workerIdentifier} lifecycle phase (${lifecycle})`); + } + }); + }; + + // a helpful cluster event called on the master thread each time a child process exits + on("exit", ({ process: { pid } }, code, signal) => { + const prompt = `server worker with process id ${pid} has exited with code ${code}${signal === null ? "" : `, having encountered signal ${signal}`}.`; + masterLog(cyan(prompt)); + // to make this a robust, continuous session, every time a child process dies, we immediately spawn a new one + spawn(); + }); + + // builds the repl that allows the following commands to be typed into stdin of the master thread + const repl = new Repl({ identifier: () => `${timestamp()} ${masterIdentifier}` }); + repl.registerCommand("exit", [], () => execSync(onWindows ? "taskkill /f /im node.exe" : "killall -9 node")); + repl.registerCommand("restart", [], restart); + repl.registerCommand("set", [/[a-zA-Z]+/, "port", /\d+/, /true|false/], args => setPort(args[0], Number(args[2]), args[3] === "true")); + // finally, set things in motion by spawning off the first child (server) process + spawn(); + + // returned to allow the caller to add custom commands + return { + addReplCommand: repl.registerCommand, + addChildMessageHandler: (message: string, handler: ActionHandler) => { childMessageHandlers[message] = handler; } + }; + } + + /** + * Effectively, each worker repairs the connection to the server by reintroducing a consistent state + * if its predecessor has died. It itself also polls the server heartbeat, and exits with a notification + * email if the server encounters an uncaught exception or if the server cannot be reached. + * @param work the function specifying the work to be done by each worker thread + */ + export async function initializeWorkerThread(work: Function): Promise<(handler: ExitHandler) => void> { + let shouldServerBeResponsive = false; + const exitHandlers: ExitHandler[] = []; + + // notify master thread (which will log update in the console) of initialization via IPC + process.send?.({ lifecycle: green("compiling and initializing...") }); + + // updates the local value of listening to the value sent from master + process.on("message", ({ setResponsiveness }) => shouldServerBeResponsive = setResponsiveness); + + // called whenever the process has a reason to terminate, either through an uncaught exception + // in the process (potentially inconsistent state) or the server cannot be reached + const activeExit = async (error: Error): Promise<void> => { + if (!shouldServerBeResponsive) { + return; + } + shouldServerBeResponsive = false; + // communicates via IPC to the master thread that it should dispatch a crash notification email + process.send?.({ + action: { + message: "notify_crash", + args: { error } + } + }); + await Promise.all(exitHandlers.map(handler => handler(error))); + // notify master thread (which will log update in the console) of crash event via IPC + process.send?.({ lifecycle: red(`crash event detected @ ${new Date().toUTCString()}`) }); + process.send?.({ lifecycle: red(error.message) }); + process.exit(1); + }; + + // one reason to exit, as the process might be in an inconsistent state after such an exception + process.on('uncaughtException', activeExit); + + const { + pollingIntervalSeconds, + pollingRoute, + serverPort + } = process.env; + // this monitors the health of the server by submitting a get request to whatever port / route specified + // by the configuration every n seconds, where n is also given by the configuration. + const pollTarget = `http://localhost:${serverPort}${pollingRoute}`; + const pollServer = async (): Promise<void> => { + await new Promise<void>(resolve => { + setTimeout(async () => { + try { + await get(pollTarget); + if (!shouldServerBeResponsive) { + // notify master thread (which will log update in the console) via IPC that the server is up and running + process.send?.({ lifecycle: green(`listening on ${serverPort}...`) }); + } + shouldServerBeResponsive = true; + resolve(); + } catch (error) { + // if we expect the server to be unavailable, i.e. during compilation, + // the listening variable is false, activeExit will return early and the child + // process will continue + activeExit(error); + } + }, 1000 * Number(pollingIntervalSeconds)); + }); + // controlled, asynchronous infinite recursion achieves a persistent poll that does not submit a new request until the previous has completed + pollServer(); + }; + + work(); + pollServer(); // begin polling + + return (handler: ExitHandler) => exitHandlers.push(handler); + } + +}
\ No newline at end of file diff --git a/src/server/Session/session_config_schema.ts b/src/server/Session/session_config_schema.ts new file mode 100644 index 000000000..76af04b9f --- /dev/null +++ b/src/server/Session/session_config_schema.ts @@ -0,0 +1,35 @@ +import { Schema } from "jsonschema"; + +export const configurationSchema: Schema = { + id: "/configuration", + type: "object", + properties: { + ports: { + type: "object", + properties: { + server: { type: "number", minimum: 1024, maximum: 65535 }, + socket: { type: "number", minimum: 1024, maximum: 65535 } + }, + required: ["server"], + additionalProperties: true + }, + pollingRoute: { + type: "string", + pattern: /\/[a-zA-Z]*/g + }, + masterIdentifier: { + type: "string", + minLength: 1 + }, + workerIdentifier: { + type: "string", + minLength: 1 + }, + showServerOutput: { type: "boolean" }, + pollingIntervalSeconds: { + type: "number", + minimum: 1, + maximum: 86400 + } + } +};
\ No newline at end of file diff --git a/src/server/Websocket/Websocket.ts b/src/server/Websocket/Websocket.ts index e1e157fc4..578147d60 100644 --- a/src/server/Websocket/Websocket.ts +++ b/src/server/Websocket/Websocket.ts @@ -13,21 +13,24 @@ import { green } from "colors"; export namespace WebSocket { + export let _socket: Socket; const clients: { [key: string]: Client } = {}; export const socketMap = new Map<SocketIO.Socket, string>(); export let disconnect: Function; - export async function start(serverPort: number, isRelease: boolean) { + export async function start(isRelease: boolean) { await preliminaryFunctions(); - initialize(serverPort, isRelease); + initialize(isRelease); } async function preliminaryFunctions() { } - export function initialize(socketPort: number, isRelease: boolean) { + function initialize(isRelease: boolean) { const endpoint = io(); endpoint.on("connection", function (socket: Socket) { + _socket = socket; + socket.use((_packet, next) => { const userEmail = socketMap.get(socket); if (userEmail) { @@ -60,6 +63,7 @@ export namespace WebSocket { }; }); + const socketPort = isRelease ? Number(process.env.socketPort) : 4321; endpoint.listen(socketPort); logPort("websocket", socketPort); } diff --git a/src/server/index.ts b/src/server/index.ts index 2cc35ccec..5e411aa3a 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -6,8 +6,8 @@ import { Database } from './database'; const serverPort = 4321; import { DashUploadUtils } from './DashUploadUtils'; import RouteSubscriber from './RouteSubscriber'; -import initializeServer from './Initialization'; -import RouteManager, { Method, _success, _permission_denied, _error, _invalid, OnUnauthenticated } from './RouteManager'; +import initializeServer from './server_Initialization'; +import RouteManager, { Method, _success, _permission_denied, _error, _invalid, PublicHandler } from './RouteManager'; import * as qs from 'query-string'; import UtilManager from './ApiManagers/UtilManager'; import { SearchManager, SolrManager } from './ApiManagers/SearchManager'; @@ -18,18 +18,20 @@ import { GoogleCredentialsLoader } from './credentials/CredentialsLoader'; import DeleteManager from "./ApiManagers/DeleteManager"; import PDFManager from "./ApiManagers/PDFManager"; import UploadManager from "./ApiManagers/UploadManager"; -import { log_execution } from "./ActionUtilities"; +import { log_execution, Email } from "./ActionUtilities"; import GeneralGoogleManager from "./ApiManagers/GeneralGoogleManager"; import GooglePhotosManager from "./ApiManagers/GooglePhotosManager"; -import { yellow, red } from "colors"; -import { disconnect } from "../server/Initialization"; -import { ProcessFactory, Logger } from "./ProcessFactory"; +import { Logger } from "./ProcessFactory"; +import { yellow } from "colors"; +import { Session } from "./Session/session"; +import { isMaster } from "cluster"; +import { execSync } from "child_process"; +import { Utils } from "../Utils"; +import { MessageStore } from "./Message"; export const publicDirectory = path.resolve(__dirname, "public"); export const filesDirectory = path.resolve(publicDirectory, "files"); -export const ExitHandlers = new Array<() => void>(); - /** * These are the functions run before the server starts * listening. Anything that must be complete @@ -79,29 +81,29 @@ function routeSetter({ isRelease, addSupervisedRoute, logRegistrationOutcome }: addSupervisedRoute({ method: Method.GET, subscription: "/", - onValidation: ({ res }) => res.redirect("/home") + secureHandler: ({ res }) => res.redirect("/home") }); addSupervisedRoute({ method: Method.GET, subscription: "/serverHeartbeat", - onValidation: ({ res }) => res.send(true) + secureHandler: ({ res }) => res.send(true) }); addSupervisedRoute({ method: Method.GET, - subscription: "/shutdown", - onValidation: async ({ res }) => { - WebSocket.disconnect(); - await disconnect(); - await Database.disconnect(); - SolrManager.SetRunning(false); - res.send("Server successfully shut down."); - process.exit(0); + subscription: new RouteSubscriber("kill").add("key"), + secureHandler: ({ req, res }) => { + if (req.params.key === process.env.session_key) { + res.send("<img src='https://media.giphy.com/media/NGIfqtcS81qi4/giphy.gif' style='width:100%;height:100%;'/>"); + process.send!({ action: { message: "kill" } }); + } else { + res.redirect("/home"); + } } }); - const serve: OnUnauthenticated = ({ req, res }) => { + const serve: PublicHandler = ({ req, res }) => { const detector = new mobileDetect(req.headers['user-agent'] || ""); const filename = detector.mobile() !== null ? 'mobile/image.html' : 'index.html'; res.sendFile(path.join(__dirname, '../../deploy/' + filename)); @@ -110,8 +112,8 @@ function routeSetter({ isRelease, addSupervisedRoute, logRegistrationOutcome }: addSupervisedRoute({ method: Method.GET, subscription: ["/home", new RouteSubscriber("doc").add("docId")], - onValidation: serve, - onUnauthenticated: ({ req, ...remaining }) => { + secureHandler: serve, + publicHandler: ({ req, ...remaining }) => { const { originalUrl: target } = req; const sharing = qs.parse(qs.extract(req.originalUrl), { sort: false }).sharing === "true"; const docAccess = target.startsWith("/doc/"); @@ -125,14 +127,70 @@ function routeSetter({ isRelease, addSupervisedRoute, logRegistrationOutcome }: // initialize the web socket (bidirectional communication: if a user changes // a field on one client, that change must be broadcast to all other clients) - WebSocket.initialize(serverPort, isRelease); + WebSocket.start(isRelease); } -(async function start() { +/** + * This function can be used in two different ways. If not in release mode, + * this is simply the logic that is invoked to start the server. In release mode, + * however, this becomes the logic invoked by a single worker thread spawned by + * the main monitor (master) thread. + */ +async function launchServer() { await log_execution({ startMessage: "\nstarting execution of preliminary functions", endMessage: "completed preliminary functions\n", action: preliminaryFunctions }); - await initializeServer({ serverPort: 1050, routeSetter }); -})(); + await initializeServer(routeSetter); +} + +/** + * If we're the monitor (master) thread, we should launch the monitor logic for the session. + * Otherwise, we must be on a worker thread that was spawned *by* the monitor (master) thread, and thus + * our job should be to run the server. + */ +async function launchMonitoredSession() { + if (isMaster) { + const recipients = ["samuel_wilkins@brown.edu"]; + const signature = "-Dash Server Session Manager"; + const customizer = await Session.initializeMonitorThread({ + key: async (key: string) => { + const content = `The key for this session (started @ ${new Date().toUTCString()}) is ${key}.\n\n${signature}`; + const failures = await Email.dispatchAll(recipients, "Server Termination Key", content); + return failures.length === 0; + }, + crash: async (error: Error) => { + const subject = "Dash Web Server Crash"; + const { name, message, stack } = error; + const body = [ + "You, as a Dash Administrator, are being notified of a server crash event. Here's what we know:", + `name:\n${name}`, + `message:\n${message}`, + `stack:\n${stack}`, + "The server is already restarting itself, but if you're concerned, use the Remote Desktop Connection to monitor progress.", + ].join("\n\n"); + const content = `${body}\n\n${signature}`; + const failures = await Email.dispatchAll(recipients, subject, content); + return failures.length === 0; + } + }); + customizer.addReplCommand("pull", [], () => execSync("git pull", { stdio: ["ignore", "inherit", "inherit"] })); + customizer.addReplCommand("solr", [/start|stop/g], args => SolrManager.SetRunning(args[0] === "start")); + } else { + const addExitHandler = await Session.initializeWorkerThread(launchServer); // server initialization delegated to worker + addExitHandler(() => Utils.Emit(WebSocket._socket, MessageStore.ConnectionTerminated, "Manual")); + } +} + +/** + * If you're in development mode, you won't need to run a session. + * The session spawns off new server processes each time an error is encountered, and doesn't + * log the output of the server process, so it's not ideal for development. + * So, the 'else' clause is exactly what we've always run when executing npm start. + */ +if (process.env.RELEASE) { + launchMonitoredSession(); +} else { + launchServer(); +}
\ No newline at end of file diff --git a/src/server/session_manager/input_manager.ts b/src/server/repl.ts index a95e6baae..bd00e48cd 100644 --- a/src/server/session_manager/input_manager.ts +++ b/src/server/repl.ts @@ -1,33 +1,39 @@ import { createInterface, Interface } from "readline"; -import { red } from "colors"; +import { red, green, white } from "colors"; export interface Configuration { - identifier: string; + identifier: () => string | string; onInvalid?: (culprit?: string) => string | string; + onValid?: (success?: string) => string | string; isCaseSensitive?: boolean; } +export type ReplAction = (parsedArgs: Array<string>) => any | Promise<any>; export interface Registration { - argPattern: RegExp[]; - action: (parsedArgs: IterableIterator<string>) => any | Promise<any>; + argPatterns: RegExp[]; + action: ReplAction; } -export default class InputManager { - private identifier: string; - private onInvalid: ((culprit?: string) => string) | string; +export default class Repl { + private identifier: () => string | string; + private onInvalid: (culprit?: string) => string | string; + private onValid: (success: string) => string | string; private isCaseSensitive: boolean; private commandMap = new Map<string, Registration[]>(); public interface: Interface; private busy = false; private keys: string | undefined; - constructor({ identifier: prompt, onInvalid, isCaseSensitive }: Configuration) { + constructor({ identifier: prompt, onInvalid, onValid, isCaseSensitive }: Configuration) { this.identifier = prompt; this.onInvalid = onInvalid || this.usage; + this.onValid = onValid || this.success; this.isCaseSensitive = isCaseSensitive ?? true; this.interface = createInterface(process.stdin, process.stdout).on('line', this.considerInput); } + private resolvedIdentifier = () => typeof this.identifier === "string" ? this.identifier : this.identifier(); + private usage = () => { const resolved = this.keys; if (resolved) { @@ -39,12 +45,15 @@ export default class InputManager { while (!(next = keys.next()).done) { members.push(next.value); } - return `${this.identifier} commands: { ${members.sort().join(", ")} }`; + return `${this.resolvedIdentifier()} commands: { ${members.sort().join(", ")} }`; } - public registerCommand = (basename: string, argPattern: RegExp[], action: any | Promise<any>) => { + private success = (command: string) => `${this.resolvedIdentifier()} completed execution of ${white(command)}`; + + public registerCommand = (basename: string, argPatterns: (RegExp | string)[], action: ReplAction) => { const existing = this.commandMap.get(basename); - const registration = { argPattern, action }; + const converted = argPatterns.map(input => input instanceof RegExp ? input : new RegExp(input)); + const registration = { argPatterns: converted, action }; if (existing) { existing.push(registration); } else { @@ -57,6 +66,11 @@ export default class InputManager { this.busy = false; } + private valid = (command: string) => { + console.log(green(typeof this.onValid === "string" ? this.onValid : this.onValid(command))); + this.busy = false; + } + private considerInput = async (line: string) => { if (this.busy) { console.log(red("Busy")); @@ -74,14 +88,14 @@ export default class InputManager { const registered = this.commandMap.get(command); if (registered) { const { length } = args; - const candidates = registered.filter(({ argPattern: { length: count } }) => count === length); - for (const { argPattern, action } of candidates) { + const candidates = registered.filter(({ argPatterns: { length: count } }) => count === length); + for (const { argPatterns, action } of candidates) { const parsed: string[] = []; let matched = false; if (length) { for (let i = 0; i < length; i++) { let matches: RegExpExecArray | null; - if ((matches = argPattern[i].exec(args[i])) === null) { + if ((matches = argPatterns[i].exec(args[i])) === null) { break; } parsed.push(matches[0]); @@ -89,8 +103,8 @@ export default class InputManager { matched = true; } if (!length || matched) { - await action(parsed[Symbol.iterator]()); - this.busy = false; + await action(parsed); + this.valid(`${command} ${parsed.join(" ")}`); return; } } diff --git a/src/server/Initialization.ts b/src/server/server_Initialization.ts index b58bc3e70..4cb1fca47 100644 --- a/src/server/Initialization.ts +++ b/src/server/server_Initialization.ts @@ -18,7 +18,7 @@ import * as whm from 'webpack-hot-middleware'; import * as fs from 'fs'; import * as request from 'request'; import RouteSubscriber from './RouteSubscriber'; -import { publicDirectory, ExitHandlers } from '.'; +import { publicDirectory } from '.'; import { logPort, } from './ActionUtilities'; import { timeMap } from './ApiManagers/UserManager'; import { blue, yellow } from 'colors'; @@ -26,15 +26,9 @@ import { blue, yellow } from 'colors'; /* RouteSetter is a wrapper around the server that prevents the server from being exposed. */ export type RouteSetter = (server: RouteManager) => void; -export interface InitializationOptions { - serverPort: number; - routeSetter: RouteSetter; -} - export let disconnect: Function; -export default async function InitializeServer(options: InitializationOptions) { - const { serverPort, routeSetter } = options; +export default async function InitializeServer(routeSetter: RouteSetter) { const app = buildWithMiddleware(express()); app.use(express.static(publicDirectory)); @@ -63,8 +57,9 @@ export default async function InitializeServer(options: InitializationOptions) { routeSetter(new RouteManager(app, isRelease)); + const serverPort = isRelease ? Number(process.env.serverPort) : 1050; const server = app.listen(serverPort, () => { - logPort("server", serverPort); + logPort("server", Number(serverPort)); console.log(); }); disconnect = async () => new Promise<Error>(resolve => server.close(resolve)); diff --git a/src/server/session_manager/config.ts b/src/server/session_manager/config.ts deleted file mode 100644 index ebbd999c6..000000000 --- a/src/server/session_manager/config.ts +++ /dev/null @@ -1,33 +0,0 @@ -import { resolve } from 'path'; -import { yellow } from "colors"; - -export const latency = 10; -export const ports = [1050, 4321]; -export const onWindows = process.platform === "win32"; -export const heartbeat = `http://localhost:1050/serverHeartbeat`; -export const recipient = "samuel_wilkins@brown.edu"; -export const { pid, platform } = process; - -/** - * Logging - */ -export const identifier = yellow("__session_manager__:"); - -/** - * Paths - */ -export const logPath = resolve(__dirname, "./logs"); -export const crashPath = resolve(logPath, "./crashes"); - -/** - * State - */ -export enum SessionState { - STARTING = "STARTING", - INITIALIZED = "INITIALIZED", - LISTENING = "LISTENING", - AUTOMATICALLY_RESTARTING = "CRASH_RESTARTING", - MANUALLY_RESTARTING = "MANUALLY_RESTARTING", - EXITING = "EXITING", - UPDATING = "UPDATING" -}
\ No newline at end of file diff --git a/src/server/session_manager/logs/current_daemon_pid.log b/src/server/session_manager/logs/current_daemon_pid.log deleted file mode 100644 index 557e3d7c3..000000000 --- a/src/server/session_manager/logs/current_daemon_pid.log +++ /dev/null @@ -1 +0,0 @@ -26860 diff --git a/src/server/session_manager/logs/current_server_pid.log b/src/server/session_manager/logs/current_server_pid.log deleted file mode 100644 index 85fdb7ae0..000000000 --- a/src/server/session_manager/logs/current_server_pid.log +++ /dev/null @@ -1 +0,0 @@ -54649 created @ 2019-12-14T08:04:42.391Z diff --git a/src/server/session_manager/logs/current_session_manager_pid.log b/src/server/session_manager/logs/current_session_manager_pid.log deleted file mode 100644 index 75c23b35a..000000000 --- a/src/server/session_manager/logs/current_session_manager_pid.log +++ /dev/null @@ -1 +0,0 @@ -54643 diff --git a/src/server/session_manager/session_manager.ts b/src/server/session_manager/session_manager.ts deleted file mode 100644 index 6084d9a77..000000000 --- a/src/server/session_manager/session_manager.ts +++ /dev/null @@ -1,201 +0,0 @@ -import * as request from "request-promise"; -import { log_execution, pathFromRoot } from "../ActionUtilities"; -import { red, yellow, cyan, green, Color } from "colors"; -import * as nodemailer from "nodemailer"; -import { MailOptions } from "nodemailer/lib/json-transport"; -import { writeFileSync, existsSync, mkdirSync } from "fs"; -import { resolve } from 'path'; -import { ChildProcess, exec, execSync } from "child_process"; -import InputManager from "./input_manager"; -import { identifier, logPath, crashPath, onWindows, pid, ports, heartbeat, recipient, latency, SessionState } from "./config"; -const killport = require("kill-port"); - -process.on('SIGINT', endPrevious); -let state: SessionState = SessionState.STARTING; -const is = (...reference: SessionState[]) => reference.includes(state); -const set = (reference: SessionState) => state = reference; - - - -const { registerCommand } = new InputManager({ identifier }); - -registerCommand("restart", [], async () => { - set(SessionState.MANUALLY_RESTARTING); - identifiedLog(cyan("Initializing manual restart...")); - await endPrevious(); -}); - -registerCommand("exit", [], exit); - -async function exit() { - set(SessionState.EXITING); - identifiedLog(cyan("Initializing session end")); - await endPrevious(); - identifiedLog("Cleanup complete. Exiting session...\n"); - execSync(killAllCommand()); -} - -registerCommand("update", [], async () => { - set(SessionState.UPDATING); - identifiedLog(cyan("Initializing server update from version control...")); - await endPrevious(); - await new Promise<void>(resolve => { - exec(updateCommand(), error => { - if (error) { - identifiedLog(red(error.message)); - } - resolve(); - }); - }); - await exit(); -}); - -registerCommand("state", [], () => identifiedLog(state)); - -if (!existsSync(logPath)) { - mkdirSync(logPath); -} -if (!existsSync(crashPath)) { - mkdirSync(crashPath); -} - -function addLogEntry(message: string, color: Color) { - const formatted = color(`${message} ${timestamp()}.`); - identifiedLog(formatted); - // appendFileSync(resolve(crashPath, `./session_crashes_${new Date().toISOString()}.log`), `${formatted}\n`); -} - -function identifiedLog(message?: any, ...optionalParams: any[]) { - console.log(identifier, message, ...optionalParams); -} - -if (!["win32", "darwin"].includes(process.platform)) { - identifiedLog(red("Invalid operating system: this script is supported only on Mac and Windows.")); - process.exit(1); -} - -const windowsPrepend = (command: string) => `"C:\\Program Files\\Git\\git-bash.exe" -c "${command}"`; -const macPrepend = (command: string) => `osascript -e 'tell app "Terminal"\ndo script "cd ${pathFromRoot()} && ${command}"\nend tell'`; - -function updateCommand() { - const command = "git pull && npm install"; - if (onWindows) { - return windowsPrepend(command); - } - return macPrepend(command); -} - -function startServerCommand() { - const command = "npm run start-release"; - if (onWindows) { - return windowsPrepend(command); - } - return macPrepend(command); -} - -function killAllCommand() { - if (onWindows) { - return "taskkill /f /im node.exe"; - } - return "killall -9 node"; -} - -identifiedLog("Initializing session..."); - -writeLocalPidLog("session_manager", pid); - -function writeLocalPidLog(filename: string, contents: any) { - const path = `./logs/current_${filename}_pid.log`; - identifiedLog(cyan(`${contents} written to ${path}`)); - writeFileSync(resolve(__dirname, path), `${contents}\n`); -} - -function timestamp() { - return `@ ${new Date().toISOString()}`; -} - -async function endPrevious() { - identifiedLog(yellow("Cleaning up previous connections...")); - current_backup?.kill("SIGKILL"); - await Promise.all(ports.map(port => { - const task = killport(port, 'tcp'); - return task.catch((error: any) => identifiedLog(red(error))); - })); - identifiedLog(yellow("Done. Any failures will be printed in red immediately above.")); -} - -let current_backup: ChildProcess | undefined = undefined; - -async function checkHeartbeat() { - const listening = is(SessionState.LISTENING); - let error: any; - try { - listening && process.stdout.write(`${identifier} 👂 `); - await request.get(heartbeat); - listening && console.log('⇠💚'); - if (!listening) { - addLogEntry(is(SessionState.INITIALIZED) ? "Server successfully started" : "Backup server successfully restarted", green); - set(SessionState.LISTENING); - } - } catch (e) { - listening && console.log("⇠💔\n"); - error = e; - } finally { - if (error && !is(SessionState.AUTOMATICALLY_RESTARTING, SessionState.INITIALIZED, SessionState.UPDATING)) { - if (is(SessionState.STARTING)) { - set(SessionState.INITIALIZED); - } else if (is(SessionState.MANUALLY_RESTARTING)) { - set(SessionState.AUTOMATICALLY_RESTARTING); - } else { - set(SessionState.AUTOMATICALLY_RESTARTING); - addLogEntry("Detected a server crash", red); - identifiedLog(red(error.message)); - await endPrevious(); - await log_execution({ - startMessage: identifier + " Sending crash notification email", - endMessage: ({ error, result }) => { - const success = error === null && result === true; - return identifier + ` ${(success ? `Notification successfully sent to` : `Failed to notify`)} ${recipient} ${timestamp()}`; - }, - action: async () => notify(error || "Hmm, no error to report..."), - color: cyan - }); - identifiedLog(green("Initiating server restart...")); - } - current_backup = exec(startServerCommand(), err => identifiedLog(err?.message || is(SessionState.INITIALIZED) ? "Spawned initial server." : "Previous server process exited.")); - writeLocalPidLog("server", `${(current_backup?.pid ?? -2) + 1} created ${timestamp()}`); - } - setTimeout(checkHeartbeat, 1000 * latency); - } -} - -function emailText(error: any) { - return [ - `Hey ${recipient.split("@")[0]},`, - "You, as a Dash Administrator, are being notified of a server crash event. Here's what we know:", - `Location: ${heartbeat}\nError: ${error}`, - "The server should already be restarting itself, but if you're concerned, use the Remote Desktop Connection to monitor progress." - ].join("\n\n"); -} - -async function notify(error: any) { - const smtpTransport = nodemailer.createTransport({ - service: 'Gmail', - auth: { - user: 'brownptcdash@gmail.com', - pass: 'browngfx1' - } - }); - const mailOptions = { - to: recipient, - from: 'brownptcdash@gmail.com', - subject: 'Dash Server Crash', - text: emailText(error) - } as MailOptions; - return new Promise<boolean>(resolve => { - smtpTransport.sendMail(mailOptions, (dispatchError: Error | null) => resolve(dispatchError === null)); - }); -} - -identifiedLog(yellow(`After initialization, will poll server heartbeat repeatedly...\n`)); -checkHeartbeat();
\ No newline at end of file |