diff options
Diffstat (limited to 'src/server')
| -rw-r--r-- | src/server/ActionUtilities.ts | 23 | ||||
| -rw-r--r-- | src/server/ApiManagers/SearchManager.ts | 26 | ||||
| -rw-r--r-- | src/server/DashSession.ts | 73 | ||||
| -rw-r--r-- | src/server/Session/session.ts | 823 | ||||
| -rw-r--r-- | src/server/Session/session_config_schema.ts | 68 | ||||
| -rw-r--r-- | src/server/index.ts | 58 | ||||
| -rw-r--r-- | src/server/repl.ts | 57 | ||||
| -rw-r--r-- | src/server/server_Initialization.ts | 8 |
8 files changed, 784 insertions, 352 deletions
diff --git a/src/server/ActionUtilities.ts b/src/server/ActionUtilities.ts index 30aed32e6..a93566fb1 100644 --- a/src/server/ActionUtilities.ts +++ b/src/server/ActionUtilities.ts @@ -118,25 +118,34 @@ export namespace Email { } }); + export interface DispatchFailure { + recipient: string; + error: Error; + } + export async function dispatchAll(recipients: string[], subject: string, content: string) { - const failures: string[] = []; + const failures: DispatchFailure[] = []; await Promise.all(recipients.map(async (recipient: string) => { - if (!await Email.dispatch(recipient, subject, content)) { - failures.push(recipient); + let error: Error | null; + if ((error = await Email.dispatch(recipient, subject, content)) !== null) { + failures.push({ + recipient, + error + }); } })); - return failures; + return failures.length ? failures : undefined; } - export async function dispatch(recipient: string, subject: string, content: string): Promise<boolean> { + export async function dispatch(recipient: string, subject: string, content: string): Promise<Error | null> { const mailOptions = { to: recipient, from: 'brownptcdash@gmail.com', subject, text: `Hello ${recipient.split("@")[0]},\n\n${content}` } as MailOptions; - return new Promise<boolean>(resolve => { - smtpTransport.sendMail(mailOptions, (dispatchError: Error | null) => resolve(dispatchError === null)); + return new Promise<Error | null>(resolve => { + smtpTransport.sendMail(mailOptions, resolve); }); } diff --git a/src/server/ApiManagers/SearchManager.ts b/src/server/ApiManagers/SearchManager.ts index c1c908088..4ce12f9f3 100644 --- a/src/server/ApiManagers/SearchManager.ts +++ b/src/server/ApiManagers/SearchManager.ts @@ -4,11 +4,11 @@ import { Search } from "../Search"; const findInFiles = require('find-in-files'); import * as path from 'path'; import { pathToDirectory, Directory } from "./UploadManager"; -import { command_line } from "../ActionUtilities"; -import request = require('request-promise'); -import { red } from "colors"; +import { red, cyan, yellow } from "colors"; import RouteSubscriber from "../RouteSubscriber"; -import { execSync } from "child_process"; +import { exec } from "child_process"; +import { onWindows } from ".."; +import { get } from "request-promise"; export class SearchManager extends ApiManager { @@ -69,15 +69,23 @@ export class SearchManager extends ApiManager { export namespace SolrManager { + const command = onWindows ? "solr.cmd" : "solr"; + export async function SetRunning(status: boolean): Promise<boolean> { const args = status ? "start" : "stop -p 8983"; + console.log(`solr management: trying to ${args}`); + exec(`${command} ${args}`, { cwd: "./solr-8.3.1/bin" }, (error, stdout, stderr) => { + if (error) { + console.log(red(`solr management error: unable to ${args} server`)); + console.log(red(error.message)); + } + console.log(cyan(stdout)); + console.log(yellow(stderr)); + }); try { - console.log(`Solr management: trying to ${args}`); - console.log(execSync(`./solr.cmd ${args}`, { cwd: "./solr-8.3.1/bin" })); + await get("http://localhost:8983"); return true; - } catch (e) { - console.log(red(`Solr management error: unable to ${args}`)); - console.log(e); + } catch { return false; } } diff --git a/src/server/DashSession.ts b/src/server/DashSession.ts new file mode 100644 index 000000000..a0e00adda --- /dev/null +++ b/src/server/DashSession.ts @@ -0,0 +1,73 @@ +import { Session } from "./Session/session"; +import { Email } from "./ActionUtilities"; +import { red, yellow, green } from "colors"; +import { get } from "request-promise"; +import { Utils } from "../Utils"; +import { WebSocket } from "./Websocket/Websocket"; +import { MessageStore } from "./Message"; +import { launchServer, onWindows } from "."; + +/** +* If we're the monitor (master) thread, we should launch the monitor logic for the session. +* Otherwise, we must be on a worker thread that was spawned *by* the monitor (master) thread, and thus +* our job should be to run the server. +*/ +export class DashSessionAgent extends Session.AppliedSessionAgent { + + private readonly notificationRecipients = ["samuel_wilkins@brown.edu"]; + private readonly signature = "-Dash Server Session Manager"; + + protected async launchMonitor() { + const monitor = Session.Monitor.Create({ + key: async key => { + // this sends a pseudorandomly generated guid to the configuration's recipients, allowing them alone + // to kill the server via the /kill/:key route + const content = `The key for this session (started @ ${new Date().toUTCString()}) is ${key}.\n\n${this.signature}`; + const failures = await Email.dispatchAll(this.notificationRecipients, "Server Termination Key", content); + if (failures) { + failures.map(({ recipient, error: { message } }) => monitor.mainLog(red(`dispatch failure @ ${recipient} (${yellow(message)})`))); + return false; + } + return true; + }, + crash: async ({ name, message, stack }) => { + const body = [ + "You, as a Dash Administrator, are being notified of a server crash event. Here's what we know:", + `name:\n${name}`, + `message:\n${message}`, + `stack:\n${stack}`, + "The server is already restarting itself, but if you're concerned, use the Remote Desktop Connection to monitor progress.", + ].join("\n\n"); + const content = `${body}\n\n${this.signature}`; + const failures = await Email.dispatchAll(this.notificationRecipients, "Dash Web Server Crash", content); + if (failures) { + failures.map(({ recipient, error: { message } }) => monitor.mainLog(red(`dispatch failure @ ${recipient} (${yellow(message)})`))); + return false; + } + return true; + } + }); + monitor.addReplCommand("pull", [], () => monitor.exec("git pull")); + monitor.addReplCommand("solr", [/start|stop/], async args => { + const command = `${onWindows ? "solr.cmd" : "solr"} ${args[0] === "start" ? "start" : "stop -p 8983"}`; + await monitor.exec(command, { cwd: "./solr-8.3.1/bin" }); + try { + await get("http://localhost:8983"); + monitor.mainLog(green("successfully connected to 8983 after running solr initialization")); + } catch { + monitor.mainLog(red("unable to connect at 8983 after running solr initialization")); + } + }); + return monitor; + } + + protected async launchServerWorker() { + const worker = Session.ServerWorker.Create(launchServer); // server initialization delegated to worker + worker.addExitHandler(() => { + const { _socket } = WebSocket; + _socket && Utils.Emit(_socket, MessageStore.ConnectionTerminated, "Manual"); + }); + return worker; + } + +}
\ No newline at end of file diff --git a/src/server/Session/session.ts b/src/server/Session/session.ts index cf2231b1f..6967ece52 100644 --- a/src/server/Session/session.ts +++ b/src/server/Session/session.ts @@ -1,338 +1,675 @@ -import { red, cyan, green, yellow, magenta, blue } from "colors"; -import { on, fork, setupMaster, Worker } from "cluster"; -import { execSync } from "child_process"; +import { red, cyan, green, yellow, magenta, blue, white, Color, grey, gray, black } from "colors"; +import { on, fork, setupMaster, Worker, isMaster, isWorker } from "cluster"; import { get } from "request-promise"; import { Utils } from "../../Utils"; import Repl, { ReplAction } from "../repl"; import { readFileSync } from "fs"; import { validate, ValidationError } from "jsonschema"; import { configurationSchema } from "./session_config_schema"; - -const onWindows = process.platform === "win32"; +import { exec, ExecOptions } from "child_process"; /** - * This namespace relies on NodeJS's cluster module, which allows a parent (master) process to share - * code with its children (workers). A simple `isMaster` flag indicates who is trying to access - * the code, and thus determines the functionality that actually gets invoked (checked by the caller, not internally). - * - * Think of the master thread as a factory, and the workers as the helpers that actually run the server. - * - * So, when we run `npm start`, given the appropriate check, initializeMaster() is called in the parent process - * This will spawn off its own child process (by default, mirrors the execution path of its parent), - * in which initializeWorker() is invoked. - */ + * This namespace relies on NodeJS's cluster module, which allows a parent (master) process to share + * code with its children (workers). A simple `isMaster` flag indicates who is trying to access + * the code, and thus determines the functionality that actually gets invoked (checked by the caller, not internally). + * + * Think of the master thread as a factory, and the workers as the helpers that actually run the server. + * + * So, when we run `npm start`, given the appropriate check, initializeMaster() is called in the parent process + * This will spawn off its own child process (by default, mirrors the execution path of its parent), + * in which initializeWorker() is invoked. + */ export namespace Session { + type ColorLabel = "yellow" | "red" | "cyan" | "green" | "blue" | "magenta" | "grey" | "gray" | "white" | "black"; + const colorMapping: Map<ColorLabel, Color> = new Map([ + ["yellow", yellow], + ["red", red], + ["cyan", cyan], + ["green", green], + ["blue", blue], + ["magenta", magenta], + ["grey", grey], + ["gray", gray], + ["white", white], + ["black", black] + ]); + + export abstract class AppliedSessionAgent { + + // the following two methods allow the developer to create a custom + // session and use the built in customization options for each thread + protected abstract async launchMonitor(): Promise<Session.Monitor>; + protected abstract async launchServerWorker(): Promise<Session.ServerWorker>; + + private launched = false; + + public killSession = (reason: string, graceful = true, errorCode = 0) => { + const target = isMaster ? this.sessionMonitor : this.serverWorker; + target.killSession(reason, graceful, errorCode); + } + + private sessionMonitorRef: Session.Monitor | undefined; + public get sessionMonitor(): Session.Monitor { + if (!isMaster) { + this.serverWorker.sendMonitorAction("kill", { + graceful: false, + reason: "Cannot access the session monitor directly from the server worker thread.", + errorCode: 1 + }); + throw new Error(); + } + return this.sessionMonitorRef!; + } + + private serverWorkerRef: Session.ServerWorker | undefined; + public get serverWorker(): Session.ServerWorker { + if (isMaster) { + throw new Error("Cannot access the server worker directly from the session monitor thread"); + } + return this.serverWorkerRef!; + } + + public async launch(): Promise<void> { + if (!this.launched) { + this.launched = true; + if (isMaster) { + this.sessionMonitorRef = await this.launchMonitor(); + } else { + this.serverWorkerRef = await this.launchServerWorker(); + } + } else { + throw new Error("Cannot launch a session thread more than once per process."); + } + } + + } + + interface Identifier { + text: string; + color: ColorLabel; + } + + interface Identifiers { + master: Identifier; + worker: Identifier; + exec: Identifier; + } + interface Configuration { showServerOutput: boolean; - masterIdentifier: string; - workerIdentifier: string; + identifiers: Identifiers; ports: { [description: string]: number }; - pollingRoute: string; - pollingIntervalSeconds: number; - [key: string]: any; + polling: { + route: string; + intervalSeconds: number; + failureTolerance: number; + }; } - const defaultConfiguration: Configuration = { + const defaultConfig: Configuration = { showServerOutput: false, - masterIdentifier: yellow("__monitor__:"), - workerIdentifier: magenta("__server__:"), + identifiers: { + master: { + text: "__monitor__", + color: "yellow" + }, + worker: { + text: "__server__", + color: "magenta" + }, + exec: { + text: "__exec__", + color: "green" + } + }, ports: { server: 3000 }, - pollingRoute: "/", - pollingIntervalSeconds: 30 + polling: { + route: "/", + intervalSeconds: 30, + failureTolerance: 0 + } }; - interface MasterExtensions { - addReplCommand: (basename: string, argPatterns: (RegExp | string)[], action: ReplAction) => void; - addChildMessageHandler: (message: string, handler: ActionHandler) => void; - } + export type ExitHandler = (reason: Error | null) => void | Promise<void>; - export interface NotifierHooks { - key?: (key: string) => boolean | Promise<boolean>; - crash?: (error: Error) => boolean | Promise<boolean>; - } + export namespace Monitor { - export interface SessionAction { - message: string; - args: any; - } + export interface NotifierHooks { + key?: (key: string) => (boolean | Promise<boolean>); + crash?: (error: Error) => (boolean | Promise<boolean>); + } + + export interface Action { + message: string; + args: any; + } + + export type ServerMessageHandler = (action: Action) => void | Promise<void>; - export type ExitHandler = (error: Error) => void | Promise<void>; - export type ActionHandler = (action: SessionAction) => void | Promise<void>; - export interface EmailTemplate { - subject: string; - body: string; } - function loadAndValidateConfiguration(): any { - try { - const configuration: Configuration = JSON.parse(readFileSync('./session.config.json', 'utf8')); - const options = { - throwError: true, - allowUnknownAttributes: false - }; - // ensure all necessary and no excess information is specified by the configuration file - validate(configuration, configurationSchema, options); - let formatMaster = true; - let formatWorker = true; - Object.keys(defaultConfiguration).forEach(property => { - if (!configuration[property]) { - if (property === "masterIdentifier") { - formatMaster = false; - } else if (property === "workerIdentifier") { - formatWorker = false; + /** + * Validates and reads the configuration file, accordingly builds a child process factory + * and spawns off an initial process that will respawn as predecessors die. + */ + export class Monitor { + + private static count = 0; + private exitHandlers: ExitHandler[] = []; + private readonly notifiers: Monitor.NotifierHooks | undefined; + private readonly config: Configuration; + private onMessage: { [message: string]: Monitor.ServerMessageHandler[] | undefined } = {}; + private activeWorker: Worker | undefined; + private key: string | undefined; + private repl: Repl; + + public static Create(notifiers?: Monitor.NotifierHooks) { + if (isWorker) { + process.send?.({ + action: { + message: "kill", + args: { + reason: "cannot create a monitor on the worker process.", + graceful: false, + errorCode: 1 + } } - configuration[property] = defaultConfiguration[property]; - } - }); - if (formatMaster) { - configuration.masterIdentifier = yellow(configuration.masterIdentifier + ":"); - } - if (formatWorker) { - configuration.workerIdentifier = magenta(configuration.workerIdentifier + ":"); + }); + process.exit(1); + } else if (++Monitor.count > 1) { + console.error(red("cannot create more than one monitor.")); + process.exit(1); + } else { + return new Monitor(notifiers); } - return configuration; - } catch (error) { - if (error instanceof ValidationError) { - console.log(red("\nSession configuration failed.")); - console.log("The given session.config.json configuration file is invalid."); - console.log(`${error.instance}: ${error.stack}`); - process.exit(0); - } else if (error.code === "ENOENT" && error.path === "./session.config.json") { - console.log(cyan("Loading default session parameters...")); - console.log("Consider including a session.config.json configuration file in your project root for customization."); - return defaultConfiguration; + } + + /** + * Kill this session and its active child + * server process, either gracefully (may wait + * indefinitely, but at least allows active networking + * requests to complete) or immediately. + */ + public killSession = async (reason: string, graceful = true, errorCode = 0) => { + this.mainLog(cyan(`exiting session ${graceful ? "clean" : "immediate"}ly`)); + this.mainLog(`reason: ${(red(reason))}`); + await this.executeExitHandlers(null); + this.killActiveWorker(graceful); + process.exit(errorCode); + } + + /** + * Execute the list of functions registered to be called + * whenever the process exits. + */ + public addExitHandler = (handler: ExitHandler) => this.exitHandlers.push(handler); + + /** + * Extend the default repl by adding in custom commands + * that can invoke application logic external to this module + */ + public addReplCommand = (basename: string, argPatterns: (RegExp | string)[], action: ReplAction) => { + this.repl.registerCommand(basename, argPatterns, action); + } + + public exec = (command: string, options?: ExecOptions) => { + return new Promise<void>(resolve => { + exec(command, { ...options, encoding: "utf8" }, (error, stdout, stderr) => { + if (error) { + this.execLog(red(`unable to execute ${white(command)}`)); + error.message.split("\n").forEach(line => line.length && this.execLog(red(`(error) ${line}`))); + } else { + let outLines: string[], errorLines: string[]; + if ((outLines = stdout.split("\n").filter(line => line.length)).length) { + outLines.forEach(line => line.length && this.execLog(cyan(`(stdout) ${line}`))); + } + if ((errorLines = stderr.split("\n").filter(line => line.length)).length) { + errorLines.forEach(line => line.length && this.execLog(yellow(`(stderr) ${line}`))); + } + } + resolve(); + }); + }); + } + + /** + * Add a listener at this message. When the monitor process + * receives a message, it will invoke all registered functions. + */ + public addServerMessageListener = (message: string, handler: Monitor.ServerMessageHandler) => { + const handlers = this.onMessage[message]; + if (handlers) { + handlers.push(handler); } else { - console.log(red("\nSession configuration failed.")); - console.log("The following unknown error occurred during configuration."); - console.log(error.stack); - process.exit(0); + this.onMessage[message] = [handler]; } } - } - function timestamp() { - return blue(`[${new Date().toUTCString()}]`); - } + /** + * Unregister a given listener at this message. + */ + public removeServerMessageListener = (message: string, handler: Monitor.ServerMessageHandler) => { + const handlers = this.onMessage[message]; + if (handlers) { + const index = handlers.indexOf(handler); + if (index > -1) { + handlers.splice(index, 1); + } + } + } - /** - * Validates and reads the configuration file, accordingly builds a child process factory - * and spawns off an initial process that will respawn as predecessors die. - */ - export async function initializeMonitorThread(notifiers?: NotifierHooks): Promise<MasterExtensions> { - let activeWorker: Worker; - const childMessageHandlers: { [message: string]: (action: SessionAction, args: any) => void } = {}; - - // read in configuration .json file only once, in the master thread - // pass down any variables the pertinent to the child processes as environment variables - const { - masterIdentifier, - workerIdentifier, - ports, - pollingRoute, - showServerOutput, - pollingIntervalSeconds - } = loadAndValidateConfiguration(); - - const masterLog = (...optionalParams: any[]) => console.log(timestamp(), masterIdentifier, ...optionalParams); - - // this sends a pseudorandomly generated guid to the configuration's recipients, allowing them alone - // to kill the server via the /kill/:key route - let key: string | undefined; - if (notifiers && notifiers.key) { - key = Utils.GenerateGuid(); - const success = await notifiers.key(key); - const statement = success ? green("distributed session key to recipients") : red("distribution of session key failed"); - masterLog(statement); + /** + * Unregister all listeners at this message. + */ + public clearServerMessageListeners = (message: string) => this.onMessage[message] = undefined; + + private constructor(notifiers?: Monitor.NotifierHooks) { + this.notifiers = notifiers; + + console.log(this.timestamp(), cyan("initializing session...")); + + this.config = this.loadAndValidateConfiguration(); + + this.initializeSessionKey(); + // determines whether or not we see the compilation / initialization / runtime output of each child server process + const output = this.config.showServerOutput ? "inherit" : "ignore"; + setupMaster({ stdio: ["ignore", output, output, "ipc"] }); + + // handle exceptions in the master thread - there shouldn't be many of these + // the IPC (inter process communication) channel closed exception can't seem + // to be caught in a try catch, and is inconsequential, so it is ignored + process.on("uncaughtException", ({ message, stack }): void => { + if (message !== "Channel closed") { + this.mainLog(red(message)); + if (stack) { + this.mainLog(`uncaught exception\n${red(stack)}`); + } + } + }); + + // a helpful cluster event called on the master thread each time a child process exits + on("exit", ({ process: { pid } }, code, signal) => { + const prompt = `server worker with process id ${pid} has exited with code ${code}${signal === null ? "" : `, having encountered signal ${signal}`}.`; + this.mainLog(cyan(prompt)); + // to make this a robust, continuous session, every time a child process dies, we immediately spawn a new one + this.spawn(); + }); + + this.repl = this.initializeRepl(); + this.spawn(); + } + + /** + * Generates a blue UTC string associated with the time + * of invocation. + */ + private timestamp = () => blue(`[${new Date().toUTCString()}]`); + + /** + * A formatted, identified and timestamped log in color + */ + public mainLog = (...optionalParams: any[]) => { + console.log(this.timestamp(), this.config.identifiers.master.text, ...optionalParams); + } + + /** + * A formatted, identified and timestamped log in color for non- + */ + private execLog = (...optionalParams: any[]) => { + console.log(this.timestamp(), this.config.identifiers.exec.text, ...optionalParams); + } + + /** + * If the caller has indicated an interest + * in being notified of this feature, creates + * a GUID for this session that can, for example, + * be used as authentication for killing the server + * (checked externally). + */ + private initializeSessionKey = async (): Promise<void> => { + if (this.notifiers?.key) { + this.key = Utils.GenerateGuid(); + const success = await this.notifiers.key(this.key); + const statement = success ? green("distributed session key to recipients") : red("distribution of session key failed"); + this.mainLog(statement); + } } - // handle exceptions in the master thread - there shouldn't be many of these - // the IPC (inter process communication) channel closed exception can't seem - // to be caught in a try catch, and is inconsequential, so it is ignored - process.on("uncaughtException", ({ message, stack }) => { - if (message !== "Channel closed") { - masterLog(red(message)); - if (stack) { - masterLog(`uncaught exception\n${red(stack)}`); + /** + * At any arbitrary layer of nesting within the configuration objects, any single value that + * is not specified by the configuration is given the default counterpart. If, within an object, + * one peer is given by configuration and two are not, the one is preserved while the two are given + * the default value. + */ + private assign = (defaultObject: any, specifiedObject: any, collector: any) => { + Array.from(new Set([...Object.keys(defaultObject), ...Object.keys(specifiedObject)])).map(property => { + let defaultValue: any, specifiedValue: any; + if (specifiedValue = specifiedObject[property]) { + if (typeof specifiedValue === "object" && typeof (defaultValue = defaultObject[property]) === "object") { + this.assign(defaultValue, specifiedValue, collector[property] = {}); + } else { + collector[property] = specifiedValue; + } + } else { + collector[property] = defaultObject[property]; + } + }); + } + + /** + * Reads in configuration .json file only once, in the master thread + * and pass down any variables the pertinent to the child processes as environment variables. + */ + private loadAndValidateConfiguration = (): Configuration => { + let config: Configuration; + try { + console.log(this.timestamp(), cyan("validating configuration...")); + config = JSON.parse(readFileSync('./session.config.json', 'utf8')); + const options = { + throwError: true, + allowUnknownAttributes: false + }; + // ensure all necessary and no excess information is specified by the configuration file + validate(config, configurationSchema, options); + const results: any = {}; + this.assign(defaultConfig, config, results); + config = results; + } catch (error) { + if (error instanceof ValidationError) { + console.log(red("\nSession configuration failed.")); + console.log("The given session.config.json configuration file is invalid."); + console.log(`${error.instance}: ${error.stack}`); + process.exit(0); + } else if (error.code === "ENOENT" && error.path === "./session.config.json") { + console.log(cyan("Loading default session parameters...")); + console.log("Consider including a session.config.json configuration file in your project root for customization."); + config = { ...defaultConfig }; + } else { + console.log(red("\nSession configuration failed.")); + console.log("The following unknown error occurred during configuration."); + console.log(error.stack); + process.exit(0); } + } finally { + const { identifiers } = config!; + Object.keys(identifiers).forEach(key => { + const resolved = key as keyof Identifiers; + const { text, color } = identifiers[resolved]; + identifiers[resolved].text = (colorMapping.get(color) || white)(`${text}:`); + }); + return config!; } - }); + } - // determines whether or not we see the compilation / initialization / runtime output of each child server process - setupMaster({ silent: !showServerOutput }); + /** + * Builds the repl that allows the following commands to be typed into stdin of the master thread. + */ + private initializeRepl = (): Repl => { + const repl = new Repl({ identifier: () => `${this.timestamp()} ${this.config.identifiers.master.text}` }); + const boolean = /true|false/; + const number = /\d+/; + const letters = /[a-zA-Z]+/; + repl.registerCommand("exit", [/clean|force/], args => this.killSession("manual exit requested by repl", args[0] === "clean", 0)); + repl.registerCommand("restart", [/clean|force/], args => this.killActiveWorker(args[0] === "clean")); + repl.registerCommand("set", [letters, "port", number, boolean], args => this.setPort(args[0], Number(args[2]), args[3] === "true")); + repl.registerCommand("set", [/polling/, number, boolean], args => { + const newPollingIntervalSeconds = Math.floor(Number(args[2])); + if (newPollingIntervalSeconds < 0) { + this.mainLog(red("the polling interval must be a non-negative integer")); + } else { + if (newPollingIntervalSeconds !== this.config.polling.intervalSeconds) { + this.config.polling.intervalSeconds = newPollingIntervalSeconds; + if (args[3] === "true") { + this.activeWorker?.send({ newPollingIntervalSeconds }); + } + } + } + }); + return repl; + } - // attempts to kills the active worker ungracefully - const tryKillActiveWorker = (graceful = false): boolean => { - if (activeWorker && !activeWorker.isDead()) { + private executeExitHandlers = async (reason: Error | null) => Promise.all(this.exitHandlers.map(handler => handler(reason))); + + /** + * Attempts to kill the active worker gracefully, unless otherwise specified. + */ + private killActiveWorker = (graceful = true): void => { + if (this.activeWorker && !this.activeWorker.isDead()) { if (graceful) { - activeWorker.kill(); + this.activeWorker.send({ manualExit: true }); } else { - activeWorker.process.kill(); + this.activeWorker.process.kill(); } - return true; } - return false; - }; - - const restart = () => { - // indicate to the worker that we are 'expecting' this restart - activeWorker.send({ setResponsiveness: false }); - tryKillActiveWorker(); - }; + } - const setPort = (port: string, value: number, immediateRestart: boolean) => { + /** + * Allows the caller to set the port at which the target (be it the server, + * the websocket, some other custom port) is listening. If an immediate restart + * is specified, this monitor will kill the active child and re-launch the server + * at the port. Otherwise, the updated port won't be used until / unless the child + * dies on its own and triggers a restart. + */ + private setPort = (port: "server" | "socket" | string, value: number, immediateRestart: boolean): void => { if (value > 1023 && value < 65536) { - ports[port] = value; + this.config.ports[port] = value; if (immediateRestart) { - restart(); + this.killActiveWorker(); } } else { - masterLog(red(`${port} is an invalid port number`)); + this.mainLog(red(`${port} is an invalid port number`)); } - }; + } - // kills the current active worker and proceeds to spawn a new worker, - // feeding in configuration information as environment variables - const spawn = (): void => { - tryKillActiveWorker(); - activeWorker = fork({ - pollingRoute, + /** + * Kills the current active worker and proceeds to spawn a new worker, + * feeding in configuration information as environment variables. + */ + private spawn = (): void => { + const { + polling: { + route, + failureTolerance, + intervalSeconds + }, + ports + } = this.config; + this.killActiveWorker(); + this.activeWorker = fork({ + pollingRoute: route, + pollingFailureTolerance: failureTolerance, serverPort: ports.server, socketPort: ports.socket, - pollingIntervalSeconds, - session_key: key + pollingIntervalSeconds: intervalSeconds, + session_key: this.key }); - masterLog(`spawned new server worker with process id ${activeWorker.process.pid}`); + this.mainLog(cyan(`spawned new server worker with process id ${this.activeWorker.process.pid}`)); // an IPC message handler that executes actions on the master thread when prompted by the active worker - activeWorker.on("message", async ({ lifecycle, action }) => { + this.activeWorker.on("message", async ({ lifecycle, action }) => { if (action) { - const { message, args } = action as SessionAction; - console.log(timestamp(), `${workerIdentifier} action requested (${cyan(message)})`); + const { message, args } = action as Monitor.Action; + console.log(this.timestamp(), `${this.config.identifiers.worker.text} action requested (${cyan(message)})`); switch (message) { case "kill": - masterLog(red("an authorized user has manually ended the server session")); - tryKillActiveWorker(true); - process.exit(0); + const { reason, graceful, errorCode } = args; + this.killSession(reason, graceful, errorCode); + break; case "notify_crash": - if (notifiers && notifiers.crash) { + if (this.notifiers?.crash) { const { error } = args; - const success = await notifiers.crash(error); + const success = await this.notifiers.crash(error); const statement = success ? green("distributed crash notification to recipients") : red("distribution of crash notification failed"); - masterLog(statement); + this.mainLog(statement); } + break; case "set_port": const { port, value, immediateRestart } = args; - setPort(port, value, immediateRestart); - default: - const handler = childMessageHandlers[message]; - if (handler) { - handler(action, args); - } + this.setPort(port, value, immediateRestart); + break; } - } else if (lifecycle) { - console.log(timestamp(), `${workerIdentifier} lifecycle phase (${lifecycle})`); + const handlers = this.onMessage[message]; + if (handlers) { + handlers.forEach(handler => handler({ message, args })); + } + } + if (lifecycle) { + console.log(this.timestamp(), `${this.config.identifiers.worker.text} lifecycle phase (${lifecycle})`); } }); - }; + } - // a helpful cluster event called on the master thread each time a child process exits - on("exit", ({ process: { pid } }, code, signal) => { - const prompt = `server worker with process id ${pid} has exited with code ${code}${signal === null ? "" : `, having encountered signal ${signal}`}.`; - masterLog(cyan(prompt)); - // to make this a robust, continuous session, every time a child process dies, we immediately spawn a new one - spawn(); - }); - - // builds the repl that allows the following commands to be typed into stdin of the master thread - const repl = new Repl({ identifier: () => `${timestamp()} ${masterIdentifier}` }); - repl.registerCommand("exit", [], () => execSync(onWindows ? "taskkill /f /im node.exe" : "killall -9 node")); - repl.registerCommand("restart", [], restart); - repl.registerCommand("set", [/[a-zA-Z]+/, "port", /\d+/, /true|false/], args => setPort(args[0], Number(args[2]), args[3] === "true")); - // finally, set things in motion by spawning off the first child (server) process - spawn(); - - // returned to allow the caller to add custom commands - return { - addReplCommand: repl.registerCommand, - addChildMessageHandler: (message: string, handler: ActionHandler) => { childMessageHandlers[message] = handler; } - }; } /** * Effectively, each worker repairs the connection to the server by reintroducing a consistent state * if its predecessor has died. It itself also polls the server heartbeat, and exits with a notification * email if the server encounters an uncaught exception or if the server cannot be reached. - * @param work the function specifying the work to be done by each worker thread */ - export async function initializeWorkerThread(work: Function): Promise<(handler: ExitHandler) => void> { - let shouldServerBeResponsive = false; - const exitHandlers: ExitHandler[] = []; - - // notify master thread (which will log update in the console) of initialization via IPC - process.send?.({ lifecycle: green("compiling and initializing...") }); + export class ServerWorker { + + private static count = 0; + private shouldServerBeResponsive = false; + private exitHandlers: ExitHandler[] = []; + private pollingFailureCount = 0; + private pollingIntervalSeconds: number; + private pollingFailureTolerance: number; + private pollTarget: string; + private serverPort: number; + + public static Create(work: Function) { + if (isMaster) { + console.error(red("cannot create a worker on the monitor process.")); + process.exit(1); + } else if (++ServerWorker.count > 1) { + process.send?.({ + action: { + message: "kill", args: { + reason: "cannot create more than one worker on a given worker process.", + graceful: false, + errorCode: 1 + } + } + }); + process.exit(1); + } else { + return new ServerWorker(work); + } + } - // updates the local value of listening to the value sent from master - process.on("message", ({ setResponsiveness }) => shouldServerBeResponsive = setResponsiveness); + /** + * Allows developers to invoke application specific logic + * by hooking into the exiting of the server process. + */ + public addExitHandler = (handler: ExitHandler) => this.exitHandlers.push(handler); + + /** + * Kill the session monitor (parent process) from this + * server worker (child process). This will also kill + * this process (child process). + */ + public killSession = (reason: string, graceful = true, errorCode = 0) => this.sendMonitorAction("kill", { reason, graceful, errorCode }); + + /** + * A convenience wrapper to tell the session monitor (parent process) + * to carry out the action with the specified message and arguments. + */ + public sendMonitorAction = (message: string, args?: any) => process.send!({ action: { message, args } }); + + private constructor(work: Function) { + this.lifecycleNotification(green(`initializing process... (${white(`${process.execPath} ${process.execArgv.join(" ")}`)})`)); + + const { pollingRoute, serverPort, pollingIntervalSeconds, pollingFailureTolerance } = process.env; + this.serverPort = Number(serverPort); + this.pollingIntervalSeconds = Number(pollingIntervalSeconds); + this.pollingFailureTolerance = Number(pollingFailureTolerance); + this.pollTarget = `http://localhost:${serverPort}${pollingRoute}`; + + this.configureProcess(); + work(); + this.pollServer(); + } - // called whenever the process has a reason to terminate, either through an uncaught exception - // in the process (potentially inconsistent state) or the server cannot be reached - const activeExit = async (error: Error): Promise<void> => { - if (!shouldServerBeResponsive) { - return; - } - shouldServerBeResponsive = false; - // communicates via IPC to the master thread that it should dispatch a crash notification email - process.send?.({ - action: { - message: "notify_crash", - args: { error } + /** + * Set up message and uncaught exception handlers for this + * server process. + */ + private configureProcess = () => { + // updates the local values of variables to the those sent from master + process.on("message", async ({ newPollingIntervalSeconds, manualExit }) => { + if (newPollingIntervalSeconds !== undefined) { + this.pollingIntervalSeconds = newPollingIntervalSeconds; + } + if (manualExit !== undefined) { + await this.executeExitHandlers(null); + process.exit(0); } }); - await Promise.all(exitHandlers.map(handler => handler(error))); + + // one reason to exit, as the process might be in an inconsistent state after such an exception + process.on('uncaughtException', this.proactiveUnplannedExit); + process.on('unhandledRejection', this.proactiveUnplannedExit); + } + + /** + * Execute the list of functions registered to be called + * whenever the process exits. + */ + private executeExitHandlers = async (reason: Error | null) => Promise.all(this.exitHandlers.map(handler => handler(reason))); + + /** + * Notify master thread (which will log update in the console) of initialization via IPC. + */ + public lifecycleNotification = (event: string) => process.send?.({ lifecycle: event }); + + /** + * Called whenever the process has a reason to terminate, either through an uncaught exception + * in the process (potentially inconsistent state) or the server cannot be reached. + */ + private proactiveUnplannedExit = async (error: any): Promise<void> => { + this.shouldServerBeResponsive = false; + // communicates via IPC to the master thread that it should dispatch a crash notification email + this.sendMonitorAction("notify_crash", { error }); + await this.executeExitHandlers(error); // notify master thread (which will log update in the console) of crash event via IPC - process.send?.({ lifecycle: red(`crash event detected @ ${new Date().toUTCString()}`) }); - process.send?.({ lifecycle: red(error.message) }); + this.lifecycleNotification(red(`crash event detected @ ${new Date().toUTCString()}`)); + this.lifecycleNotification(red(error.message || error)); process.exit(1); - }; + } - // one reason to exit, as the process might be in an inconsistent state after such an exception - process.on('uncaughtException', activeExit); - - const { - pollingIntervalSeconds, - pollingRoute, - serverPort - } = process.env; - // this monitors the health of the server by submitting a get request to whatever port / route specified - // by the configuration every n seconds, where n is also given by the configuration. - const pollTarget = `http://localhost:${serverPort}${pollingRoute}`; - const pollServer = async (): Promise<void> => { + /** + * This monitors the health of the server by submitting a get request to whatever port / route specified + * by the configuration every n seconds, where n is also given by the configuration. + */ + private pollServer = async (): Promise<void> => { await new Promise<void>(resolve => { setTimeout(async () => { try { - await get(pollTarget); - if (!shouldServerBeResponsive) { - // notify master thread (which will log update in the console) via IPC that the server is up and running - process.send?.({ lifecycle: green(`listening on ${serverPort}...`) }); + await get(this.pollTarget); + if (!this.shouldServerBeResponsive) { + // notify monitor thread that the server is up and running + this.lifecycleNotification(green(`listening on ${this.serverPort}...`)); } - shouldServerBeResponsive = true; + this.shouldServerBeResponsive = true; resolve(); } catch (error) { // if we expect the server to be unavailable, i.e. during compilation, // the listening variable is false, activeExit will return early and the child // process will continue - activeExit(error); + if (this.shouldServerBeResponsive) { + if (++this.pollingFailureCount > this.pollingFailureTolerance) { + this.proactiveUnplannedExit(error); + } else { + this.lifecycleNotification(yellow(`the server has encountered ${this.pollingFailureCount} of ${this.pollingFailureTolerance} tolerable failures`)); + } + } } - }, 1000 * Number(pollingIntervalSeconds)); + }, 1000 * this.pollingIntervalSeconds); }); // controlled, asynchronous infinite recursion achieves a persistent poll that does not submit a new request until the previous has completed - pollServer(); - }; - - work(); - pollServer(); // begin polling + this.pollServer(); + } - return (handler: ExitHandler) => exitHandlers.push(handler); } }
\ No newline at end of file diff --git a/src/server/Session/session_config_schema.ts b/src/server/Session/session_config_schema.ts index 76af04b9f..e32cf8c6a 100644 --- a/src/server/Session/session_config_schema.ts +++ b/src/server/Session/session_config_schema.ts @@ -1,35 +1,67 @@ import { Schema } from "jsonschema"; +const colorPattern = /black|red|green|yellow|blue|magenta|cyan|white|gray|grey/; + +const identifierProperties: Schema = { + type: "object", + properties: { + text: { + type: "string", + minLength: 1 + }, + color: { + type: "string", + pattern: colorPattern + } + } +}; + +const portProperties: Schema = { + type: "number", + minimum: 1024, + maximum: 65535 +}; + export const configurationSchema: Schema = { id: "/configuration", type: "object", properties: { + showServerOutput: { type: "boolean" }, ports: { type: "object", properties: { - server: { type: "number", minimum: 1024, maximum: 65535 }, - socket: { type: "number", minimum: 1024, maximum: 65535 } + server: portProperties, + socket: portProperties }, required: ["server"], additionalProperties: true }, - pollingRoute: { - type: "string", - pattern: /\/[a-zA-Z]*/g - }, - masterIdentifier: { - type: "string", - minLength: 1 + identifiers: { + type: "object", + properties: { + master: identifierProperties, + worker: identifierProperties, + exec: identifierProperties + } }, - workerIdentifier: { - type: "string", - minLength: 1 + polling: { + type: "object", + additionalProperties: false, + properties: { + intervalSeconds: { + type: "number", + minimum: 1, + maximum: 86400 + }, + route: { + type: "string", + pattern: /\/[a-zA-Z]*/g + }, + failureTolerance: { + type: "number", + minimum: 0, + } + } }, - showServerOutput: { type: "boolean" }, - pollingIntervalSeconds: { - type: "number", - minimum: 1, - maximum: 86400 - } } };
\ No newline at end of file diff --git a/src/server/index.ts b/src/server/index.ts index 5e411aa3a..6b3dfd614 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -3,10 +3,9 @@ import { GoogleApiServerUtils } from "./apis/google/GoogleApiServerUtils"; import * as mobileDetect from 'mobile-detect'; import * as path from 'path'; import { Database } from './database'; -const serverPort = 4321; import { DashUploadUtils } from './DashUploadUtils'; import RouteSubscriber from './RouteSubscriber'; -import initializeServer from './server_Initialization'; +import initializeServer from './server_initialization'; import RouteManager, { Method, _success, _permission_denied, _error, _invalid, PublicHandler } from './RouteManager'; import * as qs from 'query-string'; import UtilManager from './ApiManagers/UtilManager'; @@ -22,13 +21,12 @@ import { log_execution, Email } from "./ActionUtilities"; import GeneralGoogleManager from "./ApiManagers/GeneralGoogleManager"; import GooglePhotosManager from "./ApiManagers/GooglePhotosManager"; import { Logger } from "./ProcessFactory"; -import { yellow } from "colors"; +import { yellow, red } from "colors"; import { Session } from "./Session/session"; -import { isMaster } from "cluster"; -import { execSync } from "child_process"; -import { Utils } from "../Utils"; -import { MessageStore } from "./Message"; +import { DashSessionAgent } from "./DashSession"; +export const onWindows = process.platform === "win32"; +export let sessionAgent: Session.AppliedSessionAgent; export const publicDirectory = path.resolve(__dirname, "public"); export const filesDirectory = path.resolve(publicDirectory, "files"); @@ -96,7 +94,9 @@ function routeSetter({ isRelease, addSupervisedRoute, logRegistrationOutcome }: secureHandler: ({ req, res }) => { if (req.params.key === process.env.session_key) { res.send("<img src='https://media.giphy.com/media/NGIfqtcS81qi4/giphy.gif' style='width:100%;height:100%;'/>"); - process.send!({ action: { message: "kill" } }); + setTimeout(() => { + sessionAgent.killSession("an authorized user has manually ended the server session via the /kill route", false); + }, 5000); } else { res.redirect("/home"); } @@ -136,7 +136,7 @@ function routeSetter({ isRelease, addSupervisedRoute, logRegistrationOutcome }: * however, this becomes the logic invoked by a single worker thread spawned by * the main monitor (master) thread. */ -async function launchServer() { +export async function launchServer() { await log_execution({ startMessage: "\nstarting execution of preliminary functions", endMessage: "completed preliminary functions\n", @@ -146,51 +146,13 @@ async function launchServer() { } /** - * If we're the monitor (master) thread, we should launch the monitor logic for the session. - * Otherwise, we must be on a worker thread that was spawned *by* the monitor (master) thread, and thus - * our job should be to run the server. - */ -async function launchMonitoredSession() { - if (isMaster) { - const recipients = ["samuel_wilkins@brown.edu"]; - const signature = "-Dash Server Session Manager"; - const customizer = await Session.initializeMonitorThread({ - key: async (key: string) => { - const content = `The key for this session (started @ ${new Date().toUTCString()}) is ${key}.\n\n${signature}`; - const failures = await Email.dispatchAll(recipients, "Server Termination Key", content); - return failures.length === 0; - }, - crash: async (error: Error) => { - const subject = "Dash Web Server Crash"; - const { name, message, stack } = error; - const body = [ - "You, as a Dash Administrator, are being notified of a server crash event. Here's what we know:", - `name:\n${name}`, - `message:\n${message}`, - `stack:\n${stack}`, - "The server is already restarting itself, but if you're concerned, use the Remote Desktop Connection to monitor progress.", - ].join("\n\n"); - const content = `${body}\n\n${signature}`; - const failures = await Email.dispatchAll(recipients, subject, content); - return failures.length === 0; - } - }); - customizer.addReplCommand("pull", [], () => execSync("git pull", { stdio: ["ignore", "inherit", "inherit"] })); - customizer.addReplCommand("solr", [/start|stop/g], args => SolrManager.SetRunning(args[0] === "start")); - } else { - const addExitHandler = await Session.initializeWorkerThread(launchServer); // server initialization delegated to worker - addExitHandler(() => Utils.Emit(WebSocket._socket, MessageStore.ConnectionTerminated, "Manual")); - } -} - -/** * If you're in development mode, you won't need to run a session. * The session spawns off new server processes each time an error is encountered, and doesn't * log the output of the server process, so it's not ideal for development. * So, the 'else' clause is exactly what we've always run when executing npm start. */ if (process.env.RELEASE) { - launchMonitoredSession(); + (sessionAgent = new DashSessionAgent()).launch(); } else { launchServer(); }
\ No newline at end of file diff --git a/src/server/repl.ts b/src/server/repl.ts index bd00e48cd..ad55b6aaa 100644 --- a/src/server/repl.ts +++ b/src/server/repl.ts @@ -3,7 +3,7 @@ import { red, green, white } from "colors"; export interface Configuration { identifier: () => string | string; - onInvalid?: (culprit?: string) => string | string; + onInvalid?: (command: string, validCommand: boolean) => string | string; onValid?: (success?: string) => string | string; isCaseSensitive?: boolean; } @@ -16,8 +16,8 @@ export interface Registration { export default class Repl { private identifier: () => string | string; - private onInvalid: (culprit?: string) => string | string; - private onValid: (success: string) => string | string; + private onInvalid: ((command: string, validCommand: boolean) => string) | string; + private onValid: ((success: string) => string) | string; private isCaseSensitive: boolean; private commandMap = new Map<string, Registration[]>(); public interface: Interface; @@ -34,18 +34,24 @@ export default class Repl { private resolvedIdentifier = () => typeof this.identifier === "string" ? this.identifier : this.identifier(); - private usage = () => { - const resolved = this.keys; - if (resolved) { - return resolved; - } - const members: string[] = []; - const keys = this.commandMap.keys(); - let next: IteratorResult<string>; - while (!(next = keys.next()).done) { - members.push(next.value); + private usage = (command: string, validCommand: boolean) => { + if (validCommand) { + const formatted = white(command); + const patterns = green(this.commandMap.get(command)!.map(({ argPatterns }) => `${formatted} ${argPatterns.join(" ")}`).join('\n')); + return `${this.resolvedIdentifier()}\nthe given arguments do not match any registered patterns for ${formatted}\nthe list of valid argument patterns is given by:\n${patterns}`; + } else { + const resolved = this.keys; + if (resolved) { + return resolved; + } + const members: string[] = []; + const keys = this.commandMap.keys(); + let next: IteratorResult<string>; + while (!(next = keys.next()).done) { + members.push(next.value); + } + return `${this.resolvedIdentifier()} commands: { ${members.sort().join(", ")} }`; } - return `${this.resolvedIdentifier()} commands: { ${members.sort().join(", ")} }`; } private success = (command: string) => `${this.resolvedIdentifier()} completed execution of ${white(command)}`; @@ -61,8 +67,8 @@ export default class Repl { } } - private invalid = (culprit?: string) => { - console.log(red(typeof this.onInvalid === "string" ? this.onInvalid : this.onInvalid(culprit))); + private invalid = (command: string, validCommand: boolean) => { + console.log(red(typeof this.onInvalid === "string" ? this.onInvalid : this.onInvalid(command, validCommand))); this.busy = false; } @@ -83,7 +89,7 @@ export default class Repl { } const [command, ...args] = line.split(/\s+/g); if (!command) { - return this.invalid(); + return this.invalid(command, false); } const registered = this.commandMap.get(command); if (registered) { @@ -91,25 +97,32 @@ export default class Repl { const candidates = registered.filter(({ argPatterns: { length: count } }) => count === length); for (const { argPatterns, action } of candidates) { const parsed: string[] = []; - let matched = false; + let matched = true; if (length) { for (let i = 0; i < length; i++) { let matches: RegExpExecArray | null; if ((matches = argPatterns[i].exec(args[i])) === null) { + matched = false; break; } parsed.push(matches[0]); } - matched = true; } if (!length || matched) { - await action(parsed); - this.valid(`${command} ${parsed.join(" ")}`); + const result = action(parsed); + const resolve = () => this.valid(`${command} ${parsed.join(" ")}`); + if (result instanceof Promise) { + result.then(resolve); + } else { + resolve(); + } return; } } + this.invalid(command, true); + } else { + this.invalid(command, false); } - this.invalid(command); } }
\ No newline at end of file diff --git a/src/server/server_Initialization.ts b/src/server/server_Initialization.ts index 0f502e8fb..cbe070293 100644 --- a/src/server/server_Initialization.ts +++ b/src/server/server_Initialization.ts @@ -22,7 +22,7 @@ import { publicDirectory } from '.'; import { logPort, } from './ActionUtilities'; import { timeMap } from './ApiManagers/UserManager'; import { blue, yellow } from 'colors'; -var cors = require('cors'); +import * as cors from "cors"; /* RouteSetter is a wrapper around the server that prevents the server from being exposed. */ @@ -33,13 +33,11 @@ export default async function InitializeServer(routeSetter: RouteSetter) { const app = buildWithMiddleware(express()); app.use(express.static(publicDirectory, { - setHeaders: (res, path) => { - res.setHeader("Access-Control-Allow-Origin", "*"); - } + setHeaders: res => res.setHeader("Access-Control-Allow-Origin", "*") })); app.use("/images", express.static(publicDirectory)); const corsOptions = { - origin: function (origin: any, callback: any) { + origin: function (_origin: any, callback: any) { callback(null, true); } }; |
