From b31d54b285236dc92f7d287af6a441878f429a34 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Fri, 3 Jan 2020 23:28:34 -0800 Subject: session restructuring and schema enforced json configuration --- session.config.json | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 session.config.json (limited to 'session.config.json') diff --git a/session.config.json b/session.config.json new file mode 100644 index 000000000..1a1073c4f --- /dev/null +++ b/session.config.json @@ -0,0 +1,10 @@ +{ + "heartbeat": "http://localhost:1050/serverHeartbeat", + "signature": "Best,\nServer Session Manager", + "masterIdentifier": "__master__", + "workerIdentifier": "__worker__", + "recipients": [ + "samuel_wilkins@brown.edu" + ], + "silentChildren": true +} \ No newline at end of file -- cgit v1.2.3-70-g09d2 From 19b62446a1f05048c6fa940ea4cd7a94021d4ab1 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Sat, 4 Jan 2020 11:12:45 -0800 Subject: cleaner refactor, improved use of configuration --- session.config.json | 11 +- src/server/ActionUtilities.ts | 4 + src/server/Session/session.ts | 348 +++++++++++++++++----------- src/server/Session/session_config_schema.ts | 44 ++-- src/server/index.ts | 11 +- 5 files changed, 252 insertions(+), 166 deletions(-) (limited to 'session.config.json') diff --git a/session.config.json b/session.config.json index 1a1073c4f..7396e1135 100644 --- a/session.config.json +++ b/session.config.json @@ -1,10 +1,11 @@ { - "heartbeat": "http://localhost:1050/serverHeartbeat", - "signature": "Best,\nServer Session Manager", - "masterIdentifier": "__master__", - "workerIdentifier": "__worker__", + "showServerOutput": false, "recipients": [ "samuel_wilkins@brown.edu" ], - "silentChildren": true + "heartbeat": "http://localhost:1050/serverHeartbeat", + "pollingIntervalSeconds": 15, + "masterIdentifier": "__master__", + "workerIdentifier": "__worker__", + "signature": "Best,\nServer Session Manager" } \ No newline at end of file diff --git a/src/server/ActionUtilities.ts b/src/server/ActionUtilities.ts index 950fba093..3125f8683 100644 --- a/src/server/ActionUtilities.ts +++ b/src/server/ActionUtilities.ts @@ -118,6 +118,10 @@ export namespace Email { } }); + export async function dispatchAll(recipients: string[], subject: string, content: string) { + return Promise.all(recipients.map((recipient: string) => Email.dispatch(recipient, subject, content))); + } + export async function dispatch(recipient: string, subject: string, content: string): Promise { const mailOptions = { to: recipient, diff --git a/src/server/Session/session.ts b/src/server/Session/session.ts index 1ff4ce4de..2ff4ef0d2 100644 --- a/src/server/Session/session.ts +++ b/src/server/Session/session.ts @@ -13,162 +13,232 @@ import { configurationSchema } from "./session_config_schema"; const onWindows = process.platform === "win32"; +/** + * This namespace relies on NodeJS's cluster module, which allows a parent (master) process to share + * code with its children (workers). A simple `isMaster` flag indicates who is trying to access + * the code, and thus determines the functionality that actually gets invoked (checked by the caller, not internally). + * + * Think of the master thread as a factory, and the workers as the helpers that actually run the server. + * + * So, when we run `npm start`, given the appropriate check, initializeMaster() is called in the parent process + * This will spawn off its own child process (by default, mirrors the execution path of its parent), + * in which initializeWorker() is invoked. + */ export namespace Session { - export let key: string; - let activeWorker: Worker; - let listening = false; - - function loadConfiguration() { - try { - const raw = readFileSync('./session.config.json', 'utf8'); - const configuration = JSON.parse(raw); - const options = { - throwError: true, - allowUnknownAttributes: false - }; - validate(configuration, configurationSchema, options); - configuration.masterIdentifier = `${yellow(configuration.masterIdentifier)}:`; - configuration.workerIdentifier = `${magenta(configuration.workerIdentifier)}:`; - return configuration; - } catch (error) { - console.log(red("\nSession configuration failed.")); - if (error instanceof ValidationError) { - console.log("The given session.config.json configuration file is invalid."); - console.log(`${error.instance}: ${error.stack}`); - } else if (error.code === "ENOENT" && error.path === "./session.config.json") { - console.log("Please include a session.config.json configuration file in your project root."); - } else { - console.log(error.stack); + /** + * Validates and reads the configuration file, accordingly builds a child process factory + * and spawns off an initial process that will respawn as predecessors die. + */ + export async function initializeMaster() { + let activeWorker: Worker; + + // read in configuration .json file only once, in the master thread + // pass down any variables the pertinent to the child processes as environment variables + const { + masterIdentifier, + workerIdentifier, + recipients, + signature, + heartbeat, + showServerOutput, + pollingIntervalSeconds + } = function loadConfiguration() { + try { + const raw = readFileSync('./session.config.json', 'utf8'); + const configuration = JSON.parse(raw); + const options = { + throwError: true, + allowUnknownAttributes: false + }; + // ensure all necessary and no excess information is specified by the configuration file + validate(configuration, configurationSchema, options); + configuration.masterIdentifier = `${yellow(configuration.masterIdentifier)}:`; + configuration.workerIdentifier = `${magenta(configuration.workerIdentifier)}:`; + return configuration; + } catch (error) { + console.log(red("\nSession configuration failed.")); + if (error instanceof ValidationError) { + console.log("The given session.config.json configuration file is invalid."); + console.log(`${error.instance}: ${error.stack}`); + } else if (error.code === "ENOENT" && error.path === "./session.config.json") { + console.log("Please include a session.config.json configuration file in your project root."); + } else { + console.log("The following unknown error occurred during configuration."); + console.log(error.stack); + } + console.log(); + process.exit(0); } - console.log(); - process.exit(0); - } - } + }(); - export async function email(recipients: string[], subject: string, content: string) { - return Promise.all(recipients.map((recipient: string) => Email.dispatch(recipient, subject, content))); - } + // this sends a random guid to the configuration's recipients, allowing them alone + // to kill the server via the /kill/:password route + const key = Utils.GenerateGuid(); + const timestamp = new Date().toUTCString(); + const content = `The key for this session (started @ ${timestamp}) is ${key}.\n\n${signature}`; + await Email.dispatchAll(recipients, "Server Termination Key", content); - function tryKillActiveWorker() { - if (activeWorker && !activeWorker.isDead()) { - activeWorker.process.kill(); - return true; - } - return false; - } + console.log(masterIdentifier, "distributed session key to recipients"); - async function activeExit(error: Error) { - if (!listening) { - return; - } - listening = false; - process.send?.({ - action: { - message: "notify_crash", - args: { error } + // handle exceptions in the master thread - there shouldn't be many of these + // the IPC (inter process communication) channel closed exception can't seem to be caught in a try catch, and is inconsequential, so it is ignored + process.on("uncaughtException", ({ message, stack }) => { + if (message !== "Channel closed") { + console.log(masterIdentifier, red(message)); + if (stack) { + console.log(masterIdentifier, `\n${red(stack)}`); + } } }); - const { _socket } = WebSocket; - if (_socket) { - Utils.Emit(_socket, MessageStore.ConnectionTerminated, "Manual"); - } - process.send?.({ lifecycle: red(`Crash event detected @ ${new Date().toUTCString()}`) }); - process.send?.({ lifecycle: red(error.message) }); - process.exit(1); - } - export async function initialize(work: Function) { - if (isMaster) { - const { - masterIdentifier, - workerIdentifier, - recipients, - signature, + // determines whether or not we see the compilation / initialization / runtime output of each child server process + setupMaster({ silent: !showServerOutput }); + + // attempts to kills the active worker ungracefully + const tryKillActiveWorker = () => { + if (activeWorker && !activeWorker.isDead()) { + activeWorker.process.kill(); + return true; + } + return false; + }; + + // kills the current active worker and proceeds to spawn a new worker, feeding in configuration information as environment variables + const spawn = () => { + tryKillActiveWorker(); + activeWorker = fork({ heartbeat, - silentChildren - } = loadConfiguration(); - await (async function distributeKey() { - key = Utils.GenerateGuid(); - const timestamp = new Date().toUTCString(); - const content = `The key for this session (started @ ${timestamp}) is ${key}.\n\n${signature}`; - return email(recipients, "Server Termination Key", content); - })(); - console.log(masterIdentifier, "distributed session key to recipients"); - process.on("uncaughtException", ({ message, stack }) => { - if (message !== "Channel closed") { - console.log(masterIdentifier, red(message)); - if (stack) { - console.log(masterIdentifier, `\n${red(stack)}`); + pollingIntervalSeconds, + session_key: key + }); + console.log(masterIdentifier, `spawned new server worker with process id ${activeWorker.process.pid}`); + // an IPC message handler that executes actions on the master thread when prompted by the active worker + activeWorker.on("message", ({ lifecycle, action }) => { + if (action) { + const { message, args } = action; + console.log(`${workerIdentifier} action requested (${cyan(message)})`); + switch (message) { + case "kill": + console.log(masterIdentifier, red("An authorized user has ended the server session from the /kill route")); + tryKillActiveWorker(); + process.exit(0); + case "notify_crash": + const { error: { name, message, stack } } = args; + const content = [ + "You, as a Dash Administrator, are being notified of a server crash event. Here's what we know:", + `name:\n${name}`, + `message:\n${message}`, + `stack:\n${stack}`, + "The server is already restarting itself, but if you're concerned, use the Remote Desktop Connection to monitor progress.", + signature + ].join("\n\n"); + Email.dispatchAll(recipients, "Dash Web Server Crash", content); } + } else if (lifecycle) { + console.log(`${workerIdentifier} lifecycle phase (${lifecycle})`); } }); - setupMaster({ silent: silentChildren }); - const spawn = () => { - tryKillActiveWorker(); - activeWorker = fork({ heartbeat, session_key: key }); - console.log(masterIdentifier, `spawned new server worker with process id ${activeWorker.process.pid}`); - activeWorker.on("message", ({ lifecycle, action }) => { - if (action) { - const { message, args } = action; - console.log(`${workerIdentifier} action requested (${cyan(message)})`); - switch (message) { - case "kill": - console.log(masterIdentifier, red("An authorized user has ended the server session from the /kill route")); - tryKillActiveWorker(); - process.exit(0); - case "notify_crash": - const { error: { name, message, stack } } = args; - email(recipients, "Dash Web Server Crash", [ - "You, as a Dash Administrator, are being notified of a server crash event. Here's what we know:", - `name:\n${name}`, - `message:\n${message}`, - `stack:\n${stack}`, - "The server is already restarting itself, but if you're concerned, use the Remote Desktop Connection to monitor progress.", - signature - ].join("\n\n")); - } - } else if (lifecycle) { - console.log(`${workerIdentifier} lifecycle phase (${lifecycle})`); - } - }); - }; + }; + + // a helpful cluster event called on the master thread each time a child process exits + on("exit", ({ process: { pid } }, code, signal) => { + const prompt = `Server worker with process id ${pid} has exited with code ${code}${signal === null ? "" : `, having encountered signal ${signal}`}.`; + console.log(masterIdentifier, cyan(prompt)); + // to make this a robust, continuous session, every time a child process dies, we immediately spawn a new one spawn(); - on("exit", ({ process: { pid } }, code, signal) => { - const prompt = `Server worker with process id ${pid} has exited with code ${code}${signal === null ? "" : `, having encountered signal ${signal}`}.`; - console.log(masterIdentifier, cyan(prompt)); - spawn(); - }); - const { registerCommand } = new Repl({ identifier: masterIdentifier }); - registerCommand("exit", [], () => execSync(onWindows ? "taskkill /f /im node.exe" : "killall -9 node")); - registerCommand("pull", [], () => execSync("git pull", { stdio: ["ignore", "inherit", "inherit"] })); - registerCommand("restart", [], () => { - listening = false; - tryKillActiveWorker(); + }); + + // builds the repl that allows the following commands to be typed into stdin of the master thread + const { registerCommand } = new Repl({ identifier: masterIdentifier }); + registerCommand("exit", [], () => execSync(onWindows ? "taskkill /f /im node.exe" : "killall -9 node")); + registerCommand("pull", [], () => execSync("git pull", { stdio: ["ignore", "inherit", "inherit"] })); + registerCommand("restart", [], () => { + // indicate to the worker that we are 'expecting' this restart + activeWorker.send({ setListening: false }); + tryKillActiveWorker(); + }); + + // finally, set things in motion by spawning off the first child (server) process + spawn(); + } + + /** + * Effectively, each worker repairs the connection to the server by reintroducing a consistent state + * if its predecessor has died. It itself also polls the server heartbeat, and exits with a notification + * email if the server encounters an uncaught exception or if the server cannot be reached. + * @param work the function specifying the work to be done by each worker thread + */ + export async function initializeWorker(work: Function) { + let listening = false; + + // notify master thread (which will log update in the console) of initialization via IPC + process.send?.({ lifecycle: green("initializing...") }); + + // updates the local value of listening to the value sent from master + process.on("message", ({ setListening }) => listening = setListening); + + // called whenever the process has a reason to terminate, either through an uncaught exception + // in the process (potentially inconsistent state) or the server cannot be reached + const activeExit = (error: Error) => { + if (!listening) { + return; + } + listening = false; + // communicates via IPC to the master thread that it should dispatch a crash notification email + process.send?.({ + action: { + message: "notify_crash", + args: { error } + } }); - } else { - process.send?.({ lifecycle: green("initializing...") }); - process.on('uncaughtException', activeExit); - const checkHeartbeat = async () => { - await new Promise(resolve => { - setTimeout(async () => { - try { - await get(process.env.heartbeat!); - if (!listening) { - process.send?.({ lifecycle: green("listening...") }); - } - listening = true; - resolve(); - } catch (error) { - await activeExit(error); + const { _socket } = WebSocket; + // notifies all client users of a crash event + if (_socket) { + Utils.Emit(_socket, MessageStore.ConnectionTerminated, "Manual"); + } + // notify master thread (which will log update in the console) of crash event via IPC + process.send?.({ lifecycle: red(`Crash event detected @ ${new Date().toUTCString()}`) }); + process.send?.({ lifecycle: red(error.message) }); + process.exit(1); + }; + + // one reason to exit, as the process might be in an inconsistent state after such an exception + process.on('uncaughtException', activeExit); + + const { pollingIntervalSeconds, heartbeat } = process.env; + + // this monitors the health of the server by submitting a get request to whatever port / route specified + // by the configuration every n seconds, where n is also given by the configuration. + const checkHeartbeat = async () => { + await new Promise(resolve => { + setTimeout(async () => { + try { + await get(heartbeat!); + if (!listening) { + // notify master thread (which will log update in the console) via IPC that the server is up and running + process.send?.({ lifecycle: green("listening...") }); } - }, 1000 * 15); - }); - checkHeartbeat(); - }; - work(); + listening = true; + resolve(); + } catch (error) { + // if we expect the server to be unavailable, i.e. during compilation, + // the listening variable is false, activeExit will return early and the child + // process will continue + activeExit(error); + } + }, 1000 * Number(pollingIntervalSeconds)); + }); + // controlled, asynchronous infinite recursion achieves a persistent poll that does not submit a new request until the previous has completed checkHeartbeat(); - } + }; + + // the actual work of the process, may be asynchronous + // for Dash, this is the code that launches the server + work(); + + // begin polling + checkHeartbeat(); } } \ No newline at end of file diff --git a/src/server/Session/session_config_schema.ts b/src/server/Session/session_config_schema.ts index 25d95c243..a5010055a 100644 --- a/src/server/Session/session_config_schema.ts +++ b/src/server/Session/session_config_schema.ts @@ -1,25 +1,31 @@ import { Schema } from "jsonschema"; -export const configurationSchema: Schema = { - id: "/Configuration", - type: "object", - properties: { - recipients: { - type: "array", - items: { - type: "string", - pattern: /[^\@]+\@[^\@]+/g - }, - minLength: 1 - }, - heartbeat: { +const emailPattern = /^(([a-zA-Z0-9_.-])+@([a-zA-Z0-9_.-])+\.([a-zA-Z])+([a-zA-Z])+)?$/g; +const localPortPattern = /http\:\/\/localhost:\d+\/[a-zA-Z]+/g; + +const properties = { + recipients: { + type: "array", + items: { type: "string", - pattern: /http\:\/\/localhost:\d+\/[a-zA-Z]+/g + pattern: emailPattern }, - signature: { type: "string" }, - masterIdentifier: { type: "string", minLength: 1 }, - workerIdentifier: { type: "string", minLength: 1 }, - silentChildren: { type: "boolean" } + minLength: 1 }, - required: ["heartbeat", "recipients", "signature", "masterIdentifier", "workerIdentifier", "silentChildren"] + heartbeat: { + type: "string", + pattern: localPortPattern + }, + signature: { type: "string" }, + masterIdentifier: { type: "string", minLength: 1 }, + workerIdentifier: { type: "string", minLength: 1 }, + showServerOutput: { type: "boolean" }, + pollingIntervalSeconds: { type: "number", minimum: 1, maximum: 86400 } +}; + +export const configurationSchema: Schema = { + id: "/configuration", + type: "object", + properties, + required: Object.keys(properties) }; \ No newline at end of file diff --git a/src/server/index.ts b/src/server/index.ts index 381496c20..0a5b4afae 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -24,6 +24,13 @@ import GooglePhotosManager from "./ApiManagers/GooglePhotosManager"; import { Logger } from "./ProcessFactory"; import { yellow } from "colors"; import { Session } from "./Session/session"; +import { isMaster } from "cluster"; + +if (isMaster) { + Session.initializeMaster(); +} else { + Session.initializeWorker(launch); +} export const publicDirectory = path.resolve(__dirname, "public"); export const filesDirectory = path.resolve(publicDirectory, "files"); @@ -133,6 +140,4 @@ async function launch() { action: preliminaryFunctions }); await initializeServer({ serverPort: 1050, routeSetter }); -} - -Session.initialize(launch); \ No newline at end of file +} \ No newline at end of file -- cgit v1.2.3-70-g09d2 From 987b512d2564710e5c5c7fd2eeff1914af8180dd Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Sat, 4 Jan 2020 13:02:54 -0800 Subject: factored out socket and server ports to config, added kill response ;) --- session.config.json | 4 +++- src/server/Initialization.ts | 11 +++-------- src/server/Session/session.ts | 28 ++++++++++++++++------------ src/server/Session/session_config_schema.ts | 6 ++++-- src/server/Websocket/Websocket.ts | 7 ++++--- src/server/index.ts | 12 ++++++------ 6 files changed, 36 insertions(+), 32 deletions(-) (limited to 'session.config.json') diff --git a/session.config.json b/session.config.json index 7396e1135..d8f86d239 100644 --- a/session.config.json +++ b/session.config.json @@ -3,7 +3,9 @@ "recipients": [ "samuel_wilkins@brown.edu" ], - "heartbeat": "http://localhost:1050/serverHeartbeat", + "serverPort": 1050, + "socketPort": 4321, + "heartbeatRoute": "/serverHeartbeat", "pollingIntervalSeconds": 15, "masterIdentifier": "__master__", "workerIdentifier": "__worker__", diff --git a/src/server/Initialization.ts b/src/server/Initialization.ts index 465e7ea63..702339ca1 100644 --- a/src/server/Initialization.ts +++ b/src/server/Initialization.ts @@ -26,15 +26,9 @@ import { blue, yellow } from 'colors'; /* RouteSetter is a wrapper around the server that prevents the server from being exposed. */ export type RouteSetter = (server: RouteManager) => void; -export interface InitializationOptions { - serverPort: number; - routeSetter: RouteSetter; -} - export let disconnect: Function; -export default async function InitializeServer(options: InitializationOptions) { - const { serverPort, routeSetter } = options; +export default async function InitializeServer(routeSetter: RouteSetter) { const app = buildWithMiddleware(express()); app.use(express.static(publicDirectory)); @@ -63,8 +57,9 @@ export default async function InitializeServer(options: InitializationOptions) { routeSetter(new RouteManager(app, isRelease)); + const serverPort = Number(process.env.serverPort); const server = app.listen(serverPort, () => { - logPort("server", serverPort); + logPort("server", Number(serverPort)); console.log(); }); disconnect = async () => new Promise(resolve => server.close(resolve)); diff --git a/src/server/Session/session.ts b/src/server/Session/session.ts index e32811a18..d1d7aab87 100644 --- a/src/server/Session/session.ts +++ b/src/server/Session/session.ts @@ -40,7 +40,9 @@ export namespace Session { workerIdentifier, recipients, signature, - heartbeat, + heartbeatRoute, + serverPort, + socketPort, showServerOutput, pollingIntervalSeconds } = function loadConfiguration(): any { @@ -72,7 +74,7 @@ export namespace Session { }(); // this sends a pseudorandomly generated guid to the configuration's recipients, allowing them alone - // to kill the server via the /kill/:password route + // to kill the server via the /kill/:key route const key = Utils.GenerateGuid(); const timestamp = new Date().toUTCString(); const content = `The key for this session (started @ ${timestamp}) is ${key}.\n\n${signature}`; @@ -112,7 +114,9 @@ export namespace Session { const spawn = (): void => { tryKillActiveWorker(); activeWorker = fork({ - heartbeat, + heartbeatRoute, + serverPort, + socketPort, pollingIntervalSeconds, session_key: key }); @@ -212,18 +216,22 @@ export namespace Session { // one reason to exit, as the process might be in an inconsistent state after such an exception process.on('uncaughtException', activeExit); - const { pollingIntervalSeconds, heartbeat } = process.env; - + const { + pollingIntervalSeconds, + heartbeatRoute, + serverPort + } = process.env; // this monitors the health of the server by submitting a get request to whatever port / route specified // by the configuration every n seconds, where n is also given by the configuration. + const heartbeat = `http://localhost:${serverPort}${heartbeatRoute}`; const checkHeartbeat = async (): Promise => { await new Promise(resolve => { setTimeout(async () => { try { - await get(heartbeat!); + await get(heartbeat); if (!listening) { // notify master thread (which will log update in the console) via IPC that the server is up and running - process.send?.({ lifecycle: green("listening...") }); + process.send?.({ lifecycle: green(`listening on ${serverPort}...`) }); } listening = true; resolve(); @@ -239,12 +247,8 @@ export namespace Session { checkHeartbeat(); }; - // the actual work of the process, may be asynchronous - // for Dash, this is the code that launches the server work(); - - // begin polling - checkHeartbeat(); + checkHeartbeat(); // begin polling } } \ No newline at end of file diff --git a/src/server/Session/session_config_schema.ts b/src/server/Session/session_config_schema.ts index a5010055a..03009a351 100644 --- a/src/server/Session/session_config_schema.ts +++ b/src/server/Session/session_config_schema.ts @@ -1,7 +1,7 @@ import { Schema } from "jsonschema"; const emailPattern = /^(([a-zA-Z0-9_.-])+@([a-zA-Z0-9_.-])+\.([a-zA-Z])+([a-zA-Z])+)?$/g; -const localPortPattern = /http\:\/\/localhost:\d+\/[a-zA-Z]+/g; +const localPortPattern = /\/[a-zA-Z]+/g; const properties = { recipients: { @@ -12,7 +12,9 @@ const properties = { }, minLength: 1 }, - heartbeat: { + serverPort: { type: "number" }, + socketPort: { type: "number" }, + heartbeatRoute: { type: "string", pattern: localPortPattern }, diff --git a/src/server/Websocket/Websocket.ts b/src/server/Websocket/Websocket.ts index 0b58ca344..e26a76107 100644 --- a/src/server/Websocket/Websocket.ts +++ b/src/server/Websocket/Websocket.ts @@ -18,15 +18,15 @@ export namespace WebSocket { export const socketMap = new Map(); export let disconnect: Function; - export async function start(serverPort: number, isRelease: boolean) { + export async function start(isRelease: boolean) { await preliminaryFunctions(); - initialize(serverPort, isRelease); + initialize(isRelease); } async function preliminaryFunctions() { } - export function initialize(socketPort: number, isRelease: boolean) { + function initialize(isRelease: boolean) { const endpoint = io(); endpoint.on("connection", function (socket: Socket) { _socket = socket; @@ -63,6 +63,7 @@ export namespace WebSocket { }; }); + const socketPort = Number(process.env.socketPort); endpoint.listen(socketPort); logPort("websocket", socketPort); } diff --git a/src/server/index.ts b/src/server/index.ts index 3c1839e4d..7eb8b12be 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -90,11 +90,11 @@ function routeSetter({ isRelease, addSupervisedRoute, logRegistrationOutcome }: addSupervisedRoute({ method: Method.GET, - subscription: new RouteSubscriber("kill").add("password"), + subscription: new RouteSubscriber("kill").add("key"), secureHandler: ({ req, res }) => { - if (req.params.password === process.env.session_key) { - process.send!({ action: { message: "kill" } }); - res.send("Server successfully killed."); + if (req.params.key === process.env.session_key) { + res.send(""); + setTimeout(() => process.send!({ action: { message: "kill" } }), 1000 * 5); } else { res.redirect("/home"); } @@ -125,7 +125,7 @@ function routeSetter({ isRelease, addSupervisedRoute, logRegistrationOutcome }: // initialize the web socket (bidirectional communication: if a user changes // a field on one client, that change must be broadcast to all other clients) - WebSocket.initialize(serverPort, isRelease); + WebSocket.start(isRelease); } /** @@ -142,6 +142,6 @@ if (isMaster) { endMessage: "completed preliminary functions\n", action: preliminaryFunctions }); - await initializeServer({ serverPort: 1050, routeSetter }); + await initializeServer(routeSetter); }); } \ No newline at end of file -- cgit v1.2.3-70-g09d2 From d4e7e354ec9209f117054ead9970ea9f180dd17b Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Sun, 5 Jan 2020 14:50:54 -0800 Subject: port config, customizers --- session.config.json | 6 +- src/server/Session/session.ts | 131 ++++++++++++++++++---------- src/server/Session/session_config_schema.ts | 13 ++- src/server/index.ts | 42 ++++++--- src/server/repl.ts | 27 ++++-- 5 files changed, 145 insertions(+), 74 deletions(-) (limited to 'session.config.json') diff --git a/session.config.json b/session.config.json index d8f86d239..ebe17763e 100644 --- a/session.config.json +++ b/session.config.json @@ -3,8 +3,10 @@ "recipients": [ "samuel_wilkins@brown.edu" ], - "serverPort": 1050, - "socketPort": 4321, + "ports": { + "server": 1050, + "socket": 4321 + }, "heartbeatRoute": "/serverHeartbeat", "pollingIntervalSeconds": 15, "masterIdentifier": "__master__", diff --git a/src/server/Session/session.ts b/src/server/Session/session.ts index 789a40c42..61b8bcf16 100644 --- a/src/server/Session/session.ts +++ b/src/server/Session/session.ts @@ -1,12 +1,10 @@ import { red, cyan, green, yellow, magenta } from "colors"; -import { isMaster, on, fork, setupMaster, Worker } from "cluster"; +import { on, fork, setupMaster, Worker } from "cluster"; import { execSync } from "child_process"; import { get } from "request-promise"; -import { WebSocket } from "../Websocket/Websocket"; import { Utils } from "../../Utils"; -import { MessageStore } from "../Message"; import { Email } from "../ActionUtilities"; -import Repl from "../repl"; +import Repl, { ReplAction } from "../repl"; import { readFileSync } from "fs"; import { validate, ValidationError } from "jsonschema"; import { configurationSchema } from "./session_config_schema"; @@ -26,26 +24,35 @@ const onWindows = process.platform === "win32"; */ export namespace Session { + interface MasterCustomizer { + addReplCommand: (basename: string, argPatterns: (RegExp | string)[], action: ReplAction) => void; + addChildMessageHandler: (message: string, handler: ActionHandler) => void; + } + + export interface SessionAction { + message: string; + args: any; + } + + export type ExitHandler = (error: Error) => void | Promise; + export type ActionHandler = (action: SessionAction) => void | Promise; + export interface EmailTemplate { + subject: string; + body: string; + } + export type CrashEmailGenerator = (error: Error) => EmailTemplate | Promise; + /** * Validates and reads the configuration file, accordingly builds a child process factory * and spawns off an initial process that will respawn as predecessors die. */ - export async function initializeMonitorThread(): Promise { + export async function initializeMonitorThread(crashEmailGenerator?: CrashEmailGenerator): Promise { let activeWorker: Worker; + const childMessageHandlers: { [message: string]: (action: SessionAction, args: any) => void } = {}; // read in configuration .json file only once, in the master thread // pass down any variables the pertinent to the child processes as environment variables - const { - masterIdentifier, - workerIdentifier, - recipients, - signature, - heartbeatRoute, - serverPort, - socketPort, - showServerOutput, - pollingIntervalSeconds - } = function loadConfiguration(): any { + const configuration = function loadConfiguration(): any { try { const configuration = JSON.parse(readFileSync('./session.config.json', 'utf8')); const options = { @@ -54,8 +61,8 @@ export namespace Session { }; // ensure all necessary and no excess information is specified by the configuration file validate(configuration, configurationSchema, options); - configuration.masterIdentifier = `${yellow(configuration.masterIdentifier)}:`; - configuration.workerIdentifier = `${magenta(configuration.workerIdentifier)}:`; + configuration.masterIdentifier = yellow(configuration.masterIdentifier + ":"); + configuration.workerIdentifier = magenta(configuration.workerIdentifier + ":"); return configuration; } catch (error) { console.log(red("\nSession configuration failed.")); @@ -73,6 +80,17 @@ export namespace Session { } }(); + const { + masterIdentifier, + workerIdentifier, + recipients, + ports, + signature, + heartbeatRoute, + showServerOutput, + pollingIntervalSeconds + } = configuration; + // this sends a pseudorandomly generated guid to the configuration's recipients, allowing them alone // to kill the server via the /kill/:key route const key = Utils.GenerateGuid(); @@ -92,7 +110,7 @@ export namespace Session { if (message !== "Channel closed") { console.log(masterIdentifier, red(message)); if (stack) { - console.log(masterIdentifier, `\n${red(stack)}`); + console.log(masterIdentifier, `uncaught exception\n${red(stack)}`); } } }); @@ -113,39 +131,56 @@ export namespace Session { return false; }; + const restart = () => { + // indicate to the worker that we are 'expecting' this restart + activeWorker.send({ setListening: false }); + tryKillActiveWorker(); + }; + + const setPort = (port: string, value: number, immediateRestart: boolean) => { + ports[port] = value; + if (immediateRestart) { + restart(); + } + }; + // kills the current active worker and proceeds to spawn a new worker, // feeding in configuration information as environment variables const spawn = (): void => { tryKillActiveWorker(); activeWorker = fork({ heartbeatRoute, - serverPort, - socketPort, + serverPort: ports.server, + socketPort: ports.socket, pollingIntervalSeconds, session_key: key }); console.log(masterIdentifier, `spawned new server worker with process id ${activeWorker.process.pid}`); // an IPC message handler that executes actions on the master thread when prompted by the active worker - activeWorker.on("message", ({ lifecycle, action }) => { + activeWorker.on("message", async ({ lifecycle, action }) => { if (action) { - const { message, args } = action; + const { message, args } = action as SessionAction; console.log(`${workerIdentifier} action requested (${cyan(message)})`); switch (message) { case "kill": - console.log(masterIdentifier, red("An authorized user has ended the server session from the /kill route")); + console.log(masterIdentifier, red("An authorized user has manually ended the server session")); tryKillActiveWorker(false); process.exit(0); case "notify_crash": - const { error: { name, message, stack } } = args; - const content = [ - "You, as a Dash Administrator, are being notified of a server crash event. Here's what we know:", - `name:\n${name}`, - `message:\n${message}`, - `stack:\n${stack}`, - "The server is already restarting itself, but if you're concerned, use the Remote Desktop Connection to monitor progress.", - signature - ].join("\n\n"); - Email.dispatchAll(recipients, "Dash Web Server Crash", content); + if (crashEmailGenerator) { + const { error } = args; + const { subject, body } = await crashEmailGenerator(error); + const content = `${body}\n\n${signature}`; + Email.dispatchAll(recipients, subject, content); + } + case "set_port": + const { port, value, immediateRestart } = args; + setPort(port, value, immediateRestart); + default: + const handler = childMessageHandlers[message]; + if (handler) { + handler(action, args); + } } } else if (lifecycle) { console.log(`${workerIdentifier} lifecycle phase (${lifecycle})`); @@ -155,7 +190,7 @@ export namespace Session { // a helpful cluster event called on the master thread each time a child process exits on("exit", ({ process: { pid } }, code, signal) => { - const prompt = `Server worker with process id ${pid} has exited with code ${code}${signal === null ? "" : `, having encountered signal ${signal}`}.`; + const prompt = `server worker with process id ${pid} has exited with code ${code}${signal === null ? "" : `, having encountered signal ${signal}`}.`; console.log(masterIdentifier, cyan(prompt)); // to make this a robust, continuous session, every time a child process dies, we immediately spawn a new one spawn(); @@ -164,17 +199,18 @@ export namespace Session { // builds the repl that allows the following commands to be typed into stdin of the master thread const repl = new Repl({ identifier: masterIdentifier }); repl.registerCommand("exit", [], () => execSync(onWindows ? "taskkill /f /im node.exe" : "killall -9 node")); - repl.registerCommand("restart", [], () => { - // indicate to the worker that we are 'expecting' this restart - activeWorker.send({ setListening: false }); - tryKillActiveWorker(); + repl.registerCommand("restart", [], restart); + repl.registerCommand("set", [/[a-zA-Z]+/g, "port", /\d+/g, /true|false/g], args => { + setPort(args[0], Number(args[2]), args[3] === "true"); }); - // finally, set things in motion by spawning off the first child (server) process spawn(); // returned to allow the caller to add custom commands - return repl; + return { + addReplCommand: repl.registerCommand, + addChildMessageHandler: (message: string, handler: ActionHandler) => { childMessageHandlers[message] = handler; } + }; } /** @@ -183,8 +219,9 @@ export namespace Session { * email if the server encounters an uncaught exception or if the server cannot be reached. * @param work the function specifying the work to be done by each worker thread */ - export async function initializeWorkerThread(work: Function): Promise { + export async function initializeWorkerThread(work: Function): Promise<(handler: ExitHandler) => void> { let listening = false; + const exitHandlers: ExitHandler[] = []; // notify master thread (which will log update in the console) of initialization via IPC process.send?.({ lifecycle: green("initializing...") }); @@ -194,7 +231,7 @@ export namespace Session { // called whenever the process has a reason to terminate, either through an uncaught exception // in the process (potentially inconsistent state) or the server cannot be reached - const activeExit = (error: Error): void => { + const activeExit = async (error: Error): Promise => { if (!listening) { return; } @@ -206,11 +243,7 @@ export namespace Session { args: { error } } }); - const { _socket } = WebSocket; - // notifies all client users of a crash event - if (_socket) { - Utils.Emit(_socket, MessageStore.ConnectionTerminated, "Manual"); - } + await Promise.all(exitHandlers.map(handler => handler(error))); // notify master thread (which will log update in the console) of crash event via IPC process.send?.({ lifecycle: red(`Crash event detected @ ${new Date().toUTCString()}`) }); process.send?.({ lifecycle: red(error.message) }); @@ -253,6 +286,8 @@ export namespace Session { work(); checkHeartbeat(); // begin polling + + return (handler: ExitHandler) => exitHandlers.push(handler); } } \ No newline at end of file diff --git a/src/server/Session/session_config_schema.ts b/src/server/Session/session_config_schema.ts index 03009a351..34d1ad523 100644 --- a/src/server/Session/session_config_schema.ts +++ b/src/server/Session/session_config_schema.ts @@ -3,7 +3,7 @@ import { Schema } from "jsonschema"; const emailPattern = /^(([a-zA-Z0-9_.-])+@([a-zA-Z0-9_.-])+\.([a-zA-Z])+([a-zA-Z])+)?$/g; const localPortPattern = /\/[a-zA-Z]+/g; -const properties = { +const properties: { [name: string]: Schema } = { recipients: { type: "array", items: { @@ -12,8 +12,15 @@ const properties = { }, minLength: 1 }, - serverPort: { type: "number" }, - socketPort: { type: "number" }, + ports: { + type: "object", + properties: { + server: { type: "number" }, + socket: { type: "number" } + }, + required: ["server"], + additionalProperties: true + }, heartbeatRoute: { type: "string", pattern: localPortPattern diff --git a/src/server/index.ts b/src/server/index.ts index 4400687d8..0fee41bd8 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -26,6 +26,8 @@ import { yellow } from "colors"; import { Session } from "./Session/session"; import { isMaster } from "cluster"; import { execSync } from "child_process"; +import { Utils } from "../Utils"; +import { MessageStore } from "./Message"; export const publicDirectory = path.resolve(__dirname, "public"); export const filesDirectory = path.resolve(publicDirectory, "files"); @@ -132,17 +134,31 @@ function routeSetter({ isRelease, addSupervisedRoute, logRegistrationOutcome }: /** * Thread dependent session initialization */ -if (isMaster) { - Session.initializeMonitorThread().then(({ registerCommand }) => { - registerCommand("pull", [], () => execSync("git pull", { stdio: ["ignore", "inherit", "inherit"] })); - }); -} else { - Session.initializeWorkerThread(async () => { - await log_execution({ - startMessage: "\nstarting execution of preliminary functions", - endMessage: "completed preliminary functions\n", - action: preliminaryFunctions +(async function launch() { + if (isMaster) { + const emailGenerator = (error: Error) => { + const subject = "Dash Web Server Crash"; + const { name, message, stack } = error; + const body = [ + "You, as a Dash Administrator, are being notified of a server crash event. Here's what we know:", + `name:\n${name}`, + `message:\n${message}`, + `stack:\n${stack}`, + "The server is already restarting itself, but if you're concerned, use the Remote Desktop Connection to monitor progress.", + ].join("\n\n"); + return { subject, body }; + }; + const customizer = await Session.initializeMonitorThread(emailGenerator); + customizer.addReplCommand("pull", [], () => execSync("git pull", { stdio: ["ignore", "inherit", "inherit"] })); + } else { + const addExitHandler = await Session.initializeWorkerThread(async () => { + await log_execution({ + startMessage: "\nstarting execution of preliminary functions", + endMessage: "completed preliminary functions\n", + action: preliminaryFunctions + }); + await initializeServer(routeSetter); }); - await initializeServer(routeSetter); - }); -} \ No newline at end of file + addExitHandler(() => Utils.Emit(WebSocket._socket, MessageStore.ConnectionTerminated, "Manual")); + } +})(); \ No newline at end of file diff --git a/src/server/repl.ts b/src/server/repl.ts index ec525582b..a47d4aad4 100644 --- a/src/server/repl.ts +++ b/src/server/repl.ts @@ -1,30 +1,33 @@ import { createInterface, Interface } from "readline"; -import { red } from "colors"; +import { red, green, white } from "colors"; export interface Configuration { identifier: string; onInvalid?: (culprit?: string) => string | string; + onValid?: (success?: string) => string | string; isCaseSensitive?: boolean; } -type Action = (parsedArgs: IterableIterator) => any | Promise; +export type ReplAction = (parsedArgs: Array) => any | Promise; export interface Registration { argPatterns: RegExp[]; - action: Action; + action: ReplAction; } export default class Repl { private identifier: string; - private onInvalid: ((culprit?: string) => string) | string; + private onInvalid: (culprit?: string) => string | string; + private onValid: (success: string) => string | string; private isCaseSensitive: boolean; private commandMap = new Map(); public interface: Interface; private busy = false; private keys: string | undefined; - constructor({ identifier: prompt, onInvalid, isCaseSensitive }: Configuration) { + constructor({ identifier: prompt, onInvalid, onValid, isCaseSensitive }: Configuration) { this.identifier = prompt; this.onInvalid = onInvalid || this.usage; + this.onValid = onValid || this.success; this.isCaseSensitive = isCaseSensitive ?? true; this.interface = createInterface(process.stdin, process.stdout).on('line', this.considerInput); } @@ -43,7 +46,9 @@ export default class Repl { return `${this.identifier} commands: { ${members.sort().join(", ")} }`; } - public registerCommand = (basename: string, argPatterns: (RegExp | string)[], action: Action) => { + private success = (command: string) => `${this.identifier} completed execution of ${white(command)}`; + + public registerCommand = (basename: string, argPatterns: (RegExp | string)[], action: ReplAction) => { const existing = this.commandMap.get(basename); const converted = argPatterns.map(input => input instanceof RegExp ? input : new RegExp(input)); const registration = { argPatterns: converted, action }; @@ -59,7 +64,13 @@ export default class Repl { this.busy = false; } + private valid = (command: string) => { + console.log(green(typeof this.onValid === "string" ? this.onValid : this.onValid(command))); + this.busy = false; + } + private considerInput = async (line: string) => { + console.log("raw", line); if (this.busy) { console.log(red("Busy")); return; @@ -91,8 +102,8 @@ export default class Repl { matched = true; } if (!length || matched) { - await action(parsed[Symbol.iterator]()); - this.busy = false; + await action(parsed); + this.valid(`${command} ${parsed.join(" ")}`); return; } } -- cgit v1.2.3-70-g09d2 From 8d50b63be5dd754d9260adbf69323f3ac2b8cb08 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Sun, 5 Jan 2020 16:03:59 -0800 Subject: default configuration --- session.config.json | 14 ++-- src/server/Session/session.ts | 124 +++++++++++++++++++++------- src/server/Session/session_config_schema.ts | 46 +++++++---- src/server/index.ts | 1 - src/server/repl.ts | 1 - 5 files changed, 132 insertions(+), 54 deletions(-) (limited to 'session.config.json') diff --git a/session.config.json b/session.config.json index ebe17763e..da721efa5 100644 --- a/session.config.json +++ b/session.config.json @@ -1,15 +1,15 @@ { "showServerOutput": false, - "recipients": [ - "samuel_wilkins@brown.edu" - ], "ports": { "server": 1050, "socket": 4321 }, + "email": { + "recipients": [ + "samuel_wilkins@brown.edu" + ], + "signature": "Best,\nDash Server Session Manager" + }, "heartbeatRoute": "/serverHeartbeat", - "pollingIntervalSeconds": 15, - "masterIdentifier": "__master__", - "workerIdentifier": "__worker__", - "signature": "Best,\nServer Session Manager" + "pollingIntervalSeconds": 15 } \ No newline at end of file diff --git a/src/server/Session/session.ts b/src/server/Session/session.ts index 61b8bcf16..15f7e8292 100644 --- a/src/server/Session/session.ts +++ b/src/server/Session/session.ts @@ -24,6 +24,34 @@ const onWindows = process.platform === "win32"; */ export namespace Session { + interface EmailOptions { + recipients: string[]; + signature?: string; + } + + interface Configuration { + showServerOutput: boolean; + masterIdentifier: string; + workerIdentifier: string; + email: EmailOptions | undefined; + ports: { [description: string]: number }; + heartbeatRoute: string; + pollingIntervalSeconds: number; + [key: string]: any; + } + + const defaultConfiguration: Configuration = { + showServerOutput: false, + masterIdentifier: yellow("__monitor__:"), + workerIdentifier: magenta("__server__:"), + email: undefined, + ports: { server: 3000 }, + heartbeatRoute: "/", + pollingIntervalSeconds: 30 + }; + + const defaultSignature = "-Server Session Manager"; + interface MasterCustomizer { addReplCommand: (basename: string, argPatterns: (RegExp | string)[], action: ReplAction) => void; addChildMessageHandler: (message: string, handler: ActionHandler) => void; @@ -42,65 +70,98 @@ export namespace Session { } export type CrashEmailGenerator = (error: Error) => EmailTemplate | Promise; + function defaultEmailGenerator({ name, message, stack }: Error) { + return { + subject: "Server Crash Event", + body: [ + "You are being notified of a server crash event. Here's what we know:", + `name:\n${name}`, + `message:\n${message}`, + `stack:\n${stack}`, + "The server is already restarting itself automatically" + ].join("\n\n") + }; + } + /** * Validates and reads the configuration file, accordingly builds a child process factory * and spawns off an initial process that will respawn as predecessors die. */ - export async function initializeMonitorThread(crashEmailGenerator?: CrashEmailGenerator): Promise { + export async function initializeMonitorThread(custom?: CrashEmailGenerator): Promise { let activeWorker: Worker; const childMessageHandlers: { [message: string]: (action: SessionAction, args: any) => void } = {}; + const crashEmailGenerator = custom || defaultEmailGenerator; // read in configuration .json file only once, in the master thread // pass down any variables the pertinent to the child processes as environment variables - const configuration = function loadConfiguration(): any { + const { + masterIdentifier, + workerIdentifier, + ports, + email, + heartbeatRoute, + showServerOutput, + pollingIntervalSeconds + } = function loadAndValidateConfiguration(): any { try { - const configuration = JSON.parse(readFileSync('./session.config.json', 'utf8')); + const configuration: Configuration = JSON.parse(readFileSync('./session.config.json', 'utf8')); const options = { throwError: true, allowUnknownAttributes: false }; // ensure all necessary and no excess information is specified by the configuration file validate(configuration, configurationSchema, options); - configuration.masterIdentifier = yellow(configuration.masterIdentifier + ":"); - configuration.workerIdentifier = magenta(configuration.workerIdentifier + ":"); + let formatMaster = true; + let formatWorker = true; + Object.keys(defaultConfiguration).forEach(property => { + if (!configuration[property]) { + if (property === "masterIdentifier") { + formatMaster = false; + } else if (property === "workerIdentifier") { + formatWorker = false; + } + configuration[property] = defaultConfiguration[property]; + } + }); + if (formatMaster) { + configuration.masterIdentifier = yellow(configuration.masterIdentifier + ":"); + } + if (formatWorker) { + configuration.workerIdentifier = magenta(configuration.workerIdentifier + ":"); + } return configuration; } catch (error) { - console.log(red("\nSession configuration failed.")); if (error instanceof ValidationError) { + console.log(red("\nSession configuration failed.")); console.log("The given session.config.json configuration file is invalid."); console.log(`${error.instance}: ${error.stack}`); + process.exit(0); } else if (error.code === "ENOENT" && error.path === "./session.config.json") { - console.log("Please include a session.config.json configuration file in your project root."); + console.log(defaultConfiguration.masterIdentifier, "consider including a session.config.json configuration file in your project root."); + return defaultConfiguration; } else { + console.log(red("\nSession configuration failed.")); console.log("The following unknown error occurred during configuration."); console.log(error.stack); + process.exit(0); } - console.log(); - process.exit(0); } }(); - const { - masterIdentifier, - workerIdentifier, - recipients, - ports, - signature, - heartbeatRoute, - showServerOutput, - pollingIntervalSeconds - } = configuration; - // this sends a pseudorandomly generated guid to the configuration's recipients, allowing them alone // to kill the server via the /kill/:key route - const key = Utils.GenerateGuid(); - const timestamp = new Date().toUTCString(); - const content = `The key for this session (started @ ${timestamp}) is ${key}.\n\n${signature}`; - const results = await Email.dispatchAll(recipients, "Server Termination Key", content); - if (results.some(success => !success)) { - console.log(masterIdentifier, red("distribution of session key failed")); - } else { - console.log(masterIdentifier, green("distributed session key to recipients")); + let key: string | undefined; + if (email) { + const { recipients, signature } = email; + key = Utils.GenerateGuid(); + const timestamp = new Date().toUTCString(); + const content = `The key for this session (started @ ${timestamp}) is ${key}.\n\n${signature || defaultSignature}`; + const results = await Email.dispatchAll(recipients, "Server Termination Key", content); + if (results.some(success => !success)) { + console.log(masterIdentifier, red("distribution of session key failed")); + } else { + console.log(masterIdentifier, green("distributed session key to recipients")); + } } // handle exceptions in the master thread - there shouldn't be many of these @@ -167,10 +228,11 @@ export namespace Session { tryKillActiveWorker(false); process.exit(0); case "notify_crash": - if (crashEmailGenerator) { + if (email) { + const { recipients, signature } = email; const { error } = args; const { subject, body } = await crashEmailGenerator(error); - const content = `${body}\n\n${signature}`; + const content = `${body}\n\n${signature || defaultSignature}`; Email.dispatchAll(recipients, subject, content); } case "set_port": @@ -224,7 +286,7 @@ export namespace Session { const exitHandlers: ExitHandler[] = []; // notify master thread (which will log update in the console) of initialization via IPC - process.send?.({ lifecycle: green("initializing...") }); + process.send?.({ lifecycle: green("compiling and initializing...") }); // updates the local value of listening to the value sent from master process.on("message", ({ setListening }) => listening = setListening); diff --git a/src/server/Session/session_config_schema.ts b/src/server/Session/session_config_schema.ts index 34d1ad523..0acb304db 100644 --- a/src/server/Session/session_config_schema.ts +++ b/src/server/Session/session_config_schema.ts @@ -1,17 +1,9 @@ import { Schema } from "jsonschema"; const emailPattern = /^(([a-zA-Z0-9_.-])+@([a-zA-Z0-9_.-])+\.([a-zA-Z])+([a-zA-Z])+)?$/g; -const localPortPattern = /\/[a-zA-Z]+/g; +const localPortPattern = /\/[a-zA-Z]*/g; const properties: { [name: string]: Schema } = { - recipients: { - type: "array", - items: { - type: "string", - pattern: emailPattern - }, - minLength: 1 - }, ports: { type: "object", properties: { @@ -25,16 +17,42 @@ const properties: { [name: string]: Schema } = { type: "string", pattern: localPortPattern }, - signature: { type: "string" }, - masterIdentifier: { type: "string", minLength: 1 }, - workerIdentifier: { type: "string", minLength: 1 }, + email: { + type: "object", + properties: { + recipients: { + type: "array", + items: { + type: "string", + pattern: emailPattern + }, + minLength: 1 + }, + signature: { + type: "string", + minLength: 1 + } + }, + required: ["recipients"] + }, + masterIdentifier: { + type: "string", + minLength: 1 + }, + workerIdentifier: { + type: "string", + minLength: 1 + }, showServerOutput: { type: "boolean" }, - pollingIntervalSeconds: { type: "number", minimum: 1, maximum: 86400 } + pollingIntervalSeconds: { + type: "number", + minimum: 1, + maximum: 86400 + } }; export const configurationSchema: Schema = { id: "/configuration", type: "object", properties, - required: Object.keys(properties) }; \ No newline at end of file diff --git a/src/server/index.ts b/src/server/index.ts index 0fee41bd8..3dc0c739e 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -96,7 +96,6 @@ function routeSetter({ isRelease, addSupervisedRoute, logRegistrationOutcome }: secureHandler: ({ req, res }) => { if (req.params.key === process.env.session_key) { res.send(""); - // setTimeout(() => process.send!({ action: { message: "kill" } }), 1000 * 5); process.send!({ action: { message: "kill" } }); } else { res.redirect("/home"); diff --git a/src/server/repl.ts b/src/server/repl.ts index a47d4aad4..b77fbcefc 100644 --- a/src/server/repl.ts +++ b/src/server/repl.ts @@ -70,7 +70,6 @@ export default class Repl { } private considerInput = async (line: string) => { - console.log("raw", line); if (this.busy) { console.log(red("Busy")); return; -- cgit v1.2.3-70-g09d2 From edf8dc1c042edd126f74e6bc3669bbc52d20d375 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Sun, 5 Jan 2020 19:32:30 -0800 Subject: port constraints, master log --- session.config.json | 2 +- src/server/ApiManagers/SearchManager.ts | 3 +- src/server/Session/session.ts | 183 +++++++++++++++------------- src/server/Session/session_config_schema.ts | 94 +++++++------- src/server/index.ts | 16 +-- 5 files changed, 155 insertions(+), 143 deletions(-) (limited to 'session.config.json') diff --git a/session.config.json b/session.config.json index da721efa5..d8c9e47c8 100644 --- a/session.config.json +++ b/session.config.json @@ -10,6 +10,6 @@ ], "signature": "Best,\nDash Server Session Manager" }, - "heartbeatRoute": "/serverHeartbeat", + "pollingRoute": "/serverHeartbeat", "pollingIntervalSeconds": 15 } \ No newline at end of file diff --git a/src/server/ApiManagers/SearchManager.ts b/src/server/ApiManagers/SearchManager.ts index 75ccfe2a8..c1c908088 100644 --- a/src/server/ApiManagers/SearchManager.ts +++ b/src/server/ApiManagers/SearchManager.ts @@ -8,6 +8,7 @@ import { command_line } from "../ActionUtilities"; import request = require('request-promise'); import { red } from "colors"; import RouteSubscriber from "../RouteSubscriber"; +import { execSync } from "child_process"; export class SearchManager extends ApiManager { @@ -72,7 +73,7 @@ export namespace SolrManager { const args = status ? "start" : "stop -p 8983"; try { console.log(`Solr management: trying to ${args}`); - console.log(await command_line(`./solr.cmd ${args}`, "./solr-8.3.1/bin")); + console.log(execSync(`./solr.cmd ${args}`, { cwd: "./solr-8.3.1/bin" })); return true; } catch (e) { console.log(red(`Solr management error: unable to ${args}`)); diff --git a/src/server/Session/session.ts b/src/server/Session/session.ts index 15f7e8292..c9b49cc73 100644 --- a/src/server/Session/session.ts +++ b/src/server/Session/session.ts @@ -1,4 +1,4 @@ -import { red, cyan, green, yellow, magenta } from "colors"; +import { red, cyan, green, yellow, magenta, blue } from "colors"; import { on, fork, setupMaster, Worker } from "cluster"; import { execSync } from "child_process"; import { get } from "request-promise"; @@ -35,7 +35,7 @@ export namespace Session { workerIdentifier: string; email: EmailOptions | undefined; ports: { [description: string]: number }; - heartbeatRoute: string; + pollingRoute: string; pollingIntervalSeconds: number; [key: string]: any; } @@ -46,7 +46,7 @@ export namespace Session { workerIdentifier: magenta("__server__:"), email: undefined, ports: { server: 3000 }, - heartbeatRoute: "/", + pollingRoute: "/", pollingIntervalSeconds: 30 }; @@ -83,6 +83,57 @@ export namespace Session { }; } + function loadAndValidateConfiguration(): any { + try { + const configuration: Configuration = JSON.parse(readFileSync('./session.config.json', 'utf8')); + const options = { + throwError: true, + allowUnknownAttributes: false + }; + // ensure all necessary and no excess information is specified by the configuration file + validate(configuration, configurationSchema, options); + let formatMaster = true; + let formatWorker = true; + Object.keys(defaultConfiguration).forEach(property => { + if (!configuration[property]) { + if (property === "masterIdentifier") { + formatMaster = false; + } else if (property === "workerIdentifier") { + formatWorker = false; + } + configuration[property] = defaultConfiguration[property]; + } + }); + if (formatMaster) { + configuration.masterIdentifier = yellow(configuration.masterIdentifier + ":"); + } + if (formatWorker) { + configuration.workerIdentifier = magenta(configuration.workerIdentifier + ":"); + } + return configuration; + } catch (error) { + if (error instanceof ValidationError) { + console.log(red("\nSession configuration failed.")); + console.log("The given session.config.json configuration file is invalid."); + console.log(`${error.instance}: ${error.stack}`); + process.exit(0); + } else if (error.code === "ENOENT" && error.path === "./session.config.json") { + console.log(cyan("Loading default session parameters...")); + console.log("Consider including a session.config.json configuration file in your project root for customization."); + return defaultConfiguration; + } else { + console.log(red("\nSession configuration failed.")); + console.log("The following unknown error occurred during configuration."); + console.log(error.stack); + process.exit(0); + } + } + } + + function timestamp() { + return blue(`[${new Date().toUTCString()}]`); + } + /** * Validates and reads the configuration file, accordingly builds a child process factory * and spawns off an initial process that will respawn as predecessors die. @@ -99,54 +150,12 @@ export namespace Session { workerIdentifier, ports, email, - heartbeatRoute, + pollingRoute, showServerOutput, pollingIntervalSeconds - } = function loadAndValidateConfiguration(): any { - try { - const configuration: Configuration = JSON.parse(readFileSync('./session.config.json', 'utf8')); - const options = { - throwError: true, - allowUnknownAttributes: false - }; - // ensure all necessary and no excess information is specified by the configuration file - validate(configuration, configurationSchema, options); - let formatMaster = true; - let formatWorker = true; - Object.keys(defaultConfiguration).forEach(property => { - if (!configuration[property]) { - if (property === "masterIdentifier") { - formatMaster = false; - } else if (property === "workerIdentifier") { - formatWorker = false; - } - configuration[property] = defaultConfiguration[property]; - } - }); - if (formatMaster) { - configuration.masterIdentifier = yellow(configuration.masterIdentifier + ":"); - } - if (formatWorker) { - configuration.workerIdentifier = magenta(configuration.workerIdentifier + ":"); - } - return configuration; - } catch (error) { - if (error instanceof ValidationError) { - console.log(red("\nSession configuration failed.")); - console.log("The given session.config.json configuration file is invalid."); - console.log(`${error.instance}: ${error.stack}`); - process.exit(0); - } else if (error.code === "ENOENT" && error.path === "./session.config.json") { - console.log(defaultConfiguration.masterIdentifier, "consider including a session.config.json configuration file in your project root."); - return defaultConfiguration; - } else { - console.log(red("\nSession configuration failed.")); - console.log("The following unknown error occurred during configuration."); - console.log(error.stack); - process.exit(0); - } - } - }(); + } = loadAndValidateConfiguration(); + + const masterLog = (...optionalParams: any[]) => console.log(timestamp(), masterIdentifier, ...optionalParams); // this sends a pseudorandomly generated guid to the configuration's recipients, allowing them alone // to kill the server via the /kill/:key route @@ -154,14 +163,10 @@ export namespace Session { if (email) { const { recipients, signature } = email; key = Utils.GenerateGuid(); - const timestamp = new Date().toUTCString(); - const content = `The key for this session (started @ ${timestamp}) is ${key}.\n\n${signature || defaultSignature}`; + const content = `The key for this session (started @ ${new Date().toUTCString()}) is ${key}.\n\n${signature || defaultSignature}`; const results = await Email.dispatchAll(recipients, "Server Termination Key", content); - if (results.some(success => !success)) { - console.log(masterIdentifier, red("distribution of session key failed")); - } else { - console.log(masterIdentifier, green("distributed session key to recipients")); - } + const statement = results.some(success => !success) ? red("distribution of session key failed") : green("distributed session key to recipients"); + masterLog(statement); } // handle exceptions in the master thread - there shouldn't be many of these @@ -169,9 +174,9 @@ export namespace Session { // to be caught in a try catch, and is inconsequential, so it is ignored process.on("uncaughtException", ({ message, stack }) => { if (message !== "Channel closed") { - console.log(masterIdentifier, red(message)); + masterLog(red(message)); if (stack) { - console.log(masterIdentifier, `uncaught exception\n${red(stack)}`); + masterLog(`uncaught exception\n${red(stack)}`); } } }); @@ -180,12 +185,12 @@ export namespace Session { setupMaster({ silent: !showServerOutput }); // attempts to kills the active worker ungracefully - const tryKillActiveWorker = (strict = true): boolean => { + const tryKillActiveWorker = (graceful = false): boolean => { if (activeWorker && !activeWorker.isDead()) { - if (strict) { - activeWorker.process.kill(); - } else { + if (graceful) { activeWorker.kill(); + } else { + activeWorker.process.kill(); } return true; } @@ -194,14 +199,18 @@ export namespace Session { const restart = () => { // indicate to the worker that we are 'expecting' this restart - activeWorker.send({ setListening: false }); + activeWorker.send({ setResponsiveness: false }); tryKillActiveWorker(); }; const setPort = (port: string, value: number, immediateRestart: boolean) => { - ports[port] = value; - if (immediateRestart) { - restart(); + if (value >= 1024 && value <= 65535) { + ports[port] = value; + if (immediateRestart) { + restart(); + } + } else { + masterLog(red(`${port} is an invalid port number`)); } }; @@ -210,22 +219,22 @@ export namespace Session { const spawn = (): void => { tryKillActiveWorker(); activeWorker = fork({ - heartbeatRoute, + pollingRoute, serverPort: ports.server, socketPort: ports.socket, pollingIntervalSeconds, session_key: key }); - console.log(masterIdentifier, `spawned new server worker with process id ${activeWorker.process.pid}`); + masterLog(`spawned new server worker with process id ${activeWorker.process.pid}`); // an IPC message handler that executes actions on the master thread when prompted by the active worker activeWorker.on("message", async ({ lifecycle, action }) => { if (action) { const { message, args } = action as SessionAction; - console.log(`${workerIdentifier} action requested (${cyan(message)})`); + console.log(timestamp(), `${workerIdentifier} action requested (${cyan(message)})`); switch (message) { case "kill": - console.log(masterIdentifier, red("An authorized user has manually ended the server session")); - tryKillActiveWorker(false); + masterLog(red("An authorized user has manually ended the server session")); + tryKillActiveWorker(true); process.exit(0); case "notify_crash": if (email) { @@ -233,7 +242,9 @@ export namespace Session { const { error } = args; const { subject, body } = await crashEmailGenerator(error); const content = `${body}\n\n${signature || defaultSignature}`; - Email.dispatchAll(recipients, subject, content); + const results = await Email.dispatchAll(recipients, subject, content); + const statement = results.some(success => !success) ? red("distribution of crash notification failed") : green("distributed crash notification to recipients"); + masterLog(statement); } case "set_port": const { port, value, immediateRestart } = args; @@ -245,7 +256,7 @@ export namespace Session { } } } else if (lifecycle) { - console.log(`${workerIdentifier} lifecycle phase (${lifecycle})`); + console.log(timestamp(), `${workerIdentifier} lifecycle phase (${lifecycle})`); } }); }; @@ -253,7 +264,7 @@ export namespace Session { // a helpful cluster event called on the master thread each time a child process exits on("exit", ({ process: { pid } }, code, signal) => { const prompt = `server worker with process id ${pid} has exited with code ${code}${signal === null ? "" : `, having encountered signal ${signal}`}.`; - console.log(masterIdentifier, cyan(prompt)); + masterLog(cyan(prompt)); // to make this a robust, continuous session, every time a child process dies, we immediately spawn a new one spawn(); }); @@ -282,22 +293,22 @@ export namespace Session { * @param work the function specifying the work to be done by each worker thread */ export async function initializeWorkerThread(work: Function): Promise<(handler: ExitHandler) => void> { - let listening = false; + let shouldServerBeResponsive = false; const exitHandlers: ExitHandler[] = []; // notify master thread (which will log update in the console) of initialization via IPC process.send?.({ lifecycle: green("compiling and initializing...") }); // updates the local value of listening to the value sent from master - process.on("message", ({ setListening }) => listening = setListening); + process.on("message", ({ setResponsiveness }) => shouldServerBeResponsive = setResponsiveness); // called whenever the process has a reason to terminate, either through an uncaught exception // in the process (potentially inconsistent state) or the server cannot be reached const activeExit = async (error: Error): Promise => { - if (!listening) { + if (!shouldServerBeResponsive) { return; } - listening = false; + shouldServerBeResponsive = false; // communicates via IPC to the master thread that it should dispatch a crash notification email process.send?.({ action: { @@ -317,22 +328,22 @@ export namespace Session { const { pollingIntervalSeconds, - heartbeatRoute, + pollingRoute, serverPort } = process.env; // this monitors the health of the server by submitting a get request to whatever port / route specified // by the configuration every n seconds, where n is also given by the configuration. - const heartbeat = `http://localhost:${serverPort}${heartbeatRoute}`; - const checkHeartbeat = async (): Promise => { + const pollTarget = `http://localhost:${serverPort}${pollingRoute}`; + const pollServer = async (): Promise => { await new Promise(resolve => { setTimeout(async () => { try { - await get(heartbeat); - if (!listening) { + await get(pollTarget); + if (!shouldServerBeResponsive) { // notify master thread (which will log update in the console) via IPC that the server is up and running process.send?.({ lifecycle: green(`listening on ${serverPort}...`) }); } - listening = true; + shouldServerBeResponsive = true; resolve(); } catch (error) { // if we expect the server to be unavailable, i.e. during compilation, @@ -343,11 +354,11 @@ export namespace Session { }, 1000 * Number(pollingIntervalSeconds)); }); // controlled, asynchronous infinite recursion achieves a persistent poll that does not submit a new request until the previous has completed - checkHeartbeat(); + pollServer(); }; work(); - checkHeartbeat(); // begin polling + pollServer(); // begin polling return (handler: ExitHandler) => exitHandlers.push(handler); } diff --git a/src/server/Session/session_config_schema.ts b/src/server/Session/session_config_schema.ts index 0acb304db..72b8d388a 100644 --- a/src/server/Session/session_config_schema.ts +++ b/src/server/Session/session_config_schema.ts @@ -1,58 +1,56 @@ import { Schema } from "jsonschema"; const emailPattern = /^(([a-zA-Z0-9_.-])+@([a-zA-Z0-9_.-])+\.([a-zA-Z])+([a-zA-Z])+)?$/g; -const localPortPattern = /\/[a-zA-Z]*/g; +const routePattern = /\/[a-zA-Z]*/g; -const properties: { [name: string]: Schema } = { - ports: { - type: "object", - properties: { - server: { type: "number" }, - socket: { type: "number" } +export const configurationSchema: Schema = { + id: "/configuration", + type: "object", + properties: { + ports: { + type: "object", + properties: { + server: { type: "number", minimum: 1024, maximum: 65535 }, + socket: { type: "number", minimum: 1024, maximum: 65535 } + }, + required: ["server"], + additionalProperties: true }, - required: ["server"], - additionalProperties: true - }, - heartbeatRoute: { - type: "string", - pattern: localPortPattern - }, - email: { - type: "object", - properties: { - recipients: { - type: "array", - items: { - type: "string", - pattern: emailPattern + pollingRoute: { + type: "string", + pattern: routePattern + }, + email: { + type: "object", + properties: { + recipients: { + type: "array", + items: { + type: "string", + pattern: emailPattern + }, + minLength: 1 }, - minLength: 1 + signature: { + type: "string", + minLength: 1 + } }, - signature: { - type: "string", - minLength: 1 - } + required: ["recipients"] }, - required: ["recipients"] - }, - masterIdentifier: { - type: "string", - minLength: 1 - }, - workerIdentifier: { - type: "string", - minLength: 1 - }, - showServerOutput: { type: "boolean" }, - pollingIntervalSeconds: { - type: "number", - minimum: 1, - maximum: 86400 + masterIdentifier: { + type: "string", + minLength: 1 + }, + workerIdentifier: { + type: "string", + minLength: 1 + }, + showServerOutput: { type: "boolean" }, + pollingIntervalSeconds: { + type: "number", + minimum: 1, + maximum: 86400 + } } -}; - -export const configurationSchema: Schema = { - id: "/configuration", - type: "object", - properties, }; \ No newline at end of file diff --git a/src/server/index.ts b/src/server/index.ts index d1480e51e..bd339d65a 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -10,7 +10,7 @@ import initializeServer from './server_Initialization'; import RouteManager, { Method, _success, _permission_denied, _error, _invalid, PublicHandler } from './RouteManager'; import * as qs from 'query-string'; import UtilManager from './ApiManagers/UtilManager'; -import { SearchManager } from './ApiManagers/SearchManager'; +import { SearchManager, SolrManager } from './ApiManagers/SearchManager'; import UserManager from './ApiManagers/UserManager'; import { WebSocket } from './Websocket/Websocket'; import DownloadManager from './ApiManagers/DownloadManager'; @@ -163,14 +163,15 @@ function crashEmailGenerator(error: Error) { } /** - * If on the master thread, launches the monitor for the session. - * Otherwise, the thread must have been spawned *by* the monitor, and thus - * should run the server as a worker. + * If we're the monitor (master) thread, we should launch the monitor logic for the session. + * Otherwise, we must be on a worker thread that was spawned *by* the monitor (master) thread, and thus + * our job should be to run the server. */ async function launchMonitoredSession() { if (isMaster) { const customizer = await Session.initializeMonitorThread(crashEmailGenerator); customizer.addReplCommand("pull", [], () => execSync("git pull", { stdio: ["ignore", "inherit", "inherit"] })); + customizer.addReplCommand("solr", [/start|stop/g], args => SolrManager.SetRunning(args[0] === "start")); } else { const addExitHandler = await Session.initializeWorkerThread(launchServer); // server initialization delegated to worker addExitHandler(() => Utils.Emit(WebSocket._socket, MessageStore.ConnectionTerminated, "Manual")); @@ -178,9 +179,10 @@ async function launchMonitoredSession() { } /** - * Ensures that development mode avoids - * the overhead and lack of default output - * found in a release session. + * If you're in development mode, you won't need to run a session. + * The session spawns off new server processes each time an error is encountered, and doesn't + * log the output of the server process, so it's not ideal for development. + * So, the 'else' clause is exactly what we've always run when executing npm start. */ if (process.env.RELEASE) { launchMonitoredSession(); -- cgit v1.2.3-70-g09d2 From 11000d8959c62772374577fadd9b282c92823a2c Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Mon, 6 Jan 2020 00:37:36 -0800 Subject: factored out email, got rid of global regex --- session.config.json | 6 --- src/server/ActionUtilities.ts | 8 +++- src/server/Session/session.ts | 58 ++++++++--------------------- src/server/Session/session_config_schema.ts | 23 +----------- src/server/index.ts | 43 +++++++++++---------- 5 files changed, 47 insertions(+), 91 deletions(-) (limited to 'session.config.json') diff --git a/session.config.json b/session.config.json index d8c9e47c8..211422868 100644 --- a/session.config.json +++ b/session.config.json @@ -4,12 +4,6 @@ "server": 1050, "socket": 4321 }, - "email": { - "recipients": [ - "samuel_wilkins@brown.edu" - ], - "signature": "Best,\nDash Server Session Manager" - }, "pollingRoute": "/serverHeartbeat", "pollingIntervalSeconds": 15 } \ No newline at end of file diff --git a/src/server/ActionUtilities.ts b/src/server/ActionUtilities.ts index 3125f8683..30aed32e6 100644 --- a/src/server/ActionUtilities.ts +++ b/src/server/ActionUtilities.ts @@ -119,7 +119,13 @@ export namespace Email { }); export async function dispatchAll(recipients: string[], subject: string, content: string) { - return Promise.all(recipients.map((recipient: string) => Email.dispatch(recipient, subject, content))); + const failures: string[] = []; + await Promise.all(recipients.map(async (recipient: string) => { + if (!await Email.dispatch(recipient, subject, content)) { + failures.push(recipient); + } + })); + return failures; } export async function dispatch(recipient: string, subject: string, content: string): Promise { diff --git a/src/server/Session/session.ts b/src/server/Session/session.ts index d07bd13a2..5c74af511 100644 --- a/src/server/Session/session.ts +++ b/src/server/Session/session.ts @@ -3,7 +3,6 @@ import { on, fork, setupMaster, Worker } from "cluster"; import { execSync } from "child_process"; import { get } from "request-promise"; import { Utils } from "../../Utils"; -import { Email } from "../ActionUtilities"; import Repl, { ReplAction } from "../repl"; import { readFileSync } from "fs"; import { validate, ValidationError } from "jsonschema"; @@ -24,16 +23,10 @@ const onWindows = process.platform === "win32"; */ export namespace Session { - interface EmailOptions { - recipients: string[]; - signature?: string; - } - interface Configuration { showServerOutput: boolean; masterIdentifier: string; workerIdentifier: string; - email: EmailOptions | undefined; ports: { [description: string]: number }; pollingRoute: string; pollingIntervalSeconds: number; @@ -44,19 +37,21 @@ export namespace Session { showServerOutput: false, masterIdentifier: yellow("__monitor__:"), workerIdentifier: magenta("__server__:"), - email: undefined, ports: { server: 3000 }, pollingRoute: "/", pollingIntervalSeconds: 30 }; - const defaultSignature = "-Server Session Manager"; - interface MasterCustomizer { addReplCommand: (basename: string, argPatterns: (RegExp | string)[], action: ReplAction) => void; addChildMessageHandler: (message: string, handler: ActionHandler) => void; } + export interface NotifierHooks { + key: (key: string) => boolean | Promise; + crash: (error: Error) => boolean | Promise; + } + export interface SessionAction { message: string; args: any; @@ -68,20 +63,6 @@ export namespace Session { subject: string; body: string; } - export type CrashEmailGenerator = (error: Error) => EmailTemplate | Promise; - - function defaultEmailGenerator({ name, message, stack }: Error) { - return { - subject: "Server Crash Event", - body: [ - "You are being notified of a server crash event. Here's what we know:", - `name:\n${name}`, - `message:\n${message}`, - `stack:\n${stack}`, - "The server is already restarting itself automatically" - ].join("\n\n") - }; - } function loadAndValidateConfiguration(): any { try { @@ -138,10 +119,9 @@ export namespace Session { * Validates and reads the configuration file, accordingly builds a child process factory * and spawns off an initial process that will respawn as predecessors die. */ - export async function initializeMonitorThread(custom?: CrashEmailGenerator): Promise { + export async function initializeMonitorThread(notifiers?: NotifierHooks): Promise { let activeWorker: Worker; const childMessageHandlers: { [message: string]: (action: SessionAction, args: any) => void } = {}; - const crashEmailGenerator = custom || defaultEmailGenerator; // read in configuration .json file only once, in the master thread // pass down any variables the pertinent to the child processes as environment variables @@ -149,7 +129,6 @@ export namespace Session { masterIdentifier, workerIdentifier, ports, - email, pollingRoute, showServerOutput, pollingIntervalSeconds @@ -160,12 +139,10 @@ export namespace Session { // this sends a pseudorandomly generated guid to the configuration's recipients, allowing them alone // to kill the server via the /kill/:key route let key: string | undefined; - if (email) { - const { recipients, signature } = email; + if (notifiers && notifiers.key) { key = Utils.GenerateGuid(); - const content = `The key for this session (started @ ${new Date().toUTCString()}) is ${key}.\n\n${signature || defaultSignature}`; - const results = await Email.dispatchAll(recipients, "Server Termination Key", content); - const statement = results.some(success => !success) ? red("distribution of session key failed") : green("distributed session key to recipients"); + const success = await notifiers.key(key); + const statement = success ? green("distributed session key to recipients") : red("distribution of session key failed"); masterLog(statement); } @@ -233,17 +210,14 @@ export namespace Session { console.log(timestamp(), `${workerIdentifier} action requested (${cyan(message)})`); switch (message) { case "kill": - masterLog(red("An authorized user has manually ended the server session")); + masterLog(red("an authorized user has manually ended the server session")); tryKillActiveWorker(true); process.exit(0); case "notify_crash": - if (email) { - const { recipients, signature } = email; + if (notifiers && notifiers.crash) { const { error } = args; - const { subject, body } = await crashEmailGenerator(error); - const content = `${body}\n\n${signature || defaultSignature}`; - const results = await Email.dispatchAll(recipients, subject, content); - const statement = results.some(success => !success) ? red("distribution of crash notification failed") : green("distributed crash notification to recipients"); + const success = await notifiers.crash(error); + const statement = success ? green("distributed crash notification to recipients") : red("distribution of crash notification failed"); masterLog(statement); } case "set_port": @@ -273,9 +247,7 @@ export namespace Session { const repl = new Repl({ identifier: () => `${timestamp()} ${masterIdentifier}` }); repl.registerCommand("exit", [], () => execSync(onWindows ? "taskkill /f /im node.exe" : "killall -9 node")); repl.registerCommand("restart", [], restart); - repl.registerCommand("set", [/[a-zA-Z]+/g, "port", /\d+/g, /true|false/g], args => { - setPort(args[0], Number(args[2]), args[3] === "true"); - }); + repl.registerCommand("set", [/[a-zA-Z]+/, "port", /\d+/, /true|false/], args => setPort(args[0], Number(args[2]), args[3] === "true")); // finally, set things in motion by spawning off the first child (server) process spawn(); @@ -318,7 +290,7 @@ export namespace Session { }); await Promise.all(exitHandlers.map(handler => handler(error))); // notify master thread (which will log update in the console) of crash event via IPC - process.send?.({ lifecycle: red(`Crash event detected @ ${new Date().toUTCString()}`) }); + process.send?.({ lifecycle: red(`crash event detected @ ${new Date().toUTCString()}`) }); process.send?.({ lifecycle: red(error.message) }); process.exit(1); }; diff --git a/src/server/Session/session_config_schema.ts b/src/server/Session/session_config_schema.ts index 72b8d388a..76af04b9f 100644 --- a/src/server/Session/session_config_schema.ts +++ b/src/server/Session/session_config_schema.ts @@ -1,8 +1,5 @@ import { Schema } from "jsonschema"; -const emailPattern = /^(([a-zA-Z0-9_.-])+@([a-zA-Z0-9_.-])+\.([a-zA-Z])+([a-zA-Z])+)?$/g; -const routePattern = /\/[a-zA-Z]*/g; - export const configurationSchema: Schema = { id: "/configuration", type: "object", @@ -18,25 +15,7 @@ export const configurationSchema: Schema = { }, pollingRoute: { type: "string", - pattern: routePattern - }, - email: { - type: "object", - properties: { - recipients: { - type: "array", - items: { - type: "string", - pattern: emailPattern - }, - minLength: 1 - }, - signature: { - type: "string", - minLength: 1 - } - }, - required: ["recipients"] + pattern: /\/[a-zA-Z]*/g }, masterIdentifier: { type: "string", diff --git a/src/server/index.ts b/src/server/index.ts index bd339d65a..5e411aa3a 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -18,7 +18,7 @@ import { GoogleCredentialsLoader } from './credentials/CredentialsLoader'; import DeleteManager from "./ApiManagers/DeleteManager"; import PDFManager from "./ApiManagers/PDFManager"; import UploadManager from "./ApiManagers/UploadManager"; -import { log_execution } from "./ActionUtilities"; +import { log_execution, Email } from "./ActionUtilities"; import GeneralGoogleManager from "./ApiManagers/GeneralGoogleManager"; import GooglePhotosManager from "./ApiManagers/GooglePhotosManager"; import { Logger } from "./ProcessFactory"; @@ -145,23 +145,6 @@ async function launchServer() { await initializeServer(routeSetter); } -/** - * A function to dictate the format of the message sent on crash - * @param error the error that caused the crash - */ -function crashEmailGenerator(error: Error) { - const subject = "Dash Web Server Crash"; - const { name, message, stack } = error; - const body = [ - "You, as a Dash Administrator, are being notified of a server crash event. Here's what we know:", - `name:\n${name}`, - `message:\n${message}`, - `stack:\n${stack}`, - "The server is already restarting itself, but if you're concerned, use the Remote Desktop Connection to monitor progress.", - ].join("\n\n"); - return { subject, body }; -} - /** * If we're the monitor (master) thread, we should launch the monitor logic for the session. * Otherwise, we must be on a worker thread that was spawned *by* the monitor (master) thread, and thus @@ -169,7 +152,29 @@ function crashEmailGenerator(error: Error) { */ async function launchMonitoredSession() { if (isMaster) { - const customizer = await Session.initializeMonitorThread(crashEmailGenerator); + const recipients = ["samuel_wilkins@brown.edu"]; + const signature = "-Dash Server Session Manager"; + const customizer = await Session.initializeMonitorThread({ + key: async (key: string) => { + const content = `The key for this session (started @ ${new Date().toUTCString()}) is ${key}.\n\n${signature}`; + const failures = await Email.dispatchAll(recipients, "Server Termination Key", content); + return failures.length === 0; + }, + crash: async (error: Error) => { + const subject = "Dash Web Server Crash"; + const { name, message, stack } = error; + const body = [ + "You, as a Dash Administrator, are being notified of a server crash event. Here's what we know:", + `name:\n${name}`, + `message:\n${message}`, + `stack:\n${stack}`, + "The server is already restarting itself, but if you're concerned, use the Remote Desktop Connection to monitor progress.", + ].join("\n\n"); + const content = `${body}\n\n${signature}`; + const failures = await Email.dispatchAll(recipients, subject, content); + return failures.length === 0; + } + }); customizer.addReplCommand("pull", [], () => execSync("git pull", { stdio: ["ignore", "inherit", "inherit"] })); customizer.addReplCommand("solr", [/start|stop/g], args => SolrManager.SetRunning(args[0] === "start")); } else { -- cgit v1.2.3-70-g09d2 From 45f3eba54ffdb2e4571712a13fcf72a359ec5857 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Mon, 6 Jan 2020 15:54:05 -0800 Subject: configurable tolerance, change polling interval from repl --- session.config.json | 3 +- src/server/ActionUtilities.ts | 23 ++++-- src/server/Session/session.ts | 105 ++++++++++++++++++---------- src/server/Session/session_config_schema.ts | 4 ++ src/server/index.ts | 32 +++++---- 5 files changed, 110 insertions(+), 57 deletions(-) (limited to 'session.config.json') diff --git a/session.config.json b/session.config.json index 211422868..57ca9e3cc 100644 --- a/session.config.json +++ b/session.config.json @@ -5,5 +5,6 @@ "socket": 4321 }, "pollingRoute": "/serverHeartbeat", - "pollingIntervalSeconds": 15 + "pollingIntervalSeconds": 15, + "pollingFailureTolerance": 0 } \ No newline at end of file diff --git a/src/server/ActionUtilities.ts b/src/server/ActionUtilities.ts index 30aed32e6..a93566fb1 100644 --- a/src/server/ActionUtilities.ts +++ b/src/server/ActionUtilities.ts @@ -118,25 +118,34 @@ export namespace Email { } }); + export interface DispatchFailure { + recipient: string; + error: Error; + } + export async function dispatchAll(recipients: string[], subject: string, content: string) { - const failures: string[] = []; + const failures: DispatchFailure[] = []; await Promise.all(recipients.map(async (recipient: string) => { - if (!await Email.dispatch(recipient, subject, content)) { - failures.push(recipient); + let error: Error | null; + if ((error = await Email.dispatch(recipient, subject, content)) !== null) { + failures.push({ + recipient, + error + }); } })); - return failures; + return failures.length ? failures : undefined; } - export async function dispatch(recipient: string, subject: string, content: string): Promise { + export async function dispatch(recipient: string, subject: string, content: string): Promise { const mailOptions = { to: recipient, from: 'brownptcdash@gmail.com', subject, text: `Hello ${recipient.split("@")[0]},\n\n${content}` } as MailOptions; - return new Promise(resolve => { - smtpTransport.sendMail(mailOptions, (dispatchError: Error | null) => resolve(dispatchError === null)); + return new Promise(resolve => { + smtpTransport.sendMail(mailOptions, resolve); }); } diff --git a/src/server/Session/session.ts b/src/server/Session/session.ts index cf2231b1f..06f22d855 100644 --- a/src/server/Session/session.ts +++ b/src/server/Session/session.ts @@ -1,6 +1,5 @@ import { red, cyan, green, yellow, magenta, blue } from "colors"; import { on, fork, setupMaster, Worker } from "cluster"; -import { execSync } from "child_process"; import { get } from "request-promise"; import { Utils } from "../../Utils"; import Repl, { ReplAction } from "../repl"; @@ -8,8 +7,6 @@ import { readFileSync } from "fs"; import { validate, ValidationError } from "jsonschema"; import { configurationSchema } from "./session_config_schema"; -const onWindows = process.platform === "win32"; - /** * This namespace relies on NodeJS's cluster module, which allows a parent (master) process to share * code with its children (workers). A simple `isMaster` flag indicates who is trying to access @@ -30,6 +27,7 @@ export namespace Session { ports: { [description: string]: number }; pollingRoute: string; pollingIntervalSeconds: number; + pollingFailureTolerance: number; [key: string]: any; } @@ -39,7 +37,8 @@ export namespace Session { workerIdentifier: magenta("__server__:"), ports: { server: 3000 }, pollingRoute: "/", - pollingIntervalSeconds: 30 + pollingIntervalSeconds: 30, + pollingFailureTolerance: 1 }; interface MasterExtensions { @@ -48,8 +47,8 @@ export namespace Session { } export interface NotifierHooks { - key?: (key: string) => boolean | Promise; - crash?: (error: Error) => boolean | Promise; + key?: (key: string, masterLog: (...optionalParams: any[]) => void) => boolean | Promise; + crash?: (error: Error, masterLog: (...optionalParams: any[]) => void) => boolean | Promise; } export interface SessionAction { @@ -57,14 +56,20 @@ export namespace Session { args: any; } + export interface SessionHooks { + masterLog: (...optionalParams: any[]) => void; + killSession: (graceful?: boolean) => never; + restartServer: () => void; + } + export type ExitHandler = (error: Error) => void | Promise; - export type ActionHandler = (action: SessionAction) => void | Promise; + export type ActionHandler = (action: SessionAction, hooks: SessionHooks) => void | Promise; export interface EmailTemplate { subject: string; body: string; } - function loadAndValidateConfiguration(): any { + function loadAndValidateConfiguration(): Configuration { try { const configuration: Configuration = JSON.parse(readFileSync('./session.config.json', 'utf8')); const options = { @@ -121,7 +126,7 @@ export namespace Session { */ export async function initializeMonitorThread(notifiers?: NotifierHooks): Promise { let activeWorker: Worker; - const childMessageHandlers: { [message: string]: (action: SessionAction, args: any) => void } = {}; + const childMessageHandlers: { [message: string]: ActionHandler } = {}; // read in configuration .json file only once, in the master thread // pass down any variables the pertinent to the child processes as environment variables @@ -131,7 +136,8 @@ export namespace Session { ports, pollingRoute, showServerOutput, - pollingIntervalSeconds + pollingIntervalSeconds, + pollingFailureTolerance } = loadAndValidateConfiguration(); const masterLog = (...optionalParams: any[]) => console.log(timestamp(), masterIdentifier, ...optionalParams); @@ -141,7 +147,7 @@ export namespace Session { let key: string | undefined; if (notifiers && notifiers.key) { key = Utils.GenerateGuid(); - const success = await notifiers.key(key); + const success = await notifiers.key(key, masterLog); const statement = success ? green("distributed session key to recipients") : red("distribution of session key failed"); masterLog(statement); } @@ -161,7 +167,7 @@ export namespace Session { // determines whether or not we see the compilation / initialization / runtime output of each child server process setupMaster({ silent: !showServerOutput }); - // attempts to kills the active worker ungracefully + // attempts to kills the active worker ungracefully, unless otherwise specified const tryKillActiveWorker = (graceful = false): boolean => { if (activeWorker && !activeWorker.isDead()) { if (graceful) { @@ -174,17 +180,22 @@ export namespace Session { return false; }; - const restart = () => { + const restartServer = () => { // indicate to the worker that we are 'expecting' this restart activeWorker.send({ setResponsiveness: false }); - tryKillActiveWorker(); + tryKillActiveWorker(true); + }; + + const killSession = (graceful = true) => { + tryKillActiveWorker(graceful); + process.exit(0); }; const setPort = (port: string, value: number, immediateRestart: boolean) => { if (value > 1023 && value < 65536) { ports[port] = value; if (immediateRestart) { - restart(); + restartServer(); } } else { masterLog(red(`${port} is an invalid port number`)); @@ -197,12 +208,13 @@ export namespace Session { tryKillActiveWorker(); activeWorker = fork({ pollingRoute, + pollingFailureTolerance, serverPort: ports.server, socketPort: ports.socket, pollingIntervalSeconds, session_key: key }); - masterLog(`spawned new server worker with process id ${activeWorker.process.pid}`); + masterLog(cyan(`spawned new server worker with process id ${activeWorker.process.pid}`)); // an IPC message handler that executes actions on the master thread when prompted by the active worker activeWorker.on("message", async ({ lifecycle, action }) => { if (action) { @@ -211,12 +223,11 @@ export namespace Session { switch (message) { case "kill": masterLog(red("an authorized user has manually ended the server session")); - tryKillActiveWorker(true); - process.exit(0); + killSession(); case "notify_crash": if (notifiers && notifiers.crash) { const { error } = args; - const success = await notifiers.crash(error); + const success = await notifiers.crash(error, masterLog); const statement = success ? green("distributed crash notification to recipients") : red("distribution of crash notification failed"); masterLog(statement); } @@ -226,7 +237,7 @@ export namespace Session { default: const handler = childMessageHandlers[message]; if (handler) { - handler(action, args); + handler({ message, args }, { restartServer, killSession, masterLog }); } } } else if (lifecycle) { @@ -245,9 +256,19 @@ export namespace Session { // builds the repl that allows the following commands to be typed into stdin of the master thread const repl = new Repl({ identifier: () => `${timestamp()} ${masterIdentifier}` }); - repl.registerCommand("exit", [], () => execSync(onWindows ? "taskkill /f /im node.exe" : "killall -9 node")); - repl.registerCommand("restart", [], restart); + repl.registerCommand("exit", [/clean|force/], args => killSession(args[0] === "clean")); + repl.registerCommand("restart", [], restartServer); repl.registerCommand("set", [/[a-zA-Z]+/, "port", /\d+/, /true|false/], args => setPort(args[0], Number(args[2]), args[3] === "true")); + repl.registerCommand("set", [/polling/, /interval/, /\d+/], args => { + const newPollingIntervalSeconds = Math.floor(Number(args[2])); + if (newPollingIntervalSeconds < 0) { + masterLog(red("the polling interval must be a non-negative integer")); + } else { + if (newPollingIntervalSeconds !== pollingIntervalSeconds) { + activeWorker.send({ newPollingIntervalSeconds }); + } + } + }); // finally, set things in motion by spawning off the first child (server) process spawn(); @@ -267,19 +288,26 @@ export namespace Session { export async function initializeWorkerThread(work: Function): Promise<(handler: ExitHandler) => void> { let shouldServerBeResponsive = false; const exitHandlers: ExitHandler[] = []; + let pollingFailureCount = 0; + + const lifecycleNotification = (lifecycle: string) => process.send?.({ lifecycle }); // notify master thread (which will log update in the console) of initialization via IPC - process.send?.({ lifecycle: green("compiling and initializing...") }); + lifecycleNotification(green("compiling and initializing...")); // updates the local value of listening to the value sent from master - process.on("message", ({ setResponsiveness }) => shouldServerBeResponsive = setResponsiveness); + process.on("message", ({ setResponsiveness, newPollingIntervalSeconds }) => { + if (setResponsiveness) { + shouldServerBeResponsive = setResponsiveness; + } + if (newPollingIntervalSeconds) { + pollingIntervalSeconds = newPollingIntervalSeconds; + } + }); // called whenever the process has a reason to terminate, either through an uncaught exception // in the process (potentially inconsistent state) or the server cannot be reached const activeExit = async (error: Error): Promise => { - if (!shouldServerBeResponsive) { - return; - } shouldServerBeResponsive = false; // communicates via IPC to the master thread that it should dispatch a crash notification email process.send?.({ @@ -290,19 +318,18 @@ export namespace Session { }); await Promise.all(exitHandlers.map(handler => handler(error))); // notify master thread (which will log update in the console) of crash event via IPC - process.send?.({ lifecycle: red(`crash event detected @ ${new Date().toUTCString()}`) }); - process.send?.({ lifecycle: red(error.message) }); + lifecycleNotification(red(`crash event detected @ ${new Date().toUTCString()}`)); + lifecycleNotification(red(error.message)); process.exit(1); }; // one reason to exit, as the process might be in an inconsistent state after such an exception process.on('uncaughtException', activeExit); - const { - pollingIntervalSeconds, - pollingRoute, - serverPort - } = process.env; + const { env } = process; + const { pollingRoute, serverPort } = env; + let pollingIntervalSeconds = Number(env.pollingIntervalSeconds); + const pollingFailureTolerance = Number(env.pollingFailureTolerance); // this monitors the health of the server by submitting a get request to whatever port / route specified // by the configuration every n seconds, where n is also given by the configuration. const pollTarget = `http://localhost:${serverPort}${pollingRoute}`; @@ -321,9 +348,15 @@ export namespace Session { // if we expect the server to be unavailable, i.e. during compilation, // the listening variable is false, activeExit will return early and the child // process will continue - activeExit(error); + if (shouldServerBeResponsive) { + if (++pollingFailureCount > pollingFailureTolerance) { + activeExit(error); + } else { + lifecycleNotification(yellow(`the server has encountered ${pollingFailureCount} of ${pollingFailureTolerance} tolerable failures`)); + } + } } - }, 1000 * Number(pollingIntervalSeconds)); + }, 1000 * pollingIntervalSeconds); }); // controlled, asynchronous infinite recursion achieves a persistent poll that does not submit a new request until the previous has completed pollServer(); diff --git a/src/server/Session/session_config_schema.ts b/src/server/Session/session_config_schema.ts index 76af04b9f..5a85a45e3 100644 --- a/src/server/Session/session_config_schema.ts +++ b/src/server/Session/session_config_schema.ts @@ -30,6 +30,10 @@ export const configurationSchema: Schema = { type: "number", minimum: 1, maximum: 86400 + }, + pollingFailureTolerance: { + type: "number", + minimum: 0, } } }; \ No newline at end of file diff --git a/src/server/index.ts b/src/server/index.ts index 5e411aa3a..28b2885a1 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -22,7 +22,7 @@ import { log_execution, Email } from "./ActionUtilities"; import GeneralGoogleManager from "./ApiManagers/GeneralGoogleManager"; import GooglePhotosManager from "./ApiManagers/GooglePhotosManager"; import { Logger } from "./ProcessFactory"; -import { yellow } from "colors"; +import { yellow, red } from "colors"; import { Session } from "./Session/session"; import { isMaster } from "cluster"; import { execSync } from "child_process"; @@ -152,17 +152,19 @@ async function launchServer() { */ async function launchMonitoredSession() { if (isMaster) { - const recipients = ["samuel_wilkins@brown.edu"]; + const notificationRecipients = ["samuel_wilkins@brown.edu"]; const signature = "-Dash Server Session Manager"; - const customizer = await Session.initializeMonitorThread({ - key: async (key: string) => { + const extensions = await Session.initializeMonitorThread({ + key: async (key, masterLog) => { const content = `The key for this session (started @ ${new Date().toUTCString()}) is ${key}.\n\n${signature}`; - const failures = await Email.dispatchAll(recipients, "Server Termination Key", content); - return failures.length === 0; + const failures = await Email.dispatchAll(notificationRecipients, "Server Termination Key", content); + if (failures) { + failures.map(({ recipient, error: { message } }) => masterLog(red(`dispatch failure @ ${recipient} (${yellow(message)})`))); + return false; + } + return true; }, - crash: async (error: Error) => { - const subject = "Dash Web Server Crash"; - const { name, message, stack } = error; + crash: async ({ name, message, stack }, masterLog) => { const body = [ "You, as a Dash Administrator, are being notified of a server crash event. Here's what we know:", `name:\n${name}`, @@ -171,12 +173,16 @@ async function launchMonitoredSession() { "The server is already restarting itself, but if you're concerned, use the Remote Desktop Connection to monitor progress.", ].join("\n\n"); const content = `${body}\n\n${signature}`; - const failures = await Email.dispatchAll(recipients, subject, content); - return failures.length === 0; + const failures = await Email.dispatchAll(notificationRecipients, "Dash Web Server Crash", content); + if (failures) { + failures.map(({ recipient, error: { message } }) => masterLog(red(`dispatch failure @ ${recipient} (${yellow(message)})`))); + return false; + } + return true; } }); - customizer.addReplCommand("pull", [], () => execSync("git pull", { stdio: ["ignore", "inherit", "inherit"] })); - customizer.addReplCommand("solr", [/start|stop/g], args => SolrManager.SetRunning(args[0] === "start")); + extensions.addReplCommand("pull", [], () => execSync("git pull", { stdio: ["ignore", "inherit", "inherit"] })); + extensions.addReplCommand("solr", [/start|stop/g], args => SolrManager.SetRunning(args[0] === "start")); } else { const addExitHandler = await Session.initializeWorkerThread(launchServer); // server initialization delegated to worker addExitHandler(() => Utils.Emit(WebSocket._socket, MessageStore.ConnectionTerminated, "Manual")); -- cgit v1.2.3-70-g09d2