aboutsummaryrefslogtreecommitdiff
path: root/src/server/Session/session.ts
diff options
context:
space:
mode:
Diffstat (limited to 'src/server/Session/session.ts')
-rw-r--r--src/server/Session/session.ts116
1 files changed, 78 insertions, 38 deletions
diff --git a/src/server/Session/session.ts b/src/server/Session/session.ts
index cf2231b1f..0c6718803 100644
--- a/src/server/Session/session.ts
+++ b/src/server/Session/session.ts
@@ -1,6 +1,5 @@
import { red, cyan, green, yellow, magenta, blue } from "colors";
import { on, fork, setupMaster, Worker } from "cluster";
-import { execSync } from "child_process";
import { get } from "request-promise";
import { Utils } from "../../Utils";
import Repl, { ReplAction } from "../repl";
@@ -8,8 +7,6 @@ import { readFileSync } from "fs";
import { validate, ValidationError } from "jsonschema";
import { configurationSchema } from "./session_config_schema";
-const onWindows = process.platform === "win32";
-
/**
* This namespace relies on NodeJS's cluster module, which allows a parent (master) process to share
* code with its children (workers). A simple `isMaster` flag indicates who is trying to access
@@ -30,6 +27,7 @@ export namespace Session {
ports: { [description: string]: number };
pollingRoute: string;
pollingIntervalSeconds: number;
+ pollingFailureTolerance: number;
[key: string]: any;
}
@@ -39,7 +37,8 @@ export namespace Session {
workerIdentifier: magenta("__server__:"),
ports: { server: 3000 },
pollingRoute: "/",
- pollingIntervalSeconds: 30
+ pollingIntervalSeconds: 30,
+ pollingFailureTolerance: 1
};
interface MasterExtensions {
@@ -48,8 +47,8 @@ export namespace Session {
}
export interface NotifierHooks {
- key?: (key: string) => boolean | Promise<boolean>;
- crash?: (error: Error) => boolean | Promise<boolean>;
+ key?: (key: string, masterLog: (...optionalParams: any[]) => void) => boolean | Promise<boolean>;
+ crash?: (error: Error, masterLog: (...optionalParams: any[]) => void) => boolean | Promise<boolean>;
}
export interface SessionAction {
@@ -57,14 +56,20 @@ export namespace Session {
args: any;
}
+ export interface SessionHooks {
+ masterLog: (...optionalParams: any[]) => void;
+ killSession: (graceful?: boolean) => never;
+ restartServer: () => void;
+ }
+
export type ExitHandler = (error: Error) => void | Promise<void>;
- export type ActionHandler = (action: SessionAction) => void | Promise<void>;
+ export type ActionHandler = (action: SessionAction, hooks: SessionHooks) => void | Promise<void>;
export interface EmailTemplate {
subject: string;
body: string;
}
- function loadAndValidateConfiguration(): any {
+ function loadAndValidateConfiguration(): Configuration {
try {
const configuration: Configuration = JSON.parse(readFileSync('./session.config.json', 'utf8'));
const options = {
@@ -121,18 +126,20 @@ export namespace Session {
*/
export async function initializeMonitorThread(notifiers?: NotifierHooks): Promise<MasterExtensions> {
let activeWorker: Worker;
- const childMessageHandlers: { [message: string]: (action: SessionAction, args: any) => void } = {};
+ const childMessageHandlers: { [message: string]: ActionHandler } = {};
// read in configuration .json file only once, in the master thread
// pass down any variables the pertinent to the child processes as environment variables
+ const configuration = loadAndValidateConfiguration();
const {
masterIdentifier,
workerIdentifier,
ports,
pollingRoute,
showServerOutput,
- pollingIntervalSeconds
- } = loadAndValidateConfiguration();
+ pollingFailureTolerance
+ } = configuration;
+ let { pollingIntervalSeconds } = configuration;
const masterLog = (...optionalParams: any[]) => console.log(timestamp(), masterIdentifier, ...optionalParams);
@@ -141,7 +148,7 @@ export namespace Session {
let key: string | undefined;
if (notifiers && notifiers.key) {
key = Utils.GenerateGuid();
- const success = await notifiers.key(key);
+ const success = await notifiers.key(key, masterLog);
const statement = success ? green("distributed session key to recipients") : red("distribution of session key failed");
masterLog(statement);
}
@@ -161,7 +168,7 @@ export namespace Session {
// determines whether or not we see the compilation / initialization / runtime output of each child server process
setupMaster({ silent: !showServerOutput });
- // attempts to kills the active worker ungracefully
+ // attempts to kills the active worker ungracefully, unless otherwise specified
const tryKillActiveWorker = (graceful = false): boolean => {
if (activeWorker && !activeWorker.isDead()) {
if (graceful) {
@@ -174,17 +181,22 @@ export namespace Session {
return false;
};
- const restart = () => {
+ const restartServer = () => {
// indicate to the worker that we are 'expecting' this restart
activeWorker.send({ setResponsiveness: false });
- tryKillActiveWorker();
+ tryKillActiveWorker(true);
+ };
+
+ const killSession = (graceful = true) => {
+ tryKillActiveWorker(graceful);
+ process.exit(0);
};
const setPort = (port: string, value: number, immediateRestart: boolean) => {
if (value > 1023 && value < 65536) {
ports[port] = value;
if (immediateRestart) {
- restart();
+ restartServer();
}
} else {
masterLog(red(`${port} is an invalid port number`));
@@ -197,12 +209,13 @@ export namespace Session {
tryKillActiveWorker();
activeWorker = fork({
pollingRoute,
+ pollingFailureTolerance,
serverPort: ports.server,
socketPort: ports.socket,
pollingIntervalSeconds,
session_key: key
});
- masterLog(`spawned new server worker with process id ${activeWorker.process.pid}`);
+ masterLog(cyan(`spawned new server worker with process id ${activeWorker.process.pid}`));
// an IPC message handler that executes actions on the master thread when prompted by the active worker
activeWorker.on("message", async ({ lifecycle, action }) => {
if (action) {
@@ -211,12 +224,11 @@ export namespace Session {
switch (message) {
case "kill":
masterLog(red("an authorized user has manually ended the server session"));
- tryKillActiveWorker(true);
- process.exit(0);
+ killSession();
case "notify_crash":
if (notifiers && notifiers.crash) {
const { error } = args;
- const success = await notifiers.crash(error);
+ const success = await notifiers.crash(error, masterLog);
const statement = success ? green("distributed crash notification to recipients") : red("distribution of crash notification failed");
masterLog(statement);
}
@@ -226,7 +238,7 @@ export namespace Session {
default:
const handler = childMessageHandlers[message];
if (handler) {
- handler(action, args);
+ handler({ message, args }, { restartServer, killSession, masterLog });
}
}
} else if (lifecycle) {
@@ -245,9 +257,25 @@ export namespace Session {
// builds the repl that allows the following commands to be typed into stdin of the master thread
const repl = new Repl({ identifier: () => `${timestamp()} ${masterIdentifier}` });
- repl.registerCommand("exit", [], () => execSync(onWindows ? "taskkill /f /im node.exe" : "killall -9 node"));
- repl.registerCommand("restart", [], restart);
- repl.registerCommand("set", [/[a-zA-Z]+/, "port", /\d+/, /true|false/], args => setPort(args[0], Number(args[2]), args[3] === "true"));
+ const boolean = /true|false/;
+ const number = /\d+/;
+ const letters = /[a-zA-Z]+/;
+ repl.registerCommand("exit", [/clean|force/], args => killSession(args[0] === "clean"));
+ repl.registerCommand("restart", [], restartServer);
+ repl.registerCommand("set", [letters, "port", number, boolean], args => setPort(args[0], Number(args[2]), args[3] === "true"));
+ repl.registerCommand("set", [/polling/, /interval/, number, boolean], args => {
+ const newPollingIntervalSeconds = Math.floor(Number(args[2]));
+ if (newPollingIntervalSeconds < 0) {
+ masterLog(red("the polling interval must be a non-negative integer"));
+ } else {
+ if (newPollingIntervalSeconds !== pollingIntervalSeconds) {
+ pollingIntervalSeconds = newPollingIntervalSeconds;
+ if (args[3] === "true") {
+ activeWorker.send({ newPollingIntervalSeconds });
+ }
+ }
+ }
+ });
// finally, set things in motion by spawning off the first child (server) process
spawn();
@@ -267,19 +295,26 @@ export namespace Session {
export async function initializeWorkerThread(work: Function): Promise<(handler: ExitHandler) => void> {
let shouldServerBeResponsive = false;
const exitHandlers: ExitHandler[] = [];
+ let pollingFailureCount = 0;
+
+ const lifecycleNotification = (lifecycle: string) => process.send?.({ lifecycle });
// notify master thread (which will log update in the console) of initialization via IPC
- process.send?.({ lifecycle: green("compiling and initializing...") });
+ lifecycleNotification(green("compiling and initializing..."));
// updates the local value of listening to the value sent from master
- process.on("message", ({ setResponsiveness }) => shouldServerBeResponsive = setResponsiveness);
+ process.on("message", ({ setResponsiveness, newPollingIntervalSeconds }) => {
+ if (setResponsiveness) {
+ shouldServerBeResponsive = setResponsiveness;
+ }
+ if (newPollingIntervalSeconds) {
+ pollingIntervalSeconds = newPollingIntervalSeconds;
+ }
+ });
// called whenever the process has a reason to terminate, either through an uncaught exception
// in the process (potentially inconsistent state) or the server cannot be reached
const activeExit = async (error: Error): Promise<void> => {
- if (!shouldServerBeResponsive) {
- return;
- }
shouldServerBeResponsive = false;
// communicates via IPC to the master thread that it should dispatch a crash notification email
process.send?.({
@@ -290,19 +325,18 @@ export namespace Session {
});
await Promise.all(exitHandlers.map(handler => handler(error)));
// notify master thread (which will log update in the console) of crash event via IPC
- process.send?.({ lifecycle: red(`crash event detected @ ${new Date().toUTCString()}`) });
- process.send?.({ lifecycle: red(error.message) });
+ lifecycleNotification(red(`crash event detected @ ${new Date().toUTCString()}`));
+ lifecycleNotification(red(error.message));
process.exit(1);
};
// one reason to exit, as the process might be in an inconsistent state after such an exception
process.on('uncaughtException', activeExit);
- const {
- pollingIntervalSeconds,
- pollingRoute,
- serverPort
- } = process.env;
+ const { env } = process;
+ const { pollingRoute, serverPort } = env;
+ let pollingIntervalSeconds = Number(env.pollingIntervalSeconds);
+ const pollingFailureTolerance = Number(env.pollingFailureTolerance);
// this monitors the health of the server by submitting a get request to whatever port / route specified
// by the configuration every n seconds, where n is also given by the configuration.
const pollTarget = `http://localhost:${serverPort}${pollingRoute}`;
@@ -321,9 +355,15 @@ export namespace Session {
// if we expect the server to be unavailable, i.e. during compilation,
// the listening variable is false, activeExit will return early and the child
// process will continue
- activeExit(error);
+ if (shouldServerBeResponsive) {
+ if (++pollingFailureCount > pollingFailureTolerance) {
+ activeExit(error);
+ } else {
+ lifecycleNotification(yellow(`the server has encountered ${pollingFailureCount} of ${pollingFailureTolerance} tolerable failures`));
+ }
+ }
}
- }, 1000 * Number(pollingIntervalSeconds));
+ }, 1000 * pollingIntervalSeconds);
});
// controlled, asynchronous infinite recursion achieves a persistent poll that does not submit a new request until the previous has completed
pollServer();
generated by cgit v1.2.3-70-g09d2 (git 2.46.0) at 2025-10-09 07:46:53 +0000