From b4bf12dd03cdac8c9930ddf5e19e36c87b3696e6 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Sun, 29 Dec 2019 15:35:38 -0800 Subject: experimenting with process clustering --- src/server/ApiManagers/SearchManager.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'src/server/ApiManagers') diff --git a/src/server/ApiManagers/SearchManager.ts b/src/server/ApiManagers/SearchManager.ts index 37d66666b..1ea4d8a50 100644 --- a/src/server/ApiManagers/SearchManager.ts +++ b/src/server/ApiManagers/SearchManager.ts @@ -72,10 +72,11 @@ export namespace SolrManager { const args = status ? "start" : "stop -p 8983"; try { console.log(`Solr management: trying to ${args}`); - console.log(await command_line(`solr.cmd ${args}`, "./solr-8.3.1/bin")); + console.log(await command_line(`./solr.cmd ${args}`, "./solr-8.3.1/bin")); return true; } catch (e) { console.log(red(`Solr management error: unable to ${args}`)); + console.log(e); return false; } } -- cgit v1.2.3-70-g09d2 From e85521e0be77eb01ca34a9346a760c5f7c656a4e Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Thu, 2 Jan 2020 18:33:14 -0800 Subject: connected index to session --- src/server/ApiManagers/DeleteManager.ts | 8 ++--- src/server/ApiManagers/DownloadManager.ts | 6 ++-- src/server/ApiManagers/GeneralGoogleManager.ts | 6 ++-- src/server/ApiManagers/GooglePhotosManager.ts | 4 +-- src/server/ApiManagers/PDFManager.ts | 2 +- src/server/ApiManagers/SearchManager.ts | 6 ++-- src/server/ApiManagers/UploadManager.ts | 8 ++--- src/server/ApiManagers/UserManager.ts | 10 +++---- src/server/ApiManagers/UtilManager.ts | 8 ++--- src/server/Initialization.ts | 2 +- src/server/RouteManager.ts | 16 +++++----- src/server/index.ts | 35 +++++++++++----------- src/server/session.ts | 41 ++++++++++++++++++-------- 13 files changed, 83 insertions(+), 69 deletions(-) (limited to 'src/server/ApiManagers') diff --git a/src/server/ApiManagers/DeleteManager.ts b/src/server/ApiManagers/DeleteManager.ts index 71818c673..88dfa6a64 100644 --- a/src/server/ApiManagers/DeleteManager.ts +++ b/src/server/ApiManagers/DeleteManager.ts @@ -10,7 +10,7 @@ export default class DeleteManager extends ApiManager { register({ method: Method.GET, subscription: "/delete", - onValidation: async ({ res, isRelease }) => { + secureHandler: async ({ res, isRelease }) => { if (isRelease) { return _permission_denied(res, deletionPermissionError); } @@ -22,7 +22,7 @@ export default class DeleteManager extends ApiManager { register({ method: Method.GET, subscription: "/deleteAll", - onValidation: async ({ res, isRelease }) => { + secureHandler: async ({ res, isRelease }) => { if (isRelease) { return _permission_denied(res, deletionPermissionError); } @@ -35,7 +35,7 @@ export default class DeleteManager extends ApiManager { register({ method: Method.GET, subscription: "/deleteWithAux", - onValidation: async ({ res, isRelease }) => { + secureHandler: async ({ res, isRelease }) => { if (isRelease) { return _permission_denied(res, deletionPermissionError); } @@ -47,7 +47,7 @@ export default class DeleteManager extends ApiManager { register({ method: Method.GET, subscription: "/deleteWithGoogleCredentials", - onValidation: async ({ res, isRelease }) => { + secureHandler: async ({ res, isRelease }) => { if (isRelease) { return _permission_denied(res, deletionPermissionError); } diff --git a/src/server/ApiManagers/DownloadManager.ts b/src/server/ApiManagers/DownloadManager.ts index d9808704b..1bb84f374 100644 --- a/src/server/ApiManagers/DownloadManager.ts +++ b/src/server/ApiManagers/DownloadManager.ts @@ -33,7 +33,7 @@ export default class DownloadManager extends ApiManager { register({ method: Method.GET, subscription: new RouteSubscriber("imageHierarchyExport").add('docId'), - onValidation: async ({ req, res }) => { + secureHandler: async ({ req, res }) => { const id = req.params.docId; const hierarchy: Hierarchy = {}; await buildHierarchyRecursive(id, hierarchy); @@ -44,7 +44,7 @@ export default class DownloadManager extends ApiManager { register({ method: Method.GET, subscription: new RouteSubscriber("downloadId").add("docId"), - onValidation: async ({ req, res }) => { + secureHandler: async ({ req, res }) => { return BuildAndDispatchZip(res, async zip => { const { id, docs, files } = await getDocs(req.params.docId); const docString = JSON.stringify({ id, docs }); @@ -59,7 +59,7 @@ export default class DownloadManager extends ApiManager { register({ method: Method.GET, subscription: new RouteSubscriber("serializeDoc").add("docId"), - onValidation: async ({ req, res }) => { + secureHandler: async ({ req, res }) => { const { docs, files } = await getDocs(req.params.docId); res.send({ docs, files: Array.from(files) }); } diff --git a/src/server/ApiManagers/GeneralGoogleManager.ts b/src/server/ApiManagers/GeneralGoogleManager.ts index 3617779d5..a5240edbc 100644 --- a/src/server/ApiManagers/GeneralGoogleManager.ts +++ b/src/server/ApiManagers/GeneralGoogleManager.ts @@ -19,7 +19,7 @@ export default class GeneralGoogleManager extends ApiManager { register({ method: Method.GET, subscription: "/readGoogleAccessToken", - onValidation: async ({ user, res }) => { + secureHandler: async ({ user, res }) => { const token = await GoogleApiServerUtils.retrieveAccessToken(user.id); if (!token) { return res.send(GoogleApiServerUtils.generateAuthenticationUrl()); @@ -31,7 +31,7 @@ export default class GeneralGoogleManager extends ApiManager { register({ method: Method.POST, subscription: "/writeGoogleAccessToken", - onValidation: async ({ user, req, res }) => { + secureHandler: async ({ user, req, res }) => { res.send(await GoogleApiServerUtils.processNewUser(user.id, req.body.authenticationCode)); } }); @@ -39,7 +39,7 @@ export default class GeneralGoogleManager extends ApiManager { register({ method: Method.POST, subscription: new RouteSubscriber("googleDocs").add("sector", "action"), - onValidation: async ({ req, res, user }) => { + secureHandler: async ({ req, res, user }) => { const sector: GoogleApiServerUtils.Service = req.params.sector as GoogleApiServerUtils.Service; const action: GoogleApiServerUtils.Action = req.params.action as GoogleApiServerUtils.Action; const endpoint = await GoogleApiServerUtils.GetEndpoint(GoogleApiServerUtils.Service[sector], user.id); diff --git a/src/server/ApiManagers/GooglePhotosManager.ts b/src/server/ApiManagers/GooglePhotosManager.ts index e2539f120..107542ce2 100644 --- a/src/server/ApiManagers/GooglePhotosManager.ts +++ b/src/server/ApiManagers/GooglePhotosManager.ts @@ -41,7 +41,7 @@ export default class GooglePhotosManager extends ApiManager { register({ method: Method.POST, subscription: "/googlePhotosMediaUpload", - onValidation: async ({ user, req, res }) => { + secureHandler: async ({ user, req, res }) => { const { media } = req.body; const token = await GoogleApiServerUtils.retrieveAccessToken(user.id); if (!token) { @@ -82,7 +82,7 @@ export default class GooglePhotosManager extends ApiManager { register({ method: Method.POST, subscription: "/googlePhotosMediaDownload", - onValidation: async ({ req, res }) => { + secureHandler: async ({ req, res }) => { const contents: { mediaItems: MediaItem[] } = req.body; let failed = 0; if (contents) { diff --git a/src/server/ApiManagers/PDFManager.ts b/src/server/ApiManagers/PDFManager.ts index 7e862631d..0136b758e 100644 --- a/src/server/ApiManagers/PDFManager.ts +++ b/src/server/ApiManagers/PDFManager.ts @@ -17,7 +17,7 @@ export default class PDFManager extends ApiManager { register({ method: Method.GET, subscription: new RouteSubscriber("thumbnail").add("filename"), - onValidation: ({ req, res }) => getOrCreateThumbnail(req.params.filename, res) + secureHandler: ({ req, res }) => getOrCreateThumbnail(req.params.filename, res) }); } diff --git a/src/server/ApiManagers/SearchManager.ts b/src/server/ApiManagers/SearchManager.ts index 1ea4d8a50..75ccfe2a8 100644 --- a/src/server/ApiManagers/SearchManager.ts +++ b/src/server/ApiManagers/SearchManager.ts @@ -16,7 +16,7 @@ export class SearchManager extends ApiManager { register({ method: Method.GET, subscription: new RouteSubscriber("solr").add("action"), - onValidation: async ({ req, res }) => { + secureHandler: async ({ req, res }) => { const { action } = req.params; if (["start", "stop"].includes(action)) { const status = req.params.action === "start"; @@ -30,7 +30,7 @@ export class SearchManager extends ApiManager { register({ method: Method.GET, subscription: "/textsearch", - onValidation: async ({ req, res }) => { + secureHandler: async ({ req, res }) => { const q = req.query.q; if (q === undefined) { res.send([]); @@ -50,7 +50,7 @@ export class SearchManager extends ApiManager { register({ method: Method.GET, subscription: "/search", - onValidation: async ({ req, res }) => { + secureHandler: async ({ req, res }) => { const solrQuery: any = {}; ["q", "fq", "start", "rows", "hl", "hl.fl"].forEach(key => solrQuery[key] = req.query[key]); if (solrQuery.q === undefined) { diff --git a/src/server/ApiManagers/UploadManager.ts b/src/server/ApiManagers/UploadManager.ts index da1f83b75..74f45ae62 100644 --- a/src/server/ApiManagers/UploadManager.ts +++ b/src/server/ApiManagers/UploadManager.ts @@ -41,7 +41,7 @@ export default class UploadManager extends ApiManager { register({ method: Method.POST, subscription: "/upload", - onValidation: async ({ req, res }) => { + secureHandler: async ({ req, res }) => { const form = new formidable.IncomingForm(); form.uploadDir = pathToDirectory(Directory.parsed_files); form.keepExtensions = true; @@ -62,7 +62,7 @@ export default class UploadManager extends ApiManager { register({ method: Method.POST, subscription: "/uploadDoc", - onValidation: ({ req, res }) => { + secureHandler: ({ req, res }) => { const form = new formidable.IncomingForm(); form.keepExtensions = true; // let path = req.body.path; @@ -166,7 +166,7 @@ export default class UploadManager extends ApiManager { register({ method: Method.POST, subscription: "/inspectImage", - onValidation: async ({ req, res }) => { + secureHandler: async ({ req, res }) => { const { source } = req.body; if (typeof source === "string") { const { serverAccessPaths } = await DashUploadUtils.UploadImage(source); @@ -179,7 +179,7 @@ export default class UploadManager extends ApiManager { register({ method: Method.POST, subscription: "/uploadURI", - onValidation: ({ req, res }) => { + secureHandler: ({ req, res }) => { const uri = req.body.uri; const filename = req.body.name; if (!uri || !filename) { diff --git a/src/server/ApiManagers/UserManager.ts b/src/server/ApiManagers/UserManager.ts index 0f7d14320..f2ef22961 100644 --- a/src/server/ApiManagers/UserManager.ts +++ b/src/server/ApiManagers/UserManager.ts @@ -16,7 +16,7 @@ export default class UserManager extends ApiManager { register({ method: Method.GET, subscription: "/getUsers", - onValidation: async ({ res }) => { + secureHandler: async ({ res }) => { const cursor = await Database.Instance.query({}, { email: 1, userDocumentId: 1 }, "users"); const results = await cursor.toArray(); res.send(results.map(user => ({ email: user.email, userDocumentId: user.userDocumentId }))); @@ -26,20 +26,20 @@ export default class UserManager extends ApiManager { register({ method: Method.GET, subscription: "/getUserDocumentId", - onValidation: ({ res, user }) => res.send(user.userDocumentId) + secureHandler: ({ res, user }) => res.send(user.userDocumentId) }); register({ method: Method.GET, subscription: "/getCurrentUser", - onValidation: ({ res, user }) => res.send(JSON.stringify(user)), - onUnauthenticated: ({ res }) => res.send(JSON.stringify({ id: "__guest__", email: "" })) + secureHandler: ({ res, user }) => res.send(JSON.stringify(user)), + publicHandler: ({ res }) => res.send(JSON.stringify({ id: "__guest__", email: "" })) }); register({ method: Method.GET, subscription: "/activity", - onValidation: ({ res }) => { + secureHandler: ({ res }) => { const now = Date.now(); const activeTimes: ActivityUnit[] = []; diff --git a/src/server/ApiManagers/UtilManager.ts b/src/server/ApiManagers/UtilManager.ts index 2f1bd956f..a0d0d0f4b 100644 --- a/src/server/ApiManagers/UtilManager.ts +++ b/src/server/ApiManagers/UtilManager.ts @@ -12,7 +12,7 @@ export default class UtilManager extends ApiManager { register({ method: Method.GET, subscription: new RouteSubscriber("environment").add("key"), - onValidation: ({ req, res }) => { + secureHandler: ({ req, res }) => { const { key } = req.params; const value = process.env[key]; if (!value) { @@ -25,7 +25,7 @@ export default class UtilManager extends ApiManager { register({ method: Method.GET, subscription: "/pull", - onValidation: async ({ res }) => { + secureHandler: async ({ res }) => { return new Promise(resolve => { exec('"C:\\Program Files\\Git\\git-bash.exe" -c "git pull"', err => { if (err) { @@ -42,7 +42,7 @@ export default class UtilManager extends ApiManager { register({ method: Method.GET, subscription: "/buxton", - onValidation: async ({ res }) => { + secureHandler: async ({ res }) => { const cwd = './src/scraping/buxton'; const onResolved = (stdout: string) => { console.log(stdout); res.redirect("/"); }; @@ -56,7 +56,7 @@ export default class UtilManager extends ApiManager { register({ method: Method.GET, subscription: "/version", - onValidation: ({ res }) => { + secureHandler: ({ res }) => { return new Promise(resolve => { exec('"C:\\Program Files\\Git\\bin\\git.exe" rev-parse HEAD', (err, stdout) => { if (err) { diff --git a/src/server/Initialization.ts b/src/server/Initialization.ts index b58bc3e70..465e7ea63 100644 --- a/src/server/Initialization.ts +++ b/src/server/Initialization.ts @@ -18,7 +18,7 @@ import * as whm from 'webpack-hot-middleware'; import * as fs from 'fs'; import * as request from 'request'; import RouteSubscriber from './RouteSubscriber'; -import { publicDirectory, ExitHandlers } from '.'; +import { publicDirectory } from '.'; import { logPort, } from './ActionUtilities'; import { timeMap } from './ApiManagers/UserManager'; import { blue, yellow } from 'colors'; diff --git a/src/server/RouteManager.ts b/src/server/RouteManager.ts index 9e84b3687..25259bd88 100644 --- a/src/server/RouteManager.ts +++ b/src/server/RouteManager.ts @@ -14,16 +14,16 @@ export interface CoreArguments { isRelease: boolean; } -export type OnValidation = (core: CoreArguments & { user: DashUserModel }) => any | Promise; -export type OnUnauthenticated = (core: CoreArguments) => any | Promise; -export type OnError = (core: CoreArguments & { error: any }) => any | Promise; +export type SecureHandler = (core: CoreArguments & { user: DashUserModel }) => any | Promise; +export type PublicHandler = (core: CoreArguments) => any | Promise; +export type ErrorHandler = (core: CoreArguments & { error: any }) => any | Promise; export interface RouteInitializer { method: Method; subscription: string | RouteSubscriber | (string | RouteSubscriber)[]; - onValidation: OnValidation; - onUnauthenticated?: OnUnauthenticated; - onError?: OnError; + secureHandler: SecureHandler; + publicHandler?: PublicHandler; + errorHandler?: ErrorHandler; } const registered = new Map>(); @@ -69,7 +69,7 @@ export default class RouteManager { if (malformedCount) { console.log(`please ensure all routes adhere to ^\/$|^\/[A-Za-z]+(\/\:[A-Za-z]+)*$`); } - process.exit(0); + process.exit(1); } else { console.log(green("all server routes have been successfully registered:")); Array.from(registered.keys()).sort().forEach(route => console.log(cyan(route))); @@ -82,7 +82,7 @@ export default class RouteManager { * @param initializer */ addSupervisedRoute = (initializer: RouteInitializer): void => { - const { method, subscription, onValidation, onUnauthenticated, onError } = initializer; + const { method, subscription, secureHandler: onValidation, publicHandler: onUnauthenticated, errorHandler: onError } = initializer; const isRelease = this._isRelease; const supervised = async (req: express.Request, res: express.Response) => { const { user, originalUrl: target } = req; diff --git a/src/server/index.ts b/src/server/index.ts index c26e0ec19..597198c04 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -7,7 +7,7 @@ const serverPort = 4321; import { DashUploadUtils } from './DashUploadUtils'; import RouteSubscriber from './RouteSubscriber'; import initializeServer from './Initialization'; -import RouteManager, { Method, _success, _permission_denied, _error, _invalid, OnUnauthenticated } from './RouteManager'; +import RouteManager, { Method, _success, _permission_denied, _error, _invalid, PublicHandler } from './RouteManager'; import * as qs from 'query-string'; import UtilManager from './ApiManagers/UtilManager'; import { SearchManager, SolrManager } from './ApiManagers/SearchManager'; @@ -29,8 +29,6 @@ import { Session } from "./session"; export const publicDirectory = path.resolve(__dirname, "public"); export const filesDirectory = path.resolve(publicDirectory, "files"); -export const ExitHandlers = new Array<() => void>(); - /** * These are the functions run before the server starts * listening. Anything that must be complete @@ -80,29 +78,30 @@ function routeSetter({ isRelease, addSupervisedRoute, logRegistrationOutcome }: addSupervisedRoute({ method: Method.GET, subscription: "/", - onValidation: ({ res }) => res.redirect("/home") + secureHandler: ({ res }) => res.redirect("/home") }); addSupervisedRoute({ method: Method.GET, subscription: "/serverHeartbeat", - onValidation: ({ res }) => res.send(true) + secureHandler: ({ res }) => res.send(true) }); addSupervisedRoute({ method: Method.GET, - subscription: "/shutdown", - onValidation: async ({ res }) => { - WebSocket.disconnect(); - await disconnect(); - await Database.disconnect(); - SolrManager.SetRunning(false); - res.send("Server successfully shut down."); - process.exit(0); + subscription: "/kill", + secureHandler: ({ res }) => { + const { send } = process; + if (send) { + res.send("Server successfully killed."); + send({ action: "kill" }); + } else { + res.send("Server worker does not have a viable send protocol. Did not attempt to kill server."); + } } }); - const serve: OnUnauthenticated = ({ req, res }) => { + const serve: PublicHandler = ({ req, res }) => { const detector = new mobileDetect(req.headers['user-agent'] || ""); const filename = detector.mobile() !== null ? 'mobile/image.html' : 'index.html'; res.sendFile(path.join(__dirname, '../../deploy/' + filename)); @@ -111,8 +110,8 @@ function routeSetter({ isRelease, addSupervisedRoute, logRegistrationOutcome }: addSupervisedRoute({ method: Method.GET, subscription: ["/home", new RouteSubscriber("doc").add("docId")], - onValidation: serve, - onUnauthenticated: ({ req, ...remaining }) => { + secureHandler: serve, + publicHandler: ({ req, ...remaining }) => { const { originalUrl: target } = req; const sharing = qs.parse(qs.extract(req.originalUrl), { sort: false }).sharing === "true"; const docAccess = target.startsWith("/doc/"); @@ -129,7 +128,7 @@ function routeSetter({ isRelease, addSupervisedRoute, logRegistrationOutcome }: WebSocket.initialize(serverPort, isRelease); } -async function start() { +async function launch() { await log_execution({ startMessage: "\nstarting execution of preliminary functions", endMessage: "completed preliminary functions\n", @@ -138,4 +137,4 @@ async function start() { await initializeServer({ serverPort: 1050, routeSetter }); } -Session.initialize(start); \ No newline at end of file +Session.initialize(launch); \ No newline at end of file diff --git a/src/server/session.ts b/src/server/session.ts index dd3edccff..1fed9746e 100644 --- a/src/server/session.ts +++ b/src/server/session.ts @@ -18,15 +18,24 @@ export namespace Session { const masterIdentifier = yellow("__master__"); const workerIdentifier = magenta("__worker__"); + function killAll() { + execSync(onWindows ? "taskkill /f /im node.exe" : "killall -9 node"); + } + + function log(message?: any, ...optionalParams: any[]) { + const identifier = `${isMaster ? masterIdentifier : workerIdentifier}:`; + console.log(identifier, message, ...optionalParams); + } + export async function initialize(work: Function) { let listening = false; let active: Worker; if (isMaster) { process.on("uncaughtException", error => { if (error.message !== "Channel closed") { - console.log(`${masterIdentifier}: ${red(error.message)}`); + log(red(error.message)); if (error.stack) { - console.log(`${masterIdentifier}:\n${red(error.stack)}`); + log(`\n${red(error.stack)}`); } } }); @@ -36,30 +45,36 @@ export namespace Session { active.process.kill(); } active = fork(); - active.on("message", ({ update }) => { - if (update) { - console.log(`${workerIdentifier}: ${update}`); + active.on("message", ({ lifecycle, action }) => { + if (action) { + console.log(`${workerIdentifier}: action requested (${action})`); + switch (action) { + case "kill": + log(red("An authorized user has ended the server from the /kill route")); + } + } else if (lifecycle) { + console.log(`${workerIdentifier}: lifecycle phase (${lifecycle})`); } }); }; spawn(); on("exit", ({ process: { pid } }, code, signal) => { const prompt = `Server worker with process id ${pid} has exited with code ${code}${signal === null ? "" : `, having encountered signal ${signal}`}.`; - console.log(`${masterIdentifier}: ${cyan(prompt)}`); + log(cyan(prompt)); spawn(); }); const restart = () => { listening = false; const prompt = `Server worker with process id ${active.process.pid} has been manually killed.`; - console.log(`${masterIdentifier}: ${cyan(prompt)}`); + log(cyan(prompt)); spawn(); }; const { registerCommand } = new InputManager({ identifier }); - registerCommand("exit", [], () => execSync(onWindows ? "taskkill /f /im node.exe" : "killall -9 node")); + registerCommand("exit", [], killAll); registerCommand("restart", [], restart); } else { - const notifyMaster = (update: string) => process.send?.({ update }); - notifyMaster(green("initializing...")); + const logLifecycleEvent = (lifecycle: string) => process.send?.({ lifecycle }); + logLifecycleEvent(green("initializing...")); const activeExit = async (error: Error) => { if (!listening) { return; @@ -70,8 +85,8 @@ export namespace Session { if (_socket) { Utils.Emit(_socket, MessageStore.ConnectionTerminated, "Manual"); } - notifyMaster(red(`Crash event detected @ ${new Date().toUTCString()}`)); - notifyMaster(red(error.message)); + logLifecycleEvent(red(`Crash event detected @ ${new Date().toUTCString()}`)); + logLifecycleEvent(red(error.message)); process.exit(1); }; process.on('uncaughtException', activeExit); @@ -81,7 +96,7 @@ export namespace Session { try { await get(heartbeat); if (!listening) { - notifyMaster(green("server is now successfully listening...")); + logLifecycleEvent(green("server is now successfully listening...")); } listening = true; resolve(); -- cgit v1.2.3-70-g09d2 From edf8dc1c042edd126f74e6bc3669bbc52d20d375 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Sun, 5 Jan 2020 19:32:30 -0800 Subject: port constraints, master log --- session.config.json | 2 +- src/server/ApiManagers/SearchManager.ts | 3 +- src/server/Session/session.ts | 183 +++++++++++++++------------- src/server/Session/session_config_schema.ts | 94 +++++++------- src/server/index.ts | 16 +-- 5 files changed, 155 insertions(+), 143 deletions(-) (limited to 'src/server/ApiManagers') diff --git a/session.config.json b/session.config.json index da721efa5..d8c9e47c8 100644 --- a/session.config.json +++ b/session.config.json @@ -10,6 +10,6 @@ ], "signature": "Best,\nDash Server Session Manager" }, - "heartbeatRoute": "/serverHeartbeat", + "pollingRoute": "/serverHeartbeat", "pollingIntervalSeconds": 15 } \ No newline at end of file diff --git a/src/server/ApiManagers/SearchManager.ts b/src/server/ApiManagers/SearchManager.ts index 75ccfe2a8..c1c908088 100644 --- a/src/server/ApiManagers/SearchManager.ts +++ b/src/server/ApiManagers/SearchManager.ts @@ -8,6 +8,7 @@ import { command_line } from "../ActionUtilities"; import request = require('request-promise'); import { red } from "colors"; import RouteSubscriber from "../RouteSubscriber"; +import { execSync } from "child_process"; export class SearchManager extends ApiManager { @@ -72,7 +73,7 @@ export namespace SolrManager { const args = status ? "start" : "stop -p 8983"; try { console.log(`Solr management: trying to ${args}`); - console.log(await command_line(`./solr.cmd ${args}`, "./solr-8.3.1/bin")); + console.log(execSync(`./solr.cmd ${args}`, { cwd: "./solr-8.3.1/bin" })); return true; } catch (e) { console.log(red(`Solr management error: unable to ${args}`)); diff --git a/src/server/Session/session.ts b/src/server/Session/session.ts index 15f7e8292..c9b49cc73 100644 --- a/src/server/Session/session.ts +++ b/src/server/Session/session.ts @@ -1,4 +1,4 @@ -import { red, cyan, green, yellow, magenta } from "colors"; +import { red, cyan, green, yellow, magenta, blue } from "colors"; import { on, fork, setupMaster, Worker } from "cluster"; import { execSync } from "child_process"; import { get } from "request-promise"; @@ -35,7 +35,7 @@ export namespace Session { workerIdentifier: string; email: EmailOptions | undefined; ports: { [description: string]: number }; - heartbeatRoute: string; + pollingRoute: string; pollingIntervalSeconds: number; [key: string]: any; } @@ -46,7 +46,7 @@ export namespace Session { workerIdentifier: magenta("__server__:"), email: undefined, ports: { server: 3000 }, - heartbeatRoute: "/", + pollingRoute: "/", pollingIntervalSeconds: 30 }; @@ -83,6 +83,57 @@ export namespace Session { }; } + function loadAndValidateConfiguration(): any { + try { + const configuration: Configuration = JSON.parse(readFileSync('./session.config.json', 'utf8')); + const options = { + throwError: true, + allowUnknownAttributes: false + }; + // ensure all necessary and no excess information is specified by the configuration file + validate(configuration, configurationSchema, options); + let formatMaster = true; + let formatWorker = true; + Object.keys(defaultConfiguration).forEach(property => { + if (!configuration[property]) { + if (property === "masterIdentifier") { + formatMaster = false; + } else if (property === "workerIdentifier") { + formatWorker = false; + } + configuration[property] = defaultConfiguration[property]; + } + }); + if (formatMaster) { + configuration.masterIdentifier = yellow(configuration.masterIdentifier + ":"); + } + if (formatWorker) { + configuration.workerIdentifier = magenta(configuration.workerIdentifier + ":"); + } + return configuration; + } catch (error) { + if (error instanceof ValidationError) { + console.log(red("\nSession configuration failed.")); + console.log("The given session.config.json configuration file is invalid."); + console.log(`${error.instance}: ${error.stack}`); + process.exit(0); + } else if (error.code === "ENOENT" && error.path === "./session.config.json") { + console.log(cyan("Loading default session parameters...")); + console.log("Consider including a session.config.json configuration file in your project root for customization."); + return defaultConfiguration; + } else { + console.log(red("\nSession configuration failed.")); + console.log("The following unknown error occurred during configuration."); + console.log(error.stack); + process.exit(0); + } + } + } + + function timestamp() { + return blue(`[${new Date().toUTCString()}]`); + } + /** * Validates and reads the configuration file, accordingly builds a child process factory * and spawns off an initial process that will respawn as predecessors die. @@ -99,54 +150,12 @@ export namespace Session { workerIdentifier, ports, email, - heartbeatRoute, + pollingRoute, showServerOutput, pollingIntervalSeconds - } = function loadAndValidateConfiguration(): any { - try { - const configuration: Configuration = JSON.parse(readFileSync('./session.config.json', 'utf8')); - const options = { - throwError: true, - allowUnknownAttributes: false - }; - // ensure all necessary and no excess information is specified by the configuration file - validate(configuration, configurationSchema, options); - let formatMaster = true; - let formatWorker = true; - Object.keys(defaultConfiguration).forEach(property => { - if (!configuration[property]) { - if (property === "masterIdentifier") { - formatMaster = false; - } else if (property === "workerIdentifier") { - formatWorker = false; - } - configuration[property] = defaultConfiguration[property]; - } - }); - if (formatMaster) { - configuration.masterIdentifier = yellow(configuration.masterIdentifier + ":"); - } - if (formatWorker) { - configuration.workerIdentifier = magenta(configuration.workerIdentifier + ":"); - } - return configuration; - } catch (error) { - if (error instanceof ValidationError) { - console.log(red("\nSession configuration failed.")); - console.log("The given session.config.json configuration file is invalid."); - console.log(`${error.instance}: ${error.stack}`); - process.exit(0); - } else if (error.code === "ENOENT" && error.path === "./session.config.json") { - console.log(defaultConfiguration.masterIdentifier, "consider including a session.config.json configuration file in your project root."); - return defaultConfiguration; - } else { - console.log(red("\nSession configuration failed.")); - console.log("The following unknown error occurred during configuration."); - console.log(error.stack); - process.exit(0); - } - } - }(); + } = loadAndValidateConfiguration(); + + const masterLog = (...optionalParams: any[]) => console.log(timestamp(), masterIdentifier, ...optionalParams); // this sends a pseudorandomly generated guid to the configuration's recipients, allowing them alone // to kill the server via the /kill/:key route @@ -154,14 +163,10 @@ export namespace Session { if (email) { const { recipients, signature } = email; key = Utils.GenerateGuid(); - const timestamp = new Date().toUTCString(); - const content = `The key for this session (started @ ${timestamp}) is ${key}.\n\n${signature || defaultSignature}`; + const content = `The key for this session (started @ ${new Date().toUTCString()}) is ${key}.\n\n${signature || defaultSignature}`; const results = await Email.dispatchAll(recipients, "Server Termination Key", content); - if (results.some(success => !success)) { - console.log(masterIdentifier, red("distribution of session key failed")); - } else { - console.log(masterIdentifier, green("distributed session key to recipients")); - } + const statement = results.some(success => !success) ? red("distribution of session key failed") : green("distributed session key to recipients"); + masterLog(statement); } // handle exceptions in the master thread - there shouldn't be many of these @@ -169,9 +174,9 @@ export namespace Session { // to be caught in a try catch, and is inconsequential, so it is ignored process.on("uncaughtException", ({ message, stack }) => { if (message !== "Channel closed") { - console.log(masterIdentifier, red(message)); + masterLog(red(message)); if (stack) { - console.log(masterIdentifier, `uncaught exception\n${red(stack)}`); + masterLog(`uncaught exception\n${red(stack)}`); } } }); @@ -180,12 +185,12 @@ export namespace Session { setupMaster({ silent: !showServerOutput }); // attempts to kills the active worker ungracefully - const tryKillActiveWorker = (strict = true): boolean => { + const tryKillActiveWorker = (graceful = false): boolean => { if (activeWorker && !activeWorker.isDead()) { - if (strict) { - activeWorker.process.kill(); - } else { + if (graceful) { activeWorker.kill(); + } else { + activeWorker.process.kill(); } return true; } @@ -194,14 +199,18 @@ export namespace Session { const restart = () => { // indicate to the worker that we are 'expecting' this restart - activeWorker.send({ setListening: false }); + activeWorker.send({ setResponsiveness: false }); tryKillActiveWorker(); }; const setPort = (port: string, value: number, immediateRestart: boolean) => { - ports[port] = value; - if (immediateRestart) { - restart(); + if (value >= 1024 && value <= 65535) { + ports[port] = value; + if (immediateRestart) { + restart(); + } + } else { + masterLog(red(`${port} is an invalid port number`)); } }; @@ -210,22 +219,22 @@ export namespace Session { const spawn = (): void => { tryKillActiveWorker(); activeWorker = fork({ - heartbeatRoute, + pollingRoute, serverPort: ports.server, socketPort: ports.socket, pollingIntervalSeconds, session_key: key }); - console.log(masterIdentifier, `spawned new server worker with process id ${activeWorker.process.pid}`); + masterLog(`spawned new server worker with process id ${activeWorker.process.pid}`); // an IPC message handler that executes actions on the master thread when prompted by the active worker activeWorker.on("message", async ({ lifecycle, action }) => { if (action) { const { message, args } = action as SessionAction; - console.log(`${workerIdentifier} action requested (${cyan(message)})`); + console.log(timestamp(), `${workerIdentifier} action requested (${cyan(message)})`); switch (message) { case "kill": - console.log(masterIdentifier, red("An authorized user has manually ended the server session")); - tryKillActiveWorker(false); + masterLog(red("An authorized user has manually ended the server session")); + tryKillActiveWorker(true); process.exit(0); case "notify_crash": if (email) { @@ -233,7 +242,9 @@ export namespace Session { const { error } = args; const { subject, body } = await crashEmailGenerator(error); const content = `${body}\n\n${signature || defaultSignature}`; - Email.dispatchAll(recipients, subject, content); + const results = await Email.dispatchAll(recipients, subject, content); + const statement = results.some(success => !success) ? red("distribution of crash notification failed") : green("distributed crash notification to recipients"); + masterLog(statement); } case "set_port": const { port, value, immediateRestart } = args; @@ -245,7 +256,7 @@ export namespace Session { } } } else if (lifecycle) { - console.log(`${workerIdentifier} lifecycle phase (${lifecycle})`); + console.log(timestamp(), `${workerIdentifier} lifecycle phase (${lifecycle})`); } }); }; @@ -253,7 +264,7 @@ export namespace Session { // a helpful cluster event called on the master thread each time a child process exits on("exit", ({ process: { pid } }, code, signal) => { const prompt = `server worker with process id ${pid} has exited with code ${code}${signal === null ? "" : `, having encountered signal ${signal}`}.`; - console.log(masterIdentifier, cyan(prompt)); + masterLog(cyan(prompt)); // to make this a robust, continuous session, every time a child process dies, we immediately spawn a new one spawn(); }); @@ -282,22 +293,22 @@ export namespace Session { * @param work the function specifying the work to be done by each worker thread */ export async function initializeWorkerThread(work: Function): Promise<(handler: ExitHandler) => void> { - let listening = false; + let shouldServerBeResponsive = false; const exitHandlers: ExitHandler[] = []; // notify master thread (which will log update in the console) of initialization via IPC process.send?.({ lifecycle: green("compiling and initializing...") }); // updates the local value of listening to the value sent from master - process.on("message", ({ setListening }) => listening = setListening); + process.on("message", ({ setResponsiveness }) => shouldServerBeResponsive = setResponsiveness); // called whenever the process has a reason to terminate, either through an uncaught exception // in the process (potentially inconsistent state) or the server cannot be reached const activeExit = async (error: Error): Promise => { - if (!listening) { + if (!shouldServerBeResponsive) { return; } - listening = false; + shouldServerBeResponsive = false; // communicates via IPC to the master thread that it should dispatch a crash notification email process.send?.({ action: { @@ -317,22 +328,22 @@ export namespace Session { const { pollingIntervalSeconds, - heartbeatRoute, + pollingRoute, serverPort } = process.env; // this monitors the health of the server by submitting a get request to whatever port / route specified // by the configuration every n seconds, where n is also given by the configuration. - const heartbeat = `http://localhost:${serverPort}${heartbeatRoute}`; - const checkHeartbeat = async (): Promise => { + const pollTarget = `http://localhost:${serverPort}${pollingRoute}`; + const pollServer = async (): Promise => { await new Promise(resolve => { setTimeout(async () => { try { - await get(heartbeat); - if (!listening) { + await get(pollTarget); + if (!shouldServerBeResponsive) { // notify master thread (which will log update in the console) via IPC that the server is up and running process.send?.({ lifecycle: green(`listening on ${serverPort}...`) }); } - listening = true; + shouldServerBeResponsive = true; resolve(); } catch (error) { // if we expect the server to be unavailable, i.e. during compilation, @@ -343,11 +354,11 @@ export namespace Session { }, 1000 * Number(pollingIntervalSeconds)); }); // controlled, asynchronous infinite recursion achieves a persistent poll that does not submit a new request until the previous has completed - checkHeartbeat(); + pollServer(); }; work(); - checkHeartbeat(); // begin polling + pollServer(); // begin polling return (handler: ExitHandler) => exitHandlers.push(handler); } diff --git a/src/server/Session/session_config_schema.ts b/src/server/Session/session_config_schema.ts index 0acb304db..72b8d388a 100644 --- a/src/server/Session/session_config_schema.ts +++ b/src/server/Session/session_config_schema.ts @@ -1,58 +1,56 @@ import { Schema } from "jsonschema"; const emailPattern = /^(([a-zA-Z0-9_.-])+@([a-zA-Z0-9_.-])+\.([a-zA-Z])+([a-zA-Z])+)?$/g; -const localPortPattern = /\/[a-zA-Z]*/g; +const routePattern = /\/[a-zA-Z]*/g; -const properties: { [name: string]: Schema } = { - ports: { - type: "object", - properties: { - server: { type: "number" }, - socket: { type: "number" } +export const configurationSchema: Schema = { + id: "/configuration", + type: "object", + properties: { + ports: { + type: "object", + properties: { + server: { type: "number", minimum: 1024, maximum: 65535 }, + socket: { type: "number", minimum: 1024, maximum: 65535 } + }, + required: ["server"], + additionalProperties: true }, - required: ["server"], - additionalProperties: true - }, - heartbeatRoute: { - type: "string", - pattern: localPortPattern - }, - email: { - type: "object", - properties: { - recipients: { - type: "array", - items: { - type: "string", - pattern: emailPattern + pollingRoute: { + type: "string", + pattern: routePattern + }, + email: { + type: "object", + properties: { + recipients: { + type: "array", + items: { + type: "string", + pattern: emailPattern + }, + minLength: 1 }, - minLength: 1 + signature: { + type: "string", + minLength: 1 + } }, - signature: { - type: "string", - minLength: 1 - } + required: ["recipients"] }, - required: ["recipients"] - }, - masterIdentifier: { - type: "string", - minLength: 1 - }, - workerIdentifier: { - type: "string", - minLength: 1 - }, - showServerOutput: { type: "boolean" }, - pollingIntervalSeconds: { - type: "number", - minimum: 1, - maximum: 86400 + masterIdentifier: { + type: "string", + minLength: 1 + }, + workerIdentifier: { + type: "string", + minLength: 1 + }, + showServerOutput: { type: "boolean" }, + pollingIntervalSeconds: { + type: "number", + minimum: 1, + maximum: 86400 + } } -}; - -export const configurationSchema: Schema = { - id: "/configuration", - type: "object", - properties, }; \ No newline at end of file diff --git a/src/server/index.ts b/src/server/index.ts index d1480e51e..bd339d65a 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -10,7 +10,7 @@ import initializeServer from './server_Initialization'; import RouteManager, { Method, _success, _permission_denied, _error, _invalid, PublicHandler } from './RouteManager'; import * as qs from 'query-string'; import UtilManager from './ApiManagers/UtilManager'; -import { SearchManager } from './ApiManagers/SearchManager'; +import { SearchManager, SolrManager } from './ApiManagers/SearchManager'; import UserManager from './ApiManagers/UserManager'; import { WebSocket } from './Websocket/Websocket'; import DownloadManager from './ApiManagers/DownloadManager'; @@ -163,14 +163,15 @@ function crashEmailGenerator(error: Error) { } /** - * If on the master thread, launches the monitor for the session. - * Otherwise, the thread must have been spawned *by* the monitor, and thus - * should run the server as a worker. + * If we're the monitor (master) thread, we should launch the monitor logic for the session. + * Otherwise, we must be on a worker thread that was spawned *by* the monitor (master) thread, and thus + * our job should be to run the server. */ async function launchMonitoredSession() { if (isMaster) { const customizer = await Session.initializeMonitorThread(crashEmailGenerator); customizer.addReplCommand("pull", [], () => execSync("git pull", { stdio: ["ignore", "inherit", "inherit"] })); + customizer.addReplCommand("solr", [/start|stop/g], args => SolrManager.SetRunning(args[0] === "start")); } else { const addExitHandler = await Session.initializeWorkerThread(launchServer); // server initialization delegated to worker addExitHandler(() => Utils.Emit(WebSocket._socket, MessageStore.ConnectionTerminated, "Manual")); @@ -178,9 +179,10 @@ async function launchMonitoredSession() { } /** - * Ensures that development mode avoids - * the overhead and lack of default output - * found in a release session. + * If you're in development mode, you won't need to run a session. + * The session spawns off new server processes each time an error is encountered, and doesn't + * log the output of the server process, so it's not ideal for development. + * So, the 'else' clause is exactly what we've always run when executing npm start. */ if (process.env.RELEASE) { launchMonitoredSession(); -- cgit v1.2.3-70-g09d2