From 786d25a4f8db1db8795f04a17fba392636e5f891 Mon Sep 17 00:00:00 2001 From: Tyler Schicke Date: Tue, 7 Jan 2020 23:35:14 -0800 Subject: Various features/fixes to allow running on Linux w/o MongoDB or Solr - Added new launch config option for chromium - Changed port for TypeScript server debugger to account for worker process - Updated packages to versions that work with current node/npm - Update IDatabase interface - Updated MemoryDatabase to work properly with Dash - Added some workarounds for in memory database as they currently don't support users, so you must be guest, which means the guest needs to be able to do things it usually can't - Added environment variable to disable search. This doesn't fully disable search yet, but it is enough to not throw major errors when Solr isn't running - Added logic to support using an in memory DB instead of MongoDB --- session.config.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'session.config.json') diff --git a/session.config.json b/session.config.json index 57ca9e3cc..6d3e5f94d 100644 --- a/session.config.json +++ b/session.config.json @@ -7,4 +7,4 @@ "pollingRoute": "/serverHeartbeat", "pollingIntervalSeconds": 15, "pollingFailureTolerance": 0 -} \ No newline at end of file +} -- cgit v1.2.3-70-g09d2 From d8361df45515c9724dcf0400a2d9484118b4cd71 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Wed, 8 Jan 2020 22:04:58 -0500 Subject: configuration assignment improvements, exec log and more granularity for identifiers --- session.config.json | 8 +- .../solr/dash/data/tlog/tlog.0000000000000000014 | Bin 0 -> 56466 bytes src/server/ApiManagers/SearchManager.ts | 16 +- src/server/DashSession.ts | 13 +- src/server/Session/session.ts | 201 ++++++++++++++------- src/server/Session/session_config_schema.ts | 72 +++++--- 6 files changed, 208 insertions(+), 102 deletions(-) create mode 100644 solr-8.3.1/server/solr/dash/data/tlog/tlog.0000000000000000014 (limited to 'session.config.json') diff --git a/session.config.json b/session.config.json index 57ca9e3cc..f613dd904 100644 --- a/session.config.json +++ b/session.config.json @@ -4,7 +4,9 @@ "server": 1050, "socket": 4321 }, - "pollingRoute": "/serverHeartbeat", - "pollingIntervalSeconds": 15, - "pollingFailureTolerance": 0 + "polling": { + "route": "/serverHeartbeat", + "intervalSeconds": 15, + "failureTolerance": 0 + } } \ No newline at end of file diff --git a/solr-8.3.1/server/solr/dash/data/tlog/tlog.0000000000000000014 b/solr-8.3.1/server/solr/dash/data/tlog/tlog.0000000000000000014 new file mode 100644 index 000000000..e39ac337f Binary files /dev/null and b/solr-8.3.1/server/solr/dash/data/tlog/tlog.0000000000000000014 differ diff --git a/src/server/ApiManagers/SearchManager.ts b/src/server/ApiManagers/SearchManager.ts index 316ba09ed..4ce12f9f3 100644 --- a/src/server/ApiManagers/SearchManager.ts +++ b/src/server/ApiManagers/SearchManager.ts @@ -8,6 +8,7 @@ import { red, cyan, yellow } from "colors"; import RouteSubscriber from "../RouteSubscriber"; import { exec } from "child_process"; import { onWindows } from ".."; +import { get } from "request-promise"; export class SearchManager extends ApiManager { @@ -68,18 +69,25 @@ export class SearchManager extends ApiManager { export namespace SolrManager { + const command = onWindows ? "solr.cmd" : "solr"; + export async function SetRunning(status: boolean): Promise { const args = status ? "start" : "stop -p 8983"; - console.log(`Solr management: trying to ${args}`); - exec(`${onWindows ? "solr.cmd" : "solr"} ${args}`, { cwd: "./solr-8.3.1/bin" }, (error, stdout, stderr) => { + console.log(`solr management: trying to ${args}`); + exec(`${command} ${args}`, { cwd: "./solr-8.3.1/bin" }, (error, stdout, stderr) => { if (error) { + console.log(red(`solr management error: unable to ${args} server`)); console.log(red(error.message)); - console.log(red(`Solr management error: unable to ${args}`)); } console.log(cyan(stdout)); console.log(yellow(stderr)); }); - return true; + try { + await get("http://localhost:8983"); + return true; + } catch { + return false; + } } } \ No newline at end of file diff --git a/src/server/DashSession.ts b/src/server/DashSession.ts index 83ce7caaf..47a63c64f 100644 --- a/src/server/DashSession.ts +++ b/src/server/DashSession.ts @@ -1,8 +1,8 @@ import { Session } from "./Session/session"; import { Email } from "./ActionUtilities"; -import { red, yellow } from "colors"; +import { red, yellow, cyan } from "colors"; import { SolrManager } from "./ApiManagers/SearchManager"; -import { execSync } from "child_process"; +import { exec } from "child_process"; import { Utils } from "../Utils"; import { WebSocket } from "./Websocket/Websocket"; import { MessageStore } from "./Message"; @@ -48,7 +48,14 @@ export class DashSessionAgent extends Session.AppliedSessionAgent { return true; } }); - monitor.addReplCommand("pull", [], () => execSync("git pull", { stdio: ["ignore", "inherit", "inherit"] })); + monitor.addReplCommand("pull", [], () => exec("git pull", (error, stdout, stderr) => { + if (error) { + monitor.log(red("unable to pull from version control")); + monitor.log(red(error.message)); + } + stdout.split("\n").forEach(line => line.length && monitor.execLog(cyan(line))); + stderr.split("\n").forEach(line => line.length && monitor.execLog(yellow(line))); + })); monitor.addReplCommand("solr", [/start|stop/], args => SolrManager.SetRunning(args[0] === "start")); return monitor; } diff --git a/src/server/Session/session.ts b/src/server/Session/session.ts index 06a076ae4..9a222b2eb 100644 --- a/src/server/Session/session.ts +++ b/src/server/Session/session.ts @@ -1,4 +1,4 @@ -import { red, cyan, green, yellow, magenta, blue, white } from "colors"; +import { red, cyan, green, yellow, magenta, blue, white, Color, grey, gray, black } from "colors"; import { on, fork, setupMaster, Worker, isMaster, isWorker } from "cluster"; import { get } from "request-promise"; import { Utils } from "../../Utils"; @@ -20,6 +20,20 @@ import { configurationSchema } from "./session_config_schema"; */ export namespace Session { + type ColorLabel = "yellow" | "red" | "cyan" | "green" | "blue" | "magenta" | "grey" | "gray" | "white" | "black"; + const colorMapping: Map = new Map([ + ["yellow", yellow], + ["red", red], + ["cyan", cyan], + ["green", green], + ["blue", blue], + ["magenta", magenta], + ["grey", grey], + ["gray", gray], + ["white", white], + ["black", black] + ]); + export abstract class AppliedSessionAgent { // the following two methods allow the developer to create a custom @@ -70,25 +84,50 @@ export namespace Session { } + interface Identifier { + text: string; + color: ColorLabel; + } + + interface Identifiers { + master: Identifier; + worker: Identifier; + exec: Identifier; + } + interface Configuration { showServerOutput: boolean; - masterIdentifier: string; - workerIdentifier: string; + identifiers: Identifiers; ports: { [description: string]: number }; - pollingRoute: string; - pollingIntervalSeconds: number; - pollingFailureTolerance: number; - [key: string]: any; + polling: { + route: string; + intervalSeconds: number; + failureTolerance: number; + }; } - const defaultConfiguration: Configuration = { + const defaultConfig: Configuration = { showServerOutput: false, - masterIdentifier: yellow("__monitor__:"), - workerIdentifier: magenta("__server__:"), + identifiers: { + master: { + text: "__monitor__", + color: "yellow" + }, + worker: { + text: "__server__", + color: "magenta" + }, + exec: { + text: "__exec__", + color: "green" + } + }, ports: { server: 3000 }, - pollingRoute: "/", - pollingIntervalSeconds: 30, - pollingFailureTolerance: 0 + polling: { + route: "/", + intervalSeconds: 30, + failureTolerance: 0 + } }; export type ExitHandler = (reason: Error | null) => void | Promise; @@ -118,7 +157,7 @@ export namespace Session { private static count = 0; private exitHandlers: ExitHandler[] = []; private readonly notifiers: Monitor.NotifierHooks | undefined; - private readonly configuration: Configuration; + private readonly config: Configuration; private onMessage: { [message: string]: Monitor.ServerMessageHandler[] | undefined } = {}; private activeWorker: Worker | undefined; private key: string | undefined; @@ -209,10 +248,11 @@ export namespace Session { console.log(this.timestamp(), cyan("initializing session...")); - this.configuration = this.loadAndValidateConfiguration(); + this.config = this.loadAndValidateConfiguration(); + this.initializeSessionKey(); // determines whether or not we see the compilation / initialization / runtime output of each child server process - setupMaster({ silent: !this.configuration.showServerOutput }); + setupMaster({ silent: !this.config.showServerOutput }); // handle exceptions in the master thread - there shouldn't be many of these // the IPC (inter process communication) channel closed exception can't seem @@ -238,7 +278,6 @@ export namespace Session { this.spawn(); } - /** * Generates a blue UTC string associated with the time * of invocation. @@ -249,7 +288,14 @@ export namespace Session { * A formatted, identified and timestamped log in color */ public log = (...optionalParams: any[]) => { - console.log(this.timestamp(), this.configuration.masterIdentifier, ...optionalParams); + console.log(this.timestamp(), this.config.identifiers.master.text, ...optionalParams); + } + + /** + * A formatted, identified and timestamped log in color for non- + */ + public execLog = (...optionalParams: any[]) => { + console.log(this.timestamp(), this.config.identifiers.exec.text, ...optionalParams); } /** @@ -269,30 +315,24 @@ export namespace Session { } /** - * Builds the repl that allows the following commands to be typed into stdin of the master thread. + * At any arbitrary layer of nesting within the configuration objects, any single value that + * is not specified by the configuration is given the default counterpart. If, within an object, + * one peer is given by configuration and two are not, the one is preserved while the two are given + * the default value. */ - private initializeRepl = (): Repl => { - const repl = new Repl({ identifier: () => `${this.timestamp()} ${this.configuration.masterIdentifier}` }); - const boolean = /true|false/; - const number = /\d+/; - const letters = /[a-zA-Z]+/; - repl.registerCommand("exit", [/clean|force/], args => this.killSession("manual exit requested by repl", args[0] === "clean", 0)); - repl.registerCommand("restart", [/clean|force/], args => this.tryKillActiveWorker(args[0] === "clean")); - repl.registerCommand("set", [letters, "port", number, boolean], args => this.setPort(args[0], Number(args[2]), args[3] === "true")); - repl.registerCommand("set", [/polling/, number, boolean], args => { - const newPollingIntervalSeconds = Math.floor(Number(args[2])); - if (newPollingIntervalSeconds < 0) { - this.log(red("the polling interval must be a non-negative integer")); - } else { - if (newPollingIntervalSeconds !== this.configuration.pollingIntervalSeconds) { - this.configuration.pollingIntervalSeconds = newPollingIntervalSeconds; - if (args[3] === "true") { - this.activeWorker?.send({ newPollingIntervalSeconds }); - } + private assign = (defaultObject: any, specifiedObject: any, collector: any) => { + Array.from(new Set([...Object.keys(defaultObject), ...Object.keys(specifiedObject)])).map(property => { + let defaultValue: any, specifiedValue: any; + if (specifiedValue = specifiedObject[property]) { + if (typeof specifiedValue === "object" && typeof (defaultValue = defaultObject[property]) === "object") { + this.assign(defaultValue, specifiedValue, collector[property] = {}); + } else { + collector[property] = specifiedValue; } + } else { + collector[property] = defaultObject[property]; } }); - return repl; } /** @@ -300,34 +340,19 @@ export namespace Session { * and pass down any variables the pertinent to the child processes as environment variables. */ private loadAndValidateConfiguration = (): Configuration => { + let config: Configuration; try { console.log(this.timestamp(), cyan("validating configuration...")); - const configuration: Configuration = JSON.parse(readFileSync('./session.config.json', 'utf8')); + config = JSON.parse(readFileSync('./session.config.json', 'utf8')); const options = { throwError: true, allowUnknownAttributes: false }; // ensure all necessary and no excess information is specified by the configuration file - validate(configuration, configurationSchema, options); - let formatMaster = true; - let formatWorker = true; - Object.keys(defaultConfiguration).forEach(property => { - if (!configuration[property]) { - if (property === "masterIdentifier") { - formatMaster = false; - } else if (property === "workerIdentifier") { - formatWorker = false; - } - configuration[property] = defaultConfiguration[property]; - } - }); - if (formatMaster) { - configuration.masterIdentifier = yellow(configuration.masterIdentifier + ":"); - } - if (formatWorker) { - configuration.workerIdentifier = magenta(configuration.workerIdentifier + ":"); - } - return configuration; + validate(config, configurationSchema, options); + const results: any = {}; + this.assign(defaultConfig, config, results); + config = results; } catch (error) { if (error instanceof ValidationError) { console.log(red("\nSession configuration failed.")); @@ -337,16 +362,50 @@ export namespace Session { } else if (error.code === "ENOENT" && error.path === "./session.config.json") { console.log(cyan("Loading default session parameters...")); console.log("Consider including a session.config.json configuration file in your project root for customization."); - return defaultConfiguration; + config = { ...defaultConfig }; } else { console.log(red("\nSession configuration failed.")); console.log("The following unknown error occurred during configuration."); console.log(error.stack); process.exit(0); } + } finally { + const { identifiers } = config!; + Object.keys(identifiers).forEach(key => { + const resolved = key as keyof Identifiers; + const { text, color } = identifiers[resolved]; + identifiers[resolved].text = (colorMapping.get(color) || white)(`${text}:`); + }); + return config!; } } + /** + * Builds the repl that allows the following commands to be typed into stdin of the master thread. + */ + private initializeRepl = (): Repl => { + const repl = new Repl({ identifier: () => `${this.timestamp()} ${this.config.identifiers.master.text}` }); + const boolean = /true|false/; + const number = /\d+/; + const letters = /[a-zA-Z]+/; + repl.registerCommand("exit", [/clean|force/], args => this.killSession("manual exit requested by repl", args[0] === "clean", 0)); + repl.registerCommand("restart", [/clean|force/], args => this.tryKillActiveWorker(args[0] === "clean")); + repl.registerCommand("set", [letters, "port", number, boolean], args => this.setPort(args[0], Number(args[2]), args[3] === "true")); + repl.registerCommand("set", [/polling/, number, boolean], args => { + const newPollingIntervalSeconds = Math.floor(Number(args[2])); + if (newPollingIntervalSeconds < 0) { + this.log(red("the polling interval must be a non-negative integer")); + } else { + if (newPollingIntervalSeconds !== this.config.polling.intervalSeconds) { + this.config.polling.intervalSeconds = newPollingIntervalSeconds; + if (args[3] === "true") { + this.activeWorker?.send({ newPollingIntervalSeconds }); + } + } + } + }); + return repl; + } private executeExitHandlers = async (reason: Error | null) => Promise.all(this.exitHandlers.map(handler => handler(reason))); @@ -374,7 +433,7 @@ export namespace Session { */ private setPort = (port: "server" | "socket" | string, value: number, immediateRestart: boolean): void => { if (value > 1023 && value < 65536) { - this.configuration.ports[port] = value; + this.config.ports[port] = value; if (immediateRestart) { this.tryKillActiveWorker(); } @@ -389,18 +448,20 @@ export namespace Session { */ private spawn = (): void => { const { - pollingRoute, - pollingFailureTolerance, - pollingIntervalSeconds, + polling: { + route, + failureTolerance, + intervalSeconds + }, ports - } = this.configuration; + } = this.config; this.tryKillActiveWorker(); this.activeWorker = fork({ - pollingRoute, - pollingFailureTolerance, + pollingRoute: route, + pollingFailureTolerance: failureTolerance, serverPort: ports.server, socketPort: ports.socket, - pollingIntervalSeconds, + pollingIntervalSeconds: intervalSeconds, session_key: this.key }); this.log(cyan(`spawned new server worker with process id ${this.activeWorker.process.pid}`)); @@ -408,7 +469,7 @@ export namespace Session { this.activeWorker.on("message", async ({ lifecycle, action }) => { if (action) { const { message, args } = action as Monitor.Action; - console.log(this.timestamp(), `${this.configuration.workerIdentifier} action requested (${cyan(message)})`); + console.log(this.timestamp(), `${this.config.identifiers.worker.text} action requested (${cyan(message)})`); switch (message) { case "kill": const { reason, graceful, errorCode } = args; @@ -432,7 +493,7 @@ export namespace Session { handlers.forEach(handler => handler({ message, args })); } } else if (lifecycle) { - console.log(this.timestamp(), `${this.configuration.workerIdentifier} lifecycle phase (${lifecycle})`); + console.log(this.timestamp(), `${this.config.identifiers.worker.text} lifecycle phase (${lifecycle})`); } }); } diff --git a/src/server/Session/session_config_schema.ts b/src/server/Session/session_config_schema.ts index 5a85a45e3..e32cf8c6a 100644 --- a/src/server/Session/session_config_schema.ts +++ b/src/server/Session/session_config_schema.ts @@ -1,39 +1,67 @@ import { Schema } from "jsonschema"; +const colorPattern = /black|red|green|yellow|blue|magenta|cyan|white|gray|grey/; + +const identifierProperties: Schema = { + type: "object", + properties: { + text: { + type: "string", + minLength: 1 + }, + color: { + type: "string", + pattern: colorPattern + } + } +}; + +const portProperties: Schema = { + type: "number", + minimum: 1024, + maximum: 65535 +}; + export const configurationSchema: Schema = { id: "/configuration", type: "object", properties: { + showServerOutput: { type: "boolean" }, ports: { type: "object", properties: { - server: { type: "number", minimum: 1024, maximum: 65535 }, - socket: { type: "number", minimum: 1024, maximum: 65535 } + server: portProperties, + socket: portProperties }, required: ["server"], additionalProperties: true }, - pollingRoute: { - type: "string", - pattern: /\/[a-zA-Z]*/g - }, - masterIdentifier: { - type: "string", - minLength: 1 - }, - workerIdentifier: { - type: "string", - minLength: 1 + identifiers: { + type: "object", + properties: { + master: identifierProperties, + worker: identifierProperties, + exec: identifierProperties + } }, - showServerOutput: { type: "boolean" }, - pollingIntervalSeconds: { - type: "number", - minimum: 1, - maximum: 86400 + polling: { + type: "object", + additionalProperties: false, + properties: { + intervalSeconds: { + type: "number", + minimum: 1, + maximum: 86400 + }, + route: { + type: "string", + pattern: /\/[a-zA-Z]*/g + }, + failureTolerance: { + type: "number", + minimum: 0, + } + } }, - pollingFailureTolerance: { - type: "number", - minimum: 0, - } } }; \ No newline at end of file -- cgit v1.2.3-70-g09d2