1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
|
import * as request from "request-promise";
import { log_execution, pathFromRoot } from "../../ActionUtilities";
import { red, yellow, cyan, green, Color } from "colors";
import * as nodemailer from "nodemailer";
import { MailOptions } from "nodemailer/lib/json-transport";
import { writeFileSync, existsSync, mkdirSync } from "fs";
import { resolve } from 'path';
import { ChildProcess, exec, execSync } from "child_process";
import { createInterface } from "readline";
const killport = require("kill-port");
process.on('SIGINT', endPrevious);
const identifier = yellow("__session_manager__:");
let manualRestartActive = false;
createInterface(process.stdin, process.stdout).on('line', async line => {
const prompt = line.trim().toLowerCase();
switch (prompt) {
case "restart":
manualRestartActive = true;
identifiedLog(cyan("Initializing manual restart..."));
await endPrevious();
break;
case "exit":
identifiedLog(cyan("Initializing session end"));
await endPrevious();
identifiedLog("Cleanup complete. Exiting session...\n");
execSync(killAllCommand());
break;
default:
identifiedLog(red("commands: { exit, restart }"));
return;
}
});
const logPath = resolve(__dirname, "./logs");
const crashPath = resolve(logPath, "./crashes");
if (!existsSync(logPath)) {
mkdirSync(logPath);
}
if (!existsSync(crashPath)) {
mkdirSync(crashPath);
}
const crashLogPath = resolve(crashPath, `./session_crashes_${new Date().toISOString()}.log`);
function addLogEntry(message: string, color: Color) {
const formatted = color(`${message} ${timestamp()}.`);
identifiedLog(formatted);
// appendFileSync(crashLogPath, `${formatted}\n`);
}
function identifiedLog(message?: any, ...optionalParams: any[]) {
console.log(identifier, message, ...optionalParams);
}
if (!["win32", "darwin"].includes(process.platform)) {
identifiedLog(red("Invalid operating system: this script is supported only on Mac and Windows."));
process.exit(1);
}
const latency = 10;
const ports = [1050, 4321];
const onWindows = process.platform === "win32";
const LOCATION = "http://localhost";
const heartbeat = `${LOCATION}:1050/serverHeartbeat`;
const recipient = "samuel_wilkins@brown.edu";
const { pid } = process;
let restarting = false;
let count = 0;
function startServerCommand() {
if (onWindows) {
return '"C:\\Program Files\\Git\\git-bash.exe" -c "npm run start-release"';
}
return `osascript -e 'tell app "Terminal"\ndo script "cd ${pathFromRoot()} && npm run start-release"\nend tell'`;
}
function killAllCommand() {
if (onWindows) {
return "taskkill /f /im node.exe";
}
return "killall -9 node";
}
identifiedLog("Initializing session...");
writeLocalPidLog("session_manager", pid);
function writeLocalPidLog(filename: string, contents: any) {
const path = `./logs/current_${filename}_pid.log`;
identifiedLog(cyan(`${contents} written to ${path}`));
writeFileSync(resolve(__dirname, path), `${contents}\n`);
}
function timestamp() {
return `@ ${new Date().toISOString()}`;
}
async function endPrevious() {
identifiedLog(yellow("Cleaning up previous connections..."));
current_backup?.kill("SIGTERM");
await Promise.all(ports.map(port => {
const task = killport(port, 'tcp');
return task.catch((error: any) => identifiedLog(red(error)));
}));
identifiedLog(yellow("Done. Any failures will be printed in red immediately above."));
}
let current_backup: ChildProcess | undefined = undefined;
async function checkHeartbeat() {
let error: any;
try {
count && !restarting && process.stdout.write(`${identifier} 👂 `);
await request.get(heartbeat);
count && !restarting && console.log('⇠ 💚');
if (restarting || manualRestartActive) {
addLogEntry(count++ ? "Backup server successfully restarted" : "Server successfully started", green);
restarting = false;
}
} catch (e) {
count && !restarting && console.log("⇠ 💔");
error = e;
} finally {
if (error) {
if (!restarting || manualRestartActive) {
restarting = true;
if (count && !manualRestartActive) {
console.log();
addLogEntry("Detected a server crash", red);
identifiedLog(red(error.message));
await endPrevious();
await log_execution({
startMessage: identifier + " Sending crash notification email",
endMessage: ({ error, result }) => {
const success = error === null && result === true;
return identifier + ` ${(success ? `Notification successfully sent to` : `Failed to notify`)} ${recipient} ${timestamp()}`;
},
action: async () => notify(error || "Hmm, no error to report..."),
color: cyan
});
identifiedLog(green("Initiating server restart..."));
}
manualRestartActive = false;
current_backup = exec(startServerCommand(), err => identifiedLog(err?.message || count ? "Previous server process exited." : "Spawned initial server."));
writeLocalPidLog("server", `${(current_backup?.pid ?? -2) + 1} created ${timestamp()}`);
}
}
setTimeout(checkHeartbeat, 1000 * latency);
}
}
async function startListening() {
identifiedLog(yellow(`After initialization, will poll server heartbeat repeatedly...\n`));
if (!LOCATION) {
identifiedLog(red("No location specified for session manager. Please include as a command line environment variable or in a .env file."));
process.exit(0);
}
await checkHeartbeat();
}
function emailText(error: any) {
return [
`Hey ${recipient.split("@")[0]},`,
"You, as a Dash Administrator, are being notified of a server crash event. Here's what we know:",
`Location: ${LOCATION}\nError: ${error}`,
"The server should already be restarting itself, but if you're concerned, use the Remote Desktop Connection to monitor progress."
].join("\n\n");
}
async function notify(error: any) {
const smtpTransport = nodemailer.createTransport({
service: 'Gmail',
auth: {
user: 'brownptcdash@gmail.com',
pass: 'browngfx1'
}
});
const mailOptions = {
to: recipient,
from: 'brownptcdash@gmail.com',
subject: 'Dash Server Crash',
text: emailText(error)
} as MailOptions;
return new Promise<boolean>(resolve => {
smtpTransport.sendMail(mailOptions, (dispatchError: Error | null) => resolve(dispatchError === null));
});
}
startListening();
|
