1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
|
import * as request from "request-promise";
import { log_execution, pathFromRoot } from "../ActionUtilities";
import { red, yellow, cyan, green, Color } from "colors";
import * as nodemailer from "nodemailer";
import { MailOptions } from "nodemailer/lib/json-transport";
import { writeFileSync, existsSync, mkdirSync } from "fs";
import { resolve } from 'path';
import { ChildProcess, exec, execSync } from "child_process";
import InputManager from "./input_manager";
import { identifier, logPath, crashPath, onWindows, pid, ports, heartbeat, recipient, latency, SessionState } from "./config";
const killport = require("kill-port");
import * as io from "socket.io";
process.on('SIGINT', endPrevious);
let state: SessionState = SessionState.STARTING;
const is = (...reference: SessionState[]) => reference.includes(state);
const set = (reference: SessionState) => state = reference;
const endpoint = io();
endpoint.on("connection", socket => {
});
endpoint.listen(process.env.PORT);
const { registerCommand } = new InputManager({ identifier });
registerCommand("restart", [], async () => {
set(SessionState.MANUALLY_RESTARTING);
identifiedLog(cyan("Initializing manual restart..."));
await endPrevious();
});
registerCommand("exit", [], exit);
async function exit() {
set(SessionState.EXITING);
identifiedLog(cyan("Initializing session end"));
await endPrevious();
identifiedLog("Cleanup complete. Exiting session...\n");
execSync(killAllCommand());
}
registerCommand("update", [], async () => {
set(SessionState.UPDATING);
identifiedLog(cyan("Initializing server update from version control..."));
await endPrevious();
await new Promise<void>(resolve => {
exec(updateCommand(), error => {
if (error) {
identifiedLog(red(error.message));
}
resolve();
});
});
await exit();
});
registerCommand("state", [], () => identifiedLog(state));
if (!existsSync(logPath)) {
mkdirSync(logPath);
}
if (!existsSync(crashPath)) {
mkdirSync(crashPath);
}
function addLogEntry(message: string, color: Color) {
const formatted = color(`${message} ${timestamp()}.`);
identifiedLog(formatted);
// appendFileSync(resolve(crashPath, `./session_crashes_${new Date().toISOString()}.log`), `${formatted}\n`);
}
function identifiedLog(message?: any, ...optionalParams: any[]) {
console.log(identifier, message, ...optionalParams);
}
if (!["win32", "darwin"].includes(process.platform)) {
identifiedLog(red("Invalid operating system: this script is supported only on Mac and Windows."));
process.exit(1);
}
const windowsPrepend = (command: string) => `"C:\\Program Files\\Git\\git-bash.exe" -c "${command}"`;
const macPrepend = (command: string) => `osascript -e 'tell app "Terminal"\ndo script "cd ${pathFromRoot()} && ${command}"\nend tell'`;
function updateCommand() {
const command = "git pull && npm install";
if (onWindows) {
return windowsPrepend(command);
}
return macPrepend(command);
}
function startServerCommand() {
const command = "npm run start-release";
if (onWindows) {
return windowsPrepend(command);
}
return macPrepend(command);
}
function killAllCommand() {
if (onWindows) {
return "taskkill /f /im node.exe";
}
return "killall -9 node";
}
identifiedLog("Initializing session...");
writeLocalPidLog("session_manager", pid);
function writeLocalPidLog(filename: string, contents: any) {
const path = `./logs/current_${filename}_pid.log`;
identifiedLog(cyan(`${contents} written to ${path}`));
writeFileSync(resolve(__dirname, path), `${contents}\n`);
}
function timestamp() {
return `@ ${new Date().toISOString()}`;
}
async function endPrevious() {
identifiedLog(yellow("Cleaning up previous connections..."));
current_backup?.kill();
await Promise.all(ports.map(port => {
const task = killport(port, 'tcp');
return task.catch((error: any) => identifiedLog(red(error)));
}));
identifiedLog(yellow("Done. Any failures will be printed in red immediately above."));
}
let current_backup: ChildProcess | undefined = undefined;
async function checkHeartbeat() {
const listening = is(SessionState.LISTENING);
let error: any;
try {
listening && process.stdout.write(`${identifier} 👂 `);
await request.get(heartbeat);
listening && console.log('⇠ 💚');
if (!listening) {
addLogEntry(is(SessionState.INITIALIZED) ? "Server successfully started" : "Backup server successfully restarted", green);
set(SessionState.LISTENING);
}
} catch (e) {
listening && console.log("⇠ 💔\n");
error = e;
} finally {
if (error && !is(SessionState.AUTOMATICALLY_RESTARTING, SessionState.INITIALIZED, SessionState.UPDATING)) {
if (is(SessionState.STARTING)) {
set(SessionState.INITIALIZED);
} else if (is(SessionState.MANUALLY_RESTARTING)) {
set(SessionState.AUTOMATICALLY_RESTARTING);
} else {
set(SessionState.AUTOMATICALLY_RESTARTING);
addLogEntry("Detected a server crash", red);
identifiedLog(red(error.message));
await endPrevious();
await log_execution({
startMessage: identifier + " Sending crash notification email",
endMessage: ({ error, result }) => {
const success = error === null && result === true;
return identifier + ` ${(success ? `Notification successfully sent to` : `Failed to notify`)} ${recipient} ${timestamp()}`;
},
action: async () => notify(error || "Hmm, no error to report..."),
color: cyan
});
identifiedLog(green("Initiating server restart..."));
}
current_backup = exec(startServerCommand(), err => identifiedLog(err?.message || is(SessionState.INITIALIZED) ? "Spawned initial server." : "Previous server process exited."));
writeLocalPidLog("server", `${(current_backup?.pid ?? -2) + 1} created ${timestamp()}`);
}
setTimeout(checkHeartbeat, 1000 * latency);
}
}
function emailText(error: any) {
return [
`Hey ${recipient.split("@")[0]},`,
"You, as a Dash Administrator, are being notified of a server crash event. Here's what we know:",
`Location: ${heartbeat}\nError: ${error}`,
"The server should already be restarting itself, but if you're concerned, use the Remote Desktop Connection to monitor progress."
].join("\n\n");
}
async function notify(error: any) {
const smtpTransport = nodemailer.createTransport({
service: 'Gmail',
auth: {
user: 'brownptcdash@gmail.com',
pass: 'browngfx1'
}
});
const mailOptions = {
to: recipient,
from: 'brownptcdash@gmail.com',
subject: 'Dash Server Crash',
text: emailText(error)
} as MailOptions;
return new Promise<boolean>(resolve => {
smtpTransport.sendMail(mailOptions, (dispatchError: Error | null) => resolve(dispatchError === null));
});
}
identifiedLog(yellow(`After initialization, will poll server heartbeat repeatedly...\n`));
checkHeartbeat();
|
