1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
|
import * as request from "request-promise";
import { log_execution } from "../../ActionUtilities";
import { red, yellow, cyan, green, Color } from "colors";
import * as nodemailer from "nodemailer";
import { MailOptions } from "nodemailer/lib/json-transport";
import { writeFileSync, appendFileSync, existsSync, mkdirSync } from "fs";
import { resolve } from 'path';
import { ChildProcess, exec } from "child_process";
import * as killport from "kill-port";
const identifier = yellow("__daemon__:");
process.on('SIGINT', () => current_backup?.kill("SIGTERM"));
const logPath = resolve(__dirname, "./logs");
const crashPath = resolve(logPath, "./crashes");
if (!existsSync(logPath)) {
mkdirSync(logPath);
}
if (!existsSync(crashPath)) {
mkdirSync(crashPath);
}
const crashLogPath = resolve(crashPath, `./session_crashes_${new Date().toISOString()}.log`);
function addLogEntry(message: string, color: Color) {
const formatted = color(`${message} ${timestamp()}.`);
identifiedLog(formatted);
// appendFileSync(crashLogPath, `${formatted}\n`);
}
function identifiedLog(message?: any, ...optionalParams: any[]) {
console.log(identifier, message, ...optionalParams);
}
const LOCATION = "http://localhost";
const recipient = "samuel_wilkins@brown.edu";
const frequency = 10;
const { pid } = process;
let restarting = false;
identifiedLog("Initializing daemon...");
writeLocalPidLog("daemon", pid);
function writeLocalPidLog(filename: string, contents: any) {
const path = `./logs/current_${filename}_pid.log`;
identifiedLog(cyan(`${contents} written to ${path}`));
writeFileSync(resolve(__dirname, path), `${contents}\n`);
}
function timestamp() {
return `@ ${new Date().toISOString()}`;
}
let current_backup: ChildProcess | undefined = undefined;
async function listen() {
identifiedLog(yellow(`Beginning to poll server heartbeat every ${frequency} seconds...\n`));
if (!LOCATION) {
identifiedLog(red("No location specified for persistence daemon. Please include as a command line environment variable or in a .env file."));
process.exit(0);
}
const heartbeat = `${LOCATION}:1050/serverHeartbeat`;
// if this is on our remote server, the server must be run in release mode
// const suffix = LOCATION.includes("localhost") ? "" : "-release";
setInterval(async () => {
let error: any;
try {
await request.get(heartbeat);
if (restarting) {
addLogEntry("Backup server successfully restarted", green);
}
restarting = false;
} catch (e) {
error = e;
} finally {
if (error) {
if (!restarting) {
restarting = true;
addLogEntry("Detected a server crash", red);
current_backup?.kill();
await killport(1050, 'tcp');
await log_execution({
startMessage: identifier + " Sending crash notification email",
endMessage: ({ error, result }) => {
const success = error === null && result === true;
return identifier + ` ${(success ? `Notification successfully sent to` : `Failed to notify`)} ${recipient} ${timestamp()}`;
},
action: async () => notify(error || "Hmm, no error to report..."),
color: cyan
});
identifiedLog(green("Initiating server restart..."));
current_backup = exec('"C:\\Program Files\\Git\\git-bash.exe" -c "npm run start-release"', err => identifiedLog(err?.message || "Previous server process exited."));
writeLocalPidLog("server", `${(current_backup?.pid ?? -2) + 1} created ${timestamp()}`);
} else {
identifiedLog(yellow(`Callback ignored because restarting already initiated ${timestamp()}`));
}
}
}
}, 1000 * 10);
}
function emailText(error: any) {
return [
`Hey ${recipient.split("@")[0]},`,
"You, as a Dash Administrator, are being notified of a server crash event. Here's what we know:",
`Location: ${LOCATION}\nError: ${error}`,
"The server should already be restarting itself, but if you're concerned, use the Remote Desktop Connection to monitor progress."
].join("\n\n");
}
async function notify(error: any) {
const smtpTransport = nodemailer.createTransport({
service: 'Gmail',
auth: {
user: 'brownptcdash@gmail.com',
pass: 'browngfx1'
}
});
const mailOptions = {
to: recipient,
from: 'brownptcdash@gmail.com',
subject: 'Dash Server Crash',
text: emailText(error)
} as MailOptions;
return new Promise<boolean>(resolve => {
smtpTransport.sendMail(mailOptions, (dispatchError: Error | null) => resolve(dispatchError === null));
});
}
listen();
|
