fixed sending body of web pages that are not gzip'ed

1 files changed, 76 insertions, 66 deletions

diff --git a/src/server/server_Initialization.ts b/src/server/server_Initialization.ts
index fd000a83c..b0db71f9c 100644
--- a/src/server/server_Initialization.ts
+++ b/src/server/server_Initialization.ts
@@ -1,13 +1,13 @@
 import * as bodyParser from 'body-parser';
 import { blue, yellow } from 'colors';
 import * as cookieParser from 'cookie-parser';
-import * as cors from "cors";
+import * as cors from 'cors';
 import * as express from 'express';
 import * as session from 'express-session';
 import * as expressValidator from 'express-validator';
 import * as fs from 'fs';
-import { Server as HttpServer } from "http";
-import { createServer, Server as HttpsServer } from "https";
+import { Server as HttpServer } from 'http';
+import { createServer, Server as HttpsServer } from 'https';
 import * as passport from 'passport';
 import * as request from 'request';
 import * as webpack from 'webpack';
@@ -33,7 +33,7 @@ const compiler = webpack(config);
 export type RouteSetter = (server: RouteManager) => void;
 //export let disconnect: Function;
 
-export let resolvedPorts: { server: number, socket: number } = { server: 1050, socket: 4321 };
+export let resolvedPorts: { server: number; socket: number } = { server: 1050, socket: 4321 };
 export let resolvedServerUrl: string;
 
 export default async function InitializeServer(routeSetter: RouteSetter) {
@@ -42,33 +42,32 @@ export default async function InitializeServer(routeSetter: RouteSetter) {
 
     const compiler = webpack(config);
 
-    app.use(require("webpack-dev-middleware")(compiler, {
-        publicPath: config.output.publicPath
-    }));
+    app.use(
+        require('webpack-dev-middleware')(compiler, {
+            publicPath: config.output.publicPath,
+        })
+    );
 
-    app.use(require("webpack-hot-middleware")(compiler));
+    app.use(require('webpack-hot-middleware')(compiler));
 
     // route table managed by express.  routes are tested sequentially against each of these map rules.  when a match is found, the handler is called to process the request
-    app.get(new RegExp(/^\/+$/), (req, res) => res.redirect(req.user ? "/home" : "/login")); // target urls that consist of one or more '/'s with nothing in between
-    app.use(express.static(publicDirectory, { setHeaders: res => res.setHeader("Access-Control-Allow-Origin", "*") }));  //all urls that start with dash's public directory: /files/  (e.g., /files/images, /files/audio, etc)
+    app.get(new RegExp(/^\/+$/), (req, res) => res.redirect(req.user ? '/home' : '/login')); // target urls that consist of one or more '/'s with nothing in between
+    app.use(express.static(publicDirectory, { setHeaders: res => res.setHeader('Access-Control-Allow-Origin', '*') })); //all urls that start with dash's public directory: /files/  (e.g., /files/images, /files/audio, etc)
     app.use(cors({ origin: (_origin: any, callback: any) => callback(null, true) }));
     app.use(wdm(compiler, { publicPath: config.output.publicPath }));
     app.use(whm(compiler));
-    registerAuthenticationRoutes(app);  // this adds routes to authenticate a user (login, etc)
-    registerCorsProxy(app);  // this adds a /corsProxy/ route to allow clients to get to urls that would otherwise be blocked by cors policies
+    registerAuthenticationRoutes(app); // this adds routes to authenticate a user (login, etc)
+    registerCorsProxy(app); // this adds a /corsProxy/ route to allow clients to get to urls that would otherwise be blocked by cors policies
     isRelease && !SSL.Loaded && SSL.exit();
     routeSetter(new RouteManager(app, isRelease)); // this sets up all the regular supervised routes (things like /home, download/upload api's, pdf, search, session, etc)
     registerEmbeddedBrowseRelativePathHandler(app); // this allows renered web pages which internally have relative paths to find their content
 
     let server: HttpServer | HttpsServer;
     isRelease && process.env.serverPort && (resolvedPorts.server = Number(process.env.serverPort));
-    await new Promise<void>(resolve => server = isRelease ?
-        createServer(SSL.Credentials, app).listen(resolvedPorts.server, resolve) :
-        app.listen(resolvedPorts.server, resolve)
-    );
-    logPort("server", resolvedPorts.server);
+    await new Promise<void>(resolve => (server = isRelease ? createServer(SSL.Credentials, app).listen(resolvedPorts.server, resolve) : app.listen(resolvedPorts.server, resolve)));
+    logPort('server', resolvedPorts.server);
 
-    resolvedServerUrl = `${isRelease && process.env.serverName ? `https://${process.env.serverName}.com` : "http://localhost"}:${resolvedPorts.server}`;
+    resolvedServerUrl = `${isRelease && process.env.serverName ? `https://${process.env.serverName}.com` : 'http://localhost'}:${resolvedPorts.server}`;
 
     // initialize the web socket (bidirectional communication: if a user changes
     // a field on one client, that change must be broadcast to all other clients)
@@ -79,7 +78,7 @@ export default async function InitializeServer(routeSetter: RouteSetter) {
 }
 
 const week = 7 * 24 * 60 * 60 * 1000;
-const secret = "64d6866242d3b5a5503c675b32c9605e4e90478e9b77bcf2bc";
+const secret = '64d6866242d3b5a5503c675b32c9605e4e90478e9b77bcf2bc';
 
 function buildWithMiddleware(server: express.Express) {
     [
@@ -89,18 +88,18 @@ function buildWithMiddleware(server: express.Express) {
             resave: true,
             cookie: { maxAge: week },
             saveUninitialized: true,
-            store: process.env.DB === "MEM" ? new session.MemoryStore() : new MongoStore({ url: Database.url })
+            store: process.env.DB === 'MEM' ? new session.MemoryStore() : new MongoStore({ url: Database.url }),
         }),
         flash(),
         expressFlash(),
-        bodyParser.json({ limit: "10mb" }),
+        bodyParser.json({ limit: '10mb' }),
         bodyParser.urlencoded({ extended: true }),
         expressValidator(),
         passport.initialize(),
         passport.session(),
         (req: express.Request, res: express.Response, next: express.NextFunction) => {
             res.locals.user = req.user;
-            if ((req.originalUrl.endsWith(".png") /*|| req.originalUrl.endsWith(".js")*/) && req.method === 'GET' && (res as any)._contentLength) {
+            if (req.originalUrl.endsWith('.png') /*|| req.originalUrl.endsWith(".js")*/ && req.method === 'GET' && (res as any)._contentLength) {
                 const period = 30000;
                 res.set('Cache-control', `public, max-age=${period}`);
             } else {
@@ -108,61 +107,61 @@ function buildWithMiddleware(server: express.Express) {
                 res.set('Cache-control', `no-store`);
             }
             next();
-        }
+        },
     ].forEach(next => server.use(next));
     return server;
 }
 
 /* Determine if the enviroment is dev mode or release mode. */
 function determineEnvironment() {
-    const isRelease = process.env.RELEASE === "true";
+    const isRelease = process.env.RELEASE === 'true';
 
     const color = isRelease ? blue : yellow;
-    const label = isRelease ? "release" : "development";
+    const label = isRelease ? 'release' : 'development';
     console.log(`\nrunning server in ${color(label)} mode`);
 
     // swilkins: I don't think we need to read from ClientUtils.RELEASE anymore. Should be able to invoke process.env.RELEASE
     // on the client side, thanks to dotenv in webpack.config.js
-    let clientUtils = fs.readFileSync("./src/client/util/ClientUtils.ts.temp", "utf8");
+    let clientUtils = fs.readFileSync('./src/client/util/ClientUtils.ts.temp', 'utf8');
     clientUtils = `//AUTO-GENERATED FILE: DO NOT EDIT\n${clientUtils.replace('"mode"', String(isRelease))}`;
-    fs.writeFileSync("./src/client/util/ClientUtils.ts", clientUtils, "utf8");
+    fs.writeFileSync('./src/client/util/ClientUtils.ts', clientUtils, 'utf8');
 
     return isRelease;
 }
 
 function registerAuthenticationRoutes(server: express.Express) {
-    server.get("/signup", getSignup);
-    server.post("/signup", postSignup);
+    server.get('/signup', getSignup);
+    server.post('/signup', postSignup);
 
-    server.get("/login", getLogin);
-    server.post("/login", postLogin);
+    server.get('/login', getLogin);
+    server.post('/login', postLogin);
 
-    server.get("/logout", getLogout);
+    server.get('/logout', getLogout);
 
-    server.get("/forgotPassword", getForgot);
-    server.post("/forgotPassword", postForgot);
+    server.get('/forgotPassword', getForgot);
+    server.post('/forgotPassword', postForgot);
 
-    const reset = new RouteSubscriber("resetPassword").add("token").build;
+    const reset = new RouteSubscriber('resetPassword').add('token').build;
     server.get(reset, getReset);
     server.post(reset, postReset);
 }
 
 function registerCorsProxy(server: express.Express) {
-    server.use("/corsProxy", async (req, res) => {
-        const referer = req.headers.referer ? decodeURIComponent(req.headers.referer) : "";
+    server.use('/corsProxy', async (req, res) => {
+        const referer = req.headers.referer ? decodeURIComponent(req.headers.referer) : '';
         let requrlraw = decodeURIComponent(req.url.substring(1));
-        const qsplit = requrlraw.split("?q=");
-        const newqsplit = requrlraw.split("&q=");
+        const qsplit = requrlraw.split('?q=');
+        const newqsplit = requrlraw.split('&q=');
         if (qsplit.length > 1 && newqsplit.length > 1) {
             const lastq = newqsplit[newqsplit.length - 1];
-            requrlraw = qsplit[0] + "?q=" + lastq.split("&")[0] + "&" + qsplit[1].split("&")[1];
+            requrlraw = qsplit[0] + '?q=' + lastq.split('&')[0] + '&' + qsplit[1].split('&')[1];
         }
-        const requrl = requrlraw.startsWith("/") ? referer + requrlraw : requrlraw;
+        const requrl = requrlraw.startsWith('/') ? referer + requrlraw : requrlraw;
         // cors weirdness here...
         // if the referer is a cors page and the cors() route (I think) redirected to /corsProxy/<path> and the requested url path was relative,
         // then we redirect again to the cors referer and just add the relative path.
-        if (!requrl.startsWith("http") && req.originalUrl.startsWith("/corsProxy") && referer?.includes("corsProxy")) {
-            res.redirect(referer + (referer.endsWith("/") ? "" : "/") + requrl);
+        if (!requrl.startsWith('http') && req.originalUrl.startsWith('/corsProxy') && referer?.includes('corsProxy')) {
+            res.redirect(referer + (referer.endsWith('/') ? '' : '/') + requrl);
         } else {
             proxyServe(req, requrl, res);
         }
@@ -173,34 +172,40 @@ function proxyServe(req: any, requrl: string, response: any) {
     const htmlBodyMemoryStream = new (require('memorystream'))();
     var retrieveHTTPBody: any;
     const sendModifiedBody = () => {
-        const header = response.headers["content-encoding"];
-        if (header && header.includes("gzip")) {
+        const header = response.headers['content-encoding'];
+        if (header?.includes('gzip')) {
             try {
                 const replacer = (match: any, href: string, offset: any, string: any) => {
-                    return `href="${resolvedServerUrl + "/corsProxy/http" + href}"`;
+                    return `href="${resolvedServerUrl + '/corsProxy/http' + href}"`;
                 };
                 const zipToStringDecoder = new (require('string_decoder').StringDecoder)('utf8');
                 const bodyStream = htmlBodyMemoryStream.read();
                 if (bodyStream) {
-                    const htmlText = zipToStringDecoder.write(zlib.gunzipSync(bodyStream).toString('utf8')
-                        .replace('<head>', '<head> <style>[id ^= "google"] { display: none; } </style>')
-                        .replace(/href="https?([^"]*)"/g, replacer)
-                        .replace(/target="_blank"/g, ""));
+                    const htmlText = zipToStringDecoder.write(
+                        zlib
+                            .gunzipSync(bodyStream)
+                            .toString('utf8')
+                            .replace('<head>', '<head> <style>[id ^= "google"] { display: none; } </style>')
+                            .replace(/href="https?([^"]*)"/g, replacer)
+                            .replace(/target="_blank"/g, '')
+                    );
                     response.send(zlib.gzipSync(htmlText));
                 } else {
                     req.pipe(request(requrl)).pipe(response);
-                    console.log("EMPTY body:" + req.url);
+                    console.log('EMPTY body:' + req.url);
                 }
             } catch (e) {
-                console.log("EROR?: ", e);
+                console.log('EROR?: ', e);
             }
-        } else req.pipe(request(requrl)).pipe(response);
+        } else {
+            req.pipe(htmlBodyMemoryStream).pipe(response);
+        }
     };
     retrieveHTTPBody = () => {
-        req.headers.cookie = "";
+        req.headers.cookie = '';
         req.pipe(request(requrl))
-            .on("error", (e: any) => console.log(`Malformed CORS url: ${requrl}`, e))
-            .on("response", (res: any) => {
+            .on('error', (e: any) => console.log(`Malformed CORS url: ${requrl}`, e))
+            .on('response', (res: any) => {
                 res.headers;
                 const headers = Object.keys(res.headers);
                 const headerCharRegex = /[^\t\x20-\x7e\x80-\xff]/;
@@ -208,36 +213,41 @@ function proxyServe(req: any, requrl: string, response: any) {
                     const header = res.headers[headerName];
                     if (Array.isArray(header)) {
                         res.headers[headerName] = header.filter(h => !headerCharRegex.test(h));
-                    } else if (headerCharRegex.test(header || "")) {
+                    } else if (headerCharRegex.test(header || '')) {
                         delete res.headers[headerName];
                     } else res.headers[headerName] = header;
                 });
+                res.headers['x-permitted-cross-domain-policies'] = 'all';
+                res.headers['x-frame-options'] = '';
+                res.headers['content-security-policy'] = '';
                 response.headers = response._headers = res.headers;
             })
-            .on("end", sendModifiedBody)
+            .on('end', sendModifiedBody)
             .pipe(htmlBodyMemoryStream);
     };
     retrieveHTTPBody();
 }
 
 function registerEmbeddedBrowseRelativePathHandler(server: express.Express) {
-    server.use("*", (req, res) => {
+    server.use('*', (req, res) => {
         const relativeUrl = req.originalUrl;
-        if (!req.user) res.redirect("/home");  // When no user is logged in, we interpret a relative URL as being a reference to something they don't have access to and redirect to /home 
-        else if (!res.headersSent && req.headers.referer?.includes("corsProxy")) { // a request for something by a proxied referrer means it must be a relative reference.  So construct a proxied absolute reference here.
+        if (!req.user) res.redirect('/home'); // When no user is logged in, we interpret a relative URL as being a reference to something they don't have access to and redirect to /home
+        else if (!res.headersSent && req.headers.referer?.includes('corsProxy')) {
+            // a request for something by a proxied referrer means it must be a relative reference.  So construct a proxied absolute reference here.
             try {
                 const proxiedRefererUrl = decodeURIComponent(req.headers.referer); // (e.g., http://localhost:<port>/corsProxy/https://en.wikipedia.org/wiki/Engelbart)
                 const dashServerUrl = proxiedRefererUrl.match(/.*corsProxy\//)![0]; // the dash server url (e.g.: http://localhost:<port>/corsProxy/ )
-                const actualReferUrl = proxiedRefererUrl.replace(dashServerUrl, ""); // the url of the referer without the proxy (e.g., : https://en.wikipedia.org/wiki/Engelbart)
+                const actualReferUrl = proxiedRefererUrl.replace(dashServerUrl, ''); // the url of the referer without the proxy (e.g., : https://en.wikipedia.org/wiki/Engelbart)
                 const absoluteTargetBaseUrl = actualReferUrl.match(/https?:\/\/[^\/]*/)![0]; // the base of the original url (e.g.,  https://en.wikipedia.org)
                 const redirectedProxiedUrl = dashServerUrl + encodeURIComponent(absoluteTargetBaseUrl + relativeUrl); // the new proxied full url (e.g., http://localhost:<port>/corsProxy/https://en.wikipedia.org/<somethingelse>)
-                if (relativeUrl.startsWith("//")) res.redirect("http:" + relativeUrl);
+                if (relativeUrl.startsWith('//')) res.redirect('http:' + relativeUrl);
                 else res.redirect(redirectedProxiedUrl);
             } catch (e) {
-                console.log("Error embed: ", e);
+                console.log('Error embed: ', e);
             }
-        } else if (relativeUrl.startsWith("/search") && !req.headers.referer?.includes("corsProxy")) { // detect search query and use default search engine
-            res.redirect(req.headers.referer + "corsProxy/" + encodeURIComponent("http://www.google.com" + relativeUrl));
+        } else if (relativeUrl.startsWith('/search') && !req.headers.referer?.includes('corsProxy')) {
+            // detect search query and use default search engine
+            res.redirect(req.headers.referer + 'corsProxy/' + encodeURIComponent('http://www.google.com' + relativeUrl));
         } else {
             res.end();
         }