when document is in lightbox view and a link is followed to its sidebar, we no longer reopen the document in its context. fixed selecting annotations on web/pdf. don't change document height when shown in a linkPreview. webBox fixes for pages with scripts. fixed range bounds when clicking on web text

several fixes to web pages on server and client.
client webbox allows clicks on divs with onclick instead of doing selection
       also hacky fix so that google search url doesn't keep expanding by removing 'q=' regions
       also added prevent/allow script menu item
server grabs all html and hides id="google.." which are ads.
      also rewrites hrefs starting with http to route through corsProxy
      also removes target=_blank tags to prevent pages from opening outside of dash.
     also cleaned up routes and comments
     also when not logged in, references to anything in dash domain route to /home

1 files changed, 67 insertions, 48 deletions

diff --git a/src/server/server_Initialization.ts b/src/server/server_Initialization.ts
index 00a801e03..de93b64c3 100644
--- a/src/server/server_Initialization.ts
+++ b/src/server/server_Initialization.ts
@@ -13,6 +13,7 @@ import * as request from 'request';
 import * as webpack from 'webpack';
 import * as wdm from 'webpack-dev-middleware';
 import * as whm from 'webpack-hot-middleware';
+import * as zlib from 'zlib';
 import { publicDirectory } from '.';
 import { logPort } from './ActionUtilities';
 import { SSL } from './apis/google/CredentialsLoader';
@@ -36,39 +37,30 @@ export let resolvedPorts: { server: number, socket: number } = { server: 1050, s
 export let resolvedServerUrl: string;
 
 export default async function InitializeServer(routeSetter: RouteSetter) {
+    const isRelease = determineEnvironment();
     const app = buildWithMiddleware(express());
-    // Root route of express app
-    app.get("/", (req, res) => res.redirect("/home"));
 
-    app.use(express.static(publicDirectory, {
-        setHeaders: res => res.setHeader("Access-Control-Allow-Origin", "*")
-    }));
-    app.use("/images", express.static(publicDirectory));
+    // route table managed by express.  routes are tested sequentially against each of these map rules.  when a match is found, the handler is called to process the request
+    app.get(new RegExp(/^\/+$/), (req, res) => res.redirect(req.user ? "/home" : "/login")); // target urls that consist of one or more '/'s with nothing in between
+    app.use(express.static(publicDirectory, { setHeaders: res => res.setHeader("Access-Control-Allow-Origin", "*") }));  //all urls that start with dash's public directory: /files/  (e.g., /files/images, /files/audio, etc)
     app.use(cors({ origin: (_origin: any, callback: any) => callback(null, true) }));
-
     app.use(wdm(compiler, { publicPath: config.output.publicPath }));
     app.use(whm(compiler));
-
-    registerAuthenticationRoutes(app);
-    registerCorsProxy(app);
-
-    const isRelease = determineEnvironment();
-
+    registerAuthenticationRoutes(app);  // this adds routes to authenticate a user (login, etc)
+    registerCorsProxy(app);  // this adds a /corsProxy/ route to allow clients to get to urls that would otherwise be blocked by cors policies
     isRelease && !SSL.Loaded && SSL.exit();
-
-    routeSetter(new RouteManager(app, isRelease));
-    registerRelativePath(app);
+    routeSetter(new RouteManager(app, isRelease)); // this sets up all the regular supervised routes (things like /home, download/upload api's, pdf, search, session, etc)
+    registerEmbeddedBrowseRelativePathHandler(app); // this allows renered web pages which internally have relative paths to find their content
 
     let server: HttpServer | HttpsServer;
-    const { serverPort, serverName } = process.env;
-    isRelease && serverPort && (resolvedPorts.server = Number(serverPort));
+    isRelease && process.env.serverPort && (resolvedPorts.server = Number(process.env.serverPort));
     await new Promise<void>(resolve => server = isRelease ?
         createServer(SSL.Credentials, app).listen(resolvedPorts.server, resolve) :
         app.listen(resolvedPorts.server, resolve)
     );
     logPort("server", resolvedPorts.server);
 
-    resolvedServerUrl = `${isRelease && serverName ? `https://${serverName}.com` : "http://localhost"}:${resolvedPorts.server}`;
+    resolvedServerUrl = `${isRelease && process.env.serverName ? `https://${process.env.serverName}.com` : "http://localhost"}:${resolvedPorts.server}`;
 
     // initialize the web socket (bidirectional communication: if a user changes
     // a field on one client, that change must be broadcast to all other clients)
@@ -141,54 +133,81 @@ function registerAuthenticationRoutes(server: express.Express) {
 }
 
 function registerCorsProxy(server: express.Express) {
-    const headerCharRegex = /[^\t\x20-\x7e\x80-\xff]/;
     server.use("/corsProxy", async (req, res) => {
-
         const referer = req.headers.referer ? decodeURIComponent(req.headers.referer) : "";
-        const requrlraw = decodeURIComponent(req.url.substring(1));
+        let requrlraw = decodeURIComponent(req.url.substring(1));
+        const qsplit = requrlraw.split("?q=");
+        const newqsplit = requrlraw.split("&q=");
+        if (qsplit.length > 1 && newqsplit.length > 1) {
+            const lastq = newqsplit[newqsplit.length - 1];
+            requrlraw = qsplit[0] + "?q=" + lastq.split("&")[0] + "&" + qsplit[1].split("&")[1];
+        }
         const requrl = requrlraw.startsWith("/") ? referer + requrlraw : requrlraw;
-        // cors weirdness here... 
-        // if the referer is a cors page and the cors() route (I think) redirected to /corsProxy/<path> and the requested url path was relative, 
+        // cors weirdness here...
+        // if the referer is a cors page and the cors() route (I think) redirected to /corsProxy/<path> and the requested url path was relative,
         // then we redirect again to the cors referer and just add the relative path.
         if (!requrl.startsWith("http") && req.originalUrl.startsWith("/corsProxy") && referer?.includes("corsProxy")) {
             res.redirect(referer + (referer.endsWith("/") ? "" : "/") + requrl);
         } else {
-            try {
-                await new Promise<void>((resolve, reject) => {
-                    request(requrl).on("response", resolve).on("error", reject);
-                });
-            } catch {
-                console.log(`Malformed CORS url: ${requrl}`);
-                return res.send();
-            }
-            req.pipe(request(requrl)).on("response", res => {
-                const headers = Object.keys(res.headers);
-                headers.forEach(headerName => {
-                    const header = res.headers[headerName];
-                    if (Array.isArray(header)) {
-                        res.headers[headerName] = header.filter(h => !headerCharRegex.test(h));
-                    } else if (header) {
-                        if (headerCharRegex.test(header as any)) {
-                            delete res.headers[headerName];
-                        }
-                    }
-                });
-            }).on("error", () => console.log(`Malformed CORS url: ${requrl}`)).pipe(res);
+            proxyServe(req, requrl, res);
         }
     });
 }
 
-function registerRelativePath(server: express.Express) {
+function proxyServe(req: any, requrl: string, response: any) {
+    const htmlBodyMemoryStream = new (require('memorystream'))();
+    req.headers.cookie = "";
+    req.pipe(request(requrl))
+        .on("error", (e: any) => console.log(`Malformed CORS url: ${requrl}`, e))
+        .on("end", () => {
+            var rewrittenHtmlBody: any = undefined;
+            req.pipe(request(requrl))
+                .on("response", (res: any) => {
+                    const headers = Object.keys(res.headers);
+                    const headerCharRegex = /[^\t\x20-\x7e\x80-\xff]/;
+                    headers.forEach(headerName => {
+                        const header = res.headers[headerName];
+                        if (Array.isArray(header)) {
+                            res.headers[headerName] = header.filter(h => !headerCharRegex.test(h));
+                        } else if (headerCharRegex.test(header || "")) {
+                            delete res.headers[headerName];
+                        }
+                        if (headerName === "content-encoding" && header.includes("gzip")) {
+                            try {
+                                const replacer = (match: any, href: string, offset: any, string: any) => {
+                                    return `href="${resolvedServerUrl + "/corsProxy/http" + href}"`;
+                                };
+                                const zipToStringDecoder = new (require('string_decoder').StringDecoder)('utf8');
+                                const htmlText = zipToStringDecoder.write(zlib.gunzipSync(htmlBodyMemoryStream.read()).toString('utf8')
+                                    .replace('<head>', '<head> <style>[id ^= "google"] { display: none; } </style>')
+                                    .replace(/href="http([^"]*)"/g, replacer)
+                                    .replace(/target="_blank"/g, ""));
+                                rewrittenHtmlBody = zlib.gzipSync(htmlText);
+                            } catch (e) { console.log(e); }
+                        }
+                    });
+                })
+                .on('data', (e: any) => {
+                    rewrittenHtmlBody && response.send(rewrittenHtmlBody);
+                    rewrittenHtmlBody = undefined;
+                })
+                .pipe(response);
+        })
+        .pipe(htmlBodyMemoryStream);
+}
+
+function registerEmbeddedBrowseRelativePathHandler(server: express.Express) {
     server.use("*", (req, res) => {
         const relativeUrl = req.originalUrl;
-        if (!res.headersSent && req.headers.referer?.includes("corsProxy")) { // a request for something by a proxied referrer means it must be a relative reference.  So construct a proxied absolute reference here.
+        if (!req.user) res.redirect("/home");  // When no user is logged in, we interpret a relative URL as being a reference to something they don't have access to and redirect to /home 
+        else if (!res.headersSent && req.headers.referer?.includes("corsProxy")) { // a request for something by a proxied referrer means it must be a relative reference.  So construct a proxied absolute reference here.
             const proxiedRefererUrl = decodeURIComponent(req.headers.referer); // (e.g., http://localhost:<port>/corsProxy/https://en.wikipedia.org/wiki/Engelbart)
             const dashServerUrl = proxiedRefererUrl.match(/.*corsProxy\//)![0]; // the dash server url (e.g.: http://localhost:<port>/corsProxy/ )
             const actualReferUrl = proxiedRefererUrl.replace(dashServerUrl, ""); // the url of the referer without the proxy (e.g., : http:s//en.wikipedia.org/wiki/Engelbart)
             const absoluteTargetBaseUrl = actualReferUrl.match(/http[s]?:\/\/[^\/]*/)![0]; // the base of the original url (e.g.,  https://en.wikipedia.org)
             const redirectedProxiedUrl = dashServerUrl + encodeURIComponent(absoluteTargetBaseUrl + relativeUrl); // the new proxied full url (e..g, http://localhost:<port>/corsProxy/https://en.wikipedia.org/<somethingelse>)
             res.redirect(redirectedProxiedUrl);
-        } else if (relativeUrl.startsWith("/search")) { // detect search query and use default search engine
+        } else if (relativeUrl.startsWith("/search") && !req.headers.referer?.includes("corsProxy")) { // detect search query and use default search engine
             res.redirect(req.headers.referer + "corsProxy/" + encodeURIComponent("http://www.google.com" + relativeUrl));
         } else {
             res.end();