import * as bodyParser from 'body-parser'; import { blue, yellow } from 'colors'; import * as cors from 'cors'; import * as express from 'express'; import * as session from 'express-session'; import { createServer } from 'https'; import * as passport from 'passport'; import * as request from 'request'; import * as webpack from 'webpack'; import * as wdm from 'webpack-dev-middleware'; import * as whm from 'webpack-hot-middleware'; import * as zlib from 'zlib'; import { publicDirectory } from '.'; import { logPort } from './ActionUtilities'; import { SSL } from './apis/google/CredentialsLoader'; import { getForgot, getLogin, getLogout, getReset, getSignup, postForgot, postLogin, postReset, postSignup } from './authentication/AuthenticationManager'; import { Database } from './database'; import RouteManager from './RouteManager'; import RouteSubscriber from './RouteSubscriber'; import { WebSocket } from './websocket'; import * as expressFlash from 'express-flash'; import * as flash from 'connect-flash'; import * as brotli from 'brotli'; import * as MongoStoreConnect from 'connect-mongo'; import * as config from '../../webpack.config'; /* RouteSetter is a wrapper around the server that prevents the server from being exposed. */ export type RouteSetter = (server: RouteManager) => void; //export let disconnect: Function; export let resolvedPorts: { server: number; socket: number } = { server: 1050, socket: 4321 }; export let resolvedServerUrl: string; export default async function InitializeServer(routeSetter: RouteSetter) { const isRelease = determineEnvironment(); const app = buildWithMiddleware(express()); const compiler = webpack(config as any); // route table managed by express. routes are tested sequentially against each of these map rules. when a match is found, the handler is called to process the request app.use(wdm(compiler, { publicPath: config.output.publicPath })); app.use(whm(compiler)); app.get(new RegExp(/^\/+$/), (req, res) => res.redirect(req.user ? '/home' : '/login')); // target urls that consist of one or more '/'s with nothing in between app.use(express.static(publicDirectory, { setHeaders: res => res.setHeader('Access-Control-Allow-Origin', '*') })); //all urls that start with dash's public directory: /files/ (e.g., /files/images, /files/audio, etc) app.use(cors({ origin: (_origin: any, callback: any) => callback(null, true) })); registerAuthenticationRoutes(app); // this adds routes to authenticate a user (login, etc) registerCorsProxy(app); // this adds a /corsProxy/ route to allow clients to get to urls that would otherwise be blocked by cors policies isRelease && !SSL.Loaded && SSL.exit(); routeSetter(new RouteManager(app, isRelease)); // this sets up all the regular supervised routes (things like /home, download/upload api's, pdf, search, session, etc) registerEmbeddedBrowseRelativePathHandler(app); // this allows renered web pages which internally have relative paths to find their content isRelease && process.env.serverPort && (resolvedPorts.server = Number(process.env.serverPort)); const server = isRelease ? createServer(SSL.Credentials, app) : app; await new Promise(resolve => server.listen(resolvedPorts.server, resolve)); logPort('server', resolvedPorts.server); resolvedServerUrl = `${isRelease && process.env.serverName ? `https://${process.env.serverName}.com` : 'http://localhost'}:${resolvedPorts.server}`; // initialize the web socket (bidirectional communication: if a user changes // a field on one client, that change must be broadcast to all other clients) await WebSocket.initialize(isRelease, SSL.Credentials); //disconnect = async () => new Promise(resolve => server.close(resolve)); return isRelease; } const week = 7 * 24 * 60 * 60 * 1000; const secret = '64d6866242d3b5a5503c675b32c9605e4e90478e9b77bcf2bc'; const store = process.env.DB === 'MEM' ? new session.MemoryStore() : MongoStoreConnect.create({ mongoUrl: Database.url }); function buildWithMiddleware(server: express.Express) { [ session({ secret, resave: false, cookie: { maxAge: week }, saveUninitialized: true, store, }), flash(), expressFlash(), bodyParser.json({ limit: '10mb' }), bodyParser.urlencoded({ extended: true }), passport.initialize(), passport.session(), (req: express.Request, res: express.Response, next: express.NextFunction) => { res.locals.user = req.user; // console.log('HEADER:' + req.originalUrl + ' path = ' + req.path); if ((req.originalUrl.endsWith('.png') || req.originalUrl.endsWith('.jpg') || (process.env.RELEASE === 'true' && req.originalUrl.endsWith('.js'))) && req.method === 'GET') { const period = 30000; res.set('Cache-control', `public, max-age=${period}`); } else { // for the other requests set strict no caching parameters res.set('Cache-control', `no-store`); } next(); }, ].forEach(next => server.use(next)); return server; } /* Determine if the enviroment is dev mode or release mode. */ function determineEnvironment() { const isRelease = process.env.RELEASE === 'true'; const color = isRelease ? blue : yellow; const label = isRelease ? 'release' : 'development'; console.log(`\nrunning server in ${color(label)} mode`); // // swilkins: I don't think we need to read from ClientUtils.RELEASE anymore. Should be able to invoke process.env.RELEASE // // on the client side, thanks to dotenv in webpack.config.js // let clientUtils = fs.readFileSync('./src/client/util/ClientUtils.ts.temp', 'utf8'); // clientUtils = `//AUTO-GENERATED FILE: DO NOT EDIT\n${clientUtils.replace('"mode"', String(isRelease))}`; // fs.writeFileSync('./src/client/util/ClientUtils.ts', clientUtils, 'utf8'); return isRelease; } function registerAuthenticationRoutes(server: express.Express) { server.get('/signup', getSignup); server.post('/signup', postSignup); server.get('/login', getLogin); server.post('/login', postLogin); server.get('/logout', getLogout); server.get('/forgotPassword', getForgot); server.post('/forgotPassword', postForgot); const reset = new RouteSubscriber('resetPassword').add('token').build; server.get(reset, getReset); server.post(reset, postReset); } function registerCorsProxy(server: express.Express) { server.use('/corsProxy', async (req, res) => { res.setHeader('Access-Control-Allow-Origin', '*'); res.header('Access-Control-Allow-Methods', 'GET, PUT, PATCH, POST, DELETE'); res.header('Access-Control-Allow-Headers', req.header('access-control-request-headers')); const referer = req.headers.referer ? decodeURIComponent(req.headers.referer) : ''; let requrlraw = decodeURIComponent(req.url.substring(1)); const qsplit = requrlraw.split('?q='); const newqsplit = requrlraw.split('&q='); if (qsplit.length > 1 && newqsplit.length > 1) { const lastq = newqsplit[newqsplit.length - 1]; requrlraw = qsplit[0] + '?q=' + lastq.split('&')[0] + '&' + qsplit[1].split('&')[1]; } const requrl = requrlraw.startsWith('/') ? referer + requrlraw : requrlraw; // cors weirdness here... // if the referer is a cors page and the cors() route (I think) redirected to /corsProxy/ and the requested url path was relative, // then we redirect again to the cors referer and just add the relative path. if (!requrl.startsWith('http') && req.originalUrl.startsWith('/corsProxy') && referer?.includes('corsProxy')) { res.redirect(referer + (referer.endsWith('/') ? '' : '/') + requrl); } else { proxyServe(req, requrl, res); } }); } function proxyServe(req: any, requrl: string, response: any) { const htmlBodyMemoryStream = new (require('memorystream'))(); var wasinBrFormat = false; const sendModifiedBody = () => { const header = response.headers['content-encoding']; const refToCors = (match: any, tag: string, sym: string, href: string, offset: any, string: any) => `${tag}=${sym + resolvedServerUrl}/corsProxy/${href + sym}`; const relpathToCors = (match: any, href: string, offset: any, string: any) => `="${resolvedServerUrl + '/corsProxy/' + decodeURIComponent(req.originalUrl.split('/corsProxy/')[1].match(/https?:\/\/[^\/]*/)?.[0] ?? '') + '/' + href}"`; if (header) { try { const bodyStream = htmlBodyMemoryStream.read(); if (bodyStream) { const htmlInputText = wasinBrFormat ? Buffer.from(brotli.decompress(bodyStream)) : header.includes('gzip') ? zlib.gunzipSync(bodyStream) : bodyStream; const htmlText = htmlInputText .toString('utf8') .replace('', ' ') .replace(/(src|href)=([\'\"])(https?[^\2\n]*)\1/g, refToCors) // replace src or href='http(s)://...' or href="http(s)://.." //.replace(/= *"\/([^"]*)"/g, relpathToCors) .replace(/data-srcset="[^"]*"/g, '') .replace(/srcset="[^"]*"/g, '') .replace(/target="_blank"/g, ''); response.send(header?.includes('gzip') ? zlib.gzipSync(htmlText) : htmlText); } else { req.pipe(request(requrl)) .on('error', (e: any) => console.log('requrl ', e)) .pipe(response) .on('error', (e: any) => console.log('response pipe error', e)); console.log('EMPTY body:' + req.url); } } catch (e) { console.log('ERROR?: ', e); } } else { req.pipe(htmlBodyMemoryStream) .on('error', (e: any) => console.log('html body memorystream error', e)) .pipe(response) .on('error', (e: any) => console.log('html body memory stream response error', e)); } }; const retrieveHTTPBody = () => { //req.headers.cookie = ''; req.pipe(request(requrl)) .on('error', (e: any) => { console.log(`CORS url error: ${requrl}`, e); response.send(` Error

Failed to load: ${requrl}

${e}

`); }) .on('response', (res: any) => { res.headers; const headers = Object.keys(res.headers); const headerCharRegex = /[^\t\x20-\x7e\x80-\xff]/; headers.forEach(headerName => { const header = res.headers[headerName]; if (Array.isArray(header)) { res.headers[headerName] = header.filter(h => !headerCharRegex.test(h)); } else if (headerCharRegex.test(header || '')) { delete res.headers[headerName]; } else res.headers[headerName] = header; if (headerName === 'content-encoding') { wasinBrFormat = res.headers[headerName] === 'br'; res.headers[headerName] = 'gzip'; } }); res.headers['x-permitted-cross-domain-policies'] = 'all'; res.headers['x-frame-options'] = ''; res.headers['content-security-policy'] = ''; response.headers = response._headers = res.headers; }) .on('end', sendModifiedBody) .pipe(htmlBodyMemoryStream) .on('error', (e: any) => console.log('http body pipe error', e)); }; retrieveHTTPBody(); } function registerEmbeddedBrowseRelativePathHandler(server: express.Express) { server.use('*', (req, res) => { // res.setHeader('Access-Control-Allow-Origin', '*'); // res.header('Access-Control-Allow-Methods', 'GET, PUT, PATCH, POST, DELETE'); // res.header('Access-Control-Allow-Headers', req.header('access-control-request-headers')); const relativeUrl = req.originalUrl; if (!res.headersSent && req.headers.referer?.includes('corsProxy')) { if (!req.user) res.redirect('/home'); // When no user is logged in, we interpret a relative URL as being a reference to something they don't have access to and redirect to /home // a request for something by a proxied referrer means it must be a relative reference. So construct a proxied absolute reference here. try { const proxiedRefererUrl = decodeURIComponent(req.headers.referer); // (e.g., http://localhost:/corsProxy/https://en.wikipedia.org/wiki/Engelbart) const dashServerUrl = proxiedRefererUrl.match(/.*corsProxy\//)![0]; // the dash server url (e.g.: http://localhost:/corsProxy/ ) const actualReferUrl = proxiedRefererUrl.replace(dashServerUrl, ''); // the url of the referer without the proxy (e.g., : https://en.wikipedia.org/wiki/Engelbart) const absoluteTargetBaseUrl = actualReferUrl.match(/https?:\/\/[^\/]*/)![0]; // the base of the original url (e.g., https://en.wikipedia.org) const redirectedProxiedUrl = dashServerUrl + encodeURIComponent(absoluteTargetBaseUrl + relativeUrl); // the new proxied full url (e.g., http://localhost:/corsProxy/https://en.wikipedia.org/) const redirectUrl = relativeUrl.startsWith('//') ? 'http:' + relativeUrl : redirectedProxiedUrl; res.redirect(redirectUrl); } catch (e) { console.log('Error embed: ', e); } } else if (relativeUrl.startsWith('/search') && !req.headers.referer?.includes('corsProxy')) { // detect search query and use default search engine res.redirect(req.headers.referer + 'corsProxy/' + encodeURIComponent('http://www.google.com' + relativeUrl)); } else { res.end(); } }); }