diff options
Diffstat (limited to 'src/server')
| -rw-r--r-- | src/server/ApiManagers/UploadManager.ts | 27 | ||||
| -rw-r--r-- | src/server/DashUploadUtils.ts | 175 | ||||
| -rw-r--r-- | src/server/server_Initialization.ts | 142 |
3 files changed, 226 insertions, 118 deletions
diff --git a/src/server/ApiManagers/UploadManager.ts b/src/server/ApiManagers/UploadManager.ts index 0b6e18743..fe4c475c9 100644 --- a/src/server/ApiManagers/UploadManager.ts +++ b/src/server/ApiManagers/UploadManager.ts @@ -60,6 +60,18 @@ export default class UploadManager extends ApiManager { return new Promise<void>(resolve => { form.parse(req, async (_err, _fields, files) => { const results: Upload.FileResponse[] = []; + if (_err?.message) { + results.push({ + source: { + size: 0, + path: 'none', + name: 'none', + type: 'none', + toJSON: () => ({ name: 'none', path: '' }), + }, + result: { name: 'failed upload', message: `${_err.message}` }, + }); + } for (const key in files) { const f = files[key]; if (!Array.isArray(f)) { @@ -96,6 +108,21 @@ export default class UploadManager extends ApiManager { register({ method: Method.POST, + subscription: '/queryYoutubeProgress', + secureHandler: async ({ req, res }) => { + return new Promise<void>(async resolve => { + req.addListener('data', args => { + const payload = String.fromCharCode.apply(String, args); + const videoId = JSON.parse(payload).videoId; + _success(res, { progress: DashUploadUtils.QueryYoutubeProgress(videoId) }); + resolve(); + }); + }); + }, + }); + + register({ + method: Method.POST, subscription: new RouteSubscriber('youtubeScreenshot'), secureHandler: async ({ req, res }) => { const { id, timecode } = req.body; diff --git a/src/server/DashUploadUtils.ts b/src/server/DashUploadUtils.ts index e94ef8534..4870d218b 100644 --- a/src/server/DashUploadUtils.ts +++ b/src/server/DashUploadUtils.ts @@ -2,7 +2,7 @@ import { green, red } from 'colors'; import { ExifImage } from 'exif'; import * as exifr from 'exifr'; import { File } from 'formidable'; -import { createWriteStream, existsSync, readFileSync, rename, unlinkSync, writeFile } from 'fs'; +import { createReadStream, createWriteStream, existsSync, readFileSync, rename, unlinkSync, writeFile } from 'fs'; import * as path from 'path'; import { basename } from 'path'; import * as sharp from 'sharp'; @@ -17,11 +17,13 @@ import { resolvedServerUrl } from './server_Initialization'; import { AcceptableMedia, Upload } from './SharedMediaTypes'; import request = require('request-promise'); import formidable = require('formidable'); +const spawn = require('child_process').spawn; const { exec } = require('child_process'); const parse = require('pdf-parse'); const ffmpeg = require('fluent-ffmpeg'); const fs = require('fs'); const requestImageSize = require('../client/util/request-image-size'); +const md5File = require('md5-file'); export enum SizeSuffix { Small = '_s', @@ -99,32 +101,73 @@ export namespace DashUploadUtils { }; } + function resolveExistingFile(name: string, pat: string, directory: Directory, type?: string, duration?: number, rawText?: string) { + const data = { size: 0, path: path.basename(pat), name, type: type ?? '' }; + const file = { ...data, toJSON: () => ({ ...data, filename: data.path.replace(/.*\//, ''), mtime: duration?.toString(), mime: '', toJson: () => undefined as any }) }; + return { + source: file, + result: { + accessPaths: { + agnostic: getAccessPaths(directory, data.path), + }, + rawText, + duration, + }, + }; + } + + export function QueryYoutubeProgress(videoId: string) { + return uploadProgress.get(videoId) ?? 'failed'; + } + + let uploadProgress = new Map<string, string>(); + export function uploadYoutube(videoId: string): Promise<Upload.FileResponse> { return new Promise<Upload.FileResponse<Upload.FileInformation>>((res, rej) => { console.log('Uploading YouTube video: ' + videoId); - exec('youtube-dl -o ' + (videoId + '.mp4') + ' ' + videoId + ' -f "mp4[filesize<5M]/bestvideo[filesize<5M]+bestaudio/bestvideo+bestaudio"', (error: any, stdout: any, stderr: any) => { - if (error) { - console.log(`error: Error: ${error.message}`); - res({ - source: { - size: 0, - path: videoId, - name: videoId, - type: '', - toJSON: () => ({ name: videoId, path: videoId }), - }, - result: { name: 'failed youtube query', message: `Could not upload YouTube video (${videoId}). Error: ${error.message}` }, - }); - } else { - exec('youtube-dl -o ' + (videoId + '.mp4') + ' ' + videoId + ' --get-duration', (error: any, stdout: any, stderr: any) => { - const time = Array.from(stdout.trim().split(':')).reverse(); - const duration = (time.length > 2 ? Number(time[2]) * 1000 * 60 : 0) + (time.length > 1 ? Number(time[1]) * 60 : 0) + (time.length > 0 ? Number(time[0]) : 0); - const data = { size: 0, path: videoId + '.mp4', name: videoId, type: 'video/mp4' }; - const file = { ...data, toJSON: () => ({ ...data, filename: data.path.replace(/.*\//, ''), mtime: duration.toString(), mime: '', toJson: () => undefined as any }) }; - res(MoveParsedFile(file, Directory.videos)); - }); - } - }); + const name = videoId; + const path = name.replace(/^-/, '__') + '.mp4'; + const finalPath = serverPathToFile(Directory.videos, path); + if (existsSync(finalPath)) { + uploadProgress.set(videoId, 'computing duration'); + exec(`yt-dlp -o ${finalPath} "https://www.youtube.com/watch?v=${videoId}" --get-duration`, (error: any, stdout: any, stderr: any) => { + const time = Array.from(stdout.trim().split(':')).reverse(); + const duration = (time.length > 2 ? Number(time[2]) * 1000 * 60 : 0) + (time.length > 1 ? Number(time[1]) * 60 : 0) + (time.length > 0 ? Number(time[0]) : 0); + res(resolveExistingFile(name, finalPath, Directory.videos, 'video/mp4', duration, undefined)); + }); + } else { + uploadProgress.set(videoId, 'starting download'); + const ytdlp = spawn(`yt-dlp`, ['-o', path, `https://www.youtube.com/watch?v=${videoId}`, '--max-filesize', '100M', '-f', 'mp4']); + + ytdlp.stdout.on('data', (data: any) => !uploadProgress.get(videoId)?.includes('Aborting.') && uploadProgress.set(videoId, data.toString())); + + let errors = ''; + ytdlp.stderr.on('data', (data: any) => (errors = data.toString())); + + ytdlp.on('exit', function (code: any) { + if (code || uploadProgress.get(videoId)?.includes('Aborting.')) { + res({ + source: { + size: 0, + path, + name, + type: '', + toJSON: () => ({ name, path }), + }, + result: { name: 'failed youtube query', message: `Could not archive video. ${code ? errors : uploadProgress.get(videoId)}` }, + }); + } else { + uploadProgress.set(videoId, 'computing duration'); + exec(`yt-dlp-o ${path} "https://www.youtube.com/watch?v=${videoId}" --get-duration`, (error: any, stdout: any, stderr: any) => { + const time = Array.from(stdout.trim().split(':')).reverse(); + const duration = (time.length > 2 ? Number(time[2]) * 1000 * 60 : 0) + (time.length > 1 ? Number(time[1]) * 60 : 0) + (time.length > 0 ? Number(time[0]) : 0); + const data = { size: 0, path, name, type: 'video/mp4' }; + const file = { ...data, toJSON: () => ({ ...data, filename: data.path.replace(/.*\//, ''), mtime: duration.toString(), mime: '', toJson: () => undefined as any }) }; + res(MoveParsedFile(file, Directory.videos)); + }); + } + }); + } }); } @@ -142,6 +185,8 @@ export namespace DashUploadUtils { const result = await UploadImage(path, basename(path)); return { source: file, result }; } + fs.unlink(path, () => {}); + return { source: file, result: { name: 'Unsupported image format', message: `Could not upload unsupported file (${name}). Please convert to an .jpg` } }; case 'video': if (format.includes('x-matroska')) { console.log('case video'); @@ -164,24 +209,30 @@ export namespace DashUploadUtils { res(); }) ); - if (abort) return { source: file, result: { name: 'Unsupported video format', message: `Could not upload unsupported file (${name}). Please convert to an .mp4` } }; - // bcz: instead of aborting, we could convert the file using the code below to an mp4. Problem is that this takes a long time and will clog up the server. - // await new Promise(res => - // ffmpeg(file.path) - // .videoCodec('libx264') // this will copy the data instead of reencode it - // .audioCodec('mp2') - // .save(file.path.replace('.MOV', '.mp4').replace('.mov', '.mp4')) - // .on('end', res) - // ); - // file.path = file.path.replace('.mov', '.mp4').replace('.MOV', '.mp4'); - // format = '.mp4'; + if (abort) { + // bcz: instead of aborting, we could convert the file using the code below to an mp4. Problem is that this takes a long time and will clog up the server. + // await new Promise(res => + // ffmpeg(file.path) + // .videoCodec('libx264') // this will copy the data instead of reencode it + // .audioCodec('mp2') + // .save(file.path.replace('.MOV', '.mp4').replace('.mov', '.mp4')) + // .on('end', res) + // ); + // file.path = file.path.replace('.mov', '.mp4').replace('.MOV', '.mp4'); + // format = '.mp4'; + fs.unlink(path, () => {}); + return { source: file, result: { name: 'Unsupported video format', message: `Could not upload unsupported file (${name}). Please convert to an .mp4` } }; + } } if (videoFormats.includes(format)) { return MoveParsedFile(file, Directory.videos); } + fs.unlink(path, () => {}); + return { source: file, result: { name: 'Unsupported video format', message: `Could not upload unsupported file (${name}). Please convert to an .mp4` } }; case 'application': if (applicationFormats.includes(format)) { - return UploadPdf(file); + const val = UploadPdf(file); + if (val) return val; } case 'audio': const components = format.split(';'); @@ -191,6 +242,8 @@ export namespace DashUploadUtils { if (audioFormats.includes(format)) { return UploadAudio(file, format); } + fs.unlink(path, () => {}); + return { source: file, result: { name: 'Unsupported audio format', message: `Could not upload unsupported file (${name}). Please convert to an .mp3` } }; case 'text': if (types[1] == 'csv') { return UploadCsv(file); @@ -198,20 +251,31 @@ export namespace DashUploadUtils { } console.log(red(`Ignoring unsupported file (${name}) with upload type (${type}).`)); + fs.unlink(path, () => {}); return { source: file, result: new Error(`Could not upload unsupported file (${name}) with upload type (${type}).`) }; } async function UploadPdf(file: File) { - const { path: sourcePath } = file; - const dataBuffer = readFileSync(sourcePath); - const result: ParsedPDF = await parse(dataBuffer); - await new Promise<void>((resolve, reject) => { - const name = path.basename(sourcePath); - const textFilename = `${name.substring(0, name.length - 4)}.txt`; - const writeStream = createWriteStream(serverPathToFile(Directory.text, textFilename)); - writeStream.write(result.text, error => (error ? reject(error) : resolve())); - }); - return MoveParsedFile(file, Directory.pdfs, undefined, result.text); + const fileKey = (await md5File(file.path)) + '.pdf'; + const textFilename = `${fileKey.substring(0, fileKey.length - 4)}.txt`; + if (fExists(fileKey, Directory.pdfs) && fExists(textFilename, Directory.text)) { + return new Promise<Upload.FileResponse>(res => { + const textFilename = `${fileKey.substring(0, fileKey.length - 4)}.txt`; + const readStream = createReadStream(serverPathToFile(Directory.text, textFilename)); + var rawText = ''; + readStream.on('data', chunk => (rawText += chunk.toString())).on('end', () => res(resolveExistingFile(file.name, fileKey, Directory.pdfs, file.type, undefined, rawText))); + }); + } + const dataBuffer = readFileSync(file.path); + const result: ParsedPDF | any = await parse(dataBuffer).catch((e: any) => e); + if (!result.code) { + await new Promise<void>((resolve, reject) => { + const writeStream = createWriteStream(serverPathToFile(Directory.text, textFilename)); + writeStream.write(result?.text, error => (error ? reject(error) : resolve())); + }); + return MoveParsedFile(file, Directory.pdfs, undefined, result?.text, undefined, fileKey); + } + return { source: file, result: { name: 'faile pdf pupload', message: `Could not upload (${file.name}).${result.message}` } }; } async function UploadCsv(file: File) { @@ -250,7 +314,7 @@ export namespace DashUploadUtils { export const UploadImage = async (source: string, filename?: string, prefix: string = ''): Promise<Upload.ImageInformation | Error> => { const metadata = await InspectImage(source); if (metadata instanceof Error) { - return metadata; + return { name: metadata.name, message: metadata.message }; } return UploadInspectedImage(metadata, filename || metadata.filename, prefix); }; @@ -330,10 +394,12 @@ export namespace DashUploadUtils { exifData, requestable: resolvedUrl, }; + // Use the request library to parse out file level image information in the headers const { headers } = await new Promise<any>((resolve, reject) => { - request.head(resolvedUrl, (error, res) => (error ? reject(error) : resolve(res))); - }).catch(console.error); + return request.head(resolvedUrl, (error, res) => (error ? reject(error) : resolve(res))); + }).catch(e => console.log(e)); + try { // Compute the native width and height ofthe image with an npm module const { width: nativeWidth, height: nativeHeight } = await requestImageSize(resolvedUrl); @@ -363,9 +429,9 @@ export namespace DashUploadUtils { * @param suffix If the file doesn't have a suffix and you want to provide it one * to appear in the new location */ - export async function MoveParsedFile(file: formidable.File, destination: Directory, suffix: string | undefined = undefined, text?: string, duration?: number): Promise<Upload.FileResponse> { + export async function MoveParsedFile(file: formidable.File, destination: Directory, suffix: string | undefined = undefined, text?: string, duration?: number, targetName?: string): Promise<Upload.FileResponse> { const { path: sourcePath } = file; - let name = path.basename(sourcePath); + let name = targetName ?? path.basename(sourcePath); suffix && (name += suffix); return new Promise(resolve => { const destinationPath = serverPathToFile(destination, name); @@ -386,6 +452,11 @@ export namespace DashUploadUtils { }); } + export function fExists(name: string, destination: Directory) { + const destinationPath = serverPathToFile(destination, name); + return existsSync(destinationPath); + } + export function getAccessPaths(directory: Directory, fileName: string) { return { client: clientPathToFile(directory, fileName), @@ -438,7 +509,7 @@ export namespace DashUploadUtils { }); }); //data && bufferConverterRec(data); - return { data: await exifr.parse(image), error }; + return error ? { data: undefined, error } : { data: await exifr.parse(image), error }; }; const { pngs, jpgs, webps, tiffs } = AcceptableMedia; diff --git a/src/server/server_Initialization.ts b/src/server/server_Initialization.ts index fd000a83c..b0db71f9c 100644 --- a/src/server/server_Initialization.ts +++ b/src/server/server_Initialization.ts @@ -1,13 +1,13 @@ import * as bodyParser from 'body-parser'; import { blue, yellow } from 'colors'; import * as cookieParser from 'cookie-parser'; -import * as cors from "cors"; +import * as cors from 'cors'; import * as express from 'express'; import * as session from 'express-session'; import * as expressValidator from 'express-validator'; import * as fs from 'fs'; -import { Server as HttpServer } from "http"; -import { createServer, Server as HttpsServer } from "https"; +import { Server as HttpServer } from 'http'; +import { createServer, Server as HttpsServer } from 'https'; import * as passport from 'passport'; import * as request from 'request'; import * as webpack from 'webpack'; @@ -33,7 +33,7 @@ const compiler = webpack(config); export type RouteSetter = (server: RouteManager) => void; //export let disconnect: Function; -export let resolvedPorts: { server: number, socket: number } = { server: 1050, socket: 4321 }; +export let resolvedPorts: { server: number; socket: number } = { server: 1050, socket: 4321 }; export let resolvedServerUrl: string; export default async function InitializeServer(routeSetter: RouteSetter) { @@ -42,33 +42,32 @@ export default async function InitializeServer(routeSetter: RouteSetter) { const compiler = webpack(config); - app.use(require("webpack-dev-middleware")(compiler, { - publicPath: config.output.publicPath - })); + app.use( + require('webpack-dev-middleware')(compiler, { + publicPath: config.output.publicPath, + }) + ); - app.use(require("webpack-hot-middleware")(compiler)); + app.use(require('webpack-hot-middleware')(compiler)); // route table managed by express. routes are tested sequentially against each of these map rules. when a match is found, the handler is called to process the request - app.get(new RegExp(/^\/+$/), (req, res) => res.redirect(req.user ? "/home" : "/login")); // target urls that consist of one or more '/'s with nothing in between - app.use(express.static(publicDirectory, { setHeaders: res => res.setHeader("Access-Control-Allow-Origin", "*") })); //all urls that start with dash's public directory: /files/ (e.g., /files/images, /files/audio, etc) + app.get(new RegExp(/^\/+$/), (req, res) => res.redirect(req.user ? '/home' : '/login')); // target urls that consist of one or more '/'s with nothing in between + app.use(express.static(publicDirectory, { setHeaders: res => res.setHeader('Access-Control-Allow-Origin', '*') })); //all urls that start with dash's public directory: /files/ (e.g., /files/images, /files/audio, etc) app.use(cors({ origin: (_origin: any, callback: any) => callback(null, true) })); app.use(wdm(compiler, { publicPath: config.output.publicPath })); app.use(whm(compiler)); - registerAuthenticationRoutes(app); // this adds routes to authenticate a user (login, etc) - registerCorsProxy(app); // this adds a /corsProxy/ route to allow clients to get to urls that would otherwise be blocked by cors policies + registerAuthenticationRoutes(app); // this adds routes to authenticate a user (login, etc) + registerCorsProxy(app); // this adds a /corsProxy/ route to allow clients to get to urls that would otherwise be blocked by cors policies isRelease && !SSL.Loaded && SSL.exit(); routeSetter(new RouteManager(app, isRelease)); // this sets up all the regular supervised routes (things like /home, download/upload api's, pdf, search, session, etc) registerEmbeddedBrowseRelativePathHandler(app); // this allows renered web pages which internally have relative paths to find their content let server: HttpServer | HttpsServer; isRelease && process.env.serverPort && (resolvedPorts.server = Number(process.env.serverPort)); - await new Promise<void>(resolve => server = isRelease ? - createServer(SSL.Credentials, app).listen(resolvedPorts.server, resolve) : - app.listen(resolvedPorts.server, resolve) - ); - logPort("server", resolvedPorts.server); + await new Promise<void>(resolve => (server = isRelease ? createServer(SSL.Credentials, app).listen(resolvedPorts.server, resolve) : app.listen(resolvedPorts.server, resolve))); + logPort('server', resolvedPorts.server); - resolvedServerUrl = `${isRelease && process.env.serverName ? `https://${process.env.serverName}.com` : "http://localhost"}:${resolvedPorts.server}`; + resolvedServerUrl = `${isRelease && process.env.serverName ? `https://${process.env.serverName}.com` : 'http://localhost'}:${resolvedPorts.server}`; // initialize the web socket (bidirectional communication: if a user changes // a field on one client, that change must be broadcast to all other clients) @@ -79,7 +78,7 @@ export default async function InitializeServer(routeSetter: RouteSetter) { } const week = 7 * 24 * 60 * 60 * 1000; -const secret = "64d6866242d3b5a5503c675b32c9605e4e90478e9b77bcf2bc"; +const secret = '64d6866242d3b5a5503c675b32c9605e4e90478e9b77bcf2bc'; function buildWithMiddleware(server: express.Express) { [ @@ -89,18 +88,18 @@ function buildWithMiddleware(server: express.Express) { resave: true, cookie: { maxAge: week }, saveUninitialized: true, - store: process.env.DB === "MEM" ? new session.MemoryStore() : new MongoStore({ url: Database.url }) + store: process.env.DB === 'MEM' ? new session.MemoryStore() : new MongoStore({ url: Database.url }), }), flash(), expressFlash(), - bodyParser.json({ limit: "10mb" }), + bodyParser.json({ limit: '10mb' }), bodyParser.urlencoded({ extended: true }), expressValidator(), passport.initialize(), passport.session(), (req: express.Request, res: express.Response, next: express.NextFunction) => { res.locals.user = req.user; - if ((req.originalUrl.endsWith(".png") /*|| req.originalUrl.endsWith(".js")*/) && req.method === 'GET' && (res as any)._contentLength) { + if (req.originalUrl.endsWith('.png') /*|| req.originalUrl.endsWith(".js")*/ && req.method === 'GET' && (res as any)._contentLength) { const period = 30000; res.set('Cache-control', `public, max-age=${period}`); } else { @@ -108,61 +107,61 @@ function buildWithMiddleware(server: express.Express) { res.set('Cache-control', `no-store`); } next(); - } + }, ].forEach(next => server.use(next)); return server; } /* Determine if the enviroment is dev mode or release mode. */ function determineEnvironment() { - const isRelease = process.env.RELEASE === "true"; + const isRelease = process.env.RELEASE === 'true'; const color = isRelease ? blue : yellow; - const label = isRelease ? "release" : "development"; + const label = isRelease ? 'release' : 'development'; console.log(`\nrunning server in ${color(label)} mode`); // swilkins: I don't think we need to read from ClientUtils.RELEASE anymore. Should be able to invoke process.env.RELEASE // on the client side, thanks to dotenv in webpack.config.js - let clientUtils = fs.readFileSync("./src/client/util/ClientUtils.ts.temp", "utf8"); + let clientUtils = fs.readFileSync('./src/client/util/ClientUtils.ts.temp', 'utf8'); clientUtils = `//AUTO-GENERATED FILE: DO NOT EDIT\n${clientUtils.replace('"mode"', String(isRelease))}`; - fs.writeFileSync("./src/client/util/ClientUtils.ts", clientUtils, "utf8"); + fs.writeFileSync('./src/client/util/ClientUtils.ts', clientUtils, 'utf8'); return isRelease; } function registerAuthenticationRoutes(server: express.Express) { - server.get("/signup", getSignup); - server.post("/signup", postSignup); + server.get('/signup', getSignup); + server.post('/signup', postSignup); - server.get("/login", getLogin); - server.post("/login", postLogin); + server.get('/login', getLogin); + server.post('/login', postLogin); - server.get("/logout", getLogout); + server.get('/logout', getLogout); - server.get("/forgotPassword", getForgot); - server.post("/forgotPassword", postForgot); + server.get('/forgotPassword', getForgot); + server.post('/forgotPassword', postForgot); - const reset = new RouteSubscriber("resetPassword").add("token").build; + const reset = new RouteSubscriber('resetPassword').add('token').build; server.get(reset, getReset); server.post(reset, postReset); } function registerCorsProxy(server: express.Express) { - server.use("/corsProxy", async (req, res) => { - const referer = req.headers.referer ? decodeURIComponent(req.headers.referer) : ""; + server.use('/corsProxy', async (req, res) => { + const referer = req.headers.referer ? decodeURIComponent(req.headers.referer) : ''; let requrlraw = decodeURIComponent(req.url.substring(1)); - const qsplit = requrlraw.split("?q="); - const newqsplit = requrlraw.split("&q="); + const qsplit = requrlraw.split('?q='); + const newqsplit = requrlraw.split('&q='); if (qsplit.length > 1 && newqsplit.length > 1) { const lastq = newqsplit[newqsplit.length - 1]; - requrlraw = qsplit[0] + "?q=" + lastq.split("&")[0] + "&" + qsplit[1].split("&")[1]; + requrlraw = qsplit[0] + '?q=' + lastq.split('&')[0] + '&' + qsplit[1].split('&')[1]; } - const requrl = requrlraw.startsWith("/") ? referer + requrlraw : requrlraw; + const requrl = requrlraw.startsWith('/') ? referer + requrlraw : requrlraw; // cors weirdness here... // if the referer is a cors page and the cors() route (I think) redirected to /corsProxy/<path> and the requested url path was relative, // then we redirect again to the cors referer and just add the relative path. - if (!requrl.startsWith("http") && req.originalUrl.startsWith("/corsProxy") && referer?.includes("corsProxy")) { - res.redirect(referer + (referer.endsWith("/") ? "" : "/") + requrl); + if (!requrl.startsWith('http') && req.originalUrl.startsWith('/corsProxy') && referer?.includes('corsProxy')) { + res.redirect(referer + (referer.endsWith('/') ? '' : '/') + requrl); } else { proxyServe(req, requrl, res); } @@ -173,34 +172,40 @@ function proxyServe(req: any, requrl: string, response: any) { const htmlBodyMemoryStream = new (require('memorystream'))(); var retrieveHTTPBody: any; const sendModifiedBody = () => { - const header = response.headers["content-encoding"]; - if (header && header.includes("gzip")) { + const header = response.headers['content-encoding']; + if (header?.includes('gzip')) { try { const replacer = (match: any, href: string, offset: any, string: any) => { - return `href="${resolvedServerUrl + "/corsProxy/http" + href}"`; + return `href="${resolvedServerUrl + '/corsProxy/http' + href}"`; }; const zipToStringDecoder = new (require('string_decoder').StringDecoder)('utf8'); const bodyStream = htmlBodyMemoryStream.read(); if (bodyStream) { - const htmlText = zipToStringDecoder.write(zlib.gunzipSync(bodyStream).toString('utf8') - .replace('<head>', '<head> <style>[id ^= "google"] { display: none; } </style>') - .replace(/href="https?([^"]*)"/g, replacer) - .replace(/target="_blank"/g, "")); + const htmlText = zipToStringDecoder.write( + zlib + .gunzipSync(bodyStream) + .toString('utf8') + .replace('<head>', '<head> <style>[id ^= "google"] { display: none; } </style>') + .replace(/href="https?([^"]*)"/g, replacer) + .replace(/target="_blank"/g, '') + ); response.send(zlib.gzipSync(htmlText)); } else { req.pipe(request(requrl)).pipe(response); - console.log("EMPTY body:" + req.url); + console.log('EMPTY body:' + req.url); } } catch (e) { - console.log("EROR?: ", e); + console.log('EROR?: ', e); } - } else req.pipe(request(requrl)).pipe(response); + } else { + req.pipe(htmlBodyMemoryStream).pipe(response); + } }; retrieveHTTPBody = () => { - req.headers.cookie = ""; + req.headers.cookie = ''; req.pipe(request(requrl)) - .on("error", (e: any) => console.log(`Malformed CORS url: ${requrl}`, e)) - .on("response", (res: any) => { + .on('error', (e: any) => console.log(`Malformed CORS url: ${requrl}`, e)) + .on('response', (res: any) => { res.headers; const headers = Object.keys(res.headers); const headerCharRegex = /[^\t\x20-\x7e\x80-\xff]/; @@ -208,36 +213,41 @@ function proxyServe(req: any, requrl: string, response: any) { const header = res.headers[headerName]; if (Array.isArray(header)) { res.headers[headerName] = header.filter(h => !headerCharRegex.test(h)); - } else if (headerCharRegex.test(header || "")) { + } else if (headerCharRegex.test(header || '')) { delete res.headers[headerName]; } else res.headers[headerName] = header; }); + res.headers['x-permitted-cross-domain-policies'] = 'all'; + res.headers['x-frame-options'] = ''; + res.headers['content-security-policy'] = ''; response.headers = response._headers = res.headers; }) - .on("end", sendModifiedBody) + .on('end', sendModifiedBody) .pipe(htmlBodyMemoryStream); }; retrieveHTTPBody(); } function registerEmbeddedBrowseRelativePathHandler(server: express.Express) { - server.use("*", (req, res) => { + server.use('*', (req, res) => { const relativeUrl = req.originalUrl; - if (!req.user) res.redirect("/home"); // When no user is logged in, we interpret a relative URL as being a reference to something they don't have access to and redirect to /home - else if (!res.headersSent && req.headers.referer?.includes("corsProxy")) { // a request for something by a proxied referrer means it must be a relative reference. So construct a proxied absolute reference here. + if (!req.user) res.redirect('/home'); // When no user is logged in, we interpret a relative URL as being a reference to something they don't have access to and redirect to /home + else if (!res.headersSent && req.headers.referer?.includes('corsProxy')) { + // a request for something by a proxied referrer means it must be a relative reference. So construct a proxied absolute reference here. try { const proxiedRefererUrl = decodeURIComponent(req.headers.referer); // (e.g., http://localhost:<port>/corsProxy/https://en.wikipedia.org/wiki/Engelbart) const dashServerUrl = proxiedRefererUrl.match(/.*corsProxy\//)![0]; // the dash server url (e.g.: http://localhost:<port>/corsProxy/ ) - const actualReferUrl = proxiedRefererUrl.replace(dashServerUrl, ""); // the url of the referer without the proxy (e.g., : https://en.wikipedia.org/wiki/Engelbart) + const actualReferUrl = proxiedRefererUrl.replace(dashServerUrl, ''); // the url of the referer without the proxy (e.g., : https://en.wikipedia.org/wiki/Engelbart) const absoluteTargetBaseUrl = actualReferUrl.match(/https?:\/\/[^\/]*/)![0]; // the base of the original url (e.g., https://en.wikipedia.org) const redirectedProxiedUrl = dashServerUrl + encodeURIComponent(absoluteTargetBaseUrl + relativeUrl); // the new proxied full url (e.g., http://localhost:<port>/corsProxy/https://en.wikipedia.org/<somethingelse>) - if (relativeUrl.startsWith("//")) res.redirect("http:" + relativeUrl); + if (relativeUrl.startsWith('//')) res.redirect('http:' + relativeUrl); else res.redirect(redirectedProxiedUrl); } catch (e) { - console.log("Error embed: ", e); + console.log('Error embed: ', e); } - } else if (relativeUrl.startsWith("/search") && !req.headers.referer?.includes("corsProxy")) { // detect search query and use default search engine - res.redirect(req.headers.referer + "corsProxy/" + encodeURIComponent("http://www.google.com" + relativeUrl)); + } else if (relativeUrl.startsWith('/search') && !req.headers.referer?.includes('corsProxy')) { + // detect search query and use default search engine + res.redirect(req.headers.referer + 'corsProxy/' + encodeURIComponent('http://www.google.com' + relativeUrl)); } else { res.end(); } |