diff options
Diffstat (limited to 'src/server')
| -rw-r--r-- | src/server/ApiManagers/AzureManager.ts | 67 | ||||
| -rw-r--r-- | src/server/ApiManagers/UploadManager.ts | 9 | ||||
| -rw-r--r-- | src/server/ApiManagers/UserManager.ts | 16 | ||||
| -rw-r--r-- | src/server/DashSession/Session/utilities/session_config.ts | 2 | ||||
| -rw-r--r-- | src/server/DashUploadUtils.ts | 71 | ||||
| -rw-r--r-- | src/server/server_Initialization.ts | 32 |
6 files changed, 169 insertions, 28 deletions
diff --git a/src/server/ApiManagers/AzureManager.ts b/src/server/ApiManagers/AzureManager.ts new file mode 100644 index 000000000..12bb98ad0 --- /dev/null +++ b/src/server/ApiManagers/AzureManager.ts @@ -0,0 +1,67 @@ +import { ContainerClient, BlobServiceClient } from "@azure/storage-blob"; +import * as fs from "fs"; +import { Readable, Stream } from "stream"; +const AZURE_STORAGE_CONNECTION_STRING = process.env.AZURE_STORAGE_CONNECTION_STRING; + +export class AzureManager { + private _containerClient: ContainerClient; + private _blobServiceClient: BlobServiceClient; + private static _instance: AzureManager | undefined; + + public static CONTAINER_NAME = "dashmedia"; + public static STORAGE_ACCOUNT_NAME = "dashblobstore"; + + constructor() { + if (!AZURE_STORAGE_CONNECTION_STRING) { + throw new Error("Azure Storage Connection String Not Found"); + } + this._blobServiceClient = BlobServiceClient.fromConnectionString(AZURE_STORAGE_CONNECTION_STRING); + this._containerClient = this.BlobServiceClient.getContainerClient(AzureManager.CONTAINER_NAME); + } + + public static get Instance() { + return this._instance = this._instance ?? new AzureManager(); + } + + public get BlobServiceClient() { + return this._blobServiceClient; + } + + public get ContainerClient() { + return this._containerClient; + } + + public static UploadBlob(filename: string, filepath: string, filetype: string) { + const blockBlobClient = this.Instance.ContainerClient.getBlockBlobClient(filename); + const blobOptions = { blobHTTPHeaders: { blobContentType: filetype }}; + const stream = fs.createReadStream(filepath); + return blockBlobClient.uploadStream(stream, undefined, undefined, blobOptions); + } + + public static UploadBlobStream(stream: Readable, filename: string, filetype: string) { + const blockBlobClient = this.Instance.ContainerClient.getBlockBlobClient(filename); + const blobOptions = { blobHTTPHeaders: { blobContentType: filetype }}; + return blockBlobClient.uploadStream(stream, undefined, undefined, blobOptions); + } + + public static DeleteBlob(filename: string) { + const blockBlobClient = this.Instance.ContainerClient.getBlockBlobClient(filename); + return blockBlobClient.deleteIfExists(); + } + + public static async GetBlobs() { + const foundBlobs = []; + for await (const blob of this.Instance.ContainerClient.listBlobsFlat()) { + console.log(`${blob.name}`); + + const blobItem = { + url : `https://${AzureManager.STORAGE_ACCOUNT_NAME}.blob.core.windows.net/${AzureManager.CONTAINER_NAME}/${blob.name}`, + name : blob.name + } + + foundBlobs.push(blobItem); + } + + return foundBlobs; + } +} diff --git a/src/server/ApiManagers/UploadManager.ts b/src/server/ApiManagers/UploadManager.ts index ba6d7acfe..ebc9deab7 100644 --- a/src/server/ApiManagers/UploadManager.ts +++ b/src/server/ApiManagers/UploadManager.ts @@ -12,6 +12,7 @@ import { AcceptableMedia, Upload } from '../SharedMediaTypes'; import ApiManager, { Registration } from './ApiManager'; import { SolrManager } from './SearchManager'; import v4 = require('uuid/v4'); +import { DashVersion } from '../../fields/DocSymbols'; const AdmZip = require('adm-zip'); const imageDataUri = require('image-data-uri'); const fs = require('fs'); @@ -43,6 +44,14 @@ export default class UploadManager extends ApiManager { protected initialize(register: Registration): void { register({ method: Method.POST, + subscription: '/ping', + secureHandler: async ({ req, res }) => { + _success(res, { message: DashVersion, date: new Date() }); + }, + }); + + register({ + method: Method.POST, subscription: '/concatVideos', secureHandler: async ({ req, res }) => { // req.body contains the array of server paths to the videos diff --git a/src/server/ApiManagers/UserManager.ts b/src/server/ApiManagers/UserManager.ts index c3dadd821..8b7994eac 100644 --- a/src/server/ApiManagers/UserManager.ts +++ b/src/server/ApiManagers/UserManager.ts @@ -5,7 +5,8 @@ import { msToTime } from '../ActionUtilities'; import * as bcrypt from 'bcrypt-nodejs'; import { Opt } from '../../fields/Doc'; import { WebSocket } from '../websocket'; -import { DashStats } from '../DashStats'; +import { resolvedPorts } from '../server_Initialization'; +import { DashVersion } from '../../fields/DocSymbols'; export const timeMap: { [id: string]: number } = {}; interface ActivityUnit { @@ -68,7 +69,18 @@ export default class UserManager extends ApiManager { register({ method: Method.GET, subscription: '/getCurrentUser', - secureHandler: ({ res, user: { _id, email, cacheDocumentIds } }) => res.send(JSON.stringify({ id: _id, email, cacheDocumentIds })), + secureHandler: ({ res, user }) => + res.send( + JSON.stringify({ + version: DashVersion, + userDocumentId: user.userDocumentId, + linkDatabaseId: user.linkDatabaseId, + sharingDocumentId: user.sharingDocumentId, + email: user.email, + cacheDocumentIds: user.cacheDocumentIds, + resolvedPorts, + }) + ), publicHandler: ({ res }) => res.send(JSON.stringify({ id: '__guest__', email: 'guest' })), }); diff --git a/src/server/DashSession/Session/utilities/session_config.ts b/src/server/DashSession/Session/utilities/session_config.ts index bde98e9d2..266759929 100644 --- a/src/server/DashSession/Session/utilities/session_config.ts +++ b/src/server/DashSession/Session/utilities/session_config.ts @@ -120,7 +120,7 @@ export const defaultConfig: Configuration = { color: "green" } }, - ports: { server: 3000 }, + ports: { server: 1050 }, polling: { route: "/", intervalSeconds: 30, diff --git a/src/server/DashUploadUtils.ts b/src/server/DashUploadUtils.ts index eaaac4e6d..117981a7c 100644 --- a/src/server/DashUploadUtils.ts +++ b/src/server/DashUploadUtils.ts @@ -6,7 +6,7 @@ import { createReadStream, createWriteStream, existsSync, readFileSync, rename, import * as path from 'path'; import { basename } from 'path'; import * as sharp from 'sharp'; -import { Stream } from 'stream'; +import { Readable, Stream } from 'stream'; import { filesDirectory, publicDirectory } from '.'; import { Opt } from '../fields/Doc'; import { ParsedPDF } from '../server/PdfTypes'; @@ -17,6 +17,8 @@ import { resolvedServerUrl } from './server_Initialization'; import { AcceptableMedia, Upload } from './SharedMediaTypes'; import request = require('request-promise'); import formidable = require('formidable'); +import { AzureManager } from './ApiManagers/AzureManager'; +import axios from 'axios'; const spawn = require('child_process').spawn; const { exec } = require('child_process'); const parse = require('pdf-parse'); @@ -42,6 +44,10 @@ function isLocal() { return /Dash-Web[0-9]*[\\\/]src[\\\/]server[\\\/]public[\\\/](.*)/; } +function usingAzure() { + return process.env.USE_AZURE === 'true'; +} + export namespace DashUploadUtils { export interface Size { width: number; @@ -61,6 +67,9 @@ export namespace DashUploadUtils { const size = 'content-length'; const type = 'content-type'; + const BLOBSTORE_URL = process.env.BLOBSTORE_URL; + const RESIZE_FUNCTION_URL = process.env.RESIZE_FUNCTION_URL; + const { imageFormats, videoFormats, applicationFormats, audioFormats } = AcceptableMedia; //TODO:glr export async function concatVideos(filePaths: string[]): Promise<Upload.AccessPathInfo> { @@ -90,7 +99,7 @@ export namespace DashUploadUtils { merge .input(textFilePath) .inputOptions(['-f concat', '-safe 0']) - .outputOptions('-c copy') + // .outputOptions('-c copy') //.videoCodec("copy") .save(outputFilePath) .on('error', (err: any) => { @@ -134,7 +143,6 @@ export namespace DashUploadUtils { export function uploadYoutube(videoId: string): Promise<Upload.FileResponse> { return new Promise<Upload.FileResponse<Upload.FileInformation>>((res, rej) => { - console.log('Uploading YouTube video: ' + videoId); const name = videoId; const path = name.replace(/^-/, '__') + '.mp4'; const finalPath = serverPathToFile(Directory.videos, path); @@ -182,9 +190,10 @@ export namespace DashUploadUtils { } export async function upload(file: File, overwriteGuid?: string): Promise<Upload.FileResponse> { + const isAzureOn = usingAzure(); const { type, path, name } = file; const types = type?.split('/') ?? []; - uploadProgress.set(overwriteGuid ?? name, 'uploading'); // If the client sent a guid it uses to track upload progress, use that guid. Otherwise, use the file's name. + uploadProgress.set(overwriteGuid ?? name, 'uploading'); // If the client sent a guid it uses to track upload progress, use that guid. Otherwise, use the file's name. const category = types[0]; let format = `.${types[1]}`; @@ -478,17 +487,51 @@ export namespace DashUploadUtils { }; } + /** + * UploadInspectedImage() takes an image with its metadata. If Azure is being used, this method will call the Azure function + * to execute the resizing. If Azure is not used, the function will begin to resize the image. + * + * @param metadata metadata object from InspectImage() + * @param filename the name of the file + * @param prefix the prefix to use, which will be set to '' if none is provided. + * @param cleanUp a boolean indicating if the files should be deleted after upload. True by default. + * @returns the accessPaths for the resized files. + */ export const UploadInspectedImage = async (metadata: Upload.InspectionResults, filename?: string, prefix = '', cleanUp = true): Promise<Upload.ImageInformation> => { const { requestable, source, ...remaining } = metadata; const resolved = filename || `${prefix}upload_${Utils.GenerateGuid()}.${remaining.contentType.split('/')[1].toLowerCase()}`; const { images } = Directory; const information: Upload.ImageInformation = { accessPaths: { - agnostic: getAccessPaths(images, resolved), + agnostic: usingAzure() + ? { + client: BLOBSTORE_URL + `/${resolved}`, + server: BLOBSTORE_URL + `/${resolved}`, + } + : getAccessPaths(images, resolved), }, ...metadata, }; - const writtenFiles = await outputResizedImages(() => request(requestable), resolved, pathToDirectory(Directory.images)); + let writtenFiles: { [suffix: string]: string }; + + if (usingAzure()) { + if (!RESIZE_FUNCTION_URL) { + throw new Error('Resize function URL not provided.'); + } + + try { + const response = await axios.post(RESIZE_FUNCTION_URL, { + url: requestable, + filename: resolved, + }); + writtenFiles = response.data.writtenFiles; + } catch (err) { + console.error(err); + writtenFiles = {}; + } + } else { + writtenFiles = await outputResizedImages(() => request(requestable), resolved, pathToDirectory(Directory.images)); + } for (const suffix of Object.keys(writtenFiles)) { information.accessPaths[suffix] = getAccessPaths(images, writtenFiles[suffix]); } @@ -533,6 +576,15 @@ export namespace DashUploadUtils { force: true, }; + /** + * outputResizedImages takes in a readable stream and resizes the images according to the sizes defined at the top of this file. + * + * The new images will be saved to the server with the corresponding prefixes. + * @param streamProvider a Stream of the image to process, taken from the /parsed_files location + * @param outputFileName the basename (No suffix) of the outputted file. + * @param outputDirectory the directory to output to, usually Directory.Images + * @returns a map with suffixes as keys and resized filenames as values. + */ export async function outputResizedImages(streamProvider: () => Stream | Promise<Stream>, outputFileName: string, outputDirectory: string) { const writtenFiles: { [suffix: string]: string } = {}; for (const { resizer, suffix } of resizers(path.extname(outputFileName))) { @@ -549,11 +601,16 @@ export namespace DashUploadUtils { return writtenFiles; } + /** + * define the resizers to use + * @param ext the extension + * @returns an array of resizer functions from sharp + */ function resizers(ext: string): DashUploadUtils.ImageResizer[] { return [ { suffix: SizeSuffix.Original }, ...Object.values(DashUploadUtils.Sizes).map(({ suffix, width }) => { - let initial: sharp.Sharp | undefined = sharp().resize(width, undefined, { withoutEnlargement: true }); + let initial: sharp.Sharp | undefined = sharp({ failOnError: false }).resize(width, undefined, { withoutEnlargement: true }); if (pngs.includes(ext)) { initial = initial.png(pngOptions); } else if (jpgs.includes(ext)) { diff --git a/src/server/server_Initialization.ts b/src/server/server_Initialization.ts index 805da1d43..ee32de152 100644 --- a/src/server/server_Initialization.ts +++ b/src/server/server_Initialization.ts @@ -100,7 +100,7 @@ function buildWithMiddleware(server: express.Express) { passport.session(), (req: express.Request, res: express.Response, next: express.NextFunction) => { res.locals.user = req.user; - if (req.originalUrl.endsWith('.png') /*|| req.originalUrl.endsWith(".js")*/ && req.method === 'GET' && (res as any)._contentLength) { + if ((req.originalUrl.endsWith('.png') || req.originalUrl.endsWith('.jpg') || (process.env.RELEASE === 'true' && req.originalUrl.endsWith('.js'))) && req.method === 'GET') { const period = 30000; res.set('Cache-control', `public, max-age=${period}`); } else { @@ -149,23 +149,15 @@ function registerAuthenticationRoutes(server: express.Express) { function registerCorsProxy(server: express.Express) { server.use('/corsProxy', async (req, res) => { - const referer = req.headers.referer ? decodeURIComponent(req.headers.referer) : ''; - let requrlraw = decodeURIComponent(req.url.substring(1)); - const qsplit = requrlraw.split('?q='); - const newqsplit = requrlraw.split('&q='); + //const referer = req.headers.referer ? decodeURIComponent(req.headers.referer) : ''; + let requrl = decodeURIComponent(req.url.substring(1)); + const qsplit = requrl.split('?q='); + const newqsplit = requrl.split('&q='); if (qsplit.length > 1 && newqsplit.length > 1) { const lastq = newqsplit[newqsplit.length - 1]; - requrlraw = qsplit[0] + '?q=' + lastq.split('&')[0] + '&' + qsplit[1].split('&')[1]; - } - const requrl = requrlraw.startsWith('/') ? referer + requrlraw : requrlraw; - // cors weirdness here... - // if the referer is a cors page and the cors() route (I think) redirected to /corsProxy/<path> and the requested url path was relative, - // then we redirect again to the cors referer and just add the relative path. - if (!requrl.startsWith('http') && req.originalUrl.startsWith('/corsProxy') && referer?.includes('corsProxy')) { - res.redirect(referer + (referer.endsWith('/') ? '' : '/') + requrl); - } else { - proxyServe(req, requrl, res); + requrl = qsplit[0] + '?q=' + lastq.split('&')[0] + '&' + qsplit[1].split('&')[1]; } + proxyServe(req, requrl, res); }); } @@ -184,7 +176,9 @@ function proxyServe(req: any, requrl: string, response: any) { const htmlText = htmlInputText .toString('utf8') .replace('<head>', '<head> <style>[id ^= "google"] { display: none; } </style>') - .replace(/href="https?([^"]*)"/g, httpsToCors) + // .replace('<script', '<noscript') + // .replace('</script', '</noscript') + // .replace(/href="https?([^"]*)"/g, httpsToCors) .replace(/data-srcset="[^"]*"/g, '') .replace(/srcset="[^"]*"/g, '') .replace(/target="_blank"/g, ''); @@ -234,8 +228,10 @@ function proxyServe(req: any, requrl: string, response: any) { function registerEmbeddedBrowseRelativePathHandler(server: express.Express) { server.use('*', (req, res) => { const relativeUrl = req.originalUrl; - if (!req.user) res.redirect('/home'); // When no user is logged in, we interpret a relative URL as being a reference to something they don't have access to and redirect to /home - else if (!res.headersSent && req.headers.referer?.includes('corsProxy')) { + // if (req.originalUrl === '/css/main.css' || req.originalUrl === '/favicon.ico') res.end(); + // else + if (!res.headersSent && req.headers.referer?.includes('corsProxy')) { + if (!req.user) res.redirect('/home'); // When no user is logged in, we interpret a relative URL as being a reference to something they don't have access to and redirect to /home // a request for something by a proxied referrer means it must be a relative reference. So construct a proxied absolute reference here. try { const proxiedRefererUrl = decodeURIComponent(req.headers.referer); // (e.g., http://localhost:<port>/corsProxy/https://en.wikipedia.org/wiki/Engelbart) |