Merge branch 'master' into advanced-trails

6 files changed, 169 insertions, 28 deletions

diff --git a/src/server/ApiManagers/AzureManager.ts b/src/server/ApiManagers/AzureManager.ts
new file mode 100644
index 000000000..12bb98ad0
--- /dev/null
+++ b/src/server/ApiManagers/AzureManager.ts
@@ -0,0 +1,67 @@
+import { ContainerClient, BlobServiceClient } from "@azure/storage-blob";
+import * as fs from "fs";
+import { Readable, Stream } from "stream";
+const AZURE_STORAGE_CONNECTION_STRING = process.env.AZURE_STORAGE_CONNECTION_STRING;
+
+export class AzureManager {
+    private _containerClient: ContainerClient;
+    private _blobServiceClient: BlobServiceClient;
+    private static _instance: AzureManager | undefined;
+
+    public static CONTAINER_NAME = "dashmedia";
+    public static STORAGE_ACCOUNT_NAME = "dashblobstore";
+
+    constructor() {
+        if (!AZURE_STORAGE_CONNECTION_STRING) {
+            throw new Error("Azure Storage Connection String Not Found");
+        }
+        this._blobServiceClient = BlobServiceClient.fromConnectionString(AZURE_STORAGE_CONNECTION_STRING);
+        this._containerClient = this.BlobServiceClient.getContainerClient(AzureManager.CONTAINER_NAME);
+    }
+
+    public static get Instance() {
+        return this._instance = this._instance ?? new AzureManager();
+    }
+
+    public get BlobServiceClient() {
+        return this._blobServiceClient;
+    }
+
+    public get ContainerClient() {
+        return this._containerClient;
+    }
+
+    public static UploadBlob(filename: string, filepath: string, filetype: string) {
+        const blockBlobClient = this.Instance.ContainerClient.getBlockBlobClient(filename);
+        const blobOptions = { blobHTTPHeaders: { blobContentType: filetype }};
+        const stream = fs.createReadStream(filepath);
+        return blockBlobClient.uploadStream(stream, undefined, undefined, blobOptions);
+    }
+
+    public static UploadBlobStream(stream: Readable, filename: string, filetype: string) {
+        const blockBlobClient = this.Instance.ContainerClient.getBlockBlobClient(filename);
+        const blobOptions = { blobHTTPHeaders: { blobContentType: filetype }};
+        return blockBlobClient.uploadStream(stream, undefined, undefined, blobOptions);
+    }
+
+    public static DeleteBlob(filename: string) {
+        const blockBlobClient = this.Instance.ContainerClient.getBlockBlobClient(filename);
+        return blockBlobClient.deleteIfExists();
+    }
+
+    public static async GetBlobs() {
+        const foundBlobs = [];
+        for await (const blob of this.Instance.ContainerClient.listBlobsFlat()) {
+            console.log(`${blob.name}`);
+
+            const blobItem = {
+                url : `https://${AzureManager.STORAGE_ACCOUNT_NAME}.blob.core.windows.net/${AzureManager.CONTAINER_NAME}/${blob.name}`,
+                name : blob.name
+            }
+
+            foundBlobs.push(blobItem);
+        }
+
+        return foundBlobs;
+    }
+}
diff --git a/src/server/ApiManagers/UploadManager.ts b/src/server/ApiManagers/UploadManager.ts
index ba6d7acfe..ebc9deab7 100644
--- a/src/server/ApiManagers/UploadManager.ts
+++ b/src/server/ApiManagers/UploadManager.ts
@@ -12,6 +12,7 @@ import { AcceptableMedia, Upload } from '../SharedMediaTypes';
 import ApiManager, { Registration } from './ApiManager';
 import { SolrManager } from './SearchManager';
 import v4 = require('uuid/v4');
+import { DashVersion } from '../../fields/DocSymbols';
 const AdmZip = require('adm-zip');
 const imageDataUri = require('image-data-uri');
 const fs = require('fs');
@@ -43,6 +44,14 @@ export default class UploadManager extends ApiManager {
     protected initialize(register: Registration): void {
         register({
             method: Method.POST,
+            subscription: '/ping',
+            secureHandler: async ({ req, res }) => {
+                _success(res, { message: DashVersion, date: new Date() });
+            },
+        });
+
+        register({
+            method: Method.POST,
             subscription: '/concatVideos',
             secureHandler: async ({ req, res }) => {
                 // req.body contains the array of server paths to the videos
diff --git a/src/server/ApiManagers/UserManager.ts b/src/server/ApiManagers/UserManager.ts
index c3dadd821..8b7994eac 100644
--- a/src/server/ApiManagers/UserManager.ts
+++ b/src/server/ApiManagers/UserManager.ts
@@ -5,7 +5,8 @@ import { msToTime } from '../ActionUtilities';
 import * as bcrypt from 'bcrypt-nodejs';
 import { Opt } from '../../fields/Doc';
 import { WebSocket } from '../websocket';
-import { DashStats } from '../DashStats';
+import { resolvedPorts } from '../server_Initialization';
+import { DashVersion } from '../../fields/DocSymbols';
 
 export const timeMap: { [id: string]: number } = {};
 interface ActivityUnit {
@@ -68,7 +69,18 @@ export default class UserManager extends ApiManager {
         register({
             method: Method.GET,
             subscription: '/getCurrentUser',
-            secureHandler: ({ res, user: { _id, email, cacheDocumentIds } }) => res.send(JSON.stringify({ id: _id, email, cacheDocumentIds })),
+            secureHandler: ({ res, user }) =>
+                res.send(
+                    JSON.stringify({
+                        version: DashVersion,
+                        userDocumentId: user.userDocumentId,
+                        linkDatabaseId: user.linkDatabaseId,
+                        sharingDocumentId: user.sharingDocumentId,
+                        email: user.email,
+                        cacheDocumentIds: user.cacheDocumentIds,
+                        resolvedPorts,
+                    })
+                ),
             publicHandler: ({ res }) => res.send(JSON.stringify({ id: '__guest__', email: 'guest' })),
         });
 
diff --git a/src/server/DashSession/Session/utilities/session_config.ts b/src/server/DashSession/Session/utilities/session_config.ts
index bde98e9d2..266759929 100644
--- a/src/server/DashSession/Session/utilities/session_config.ts
+++ b/src/server/DashSession/Session/utilities/session_config.ts
@@ -120,7 +120,7 @@ export const defaultConfig: Configuration = {
             color: "green"
         }
     },
-    ports: { server: 3000 },
+    ports: { server: 1050 },
     polling: {
         route: "/",
         intervalSeconds: 30,
diff --git a/src/server/DashUploadUtils.ts b/src/server/DashUploadUtils.ts
index eaaac4e6d..117981a7c 100644
--- a/src/server/DashUploadUtils.ts
+++ b/src/server/DashUploadUtils.ts
@@ -6,7 +6,7 @@ import { createReadStream, createWriteStream, existsSync, readFileSync, rename,
 import * as path from 'path';
 import { basename } from 'path';
 import * as sharp from 'sharp';
-import { Stream } from 'stream';
+import { Readable, Stream } from 'stream';
 import { filesDirectory, publicDirectory } from '.';
 import { Opt } from '../fields/Doc';
 import { ParsedPDF } from '../server/PdfTypes';
@@ -17,6 +17,8 @@ import { resolvedServerUrl } from './server_Initialization';
 import { AcceptableMedia, Upload } from './SharedMediaTypes';
 import request = require('request-promise');
 import formidable = require('formidable');
+import { AzureManager } from './ApiManagers/AzureManager';
+import axios from 'axios';
 const spawn = require('child_process').spawn;
 const { exec } = require('child_process');
 const parse = require('pdf-parse');
@@ -42,6 +44,10 @@ function isLocal() {
     return /Dash-Web[0-9]*[\\\/]src[\\\/]server[\\\/]public[\\\/](.*)/;
 }
 
+function usingAzure() {
+    return process.env.USE_AZURE === 'true';
+}
+
 export namespace DashUploadUtils {
     export interface Size {
         width: number;
@@ -61,6 +67,9 @@ export namespace DashUploadUtils {
     const size = 'content-length';
     const type = 'content-type';
 
+    const BLOBSTORE_URL = process.env.BLOBSTORE_URL;
+    const RESIZE_FUNCTION_URL = process.env.RESIZE_FUNCTION_URL;
+
     const { imageFormats, videoFormats, applicationFormats, audioFormats } = AcceptableMedia; //TODO:glr
 
     export async function concatVideos(filePaths: string[]): Promise<Upload.AccessPathInfo> {
@@ -90,7 +99,7 @@ export namespace DashUploadUtils {
             merge
                 .input(textFilePath)
                 .inputOptions(['-f concat', '-safe 0'])
-                .outputOptions('-c copy')
+                // .outputOptions('-c copy')
                 //.videoCodec("copy")
                 .save(outputFilePath)
                 .on('error', (err: any) => {
@@ -134,7 +143,6 @@ export namespace DashUploadUtils {
 
     export function uploadYoutube(videoId: string): Promise<Upload.FileResponse> {
         return new Promise<Upload.FileResponse<Upload.FileInformation>>((res, rej) => {
-            console.log('Uploading YouTube video: ' + videoId);
             const name = videoId;
             const path = name.replace(/^-/, '__') + '.mp4';
             const finalPath = serverPathToFile(Directory.videos, path);
@@ -182,9 +190,10 @@ export namespace DashUploadUtils {
     }
 
     export async function upload(file: File, overwriteGuid?: string): Promise<Upload.FileResponse> {
+        const isAzureOn = usingAzure();
         const { type, path, name } = file;
         const types = type?.split('/') ?? [];
-        uploadProgress.set(overwriteGuid ?? name, 'uploading'); // If the client sent a guid it uses to track upload progress, use that guid. Otherwise, use the file's name. 
+        uploadProgress.set(overwriteGuid ?? name, 'uploading'); // If the client sent a guid it uses to track upload progress, use that guid. Otherwise, use the file's name.
 
         const category = types[0];
         let format = `.${types[1]}`;
@@ -478,17 +487,51 @@ export namespace DashUploadUtils {
         };
     }
 
+    /**
+     * UploadInspectedImage() takes an image with its metadata. If Azure is being used, this method will call the Azure function
+     * to execute the resizing. If Azure is not used, the function will begin to resize the image.
+     *
+     * @param metadata metadata object from InspectImage()
+     * @param filename the name of the file
+     * @param prefix the prefix to use, which will be set to '' if none is provided.
+     * @param cleanUp a boolean indicating if the files should be deleted after upload. True by default.
+     * @returns the accessPaths for the resized files.
+     */
     export const UploadInspectedImage = async (metadata: Upload.InspectionResults, filename?: string, prefix = '', cleanUp = true): Promise<Upload.ImageInformation> => {
         const { requestable, source, ...remaining } = metadata;
         const resolved = filename || `${prefix}upload_${Utils.GenerateGuid()}.${remaining.contentType.split('/')[1].toLowerCase()}`;
         const { images } = Directory;
         const information: Upload.ImageInformation = {
             accessPaths: {
-                agnostic: getAccessPaths(images, resolved),
+                agnostic: usingAzure()
+                    ? {
+                          client: BLOBSTORE_URL + `/${resolved}`,
+                          server: BLOBSTORE_URL + `/${resolved}`,
+                      }
+                    : getAccessPaths(images, resolved),
             },
             ...metadata,
         };
-        const writtenFiles = await outputResizedImages(() => request(requestable), resolved, pathToDirectory(Directory.images));
+        let writtenFiles: { [suffix: string]: string };
+
+        if (usingAzure()) {
+            if (!RESIZE_FUNCTION_URL) {
+                throw new Error('Resize function URL not provided.');
+            }
+
+            try {
+                const response = await axios.post(RESIZE_FUNCTION_URL, {
+                    url: requestable,
+                    filename: resolved,
+                });
+                writtenFiles = response.data.writtenFiles;
+            } catch (err) {
+                console.error(err);
+                writtenFiles = {};
+            }
+        } else {
+            writtenFiles = await outputResizedImages(() => request(requestable), resolved, pathToDirectory(Directory.images));
+        }
         for (const suffix of Object.keys(writtenFiles)) {
             information.accessPaths[suffix] = getAccessPaths(images, writtenFiles[suffix]);
         }
@@ -533,6 +576,15 @@ export namespace DashUploadUtils {
         force: true,
     };
 
+    /**
+     * outputResizedImages takes in a readable stream and resizes the images according to the sizes defined at the top of this file.
+     *
+     * The new images will be saved to the server with the corresponding prefixes.
+     * @param streamProvider a Stream of the image to process, taken from the /parsed_files location
+     * @param outputFileName the basename (No suffix) of the outputted file.
+     * @param outputDirectory the directory to output to, usually Directory.Images
+     * @returns a map with suffixes as keys and resized filenames as values.
+     */
     export async function outputResizedImages(streamProvider: () => Stream | Promise<Stream>, outputFileName: string, outputDirectory: string) {
         const writtenFiles: { [suffix: string]: string } = {};
         for (const { resizer, suffix } of resizers(path.extname(outputFileName))) {
@@ -549,11 +601,16 @@ export namespace DashUploadUtils {
         return writtenFiles;
     }
 
+    /**
+     * define the resizers to use
+     * @param ext the extension
+     * @returns an array of resizer functions from sharp
+     */
     function resizers(ext: string): DashUploadUtils.ImageResizer[] {
         return [
             { suffix: SizeSuffix.Original },
             ...Object.values(DashUploadUtils.Sizes).map(({ suffix, width }) => {
-                let initial: sharp.Sharp | undefined = sharp().resize(width, undefined, { withoutEnlargement: true });
+                let initial: sharp.Sharp | undefined = sharp({ failOnError: false }).resize(width, undefined, { withoutEnlargement: true });
                 if (pngs.includes(ext)) {
                     initial = initial.png(pngOptions);
                 } else if (jpgs.includes(ext)) {
diff --git a/src/server/server_Initialization.ts b/src/server/server_Initialization.ts
index 805da1d43..ee32de152 100644
--- a/src/server/server_Initialization.ts
+++ b/src/server/server_Initialization.ts
@@ -100,7 +100,7 @@ function buildWithMiddleware(server: express.Express) {
         passport.session(),
         (req: express.Request, res: express.Response, next: express.NextFunction) => {
             res.locals.user = req.user;
-            if (req.originalUrl.endsWith('.png') /*|| req.originalUrl.endsWith(".js")*/ && req.method === 'GET' && (res as any)._contentLength) {
+            if ((req.originalUrl.endsWith('.png') || req.originalUrl.endsWith('.jpg') || (process.env.RELEASE === 'true' && req.originalUrl.endsWith('.js'))) && req.method === 'GET') {
                 const period = 30000;
                 res.set('Cache-control', `public, max-age=${period}`);
             } else {
@@ -149,23 +149,15 @@ function registerAuthenticationRoutes(server: express.Express) {
 
 function registerCorsProxy(server: express.Express) {
     server.use('/corsProxy', async (req, res) => {
-        const referer = req.headers.referer ? decodeURIComponent(req.headers.referer) : '';
-        let requrlraw = decodeURIComponent(req.url.substring(1));
-        const qsplit = requrlraw.split('?q=');
-        const newqsplit = requrlraw.split('&q=');
+        //const referer = req.headers.referer ? decodeURIComponent(req.headers.referer) : '';
+        let requrl = decodeURIComponent(req.url.substring(1));
+        const qsplit = requrl.split('?q=');
+        const newqsplit = requrl.split('&q=');
         if (qsplit.length > 1 && newqsplit.length > 1) {
             const lastq = newqsplit[newqsplit.length - 1];
-            requrlraw = qsplit[0] + '?q=' + lastq.split('&')[0] + '&' + qsplit[1].split('&')[1];
-        }
-        const requrl = requrlraw.startsWith('/') ? referer + requrlraw : requrlraw;
-        // cors weirdness here...
-        // if the referer is a cors page and the cors() route (I think) redirected to /corsProxy/<path> and the requested url path was relative,
-        // then we redirect again to the cors referer and just add the relative path.
-        if (!requrl.startsWith('http') && req.originalUrl.startsWith('/corsProxy') && referer?.includes('corsProxy')) {
-            res.redirect(referer + (referer.endsWith('/') ? '' : '/') + requrl);
-        } else {
-            proxyServe(req, requrl, res);
+            requrl = qsplit[0] + '?q=' + lastq.split('&')[0] + '&' + qsplit[1].split('&')[1];
         }
+        proxyServe(req, requrl, res);
     });
 }
 
@@ -184,7 +176,9 @@ function proxyServe(req: any, requrl: string, response: any) {
                     const htmlText = htmlInputText
                         .toString('utf8')
                         .replace('<head>', '<head> <style>[id ^= "google"] { display: none; } </style>')
-                        .replace(/href="https?([^"]*)"/g, httpsToCors)
+                        // .replace('<script', '<noscript')
+                        // .replace('</script', '</noscript')
+                        // .replace(/href="https?([^"]*)"/g, httpsToCors)
                         .replace(/data-srcset="[^"]*"/g, '')
                         .replace(/srcset="[^"]*"/g, '')
                         .replace(/target="_blank"/g, '');
@@ -234,8 +228,10 @@ function proxyServe(req: any, requrl: string, response: any) {
 function registerEmbeddedBrowseRelativePathHandler(server: express.Express) {
     server.use('*', (req, res) => {
         const relativeUrl = req.originalUrl;
-        if (!req.user) res.redirect('/home'); // When no user is logged in, we interpret a relative URL as being a reference to something they don't have access to and redirect to /home
-        else if (!res.headersSent && req.headers.referer?.includes('corsProxy')) {
+        // if (req.originalUrl === '/css/main.css' || req.originalUrl === '/favicon.ico') res.end();
+        // else
+        if (!res.headersSent && req.headers.referer?.includes('corsProxy')) {
+            if (!req.user) res.redirect('/home'); // When no user is logged in, we interpret a relative URL as being a reference to something they don't have access to and redirect to /home
             // a request for something by a proxied referrer means it must be a relative reference.  So construct a proxied absolute reference here.
             try {
                 const proxiedRefererUrl = decodeURIComponent(req.headers.referer); // (e.g., http://localhost:<port>/corsProxy/https://en.wikipedia.org/wiki/Engelbart)