Merge branch 'parker' of https://github.com/brown-dash/Dash-Web into parker

1 files changed, 120 insertions, 88 deletions

diff --git a/src/server/ApiManagers/UploadManager.ts b/src/server/ApiManagers/UploadManager.ts
index bfa07d47a..787e331c5 100644
--- a/src/server/ApiManagers/UploadManager.ts
+++ b/src/server/ApiManagers/UploadManager.ts
@@ -1,31 +1,30 @@
-import ApiManager, { Registration } from "./ApiManager";
-import { Method, _success } from "../RouteManager";
+import ApiManager, { Registration } from './ApiManager';
+import { Method, _success } from '../RouteManager';
 import * as formidable from 'formidable';
 import v4 = require('uuid/v4');
 const AdmZip = require('adm-zip');
 import { extname, basename, dirname } from 'path';
-import { createReadStream, createWriteStream, unlink, writeFile } from "fs";
-import { publicDirectory, filesDirectory } from "..";
-import { Database } from "../database";
-import { DashUploadUtils, InjectSize, SizeSuffix } from "../DashUploadUtils";
+import { createReadStream, createWriteStream, unlink, writeFile } from 'fs';
+import { publicDirectory, filesDirectory } from '..';
+import { Database } from '../database';
+import { DashUploadUtils, InjectSize, SizeSuffix } from '../DashUploadUtils';
 import * as sharp from 'sharp';
-import { AcceptableMedia, Upload } from "../SharedMediaTypes";
-import { normalize } from "path";
-import RouteSubscriber from "../RouteSubscriber";
+import { AcceptableMedia, Upload } from '../SharedMediaTypes';
+import { normalize } from 'path';
+import RouteSubscriber from '../RouteSubscriber';
 const imageDataUri = require('image-data-uri');
-import { isWebUri } from "valid-url";
-import { Opt } from "../../fields/Doc";
-import { SolrManager } from "./SearchManager";
-import { StringDecoder } from "string_decoder";
+import { SolrManager } from './SearchManager';
+const fs = require('fs');
 
 export enum Directory {
-    parsed_files = "parsed_files",
-    images = "images",
-    videos = "videos",
-    pdfs = "pdfs",
-    text = "text",
-    pdf_thumbnails = "pdf_thumbnails",
-    audio = "audio",
+    parsed_files = 'parsed_files',
+    images = 'images',
+    videos = 'videos',
+    pdfs = 'pdfs',
+    text = 'text',
+    pdf_thumbnails = 'pdf_thumbnails',
+    audio = 'audio',
+    csv = 'csv',
 }
 
 export function serverPathToFile(directory: Directory, filename: string) {
@@ -41,12 +40,19 @@ export function clientPathToFile(directory: Directory, filename: string) {
 }
 
 export default class UploadManager extends ApiManager {
-
     protected initialize(register: Registration): void {
+        register({
+            method: Method.POST,
+            subscription: '/concatVideos',
+            secureHandler: async ({ req, res }) => {
+                // req.body contains the array of server paths to the videos
+                _success(res, await DashUploadUtils.concatVideos(req.body));
+            },
+        });
 
         register({
             method: Method.POST,
-            subscription: "/uploadFormData",
+            subscription: '/uploadFormData',
             secureHandler: async ({ req, res }) => {
                 const form = new formidable.IncomingForm();
                 form.keepExtensions = true;
@@ -65,16 +71,16 @@ export default class UploadManager extends ApiManager {
                         resolve();
                     });
                 });
-            }
+            },
         });
 
         register({
             method: Method.POST,
-            subscription: "/uploadYoutubeVideo",
+            subscription: '/uploadYoutubeVideo',
             secureHandler: async ({ req, res }) => {
                 //req.readableBuffer.head.data
                 return new Promise<void>(async resolve => {
-                    req.addListener("data", async (args) => {
+                    req.addListener('data', async args => {
                         console.log(args);
                         const payload = String.fromCharCode.apply(String, args);
                         const videoId = JSON.parse(payload).videoId;
@@ -85,12 +91,12 @@ export default class UploadManager extends ApiManager {
                         resolve();
                     });
                 });
-            }
+            },
         });
 
         register({
             method: Method.POST,
-            subscription: new RouteSubscriber("youtubeScreenshot"),
+            subscription: new RouteSubscriber('youtubeScreenshot'),
             secureHandler: async ({ req, res }) => {
                 const { id, timecode } = req.body;
                 const convert = (raw: string) => {
@@ -115,34 +121,32 @@ export default class UploadManager extends ApiManager {
                         await DashUploadUtils.outputResizedImages(() => createReadStream(resolvedPath), resolvedName, pathToDirectory(Directory.images));
                         res.send({
                             accessPaths: {
-                                agnostic: DashUploadUtils.getAccessPaths(Directory.images, resolvedName)
-                            }
+                                agnostic: DashUploadUtils.getAccessPaths(Directory.images, resolvedName),
+                            },
                         } as Upload.FileInformation);
                         resolve();
                     });
                 });
-            }
+            },
         });
 
         register({
             method: Method.POST,
-            subscription: "/uploadRemoteImage",
+            subscription: '/uploadRemoteImage',
             secureHandler: async ({ req, res }) => {
-
                 const { sources } = req.body;
                 if (Array.isArray(sources)) {
                     const results = await Promise.all(sources.map(source => DashUploadUtils.UploadImage(source)));
                     return res.send(results);
                 }
                 res.send();
-            }
+            },
         });
 
         register({
             method: Method.POST,
-            subscription: "/uploadDoc",
+            subscription: '/uploadDoc',
             secureHandler: ({ req, res }) => {
-
                 const form = new formidable.IncomingForm();
                 form.keepExtensions = true;
                 // let path = req.body.path;
@@ -150,11 +154,11 @@ export default class UploadManager extends ApiManager {
                 let remap = true;
                 const getId = (id: string): string => {
                     if (!remap) return id;
-                    if (id.endsWith("Proto")) return id;
+                    if (id.endsWith('Proto')) return id;
                     if (id in ids) {
                         return ids[id];
                     } else {
-                        return ids[id] = v4();
+                        return (ids[id] = v4());
                     }
                 };
                 const mapFn = (doc: any) => {
@@ -162,26 +166,30 @@ export default class UploadManager extends ApiManager {
                         doc.id = getId(doc.id);
                     }
                     for (const key in doc.fields) {
-                        if (!doc.fields.hasOwnProperty(key)) { continue; }
+                        if (!doc.fields.hasOwnProperty(key)) {
+                            continue;
+                        }
                         const field = doc.fields[key];
-                        if (field === undefined || field === null) { continue; }
+                        if (field === undefined || field === null) {
+                            continue;
+                        }
 
-                        if (field.__type === "Doc") {
+                        if (field.__type === 'Doc') {
                             mapFn(field);
-                        } else if (field.__type === "proxy" || field.__type === "prefetch_proxy") {
+                        } else if (field.__type === 'proxy' || field.__type === 'prefetch_proxy') {
                             field.fieldId = getId(field.fieldId);
-                        } else if (field.__type === "script" || field.__type === "computed") {
+                        } else if (field.__type === 'script' || field.__type === 'computed') {
                             if (field.captures) {
                                 field.captures.fieldId = getId(field.captures.fieldId);
                             }
-                        } else if (field.__type === "list") {
+                        } else if (field.__type === 'list') {
                             mapFn(field);
-                        } else if (typeof field === "string") {
+                        } else if (typeof field === 'string') {
                             const re = /("(?:dataD|d)ocumentId"\s*:\s*")([\w\-]*)"/g;
                             doc.fields[key] = (field as any).replace(re, (match: any, p1: string, p2: string) => {
                                 return `${p1}${getId(p2)}"`;
                             });
-                        } else if (field.__type === "RichTextField") {
+                        } else if (field.__type === 'RichTextField') {
                             const re = /("href"\s*:\s*")(.*?)"/g;
                             field.Data = field.Data.replace(re, (match: any, p1: string, p2: string) => {
                                 return `${p1}${getId(p2)}"`;
@@ -191,87 +199,110 @@ export default class UploadManager extends ApiManager {
                 };
                 return new Promise<void>(resolve => {
                     form.parse(req, async (_err, fields, files) => {
-                        remap = fields.remap !== "false";
-                        let id: string = "";
+                        remap = fields.remap !== 'false';
+                        let id: string = '';
                         try {
                             for (const name in files) {
                                 const f = files[name];
-                                const path_2 = Array.isArray(f) ? "" : f.path;
+                                const path_2 = Array.isArray(f) ? '' : f.path;
                                 const zip = new AdmZip(path_2);
                                 zip.getEntries().forEach((entry: any) => {
-                                    if (!entry.entryName.startsWith("files/")) return;
-                                    let directory = dirname(entry.entryName) + "/";
+                                    if (!entry.entryName.startsWith('files/')) return;
+                                    let directory = dirname(entry.entryName) + '/';
                                     const extension = extname(entry.entryName);
-                                    const base = basename(entry.entryName).split(".")[0];
+                                    const base = basename(entry.entryName).split('.')[0];
                                     try {
                                         zip.extractEntryTo(entry.entryName, publicDirectory, true, false);
-                                        directory = "/" + directory;
+                                        directory = '/' + directory;
 
-                                        createReadStream(publicDirectory + directory + base + extension).pipe(createWriteStream(publicDirectory + directory + base + "_o" + extension));
-                                        createReadStream(publicDirectory + directory + base + extension).pipe(createWriteStream(publicDirectory + directory + base + "_s" + extension));
-                                        createReadStream(publicDirectory + directory + base + extension).pipe(createWriteStream(publicDirectory + directory + base + "_m" + extension));
-                                        createReadStream(publicDirectory + directory + base + extension).pipe(createWriteStream(publicDirectory + directory + base + "_l" + extension));
+                                        createReadStream(publicDirectory + directory + base + extension).pipe(createWriteStream(publicDirectory + directory + base + '_o' + extension));
+                                        createReadStream(publicDirectory + directory + base + extension).pipe(createWriteStream(publicDirectory + directory + base + '_s' + extension));
+                                        createReadStream(publicDirectory + directory + base + extension).pipe(createWriteStream(publicDirectory + directory + base + '_m' + extension));
+                                        createReadStream(publicDirectory + directory + base + extension).pipe(createWriteStream(publicDirectory + directory + base + '_l' + extension));
                                     } catch (e) {
                                         console.log(e);
                                     }
                                 });
-                                const json = zip.getEntry("doc.json");
+                                const json = zip.getEntry('doc.json');
                                 try {
-                                    const data = JSON.parse(json.getData().toString("utf8"));
+                                    const data = JSON.parse(json.getData().toString('utf8'));
                                     const datadocs = data.docs;
                                     id = getId(data.id);
                                     const docs = Object.keys(datadocs).map(key => datadocs[key]);
                                     docs.forEach(mapFn);
-                                    await Promise.all(docs.map((doc: any) => new Promise<void>(res => {
-                                        Database.Instance.replace(doc.id, doc, (err, r) => {
-                                            err && console.log(err);
-                                            res();
-                                        }, true);
-                                    })));
-                                } catch (e) { console.log(e); }
-                                unlink(path_2, () => { });
+                                    await Promise.all(
+                                        docs.map(
+                                            (doc: any) =>
+                                                new Promise<void>(res => {
+                                                    Database.Instance.replace(
+                                                        doc.id,
+                                                        doc,
+                                                        (err, r) => {
+                                                            err && console.log(err);
+                                                            res();
+                                                        },
+                                                        true
+                                                    );
+                                                })
+                                        )
+                                    );
+                                } catch (e) {
+                                    console.log(e);
+                                }
+                                unlink(path_2, () => {});
                             }
                             SolrManager.update();
-                            res.send(JSON.stringify(id || "error"));
-                        } catch (e) { console.log(e); }
+                            res.send(JSON.stringify(id || 'error'));
+                        } catch (e) {
+                            console.log(e);
+                        }
                         resolve();
                     });
                 });
-            }
+            },
         });
 
         register({
             method: Method.POST,
-            subscription: "/inspectImage",
+            subscription: '/inspectImage',
             secureHandler: async ({ req, res }) => {
-
                 const { source } = req.body;
-                if (typeof source === "string") {
+                if (typeof source === 'string') {
                     return res.send(await DashUploadUtils.InspectImage(source));
                 }
                 res.send({});
-            }
+            },
         });
 
         register({
             method: Method.POST,
-            subscription: "/uploadURI",
+            subscription: '/uploadURI',
             secureHandler: ({ req, res }) => {
                 const uri = req.body.uri;
                 const filename = req.body.name;
                 const origSuffix = req.body.nosuffix ? SizeSuffix.None : SizeSuffix.Original;
+                const deleteFiles = req.body.replaceRootFilename;
                 if (!uri || !filename) {
-                    res.status(401).send("incorrect parameters specified");
+                    res.status(401).send('incorrect parameters specified');
                     return;
                 }
+                if (deleteFiles) {
+                    const path = serverPathToFile(Directory.images, '');
+                    const regex = new RegExp(`${deleteFiles}.*`);
+                    fs.readdirSync(path)
+                        .filter((f: any) => regex.test(f))
+                        .map((f: any) => fs.unlinkSync(path + f));
+                }
                 return imageDataUri.outputFile(uri, serverPathToFile(Directory.images, InjectSize(filename, origSuffix))).then((savedName: string) => {
                     const ext = extname(savedName).toLowerCase();
                     const { pngs, jpgs } = AcceptableMedia;
-                    const resizers = !origSuffix ? [{ resizer: sharp().resize(400, undefined, { withoutEnlargement: true }), suffix: "_m" }] : [
-                        { resizer: sharp().resize(100, undefined, { withoutEnlargement: true }), suffix: "_s" },
-                        { resizer: sharp().resize(400, undefined, { withoutEnlargement: true }), suffix: "_m" },
-                        { resizer: sharp().resize(900, undefined, { withoutEnlargement: true }), suffix: "_l" },
-                    ];
+                    const resizers = !origSuffix
+                        ? [{ resizer: sharp().resize(400, undefined, { withoutEnlargement: true }), suffix: SizeSuffix.Medium }]
+                        : [
+                              { resizer: sharp().resize(100, undefined, { withoutEnlargement: true }), suffix: SizeSuffix.Small },
+                              { resizer: sharp().resize(400, undefined, { withoutEnlargement: true }), suffix: SizeSuffix.Medium },
+                              { resizer: sharp().resize(900, undefined, { withoutEnlargement: true }), suffix: SizeSuffix.Large },
+                          ];
                     let isImage = false;
                     if (pngs.includes(ext)) {
                         resizers.forEach(element => {
@@ -286,18 +317,19 @@ export default class UploadManager extends ApiManager {
                     }
                     if (isImage) {
                         resizers.forEach(resizer => {
-                            const path = serverPathToFile(Directory.images, filename + resizer.suffix + ext);
-                            createReadStream(savedName).pipe(resizer.resizer).pipe(createWriteStream(path));
+                            const path = serverPathToFile(Directory.images, InjectSize(filename, resizer.suffix) + ext);
+                            createReadStream(savedName)
+                                .on('error', e => console.log('Resizing read:' + e))
+                                .pipe(resizer.resizer)
+                                .on('error', e => console.log('Resizing write: ' + e))
+                                .pipe(createWriteStream(path).on('error', e => console.log('Resizing write: ' + e)));
                         });
-
                     }
                     res.send(clientPathToFile(Directory.images, filename + ext));
                 });
-            }
+            },
         });
-
     }
-
 }
 function delay(ms: number) {
     return new Promise(resolve => setTimeout(resolve, ms));
@@ -305,7 +337,7 @@ function delay(ms: number) {
 /**
  * On success, returns a buffer containing the bytes of a screenshot
  * of the video (optionally, at a timecode) specified by @param targetUrl.
- * 
+ *
  * On failure, returns undefined.
  */
 async function captureYoutubeScreenshot(targetUrl: string) {
@@ -330,4 +362,4 @@ async function captureYoutubeScreenshot(targetUrl: string) {
 
     // return buffer;
     return null;
-}
\ No newline at end of file
+}