Merge branch 'master' into parker

1 files changed, 247 insertions, 241 deletions

diff --git a/src/server/ApiManagers/UploadManager.ts b/src/server/ApiManagers/UploadManager.ts
index bfa07d47a..e7b7056a1 100644
--- a/src/server/ApiManagers/UploadManager.ts
+++ b/src/server/ApiManagers/UploadManager.ts
@@ -3,7 +3,7 @@ import { Method, _success } from "../RouteManager";
 import * as formidable from 'formidable';
 import v4 = require('uuid/v4');
 const AdmZip = require('adm-zip');
-import { extname, basename, dirname } from 'path';
+import { extname, basename, dirname, } from 'path';
 import { createReadStream, createWriteStream, unlink, writeFile } from "fs";
 import { publicDirectory, filesDirectory } from "..";
 import { Database } from "../database";
@@ -13,245 +13,243 @@ import { AcceptableMedia, Upload } from "../SharedMediaTypes";
 import { normalize } from "path";
 import RouteSubscriber from "../RouteSubscriber";
 const imageDataUri = require('image-data-uri');
-import { isWebUri } from "valid-url";
-import { Opt } from "../../fields/Doc";
 import { SolrManager } from "./SearchManager";
-import { StringDecoder } from "string_decoder";
+const fs = require('fs');
 
 export enum Directory {
-    parsed_files = "parsed_files",
-    images = "images",
-    videos = "videos",
-    pdfs = "pdfs",
-    text = "text",
-    pdf_thumbnails = "pdf_thumbnails",
-    audio = "audio",
+      parsed_files = "parsed_files",
+      images = "images",
+      videos = "videos",
+      pdfs = "pdfs",
+      text = "text",
+      pdf_thumbnails = "pdf_thumbnails",
+      audio = "audio",
 }
 
 export function serverPathToFile(directory: Directory, filename: string) {
-    return normalize(`${filesDirectory}/${directory}/${filename}`);
+      return normalize(`${filesDirectory}/${directory}/${filename}`);
 }
 
 export function pathToDirectory(directory: Directory) {
-    return normalize(`${filesDirectory}/${directory}`);
+      return normalize(`${filesDirectory}/${directory}`);
 }
 
 export function clientPathToFile(directory: Directory, filename: string) {
-    return `/files/${directory}/${filename}`;
+      return `/files/${directory}/${filename}`;
 }
 
 export default class UploadManager extends ApiManager {
 
-    protected initialize(register: Registration): void {
+      protected initialize(register: Registration): void {
 
-        register({
-            method: Method.POST,
-            subscription: "/uploadFormData",
-            secureHandler: async ({ req, res }) => {
-                const form = new formidable.IncomingForm();
-                form.keepExtensions = true;
-                form.uploadDir = pathToDirectory(Directory.parsed_files);
-                return new Promise<void>(resolve => {
-                    form.parse(req, async (_err, _fields, files) => {
-                        const results: Upload.FileResponse[] = [];
-                        for (const key in files) {
-                            const f = files[key];
-                            if (!Array.isArray(f)) {
-                                const result = await DashUploadUtils.upload(f);
-                                result && !(result.result instanceof Error) && results.push(result);
-                            }
-                        }
-                        _success(res, results);
-                        resolve();
-                    });
-                });
-            }
-        });
+            register({
+                  method: Method.POST,
+                  subscription: "/uploadFormData",
+                  secureHandler: async ({ req, res }) => {
+                        const form = new formidable.IncomingForm();
+                        form.keepExtensions = true;
+                        form.uploadDir = pathToDirectory(Directory.parsed_files);
+                        return new Promise<void>(resolve => {
+                              form.parse(req, async (_err, _fields, files) => {
+                                    const results: Upload.FileResponse[] = [];
+                                    for (const key in files) {
+                                          const f = files[key];
+                                          if (!Array.isArray(f)) {
+                                                const result = await DashUploadUtils.upload(f);
+                                                result && !(result.result instanceof Error) && results.push(result);
+                                          }
+                                    }
+                                    _success(res, results);
+                                    resolve();
+                              });
+                        });
+                  }
+            });
 
-        register({
-            method: Method.POST,
-            subscription: "/uploadYoutubeVideo",
-            secureHandler: async ({ req, res }) => {
-                //req.readableBuffer.head.data
-                return new Promise<void>(async resolve => {
-                    req.addListener("data", async (args) => {
-                        console.log(args);
-                        const payload = String.fromCharCode.apply(String, args);
-                        const videoId = JSON.parse(payload).videoId;
-                        const results: Upload.FileResponse[] = [];
-                        const result = await DashUploadUtils.uploadYoutube(videoId);
-                        result && !(result.result instanceof Error) && results.push(result);
-                        _success(res, results);
-                        resolve();
-                    });
-                });
-            }
-        });
+            register({
+                  method: Method.POST,
+                  subscription: "/uploadYoutubeVideo",
+                  secureHandler: async ({ req, res }) => {
+                        //req.readableBuffer.head.data
+                        return new Promise<void>(async resolve => {
+                              req.addListener("data", async (args) => {
+                                    console.log(args);
+                                    const payload = String.fromCharCode.apply(String, args);
+                                    const videoId = JSON.parse(payload).videoId;
+                                    const results: Upload.FileResponse[] = [];
+                                    const result = await DashUploadUtils.uploadYoutube(videoId);
+                                    result && !(result.result instanceof Error) && results.push(result);
+                                    _success(res, results);
+                                    resolve();
+                              });
+                        });
+                  }
+            });
 
-        register({
-            method: Method.POST,
-            subscription: new RouteSubscriber("youtubeScreenshot"),
-            secureHandler: async ({ req, res }) => {
-                const { id, timecode } = req.body;
-                const convert = (raw: string) => {
-                    const number = Math.floor(Number(raw));
-                    const seconds = number % 60;
-                    const minutes = (number - seconds) / 60;
-                    return `${minutes}m${seconds}s`;
-                };
-                const suffix = timecode ? `&t=${convert(timecode)}` : ``;
-                const targetUrl = `https://www.youtube.com/watch?v=${id}${suffix}`;
-                const buffer = await captureYoutubeScreenshot(targetUrl);
-                if (!buffer) {
-                    return res.send();
-                }
-                const resolvedName = `youtube_capture_${id}_${suffix}.png`;
-                const resolvedPath = serverPathToFile(Directory.images, resolvedName);
-                return new Promise<void>(resolve => {
-                    writeFile(resolvedPath, buffer, async error => {
-                        if (error) {
-                            return res.send();
+            register({
+                  method: Method.POST,
+                  subscription: new RouteSubscriber("youtubeScreenshot"),
+                  secureHandler: async ({ req, res }) => {
+                        const { id, timecode } = req.body;
+                        const convert = (raw: string) => {
+                              const number = Math.floor(Number(raw));
+                              const seconds = number % 60;
+                              const minutes = (number - seconds) / 60;
+                              return `${minutes}m${seconds}s`;
+                        };
+                        const suffix = timecode ? `&t=${convert(timecode)}` : ``;
+                        const targetUrl = `https://www.youtube.com/watch?v=${id}${suffix}`;
+                        const buffer = await captureYoutubeScreenshot(targetUrl);
+                        if (!buffer) {
+                              return res.send();
                         }
-                        await DashUploadUtils.outputResizedImages(() => createReadStream(resolvedPath), resolvedName, pathToDirectory(Directory.images));
-                        res.send({
-                            accessPaths: {
-                                agnostic: DashUploadUtils.getAccessPaths(Directory.images, resolvedName)
-                            }
-                        } as Upload.FileInformation);
-                        resolve();
-                    });
-                });
-            }
-        });
+                        const resolvedName = `youtube_capture_${id}_${suffix}.png`;
+                        const resolvedPath = serverPathToFile(Directory.images, resolvedName);
+                        return new Promise<void>(resolve => {
+                              writeFile(resolvedPath, buffer, async error => {
+                                    if (error) {
+                                          return res.send();
+                                    }
+                                    await DashUploadUtils.outputResizedImages(() => createReadStream(resolvedPath), resolvedName, pathToDirectory(Directory.images));
+                                    res.send({
+                                          accessPaths: {
+                                                agnostic: DashUploadUtils.getAccessPaths(Directory.images, resolvedName)
+                                          }
+                                    } as Upload.FileInformation);
+                                    resolve();
+                              });
+                        });
+                  }
+            });
 
-        register({
-            method: Method.POST,
-            subscription: "/uploadRemoteImage",
-            secureHandler: async ({ req, res }) => {
+            register({
+                  method: Method.POST,
+                  subscription: "/uploadRemoteImage",
+                  secureHandler: async ({ req, res }) => {
 
-                const { sources } = req.body;
-                if (Array.isArray(sources)) {
-                    const results = await Promise.all(sources.map(source => DashUploadUtils.UploadImage(source)));
-                    return res.send(results);
-                }
-                res.send();
-            }
-        });
+                        const { sources } = req.body;
+                        if (Array.isArray(sources)) {
+                              const results = await Promise.all(sources.map(source => DashUploadUtils.UploadImage(source)));
+                              return res.send(results);
+                        }
+                        res.send();
+                  }
+            });
 
-        register({
-            method: Method.POST,
-            subscription: "/uploadDoc",
-            secureHandler: ({ req, res }) => {
+            register({
+                  method: Method.POST,
+                  subscription: "/uploadDoc",
+                  secureHandler: ({ req, res }) => {
 
-                const form = new formidable.IncomingForm();
-                form.keepExtensions = true;
-                // let path = req.body.path;
-                const ids: { [id: string]: string } = {};
-                let remap = true;
-                const getId = (id: string): string => {
-                    if (!remap) return id;
-                    if (id.endsWith("Proto")) return id;
-                    if (id in ids) {
-                        return ids[id];
-                    } else {
-                        return ids[id] = v4();
-                    }
-                };
-                const mapFn = (doc: any) => {
-                    if (doc.id) {
-                        doc.id = getId(doc.id);
-                    }
-                    for (const key in doc.fields) {
-                        if (!doc.fields.hasOwnProperty(key)) { continue; }
-                        const field = doc.fields[key];
-                        if (field === undefined || field === null) { continue; }
+                        const form = new formidable.IncomingForm();
+                        form.keepExtensions = true;
+                        // let path = req.body.path;
+                        const ids: { [id: string]: string } = {};
+                        let remap = true;
+                        const getId = (id: string): string => {
+                              if (!remap) return id;
+                              if (id.endsWith("Proto")) return id;
+                              if (id in ids) {
+                                    return ids[id];
+                              } else {
+                                    return ids[id] = v4();
+                              }
+                        };
+                        const mapFn = (doc: any) => {
+                              if (doc.id) {
+                                    doc.id = getId(doc.id);
+                              }
+                              for (const key in doc.fields) {
+                                    if (!doc.fields.hasOwnProperty(key)) { continue; }
+                                    const field = doc.fields[key];
+                                    if (field === undefined || field === null) { continue; }
 
-                        if (field.__type === "Doc") {
-                            mapFn(field);
-                        } else if (field.__type === "proxy" || field.__type === "prefetch_proxy") {
-                            field.fieldId = getId(field.fieldId);
-                        } else if (field.__type === "script" || field.__type === "computed") {
-                            if (field.captures) {
-                                field.captures.fieldId = getId(field.captures.fieldId);
-                            }
-                        } else if (field.__type === "list") {
-                            mapFn(field);
-                        } else if (typeof field === "string") {
-                            const re = /("(?:dataD|d)ocumentId"\s*:\s*")([\w\-]*)"/g;
-                            doc.fields[key] = (field as any).replace(re, (match: any, p1: string, p2: string) => {
-                                return `${p1}${getId(p2)}"`;
-                            });
-                        } else if (field.__type === "RichTextField") {
-                            const re = /("href"\s*:\s*")(.*?)"/g;
-                            field.Data = field.Data.replace(re, (match: any, p1: string, p2: string) => {
-                                return `${p1}${getId(p2)}"`;
-                            });
-                        }
-                    }
-                };
-                return new Promise<void>(resolve => {
-                    form.parse(req, async (_err, fields, files) => {
-                        remap = fields.remap !== "false";
-                        let id: string = "";
-                        try {
-                            for (const name in files) {
-                                const f = files[name];
-                                const path_2 = Array.isArray(f) ? "" : f.path;
-                                const zip = new AdmZip(path_2);
-                                zip.getEntries().forEach((entry: any) => {
-                                    if (!entry.entryName.startsWith("files/")) return;
-                                    let directory = dirname(entry.entryName) + "/";
-                                    const extension = extname(entry.entryName);
-                                    const base = basename(entry.entryName).split(".")[0];
+                                    if (field.__type === "Doc") {
+                                          mapFn(field);
+                                    } else if (field.__type === "proxy" || field.__type === "prefetch_proxy") {
+                                          field.fieldId = getId(field.fieldId);
+                                    } else if (field.__type === "script" || field.__type === "computed") {
+                                          if (field.captures) {
+                                                field.captures.fieldId = getId(field.captures.fieldId);
+                                          }
+                                    } else if (field.__type === "list") {
+                                          mapFn(field);
+                                    } else if (typeof field === "string") {
+                                          const re = /("(?:dataD|d)ocumentId"\s*:\s*")([\w\-]*)"/g;
+                                          doc.fields[key] = (field as any).replace(re, (match: any, p1: string, p2: string) => {
+                                                return `${p1}${getId(p2)}"`;
+                                          });
+                                    } else if (field.__type === "RichTextField") {
+                                          const re = /("href"\s*:\s*")(.*?)"/g;
+                                          field.Data = field.Data.replace(re, (match: any, p1: string, p2: string) => {
+                                                return `${p1}${getId(p2)}"`;
+                                          });
+                                    }
+                              }
+                        };
+                        return new Promise<void>(resolve => {
+                              form.parse(req, async (_err, fields, files) => {
+                                    remap = fields.remap !== "false";
+                                    let id: string = "";
                                     try {
-                                        zip.extractEntryTo(entry.entryName, publicDirectory, true, false);
-                                        directory = "/" + directory;
+                                          for (const name in files) {
+                                                const f = files[name];
+                                                const path_2 = Array.isArray(f) ? "" : f.path;
+                                                const zip = new AdmZip(path_2);
+                                                zip.getEntries().forEach((entry: any) => {
+                                                      if (!entry.entryName.startsWith("files/")) return;
+                                                      let directory = dirname(entry.entryName) + "/";
+                                                      const extension = extname(entry.entryName);
+                                                      const base = basename(entry.entryName).split(".")[0];
+                                                      try {
+                                                            zip.extractEntryTo(entry.entryName, publicDirectory, true, false);
+                                                            directory = "/" + directory;
 
-                                        createReadStream(publicDirectory + directory + base + extension).pipe(createWriteStream(publicDirectory + directory + base + "_o" + extension));
-                                        createReadStream(publicDirectory + directory + base + extension).pipe(createWriteStream(publicDirectory + directory + base + "_s" + extension));
-                                        createReadStream(publicDirectory + directory + base + extension).pipe(createWriteStream(publicDirectory + directory + base + "_m" + extension));
-                                        createReadStream(publicDirectory + directory + base + extension).pipe(createWriteStream(publicDirectory + directory + base + "_l" + extension));
-                                    } catch (e) {
-                                        console.log(e);
-                                    }
-                                });
-                                const json = zip.getEntry("doc.json");
-                                try {
-                                    const data = JSON.parse(json.getData().toString("utf8"));
-                                    const datadocs = data.docs;
-                                    id = getId(data.id);
-                                    const docs = Object.keys(datadocs).map(key => datadocs[key]);
-                                    docs.forEach(mapFn);
-                                    await Promise.all(docs.map((doc: any) => new Promise<void>(res => {
-                                        Database.Instance.replace(doc.id, doc, (err, r) => {
-                                            err && console.log(err);
-                                            res();
-                                        }, true);
-                                    })));
-                                } catch (e) { console.log(e); }
-                                unlink(path_2, () => { });
-                            }
-                            SolrManager.update();
-                            res.send(JSON.stringify(id || "error"));
-                        } catch (e) { console.log(e); }
-                        resolve();
-                    });
-                });
-            }
-        });
+                                                            createReadStream(publicDirectory + directory + base + extension).pipe(createWriteStream(publicDirectory + directory + base + "_o" + extension));
+                                                            createReadStream(publicDirectory + directory + base + extension).pipe(createWriteStream(publicDirectory + directory + base + "_s" + extension));
+                                                            createReadStream(publicDirectory + directory + base + extension).pipe(createWriteStream(publicDirectory + directory + base + "_m" + extension));
+                                                            createReadStream(publicDirectory + directory + base + extension).pipe(createWriteStream(publicDirectory + directory + base + "_l" + extension));
+                                                      } catch (e) {
+                                                            console.log(e);
+                                                      }
+                                                });
+                                                const json = zip.getEntry("doc.json");
+                                                try {
+                                                      const data = JSON.parse(json.getData().toString("utf8"));
+                                                      const datadocs = data.docs;
+                                                      id = getId(data.id);
+                                                      const docs = Object.keys(datadocs).map(key => datadocs[key]);
+                                                      docs.forEach(mapFn);
+                                                      await Promise.all(docs.map((doc: any) => new Promise<void>(res => {
+                                                            Database.Instance.replace(doc.id, doc, (err, r) => {
+                                                                  err && console.log(err);
+                                                                  res();
+                                                            }, true);
+                                                      })));
+                                                } catch (e) { console.log(e); }
+                                                unlink(path_2, () => { });
+                                          }
+                                          SolrManager.update();
+                                          res.send(JSON.stringify(id || "error"));
+                                    } catch (e) { console.log(e); }
+                                    resolve();
+                              });
+                        });
+                  }
+            });
 
-        register({
-            method: Method.POST,
-            subscription: "/inspectImage",
-            secureHandler: async ({ req, res }) => {
+            register({
+                  method: Method.POST,
+                  subscription: "/inspectImage",
+                  secureHandler: async ({ req, res }) => {
 
-                const { source } = req.body;
-                if (typeof source === "string") {
-                    return res.send(await DashUploadUtils.InspectImage(source));
-                }
-                res.send({});
-            }
-        });
+                        const { source } = req.body;
+                        if (typeof source === "string") {
+                              return res.send(await DashUploadUtils.InspectImage(source));
+                        }
+                        res.send({});
+                  }
+            });
 
         register({
             method: Method.POST,
@@ -260,17 +258,23 @@ export default class UploadManager extends ApiManager {
                 const uri = req.body.uri;
                 const filename = req.body.name;
                 const origSuffix = req.body.nosuffix ? SizeSuffix.None : SizeSuffix.Original;
+                const deleteFiles = req.body.replaceRootFilename;
                 if (!uri || !filename) {
                     res.status(401).send("incorrect parameters specified");
                     return;
                 }
+                if (deleteFiles) {
+                    const path = serverPathToFile(Directory.images, "");
+                    const regex = new RegExp(`${deleteFiles}.*`);
+                    fs.readdirSync(path).filter((f: any) => regex.test(f)).map((f: any) => fs.unlinkSync(path + f));
+                }
                 return imageDataUri.outputFile(uri, serverPathToFile(Directory.images, InjectSize(filename, origSuffix))).then((savedName: string) => {
                     const ext = extname(savedName).toLowerCase();
                     const { pngs, jpgs } = AcceptableMedia;
-                    const resizers = !origSuffix ? [{ resizer: sharp().resize(400, undefined, { withoutEnlargement: true }), suffix: "_m" }] : [
-                        { resizer: sharp().resize(100, undefined, { withoutEnlargement: true }), suffix: "_s" },
-                        { resizer: sharp().resize(400, undefined, { withoutEnlargement: true }), suffix: "_m" },
-                        { resizer: sharp().resize(900, undefined, { withoutEnlargement: true }), suffix: "_l" },
+                    const resizers = !origSuffix ? [{ resizer: sharp().resize(400, undefined, { withoutEnlargement: true }), suffix: SizeSuffix.Medium }] : [
+                        { resizer: sharp().resize(100, undefined, { withoutEnlargement: true }), suffix: SizeSuffix.Small },
+                        { resizer: sharp().resize(400, undefined, { withoutEnlargement: true }), suffix: SizeSuffix.Medium },
+                        { resizer: sharp().resize(900, undefined, { withoutEnlargement: true }), suffix: SizeSuffix.Large },
                     ];
                     let isImage = false;
                     if (pngs.includes(ext)) {
@@ -286,21 +290,23 @@ export default class UploadManager extends ApiManager {
                     }
                     if (isImage) {
                         resizers.forEach(resizer => {
-                            const path = serverPathToFile(Directory.images, filename + resizer.suffix + ext);
-                            createReadStream(savedName).pipe(resizer.resizer).pipe(createWriteStream(path));
+                            const path = serverPathToFile(Directory.images, InjectSize(filename, resizer.suffix) + ext);
+                            createReadStream(savedName).on("error", e => console.log("Resizing read:" + e))
+                                .pipe(resizer.resizer)
+                                .pipe(createWriteStream(path).on("error", e => console.log("Resizing write: " + e)));
                         });
 
-                    }
-                    res.send(clientPathToFile(Directory.images, filename + ext));
-                });
-            }
-        });
+                              }
+                              res.send(clientPathToFile(Directory.images, filename + ext));
+                        });
+                  }
+            });
 
-    }
+      }
 
 }
 function delay(ms: number) {
-    return new Promise(resolve => setTimeout(resolve, ms));
+      return new Promise(resolve => setTimeout(resolve, ms));
 }
 /**
  * On success, returns a buffer containing the bytes of a screenshot
@@ -309,25 +315,25 @@ function delay(ms: number) {
  * On failure, returns undefined.
  */
 async function captureYoutubeScreenshot(targetUrl: string) {
-    // const browser = await launch({ args: ['--no-sandbox', '--disable-setuid-sandbox'] });
-    // const page = await browser.newPage();
-    // // await page.setViewport({ width: 1920, height: 1080 });
+      // const browser = await launch({ args: ['--no-sandbox', '--disable-setuid-sandbox'] });
+      // const page = await browser.newPage();
+      // // await page.setViewport({ width: 1920, height: 1080 });
 
-    // // await page.goto(targetUrl, { waitUntil: 'domcontentloaded' as any });
+      // // await page.goto(targetUrl, { waitUntil: 'domcontentloaded' as any });
 
-    // const videoPlayer = await page.$('.html5-video-player');
-    // videoPlayer && await page.focus("video");
-    // await delay(7000);
-    // const ad = await page.$('.ytp-ad-skip-button-text');
-    // await ad?.click();
-    // await videoPlayer?.click();
-    // await delay(1000);
-    // // hide youtube player controls.
-    // await page.evaluate(() => (document.querySelector('.ytp-chrome-bottom') as HTMLElement).style.display = 'none');
+      // const videoPlayer = await page.$('.html5-video-player');
+      // videoPlayer && await page.focus("video");
+      // await delay(7000);
+      // const ad = await page.$('.ytp-ad-skip-button-text');
+      // await ad?.click();
+      // await videoPlayer?.click();
+      // await delay(1000);
+      // // hide youtube player controls.
+      // await page.evaluate(() => (document.querySelector('.ytp-chrome-bottom') as HTMLElement).style.display = 'none');
 
-    // const buffer = await videoPlayer?.screenshot({ encoding: "binary" });
-    // await browser.close();
+      // const buffer = await videoPlayer?.screenshot({ encoding: "binary" });
+      // await browser.close();
 
-    // return buffer;
-    return null;
+      // return buffer;
+      return null;
 }
\ No newline at end of file