improved file partitioning in server and generified upload method

1 files changed, 204 insertions, 184 deletions

diff --git a/src/server/DashUploadUtils.ts b/src/server/DashUploadUtils.ts
index 839aada4b..0a670ec01 100644
--- a/src/server/DashUploadUtils.ts
+++ b/src/server/DashUploadUtils.ts
@@ -5,18 +5,32 @@ import * as sharp from 'sharp';
 import request = require('request-promise');
 import { ExifData, ExifImage } from 'exif';
 import { Opt } from '../new_fields/Doc';
-import { SharedMediaTypes } from './SharedMediaTypes';
+import { AcceptibleMedia } from './SharedMediaTypes';
 import { filesDirectory } from '.';
 import { File } from 'formidable';
-import { extname, basename } from "path";
+import { basename } from "path";
+import { ConsoleColors, createIfNotExists } from './ActionUtilities';
+import { ParsedPDF } from "../server/PdfTypes";
+const parse = require('pdf-parse');
+import { Directory, serverPathToFile, clientPathToFile } from './ApiManagers/UploadManager';
 
-const uploadDirectory = path.join(__dirname, './public/files/');
+export enum SizeSuffix {
+    Small = "_s",
+    Medium = "_m",
+    Large = "_l",
+    Original = "_o"
+}
 
 export namespace DashUploadUtils {
 
+    function InjectSize(filename: string, size: SizeSuffix) {
+        const extension = path.extname(filename).toLowerCase();
+        return filename.substring(0, filename.length - extension.length) + size + extension;
+    }
+
     export interface Size {
         width: number;
-        suffix: string;
+        suffix: SizeSuffix;
     }
 
     export interface ImageFileResponse {
@@ -27,215 +41,221 @@ export namespace DashUploadUtils {
     }
 
     export const Sizes: { [size: string]: Size } = {
-        SMALL: { width: 100, suffix: "_s" },
-        MEDIUM: { width: 400, suffix: "_m" },
-        LARGE: { width: 900, suffix: "_l" },
+        SMALL: { width: 100, suffix: SizeSuffix.Small },
+        MEDIUM: { width: 400, suffix: SizeSuffix.Medium },
+        LARGE: { width: 900, suffix: SizeSuffix.Large },
     };
 
     export function validateExtension(url: string) {
-        return SharedMediaTypes.imageFormats.includes(path.extname(url).toLowerCase());
+        return AcceptibleMedia.imageFormats.includes(path.extname(url).toLowerCase());
     }
 
     const size = "content-length";
     const type = "content-type";
 
-    export interface UploadInformation {
-        mediaPaths: string[];
-        fileNames: { [key: string]: string };
+    export interface ImageUploadInformation {
+        clientAccessPath: string;
+        serverAccessPaths: { [key: string]: string };
         exifData: EnrichedExifData;
         contentSize?: number;
         contentType?: string;
     }
 
-    export function upload(file: File): any {
+    export async function upload(file: File): Promise<any> {
         const { type, path, name } = file;
-        const filename = basename(path);
-        const extension = extname(path).toLowerCase();
-        if (extension === ".pdf") {
-
-        } else if  {
-            let partition: Opt<string>;
-            if(imageFormats.includes(extension)) {
-            partition = DashUploadUtils.Partitions.images;
-        } else if (videoFormats.includes(extension)) {
-            partition = DashUploadUtils.Partitions.videos;
-        }
-        let uploadInformation: Opt<DashUploadUtils.UploadInformation>;
-        if (partition) {
-            uploadInformation = await DashUploadUtils.UploadImage(`${filesDirectory}/${partition}/${filename}`, filename);
-        } else {
-            console.log(`Unable to accommodate, and ignored, the following file upload: ${filename}`);
+        const { imageFormats, videoFormats, applicationFormats } = AcceptibleMedia;
+        const types = type.split("/");
+
+        const category = types[0];
+        const format = `.${types[1]}`;
+
+        switch (category) {
+            case "image":
+                if (imageFormats.includes(format)) {
+                    const { clientAccessPath } = await UploadImage(path, basename(path), format);
+                    return { clientAccessPath, name, type };
+                }
+            case "video":
+                if (videoFormats.includes(format)) {
+                    return MoveParsedFile(path, Directory.videos);
+                }
+            case "application":
+                if (applicationFormats.includes(format)) {
+                    return UploadPdf(path);
+                }
         }
+        console.log(ConsoleColors.Red, `Ignoring unsupported file ${name} with upload type (${type}).`);
+        return { clientAccessPath: undefined };
     }
-    const exif = uploadInformation ? uploadInformation.exifData : undefined;
-    results.push({ name, type, path: `/files/${filename}`, exif });
 
-}
+    async function UploadPdf(absolutePath: string) {
+        let dataBuffer = fs.readFileSync(absolutePath);
+        const result: ParsedPDF = await parse(dataBuffer);
+        const parsedName = basename(absolutePath);
+        await new Promise<void>((resolve, reject) => {
+            const textFilename = `${parsedName.substring(0, parsedName.length - 4)}.txt`;
+            const writeStream = fs.createWriteStream(serverPathToFile(Directory.text, textFilename));
+            writeStream.write(result.text, error => error ? reject(error) : resolve());
+        });
+        return MoveParsedFile(absolutePath, Directory.pdfs);
+    }
 
-const generate = (prefix: string, url: string) => `${prefix}upload_${Utils.GenerateGuid()}${sanitizeExtension(url)}`;
-const sanitize = (filename: string) => filename.replace(/\s+/g, "_");
-const sanitizeExtension = (source: string) => {
-    let extension = path.extname(source);
-    extension = extension.toLowerCase();
-    extension = extension.split("?")[0];
-    return extension;
-};
-
-/**
- * Uploads an image specified by the @param source to Dash's /public/files/
- * directory, and returns information generated during that upload 
- * 
- * @param {string} source is either the absolute path of an already uploaded image or
- * the url of a remote image
- * @param {string} filename dictates what to call the image. If not specified,
- * the name {@param prefix}_upload_{GUID}
- * @param {string} prefix is a string prepended to the generated image name in the
- * event that @param filename is not specified
- * 
- * @returns {UploadInformation} This method returns
- * 1) the paths to the uploaded images (plural due to resizing)
- * 2) the file name of each of the resized images
- * 3) the size of the image, in bytes (4432130)
- * 4) the content type of the image, i.e. image/(jpeg | png | ...)
- */
-export const UploadImage = async (source: string, filename?: string, prefix: string = ""): Promise<UploadInformation> => {
-    const metadata = await InspectImage(source);
-    return UploadInspectedImage(metadata, filename, prefix);
-};
-
-export interface InspectionResults {
-    isLocal: boolean;
-    stream: any;
-    normalizedUrl: string;
-    exifData: EnrichedExifData;
-    contentSize?: number;
-    contentType?: string;
-}
+    const generate = (prefix: string, url: string) => `${prefix}upload_${Utils.GenerateGuid()}${sanitizeExtension(url)}`;
+    const sanitizeExtension = (source: string) => {
+        let extension = path.extname(source);
+        extension = extension.toLowerCase();
+        extension = extension.split("?")[0];
+        return extension;
+    };
 
-export interface EnrichedExifData {
-    data: ExifData;
-    error?: string;
-}
+    /**
+     * Uploads an image specified by the @param source to Dash's /public/files/
+     * directory, and returns information generated during that upload 
+     * 
+     * @param {string} source is either the absolute path of an already uploaded image or
+     * the url of a remote image
+     * @param {string} filename dictates what to call the image. If not specified,
+     * the name {@param prefix}_upload_{GUID}
+     * @param {string} prefix is a string prepended to the generated image name in the
+     * event that @param filename is not specified
+     * 
+     * @returns {ImageUploadInformation} This method returns
+     * 1) the paths to the uploaded images (plural due to resizing)
+     * 2) the file name of each of the resized images
+     * 3) the size of the image, in bytes (4432130)
+     * 4) the content type of the image, i.e. image/(jpeg | png | ...)
+     */
+    export const UploadImage = async (source: string, filename?: string, format?: string, prefix: string = ""): Promise<ImageUploadInformation> => {
+        const metadata = await InspectImage(source);
+        return UploadInspectedImage(metadata, filename, format, prefix);
+    };
 
-export enum Partitions {
-    pdf_text = "pdf_text",
-    images = "images",
-    videos = "videos"
-}
+    export interface InspectionResults {
+        isLocal: boolean;
+        stream: any;
+        normalizedUrl: string;
+        exifData: EnrichedExifData;
+        contentSize?: number;
+        contentType?: string;
+    }
 
-export async function buildFilePartitions() {
-    const pending = Object.keys(Partitions).map(sub => createIfNotExists(filesDirectory + sub));
-    return Promise.all(pending);
-}
+    export interface EnrichedExifData {
+        data: ExifData;
+        error?: string;
+    }
 
-/**
- * Based on the url's classification as local or remote, gleans
- * as much information as possible about the specified image
- * 
- * @param source is the path or url to the image in question
- */
-export const InspectImage = async (source: string): Promise<InspectionResults> => {
-    const { isLocal, stream, normalized: normalizedUrl } = classify(source);
-    const exifData = await parseExifData(source);
-    const results = {
-        exifData,
-        isLocal,
-        stream,
-        normalizedUrl
-    };
-    // stop here if local, since request.head() can't handle local paths, only urls on the web
-    if (isLocal) {
-        return results;
+    export async function buildFileDirectories() {
+        const pending = Object.keys(Directory).map(sub => createIfNotExists(`${filesDirectory}/${sub}`));
+        return Promise.all(pending);
     }
-    const metadata = (await new Promise<any>((resolve, reject) => {
-        request.head(source, async (error, res) => {
-            if (error) {
-                return reject(error);
-            }
-            resolve(res);
-        });
-    })).headers;
-    return {
-        contentSize: parseInt(metadata[size]),
-        contentType: metadata[type],
-        ...results
-    };
-};
-
-export const UploadInspectedImage = async (metadata: InspectionResults, filename?: string, prefix = ""): Promise<UploadInformation> => {
-    const { isLocal, stream, normalizedUrl, contentSize, contentType, exifData } = metadata;
-    const resolved = filename ? sanitize(filename) : generate(prefix, normalizedUrl);
-    const extension = sanitizeExtension(normalizedUrl || resolved);
-    let information: UploadInformation = {
-        mediaPaths: [],
-        fileNames: { clean: resolved },
-        exifData,
-        contentSize,
-        contentType,
-    };
-    const { pngs, jpgs } = SharedMediaTypes;
-    return new Promise<UploadInformation>(async (resolve, reject) => {
-        const resizers = [
-            { resizer: sharp().rotate(), suffix: "_o" },
-            ...Object.values(Sizes).map(size => ({
-                resizer: sharp().resize(size.width, undefined, { withoutEnlargement: true }).rotate(),
-                suffix: size.suffix
-            }))
-        ];
-        if (pngs.includes(extension)) {
-            resizers.forEach(element => element.resizer = element.resizer.png());
-        } else if (jpgs.includes(extension)) {
-            resizers.forEach(element => element.resizer = element.resizer.jpeg());
-        }
-        for (let resizer of resizers) {
-            const suffix = resizer.suffix;
-            let mediaPath: string;
-            await new Promise<void>(resolve => {
-                const filename = resolved.substring(0, resolved.length - extension.length) + suffix + extension;
-                information.mediaPaths.push(mediaPath = uploadDirectory + filename);
-                information.fileNames[suffix] = filename;
-                stream(normalizedUrl).pipe(resizer.resizer).pipe(fs.createWriteStream(mediaPath))
-                    .on('close', resolve)
-                    .on('error', reject);
-            });
+
+    /**
+     * Based on the url's classification as local or remote, gleans
+     * as much information as possible about the specified image
+     * 
+     * @param source is the path or url to the image in question
+     */
+    export const InspectImage = async (source: string): Promise<InspectionResults> => {
+        const { isLocal, stream, normalized: normalizedUrl } = classify(source);
+        const exifData = await parseExifData(source);
+        const results = {
+            exifData,
+            isLocal,
+            stream,
+            normalizedUrl
+        };
+        // stop here if local, since request.head() can't handle local paths, only urls on the web
+        if (isLocal) {
+            return results;
         }
-        if (!isLocal) {
-            await new Promise<void>(resolve => {
-                stream(normalizedUrl).pipe(fs.createWriteStream(uploadDirectory + resolved)).on('close', resolve);
+        const metadata = (await new Promise<any>((resolve, reject) => {
+            request.head(source, async (error, res) => {
+                if (error) {
+                    return reject(error);
+                }
+                resolve(res);
             });
-        }
-        resolve(information);
-    });
-};
-
-const classify = (url: string) => {
-    const isLocal = /Dash-Web(\\|\/)src(\\|\/)server(\\|\/)public(\\|\/)files/g.test(url);
-    return {
-        isLocal,
-        stream: isLocal ? fs.createReadStream : request,
-        normalized: isLocal ? path.normalize(url) : url
+        })).headers;
+        return {
+            contentSize: parseInt(metadata[size]),
+            contentType: metadata[type],
+            ...results
+        };
     };
-};
-
-const parseExifData = async (source: string): Promise<EnrichedExifData> => {
-    return new Promise<EnrichedExifData>(resolve => {
-        new ExifImage(source, (error, data) => {
-            let reason: Opt<string> = undefined;
-            if (error) {
-                reason = (error as any).code;
+
+    export async function MoveParsedFile(absolutePath: string, destination: Directory): Promise<{ clientAccessPath: Opt<string> }> {
+        return new Promise<{ clientAccessPath: Opt<string> }>(resolve => {
+            const filename = basename(absolutePath);
+            const destinationPath = serverPathToFile(destination, filename);
+            fs.rename(absolutePath, destinationPath, error => {
+                resolve({ clientAccessPath: error ? undefined : clientPathToFile(destination, filename) });
+            });
+        });
+    }
+
+    export const UploadInspectedImage = async (metadata: InspectionResults, filename?: string, format?: string, prefix = ""): Promise<ImageUploadInformation> => {
+        const { isLocal, stream, normalizedUrl, contentSize, contentType, exifData } = metadata;
+        const resolved = filename || generate(prefix, normalizedUrl);
+        const extension = format || sanitizeExtension(normalizedUrl || resolved);
+        let information: ImageUploadInformation = {
+            clientAccessPath: clientPathToFile(Directory.images, resolved),
+            serverAccessPaths: {},
+            exifData,
+            contentSize,
+            contentType,
+        };
+        const { pngs, jpgs } = AcceptibleMedia;
+        return new Promise<ImageUploadInformation>(async (resolve, reject) => {
+            const resizers = [
+                { resizer: sharp().rotate(), suffix: SizeSuffix.Original },
+                ...Object.values(Sizes).map(size => ({
+                    resizer: sharp().resize(size.width, undefined, { withoutEnlargement: true }).rotate(),
+                    suffix: size.suffix
+                }))
+            ];
+            if (pngs.includes(extension)) {
+                resizers.forEach(element => element.resizer = element.resizer.png());
+            } else if (jpgs.includes(extension)) {
+                resizers.forEach(element => element.resizer = element.resizer.jpeg());
+            }
+            for (let { resizer, suffix } of resizers) {
+                let mediaPath: string;
+                await new Promise<void>(resolve => {
+                    const filename = InjectSize(resolved, suffix);
+                    information.serverAccessPaths[suffix] = serverPathToFile(Directory.images, filename);
+                    stream(normalizedUrl).pipe(resizer).pipe(fs.createWriteStream(serverPathToFile(Directory.images, filename)))
+                        .on('close', resolve)
+                        .on('error', reject);
+                });
             }
-            resolve({ data, error: reason });
+            if (isLocal) {
+                await new Promise<boolean>(resolve => {
+                    fs.unlink(normalizedUrl, error => resolve(error === null));
+                });
+            }
+            resolve(information);
         });
-    });
-};
+    };
 
-export const createIfNotExists = async (path: string) => {
-    if (await new Promise<boolean>(resolve => fs.exists(path, resolve))) {
-        return true;
-    }
-    return new Promise<boolean>(resolve => fs.mkdir(path, error => resolve(error === null)));
-};
+    const classify = (url: string) => {
+        const isLocal = /Dash-Web(\\|\/)src(\\|\/)server(\\|\/)public(\\|\/)files/g.test(url);
+        return {
+            isLocal,
+            stream: isLocal ? fs.createReadStream : request,
+            normalized: isLocal ? path.normalize(url) : url
+        };
+    };
 
-export const Destroy = (mediaPath: string) => new Promise<boolean>(resolve => fs.unlink(mediaPath, error => resolve(error === null)));
+    const parseExifData = async (source: string): Promise<EnrichedExifData> => {
+        return new Promise<EnrichedExifData>(resolve => {
+            new ExifImage(source, (error, data) => {
+                let reason: Opt<string> = undefined;
+                if (error) {
+                    reason = (error as any).code;
+                }
+                resolve({ data, error: reason });
+            });
+        });
+    };
 
 }
\ No newline at end of file