improved file upload api

5 files changed, 85 insertions, 80 deletions

diff --git a/src/server/ApiManagers/GooglePhotosManager.ts b/src/server/ApiManagers/GooglePhotosManager.ts
index 04b724f4b..25c54ee2e 100644
--- a/src/server/ApiManagers/GooglePhotosManager.ts
+++ b/src/server/ApiManagers/GooglePhotosManager.ts
@@ -8,6 +8,7 @@ import { Opt } from "../../new_fields/Doc";
 import { DashUploadUtils, InjectSize, SizeSuffix } from "../DashUploadUtils";
 import { Database } from "../database";
 import { red } from "colors";
+import { Upload } from "../SharedMediaTypes";
 
 const prefix = "google_photos_";
 const remoteUploadError = "None of the preliminary uploads to Google's servers was successful.";
@@ -139,7 +140,7 @@ export default class GooglePhotosManager extends ApiManager {
                     return;
                 }
                 let failed = 0;
-                const completed: Opt<DashUploadUtils.ImageUploadInformation>[] = [];
+                const completed: Opt<Upload.ImageInformation>[] = [];
                 for (const { baseUrl } of mediaItems) {
                     // start by getting the content size of the remote image
                     const results = await DashUploadUtils.InspectImage(baseUrl);
@@ -151,7 +152,7 @@ export default class GooglePhotosManager extends ApiManager {
                     const { contentSize, ...attributes } = results;
                     // check to see if we have uploaded a Google user content image *specifically via this route* already
                     // that has this exact content size
-                    const found: Opt<DashUploadUtils.ImageUploadInformation> = await Database.Auxiliary.QueryUploadHistory(contentSize);
+                    const found: Opt<Upload.ImageInformation> = await Database.Auxiliary.QueryUploadHistory(contentSize);
                     if (!found) {
                         // if we haven't, then upload it locally to Dash's server
                         const upload = await DashUploadUtils.UploadInspectedImage({ contentSize, ...attributes }, undefined, prefix, false).catch(error => _error(res, downloadError, error));
diff --git a/src/server/ApiManagers/UploadManager.ts b/src/server/ApiManagers/UploadManager.ts
index 4d09528f4..8f2a5ea3e 100644
--- a/src/server/ApiManagers/UploadManager.ts
+++ b/src/server/ApiManagers/UploadManager.ts
@@ -4,12 +4,12 @@ import * as formidable from 'formidable';
 import v4 = require('uuid/v4');
 const AdmZip = require('adm-zip');
 import { extname, basename, dirname } from 'path';
-import { createReadStream, createWriteStream, unlink, readFileSync } from "fs";
+import { createReadStream, createWriteStream, unlink } from "fs";
 import { publicDirectory, filesDirectory } from "..";
 import { Database } from "../database";
-import { DashUploadUtils, SizeSuffix } from "../DashUploadUtils";
+import { DashUploadUtils } from "../DashUploadUtils";
 import * as sharp from 'sharp';
-import { AcceptibleMedia } from "../SharedMediaTypes";
+import { AcceptibleMedia, Upload } from "../SharedMediaTypes";
 import { normalize } from "path";
 const imageDataUri = require('image-data-uri');
 
@@ -47,7 +47,7 @@ export default class UploadManager extends ApiManager {
                 form.keepExtensions = true;
                 return new Promise<void>(resolve => {
                     form.parse(req, async (_err, _fields, files) => {
-                        const results: any[] = [];
+                        const results: Upload.FileResponse[] = [];
                         for (const key in files) {
                             const result = await DashUploadUtils.upload(files[key]);
                             result && results.push(result);
diff --git a/src/server/DashUploadUtils.ts b/src/server/DashUploadUtils.ts
index 913ddc1c3..b66651ef2 100644
--- a/src/server/DashUploadUtils.ts
+++ b/src/server/DashUploadUtils.ts
@@ -3,9 +3,9 @@ import { Utils } from '../Utils';
 import * as path from 'path';
 import * as sharp from 'sharp';
 import request = require('request-promise');
-import { ExifData, ExifImage } from 'exif';
+import { ExifImage } from 'exif';
 import { Opt } from '../new_fields/Doc';
-import { AcceptibleMedia } from './SharedMediaTypes';
+import { AcceptibleMedia, Upload } from './SharedMediaTypes';
 import { filesDirectory } from '.';
 import { File } from 'formidable';
 import { basename } from "path";
@@ -14,7 +14,7 @@ import { ParsedPDF } from "../server/PdfTypes";
 const parse = require('pdf-parse');
 import { Directory, serverPathToFile, clientPathToFile, pathToDirectory } from './ApiManagers/UploadManager';
 import { red } from 'colors';
-import { Writable } from 'stream';
+import { Stream } from 'stream';
 const requestImageSize = require("../client/util/request-image-size");
 
 export enum SizeSuffix {
@@ -40,13 +40,6 @@ export namespace DashUploadUtils {
         suffix: SizeSuffix;
     }
 
-    export interface ImageFileResponse {
-        name: string;
-        path: string;
-        type: string;
-        exif: Opt<DashUploadUtils.EnrichedExifData>;
-    }
-
     export const Sizes: { [size: string]: Size } = {
         SMALL: { width: 100, suffix: SizeSuffix.Small },
         MEDIUM: { width: 400, suffix: SizeSuffix.Medium },
@@ -60,20 +53,9 @@ export namespace DashUploadUtils {
     const size = "content-length";
     const type = "content-type";
 
-    export interface ImageUploadInformation {
-        accessPaths: AccessPathInfo;
-        exifData: EnrichedExifData;
-        contentSize?: number;
-        contentType?: string;
-    }
-
-    export interface AccessPathInfo {
-        [suffix: string]: { client: string, server: string };
-    }
-
     const { imageFormats, videoFormats, applicationFormats } = AcceptibleMedia;
 
-    export async function upload(file: File): Promise<any> {
+    export async function upload(file: File): Promise<Upload.FileResponse> {
         const { type, path, name } = file;
         const types = type.split("/");
 
@@ -83,33 +65,33 @@ export namespace DashUploadUtils {
         switch (category) {
             case "image":
                 if (imageFormats.includes(format)) {
-                    const results = await UploadImage(path, basename(path));
-                    return { ...results, name, type };
+                    const result = await UploadImage(path, basename(path));
+                    return { source: file, result };
                 }
             case "video":
                 if (videoFormats.includes(format)) {
-                    return MoveParsedFile(path, Directory.videos);
+                    return MoveParsedFile(file, Directory.videos);
                 }
             case "application":
                 if (applicationFormats.includes(format)) {
-                    return UploadPdf(path);
+                    return UploadPdf(file);
                 }
         }
 
         console.log(red(`Ignoring unsupported file (${name}) with upload type (${type}).`));
-        return { accessPaths: {} };
+        return { source: file, result: new Error(`Could not upload unsupported file (${name}) with upload type (${type}).`) };
     }
 
-    async function UploadPdf(absolutePath: string) {
-        const dataBuffer = readFileSync(absolutePath);
+    async function UploadPdf(file: File) {
+        const { path, name } = file;
+        const dataBuffer = readFileSync(path);
         const result: ParsedPDF = await parse(dataBuffer);
-        const parsedName = basename(absolutePath);
         await new Promise<void>((resolve, reject) => {
-            const textFilename = `${parsedName.substring(0, parsedName.length - 4)}.txt`;
+            const textFilename = `${name.substring(0, name.length - 4)}.txt`;
             const writeStream = createWriteStream(serverPathToFile(Directory.text, textFilename));
             writeStream.write(result.text, error => error ? reject(error) : resolve());
         });
-        return MoveParsedFile(absolutePath, Directory.pdfs);
+        return MoveParsedFile(file, Directory.pdfs);
     }
 
     /**
@@ -123,13 +105,13 @@ export namespace DashUploadUtils {
      * @param {string} prefix is a string prepended to the generated image name in the
      * event that @param filename is not specified
      * 
-     * @returns {ImageUploadInformation} This method returns
+     * @returns {ImageUploadInformation | Error} This method returns
      * 1) the paths to the uploaded images (plural due to resizing)
-     * 2) the file name of each of the resized images
+     * 2) the exif data embedded in the image, or the error explaining why exif couldn't be parsed
      * 3) the size of the image, in bytes (4432130)
      * 4) the content type of the image, i.e. image/(jpeg | png | ...)
      */
-    export const UploadImage = async (source: string, filename?: string, prefix: string = ""): Promise<ImageUploadInformation | Error> => {
+    export const UploadImage = async (source: string, filename?: string, prefix: string = ""): Promise<Upload.ImageInformation | Error> => {
         const metadata = await InspectImage(source);
         if (metadata instanceof Error) {
             return metadata;
@@ -137,22 +119,6 @@ export namespace DashUploadUtils {
         return UploadInspectedImage(metadata, filename || metadata.filename, prefix);
     };
 
-    export interface InspectionResults {
-        source: string;
-        requestable: string;
-        exifData: EnrichedExifData;
-        contentSize: number;
-        contentType: string;
-        nativeWidth: number;
-        nativeHeight: number;
-        filename?: string;
-    }
-
-    export interface EnrichedExifData {
-        data: ExifData;
-        error?: string;
-    }
-
     export async function buildFileDirectories() {
         const pending = Object.keys(Directory).map(sub => createIfNotExists(`${filesDirectory}/${sub}`));
         return Promise.all(pending);
@@ -175,7 +141,7 @@ export namespace DashUploadUtils {
      * 
      * @param source is the path or url to the image in question
      */
-    export const InspectImage = async (source: string): Promise<InspectionResults | Error> => {
+    export const InspectImage = async (source: string): Promise<Upload.InspectionResults | Error> => {
         let rawMatches: RegExpExecArray | null;
         let filename: string | undefined;
         if ((rawMatches = /^data:image\/([a-z]+);base64,(.*)/.exec(source)) !== null) {
@@ -216,14 +182,17 @@ export namespace DashUploadUtils {
         };
     };
 
-    export async function MoveParsedFile(absolutePath: string, destination: Directory): Promise<Opt<{ accessPaths: AccessPathInfo }>> {
+    export async function MoveParsedFile(file: File, destination: Directory): Promise<Upload.FileResponse> {
+        const { name, path: sourcePath } = file;
         return new Promise(resolve => {
-            const filename = basename(absolutePath);
-            const destinationPath = serverPathToFile(destination, filename);
-            rename(absolutePath, destinationPath, error => {
-                resolve(error ? undefined : {
-                    accessPaths: {
-                        agnostic: getAccessPaths(destination, filename)
+            const destinationPath = serverPathToFile(destination, name);
+            rename(sourcePath, destinationPath, error => {
+                resolve({
+                    source: file,
+                    result: error ? error : {
+                        accessPaths: {
+                            agnostic: getAccessPaths(destination, name)
+                        }
                     }
                 });
             });
@@ -237,16 +206,16 @@ export namespace DashUploadUtils {
         };
     }
 
-    export const UploadInspectedImage = async (metadata: InspectionResults, filename?: string, prefix = "", cleanUp = true): Promise<ImageUploadInformation> => {
+    export const UploadInspectedImage = async (metadata: Upload.InspectionResults, filename?: string, prefix = "", cleanUp = true): Promise<Upload.ImageInformation> => {
         const { requestable, source, ...remaining } = metadata;
         const extension = `.${remaining.contentType.split("/")[1].toLowerCase()}`;
         const resolved = filename || `${prefix}upload_${Utils.GenerateGuid()}${extension}`;
         const { images } = Directory;
-        const information: ImageUploadInformation = {
+        const information: Upload.ImageInformation = {
             accessPaths: {
                 agnostic: getAccessPaths(images, resolved)
             },
-            ...remaining
+            ...metadata
         };
         const outputPath = pathToDirectory(Directory.images);
         const writtenFiles = await outputResizedImages(() => request(requestable), outputPath, resolved, extension);
@@ -259,9 +228,9 @@ export namespace DashUploadUtils {
         return information;
     };
 
-    const parseExifData = async (source: string): Promise<EnrichedExifData> => {
+    const parseExifData = async (source: string): Promise<Upload.EnrichedExifData> => {
         const image = await request.get(source, { encoding: null });
-        return new Promise<EnrichedExifData>(resolve => {
+        return new Promise(resolve => {
             new ExifImage({ image }, (error, data) => {
                 let reason: Opt<string> = undefined;
                 if (error) {
@@ -279,18 +248,14 @@ export namespace DashUploadUtils {
         force: true
     };
 
-    export interface ReadStreamLike {
-        pipe: (dest: Writable) => Writable;
-    }
-
-    export async function outputResizedImages(readStreamSource: () => ReadStreamLike | Promise<ReadStreamLike>, outputPath: string, fileName: string, ext: string) {
+    export async function outputResizedImages(readStreamSource: () => Stream | Promise<Stream>, outputPath: string, fileName: string, ext: string) {
         const writtenFiles: { [suffix: string]: string } = {};
         for (const { resizer, suffix } of resizers(ext)) {
             const resolved = writtenFiles[suffix] = InjectSize(fileName, suffix);
             await new Promise<void>(async (resolve, reject) => {
                 const writeStream = createWriteStream(path.resolve(outputPath, resolved));
-                let readStream: ReadStreamLike;
                 const source = readStreamSource();
+                let readStream: Stream;
                 if (source instanceof Promise) {
                     readStream = await source;
                 } else {
@@ -320,7 +285,7 @@ export namespace DashUploadUtils {
                     initial = initial.webp();
                 } else if (tiffs.includes(ext)) {
                     initial = initial.tiff();
-                } else {
+                } else if (ext === ".gif") {
                     initial = undefined;
                 }
                 return {
diff --git a/src/server/SharedMediaTypes.ts b/src/server/SharedMediaTypes.ts
index 274b4f01e..185e787cc 100644
--- a/src/server/SharedMediaTypes.ts
+++ b/src/server/SharedMediaTypes.ts
@@ -1,3 +1,6 @@
+import { ExifData } from 'exif';
+import { File } from 'formidable';
+
 export namespace AcceptibleMedia {
     export const gifs = [".gif"];
     export const pngs = [".png"];
@@ -7,4 +10,40 @@ export namespace AcceptibleMedia {
     export const imageFormats = [...pngs, ...jpgs, ...gifs, ...webps, ...tiffs];
     export const videoFormats = [".mov", ".mp4"];
     export const applicationFormats = [".pdf"];
+}
+
+export namespace Upload {
+
+    export function isImageInformation(uploadResponse: Upload.FileInformation): uploadResponse is Upload.ImageInformation {
+        return "nativeWidth" in uploadResponse;
+    }
+
+    export interface FileInformation {
+        accessPaths: AccessPathInfo;
+    }
+
+    export type FileResponse<T extends FileInformation = FileInformation> = { source: File, result: T | Error };
+
+    export type ImageInformation = FileInformation & InspectionResults;
+
+    export interface AccessPathInfo {
+        [suffix: string]: { client: string, server: string };
+    }
+
+    export interface InspectionResults {
+        source: string;
+        requestable: string;
+        exifData: EnrichedExifData;
+        contentSize: number;
+        contentType: string;
+        nativeWidth: number;
+        nativeHeight: number;
+        filename?: string;
+    }
+
+    export interface EnrichedExifData {
+        data: ExifData;
+        error?: string;
+    }
+
 }
\ No newline at end of file
diff --git a/src/server/database.ts b/src/server/database.ts
index 83ce865c6..055f04c49 100644
--- a/src/server/database.ts
+++ b/src/server/database.ts
@@ -2,12 +2,12 @@ import * as mongodb from 'mongodb';
 import { Transferable } from './Message';
 import { Opt } from '../new_fields/Doc';
 import { Utils, emptyFunction } from '../Utils';
-import { DashUploadUtils } from './DashUploadUtils';
 import { Credentials } from 'google-auth-library';
 import { GoogleApiServerUtils } from './apis/google/GoogleApiServerUtils';
 import { IDatabase } from './IDatabase';
 import { MemoryDatabase } from './MemoryDatabase';
 import * as mongoose from 'mongoose';
+import { Upload } from './SharedMediaTypes';
 
 export namespace Database {
 
@@ -297,7 +297,7 @@ export namespace Database {
         };
 
         export const QueryUploadHistory = async (contentSize: number) => {
-            return SanitizedSingletonQuery<DashUploadUtils.ImageUploadInformation>({ contentSize }, AuxiliaryCollections.GooglePhotosUploadHistory);
+            return SanitizedSingletonQuery<Upload.ImageInformation>({ contentSize }, AuxiliaryCollections.GooglePhotosUploadHistory);
         };
 
         export namespace GoogleAuthenticationToken {
@@ -326,7 +326,7 @@ export namespace Database {
 
         }
 
-        export const LogUpload = async (information: DashUploadUtils.ImageUploadInformation) => {
+        export const LogUpload = async (information: Upload.ImageInformation) => {
             const bundle = {
                 _id: Utils.GenerateDeterministicGuid(String(information.contentSize!)),
                 ...information