improved file partitioning in server and generified upload method

16 files changed, 312 insertions, 257 deletions

diff --git a/src/client/documents/Documents.ts b/src/client/documents/Documents.ts
index dea057b93..a2f4d23c7 100644
--- a/src/client/documents/Documents.ts
+++ b/src/client/documents/Documents.ts
@@ -377,7 +377,6 @@ export namespace Docs {
                 let extension = path.extname(target);
                 target = `${target.substring(0, target.length - extension.length)}_o${extension}`;
             }
-            // if (target !== "http://www.cs.brown.edu/") {
             requestImageSize(target)
                 .then((size: any) => {
                     let aspect = size.height / size.width;
diff --git a/src/client/util/ClientDiagnostics.ts b/src/client/util/ClientDiagnostics.ts
index 24f196252..7eef935fd 100644
--- a/src/client/util/ClientDiagnostics.ts
+++ b/src/client/util/ClientDiagnostics.ts
@@ -1,9 +1,6 @@
-import { observable, runInAction } from "mobx";
-import { MainView } from "../views/MainView";
-
 export namespace ClientDiagnostics {
 
-    export function start() {
+    export async function start() {
 
         let serverPolls = 0;
         const serverHandle = setInterval(async () => {
@@ -17,14 +14,16 @@ export namespace ClientDiagnostics {
 
 
         let executed = false;
-        const solrHandle = setInterval(async () => {
+        const handle = async () => {
             const response = await fetch("/solrHeartbeat");
             if (!(await response.json()).running) {
                 !executed && alert("Looks like SOLR is not running on your machine.");
                 executed = true;
                 clearInterval(solrHandle);
             }
-        }, 1000 * 15);
+        };
+        await handle();
+        const solrHandle = setInterval(handle, 1000 * 15);
 
     }
 
diff --git a/src/client/util/Import & Export/DirectoryImportBox.tsx b/src/client/util/Import & Export/DirectoryImportBox.tsx
index f0880f193..16ae50685 100644
--- a/src/client/util/Import & Export/DirectoryImportBox.tsx
+++ b/src/client/util/Import & Export/DirectoryImportBox.tsx
@@ -22,18 +22,10 @@ import "./DirectoryImportBox.scss";
 import { Networking } from "../../Network";
 import { BatchedArray } from "array-batcher";
 import * as path from 'path';
-import { DashUploadUtils } from "../../../server/DashUploadUtils";
-import { SharedMediaTypes } from "../../../server/SharedMediaTypes";
+import { AcceptibleMedia } from "../../../server/SharedMediaTypes";
 
 const unsupported = ["text/html", "text/plain"];
 
-interface ImageUploadResponse {
-    name: string;
-    path: string;
-    type: string;
-    exif: any;
-}
-
 @observer
 export default class DirectoryImportBox extends React.Component<FieldViewProps> {
     private selector = React.createRef<HTMLInputElement>();
@@ -98,7 +90,7 @@ export default class DirectoryImportBox extends React.Component<FieldViewProps>
             let file = files.item(i);
             if (file && !unsupported.includes(file.type)) {
                 const ext = path.extname(file.name).toLowerCase();
-                if (SharedMediaTypes.imageFormats.includes(ext)) {
+                if (AcceptibleMedia.imageFormats.includes(ext)) {
                     validated.push(file);
                 }
             }
@@ -114,7 +106,7 @@ export default class DirectoryImportBox extends React.Component<FieldViewProps>
 
         runInAction(() => this.phase = `Internal: uploading ${this.quota - this.completed} files to Dash...`);
 
-        const uploads = await BatchedArray.from(validated, { batchSize: 15 }).batchedMapAsync<ImageUploadResponse>(async (batch, collector) => {
+        const uploads = await BatchedArray.from(validated, { batchSize: 15 }).batchedMapAsync<any>(async (batch, collector) => {
             const formData = new FormData();
 
             batch.forEach(file => {
@@ -127,16 +119,17 @@ export default class DirectoryImportBox extends React.Component<FieldViewProps>
             runInAction(() => this.completed += batch.length);
         });
 
+        const size = "_o";
         await Promise.all(uploads.map(async upload => {
             const type = upload.type;
-            const path = Utils.prepend(upload.path);
+            const path = Utils.prepend(upload.clientAccessPath);
             const options = {
                 nativeWidth: 300,
                 width: 300,
                 title: upload.name
             };
             const document = await Docs.Get.DocumentFromType(type, path, options);
-            const { data, error } = upload.exif;
+            const { data, error } = upload.exifData;
             if (document) {
                 Doc.GetProto(document).exif = error || Docs.Get.DocumentHierarchyFromJson(data);
                 docs.push(document);
diff --git a/src/client/views/Main.tsx b/src/client/views/Main.tsx
index dec4a24e4..9e699978f 100644
--- a/src/client/views/Main.tsx
+++ b/src/client/views/Main.tsx
@@ -10,7 +10,7 @@ import { ClientDiagnostics } from "../util/ClientDiagnostics";
 AssignAllExtensions();
 
 (async () => {
-    ClientDiagnostics.start();
+    await ClientDiagnostics.start();
     const info = await CurrentUserUtils.loadCurrentUser();
     DocServer.init(window.location.protocol, window.location.hostname, 4321, info.email);
     await Docs.Prototypes.initialize();
diff --git a/src/client/views/collections/CollectionSubView.tsx b/src/client/views/collections/CollectionSubView.tsx
index 1c3ff37ee..a1bd1527e 100644
--- a/src/client/views/collections/CollectionSubView.tsx
+++ b/src/client/views/collections/CollectionSubView.tsx
@@ -22,6 +22,7 @@ import React = require("react");
 var path = require('path');
 import { GooglePhotos } from "../../apis/google_docs/GooglePhotosClientUtils";
 import { ImageUtils } from "../../util/Import & Export/ImageUtils";
+import { Networking } from "../../Network";
 
 export interface CollectionViewProps extends FieldViewProps {
     addDocument: (document: Doc) => boolean;
@@ -271,28 +272,25 @@ export function CollectionSubView<T>(schemaCtor: (doc: Doc) => T) {
                     let file = item.getAsFile();
                     let formData = new FormData();
 
-                    if (file) {
-                        formData.append('file', file);
+                    if (!file || !file.type) {
+                        continue;
                     }
-                    let dropFileName = file ? file.name : "-empty-";
 
-                    let prom = fetch(Utils.prepend("/upload"), {
-                        method: 'POST',
-                        body: formData
-                    }).then(async (res: Response) => {
-                        (await res.json()).map(action((file: any) => {
+                    formData.append('file', file);
+                    let dropFileName = file ? file.name : "-empty-";
+                    promises.push(Networking.PostFormDataToServer("/upload", formData).then(results => {
+                        results.map(action((file: any) => {
                             let full = { ...options, nativeWidth: type.indexOf("video") !== -1 ? 600 : 300, width: 300, title: dropFileName };
-                            let pathname = Utils.prepend(file.path);
+                            let pathname = Utils.prepend(file.clientAccessPath);
                             Docs.Get.DocumentFromType(type, pathname, full).then(doc => {
                                 doc && (Doc.GetProto(doc).fileUpload = path.basename(pathname).replace("upload_", "").replace(/\.[a-z0-9]*$/, ""));
                                 doc && this.props.addDocument(doc);
                             });
                         }));
-                    });
-                    promises.push(prom);
+                    }));
                 }
             }
-            if (text) {
+            if (text && !text.includes("https://")) {
                 this.props.addDocument(Docs.Create.TextDocument({ ...options, documentText: "@@@" + text, width: 400, height: 315 }));
                 return;
             }
diff --git a/src/client/views/pdf/PDFViewer.tsx b/src/client/views/pdf/PDFViewer.tsx
index b737ce221..c075a4f99 100644
--- a/src/client/views/pdf/PDFViewer.tsx
+++ b/src/client/views/pdf/PDFViewer.tsx
@@ -125,7 +125,8 @@ export class PDFViewer extends DocAnnotatableComponent<IViewerProps, PdfDocument
         !this.props.Document.lockedTransform && (this.props.Document.lockedTransform = true);
         // change the address to be the file address of the PNG version of each page
         // file address of the pdf
-        this._coverPath = JSON.parse(await rp.get(Utils.prepend(`/thumbnail${this.props.url.substring("files/".length, this.props.url.length - ".pdf".length)}-${(this.Document.curPage || 1)}.PNG`)));
+        const path = Utils.prepend(`/thumbnail${this.props.url.substring("files/pdfs/".length, this.props.url.length - ".pdf".length)}-${(this.Document.curPage || 1)}.png`);
+        this._coverPath = JSON.parse(await rp.get(path));
         runInAction(() => this._showWaiting = this._showCover = true);
         this.props.startupLive && this.setupPdfJsViewer();
         this._searchReactionDisposer = reaction(() => this.Document.search_string, searchString => {
diff --git a/src/server/ActionUtilities.ts b/src/server/ActionUtilities.ts
index 7f493dd70..c9fc86fea 100644
--- a/src/server/ActionUtilities.ts
+++ b/src/server/ActionUtilities.ts
@@ -2,6 +2,7 @@ import * as fs from 'fs';
 import { ExecOptions } from 'shelljs';
 import { exec } from 'child_process';
 import * as path from 'path';
+import * as rimraf from "rimraf";
 
 export const command_line = (command: string, fromDirectory?: string) => {
     return new Promise<string>((resolve, reject) => {
@@ -68,4 +69,18 @@ export function msToTime(duration: number) {
     let secondsS = (seconds < 10) ? "0" + seconds : seconds;
 
     return hoursS + ":" + minutesS + ":" + secondsS + "." + milliseconds;
-}
\ No newline at end of file
+}
+
+export const createIfNotExists = async (path: string) => {
+    if (await new Promise<boolean>(resolve => fs.exists(path, resolve))) {
+        return true;
+    }
+    return new Promise<boolean>(resolve => fs.mkdir(path, error => resolve(error === null)));
+};
+
+export async function Prune(rootDirectory: string): Promise<boolean> {
+    const error = await new Promise<Error>(resolve => rimraf(rootDirectory, resolve));
+    return error === null;
+}
+
+export const Destroy = (mediaPath: string) => new Promise<boolean>(resolve => fs.unlink(mediaPath, error => resolve(error === null)));
\ No newline at end of file
diff --git a/src/server/ApiManagers/DownloadManager.ts b/src/server/ApiManagers/DownloadManager.ts
index fc6ba0d22..5bad46eda 100644
--- a/src/server/ApiManagers/DownloadManager.ts
+++ b/src/server/ApiManagers/DownloadManager.ts
@@ -5,7 +5,7 @@ import * as Archiver from 'archiver';
 import * as express from 'express';
 import { Database } from "../database";
 import * as path from "path";
-import { DashUploadUtils } from "../DashUploadUtils";
+import { DashUploadUtils, SizeSuffix } from "../DashUploadUtils";
 import { publicDirectory } from "..";
 
 export type Hierarchy = { [id: string]: string | Hierarchy };
@@ -254,7 +254,7 @@ async function writeHierarchyRecursive(file: Archiver.Archiver, hierarchy: Hiera
                 // and dropped in the browser and thus hosted remotely) so we upload it
                 // to our server and point the zip file to it, so it can bundle up the bytes
                 const information = await DashUploadUtils.UploadImage(result);
-                path = information.mediaPaths[0];
+                path = information.serverAccessPaths[SizeSuffix.Original];
             }
             // write the file specified by the path to the directory in the
             // zip file given by the prefix.
diff --git a/src/server/ApiManagers/GooglePhotosManager.ts b/src/server/ApiManagers/GooglePhotosManager.ts
index c7af69375..5a709688b 100644
--- a/src/server/ApiManagers/GooglePhotosManager.ts
+++ b/src/server/ApiManagers/GooglePhotosManager.ts
@@ -86,10 +86,10 @@ export default class GooglePhotosManager extends ApiManager {
                 const contents: { mediaItems: MediaItem[] } = req.body;
                 let failed = 0;
                 if (contents) {
-                    const completed: Opt<DashUploadUtils.UploadInformation>[] = [];
+                    const completed: Opt<DashUploadUtils.ImageUploadInformation>[] = [];
                     for (let item of contents.mediaItems) {
                         const { contentSize, ...attributes } = await DashUploadUtils.InspectImage(item.baseUrl);
-                        const found: Opt<DashUploadUtils.UploadInformation> = await Database.Auxiliary.QueryUploadHistory(contentSize!);
+                        const found: Opt<DashUploadUtils.ImageUploadInformation> = await Database.Auxiliary.QueryUploadHistory(contentSize!);
                         if (!found) {
                             const upload = await DashUploadUtils.UploadInspectedImage({ contentSize, ...attributes }, item.filename, prefix).catch(error => _error(res, downloadError, error));
                             if (upload) {
diff --git a/src/server/ApiManagers/PDFManager.ts b/src/server/ApiManagers/PDFManager.ts
index 632b4965a..4bd750aaf 100644
--- a/src/server/ApiManagers/PDFManager.ts
+++ b/src/server/ApiManagers/PDFManager.ts
@@ -2,12 +2,12 @@ import ApiManager, { Registration } from "./ApiManager";
 import { Method } from "../RouteManager";
 import RouteSubscriber from "../RouteSubscriber";
 import { exists, createReadStream, createWriteStream } from "fs";
-import { filesDirectory } from "..";
 import * as Pdfjs from 'pdfjs-dist';
 import { createCanvas } from "canvas";
 const probe = require("probe-image-size");
 import * as express from "express";
 import * as path from "path";
+import { Directory, serverPathToFile, clientPathToFile } from "./UploadManager";
 
 export default class PDFManager extends ApiManager {
 
@@ -21,21 +21,27 @@ export default class PDFManager extends ApiManager {
                 let noExt = filename.substring(0, filename.length - ".png".length);
                 let pagenumber = parseInt(noExt.split('-')[1]);
                 return new Promise<void>(resolve => {
-                    exists(filesDirectory + filename, (exists: boolean) => {
-                        console.log(`${filesDirectory + filename} ${exists ? "exists" : "does not exist"}`);
+                    const path = serverPathToFile(Directory.pdf_thumbnails, filename);
+                    exists(path, (exists: boolean) => {
+                        console.log(`${path} ${exists ? "exists" : "does not exist"}`);
                         if (exists) {
-                            let input = createReadStream(filesDirectory + filename);
-                            probe(input, (err: any, result: any) => {
+                            let input = createReadStream(path);
+                            probe(input, (err: any, { width, height }: any) => {
                                 if (err) {
                                     console.log(err);
                                     console.log(`error on ${filename}`);
                                     return;
                                 }
-                                res.send({ path: "/files/" + filename, width: result.width, height: result.height });
+                                res.send({
+                                    path: clientPathToFile(Directory.pdf_thumbnails, filename),
+                                    width,
+                                    height
+                                });
                             });
                         }
                         else {
-                            LoadPage(filesDirectory + filename.substring(0, filename.length - noExt.split('-')[1].length - ".PNG".length - 1) + ".pdf", pagenumber, res);
+                            const name = filename.substring(0, filename.length - noExt.split('-')[1].length - ".PNG".length - 1) + ".pdf";
+                            LoadPage(serverPathToFile(Directory.pdfs, name), pagenumber, res);
                         }
                         resolve();
                     });
@@ -55,8 +61,8 @@ export default class PDFManager extends ApiManager {
                         let canvasAndContext = factory.create(viewport.width, viewport.height);
                         let renderContext = {
                             canvasContext: canvasAndContext.context,
-                            viewport: viewport,
-                            canvasFactory: factory
+                            canvasFactory: factory,
+                            viewport
                         };
                         console.log("read " + pageNumber);
 
@@ -64,13 +70,17 @@ export default class PDFManager extends ApiManager {
                             .then(() => {
                                 console.log("saving " + pageNumber);
                                 let stream = canvasAndContext.canvas.createPNGStream();
-                                let pngFile = `${file.substring(0, file.length - ".pdf".length)}-${pageNumber}.PNG`;
+                                let filenames = path.basename(file).split(".");
+                                const pngFile = serverPathToFile(Directory.pdf_thumbnails, `${filenames[0]}-${pageNumber}.png`);
                                 let out = createWriteStream(pngFile);
                                 stream.pipe(out);
                                 out.on("finish", () => {
                                     console.log(`Success! Saved to ${pngFile}`);
-                                    let name = path.basename(pngFile);
-                                    res.send({ path: "/files/" + name, width: viewport.width, height: viewport.height });
+                                    res.send({
+                                        path: pngFile,
+                                        width: viewport.width,
+                                        height: viewport.height
+                                    });
                                 });
                             }, (reason: string) => {
                                 console.error(reason + ` ${pageNumber}`);
diff --git a/src/server/ApiManagers/SearchManager.ts b/src/server/ApiManagers/SearchManager.ts
index 1c801715a..d3f8995b0 100644
--- a/src/server/ApiManagers/SearchManager.ts
+++ b/src/server/ApiManagers/SearchManager.ts
@@ -3,7 +3,7 @@ import { Method } from "../RouteManager";
 import { Search } from "../Search";
 var findInFiles = require('find-in-files');
 import * as path from 'path';
-import { filesDirectory } from "..";
+import { pathToDirectory, Directory } from "./UploadManager";
 
 export default class SearchManager extends ApiManager {
 
@@ -18,7 +18,7 @@ export default class SearchManager extends ApiManager {
                     res.send([]);
                     return;
                 }
-                let results = await findInFiles.find({ 'term': q, 'flags': 'ig' }, filesDirectory + "text", ".txt$");
+                let results = await findInFiles.find({ 'term': q, 'flags': 'ig' }, pathToDirectory(Directory.text), ".txt$");
                 let resObj: { ids: string[], numFound: number, lines: string[] } = { ids: [], numFound: 0, lines: [] };
                 for (var result in results) {
                     resObj.ids.push(path.basename(result, ".txt").replace(/upload_/, ""));
diff --git a/src/server/ApiManagers/UploadManager.ts b/src/server/ApiManagers/UploadManager.ts
index 2a9faacd8..2f76871a6 100644
--- a/src/server/ApiManagers/UploadManager.ts
+++ b/src/server/ApiManagers/UploadManager.ts
@@ -7,14 +7,33 @@ import { extname, basename, dirname } from 'path';
 import { createReadStream, createWriteStream, unlink, readFileSync } from "fs";
 import { publicDirectory, filesDirectory } from "..";
 import { Database } from "../database";
-import { DashUploadUtils } from "../DashUploadUtils";
-import { Opt } from "../../new_fields/Doc";
-import { ParsedPDF } from "../PdfTypes";
-const pdf = require('pdf-parse');
+import { DashUploadUtils, SizeSuffix } from "../DashUploadUtils";
 import * as sharp from 'sharp';
-import { SharedMediaTypes } from "../SharedMediaTypes";
+import { AcceptibleMedia } from "../SharedMediaTypes";
+import { normalize } from "path";
 const imageDataUri = require('image-data-uri');
 
+export enum Directory {
+    parsed_files = "parsed_files",
+    images = "images",
+    videos = "videos",
+    pdfs = "pdfs",
+    text = "text",
+    pdf_thumbnails = "pdf_thumbnails"
+}
+
+export function serverPathToFile(directory: Directory, filename: string) {
+    return normalize(`${filesDirectory}/${directory}/${filename}`);
+}
+
+export function pathToDirectory(directory: Directory) {
+    return normalize(`${filesDirectory}/${directory}`);
+}
+
+export function clientPathToFile(directory: Directory, filename: string) {
+    return `/files/${directory}/${filename}`;
+}
+
 export default class UploadManager extends ApiManager {
 
     protected initialize(register: Registration): void {
@@ -129,13 +148,14 @@ export default class UploadManager extends ApiManager {
             subscription: "/upload",
             onValidation: async ({ req, res }) => {
                 let form = new formidable.IncomingForm();
-                form.uploadDir = filesDirectory;
+                form.uploadDir = pathToDirectory(Directory.parsed_files);
                 form.keepExtensions = true;
                 return new Promise<void>(resolve => {
                     form.parse(req, async (_err, _fields, files) => {
                         let results: any[] = [];
                         for (const key in files) {
-                            results.push(DashUploadUtils.upload(files[key]));
+                            const result = await DashUploadUtils.upload(files[key]);
+                            result && results.push(result);
                         }
                         _success(res, results);
                         resolve();
@@ -150,8 +170,8 @@ export default class UploadManager extends ApiManager {
             onValidation: async ({ req, res }) => {
                 const { source } = req.body;
                 if (typeof source === "string") {
-                    const uploadInformation = await DashUploadUtils.UploadImage(source);
-                    return res.send(await DashUploadUtils.InspectImage(uploadInformation.mediaPaths[0]));
+                    const { serverAccessPaths } = await DashUploadUtils.UploadImage(source);
+                    return res.send(await DashUploadUtils.InspectImage(serverAccessPaths[SizeSuffix.Original]));
                 }
                 res.send({});
             }
@@ -167,9 +187,9 @@ export default class UploadManager extends ApiManager {
                     res.status(401).send("incorrect parameters specified");
                     return;
                 }
-                return imageDataUri.outputFile(uri, filesDirectory + filename).then((savedName: string) => {
+                return imageDataUri.outputFile(uri, serverPathToFile(Directory.images, filename)).then((savedName: string) => {
                     const ext = extname(savedName).toLowerCase();
-                    const { pngs, jpgs } = SharedMediaTypes;
+                    const { pngs, jpgs } = AcceptibleMedia;
                     let resizers = [
                         { resizer: sharp().resize(100, undefined, { withoutEnlargement: true }), suffix: "_s" },
                         { resizer: sharp().resize(400, undefined, { withoutEnlargement: true }), suffix: "_m" },
@@ -189,10 +209,11 @@ export default class UploadManager extends ApiManager {
                     }
                     if (isImage) {
                         resizers.forEach(resizer => {
-                            createReadStream(savedName).pipe(resizer.resizer).pipe(createWriteStream(filesDirectory + filename + resizer.suffix + ext));
+                            const path = serverPathToFile(Directory.images, filename + resizer.suffix + ext);
+                            createReadStream(savedName).pipe(resizer.resizer).pipe(createWriteStream(path));
                         });
                     }
-                    res.send("/files/" + filename + ext);
+                    res.send(clientPathToFile(Directory.images, filename + ext));
                 });
             }
         });
diff --git a/src/server/DashUploadUtils.ts b/src/server/DashUploadUtils.ts
index 839aada4b..0a670ec01 100644
--- a/src/server/DashUploadUtils.ts
+++ b/src/server/DashUploadUtils.ts
@@ -5,18 +5,32 @@ import * as sharp from 'sharp';
 import request = require('request-promise');
 import { ExifData, ExifImage } from 'exif';
 import { Opt } from '../new_fields/Doc';
-import { SharedMediaTypes } from './SharedMediaTypes';
+import { AcceptibleMedia } from './SharedMediaTypes';
 import { filesDirectory } from '.';
 import { File } from 'formidable';
-import { extname, basename } from "path";
+import { basename } from "path";
+import { ConsoleColors, createIfNotExists } from './ActionUtilities';
+import { ParsedPDF } from "../server/PdfTypes";
+const parse = require('pdf-parse');
+import { Directory, serverPathToFile, clientPathToFile } from './ApiManagers/UploadManager';
 
-const uploadDirectory = path.join(__dirname, './public/files/');
+export enum SizeSuffix {
+    Small = "_s",
+    Medium = "_m",
+    Large = "_l",
+    Original = "_o"
+}
 
 export namespace DashUploadUtils {
 
+    function InjectSize(filename: string, size: SizeSuffix) {
+        const extension = path.extname(filename).toLowerCase();
+        return filename.substring(0, filename.length - extension.length) + size + extension;
+    }
+
     export interface Size {
         width: number;
-        suffix: string;
+        suffix: SizeSuffix;
     }
 
     export interface ImageFileResponse {
@@ -27,215 +41,221 @@ export namespace DashUploadUtils {
     }
 
     export const Sizes: { [size: string]: Size } = {
-        SMALL: { width: 100, suffix: "_s" },
-        MEDIUM: { width: 400, suffix: "_m" },
-        LARGE: { width: 900, suffix: "_l" },
+        SMALL: { width: 100, suffix: SizeSuffix.Small },
+        MEDIUM: { width: 400, suffix: SizeSuffix.Medium },
+        LARGE: { width: 900, suffix: SizeSuffix.Large },
     };
 
     export function validateExtension(url: string) {
-        return SharedMediaTypes.imageFormats.includes(path.extname(url).toLowerCase());
+        return AcceptibleMedia.imageFormats.includes(path.extname(url).toLowerCase());
     }
 
     const size = "content-length";
     const type = "content-type";
 
-    export interface UploadInformation {
-        mediaPaths: string[];
-        fileNames: { [key: string]: string };
+    export interface ImageUploadInformation {
+        clientAccessPath: string;
+        serverAccessPaths: { [key: string]: string };
         exifData: EnrichedExifData;
         contentSize?: number;
         contentType?: string;
     }
 
-    export function upload(file: File): any {
+    export async function upload(file: File): Promise<any> {
         const { type, path, name } = file;
-        const filename = basename(path);
-        const extension = extname(path).toLowerCase();
-        if (extension === ".pdf") {
-
-        } else if  {
-            let partition: Opt<string>;
-            if(imageFormats.includes(extension)) {
-            partition = DashUploadUtils.Partitions.images;
-        } else if (videoFormats.includes(extension)) {
-            partition = DashUploadUtils.Partitions.videos;
-        }
-        let uploadInformation: Opt<DashUploadUtils.UploadInformation>;
-        if (partition) {
-            uploadInformation = await DashUploadUtils.UploadImage(`${filesDirectory}/${partition}/${filename}`, filename);
-        } else {
-            console.log(`Unable to accommodate, and ignored, the following file upload: ${filename}`);
+        const { imageFormats, videoFormats, applicationFormats } = AcceptibleMedia;
+        const types = type.split("/");
+
+        const category = types[0];
+        const format = `.${types[1]}`;
+
+        switch (category) {
+            case "image":
+                if (imageFormats.includes(format)) {
+                    const { clientAccessPath } = await UploadImage(path, basename(path), format);
+                    return { clientAccessPath, name, type };
+                }
+            case "video":
+                if (videoFormats.includes(format)) {
+                    return MoveParsedFile(path, Directory.videos);
+                }
+            case "application":
+                if (applicationFormats.includes(format)) {
+                    return UploadPdf(path);
+                }
         }
+        console.log(ConsoleColors.Red, `Ignoring unsupported file ${name} with upload type (${type}).`);
+        return { clientAccessPath: undefined };
     }
-    const exif = uploadInformation ? uploadInformation.exifData : undefined;
-    results.push({ name, type, path: `/files/${filename}`, exif });
 
-}
+    async function UploadPdf(absolutePath: string) {
+        let dataBuffer = fs.readFileSync(absolutePath);
+        const result: ParsedPDF = await parse(dataBuffer);
+        const parsedName = basename(absolutePath);
+        await new Promise<void>((resolve, reject) => {
+            const textFilename = `${parsedName.substring(0, parsedName.length - 4)}.txt`;
+            const writeStream = fs.createWriteStream(serverPathToFile(Directory.text, textFilename));
+            writeStream.write(result.text, error => error ? reject(error) : resolve());
+        });
+        return MoveParsedFile(absolutePath, Directory.pdfs);
+    }
 
-const generate = (prefix: string, url: string) => `${prefix}upload_${Utils.GenerateGuid()}${sanitizeExtension(url)}`;
-const sanitize = (filename: string) => filename.replace(/\s+/g, "_");
-const sanitizeExtension = (source: string) => {
-    let extension = path.extname(source);
-    extension = extension.toLowerCase();
-    extension = extension.split("?")[0];
-    return extension;
-};
-
-/**
- * Uploads an image specified by the @param source to Dash's /public/files/
- * directory, and returns information generated during that upload 
- * 
- * @param {string} source is either the absolute path of an already uploaded image or
- * the url of a remote image
- * @param {string} filename dictates what to call the image. If not specified,
- * the name {@param prefix}_upload_{GUID}
- * @param {string} prefix is a string prepended to the generated image name in the
- * event that @param filename is not specified
- * 
- * @returns {UploadInformation} This method returns
- * 1) the paths to the uploaded images (plural due to resizing)
- * 2) the file name of each of the resized images
- * 3) the size of the image, in bytes (4432130)
- * 4) the content type of the image, i.e. image/(jpeg | png | ...)
- */
-export const UploadImage = async (source: string, filename?: string, prefix: string = ""): Promise<UploadInformation> => {
-    const metadata = await InspectImage(source);
-    return UploadInspectedImage(metadata, filename, prefix);
-};
-
-export interface InspectionResults {
-    isLocal: boolean;
-    stream: any;
-    normalizedUrl: string;
-    exifData: EnrichedExifData;
-    contentSize?: number;
-    contentType?: string;
-}
+    const generate = (prefix: string, url: string) => `${prefix}upload_${Utils.GenerateGuid()}${sanitizeExtension(url)}`;
+    const sanitizeExtension = (source: string) => {
+        let extension = path.extname(source);
+        extension = extension.toLowerCase();
+        extension = extension.split("?")[0];
+        return extension;
+    };
 
-export interface EnrichedExifData {
-    data: ExifData;
-    error?: string;
-}
+    /**
+     * Uploads an image specified by the @param source to Dash's /public/files/
+     * directory, and returns information generated during that upload 
+     * 
+     * @param {string} source is either the absolute path of an already uploaded image or
+     * the url of a remote image
+     * @param {string} filename dictates what to call the image. If not specified,
+     * the name {@param prefix}_upload_{GUID}
+     * @param {string} prefix is a string prepended to the generated image name in the
+     * event that @param filename is not specified
+     * 
+     * @returns {ImageUploadInformation} This method returns
+     * 1) the paths to the uploaded images (plural due to resizing)
+     * 2) the file name of each of the resized images
+     * 3) the size of the image, in bytes (4432130)
+     * 4) the content type of the image, i.e. image/(jpeg | png | ...)
+     */
+    export const UploadImage = async (source: string, filename?: string, format?: string, prefix: string = ""): Promise<ImageUploadInformation> => {
+        const metadata = await InspectImage(source);
+        return UploadInspectedImage(metadata, filename, format, prefix);
+    };
 
-export enum Partitions {
-    pdf_text = "pdf_text",
-    images = "images",
-    videos = "videos"
-}
+    export interface InspectionResults {
+        isLocal: boolean;
+        stream: any;
+        normalizedUrl: string;
+        exifData: EnrichedExifData;
+        contentSize?: number;
+        contentType?: string;
+    }
 
-export async function buildFilePartitions() {
-    const pending = Object.keys(Partitions).map(sub => createIfNotExists(filesDirectory + sub));
-    return Promise.all(pending);
-}
+    export interface EnrichedExifData {
+        data: ExifData;
+        error?: string;
+    }
 
-/**
- * Based on the url's classification as local or remote, gleans
- * as much information as possible about the specified image
- * 
- * @param source is the path or url to the image in question
- */
-export const InspectImage = async (source: string): Promise<InspectionResults> => {
-    const { isLocal, stream, normalized: normalizedUrl } = classify(source);
-    const exifData = await parseExifData(source);
-    const results = {
-        exifData,
-        isLocal,
-        stream,
-        normalizedUrl
-    };
-    // stop here if local, since request.head() can't handle local paths, only urls on the web
-    if (isLocal) {
-        return results;
+    export async function buildFileDirectories() {
+        const pending = Object.keys(Directory).map(sub => createIfNotExists(`${filesDirectory}/${sub}`));
+        return Promise.all(pending);
     }
-    const metadata = (await new Promise<any>((resolve, reject) => {
-        request.head(source, async (error, res) => {
-            if (error) {
-                return reject(error);
-            }
-            resolve(res);
-        });
-    })).headers;
-    return {
-        contentSize: parseInt(metadata[size]),
-        contentType: metadata[type],
-        ...results
-    };
-};
-
-export const UploadInspectedImage = async (metadata: InspectionResults, filename?: string, prefix = ""): Promise<UploadInformation> => {
-    const { isLocal, stream, normalizedUrl, contentSize, contentType, exifData } = metadata;
-    const resolved = filename ? sanitize(filename) : generate(prefix, normalizedUrl);
-    const extension = sanitizeExtension(normalizedUrl || resolved);
-    let information: UploadInformation = {
-        mediaPaths: [],
-        fileNames: { clean: resolved },
-        exifData,
-        contentSize,
-        contentType,
-    };
-    const { pngs, jpgs } = SharedMediaTypes;
-    return new Promise<UploadInformation>(async (resolve, reject) => {
-        const resizers = [
-            { resizer: sharp().rotate(), suffix: "_o" },
-            ...Object.values(Sizes).map(size => ({
-                resizer: sharp().resize(size.width, undefined, { withoutEnlargement: true }).rotate(),
-                suffix: size.suffix
-            }))
-        ];
-        if (pngs.includes(extension)) {
-            resizers.forEach(element => element.resizer = element.resizer.png());
-        } else if (jpgs.includes(extension)) {
-            resizers.forEach(element => element.resizer = element.resizer.jpeg());
-        }
-        for (let resizer of resizers) {
-            const suffix = resizer.suffix;
-            let mediaPath: string;
-            await new Promise<void>(resolve => {
-                const filename = resolved.substring(0, resolved.length - extension.length) + suffix + extension;
-                information.mediaPaths.push(mediaPath = uploadDirectory + filename);
-                information.fileNames[suffix] = filename;
-                stream(normalizedUrl).pipe(resizer.resizer).pipe(fs.createWriteStream(mediaPath))
-                    .on('close', resolve)
-                    .on('error', reject);
-            });
+
+    /**
+     * Based on the url's classification as local or remote, gleans
+     * as much information as possible about the specified image
+     * 
+     * @param source is the path or url to the image in question
+     */
+    export const InspectImage = async (source: string): Promise<InspectionResults> => {
+        const { isLocal, stream, normalized: normalizedUrl } = classify(source);
+        const exifData = await parseExifData(source);
+        const results = {
+            exifData,
+            isLocal,
+            stream,
+            normalizedUrl
+        };
+        // stop here if local, since request.head() can't handle local paths, only urls on the web
+        if (isLocal) {
+            return results;
         }
-        if (!isLocal) {
-            await new Promise<void>(resolve => {
-                stream(normalizedUrl).pipe(fs.createWriteStream(uploadDirectory + resolved)).on('close', resolve);
+        const metadata = (await new Promise<any>((resolve, reject) => {
+            request.head(source, async (error, res) => {
+                if (error) {
+                    return reject(error);
+                }
+                resolve(res);
             });
-        }
-        resolve(information);
-    });
-};
-
-const classify = (url: string) => {
-    const isLocal = /Dash-Web(\\|\/)src(\\|\/)server(\\|\/)public(\\|\/)files/g.test(url);
-    return {
-        isLocal,
-        stream: isLocal ? fs.createReadStream : request,
-        normalized: isLocal ? path.normalize(url) : url
+        })).headers;
+        return {
+            contentSize: parseInt(metadata[size]),
+            contentType: metadata[type],
+            ...results
+        };
     };
-};
-
-const parseExifData = async (source: string): Promise<EnrichedExifData> => {
-    return new Promise<EnrichedExifData>(resolve => {
-        new ExifImage(source, (error, data) => {
-            let reason: Opt<string> = undefined;
-            if (error) {
-                reason = (error as any).code;
+
+    export async function MoveParsedFile(absolutePath: string, destination: Directory): Promise<{ clientAccessPath: Opt<string> }> {
+        return new Promise<{ clientAccessPath: Opt<string> }>(resolve => {
+            const filename = basename(absolutePath);
+            const destinationPath = serverPathToFile(destination, filename);
+            fs.rename(absolutePath, destinationPath, error => {
+                resolve({ clientAccessPath: error ? undefined : clientPathToFile(destination, filename) });
+            });
+        });
+    }
+
+    export const UploadInspectedImage = async (metadata: InspectionResults, filename?: string, format?: string, prefix = ""): Promise<ImageUploadInformation> => {
+        const { isLocal, stream, normalizedUrl, contentSize, contentType, exifData } = metadata;
+        const resolved = filename || generate(prefix, normalizedUrl);
+        const extension = format || sanitizeExtension(normalizedUrl || resolved);
+        let information: ImageUploadInformation = {
+            clientAccessPath: clientPathToFile(Directory.images, resolved),
+            serverAccessPaths: {},
+            exifData,
+            contentSize,
+            contentType,
+        };
+        const { pngs, jpgs } = AcceptibleMedia;
+        return new Promise<ImageUploadInformation>(async (resolve, reject) => {
+            const resizers = [
+                { resizer: sharp().rotate(), suffix: SizeSuffix.Original },
+                ...Object.values(Sizes).map(size => ({
+                    resizer: sharp().resize(size.width, undefined, { withoutEnlargement: true }).rotate(),
+                    suffix: size.suffix
+                }))
+            ];
+            if (pngs.includes(extension)) {
+                resizers.forEach(element => element.resizer = element.resizer.png());
+            } else if (jpgs.includes(extension)) {
+                resizers.forEach(element => element.resizer = element.resizer.jpeg());
+            }
+            for (let { resizer, suffix } of resizers) {
+                let mediaPath: string;
+                await new Promise<void>(resolve => {
+                    const filename = InjectSize(resolved, suffix);
+                    information.serverAccessPaths[suffix] = serverPathToFile(Directory.images, filename);
+                    stream(normalizedUrl).pipe(resizer).pipe(fs.createWriteStream(serverPathToFile(Directory.images, filename)))
+                        .on('close', resolve)
+                        .on('error', reject);
+                });
             }
-            resolve({ data, error: reason });
+            if (isLocal) {
+                await new Promise<boolean>(resolve => {
+                    fs.unlink(normalizedUrl, error => resolve(error === null));
+                });
+            }
+            resolve(information);
         });
-    });
-};
+    };
 
-export const createIfNotExists = async (path: string) => {
-    if (await new Promise<boolean>(resolve => fs.exists(path, resolve))) {
-        return true;
-    }
-    return new Promise<boolean>(resolve => fs.mkdir(path, error => resolve(error === null)));
-};
+    const classify = (url: string) => {
+        const isLocal = /Dash-Web(\\|\/)src(\\|\/)server(\\|\/)public(\\|\/)files/g.test(url);
+        return {
+            isLocal,
+            stream: isLocal ? fs.createReadStream : request,
+            normalized: isLocal ? path.normalize(url) : url
+        };
+    };
 
-export const Destroy = (mediaPath: string) => new Promise<boolean>(resolve => fs.unlink(mediaPath, error => resolve(error === null)));
+    const parseExifData = async (source: string): Promise<EnrichedExifData> => {
+        return new Promise<EnrichedExifData>(resolve => {
+            new ExifImage(source, (error, data) => {
+                let reason: Opt<string> = undefined;
+                if (error) {
+                    reason = (error as any).code;
+                }
+                resolve({ data, error: reason });
+            });
+        });
+    };
 
 }
\ No newline at end of file
diff --git a/src/server/SharedMediaTypes.ts b/src/server/SharedMediaTypes.ts
index 3d3234125..8d0f441f0 100644
--- a/src/server/SharedMediaTypes.ts
+++ b/src/server/SharedMediaTypes.ts
@@ -1,9 +1,8 @@
-export namespace SharedMediaTypes {
-
+export namespace AcceptibleMedia {
     export const gifs = [".gif"];
     export const pngs = [".png"];
     export const jpgs = [".jpg", ".jpeg"];
     export const imageFormats = [...pngs, ...jpgs, ...gifs];
     export const videoFormats = [".mov", ".mp4"];
-
+    export const applicationFormats = [".pdf"];
 }
\ No newline at end of file
diff --git a/src/server/database.ts b/src/server/database.ts
index b81fc03a4..db81245c1 100644
--- a/src/server/database.ts
+++ b/src/server/database.ts
@@ -293,7 +293,7 @@ export namespace Database {
         };
 
         export const QueryUploadHistory = async (contentSize: number) => {
-            return SanitizedSingletonQuery<DashUploadUtils.UploadInformation>({ contentSize }, AuxiliaryCollections.GooglePhotosUploadHistory);
+            return SanitizedSingletonQuery<DashUploadUtils.ImageUploadInformation>({ contentSize }, AuxiliaryCollections.GooglePhotosUploadHistory);
         };
 
         export namespace GoogleAuthenticationToken {
@@ -322,7 +322,7 @@ export namespace Database {
 
         }
 
-        export const LogUpload = async (information: DashUploadUtils.UploadInformation) => {
+        export const LogUpload = async (information: DashUploadUtils.ImageUploadInformation) => {
             const bundle = {
                 _id: Utils.GenerateDeterministicGuid(String(information.contentSize!)),
                 ...information
diff --git a/src/server/index.ts b/src/server/index.ts
index d02a6005e..d77923710 100644
--- a/src/server/index.ts
+++ b/src/server/index.ts
@@ -24,7 +24,7 @@ import GooglePhotosManager from "./ApiManagers/GooglePhotosManager";
 import DiagnosticManager from "./ApiManagers/DiagnosticManager";
 
 export const publicDirectory = path.resolve(__dirname, "public");
-export const filesDirectory = path.resolve(publicDirectory, "files") + "/";
+export const filesDirectory = path.resolve(publicDirectory, "files");
 
 /**
  * These are the functions run before the server starts
@@ -34,7 +34,7 @@ export const filesDirectory = path.resolve(publicDirectory, "files") + "/";
 async function preliminaryFunctions() {
     await GoogleCredentialsLoader.loadCredentials();
     GoogleApiServerUtils.processProjectCredentials();
-    await DashUploadUtils.buildFilePartitions();
+    await DashUploadUtils.buildFileDirectories();
     await log_execution({
         startMessage: "attempting to initialize mongodb connection",
         endMessage: "connection outcome determined",