Merge pull request #293 from browngraphicslab/exif

Exif

4 files changed, 126 insertions, 12 deletions

diff --git a/src/server/DashUploadUtils.ts b/src/server/DashUploadUtils.ts
index 57b46714a..46d897339 100644
--- a/src/server/DashUploadUtils.ts
+++ b/src/server/DashUploadUtils.ts
@@ -3,6 +3,8 @@ import { Utils } from '../Utils';
 import * as path from 'path';
 import * as sharp from 'sharp';
 import request = require('request-promise');
+import { ExifData, ExifImage } from 'exif';
+import { Opt } from '../new_fields/Doc';
 
 const uploadDirectory = path.join(__dirname, './public/files/');
 
@@ -31,12 +33,19 @@ export namespace DashUploadUtils {
     export interface UploadInformation {
         mediaPaths: string[];
         fileNames: { [key: string]: string };
+        exifData: EnrichedExifData;
         contentSize?: number;
         contentType?: string;
     }
 
-    const generate = (prefix: string, url: string) => `${prefix}upload_${Utils.GenerateGuid()}${path.extname(url).toLowerCase()}`;
+    const generate = (prefix: string, url: string) => `${prefix}upload_${Utils.GenerateGuid()}${sanitizeExtension(url)}`;
     const sanitize = (filename: string) => filename.replace(/\s+/g, "_");
+    const sanitizeExtension = (source: string) => {
+        let extension = path.extname(source);
+        extension = extension.toLowerCase();
+        extension = extension.split("?")[0];
+        return extension;
+    };
 
     /**
      * Uploads an image specified by the @param source to Dash's /public/files/
@@ -64,10 +73,16 @@ export namespace DashUploadUtils {
         isLocal: boolean;
         stream: any;
         normalizedUrl: string;
+        exifData: EnrichedExifData;
         contentSize?: number;
         contentType?: string;
     }
 
+    export interface EnrichedExifData {
+        data: ExifData;
+        error?: string;
+    }
+
     /**
      * Based on the url's classification as local or remote, gleans
      * as much information as possible about the specified image
@@ -76,7 +91,9 @@ export namespace DashUploadUtils {
      */
     export const InspectImage = async (source: string): Promise<InspectionResults> => {
         const { isLocal, stream, normalized: normalizedUrl } = classify(source);
+        const exifData = await parseExifData(source);
         const results = {
+            exifData,
             isLocal,
             stream,
             normalizedUrl
@@ -101,13 +118,13 @@ export namespace DashUploadUtils {
     };
 
     export const UploadInspectedImage = async (metadata: InspectionResults, filename?: string, prefix = ""): Promise<UploadInformation> => {
-        const { isLocal, stream, normalizedUrl, contentSize, contentType } = metadata;
+        const { isLocal, stream, normalizedUrl, contentSize, contentType, exifData } = metadata;
         const resolved = filename ? sanitize(filename) : generate(prefix, normalizedUrl);
-        let extension = path.extname(normalizedUrl) || path.extname(resolved);
-        extension && (extension = extension.toLowerCase());
+        const extension = sanitizeExtension(normalizedUrl || resolved);
         let information: UploadInformation = {
             mediaPaths: [],
             fileNames: { clean: resolved },
+            exifData,
             contentSize,
             contentType,
         };
@@ -159,6 +176,18 @@ export namespace DashUploadUtils {
         };
     };
 
+    const parseExifData = async (source: string): Promise<EnrichedExifData> => {
+        return new Promise<EnrichedExifData>(resolve => {
+            new ExifImage(source, (error, data) => {
+                let reason: Opt<string> = undefined;
+                if (error) {
+                    reason = (error as any).code;
+                }
+                resolve({ data, error: reason });
+            });
+        });
+    };
+
     export const createIfNotExists = async (path: string) => {
         if (await new Promise<boolean>(resolve => fs.exists(path, resolve))) {
             return true;
diff --git a/src/server/RouteStore.ts b/src/server/RouteStore.ts
index 391d9dc0c..7426ffb39 100644
--- a/src/server/RouteStore.ts
+++ b/src/server/RouteStore.ts
@@ -13,6 +13,8 @@ export enum RouteStore {
     upload = "/upload",
     dataUriToImage = "/uploadURI",
     images = "/images",
+    inspectImage = "/inspectImage",
+    imageHierarchyExport = "/imageHierarchyExport",
 
     // USER AND WORKSPACES
     getCurrUser = "/getCurrentUser",
diff --git a/src/server/database.ts b/src/server/database.ts
index d2375ebd9..990441d5a 100644
--- a/src/server/database.ts
+++ b/src/server/database.ts
@@ -131,7 +131,7 @@ export namespace Database {
             }
         }
 
-        public getDocument(id: string, fn: (result?: Transferable) => void, collectionName = Database.DocumentsCollection) {
+        public getDocument(id: string, fn: (result?: Transferable) => void, collectionName = "newDocuments") {
             if (this.db) {
                 this.db.collection(collectionName).findOne({ _id: id }, (err, result) => {
                     if (result) {
@@ -165,7 +165,7 @@ export namespace Database {
             }
         }
 
-        public async visit(ids: string[], fn: (result: any) => string[], collectionName = "newDocuments"): Promise<void> {
+        public async visit(ids: string[], fn: (result: any) => string[] | Promise<string[]>, collectionName = "newDocuments"): Promise<void> {
             if (this.db) {
                 const visited = new Set<string>();
                 while (ids.length) {
@@ -179,10 +179,9 @@ export namespace Database {
                     for (const doc of docs) {
                         const id = doc.id;
                         visited.add(id);
-                        ids.push(...fn(doc));
+                        ids.push(...(await fn(doc)));
                     }
                 }
-
             } else {
                 return new Promise(res => {
                     this.onConnect.push(() => {
diff --git a/src/server/index.ts b/src/server/index.ts
index dd44a0ce8..62938b9c7 100644
--- a/src/server/index.ts
+++ b/src/server/index.ts
@@ -53,6 +53,8 @@ import { DashUploadUtils } from './DashUploadUtils';
 import { BatchedArray, TimeUnit } from 'array-batcher';
 import { ParsedPDF } from "./PdfTypes";
 import { reject } from 'bluebird';
+import { ExifData } from 'exif';
+import { Result } from '../client/northstar/model/idea/idea';
 
 const download = (url: string, dest: fs.PathLike) => request.get(url).pipe(fs.createWriteStream(dest));
 let youtubeApiKey: string;
@@ -319,6 +321,76 @@ app.get("/serializeDoc/:docId", async (req, res) => {
     res.send({ docs, files: Array.from(files) });
 });
 
+export type Hierarchy = { [id: string]: string | Hierarchy };
+export type ZipMutator = (file: Archiver.Archiver) => void | Promise<void>;
+
+app.get(`${RouteStore.imageHierarchyExport}/:docId`, async (req, res) => {
+    const id = req.params.docId;
+    const hierarchy: Hierarchy = {};
+    await targetedVisitorRecursive(id, hierarchy);
+    BuildAndDispatchZip(res, async zip => {
+        await hierarchyTraverserRecursive(zip, hierarchy);
+    });
+});
+
+const BuildAndDispatchZip = async (res: Response, mutator: ZipMutator): Promise<void> => {
+    const zip = Archiver('zip');
+    zip.pipe(res);
+    await mutator(zip);
+    return zip.finalize();
+};
+
+const targetedVisitorRecursive = async (seedId: string, hierarchy: Hierarchy): Promise<void> => {
+    const local: Hierarchy = {};
+    const { title, data } = await getData(seedId);
+    const label = `${title} (${seedId})`;
+    if (Array.isArray(data)) {
+        hierarchy[label] = local;
+        await Promise.all(data.map(proxy => targetedVisitorRecursive(proxy.fieldId, local)));
+    } else {
+        hierarchy[label + path.extname(data)] = data;
+    }
+};
+
+const getData = async (seedId: string): Promise<{ data: string | any[], title: string }> => {
+    return new Promise<{ data: string | any[], title: string }>((resolve, reject) => {
+        Database.Instance.getDocument(seedId, async (result: any) => {
+            const { data, proto, title } = result.fields;
+            if (data) {
+                if (data.url) {
+                    resolve({ data: data.url, title });
+                } else if (data.fields) {
+                    resolve({ data: data.fields, title });
+                } else {
+                    reject();
+                }
+            }
+            if (proto) {
+                getData(proto.fieldId).then(resolve, reject);
+            }
+        });
+    });
+};
+
+const hierarchyTraverserRecursive = async (file: Archiver.Archiver, hierarchy: Hierarchy, prefix = "Dash Export"): Promise<void> => {
+    for (const key of Object.keys(hierarchy)) {
+        const result = hierarchy[key];
+        if (typeof result === "string") {
+            let path: string;
+            let matches: RegExpExecArray | null;
+            if ((matches = /\:1050\/files\/(upload\_[\da-z]{32}.*)/g.exec(result)) !== null) {
+                path = `${__dirname}/public/files/${matches[1]}`;
+            } else {
+                const information = await DashUploadUtils.UploadImage(result);
+                path = information.mediaPaths[0];
+            }
+            file.file(path, { name: key, prefix });
+        } else {
+            await hierarchyTraverserRecursive(file, result, `${prefix}/${key}`);
+        }
+    }
+};
+
 app.get("/downloadId/:docId", async (req, res) => {
     res.set('Content-disposition', `attachment;`);
     res.set('Content-Type', "application/zip");
@@ -590,10 +662,11 @@ const uploadDirectory = __dirname + "/public/files/";
 const pdfDirectory = uploadDirectory + "text";
 DashUploadUtils.createIfNotExists(pdfDirectory);
 
-interface FileResponse {
+interface ImageFileResponse {
     name: string;
     path: string;
     type: string;
+    exif: Opt<DashUploadUtils.EnrichedExifData>;
 }
 
 // SETTERS
@@ -604,10 +677,11 @@ app.post(
         form.uploadDir = uploadDirectory;
         form.keepExtensions = true;
         form.parse(req, async (_err, _fields, files) => {
-            let results: FileResponse[] = [];
+            let results: ImageFileResponse[] = [];
             for (const key in files) {
                 const { type, path: location, name } = files[key];
                 const filename = path.basename(location);
+                let uploadInformation: Opt<DashUploadUtils.UploadInformation>;
                 if (filename.endsWith(".pdf")) {
                     let dataBuffer = fs.readFileSync(uploadDirectory + filename);
                     const result: ParsedPDF = await pdf(dataBuffer);
@@ -622,9 +696,10 @@ app.post(
                         });
                     });
                 } else {
-                    await DashUploadUtils.UploadImage(uploadDirectory + filename, filename).catch(() => console.log(`Unable to process ${filename}`));
+                    uploadInformation = await DashUploadUtils.UploadImage(uploadDirectory + filename, filename);
                 }
-                results.push({ name, type, path: `/files/${filename}` });
+                const exif = uploadInformation ? uploadInformation.exifData : undefined;
+                results.push({ name, type, path: `/files/${filename}`, exif });
 
             }
             _success(res, results);
@@ -632,6 +707,15 @@ app.post(
     }
 );
 
+app.post(RouteStore.inspectImage, async (req, res) => {
+    const { source } = req.body;
+    if (typeof source === "string") {
+        const uploadInformation = await DashUploadUtils.UploadImage(source);
+        return res.send(await DashUploadUtils.InspectImage(uploadInformation.mediaPaths[0]));
+    }
+    res.send({});
+});
+
 addSecureRoute(
     Method.POST,
     (user, res, req) => {