From af25eaf2a848278a58f0993cba2e68c05da0760c Mon Sep 17 00:00:00 2001
From: Sam Wilkins <samwilkins333@gmail.com>
Date: Tue, 29 Oct 2019 20:00:54 -0400
Subject: comments and fixes for google photos server sid

---
 src/client/util/Import & Export/DirectoryImportBox.tsx | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

(limited to 'src/client/util/Import & Export/DirectoryImportBox.tsx')

diff --git a/src/client/util/Import & Export/DirectoryImportBox.tsx b/src/client/util/Import & Export/DirectoryImportBox.tsx
index d74b51993..2d1b6fe20 100644
--- a/src/client/util/Import & Export/DirectoryImportBox.tsx	
+++ b/src/client/util/Import & Export/DirectoryImportBox.tsx	
@@ -20,9 +20,8 @@ import { listSpec } from "../../../new_fields/Schema";
 import { GooglePhotos } from "../../apis/google_docs/GooglePhotosClientUtils";
 import { SchemaHeaderField } from "../../../new_fields/SchemaHeaderField";
 import "./DirectoryImportBox.scss";
-import { Identified } from "../../Network";
+import { Networking } from "../../Network";
 import { BatchedArray } from "array-batcher";
-import { ExifData } from "exif";
 
 const unsupported = ["text/html", "text/plain"];
 
@@ -117,7 +116,7 @@ export default class DirectoryImportBox extends React.Component<FieldViewProps>
                 formData.append(Utils.GenerateGuid(), file);
             });
 
-            const responses = await Identified.PostFormDataToServer(RouteStore.upload, formData);
+            const responses = await Networking.PostFormDataToServer(RouteStore.upload, formData);
             runInAction(() => this.completed += batch.length);
             return responses as ImageUploadResponse[];
         });
-- 
cgit v1.2.3-70-g09d2


From 9c7e619fb9d3116649ec3779bd528b947235d5a4 Mon Sep 17 00:00:00 2001
From: Sam Wilkins <samwilkins333@gmail.com>
Date: Wed, 30 Oct 2019 15:51:29 -0400
Subject: updated array batcher

---
 package.json                                       |  2 +-
 .../util/Import & Export/DirectoryImportBox.tsx    |  5 +--
 src/server/RouteManager.ts                         |  1 +
 src/server/apis/google/GooglePhotosUploadUtils.ts  | 11 ++---
 src/server/index.ts                                | 48 ++++++++++++----------
 5 files changed, 36 insertions(+), 31 deletions(-)

(limited to 'src/client/util/Import & Export/DirectoryImportBox.tsx')

diff --git a/package.json b/package.json
index 8cbbb84af..4572a3f73 100644
--- a/package.json
+++ b/package.json
@@ -115,7 +115,7 @@
         "@types/youtube": "0.0.38",
         "adm-zip": "^0.4.13",
         "archiver": "^3.0.3",
-        "array-batcher": "^1.1.3",
+        "array-batcher": "^1.2.3",
         "async": "^2.6.2",
         "babel-runtime": "^6.26.0",
         "bcrypt-nodejs": "0.0.3",
diff --git a/src/client/util/Import & Export/DirectoryImportBox.tsx b/src/client/util/Import & Export/DirectoryImportBox.tsx
index 2d1b6fe20..bdd59cb16 100644
--- a/src/client/util/Import & Export/DirectoryImportBox.tsx	
+++ b/src/client/util/Import & Export/DirectoryImportBox.tsx	
@@ -107,7 +107,7 @@ export default class DirectoryImportBox extends React.Component<FieldViewProps>
 
         runInAction(() => this.phase = `Internal: uploading ${this.quota - this.completed} files to Dash...`);
 
-        const uploads = await BatchedArray.from(validated, { batchSize: 15 }).batchedMapAsync(async batch => {
+        const uploads = await BatchedArray.from(validated, { batchSize: 15 }).batchedMapAsync<ImageUploadResponse>(async (batch, collector) => {
             const formData = new FormData();
 
             batch.forEach(file => {
@@ -116,9 +116,8 @@ export default class DirectoryImportBox extends React.Component<FieldViewProps>
                 formData.append(Utils.GenerateGuid(), file);
             });
 
-            const responses = await Networking.PostFormDataToServer(RouteStore.upload, formData);
+            collector.push(...(await Networking.PostFormDataToServer(RouteStore.upload, formData)));
             runInAction(() => this.completed += batch.length);
-            return responses as ImageUploadResponse[];
         });
 
         await Promise.all(uploads.map(async upload => {
diff --git a/src/server/RouteManager.ts b/src/server/RouteManager.ts
index eda2a49d2..c1d38327f 100644
--- a/src/server/RouteManager.ts
+++ b/src/server/RouteManager.ts
@@ -114,6 +114,7 @@ export const STATUS = {
 };
 
 export function _error(res: express.Response, message: string, error?: any) {
+    console.error(message);
     res.statusMessage = message;
     res.status(STATUS.EXECUTION_ERROR).send(error);
 }
diff --git a/src/server/apis/google/GooglePhotosUploadUtils.ts b/src/server/apis/google/GooglePhotosUploadUtils.ts
index 172fa8d46..d3442338b 100644
--- a/src/server/apis/google/GooglePhotosUploadUtils.ts
+++ b/src/server/apis/google/GooglePhotosUploadUtils.ts
@@ -1,7 +1,7 @@
 import request = require('request-promise');
 import { GoogleApiServerUtils } from './GoogleApiServerUtils';
 import * as path from 'path';
-import { MediaItemCreationResult } from './SharedTypes';
+import { MediaItemCreationResult, NewMediaItemResult } from './SharedTypes';
 import { NewMediaItem } from "../../index";
 import { BatchedArray, TimeUnit } from 'array-batcher';
 import { DashUploadUtils } from '../../DashUploadUtils';
@@ -50,9 +50,9 @@ export namespace GooglePhotosUploadUtils {
     };
 
     export const CreateMediaItems = async (bearerToken: string, newMediaItems: NewMediaItem[], album?: { id: string }): Promise<MediaItemCreationResult> => {
-        const newMediaItemResults = await BatchedArray.from(newMediaItems, { batchSize: 50 }).batchedMapPatientInterval(
+        const newMediaItemResults = await BatchedArray.from(newMediaItems, { batchSize: 50 }).batchedMapPatientInterval<NewMediaItemResult>(
             { magnitude: 100, unit: TimeUnit.Milliseconds },
-            async (batch: NewMediaItem[]) => {
+            async (batch: NewMediaItem[], collector) => {
                 const parameters = {
                     method: 'POST',
                     headers: headers('json', bearerToken),
@@ -61,7 +61,7 @@ export namespace GooglePhotosUploadUtils {
                     json: true
                 };
                 album && (parameters.body.albumId = album.id);
-                return (await new Promise<MediaItemCreationResult>((resolve, reject) => {
+                const { newMediaItemResults } = await new Promise<MediaItemCreationResult>((resolve, reject) => {
                     request(parameters, (error, _response, body) => {
                         if (error) {
                             reject(error);
@@ -69,7 +69,8 @@ export namespace GooglePhotosUploadUtils {
                             resolve(body);
                         }
                     });
-                })).newMediaItemResults;
+                });
+                collector.push(...newMediaItemResults);
             }
         );
         return { newMediaItemResults };
diff --git a/src/server/index.ts b/src/server/index.ts
index 860cde3b5..05c866eae 100644
--- a/src/server/index.ts
+++ b/src/server/index.ts
@@ -573,18 +573,15 @@ function routeSetter(router: RouteManager) {
         onValidation: async ({ req, res, user }) => {
             let sector: GoogleApiServerUtils.Service = req.params.sector as GoogleApiServerUtils.Service;
             let action: GoogleApiServerUtils.Action = req.params.action as GoogleApiServerUtils.Action;
-            return GoogleApiServerUtils.GetEndpoint(GoogleApiServerUtils.Service[sector], user.id).then(endpoint => {
-                let handler = EndpointHandlerMap.get(action);
-                if (endpoint && handler) {
-                    let execute = handler(endpoint, req.body).then(
-                        response => res.send(response.data),
-                        rejection => res.send(rejection)
-                    );
-                    execute.catch(exception => res.send(exception));
-                    return;
-                }
-                res.send(undefined);
-            });
+            const endpoint = await GoogleApiServerUtils.GetEndpoint(GoogleApiServerUtils.Service[sector], user.id);
+            let handler = EndpointHandlerMap.get(action);
+            if (endpoint && handler) {
+                handler(endpoint, req.body)
+                    .then(response => res.send(response.data))
+                    .catch(exception => res.send(exception));
+                return;
+            }
+            res.send(undefined);
         }
     });
 
@@ -611,6 +608,12 @@ function routeSetter(router: RouteManager) {
 
     const authenticationError = "Unable to authenticate Google credentials before uploading to Google Photos!";
     const mediaError = "Unable to convert all uploaded bytes to media items!";
+    interface GooglePhotosUploadFailure {
+        batch: number;
+        index: number;
+        url: string;
+        reason: string;
+    }
 
     router.addSupervisedRoute({
         method: Method.POST,
@@ -623,30 +626,31 @@ function routeSetter(router: RouteManager) {
                 return _error(res, authenticationError);
             }
 
-            let failed: number[] = [];
-            const newMediaItems = await BatchedArray.from<GooglePhotosUploadUtils.MediaInput>(media, { batchSize: 25 }).batchedMapPatientInterval(
+            let failed: GooglePhotosUploadFailure[] = [];
+            const batched = BatchedArray.from<GooglePhotosUploadUtils.MediaInput>(media, { batchSize: 25 });
+            const newMediaItems = await batched.batchedMapPatientInterval<NewMediaItem>(
                 { magnitude: 100, unit: TimeUnit.Milliseconds },
-                async (batch: GooglePhotosUploadUtils.MediaInput[]) => {
-                    const newMediaItems: NewMediaItem[] = [];
+                async (batch, collector, { completedBatches }) => {
                     for (let index = 0; index < batch.length; index++) {
-                        const element = batch[index];
-                        const uploadToken = await GooglePhotosUploadUtils.DispatchGooglePhotosUpload(token, element.url);
+                        const { url, description } = batch[index];
+                        const fail = (reason: string) => failed.push({ reason, batch: completedBatches + 1, index, url });
+                        const uploadToken = await GooglePhotosUploadUtils.DispatchGooglePhotosUpload(token, url).catch(fail);
                         if (!uploadToken) {
-                            failed.push(index);
+                            fail(`${path.extname(url)} is not an accepted extension`);
                         } else {
-                            newMediaItems.push({
-                                description: element.description,
+                            collector.push({
+                                description,
                                 simpleMediaItem: { uploadToken }
                             });
                         }
                     }
-                    return newMediaItems;
                 }
             );
 
             const failedCount = failed.length;
             if (failedCount) {
                 console.error(`Unable to upload ${failedCount} image${failedCount === 1 ? "" : "s"} to Google's servers`);
+                console.log(failed.map(({ reason, batch, index, url }) => `@${batch}.${index}: ${url} failed: ${reason}`).join('\n'));
             }
 
             return GooglePhotosUploadUtils.CreateMediaItems(token, newMediaItems, req.body.album).then(
-- 
cgit v1.2.3-70-g09d2

