integrated with array batcher npm module

4 files changed, 60 insertions, 58 deletions

diff --git a/src/client/util/Import & Export/DirectoryImportBox.tsx b/src/client/util/Import & Export/DirectoryImportBox.tsx
index 762302bc8..26a00dc7c 100644
--- a/src/client/util/Import & Export/DirectoryImportBox.tsx
+++ b/src/client/util/Import & Export/DirectoryImportBox.tsx
@@ -20,7 +20,7 @@ import { listSpec } from "../../../new_fields/Schema";
 import { GooglePhotos } from "../../apis/google_docs/GooglePhotosClientUtils";
 import { SchemaHeaderField } from "../../../new_fields/SchemaHeaderField";
 import "./DirectoryImportBox.scss";
-import { batchedMapAsync } from "array-batcher";
+import BatchedArray from "array-batcher";
 
 const unsupported = ["text/html", "text/plain"];
 interface FileResponse {
@@ -104,19 +104,22 @@ export default class DirectoryImportBox extends React.Component<FieldViewProps>
 
         runInAction(() => this.phase = `Internal: uploading ${this.quota - this.completed} files to Dash...`);
 
-        const uploads = await batchedMapAsync(validated, { batchSize: 15 }, async batch => {
-            const formData = new FormData();
-            const parameters = { method: 'POST', body: formData };
-
-            batch.forEach(file => {
-                sizes.push(file.size);
-                modifiedDates.push(file.lastModified);
-                formData.append(Utils.GenerateGuid(), file);
-            });
-
-            const responses = await (await fetch(RouteStore.upload, parameters)).json();
-            runInAction(() => this.completed += batch.length);
-            return responses as FileResponse[];
+        const uploads = await BatchedArray.from(validated).batchedMapAsync({
+            batcher: { batchSize: 15 },
+            converter: async batch => {
+                const formData = new FormData();
+                const parameters = { method: 'POST', body: formData };
+
+                batch.forEach(file => {
+                    sizes.push(file.size);
+                    modifiedDates.push(file.lastModified);
+                    formData.append(Utils.GenerateGuid(), file);
+                });
+
+                const responses = await (await fetch(RouteStore.upload, parameters)).json();
+                runInAction(() => this.completed += batch.length);
+                return responses as FileResponse[];
+            }
         });
 
         await Promise.all(uploads.map(async upload => {
diff --git a/src/server/apis/google/GooglePhotosUploadUtils.ts b/src/server/apis/google/GooglePhotosUploadUtils.ts
index 29575763c..40564ff3b 100644
--- a/src/server/apis/google/GooglePhotosUploadUtils.ts
+++ b/src/server/apis/google/GooglePhotosUploadUtils.ts
@@ -7,7 +7,7 @@ import { Opt } from '../../../new_fields/Doc';
 import * as sharp from 'sharp';
 import { MediaItemCreationResult } from './SharedTypes';
 import { NewMediaItem } from "../../index";
-import { batchedMapInterval, FixedBatcher, TimeUnit, Interval } from "array-batcher";
+import BatchedArray, { FixedBatcher, TimeUnit, Interval } from "array-batcher";
 
 const uploadDirectory = path.join(__dirname, "../../public/files/");
 
@@ -62,30 +62,29 @@ export namespace GooglePhotosUploadUtils {
     };
 
     export const CreateMediaItems = async (newMediaItems: NewMediaItem[], album?: { id: string }): Promise<MediaItemCreationResult> => {
-        const createFromUploadTokens = async (batch: NewMediaItem[]) => {
-            const parameters = {
-                method: 'POST',
-                headers: headers('json'),
-                uri: prepend('mediaItems:batchCreate'),
-                body: { newMediaItems: batch } as any,
-                json: true
-            };
-            album && (parameters.body.albumId = album.id);
-            return (await new Promise<MediaItemCreationResult>((resolve, reject) => {
-                request(parameters, (error, _response, body) => {
-                    if (error) {
-                        reject(error);
-                    } else {
-                        resolve(body);
-                    }
-                });
-            })).newMediaItemResults;
-        };
-        const batcher: FixedBatcher = { batchSize: 50 };
-        const interval: Interval = { magnitude: 100, unit: TimeUnit.Milliseconds };
-
-        const newMediaItemResults = await batchedMapInterval(newMediaItems, batcher, createFromUploadTokens, interval);
-
+        const newMediaItemResults = await BatchedArray.from(newMediaItems).batchedMapInterval({
+            batcher: { batchSize: 50 },
+            interval: { magnitude: 100, unit: TimeUnit.Milliseconds },
+            converter: async (batch: NewMediaItem[]) => {
+                const parameters = {
+                    method: 'POST',
+                    headers: headers('json'),
+                    uri: prepend('mediaItems:batchCreate'),
+                    body: { newMediaItems: batch } as any,
+                    json: true
+                };
+                album && (parameters.body.albumId = album.id);
+                return (await new Promise<MediaItemCreationResult>((resolve, reject) => {
+                    request(parameters, (error, _response, body) => {
+                        if (error) {
+                            reject(error);
+                        } else {
+                            resolve(body);
+                        }
+                    });
+                })).newMediaItemResults;
+            }
+        });
         return { newMediaItemResults };
     };
 
diff --git a/src/server/credentials/google_docs_token.json b/src/server/credentials/google_docs_token.json
index cdea139a3..9b0a649ac 100644
--- a/src/server/credentials/google_docs_token.json
+++ b/src/server/credentials/google_docs_token.json
@@ -1 +1 @@
-{"access_token":"ya29.ImCJB_jd-XlGcIHAHgN2Zl3BWQ6sMHdeMMuRxU6sPCbAYIT8hXws-WDmQf65ZY1f-0d3y7HcCcuOxtZJ_0IcBb1-yIBxiOf3VJWPmvjGiJQq_mANGVSSmsBHhqpIaYkeQN0","refresh_token":"1/HTv_xFHszu2Nf3iiFrUTaeKzC_Vp2-6bpIB06xW_WHI","scope":"https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/documents.readonly https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/photoslibrary https://www.googleapis.com/auth/photoslibrary.appendonly https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/photoslibrary.sharing","token_type":"Bearer","expiry_date":1568973665276}
\ No newline at end of file
+{"access_token":"ya29.ImCJB3_-mtKbCG8L7nQt3CDfn4HR2_xqUw8bUQkvaZZgxJZ8-WUlHe6UaSOmGtrvI4QabpYRGutYGSTUdc9bvIYooerZgAz80uk7-KEbnAxpwydRE-TK_1_VyWYZ2YW6Ht8","refresh_token":"1/HTv_xFHszu2Nf3iiFrUTaeKzC_Vp2-6bpIB06xW_WHI","scope":"https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/documents.readonly https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/photoslibrary https://www.googleapis.com/auth/photoslibrary.appendonly https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/photoslibrary.sharing","token_type":"Bearer","expiry_date":1569013995678}
\ No newline at end of file
diff --git a/src/server/index.ts b/src/server/index.ts
index e03079d66..12ceb9886 100644
--- a/src/server/index.ts
+++ b/src/server/index.ts
@@ -47,7 +47,7 @@ const mongoose = require('mongoose');
 const probe = require("probe-image-size");
 import * as qs from 'query-string';
 import { Opt } from '../new_fields/Doc';
-import { batchedMapInterval, TimeUnit } from "array-batcher";
+import BatchedArray, { TimeUnit } from "array-batcher";
 
 const download = (url: string, dest: fs.PathLike) => request.get(url).pipe(fs.createWriteStream(dest));
 let youtubeApiKey: string;
@@ -841,25 +841,25 @@ app.post(RouteStore.googlePhotosMediaUpload, async (req, res) => {
 
     let failed = 0;
 
-    const dispatchUpload = async (batch: GooglePhotosUploadUtils.MediaInput[]) => {
-        const newMediaItems: NewMediaItem[] = [];
-        for (let element of batch) {
-            const uploadToken = await GooglePhotosUploadUtils.DispatchGooglePhotosUpload(element.url);
-            if (!uploadToken) {
-                failed++;
-            } else {
-                newMediaItems.push({
-                    description: element.description,
-                    simpleMediaItem: { uploadToken }
-                });
+    const newMediaItems = await BatchedArray.from(mediaInput).batchedMapInterval({
+        batcher: { batchSize: 25 },
+        interval: { magnitude: 100, unit: TimeUnit.Milliseconds },
+        converter: async (batch: GooglePhotosUploadUtils.MediaInput[]) => {
+            const newMediaItems: NewMediaItem[] = [];
+            for (let element of batch) {
+                const uploadToken = await GooglePhotosUploadUtils.DispatchGooglePhotosUpload(element.url);
+                if (!uploadToken) {
+                    failed++;
+                } else {
+                    newMediaItems.push({
+                        description: element.description,
+                        simpleMediaItem: { uploadToken }
+                    });
+                }
             }
+            return newMediaItems;
         }
-        return newMediaItems;
-    };
-    const batcher = { batchSize: 25 };
-    const interval = { magnitude: 100, unit: TimeUnit.Milliseconds };
-
-    const newMediaItems = await batchedMapInterval(mediaInput, batcher, dispatchUpload, interval);
+    });
 
     if (failed) {
         return _error(res, tokenError);