added batching, generically

7 files changed, 155 insertions, 65 deletions

diff --git a/src/client/northstar/utils/Extensions.ts b/src/client/northstar/utils/Extensions.ts
index ab9384f1f..720f4a062 100644
--- a/src/client/northstar/utils/Extensions.ts
+++ b/src/client/northstar/utils/Extensions.ts
@@ -5,6 +5,8 @@ interface String {
     hasNewline(): boolean;
 }
 
+const extensions = require(".././/.//../util/UtilExtensions");
+
 String.prototype.ReplaceAll = function (toReplace: string, replacement: string): string {
     var target = this;
     return target.split(toReplace).join(replacement);
@@ -18,6 +20,31 @@ String.prototype.Truncate = function (length: number, replacement: string): Stri
     return target;
 };
 
+interface Action<T> {
+    handler: (batch: T[]) => any;
+    interval?: number;
+}
+
+interface BatchParameters<T> {
+    size: number;
+    action?: Action<T>;
+}
+
+interface Array<T> {
+    batch(parameters: BatchParameters<T>): Promise<T[][]>;
+    lastElement(): T;
+}
+
+Array.prototype.batch = extensions.Batch;
+
+Array.prototype.lastElement = function <T>() {
+    if (!this.length) {
+        return undefined;
+    }
+    const last: T = this[this.length - 1];
+    return last;
+};
+
 interface Math {
     log10(val: number): number;
 }
diff --git a/src/client/util/Import & Export/DirectoryImportBox.tsx b/src/client/util/Import & Export/DirectoryImportBox.tsx
index 260c6a629..5915f3412 100644
--- a/src/client/util/Import & Export/DirectoryImportBox.tsx
+++ b/src/client/util/Import & Export/DirectoryImportBox.tsx
@@ -92,29 +92,28 @@ export default class DirectoryImportBox extends React.Component<FieldViewProps>
             this.completed = 0;
         });
 
-        let sizes = [];
-        let modifiedDates = [];
+        let sizes: number[] = [];
+        let modifiedDates: number[] = [];
 
-        let i = 0;
         const uploads: FileResponse[] = [];
-        const batchSize = 15;
 
-        while (i < validated.length) {
-            const cap = Math.min(validated.length, i + batchSize);
-            let formData = new FormData();
-            const batch = validated.slice(i, cap);
+        await validated.batch({
+            size: 15,
+            action: {
+                handler: async (batch: File[]) => {
+                    sizes.push(...batch.map(file => file.size));
+                    modifiedDates.push(...batch.map(file => file.lastModified));
 
-            sizes.push(...batch.map(file => file.size));
-            modifiedDates.push(...batch.map(file => file.lastModified));
+                    let formData = new FormData();
+                    batch.forEach(file => formData.append(Utils.GenerateGuid(), file));
+                    const parameters = { method: 'POST', body: formData };
 
-            batch.forEach(file => formData.append(Utils.GenerateGuid(), file));
-            const parameters = { method: 'POST', body: formData };
-            uploads.push(...(await (await fetch(Utils.prepend(RouteStore.upload), parameters)).json()));
-
-            runInAction(() => this.completed += batch.length);
-            i = cap;
-        }
+                    uploads.push(...(await (await fetch(Utils.prepend(RouteStore.upload), parameters)).json()));
 
+                    runInAction(() => this.completed += batch.length);
+                }
+            }
+        });
 
         await Promise.all(uploads.map(async upload => {
             const type = upload.type;
diff --git a/src/client/util/UtilExtensions.ts b/src/client/util/UtilExtensions.ts
new file mode 100644
index 000000000..1e277b242
--- /dev/null
+++ b/src/client/util/UtilExtensions.ts
@@ -0,0 +1,39 @@
+module.exports.Batch = async function <T>(parameters: BatchParameters<T>) {
+    const { size, action } = parameters;
+    const batches: T[][] = [];
+    let i = 0;
+    while (i < this.length) {
+        const cap = Math.min(i + size, this.length);
+        batches.push(this.slice(i, cap));
+        i = cap;
+    }
+    console.log(`Beginning action on ${this.length} elements, split into ${batches.length} groups => ${batches.map(batch => batch.length).join(", ")}`);
+    if (action) {
+        const { handler, interval } = action;
+        if (!interval || batches.length === 1) {
+            for (let batch of batches) {
+                await handler(batch);
+            }
+        } else {
+            return new Promise<T[][]>(resolve => {
+                const iterator = batches[Symbol.iterator]();
+                const quota = batches.length;
+                let completed = 0;
+                const tag = setInterval(async () => {
+                    const next = iterator.next();
+                    if (next.done) {
+                        clearInterval(tag);
+                        return;
+                    }
+                    const batch = next.value;
+                    console.log(`Handling next batch with ${batch.length} elements`);
+                    await handler(batch);
+                    if (++completed === quota) {
+                        resolve(batches);
+                    }
+                }, interval);
+            });
+        }
+    }
+    return batches;
+};
\ No newline at end of file
diff --git a/src/client/views/Main.tsx b/src/client/views/Main.tsx
index b623cab4e..aa002cee9 100644
--- a/src/client/views/Main.tsx
+++ b/src/client/views/Main.tsx
@@ -20,14 +20,6 @@ String.prototype.hasNewline = function () {
     return this.endsWith("\n");
 };
 
-(Array.prototype as any).lastElement = function (this: any[]) {
-    if (!this.length) {
-        return undefined;
-    }
-    return this[this.length - 1];
-};
-
-
 let swapDocs = async () => {
     let oldDoc = await Cast(CurrentUserUtils.UserDocument.linkManagerDoc, Doc);
     // Docs.Prototypes.MainLinkDocument().allLinks = new List<Doc>();
diff --git a/src/server/apis/google/GooglePhotosUploadUtils.ts b/src/server/apis/google/GooglePhotosUploadUtils.ts
index e91f8352b..3989590c6 100644
--- a/src/server/apis/google/GooglePhotosUploadUtils.ts
+++ b/src/server/apis/google/GooglePhotosUploadUtils.ts
@@ -6,6 +6,7 @@ import * as path from 'path';
 import { Opt } from '../../../new_fields/Doc';
 import * as sharp from 'sharp';
 import { MediaItemCreationResult, NewMediaItemResult } from './SharedTypes';
+import { NewMediaItem } from '../..';
 
 const uploadDirectory = path.join(__dirname, "../../public/files/");
 
@@ -39,7 +40,7 @@ export namespace GooglePhotosUploadUtils {
     };
 
     export const DispatchGooglePhotosUpload = async (url: string) => {
-        const body = await request(url, { encoding: null });
+        const body = await request(url, { encoding: null }).catch(error => console.log("Error in streaming body!", error));
         const parameters = {
             method: 'POST',
             headers: {
@@ -56,36 +57,37 @@ export namespace GooglePhotosUploadUtils {
                 return reject(error);
             }
             resolve(body);
-        }));
+        }).catch(error => console.log("Error in literal uploading process to Google's servers!", error))).catch(error => console.log("Error in literal uploading process to Google's servers!", error));
     };
 
-    export const CreateMediaItems = async (newMediaItems: any[], album?: { id: string }): Promise<MediaItemCreationResult> => {
-        const quota = newMediaItems.length;
-        let handled = 0;
+    export const CreateMediaItems = async (newMediaItems: NewMediaItem[], album?: { id: string }): Promise<MediaItemCreationResult> => {
         const newMediaItemResults: NewMediaItemResult[] = [];
-        while (handled < quota) {
-            const cap = Math.min(newMediaItems.length, handled + 50);
-            const batch = newMediaItems.slice(handled, cap);
-            console.log(batch.length);
-            const parameters = {
-                method: 'POST',
-                headers: headers('json'),
-                uri: prepend('mediaItems:batchCreate'),
-                body: { newMediaItems: batch } as any,
-                json: true
-            };
-            album && (parameters.body.albumId = album.id);
-            newMediaItemResults.push(...(await new Promise<MediaItemCreationResult>((resolve, reject) => {
-                request(parameters, (error, _response, body) => {
-                    if (error) {
-                        reject(error);
-                    } else {
-                        resolve(body);
-                    }
-                });
-            })).newMediaItemResults);
-            handled = cap;
-        }
+        await newMediaItems.batch({
+            size: 50,
+            action: {
+                handler: async (batch: NewMediaItem[]) => {
+                    console.log(batch.length);
+                    const parameters = {
+                        method: 'POST',
+                        headers: headers('json'),
+                        uri: prepend('mediaItems:batchCreate'),
+                        body: { newMediaItems: batch } as any,
+                        json: true
+                    };
+                    album && (parameters.body.albumId = album.id);
+                    newMediaItemResults.push(...(await new Promise<MediaItemCreationResult>((resolve, reject) => {
+                        request(parameters, (error, _response, body) => {
+                            if (error) {
+                                reject(error);
+                            } else {
+                                resolve(body);
+                            }
+                        });
+                    })).newMediaItemResults);
+                },
+                interval: 1000
+            }
+        });
         return { newMediaItemResults };
     };
 
diff --git a/src/server/credentials/google_docs_token.json b/src/server/credentials/google_docs_token.json
index bdeca837b..d8e0eae21 100644
--- a/src/server/credentials/google_docs_token.json
+++ b/src/server/credentials/google_docs_token.json
@@ -1 +1 @@
-{"access_token":"ya29.ImCCBwLh8M4qd5ApvvhgMeCvbQidOUehUNU2fj3RH6Zx8D3rnCooiVgxoWbJ2ddS3a0_PGAQvCA7-GAeS70wUny80VKgCLjNbTlZkuxaRqpAd5yFGuWzcRljXrEIuA7EVu0","refresh_token":"1/HTv_xFHszu2Nf3iiFrUTaeKzC_Vp2-6bpIB06xW_WHI","scope":"https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/documents.readonly https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/photoslibrary https://www.googleapis.com/auth/photoslibrary.appendonly https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/photoslibrary.sharing","token_type":"Bearer","expiry_date":1568394019509}
\ No newline at end of file
+{"access_token":"ya29.GlyDB0wsV3-oS6q5TFuJSmH1YP_SPf_X6RHaJVmfqj0NTCtaPLFonZRxdT52kUkiHJgAoRizxZvlSIGptXKfnmG4BFouhgyo9ZKP0QtOH-kPR9b9x5WhGCd5NWqz0A","refresh_token":"1/HTv_xFHszu2Nf3iiFrUTaeKzC_Vp2-6bpIB06xW_WHI","scope":"https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/documents.readonly https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/photoslibrary https://www.googleapis.com/auth/photoslibrary.appendonly https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/photoslibrary.sharing","token_type":"Bearer","expiry_date":1568412547334}
\ No newline at end of file
diff --git a/src/server/index.ts b/src/server/index.ts
index fdcc79b4d..542a4ea65 100644
--- a/src/server/index.ts
+++ b/src/server/index.ts
@@ -46,6 +46,7 @@ const MongoStore = require('connect-mongo')(session);
 const mongoose = require('mongoose');
 const probe = require("probe-image-size");
 import * as qs from 'query-string';
+const extensions = require("../client/util/UtilExtensions");
 
 const download = (url: string, dest: fs.PathLike) => request.get(url).pipe(fs.createWriteStream(dest));
 let youtubeApiKey: string;
@@ -827,20 +828,50 @@ app.get(RouteStore.googlePhotosAccessToken, (req, res) => GoogleApiServerUtils.R
 const tokenError = "Unable to successfully upload bytes for all images!";
 const mediaError = "Unable to convert all uploaded bytes to media items!";
 
+export interface NewMediaItem {
+    description: string;
+    simpleMediaItem: {
+        uploadToken: string;
+    };
+}
+
+Array.prototype.batch = extensions.Batch;
+
 app.post(RouteStore.googlePhotosMediaUpload, async (req, res) => {
-    const media: GooglePhotosUploadUtils.MediaInput[] = req.body.media;
+    const mediaInput: GooglePhotosUploadUtils.MediaInput[] = req.body.media;
     await GooglePhotosUploadUtils.initialize({ uploadDirectory, credentialsPath, tokenPath });
-    const newMediaItems = await Promise.all(media.map(async element => {
-        const uploadToken = await GooglePhotosUploadUtils.DispatchGooglePhotosUpload(element.url).catch(error => {
-            console.log("Dispatching upload error!");
-            console.log(error);
+
+    const newMediaItems: NewMediaItem[] = [];
+    let failed = 0;
+    const size = 25;
+
+    try {
+        await mediaInput.batch({
+            size,
+            action: {
+                handler: async (batch: GooglePhotosUploadUtils.MediaInput[]) => {
+                    await Promise.all(batch.map(async element => {
+                        console.log(`Uploading ${element.url} to Google's servers...`);
+                        const uploadToken = await GooglePhotosUploadUtils.DispatchGooglePhotosUpload(element.url);
+                        if (uploadToken) {
+                            newMediaItems.push({
+                                description: element.description,
+                                simpleMediaItem: { uploadToken }
+                            });
+                        } else {
+                            console.log("FAIL!", element.url, element.description);
+                            failed++;
+                        }
+                    }));
+                },
+                interval: 3000
+            }
         });
-        return !uploadToken ? undefined : {
-            description: element.description,
-            simpleMediaItem: { uploadToken }
-        };
-    }));
-    if (!newMediaItems.every(item => item)) {
+    } catch (e) {
+        console.log("WHAT HAPPENED?");
+        console.log(e);
+    }
+    if (failed) {
         return _error(res, tokenError);
     }
     GooglePhotosUploadUtils.CreateMediaItems(newMediaItems, req.body.album).then(