fixed batching and refactor

6 files changed, 95 insertions, 116 deletions

diff --git a/src/client/northstar/utils/Extensions.ts b/src/client/northstar/utils/Extensions.ts
index 720f4a062..c866d1bc3 100644
--- a/src/client/northstar/utils/Extensions.ts
+++ b/src/client/northstar/utils/Extensions.ts
@@ -20,22 +20,17 @@ String.prototype.Truncate = function (length: number, replacement: string): Stri
     return target;
 };
 
-interface Action<T> {
-    handler: (batch: T[]) => any;
-    interval?: number;
-}
 
-interface BatchParameters<T> {
-    size: number;
-    action?: Action<T>;
-}
+type BatchHandler<I, O> = (batch: I[]) => O[] | Promise<O[]>;
 
 interface Array<T> {
-    batch(parameters: BatchParameters<T>): Promise<T[][]>;
+    batch(batchSize: number): T[][];
+    batchAction<O>(batchSize: number, handler: BatchHandler<T, O>, interval?: number): Promise<O[]>;
     lastElement(): T;
 }
 
 Array.prototype.batch = extensions.Batch;
+Array.prototype.batchAction = extensions.BatchAction;
 
 Array.prototype.lastElement = function <T>() {
     if (!this.length) {
diff --git a/src/client/util/Import & Export/DirectoryImportBox.tsx b/src/client/util/Import & Export/DirectoryImportBox.tsx
index 5915f3412..1ae0e7525 100644
--- a/src/client/util/Import & Export/DirectoryImportBox.tsx
+++ b/src/client/util/Import & Export/DirectoryImportBox.tsx
@@ -95,25 +95,19 @@ export default class DirectoryImportBox extends React.Component<FieldViewProps>
         let sizes: number[] = [];
         let modifiedDates: number[] = [];
 
-        const uploads: FileResponse[] = [];
+        const localUpload = async (batch: File[]) => {
+            sizes.push(...batch.map(file => file.size));
+            modifiedDates.push(...batch.map(file => file.lastModified));
 
-        await validated.batch({
-            size: 15,
-            action: {
-                handler: async (batch: File[]) => {
-                    sizes.push(...batch.map(file => file.size));
-                    modifiedDates.push(...batch.map(file => file.lastModified));
+            let formData = new FormData();
+            batch.forEach(file => formData.append(Utils.GenerateGuid(), file));
+            const parameters = { method: 'POST', body: formData };
 
-                    let formData = new FormData();
-                    batch.forEach(file => formData.append(Utils.GenerateGuid(), file));
-                    const parameters = { method: 'POST', body: formData };
-
-                    uploads.push(...(await (await fetch(Utils.prepend(RouteStore.upload), parameters)).json()));
+            runInAction(() => this.completed += batch.length);
+            return (await fetch(Utils.prepend(RouteStore.upload), parameters)).json();
+        };
 
-                    runInAction(() => this.completed += batch.length);
-                }
-            }
-        });
+        const uploads = await validated.batchAction<FileResponse>(15, localUpload);
 
         await Promise.all(uploads.map(async upload => {
             const type = upload.type;
@@ -149,7 +143,12 @@ export default class DirectoryImportBox extends React.Component<FieldViewProps>
         };
         let parent = this.props.ContainingCollectionView;
         if (parent) {
-            let importContainer = Docs.Create.MasonryDocument(docs, options);
+            let importContainer: Doc;
+            if (docs.length < 50) {
+                importContainer = Docs.Create.MasonryDocument(docs, options);
+            } else {
+                importContainer = Docs.Create.SchemaDocument([], docs, options);
+            }
             await GooglePhotos.Export.CollectionToAlbum({ collection: importContainer });
             importContainer.singleColumn = false;
             Doc.AddDocToList(Doc.GetProto(parent.props.Document), "data", importContainer);
diff --git a/src/client/util/UtilExtensions.ts b/src/client/util/UtilExtensions.ts
index 1e277b242..0bf9f4e97 100644
--- a/src/client/util/UtilExtensions.ts
+++ b/src/client/util/UtilExtensions.ts
@@ -1,39 +1,41 @@
-module.exports.Batch = async function <T>(parameters: BatchParameters<T>) {
-    const { size, action } = parameters;
+module.exports.Batch = function <T>(batchSize: number): T[][] {
     const batches: T[][] = [];
     let i = 0;
     while (i < this.length) {
-        const cap = Math.min(i + size, this.length);
+        const cap = Math.min(i + batchSize, this.length);
         batches.push(this.slice(i, cap));
         i = cap;
     }
-    console.log(`Beginning action on ${this.length} elements, split into ${batches.length} groups => ${batches.map(batch => batch.length).join(", ")}`);
-    if (action) {
-        const { handler, interval } = action;
-        if (!interval || batches.length === 1) {
-            for (let batch of batches) {
-                await handler(batch);
-            }
-        } else {
-            return new Promise<T[][]>(resolve => {
-                const iterator = batches[Symbol.iterator]();
-                const quota = batches.length;
-                let completed = 0;
-                const tag = setInterval(async () => {
-                    const next = iterator.next();
-                    if (next.done) {
-                        clearInterval(tag);
-                        return;
-                    }
-                    const batch = next.value;
-                    console.log(`Handling next batch with ${batch.length} elements`);
-                    await handler(batch);
-                    if (++completed === quota) {
-                        resolve(batches);
-                    }
-                }, interval);
-            });
+    return batches;
+};
+
+module.exports.BatchAction = async function <I, O>(batchSize: number, handler: BatchHandler<I, O>, interval?: number): Promise<O[]> {
+    if (!this.length) {
+        return [];
+    }
+    let collector: O[] = [];
+    const batches = this.batch(batchSize);
+    if (!interval || batches.length === 1) {
+        for (let batch of batches) {
+            collector.push(...(await handler(batch)));
         }
+    } else {
+        return new Promise<O[]>(resolve => {
+            const iterator = batches[Symbol.iterator]();
+            let completed = 0;
+            const tag = setInterval(async () => {
+                const next = iterator.next();
+                if (next.done) {
+                    clearInterval(tag);
+                    return;
+                }
+                const batch = next.value;
+                collector.push(...(await handler(batch)));
+                if (++completed === batches.length) {
+                    resolve(collector);
+                }
+            }, interval);
+        });
     }
-    return batches;
+    return collector;
 };
\ No newline at end of file
diff --git a/src/server/apis/google/GooglePhotosUploadUtils.ts b/src/server/apis/google/GooglePhotosUploadUtils.ts
index 3989590c6..e640f2a85 100644
--- a/src/server/apis/google/GooglePhotosUploadUtils.ts
+++ b/src/server/apis/google/GooglePhotosUploadUtils.ts
@@ -40,7 +40,7 @@ export namespace GooglePhotosUploadUtils {
     };
 
     export const DispatchGooglePhotosUpload = async (url: string) => {
-        const body = await request(url, { encoding: null }).catch(error => console.log("Error in streaming body!", error));
+        const body = await request(url, { encoding: null });
         const parameters = {
             method: 'POST',
             headers: {
@@ -57,37 +57,30 @@ export namespace GooglePhotosUploadUtils {
                 return reject(error);
             }
             resolve(body);
-        }).catch(error => console.log("Error in literal uploading process to Google's servers!", error))).catch(error => console.log("Error in literal uploading process to Google's servers!", error));
+        }));
     };
 
     export const CreateMediaItems = async (newMediaItems: NewMediaItem[], album?: { id: string }): Promise<MediaItemCreationResult> => {
-        const newMediaItemResults: NewMediaItemResult[] = [];
-        await newMediaItems.batch({
-            size: 50,
-            action: {
-                handler: async (batch: NewMediaItem[]) => {
-                    console.log(batch.length);
-                    const parameters = {
-                        method: 'POST',
-                        headers: headers('json'),
-                        uri: prepend('mediaItems:batchCreate'),
-                        body: { newMediaItems: batch } as any,
-                        json: true
-                    };
-                    album && (parameters.body.albumId = album.id);
-                    newMediaItemResults.push(...(await new Promise<MediaItemCreationResult>((resolve, reject) => {
-                        request(parameters, (error, _response, body) => {
-                            if (error) {
-                                reject(error);
-                            } else {
-                                resolve(body);
-                            }
-                        });
-                    })).newMediaItemResults);
-                },
-                interval: 1000
-            }
-        });
+        const createFromUploadTokens = async (batch: NewMediaItem[]) => {
+            const parameters = {
+                method: 'POST',
+                headers: headers('json'),
+                uri: prepend('mediaItems:batchCreate'),
+                body: { newMediaItems: batch } as any,
+                json: true
+            };
+            album && (parameters.body.albumId = album.id);
+            return (await new Promise<MediaItemCreationResult>((resolve, reject) => {
+                request(parameters, (error, _response, body) => {
+                    if (error) {
+                        reject(error);
+                    } else {
+                        resolve(body);
+                    }
+                });
+            })).newMediaItemResults;
+        };
+        const newMediaItemResults = await newMediaItems.batchAction(50, createFromUploadTokens, 1000);
         return { newMediaItemResults };
     };
 
diff --git a/src/server/credentials/google_docs_token.json b/src/server/credentials/google_docs_token.json
index d8e0eae21..98d735acd 100644
--- a/src/server/credentials/google_docs_token.json
+++ b/src/server/credentials/google_docs_token.json
@@ -1 +1 @@
-{"access_token":"ya29.GlyDB0wsV3-oS6q5TFuJSmH1YP_SPf_X6RHaJVmfqj0NTCtaPLFonZRxdT52kUkiHJgAoRizxZvlSIGptXKfnmG4BFouhgyo9ZKP0QtOH-kPR9b9x5WhGCd5NWqz0A","refresh_token":"1/HTv_xFHszu2Nf3iiFrUTaeKzC_Vp2-6bpIB06xW_WHI","scope":"https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/documents.readonly https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/photoslibrary https://www.googleapis.com/auth/photoslibrary.appendonly https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/photoslibrary.sharing","token_type":"Bearer","expiry_date":1568412547334}
\ No newline at end of file
+{"access_token":"ya29.ImCDB7HE7-5O12GlhloCz2YWbDC5s8drlIs65oaPUVjAgL66RZMmIV8BptOs2X66ZWvQLCbRourz3ubcQooIuyzgpR8D1IVVm577RC5iyA2xB1Y1GNKZbHgpX3g8yGGmbS8","refresh_token":"1/HTv_xFHszu2Nf3iiFrUTaeKzC_Vp2-6bpIB06xW_WHI","scope":"https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/documents.readonly https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/photoslibrary https://www.googleapis.com/auth/photoslibrary.appendonly https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/photoslibrary.sharing","token_type":"Bearer","expiry_date":1568423265295}
\ No newline at end of file
diff --git a/src/server/index.ts b/src/server/index.ts
index 542a4ea65..79e9155d2 100644
--- a/src/server/index.ts
+++ b/src/server/index.ts
@@ -582,9 +582,7 @@ app.post(
                 const filename = path.basename(location);
                 await UploadUtils.UploadImage(uploadDirectory + filename, filename).catch(() => console.log(`Unable to process ${filename}`));
                 results.push({ name, type, path: `/files/${filename}` });
-                console.log(path.basename(name));
             }
-            console.log("All files traversed!");
             _success(res, results);
         });
     }
@@ -836,44 +834,36 @@ export interface NewMediaItem {
 }
 
 Array.prototype.batch = extensions.Batch;
+Array.prototype.batchAction = extensions.BatchAction;
 
 app.post(RouteStore.googlePhotosMediaUpload, async (req, res) => {
     const mediaInput: GooglePhotosUploadUtils.MediaInput[] = req.body.media;
     await GooglePhotosUploadUtils.initialize({ uploadDirectory, credentialsPath, tokenPath });
 
-    const newMediaItems: NewMediaItem[] = [];
     let failed = 0;
-    const size = 25;
-
-    try {
-        await mediaInput.batch({
-            size,
-            action: {
-                handler: async (batch: GooglePhotosUploadUtils.MediaInput[]) => {
-                    await Promise.all(batch.map(async element => {
-                        console.log(`Uploading ${element.url} to Google's servers...`);
-                        const uploadToken = await GooglePhotosUploadUtils.DispatchGooglePhotosUpload(element.url);
-                        if (uploadToken) {
-                            newMediaItems.push({
-                                description: element.description,
-                                simpleMediaItem: { uploadToken }
-                            });
-                        } else {
-                            console.log("FAIL!", element.url, element.description);
-                            failed++;
-                        }
-                    }));
-                },
-                interval: 3000
+
+    const dispatchUpload = async (batch: GooglePhotosUploadUtils.MediaInput[]) => {
+        const newMediaItems: NewMediaItem[] = [];
+        for (let element of batch) {
+            const uploadToken = await GooglePhotosUploadUtils.DispatchGooglePhotosUpload(element.url);
+            if (!uploadToken) {
+                failed++;
+            } else {
+                newMediaItems.push({
+                    description: element.description,
+                    simpleMediaItem: { uploadToken }
+                });
             }
-        });
-    } catch (e) {
-        console.log("WHAT HAPPENED?");
-        console.log(e);
-    }
+        }
+        return newMediaItems;
+    };
+
+    const newMediaItems = await mediaInput.batchAction<NewMediaItem>(25, dispatchUpload, 3000);
+
     if (failed) {
         return _error(res, tokenError);
     }
+
     GooglePhotosUploadUtils.CreateMediaItems(newMediaItems, req.body.album).then(
         result => _success(res, result.newMediaItemResults),
         error => _error(res, mediaError, error)