fixed batching and refactor

3 files changed, 43 insertions, 60 deletions

diff --git a/src/server/apis/google/GooglePhotosUploadUtils.ts b/src/server/apis/google/GooglePhotosUploadUtils.ts
index 3989590c6..e640f2a85 100644
--- a/src/server/apis/google/GooglePhotosUploadUtils.ts
+++ b/src/server/apis/google/GooglePhotosUploadUtils.ts
@@ -40,7 +40,7 @@ export namespace GooglePhotosUploadUtils {
     };
 
     export const DispatchGooglePhotosUpload = async (url: string) => {
-        const body = await request(url, { encoding: null }).catch(error => console.log("Error in streaming body!", error));
+        const body = await request(url, { encoding: null });
         const parameters = {
             method: 'POST',
             headers: {
@@ -57,37 +57,30 @@ export namespace GooglePhotosUploadUtils {
                 return reject(error);
             }
             resolve(body);
-        }).catch(error => console.log("Error in literal uploading process to Google's servers!", error))).catch(error => console.log("Error in literal uploading process to Google's servers!", error));
+        }));
     };
 
     export const CreateMediaItems = async (newMediaItems: NewMediaItem[], album?: { id: string }): Promise<MediaItemCreationResult> => {
-        const newMediaItemResults: NewMediaItemResult[] = [];
-        await newMediaItems.batch({
-            size: 50,
-            action: {
-                handler: async (batch: NewMediaItem[]) => {
-                    console.log(batch.length);
-                    const parameters = {
-                        method: 'POST',
-                        headers: headers('json'),
-                        uri: prepend('mediaItems:batchCreate'),
-                        body: { newMediaItems: batch } as any,
-                        json: true
-                    };
-                    album && (parameters.body.albumId = album.id);
-                    newMediaItemResults.push(...(await new Promise<MediaItemCreationResult>((resolve, reject) => {
-                        request(parameters, (error, _response, body) => {
-                            if (error) {
-                                reject(error);
-                            } else {
-                                resolve(body);
-                            }
-                        });
-                    })).newMediaItemResults);
-                },
-                interval: 1000
-            }
-        });
+        const createFromUploadTokens = async (batch: NewMediaItem[]) => {
+            const parameters = {
+                method: 'POST',
+                headers: headers('json'),
+                uri: prepend('mediaItems:batchCreate'),
+                body: { newMediaItems: batch } as any,
+                json: true
+            };
+            album && (parameters.body.albumId = album.id);
+            return (await new Promise<MediaItemCreationResult>((resolve, reject) => {
+                request(parameters, (error, _response, body) => {
+                    if (error) {
+                        reject(error);
+                    } else {
+                        resolve(body);
+                    }
+                });
+            })).newMediaItemResults;
+        };
+        const newMediaItemResults = await newMediaItems.batchAction(50, createFromUploadTokens, 1000);
         return { newMediaItemResults };
     };
 
diff --git a/src/server/credentials/google_docs_token.json b/src/server/credentials/google_docs_token.json
index d8e0eae21..98d735acd 100644
--- a/src/server/credentials/google_docs_token.json
+++ b/src/server/credentials/google_docs_token.json
@@ -1 +1 @@
-{"access_token":"ya29.GlyDB0wsV3-oS6q5TFuJSmH1YP_SPf_X6RHaJVmfqj0NTCtaPLFonZRxdT52kUkiHJgAoRizxZvlSIGptXKfnmG4BFouhgyo9ZKP0QtOH-kPR9b9x5WhGCd5NWqz0A","refresh_token":"1/HTv_xFHszu2Nf3iiFrUTaeKzC_Vp2-6bpIB06xW_WHI","scope":"https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/documents.readonly https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/photoslibrary https://www.googleapis.com/auth/photoslibrary.appendonly https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/photoslibrary.sharing","token_type":"Bearer","expiry_date":1568412547334}
\ No newline at end of file
+{"access_token":"ya29.ImCDB7HE7-5O12GlhloCz2YWbDC5s8drlIs65oaPUVjAgL66RZMmIV8BptOs2X66ZWvQLCbRourz3ubcQooIuyzgpR8D1IVVm577RC5iyA2xB1Y1GNKZbHgpX3g8yGGmbS8","refresh_token":"1/HTv_xFHszu2Nf3iiFrUTaeKzC_Vp2-6bpIB06xW_WHI","scope":"https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/documents.readonly https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/photoslibrary https://www.googleapis.com/auth/photoslibrary.appendonly https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/photoslibrary.sharing","token_type":"Bearer","expiry_date":1568423265295}
\ No newline at end of file
diff --git a/src/server/index.ts b/src/server/index.ts
index 542a4ea65..79e9155d2 100644
--- a/src/server/index.ts
+++ b/src/server/index.ts
@@ -582,9 +582,7 @@ app.post(
                 const filename = path.basename(location);
                 await UploadUtils.UploadImage(uploadDirectory + filename, filename).catch(() => console.log(`Unable to process ${filename}`));
                 results.push({ name, type, path: `/files/${filename}` });
-                console.log(path.basename(name));
             }
-            console.log("All files traversed!");
             _success(res, results);
         });
     }
@@ -836,44 +834,36 @@ export interface NewMediaItem {
 }
 
 Array.prototype.batch = extensions.Batch;
+Array.prototype.batchAction = extensions.BatchAction;
 
 app.post(RouteStore.googlePhotosMediaUpload, async (req, res) => {
     const mediaInput: GooglePhotosUploadUtils.MediaInput[] = req.body.media;
     await GooglePhotosUploadUtils.initialize({ uploadDirectory, credentialsPath, tokenPath });
 
-    const newMediaItems: NewMediaItem[] = [];
     let failed = 0;
-    const size = 25;
-
-    try {
-        await mediaInput.batch({
-            size,
-            action: {
-                handler: async (batch: GooglePhotosUploadUtils.MediaInput[]) => {
-                    await Promise.all(batch.map(async element => {
-                        console.log(`Uploading ${element.url} to Google's servers...`);
-                        const uploadToken = await GooglePhotosUploadUtils.DispatchGooglePhotosUpload(element.url);
-                        if (uploadToken) {
-                            newMediaItems.push({
-                                description: element.description,
-                                simpleMediaItem: { uploadToken }
-                            });
-                        } else {
-                            console.log("FAIL!", element.url, element.description);
-                            failed++;
-                        }
-                    }));
-                },
-                interval: 3000
+
+    const dispatchUpload = async (batch: GooglePhotosUploadUtils.MediaInput[]) => {
+        const newMediaItems: NewMediaItem[] = [];
+        for (let element of batch) {
+            const uploadToken = await GooglePhotosUploadUtils.DispatchGooglePhotosUpload(element.url);
+            if (!uploadToken) {
+                failed++;
+            } else {
+                newMediaItems.push({
+                    description: element.description,
+                    simpleMediaItem: { uploadToken }
+                });
             }
-        });
-    } catch (e) {
-        console.log("WHAT HAPPENED?");
-        console.log(e);
-    }
+        }
+        return newMediaItems;
+    };
+
+    const newMediaItems = await mediaInput.batchAction<NewMediaItem>(25, dispatchUpload, 3000);
+
     if (failed) {
         return _error(res, tokenError);
     }
+
     GooglePhotosUploadUtils.CreateMediaItems(newMediaItems, req.body.album).then(
         result => _success(res, result.newMediaItemResults),
         error => _error(res, mediaError, error)