From f529d7d22162689c08830418da67a89778111e16 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Sat, 21 Sep 2019 14:27:10 -0400 Subject: final batcher fixes --- .../util/Import & Export/DirectoryImportBox.tsx | 29 ++++++++++------------ src/server/apis/google/GooglePhotosUploadUtils.ts | 11 ++++---- src/server/credentials/google_docs_token.json | 2 +- src/server/index.ts | 11 ++++---- 4 files changed, 24 insertions(+), 29 deletions(-) (limited to 'src') diff --git a/src/client/util/Import & Export/DirectoryImportBox.tsx b/src/client/util/Import & Export/DirectoryImportBox.tsx index 26a00dc7c..8948b73f7 100644 --- a/src/client/util/Import & Export/DirectoryImportBox.tsx +++ b/src/client/util/Import & Export/DirectoryImportBox.tsx @@ -104,22 +104,19 @@ export default class DirectoryImportBox extends React.Component runInAction(() => this.phase = `Internal: uploading ${this.quota - this.completed} files to Dash...`); - const uploads = await BatchedArray.from(validated).batchedMapAsync({ - batcher: { batchSize: 15 }, - converter: async batch => { - const formData = new FormData(); - const parameters = { method: 'POST', body: formData }; - - batch.forEach(file => { - sizes.push(file.size); - modifiedDates.push(file.lastModified); - formData.append(Utils.GenerateGuid(), file); - }); - - const responses = await (await fetch(RouteStore.upload, parameters)).json(); - runInAction(() => this.completed += batch.length); - return responses as FileResponse[]; - } + const uploads = await BatchedArray.from(validated, { batchSize: 15 }).batchedMapAsync(async batch => { + const formData = new FormData(); + const parameters = { method: 'POST', body: formData }; + + batch.forEach(file => { + sizes.push(file.size); + modifiedDates.push(file.lastModified); + formData.append(Utils.GenerateGuid(), file); + }); + + const responses = await (await fetch(RouteStore.upload, parameters)).json(); + runInAction(() => this.completed += batch.length); + return responses as FileResponse[]; }); await Promise.all(uploads.map(async upload => { diff --git a/src/server/apis/google/GooglePhotosUploadUtils.ts b/src/server/apis/google/GooglePhotosUploadUtils.ts index 40564ff3b..4dc252577 100644 --- a/src/server/apis/google/GooglePhotosUploadUtils.ts +++ b/src/server/apis/google/GooglePhotosUploadUtils.ts @@ -62,10 +62,8 @@ export namespace GooglePhotosUploadUtils { }; export const CreateMediaItems = async (newMediaItems: NewMediaItem[], album?: { id: string }): Promise => { - const newMediaItemResults = await BatchedArray.from(newMediaItems).batchedMapInterval({ - batcher: { batchSize: 50 }, - interval: { magnitude: 100, unit: TimeUnit.Milliseconds }, - converter: async (batch: NewMediaItem[]) => { + const newMediaItemResults = await BatchedArray.from(newMediaItems, { batchSize: 50 }).batchedMapInterval( + async (batch: NewMediaItem[]) => { const parameters = { method: 'POST', headers: headers('json'), @@ -83,8 +81,9 @@ export namespace GooglePhotosUploadUtils { } }); })).newMediaItemResults; - } - }); + }, + { magnitude: 100, unit: TimeUnit.Milliseconds } + ); return { newMediaItemResults }; }; diff --git a/src/server/credentials/google_docs_token.json b/src/server/credentials/google_docs_token.json index 9b0a649ac..ee44c3f30 100644 --- a/src/server/credentials/google_docs_token.json +++ b/src/server/credentials/google_docs_token.json @@ -1 +1 @@ -{"access_token":"ya29.ImCJB3_-mtKbCG8L7nQt3CDfn4HR2_xqUw8bUQkvaZZgxJZ8-WUlHe6UaSOmGtrvI4QabpYRGutYGSTUdc9bvIYooerZgAz80uk7-KEbnAxpwydRE-TK_1_VyWYZ2YW6Ht8","refresh_token":"1/HTv_xFHszu2Nf3iiFrUTaeKzC_Vp2-6bpIB06xW_WHI","scope":"https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/documents.readonly https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/photoslibrary https://www.googleapis.com/auth/photoslibrary.appendonly https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/photoslibrary.sharing","token_type":"Bearer","expiry_date":1569013995678} \ No newline at end of file +{"access_token":"ya29.GlyKBznz91v_qb8RYt4PT40Hp106N08Yk64UjMAKllBsIqJQEzBkxLbB5q5paydywHzguQYSNup5fT7ojJTDU4CMZdPbPKGcjQz17w_CospcG-8Buz94KZptvlQ_pQ","refresh_token":"1/HTv_xFHszu2Nf3iiFrUTaeKzC_Vp2-6bpIB06xW_WHI","scope":"https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/documents.readonly https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/photoslibrary https://www.googleapis.com/auth/photoslibrary.appendonly https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/photoslibrary.sharing","token_type":"Bearer","expiry_date":1569093749804} \ No newline at end of file diff --git a/src/server/index.ts b/src/server/index.ts index 12ceb9886..4c4cb84d6 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -841,10 +841,8 @@ app.post(RouteStore.googlePhotosMediaUpload, async (req, res) => { let failed = 0; - const newMediaItems = await BatchedArray.from(mediaInput).batchedMapInterval({ - batcher: { batchSize: 25 }, - interval: { magnitude: 100, unit: TimeUnit.Milliseconds }, - converter: async (batch: GooglePhotosUploadUtils.MediaInput[]) => { + const newMediaItems = await BatchedArray.from(mediaInput, { batchSize: 25 }).batchedMapInterval( + async (batch: GooglePhotosUploadUtils.MediaInput[]) => { const newMediaItems: NewMediaItem[] = []; for (let element of batch) { const uploadToken = await GooglePhotosUploadUtils.DispatchGooglePhotosUpload(element.url); @@ -858,8 +856,9 @@ app.post(RouteStore.googlePhotosMediaUpload, async (req, res) => { } } return newMediaItems; - } - }); + }, + { magnitude: 100, unit: TimeUnit.Milliseconds } + ); if (failed) { return _error(res, tokenError); -- cgit v1.2.3-70-g09d2