From b013be665b704cb6241979f1bf53403a4b6bc1a3 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Tue, 5 Nov 2019 16:56:04 -0500 Subject: fixes for batched array --- src/client/util/Import & Export/DirectoryImportBox.tsx | 6 +++--- src/server/apis/google/GooglePhotosUploadUtils.ts | 17 +++++++++-------- src/server/index.ts | 15 +++++++-------- 3 files changed, 19 insertions(+), 19 deletions(-) (limited to 'src') diff --git a/src/client/util/Import & Export/DirectoryImportBox.tsx b/src/client/util/Import & Export/DirectoryImportBox.tsx index f27d05487..5904088fc 100644 --- a/src/client/util/Import & Export/DirectoryImportBox.tsx +++ b/src/client/util/Import & Export/DirectoryImportBox.tsx @@ -108,7 +108,8 @@ export default class DirectoryImportBox extends React.Component runInAction(() => this.phase = `Internal: uploading ${this.quota - this.completed} files to Dash...`); - const uploads = await BatchedArray.from(validated, { batchSize: 15 }).batchedMapAsync(async batch => { + const batched = BatchedArray.from(validated, { batchSize: 15 }); + const uploads = await batched.batchedMapAsync(async (batch, collector) => { const formData = new FormData(); batch.forEach(file => { @@ -117,9 +118,8 @@ export default class DirectoryImportBox extends React.Component formData.append(Utils.GenerateGuid(), file); }); - const responses = await Identified.PostFormDataToServer(RouteStore.upload, formData); + collector.push(...(await Identified.PostFormDataToServer(RouteStore.upload, formData))); runInAction(() => this.completed += batch.length); - return responses as ImageUploadResponse[]; }); await Promise.all(uploads.map(async upload => { diff --git a/src/server/apis/google/GooglePhotosUploadUtils.ts b/src/server/apis/google/GooglePhotosUploadUtils.ts index 4a67e57cc..36256822c 100644 --- a/src/server/apis/google/GooglePhotosUploadUtils.ts +++ b/src/server/apis/google/GooglePhotosUploadUtils.ts @@ -1,7 +1,8 @@ + import request = require('request-promise'); import { GoogleApiServerUtils } from './GoogleApiServerUtils'; import * as path from 'path'; -import { MediaItemCreationResult } from './SharedTypes'; +import { MediaItemCreationResult, NewMediaItemResult } from './SharedTypes'; import { NewMediaItem } from "../../index"; import { BatchedArray, TimeUnit } from 'array-batcher'; import { DashUploadUtils } from '../../DashUploadUtils'; @@ -56,10 +57,11 @@ export namespace GooglePhotosUploadUtils { })); }; - export const CreateMediaItems = async (newMediaItems: NewMediaItem[], album?: { id: string }): Promise => { - const newMediaItemResults = await BatchedArray.from(newMediaItems, { batchSize: 50 }).batchedMapPatientInterval( + export const CreateMediaItems = async (newMediaItems: NewMediaItem[], album?: { id: string }): Promise => { + const batched = BatchedArray.from(newMediaItems, { batchSize: 50 }); + return batched.batchedMapPatientInterval( { magnitude: 100, unit: TimeUnit.Milliseconds }, - async (batch: NewMediaItem[]) => { + async (batch, collector) => { const parameters = { method: 'POST', headers: headers('json'), @@ -68,18 +70,17 @@ export namespace GooglePhotosUploadUtils { json: true }; album && (parameters.body.albumId = album.id); - return (await new Promise((resolve, reject) => { + collector.push(...(await new Promise((resolve, reject) => { request(parameters, (error, _response, body) => { if (error) { reject(error); } else { - resolve(body); + resolve(body.newMediaItemResults); } }); - })).newMediaItemResults; + }))); } ); - return { newMediaItemResults }; }; } \ No newline at end of file diff --git a/src/server/index.ts b/src/server/index.ts index ee6a497ba..89b5085b1 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -1049,23 +1049,22 @@ addSecureRoute({ let failed: number[] = []; - const newMediaItems = await BatchedArray.from(media, { batchSize: 25 }).batchedMapPatientInterval( + const batched = BatchedArray.from(media, { batchSize: 25 }); + const newMediaItems = await batched.batchedMapPatientInterval( { magnitude: 100, unit: TimeUnit.Milliseconds }, - async (batch: GooglePhotosUploadUtils.MediaInput[]) => { - const newMediaItems: NewMediaItem[] = []; + async (batch, collector) => { for (let index = 0; index < batch.length; index++) { - const element = batch[index]; - const uploadToken = await GooglePhotosUploadUtils.DispatchGooglePhotosUpload(element.url); + const { url, description } = batch[index]; + const uploadToken = await GooglePhotosUploadUtils.DispatchGooglePhotosUpload(url); if (!uploadToken) { failed.push(index); } else { - newMediaItems.push({ - description: element.description, + collector.push({ + description, simpleMediaItem: { uploadToken } }); } } - return newMediaItems; } ); -- cgit v1.2.3-70-g09d2 From bcc45bf8d448e56935172398a52327c68ff6117e Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Tue, 5 Nov 2019 17:00:12 -0500 Subject: final tweak --- src/server/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/server/index.ts b/src/server/index.ts index 89b5085b1..1595781dc 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -1074,7 +1074,7 @@ addSecureRoute({ } GooglePhotosUploadUtils.CreateMediaItems(newMediaItems, req.body.album).then( - result => _success(res, { results: result.newMediaItemResults, failed }), + results => _success(res, { results, failed }), error => _error(res, mediaError, error) ); } -- cgit v1.2.3-70-g09d2