diff options
-rw-r--r-- | src/client/northstar/utils/Extensions.ts | 22 | ||||
-rw-r--r-- | src/client/util/Import & Export/DirectoryImportBox.tsx | 4 | ||||
-rw-r--r-- | src/client/util/UtilExtensions.ts | 92 | ||||
-rw-r--r-- | src/server/apis/google/GooglePhotosUploadUtils.ts | 2 | ||||
-rw-r--r-- | src/server/index.ts | 9 |
5 files changed, 100 insertions, 29 deletions
diff --git a/src/client/northstar/utils/Extensions.ts b/src/client/northstar/utils/Extensions.ts index c866d1bc3..00c1e113c 100644 --- a/src/client/northstar/utils/Extensions.ts +++ b/src/client/northstar/utils/Extensions.ts @@ -20,17 +20,31 @@ String.prototype.Truncate = function (length: number, replacement: string): Stri return target; }; - -type BatchHandler<I, O> = (batch: I[]) => O[] | Promise<O[]>; +type BatchConverterSync<I, O> = (batch: I[]) => O[]; +type BatchHandlerSync<I> = (batch: I[]) => void; +type BatchConverterAsync<I, O> = (batch: I[]) => Promise<O[]>; +type BatchHandlerAsync<I> = (batch: I[]) => Promise<void>; +type BatchConverter<I, O> = BatchConverterSync<I, O> | BatchConverterAsync<I, O>; +type BatchHandler<I> = BatchHandlerSync<I> | BatchHandlerAsync<I>; interface Array<T> { batch(batchSize: number): T[][]; - batchAction<O>(batchSize: number, handler: BatchHandler<T, O>, interval?: number): Promise<O[]>; + executeInBatches(batchSize: number, handler: BatchHandlerSync<T>): void; + convertInBatches<O>(batchSize: number, handler: BatchConverterSync<T, O>): O[]; + executeInBatchesAsync(batchSize: number, handler: BatchHandler<T>): Promise<void>; + convertInBatchesAsync<O>(batchSize: number, handler: BatchConverter<T, O>): Promise<O[]>; + executeInBatchesAtInterval(batchSize: number, handler: BatchHandler<T>, interval: number): Promise<void>; + convertInBatchesAtInterval<O>(batchSize: number, handler: BatchConverter<T, O>, interval: number): Promise<O[]>; lastElement(): T; } Array.prototype.batch = extensions.Batch; -Array.prototype.batchAction = extensions.BatchAction; +Array.prototype.executeInBatches = extensions.ExecuteInBatches; +Array.prototype.convertInBatches = extensions.ConvertInBatches; +Array.prototype.executeInBatchesAsync = extensions.ExecuteInBatchesAsync; +Array.prototype.convertInBatchesAsync = extensions.ConvertInBatchesAsync; +Array.prototype.executeInBatchesAtInterval = extensions.ExecuteInBatchesAtInterval; +Array.prototype.convertInBatchesAtInterval = extensions.ConvertInBatchesAtInterval; Array.prototype.lastElement = function <T>() { if (!this.length) { diff --git a/src/client/util/Import & Export/DirectoryImportBox.tsx b/src/client/util/Import & Export/DirectoryImportBox.tsx index 1ae0e7525..a625e06c0 100644 --- a/src/client/util/Import & Export/DirectoryImportBox.tsx +++ b/src/client/util/Import & Export/DirectoryImportBox.tsx @@ -95,7 +95,7 @@ export default class DirectoryImportBox extends React.Component<FieldViewProps> let sizes: number[] = []; let modifiedDates: number[] = []; - const localUpload = async (batch: File[]) => { + const uploadLocally = async (batch: File[]) => { sizes.push(...batch.map(file => file.size)); modifiedDates.push(...batch.map(file => file.lastModified)); @@ -107,7 +107,7 @@ export default class DirectoryImportBox extends React.Component<FieldViewProps> return (await fetch(Utils.prepend(RouteStore.upload), parameters)).json(); }; - const uploads = await validated.batchAction<FileResponse>(15, localUpload); + const uploads = await validated.convertInBatchesAsync<FileResponse>(15, uploadLocally); await Promise.all(uploads.map(async upload => { const type = upload.type; diff --git a/src/client/util/UtilExtensions.ts b/src/client/util/UtilExtensions.ts index 0bf9f4e97..3eeec6ca7 100644 --- a/src/client/util/UtilExtensions.ts +++ b/src/client/util/UtilExtensions.ts @@ -9,33 +9,85 @@ module.exports.Batch = function <T>(batchSize: number): T[][] { return batches; }; -module.exports.BatchAction = async function <I, O>(batchSize: number, handler: BatchHandler<I, O>, interval?: number): Promise<O[]> { +module.exports.ExecuteBatches = function <I, O>(batchSize: number, handler: BatchHandlerSync<I>): void { + if (this.length) { + for (let batch of this.batch(batchSize)) { + handler(batch); + } + } +}; + +module.exports.ConvertInBatches = function <I, O>(batchSize: number, handler: BatchConverterSync<I, O>): O[] { if (!this.length) { return []; } let collector: O[] = []; - const batches = this.batch(batchSize); - if (!interval || batches.length === 1) { - for (let batch of batches) { - collector.push(...(await handler(batch))); + for (let batch of this.batch(batchSize)) { + collector.push(...handler(batch)); + } + return collector; +}; + +module.exports.ExecuteInBatchesAsync = async function <I>(batchSize: number, handler: BatchHandler<I>): Promise<void> { + if (this.length) { + for (let batch of this.batch(batchSize)) { + await handler(batch); } - } else { - return new Promise<O[]>(resolve => { - const iterator = batches[Symbol.iterator](); - let completed = 0; - const tag = setInterval(async () => { - const next = iterator.next(); - if (next.done) { - clearInterval(tag); - return; + } +}; + +module.exports.ConvertInBatchesAsync = async function <I, O>(batchSize: number, handler: BatchConverter<I, O>): Promise<O[]> { + if (!this.length) { + return []; + } + let collector: O[] = []; + for (let batch of this.batch(batchSize)) { + collector.push(...(await handler(batch))); + } + return collector; +}; + +module.exports.ExecuteInBatchesAtInterval = async function <I>(batchSize: number, handler: BatchHandler<I>, interval: number): Promise<void> { + if (!this.length) { + return; + } + const batches = this.batch(batchSize); + return new Promise<void>(resolve => { + const iterator = batches[Symbol.iterator](); + let completed = 0; + const tag = setInterval(async () => { + const next = iterator.next(); + if (next.done) { + clearInterval(tag); + } else { + await handler(next.value); + if (++completed === batches.length) { + resolve(); } - const batch = next.value; - collector.push(...(await handler(batch))); + } + }, interval * 1000); + }); +}; + +module.exports.ConvertInBatchesAtInterval = async function <I, O>(batchSize: number, handler: BatchConverter<I, O>, interval: number): Promise<O[]> { + if (!this.length) { + return []; + } + let collector: O[] = []; + const batches = this.batch(batchSize); + return new Promise<O[]>(resolve => { + const iterator = batches[Symbol.iterator](); + let completed = 0; + const tag = setInterval(async () => { + const next = iterator.next(); + if (next.done) { + clearInterval(tag); + } else { + collector.push(...(await handler(next.value))); if (++completed === batches.length) { resolve(collector); } - }, interval); - }); - } - return collector; + } + }, interval * 1000); + }); };
\ No newline at end of file diff --git a/src/server/apis/google/GooglePhotosUploadUtils.ts b/src/server/apis/google/GooglePhotosUploadUtils.ts index e640f2a85..e1478a097 100644 --- a/src/server/apis/google/GooglePhotosUploadUtils.ts +++ b/src/server/apis/google/GooglePhotosUploadUtils.ts @@ -80,7 +80,7 @@ export namespace GooglePhotosUploadUtils { }); })).newMediaItemResults; }; - const newMediaItemResults = await newMediaItems.batchAction(50, createFromUploadTokens, 1000); + const newMediaItemResults = await newMediaItems.convertInBatchesAtInterval(50, createFromUploadTokens, 1); return { newMediaItemResults }; }; diff --git a/src/server/index.ts b/src/server/index.ts index 79e9155d2..8767be17d 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -834,7 +834,12 @@ export interface NewMediaItem { } Array.prototype.batch = extensions.Batch; -Array.prototype.batchAction = extensions.BatchAction; +Array.prototype.executeInBatches = extensions.ExecuteInBatches; +Array.prototype.convertInBatches = extensions.ConvertInBatches; +Array.prototype.executeInBatchesAsync = extensions.ExecuteInBatchesAsync; +Array.prototype.convertInBatchesAsync = extensions.ConvertInBatchesAsync; +Array.prototype.executeInBatchesAtInterval = extensions.ExecuteInBatchesAtInterval; +Array.prototype.convertInBatchesAtInterval = extensions.ConvertInBatchesAtInterval; app.post(RouteStore.googlePhotosMediaUpload, async (req, res) => { const mediaInput: GooglePhotosUploadUtils.MediaInput[] = req.body.media; @@ -858,7 +863,7 @@ app.post(RouteStore.googlePhotosMediaUpload, async (req, res) => { return newMediaItems; }; - const newMediaItems = await mediaInput.batchAction<NewMediaItem>(25, dispatchUpload, 3000); + const newMediaItems = await mediaInput.convertInBatchesAtInterval<NewMediaItem>(25, dispatchUpload, 3); if (failed) { return _error(res, tokenError); |