From 3f6f0a878b15aa5c569a8b522d1b24915749498e Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Tue, 17 Sep 2019 21:57:43 -0400 Subject: batching developments --- .../util/Import & Export/DirectoryImportBox.tsx | 2 +- src/client/util/UtilExtensions.ts | 80 +++++++++++++++++----- 2 files changed, 62 insertions(+), 20 deletions(-) (limited to 'src/client/util') diff --git a/src/client/util/Import & Export/DirectoryImportBox.tsx b/src/client/util/Import & Export/DirectoryImportBox.tsx index c9d34b594..a86ef9a31 100644 --- a/src/client/util/Import & Export/DirectoryImportBox.tsx +++ b/src/client/util/Import & Export/DirectoryImportBox.tsx @@ -103,7 +103,7 @@ export default class DirectoryImportBox extends React.Component runInAction(() => this.phase = `Internal: uploading ${this.quota - this.completed} files to Dash...`); - const uploads = await validated.batchedMapAsync(15, async (batch: File[]) => { + const uploads = await validated.batchedMapAsync(15, async batch => { const formData = new FormData(); const parameters = { method: 'POST', body: formData }; diff --git a/src/client/util/UtilExtensions.ts b/src/client/util/UtilExtensions.ts index 0b325bc51..792cd00a1 100644 --- a/src/client/util/UtilExtensions.ts +++ b/src/client/util/UtilExtensions.ts @@ -9,11 +9,19 @@ module.exports.Batch = function (batchSize: number): T[][] { return batches; }; -module.exports.ExecuteBatches = function (batchSize: number, handler: BatchHandlerSync): void { +module.exports.ExecuteBatches = function (batchSize: number, handler: BatchHandlerSync): void { if (this.length) { - for (let batch of this.batch(batchSize)) { - const isFullBatch = batch.length === batchSize; - handler(batch, isFullBatch); + let completed = 0; + const batches = this.batch(batchSize); + const quota = batches.length; + for (let batch of batches) { + const context: BatchContext = { + completedBatches: completed, + remainingBatches: quota - completed, + isFullBatch: batch.length === batchSize + }; + handler(batch, context); + completed++; } } }; @@ -23,18 +31,34 @@ module.exports.ConvertInBatches = function (batchSize: number, handler: Ba return []; } let collector: O[] = []; - for (let batch of this.batch(batchSize)) { - const isFullBatch = batch.length === batchSize; - collector.push(...handler(batch, isFullBatch)); + let completed = 0; + const batches = this.batch(batchSize); + const quota = batches.length; + for (let batch of batches) { + const context: BatchContext = { + completedBatches: completed, + remainingBatches: quota - completed, + isFullBatch: batch.length === batchSize + }; + collector.push(...handler(batch, context)); + completed++; } return collector; }; module.exports.ExecuteInBatchesAsync = async function (batchSize: number, handler: BatchHandler): Promise { if (this.length) { - for (let batch of this.batch(batchSize)) { - const isFullBatch = batch.length === batchSize; - await handler(batch, isFullBatch); + let completed = 0; + const batches = this.batch(batchSize); + const quota = batches.length; + for (let batch of batches) { + const context: BatchContext = { + completedBatches: completed, + remainingBatches: quota - completed, + isFullBatch: batch.length === batchSize + }; + await handler(batch, context); + completed++; } } }; @@ -44,9 +68,17 @@ module.exports.ConvertInBatchesAsync = async function (batchSize: number, return []; } let collector: O[] = []; - for (let batch of this.batch(batchSize)) { - const isFullBatch = batch.length === batchSize; - collector.push(...(await handler(batch, isFullBatch))); + let completed = 0; + const batches = this.batch(batchSize); + const quota = batches.length; + for (let batch of batches) { + const context: BatchContext = { + completedBatches: completed, + remainingBatches: quota - completed, + isFullBatch: batch.length === batchSize + }; + collector.push(...(await handler(batch, context))); + completed++; } return collector; }; @@ -56,6 +88,7 @@ module.exports.ExecuteInBatchesAtInterval = async function (batchSize: number return; } const batches = this.batch(batchSize); + const quota = batches.length; return new Promise(async resolve => { const iterator = batches[Symbol.iterator](); let completed = 0; @@ -64,12 +97,16 @@ module.exports.ExecuteInBatchesAtInterval = async function (batchSize: number await new Promise(resolve => { setTimeout(async () => { const batch = next.value; - const isFullBatch = batch.length === batchSize; - await handler(batch, isFullBatch); + const context: BatchContext = { + completedBatches: completed, + remainingBatches: quota - completed, + isFullBatch: batch.length === batchSize + }; + await handler(batch, context); resolve(); }, interval * 1000); }); - if (++completed === batches.length) { + if (++completed === quota) { break; } } @@ -83,6 +120,7 @@ module.exports.ConvertInBatchesAtInterval = async function (batchSize: num } let collector: O[] = []; const batches = this.batch(batchSize); + const quota = batches.length; return new Promise(async resolve => { const iterator = batches[Symbol.iterator](); let completed = 0; @@ -91,12 +129,16 @@ module.exports.ConvertInBatchesAtInterval = async function (batchSize: num await new Promise(resolve => { setTimeout(async () => { const batch = next.value; - const isFullBatch = batch.length === batchSize; - collector.push(...(await handler(batch, isFullBatch))); + const context: BatchContext = { + completedBatches: completed, + remainingBatches: quota - completed, + isFullBatch: batch.length === batchSize + }; + collector.push(...(await handler(batch, context))); resolve(); }, interval * 1000); }); - if (++completed === batches.length) { + if (++completed === quota) { resolve(collector); break; } -- cgit v1.2.3-70-g09d2