aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--src/client/northstar/utils/Extensions.ts22
-rw-r--r--src/client/util/Import & Export/DirectoryImportBox.tsx4
-rw-r--r--src/client/util/UtilExtensions.ts92
-rw-r--r--src/server/apis/google/GooglePhotosUploadUtils.ts2
-rw-r--r--src/server/index.ts9
5 files changed, 100 insertions, 29 deletions
diff --git a/src/client/northstar/utils/Extensions.ts b/src/client/northstar/utils/Extensions.ts
index c866d1bc3..00c1e113c 100644
--- a/src/client/northstar/utils/Extensions.ts
+++ b/src/client/northstar/utils/Extensions.ts
@@ -20,17 +20,31 @@ String.prototype.Truncate = function (length: number, replacement: string): Stri
return target;
};
-
-type BatchHandler<I, O> = (batch: I[]) => O[] | Promise<O[]>;
+type BatchConverterSync<I, O> = (batch: I[]) => O[];
+type BatchHandlerSync<I> = (batch: I[]) => void;
+type BatchConverterAsync<I, O> = (batch: I[]) => Promise<O[]>;
+type BatchHandlerAsync<I> = (batch: I[]) => Promise<void>;
+type BatchConverter<I, O> = BatchConverterSync<I, O> | BatchConverterAsync<I, O>;
+type BatchHandler<I> = BatchHandlerSync<I> | BatchHandlerAsync<I>;
interface Array<T> {
batch(batchSize: number): T[][];
- batchAction<O>(batchSize: number, handler: BatchHandler<T, O>, interval?: number): Promise<O[]>;
+ executeInBatches(batchSize: number, handler: BatchHandlerSync<T>): void;
+ convertInBatches<O>(batchSize: number, handler: BatchConverterSync<T, O>): O[];
+ executeInBatchesAsync(batchSize: number, handler: BatchHandler<T>): Promise<void>;
+ convertInBatchesAsync<O>(batchSize: number, handler: BatchConverter<T, O>): Promise<O[]>;
+ executeInBatchesAtInterval(batchSize: number, handler: BatchHandler<T>, interval: number): Promise<void>;
+ convertInBatchesAtInterval<O>(batchSize: number, handler: BatchConverter<T, O>, interval: number): Promise<O[]>;
lastElement(): T;
}
Array.prototype.batch = extensions.Batch;
-Array.prototype.batchAction = extensions.BatchAction;
+Array.prototype.executeInBatches = extensions.ExecuteInBatches;
+Array.prototype.convertInBatches = extensions.ConvertInBatches;
+Array.prototype.executeInBatchesAsync = extensions.ExecuteInBatchesAsync;
+Array.prototype.convertInBatchesAsync = extensions.ConvertInBatchesAsync;
+Array.prototype.executeInBatchesAtInterval = extensions.ExecuteInBatchesAtInterval;
+Array.prototype.convertInBatchesAtInterval = extensions.ConvertInBatchesAtInterval;
Array.prototype.lastElement = function <T>() {
if (!this.length) {
diff --git a/src/client/util/Import & Export/DirectoryImportBox.tsx b/src/client/util/Import & Export/DirectoryImportBox.tsx
index 1ae0e7525..a625e06c0 100644
--- a/src/client/util/Import & Export/DirectoryImportBox.tsx
+++ b/src/client/util/Import & Export/DirectoryImportBox.tsx
@@ -95,7 +95,7 @@ export default class DirectoryImportBox extends React.Component<FieldViewProps>
let sizes: number[] = [];
let modifiedDates: number[] = [];
- const localUpload = async (batch: File[]) => {
+ const uploadLocally = async (batch: File[]) => {
sizes.push(...batch.map(file => file.size));
modifiedDates.push(...batch.map(file => file.lastModified));
@@ -107,7 +107,7 @@ export default class DirectoryImportBox extends React.Component<FieldViewProps>
return (await fetch(Utils.prepend(RouteStore.upload), parameters)).json();
};
- const uploads = await validated.batchAction<FileResponse>(15, localUpload);
+ const uploads = await validated.convertInBatchesAsync<FileResponse>(15, uploadLocally);
await Promise.all(uploads.map(async upload => {
const type = upload.type;
diff --git a/src/client/util/UtilExtensions.ts b/src/client/util/UtilExtensions.ts
index 0bf9f4e97..3eeec6ca7 100644
--- a/src/client/util/UtilExtensions.ts
+++ b/src/client/util/UtilExtensions.ts
@@ -9,33 +9,85 @@ module.exports.Batch = function <T>(batchSize: number): T[][] {
return batches;
};
-module.exports.BatchAction = async function <I, O>(batchSize: number, handler: BatchHandler<I, O>, interval?: number): Promise<O[]> {
+module.exports.ExecuteBatches = function <I, O>(batchSize: number, handler: BatchHandlerSync<I>): void {
+ if (this.length) {
+ for (let batch of this.batch(batchSize)) {
+ handler(batch);
+ }
+ }
+};
+
+module.exports.ConvertInBatches = function <I, O>(batchSize: number, handler: BatchConverterSync<I, O>): O[] {
if (!this.length) {
return [];
}
let collector: O[] = [];
- const batches = this.batch(batchSize);
- if (!interval || batches.length === 1) {
- for (let batch of batches) {
- collector.push(...(await handler(batch)));
+ for (let batch of this.batch(batchSize)) {
+ collector.push(...handler(batch));
+ }
+ return collector;
+};
+
+module.exports.ExecuteInBatchesAsync = async function <I>(batchSize: number, handler: BatchHandler<I>): Promise<void> {
+ if (this.length) {
+ for (let batch of this.batch(batchSize)) {
+ await handler(batch);
}
- } else {
- return new Promise<O[]>(resolve => {
- const iterator = batches[Symbol.iterator]();
- let completed = 0;
- const tag = setInterval(async () => {
- const next = iterator.next();
- if (next.done) {
- clearInterval(tag);
- return;
+ }
+};
+
+module.exports.ConvertInBatchesAsync = async function <I, O>(batchSize: number, handler: BatchConverter<I, O>): Promise<O[]> {
+ if (!this.length) {
+ return [];
+ }
+ let collector: O[] = [];
+ for (let batch of this.batch(batchSize)) {
+ collector.push(...(await handler(batch)));
+ }
+ return collector;
+};
+
+module.exports.ExecuteInBatchesAtInterval = async function <I>(batchSize: number, handler: BatchHandler<I>, interval: number): Promise<void> {
+ if (!this.length) {
+ return;
+ }
+ const batches = this.batch(batchSize);
+ return new Promise<void>(resolve => {
+ const iterator = batches[Symbol.iterator]();
+ let completed = 0;
+ const tag = setInterval(async () => {
+ const next = iterator.next();
+ if (next.done) {
+ clearInterval(tag);
+ } else {
+ await handler(next.value);
+ if (++completed === batches.length) {
+ resolve();
}
- const batch = next.value;
- collector.push(...(await handler(batch)));
+ }
+ }, interval * 1000);
+ });
+};
+
+module.exports.ConvertInBatchesAtInterval = async function <I, O>(batchSize: number, handler: BatchConverter<I, O>, interval: number): Promise<O[]> {
+ if (!this.length) {
+ return [];
+ }
+ let collector: O[] = [];
+ const batches = this.batch(batchSize);
+ return new Promise<O[]>(resolve => {
+ const iterator = batches[Symbol.iterator]();
+ let completed = 0;
+ const tag = setInterval(async () => {
+ const next = iterator.next();
+ if (next.done) {
+ clearInterval(tag);
+ } else {
+ collector.push(...(await handler(next.value)));
if (++completed === batches.length) {
resolve(collector);
}
- }, interval);
- });
- }
- return collector;
+ }
+ }, interval * 1000);
+ });
}; \ No newline at end of file
diff --git a/src/server/apis/google/GooglePhotosUploadUtils.ts b/src/server/apis/google/GooglePhotosUploadUtils.ts
index e640f2a85..e1478a097 100644
--- a/src/server/apis/google/GooglePhotosUploadUtils.ts
+++ b/src/server/apis/google/GooglePhotosUploadUtils.ts
@@ -80,7 +80,7 @@ export namespace GooglePhotosUploadUtils {
});
})).newMediaItemResults;
};
- const newMediaItemResults = await newMediaItems.batchAction(50, createFromUploadTokens, 1000);
+ const newMediaItemResults = await newMediaItems.convertInBatchesAtInterval(50, createFromUploadTokens, 1);
return { newMediaItemResults };
};
diff --git a/src/server/index.ts b/src/server/index.ts
index 79e9155d2..8767be17d 100644
--- a/src/server/index.ts
+++ b/src/server/index.ts
@@ -834,7 +834,12 @@ export interface NewMediaItem {
}
Array.prototype.batch = extensions.Batch;
-Array.prototype.batchAction = extensions.BatchAction;
+Array.prototype.executeInBatches = extensions.ExecuteInBatches;
+Array.prototype.convertInBatches = extensions.ConvertInBatches;
+Array.prototype.executeInBatchesAsync = extensions.ExecuteInBatchesAsync;
+Array.prototype.convertInBatchesAsync = extensions.ConvertInBatchesAsync;
+Array.prototype.executeInBatchesAtInterval = extensions.ExecuteInBatchesAtInterval;
+Array.prototype.convertInBatchesAtInterval = extensions.ConvertInBatchesAtInterval;
app.post(RouteStore.googlePhotosMediaUpload, async (req, res) => {
const mediaInput: GooglePhotosUploadUtils.MediaInput[] = req.body.media;
@@ -858,7 +863,7 @@ app.post(RouteStore.googlePhotosMediaUpload, async (req, res) => {
return newMediaItems;
};
- const newMediaItems = await mediaInput.batchAction<NewMediaItem>(25, dispatchUpload, 3000);
+ const newMediaItems = await mediaInput.convertInBatchesAtInterval<NewMediaItem>(25, dispatchUpload, 3);
if (failed) {
return _error(res, tokenError);