From 3fb212eb601dbaff28a61100fc0478b1c706a3b8 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Thu, 19 Sep 2019 16:30:14 -0400 Subject: organized and genericized --- src/extensions/ArrayExtensions.ts | 537 +++++++++++----------- src/extensions/Extensions.ts | 4 +- src/extensions/StringExtensions.ts | 2 +- src/server/apis/google/GooglePhotosUploadUtils.ts | 6 +- src/server/index.ts | 8 +- 5 files changed, 270 insertions(+), 287 deletions(-) (limited to 'src') diff --git a/src/extensions/ArrayExtensions.ts b/src/extensions/ArrayExtensions.ts index 1190aa48c..70508f6df 100644 --- a/src/extensions/ArrayExtensions.ts +++ b/src/extensions/ArrayExtensions.ts @@ -1,61 +1,3 @@ -interface BatchContext { - completedBatches: number; - remainingBatches: number; -} -type BatchConverterSync = (batch: I[], context: BatchContext) => O[]; -type BatchHandlerSync = (batch: I[], context: BatchContext) => void; -type BatchConverterAsync = (batch: I[], context: BatchContext) => Promise; -type BatchHandlerAsync = (batch: I[], context: BatchContext) => Promise; -type BatchConverter = BatchConverterSync | BatchConverterAsync; -type BatchHandler = BatchHandlerSync | BatchHandlerAsync; -type FixedBatcher = { batchSize: number } | { batchCount: number, mode?: Mode }; -interface ExecutorResult { - updated: A; - makeNextBatch: boolean; -} -interface PredicateBatcher { - executor: (element: I, accumulator: A) => ExecutorResult; - initial: A; - persistAccumulator?: boolean; -} -interface PredicateBatcherAsyncInterface { - executor: (element: I, accumulator: A) => Promise>; - initial: A; - persistAccumulator?: boolean; -} -type PredicateBatcherAsync = PredicateBatcher | PredicateBatcherAsyncInterface; -type Batcher = FixedBatcher | PredicateBatcher; -type BatcherAsync = Batcher | PredicateBatcherAsync; - -enum TimeUnit { - Milliseconds, - Seconds, - Minutes -} - -interface Interval { - magnitude: number; - unit: TimeUnit; -} - -enum Mode { - Balanced, - Even -} - -const convert = (interval: Interval) => { - const { magnitude, unit } = interval; - switch (unit) { - default: - case TimeUnit.Milliseconds: - return magnitude; - case TimeUnit.Seconds: - return magnitude * 1000; - case TimeUnit.Minutes: - return magnitude * 1000 * 60; - } -}; - interface Array { fixedBatch(batcher: FixedBatcher): T[][]; predicateBatch(batcher: PredicateBatcher): T[][]; @@ -75,134 +17,185 @@ interface Array { lastElement(): T; } -module.exports.AssignArrayExtensions = function () { - Array.prototype.fixedBatch = module.exports.fixedBatch; - Array.prototype.predicateBatch = module.exports.predicateBatch; - Array.prototype.predicateBatchAsync = module.exports.predicateBatchAsync; - Array.prototype.batch = module.exports.batch; - Array.prototype.batchAsync = module.exports.batchAsync; - Array.prototype.batchedForEach = module.exports.batchedForEach; - Array.prototype.batchedMap = module.exports.batchedMap; - Array.prototype.batchedForEachAsync = module.exports.batchedForEachAsync; - Array.prototype.batchedMapAsync = module.exports.batchedMapAsync; - Array.prototype.batchedForEachInterval = module.exports.batchedForEachInterval; - Array.prototype.batchedMapInterval = module.exports.batchedMapInterval; - Array.prototype.lastElement = module.exports.lastElement; +interface BatchContext { + completedBatches: number; + remainingBatches: number; +} + +interface ExecutorResult { + updated: A; + makeNextBatch: boolean; +} + +interface PredicateBatcherCommon { + initial: A; + persistAccumulator?: boolean; +} + +interface Interval { + magnitude: number; + unit: typeof module.exports.TimeUnit; +} + +type BatchConverterSync = (batch: I[], context: BatchContext) => O[]; +type BatchHandlerSync = (batch: I[], context: BatchContext) => void; +type BatchConverterAsync = (batch: I[], context: BatchContext) => Promise; +type BatchHandlerAsync = (batch: I[], context: BatchContext) => Promise; +type BatchConverter = BatchConverterSync | BatchConverterAsync; +type BatchHandler = BatchHandlerSync | BatchHandlerAsync; + +type FixedBatcher = { batchSize: number } | { batchCount: number, mode?: typeof module.exports.Mode }; +type PredicateBatcher = PredicateBatcherCommon & { executor: (element: I, accumulator: A) => ExecutorResult }; +type PredicateBatcherAsync = PredicateBatcherCommon & { executor: (element: I, accumulator: A) => ExecutorResult | Promise> }; + +type Batcher = FixedBatcher | PredicateBatcher; +type BatcherAsync = Batcher | PredicateBatcherAsync; + +module.exports.Mode = { + Balanced: 0, + Even: 1 }; -module.exports.fixedBatch = function (batcher: FixedBatcher): T[][] { - const batches: T[][] = []; - const length = this.length; - let i = 0; - if ("batchSize" in batcher) { - const { batchSize } = batcher; - while (i < this.length) { - const cap = Math.min(i + batchSize, length); - batches.push(this.slice(i, i = cap)); - } - } else if ("batchCount" in batcher) { - let { batchCount, mode } = batcher; - const resolved = mode || Mode.Balanced; - if (batchCount < 1) { - throw new Error("Batch count must be a positive integer!"); - } - if (batchCount === 1) { - return [this]; - } - if (batchCount >= this.length) { - return this.map((element: T) => [element]); - } +module.exports.TimeUnit = { + Milliseconds: 0, + Seconds: 1, + Minutes: 2 +}; - let length = this.length; - let size: number; +module.exports.Assign = function () { - if (length % batchCount === 0) { - size = Math.floor(length / batchCount); - while (i < length) { - batches.push(this.slice(i, i += size)); + Array.prototype.fixedBatch = function (batcher: FixedBatcher): T[][] { + const batches: T[][] = []; + const length = this.length; + let i = 0; + if ("batchSize" in batcher) { + const { batchSize } = batcher; + while (i < this.length) { + const cap = Math.min(i + batchSize, length); + batches.push(this.slice(i, i = cap)); } - } else if (resolved === Mode.Balanced) { - while (i < length) { - size = Math.ceil((length - i) / batchCount--); - batches.push(this.slice(i, i += size)); + } else if ("batchCount" in batcher) { + let { batchCount, mode } = batcher; + const resolved = mode || module.exports.Mode.Balanced; + if (batchCount < 1) { + throw new Error("Batch count must be a positive integer!"); } - } else { - batchCount--; - size = Math.floor(length / batchCount); - if (length % size === 0) { - size--; + if (batchCount === 1) { + return [this]; } - while (i < size * batchCount) { - batches.push(this.slice(i, i += size)); + if (batchCount >= this.length) { + return this.map((element: T) => [element]); + } + + let length = this.length; + let size: number; + + if (length % batchCount === 0) { + size = Math.floor(length / batchCount); + while (i < length) { + batches.push(this.slice(i, i += size)); + } + } else if (resolved === module.exports.Mode.Balanced) { + while (i < length) { + size = Math.ceil((length - i) / batchCount--); + batches.push(this.slice(i, i += size)); + } + } else { + batchCount--; + size = Math.floor(length / batchCount); + if (length % size === 0) { + size--; + } + while (i < size * batchCount) { + batches.push(this.slice(i, i += size)); + } + batches.push(this.slice(size * batchCount)); } - batches.push(this.slice(size * batchCount)); } - } - return batches; -}; + return batches; + }; -module.exports.predicateBatch = function (batcher: PredicateBatcher): T[][] { - const batches: T[][] = []; - let batch: T[] = []; - const { executor, initial, persistAccumulator } = batcher; - let accumulator = initial; - for (let element of this) { - const { updated, makeNextBatch } = executor(element, accumulator); - accumulator = updated; - if (!makeNextBatch) { - batch.push(element); - } else { - batches.push(batch); - batch = [element]; - if (!persistAccumulator) { - accumulator = initial; + Array.prototype.predicateBatch = function (batcher: PredicateBatcher): T[][] { + const batches: T[][] = []; + let batch: T[] = []; + const { executor, initial, persistAccumulator } = batcher; + let accumulator = initial; + for (let element of this) { + const { updated, makeNextBatch } = executor(element, accumulator); + accumulator = updated; + if (!makeNextBatch) { + batch.push(element); + } else { + batches.push(batch); + batch = [element]; + if (!persistAccumulator) { + accumulator = initial; + } } } - } - batches.push(batch); - return batches; -}; + batches.push(batch); + return batches; + }; -module.exports.predicateBatchAsync = async function (batcher: PredicateBatcherAsync): Promise { - const batches: T[][] = []; - let batch: T[] = []; - const { executor, initial, persistAccumulator } = batcher; - let accumulator: A = initial; - for (let element of this) { - const { updated, makeNextBatch } = await executor(element, accumulator); - accumulator = updated; - if (!makeNextBatch) { - batch.push(element); - } else { - batches.push(batch); - batch = [element]; - if (!persistAccumulator) { - accumulator = initial; + Array.prototype.predicateBatchAsync = async function (batcher: PredicateBatcherAsync): Promise { + const batches: T[][] = []; + let batch: T[] = []; + const { executor, initial, persistAccumulator } = batcher; + let accumulator: A = initial; + for (let element of this) { + const { updated, makeNextBatch } = await executor(element, accumulator); + accumulator = updated; + if (!makeNextBatch) { + batch.push(element); + } else { + batches.push(batch); + batch = [element]; + if (!persistAccumulator) { + accumulator = initial; + } } } - } - batches.push(batch); - return batches; -}; + batches.push(batch); + return batches; + }; -module.exports.batch = function (batcher: Batcher): T[][] { - if ("executor" in batcher) { - return this.predicateBatch(batcher); - } else { - return this.fixedBatch(batcher); - } -}; + Array.prototype.batch = function (batcher: Batcher): T[][] { + if ("executor" in batcher) { + return this.predicateBatch(batcher); + } else { + return this.fixedBatch(batcher); + } + }; -module.exports.batchAsync = async function (batcher: BatcherAsync): Promise { - if ("executor" in batcher) { - return this.predicateBatchAsync(batcher); - } else { - return this.fixedBatch(batcher); - } -}; + Array.prototype.batchAsync = async function (batcher: BatcherAsync): Promise { + if ("executor" in batcher) { + return this.predicateBatchAsync(batcher); + } else { + return this.fixedBatch(batcher); + } + }; + + Array.prototype.batchedForEach = function (batcher: Batcher, handler: BatchHandlerSync): void { + if (this.length) { + let completed = 0; + const batches = this.batch(batcher); + const quota = batches.length; + for (let batch of batches) { + const context: BatchContext = { + completedBatches: completed, + remainingBatches: quota - completed, + }; + handler(batch, context); + completed++; + } + } + }; -module.exports.batchedForEach = function (batcher: Batcher, handler: BatchHandlerSync): void { - if (this.length) { + Array.prototype.batchedMap = function (batcher: Batcher, handler: BatchConverterSync): O[] { + if (!this.length) { + return []; + } + let collector: O[] = []; let completed = 0; const batches = this.batch(batcher); const quota = batches.length; @@ -211,33 +204,33 @@ module.exports.batchedForEach = function (batcher: Batcher, handler: completedBatches: completed, remainingBatches: quota - completed, }; - handler(batch, context); + collector.push(...handler(batch, context)); completed++; } - } -}; + return collector; + }; -module.exports.batchedMap = function (batcher: Batcher, handler: BatchConverterSync): O[] { - if (!this.length) { - return []; - } - let collector: O[] = []; - let completed = 0; - const batches = this.batch(batcher); - const quota = batches.length; - for (let batch of batches) { - const context: BatchContext = { - completedBatches: completed, - remainingBatches: quota - completed, - }; - collector.push(...handler(batch, context)); - completed++; - } - return collector; -}; + Array.prototype.batchedForEachAsync = async function (batcher: BatcherAsync, handler: BatchHandler): Promise { + if (this.length) { + let completed = 0; + const batches = await this.batchAsync(batcher); + const quota = batches.length; + for (let batch of batches) { + const context: BatchContext = { + completedBatches: completed, + remainingBatches: quota - completed, + }; + await handler(batch, context); + completed++; + } + } + }; -module.exports.batchedForEachAsync = async function (batcher: BatcherAsync, handler: BatchHandler): Promise { - if (this.length) { + Array.prototype.batchedMapAsync = async function (batcher: BatcherAsync, handler: BatchConverter): Promise { + if (!this.length) { + return []; + } + let collector: O[] = []; let completed = 0; const batches = await this.batchAsync(batcher); const quota = batches.length; @@ -246,96 +239,92 @@ module.exports.batchedForEachAsync = async function (batcher: BatcherAsync completedBatches: completed, remainingBatches: quota - completed, }; - await handler(batch, context); + collector.push(...(await handler(batch, context))); completed++; } - } -}; + return collector; + }; -module.exports.batchedMapAsync = async function (batcher: BatcherAsync, handler: BatchConverter): Promise { - if (!this.length) { - return []; - } - let collector: O[] = []; - let completed = 0; - const batches = await this.batchAsync(batcher); - const quota = batches.length; - for (let batch of batches) { - const context: BatchContext = { - completedBatches: completed, - remainingBatches: quota - completed, - }; - collector.push(...(await handler(batch, context))); - completed++; - } - return collector; -}; - -module.exports.batchedForEachInterval = async function (batcher: BatcherAsync, handler: BatchHandler, interval: Interval): Promise { - if (!this.length) { - return; - } - const batches = await this.batchAsync(batcher); - const quota = batches.length; - return new Promise(async resolve => { - const iterator = batches[Symbol.iterator](); - let completed = 0; - while (true) { - const next = iterator.next(); - await new Promise(resolve => { - setTimeout(async () => { - const batch = next.value; - const context: BatchContext = { - completedBatches: completed, - remainingBatches: quota - completed, - }; - await handler(batch, context); - resolve(); - }, convert(interval)); - }); - if (++completed === quota) { - break; - } + Array.prototype.batchedForEachInterval = async function (batcher: BatcherAsync, handler: BatchHandler, interval: Interval): Promise { + if (!this.length) { + return; } - resolve(); - }); -}; + const batches = await this.batchAsync(batcher); + const quota = batches.length; + return new Promise(async resolve => { + const iterator = batches[Symbol.iterator](); + let completed = 0; + while (true) { + const next = iterator.next(); + await new Promise(resolve => { + setTimeout(async () => { + const batch = next.value; + const context: BatchContext = { + completedBatches: completed, + remainingBatches: quota - completed, + }; + await handler(batch, context); + resolve(); + }, convert(interval)); + }); + if (++completed === quota) { + break; + } + } + resolve(); + }); + }; -module.exports.batchedMapInterval = async function (batcher: BatcherAsync, handler: BatchConverter, interval: Interval): Promise { - if (!this.length) { - return []; - } - let collector: O[] = []; - const batches = await this.batchAsync(batcher); - const quota = batches.length; - return new Promise(async resolve => { - const iterator = batches[Symbol.iterator](); - let completed = 0; - while (true) { - const next = iterator.next(); - await new Promise(resolve => { - setTimeout(async () => { - const batch = next.value; - const context: BatchContext = { - completedBatches: completed, - remainingBatches: quota - completed, - }; - collector.push(...(await handler(batch, context))); - resolve(); - }, convert(interval)); - }); - if (++completed === quota) { - resolve(collector); - break; + Array.prototype.batchedMapInterval = async function (batcher: BatcherAsync, handler: BatchConverter, interval: Interval): Promise { + if (!this.length) { + return []; + } + let collector: O[] = []; + const batches = await this.batchAsync(batcher); + const quota = batches.length; + return new Promise(async resolve => { + const iterator = batches[Symbol.iterator](); + let completed = 0; + while (true) { + const next = iterator.next(); + await new Promise(resolve => { + setTimeout(async () => { + const batch = next.value; + const context: BatchContext = { + completedBatches: completed, + remainingBatches: quota - completed, + }; + collector.push(...(await handler(batch, context))); + resolve(); + }, convert(interval)); + }); + if (++completed === quota) { + resolve(collector); + break; + } } + }); + }; + + Array.prototype.lastElement = function () { + if (!this.length) { + return undefined; } - }); + const last: T = this[this.length - 1]; + return last; + }; + }; -module.exports.lastElement = function () { - if (!this.length) { - return undefined; +const convert = (interval: Interval) => { + const { magnitude, unit } = interval; + switch (unit) { + default: + case module.exports.Mode.TimeUnit.Milliseconds: + return magnitude; + case module.exports.Mode.TimeUnit.Seconds: + return magnitude * 1000; + case module.exports.Mode.TimeUnit.Minutes: + return magnitude * 1000 * 60; } - const last: T = this[this.length - 1]; - return last; -}; +}; \ No newline at end of file diff --git a/src/extensions/Extensions.ts b/src/extensions/Extensions.ts index 774236ea4..1bcebd0e2 100644 --- a/src/extensions/Extensions.ts +++ b/src/extensions/Extensions.ts @@ -2,6 +2,6 @@ const ArrayExtensions = require("./ArrayExtensions"); const StringExtensions = require("./StringExtensions"); module.exports.AssignExtensions = function () { - ArrayExtensions.Assign; - StringExtensions.Assign; + ArrayExtensions.Assign(); + StringExtensions.Assign(); }; \ No newline at end of file diff --git a/src/extensions/StringExtensions.ts b/src/extensions/StringExtensions.ts index 2ef31ec84..4cdbdebf7 100644 --- a/src/extensions/StringExtensions.ts +++ b/src/extensions/StringExtensions.ts @@ -3,7 +3,7 @@ interface String { hasNewline(): boolean; } -module.exports.AssignStringExtensions = function () { +module.exports.Assign = function () { String.prototype.removeTrailingNewlines = function () { let sliced = this; diff --git a/src/server/apis/google/GooglePhotosUploadUtils.ts b/src/server/apis/google/GooglePhotosUploadUtils.ts index 7eaf8a8b7..fc6772ffd 100644 --- a/src/server/apis/google/GooglePhotosUploadUtils.ts +++ b/src/server/apis/google/GooglePhotosUploadUtils.ts @@ -5,9 +5,9 @@ import { Utils } from '../../../Utils'; import * as path from 'path'; import { Opt } from '../../../new_fields/Doc'; import * as sharp from 'sharp'; -import { MediaItemCreationResult, NewMediaItemResult } from './SharedTypes'; -import { NewMediaItem } from '../..'; -import { TimeUnit } from "../../index"; +import { MediaItemCreationResult } from './SharedTypes'; +import { NewMediaItem } from "../../index"; +const { TimeUnit } = require("../../../extensions/ArrayExtensions"); const uploadDirectory = path.join(__dirname, "../../public/files/"); diff --git a/src/server/index.ts b/src/server/index.ts index 2d6c99d8a..1f411ade2 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -837,12 +837,6 @@ export interface NewMediaItem { }; } -export enum TimeUnit { - Milliseconds, - Seconds, - Minutes -} - app.post(RouteStore.googlePhotosMediaUpload, async (req, res) => { const mediaInput: GooglePhotosUploadUtils.MediaInput[] = req.body.media; await GooglePhotosUploadUtils.initialize({ uploadDirectory, credentialsPath, tokenPath }); @@ -865,7 +859,7 @@ app.post(RouteStore.googlePhotosMediaUpload, async (req, res) => { return newMediaItems; }; const batcher = { batchSize: 25 }; - const interval = { magnitude: 100, unit: TimeUnit.Milliseconds }; + const interval = { magnitude: 100, unit: ArrayExtensions.TimeUnit.Milliseconds }; const newMediaItems = await mediaInput.batchedMapInterval(batcher, dispatchUpload, interval); -- cgit v1.2.3-70-g09d2