diff options
| -rw-r--r-- | package.json | 2 | ||||
| -rw-r--r-- | src/client/util/Import & Export/DirectoryImportBox.tsx | 5 | ||||
| -rw-r--r-- | src/server/RouteManager.ts | 1 | ||||
| -rw-r--r-- | src/server/apis/google/GooglePhotosUploadUtils.ts | 11 | ||||
| -rw-r--r-- | src/server/index.ts | 48 |
5 files changed, 36 insertions, 31 deletions
diff --git a/package.json b/package.json index 8cbbb84af..4572a3f73 100644 --- a/package.json +++ b/package.json @@ -115,7 +115,7 @@ "@types/youtube": "0.0.38", "adm-zip": "^0.4.13", "archiver": "^3.0.3", - "array-batcher": "^1.1.3", + "array-batcher": "^1.2.3", "async": "^2.6.2", "babel-runtime": "^6.26.0", "bcrypt-nodejs": "0.0.3", diff --git a/src/client/util/Import & Export/DirectoryImportBox.tsx b/src/client/util/Import & Export/DirectoryImportBox.tsx index 2d1b6fe20..bdd59cb16 100644 --- a/src/client/util/Import & Export/DirectoryImportBox.tsx +++ b/src/client/util/Import & Export/DirectoryImportBox.tsx @@ -107,7 +107,7 @@ export default class DirectoryImportBox extends React.Component<FieldViewProps> runInAction(() => this.phase = `Internal: uploading ${this.quota - this.completed} files to Dash...`); - const uploads = await BatchedArray.from(validated, { batchSize: 15 }).batchedMapAsync(async batch => { + const uploads = await BatchedArray.from(validated, { batchSize: 15 }).batchedMapAsync<ImageUploadResponse>(async (batch, collector) => { const formData = new FormData(); batch.forEach(file => { @@ -116,9 +116,8 @@ export default class DirectoryImportBox extends React.Component<FieldViewProps> formData.append(Utils.GenerateGuid(), file); }); - const responses = await Networking.PostFormDataToServer(RouteStore.upload, formData); + collector.push(...(await Networking.PostFormDataToServer(RouteStore.upload, formData))); runInAction(() => this.completed += batch.length); - return responses as ImageUploadResponse[]; }); await Promise.all(uploads.map(async upload => { diff --git a/src/server/RouteManager.ts b/src/server/RouteManager.ts index eda2a49d2..c1d38327f 100644 --- a/src/server/RouteManager.ts +++ b/src/server/RouteManager.ts @@ -114,6 +114,7 @@ export const STATUS = { }; export function _error(res: express.Response, message: string, error?: any) { + console.error(message); res.statusMessage = message; res.status(STATUS.EXECUTION_ERROR).send(error); } diff --git a/src/server/apis/google/GooglePhotosUploadUtils.ts b/src/server/apis/google/GooglePhotosUploadUtils.ts index 172fa8d46..d3442338b 100644 --- a/src/server/apis/google/GooglePhotosUploadUtils.ts +++ b/src/server/apis/google/GooglePhotosUploadUtils.ts @@ -1,7 +1,7 @@ import request = require('request-promise'); import { GoogleApiServerUtils } from './GoogleApiServerUtils'; import * as path from 'path'; -import { MediaItemCreationResult } from './SharedTypes'; +import { MediaItemCreationResult, NewMediaItemResult } from './SharedTypes'; import { NewMediaItem } from "../../index"; import { BatchedArray, TimeUnit } from 'array-batcher'; import { DashUploadUtils } from '../../DashUploadUtils'; @@ -50,9 +50,9 @@ export namespace GooglePhotosUploadUtils { }; export const CreateMediaItems = async (bearerToken: string, newMediaItems: NewMediaItem[], album?: { id: string }): Promise<MediaItemCreationResult> => { - const newMediaItemResults = await BatchedArray.from(newMediaItems, { batchSize: 50 }).batchedMapPatientInterval( + const newMediaItemResults = await BatchedArray.from(newMediaItems, { batchSize: 50 }).batchedMapPatientInterval<NewMediaItemResult>( { magnitude: 100, unit: TimeUnit.Milliseconds }, - async (batch: NewMediaItem[]) => { + async (batch: NewMediaItem[], collector) => { const parameters = { method: 'POST', headers: headers('json', bearerToken), @@ -61,7 +61,7 @@ export namespace GooglePhotosUploadUtils { json: true }; album && (parameters.body.albumId = album.id); - return (await new Promise<MediaItemCreationResult>((resolve, reject) => { + const { newMediaItemResults } = await new Promise<MediaItemCreationResult>((resolve, reject) => { request(parameters, (error, _response, body) => { if (error) { reject(error); @@ -69,7 +69,8 @@ export namespace GooglePhotosUploadUtils { resolve(body); } }); - })).newMediaItemResults; + }); + collector.push(...newMediaItemResults); } ); return { newMediaItemResults }; diff --git a/src/server/index.ts b/src/server/index.ts index 860cde3b5..05c866eae 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -573,18 +573,15 @@ function routeSetter(router: RouteManager) { onValidation: async ({ req, res, user }) => { let sector: GoogleApiServerUtils.Service = req.params.sector as GoogleApiServerUtils.Service; let action: GoogleApiServerUtils.Action = req.params.action as GoogleApiServerUtils.Action; - return GoogleApiServerUtils.GetEndpoint(GoogleApiServerUtils.Service[sector], user.id).then(endpoint => { - let handler = EndpointHandlerMap.get(action); - if (endpoint && handler) { - let execute = handler(endpoint, req.body).then( - response => res.send(response.data), - rejection => res.send(rejection) - ); - execute.catch(exception => res.send(exception)); - return; - } - res.send(undefined); - }); + const endpoint = await GoogleApiServerUtils.GetEndpoint(GoogleApiServerUtils.Service[sector], user.id); + let handler = EndpointHandlerMap.get(action); + if (endpoint && handler) { + handler(endpoint, req.body) + .then(response => res.send(response.data)) + .catch(exception => res.send(exception)); + return; + } + res.send(undefined); } }); @@ -611,6 +608,12 @@ function routeSetter(router: RouteManager) { const authenticationError = "Unable to authenticate Google credentials before uploading to Google Photos!"; const mediaError = "Unable to convert all uploaded bytes to media items!"; + interface GooglePhotosUploadFailure { + batch: number; + index: number; + url: string; + reason: string; + } router.addSupervisedRoute({ method: Method.POST, @@ -623,30 +626,31 @@ function routeSetter(router: RouteManager) { return _error(res, authenticationError); } - let failed: number[] = []; - const newMediaItems = await BatchedArray.from<GooglePhotosUploadUtils.MediaInput>(media, { batchSize: 25 }).batchedMapPatientInterval( + let failed: GooglePhotosUploadFailure[] = []; + const batched = BatchedArray.from<GooglePhotosUploadUtils.MediaInput>(media, { batchSize: 25 }); + const newMediaItems = await batched.batchedMapPatientInterval<NewMediaItem>( { magnitude: 100, unit: TimeUnit.Milliseconds }, - async (batch: GooglePhotosUploadUtils.MediaInput[]) => { - const newMediaItems: NewMediaItem[] = []; + async (batch, collector, { completedBatches }) => { for (let index = 0; index < batch.length; index++) { - const element = batch[index]; - const uploadToken = await GooglePhotosUploadUtils.DispatchGooglePhotosUpload(token, element.url); + const { url, description } = batch[index]; + const fail = (reason: string) => failed.push({ reason, batch: completedBatches + 1, index, url }); + const uploadToken = await GooglePhotosUploadUtils.DispatchGooglePhotosUpload(token, url).catch(fail); if (!uploadToken) { - failed.push(index); + fail(`${path.extname(url)} is not an accepted extension`); } else { - newMediaItems.push({ - description: element.description, + collector.push({ + description, simpleMediaItem: { uploadToken } }); } } - return newMediaItems; } ); const failedCount = failed.length; if (failedCount) { console.error(`Unable to upload ${failedCount} image${failedCount === 1 ? "" : "s"} to Google's servers`); + console.log(failed.map(({ reason, batch, index, url }) => `@${batch}.${index}: ${url} failed: ${reason}`).join('\n')); } return GooglePhotosUploadUtils.CreateMediaItems(token, newMediaItems, req.body.album).then( |