diff options
Diffstat (limited to 'src/server/ApiManagers/GooglePhotosManager.ts')
| -rw-r--r-- | src/server/ApiManagers/GooglePhotosManager.ts | 155 |
1 files changed, 74 insertions, 81 deletions
diff --git a/src/server/ApiManagers/GooglePhotosManager.ts b/src/server/ApiManagers/GooglePhotosManager.ts index c0da39a03..4c2004681 100644 --- a/src/server/ApiManagers/GooglePhotosManager.ts +++ b/src/server/ApiManagers/GooglePhotosManager.ts @@ -1,23 +1,23 @@ -import ApiManager, { Registration } from "./ApiManager"; -import { Method, _error, _success, _invalid } from "../RouteManager"; -import * as path from "path"; -import { GoogleApiServerUtils } from "../apis/google/GoogleApiServerUtils"; -import { BatchedArray, TimeUnit } from "array-batcher"; -import { Opt } from "../../fields/Doc"; -import { DashUploadUtils, InjectSize, SizeSuffix } from "../DashUploadUtils"; -import { Database } from "../database"; -import { red } from "colors"; -import { Upload } from "../SharedMediaTypes"; +import ApiManager, { Registration } from './ApiManager'; +import { Method, _error, _success, _invalid } from '../RouteManager'; +import * as path from 'path'; +import { GoogleApiServerUtils } from '../apis/google/GoogleApiServerUtils'; +import { BatchedArray, TimeUnit } from 'array-batcher'; +import { Opt } from '../../fields/Doc'; +import { DashUploadUtils, InjectSize, SizeSuffix } from '../DashUploadUtils'; +import { Database } from '../database'; +import { red } from 'colors'; +import { Upload } from '../SharedMediaTypes'; import * as request from 'request-promise'; -import { NewMediaItemResult } from "../apis/google/SharedTypes"; +import { NewMediaItemResult } from '../apis/google/SharedTypes'; -const prefix = "google_photos_"; +const prefix = 'google_photos_'; const remoteUploadError = "None of the preliminary uploads to Google's servers was successful."; -const authenticationError = "Unable to authenticate Google credentials before uploading to Google Photos!"; -const mediaError = "Unable to convert all uploaded bytes to media items!"; +const authenticationError = 'Unable to authenticate Google credentials before uploading to Google Photos!'; +const mediaError = 'Unable to convert all uploaded bytes to media items!'; const localUploadError = (count: number) => `Unable to upload ${count} images to Dash's server`; const requestError = "Unable to execute download: the body's media items were malformed."; -const downloadError = "Encountered an error while executing downloads."; +const downloadError = 'Encountered an error while executing downloads.'; interface GooglePhotosUploadFailure { batch: number; @@ -41,17 +41,15 @@ interface NewMediaItem { * This manager handles the creation of routes for google photos functionality. */ export default class GooglePhotosManager extends ApiManager { - protected initialize(register: Registration): void { - /** * This route receives a list of urls that point to images stored * on Dash's file system, and, in a two step process, uploads them to Google's servers and - * returns the information Google generates about the associated uploaded remote images. + * returns the information Google generates about the associated uploaded remote images. */ register({ method: Method.POST, - subscription: "/googlePhotosMediaPost", + subscription: '/googlePhotosMediaPost', secureHandler: async ({ user, req, res }) => { const { media } = req.body; @@ -67,38 +65,35 @@ export default class GooglePhotosManager extends ApiManager { const failed: GooglePhotosUploadFailure[] = []; const batched = BatchedArray.from<Uploader.UploadSource>(media, { batchSize: 25 }); const interval = { magnitude: 100, unit: TimeUnit.Milliseconds }; - const newMediaItems = await batched.batchedMapPatientInterval<NewMediaItem>( - interval, - async (batch, collector, { completedBatches }) => { - for (let index = 0; index < batch.length; index++) { - const { url, description } = batch[index]; - // a local function used to record failure of an upload - const fail = (reason: string) => failed.push({ reason, batch: completedBatches + 1, index, url }); - // see image resizing - we store the size-agnostic url in our logic, but write out size-suffixed images to the file system - // so here, given a size agnostic url, we're just making that conversion so that the file system knows which bytes to actually upload - const imageToUpload = InjectSize(url, SizeSuffix.Original); - // STEP 1/2: send the raw bytes of the image from our server to Google's servers. We'll get back an upload token - // which acts as a pointer to those bytes that we can use to locate them later on - const uploadToken = await Uploader.SendBytes(token, imageToUpload).catch(fail); - if (!uploadToken) { - fail(`${path.extname(url)} is not an accepted extension`); - } else { - // gather the upload token return from Google (a pointer they give us to the raw, currently useless bytes - // we've uploaded to their servers) and put in the JSON format that the API accepts for image creation (used soon, below) - collector.push({ - description, - simpleMediaItem: { uploadToken } - }); - } + const newMediaItems = await batched.batchedMapPatientInterval<NewMediaItem>(interval, async (batch, collector, { completedBatches }) => { + for (let index = 0; index < batch.length; index++) { + const { url, description } = batch[index]; + // a local function used to record failure of an upload + const fail = (reason: string) => failed.push({ reason, batch: completedBatches + 1, index, url }); + // see image resizing - we store the size-agnostic url in our logic, but write out size-suffixed images to the file system + // so here, given a size agnostic url, we're just making that conversion so that the file system knows which bytes to actually upload + const imageToUpload = InjectSize(url, SizeSuffix.Original); + // STEP 1/2: send the raw bytes of the image from our server to Google's servers. We'll get back an upload token + // which acts as a pointer to those bytes that we can use to locate them later on + const uploadToken = await Uploader.SendBytes(token, imageToUpload).catch(fail); + if (!uploadToken) { + fail(`${path.extname(url)} is not an accepted extension`); + } else { + // gather the upload token return from Google (a pointer they give us to the raw, currently useless bytes + // we've uploaded to their servers) and put in the JSON format that the API accepts for image creation (used soon, below) + collector.push({ + description, + simpleMediaItem: { uploadToken }, + }); } } - ); + }); // inform the developer / server console of any failed upload attempts // does not abort the operation, since some subset of the uploads may have been successful const { length } = failed; if (length) { - console.error(`Unable to upload ${length} image${length === 1 ? "" : "s"} to Google's servers`); + console.error(`Unable to upload ${length} image${length === 1 ? '' : 's'} to Google's servers`); console.log(failed.map(({ reason, batch, index, url }) => `@${batch}.${index}: ${url} failed:\n${reason}`).join('\n\n')); } @@ -115,7 +110,7 @@ export default class GooglePhotosManager extends ApiManager { results => _success(res, { results, failed }), error => _error(res, mediaError, error) ); - } + }, }); /** @@ -128,11 +123,11 @@ export default class GooglePhotosManager extends ApiManager { * since the same bytes on their server might now be associated with a new, random url. * So, we do the next best thing and try to use an intrinsic attribute of those bytes as * an identifier: the precise content size. This works in small cases, but has the obvious flaw of failing to upload - * an image locally if we already have uploaded another Google user content image with the exact same content size. + * an image locally if we already have uploaded another Google user content image with the exact same content size. */ register({ method: Method.POST, - subscription: "/googlePhotosMediaGet", + subscription: '/googlePhotosMediaGet', secureHandler: async ({ req, res }) => { const { mediaItems } = req.body as { mediaItems: MediaItem[] }; if (!mediaItems) { @@ -167,7 +162,7 @@ export default class GooglePhotosManager extends ApiManager { failed++; } } else { - // if we have, the variable 'found' is handily the upload information of the + // if we have, the variable 'found' is handily the upload information of the // existing image, so we add it to the list as if we had just uploaded it now without actually // making a duplicate write completed.push(found); @@ -180,9 +175,8 @@ export default class GooglePhotosManager extends ApiManager { // otherwise, return the image upload information list corresponding to the newly (or previously) // uploaded images _success(res, completed); - } + }, }); - } } @@ -192,11 +186,10 @@ export default class GooglePhotosManager extends ApiManager { * and then initialize / create those images in the Photos * API given the upload tokens returned from the initial * uploading process. - * + * * https://developers.google.com/photos/library/reference/rest/v1/mediaItems/batchCreate */ export namespace Uploader { - /** * Specifies the structure of the object * necessary to upload bytes to Google's servers. @@ -214,7 +207,7 @@ export namespace Uploader { * into the BatchCreate API request * to take a reference to raw uploaded bytes * and actually create an image in Google Photos. - * + * * So, to instantiate this interface you must have already dispatched an upload * and received an upload token. */ @@ -245,7 +238,7 @@ export namespace Uploader { function headers(type: string, token: string) { return { 'Content-Type': `application/${type}`, - 'Authorization': `Bearer ${token}`, + Authorization: `Bearer ${token}`, }; } @@ -272,17 +265,19 @@ export namespace Uploader { headers: { ...headers('octet-stream', bearerToken), 'X-Goog-Upload-File-Name': filename || path.basename(url), - 'X-Goog-Upload-Protocol': 'raw' + 'X-Goog-Upload-Protocol': 'raw', }, - body + body, }; - return new Promise((resolve, reject) => request(parameters, (error, _response, body) => { - if (error) { - // on rejection, the server logs the error and the offending image - return reject(error); - } - resolve(body); - })); + return new Promise((resolve, reject) => + request(parameters, (error, _response, body) => { + if (error) { + // on rejection, the server logs the error and the offending image + return reject(error); + } + resolve(body); + }) + ); }; /** @@ -303,19 +298,18 @@ export namespace Uploader { // seems to need at least some latency between requests (spamming it synchronously has led to the server returning errors)... const batched = BatchedArray.from(newMediaItems, { batchSize: 50 }); // ...so we execute them in delayed batches and await the entire execution - return batched.batchedMapPatientInterval( - { magnitude: 100, unit: TimeUnit.Milliseconds }, - async (batch: NewMediaItem[], collector): Promise<void> => { - const parameters = { - method: 'POST', - headers: headers('json', bearerToken), - uri: prepend('mediaItems:batchCreate'), - body: { newMediaItems: batch } as any, - json: true - }; - // register the target album, if provided - album && (parameters.body.albumId = album.id); - collector.push(...(await new Promise<NewMediaItemResult[]>((resolve, reject) => { + return batched.batchedMapPatientInterval({ magnitude: 100, unit: TimeUnit.Milliseconds }, async (batch: NewMediaItem[], collector): Promise<void> => { + const parameters = { + method: 'POST', + headers: headers('json', bearerToken), + uri: prepend('mediaItems:batchCreate'), + body: { newMediaItems: batch } as any, + json: true, + }; + // register the target album, if provided + album && (parameters.body.albumId = album.id); + collector.push( + ...(await new Promise<NewMediaItemResult[]>((resolve, reject) => { request(parameters, (error, _response, body) => { if (error) { reject(error); @@ -323,9 +317,8 @@ export namespace Uploader { resolve(body.newMediaItemResults); } }); - }))); - } - ); + })) + ); + }); }; - -}
\ No newline at end of file +} |