diff options
Diffstat (limited to 'src/server')
| -rw-r--r-- | src/server/DashUploadUtils.ts | 6 | ||||
| -rw-r--r-- | src/server/RouteManager.ts | 5 | ||||
| -rw-r--r-- | src/server/RouteStore.ts | 3 | ||||
| -rw-r--r-- | src/server/apis/google/GoogleApiServerUtils.ts | 361 | ||||
| -rw-r--r-- | src/server/apis/google/GooglePhotosUploadUtils.ts | 113 | ||||
| -rw-r--r-- | src/server/database.ts | 4 | ||||
| -rw-r--r-- | src/server/index.ts | 62 |
7 files changed, 320 insertions, 234 deletions
diff --git a/src/server/DashUploadUtils.ts b/src/server/DashUploadUtils.ts index 46d897339..9fddb466c 100644 --- a/src/server/DashUploadUtils.ts +++ b/src/server/DashUploadUtils.ts @@ -24,9 +24,13 @@ export namespace DashUploadUtils { const gifs = [".gif"]; const pngs = [".png"]; const jpgs = [".jpg", ".jpeg"]; - export const imageFormats = [...pngs, ...jpgs, ...gifs]; + const imageFormats = [...pngs, ...jpgs, ...gifs]; const videoFormats = [".mov", ".mp4"]; + export function validateExtension(url: string) { + return imageFormats.includes(path.extname(url).toLowerCase()); + } + const size = "content-length"; const type = "content-type"; diff --git a/src/server/RouteManager.ts b/src/server/RouteManager.ts index 21ce9c9e4..c1d38327f 100644 --- a/src/server/RouteManager.ts +++ b/src/server/RouteManager.ts @@ -49,9 +49,9 @@ export default class RouteManager { let supervised = async (req: express.Request, res: express.Response) => { const { user, originalUrl: target } = req; const core = { req, res, isRelease }; - const tryExecute = async (target: (args: any) => any | Promise<any>, args: any) => { + const tryExecute = async (toExecute: (args: any) => any | Promise<any>, args: any) => { try { - await target(args); + await toExecute(args); } catch (e) { if (onError) { onError({ ...core, error: e }); @@ -114,6 +114,7 @@ export const STATUS = { }; export function _error(res: express.Response, message: string, error?: any) { + console.error(message); res.statusMessage = message; res.status(STATUS.EXECUTION_ERROR).send(error); } diff --git a/src/server/RouteStore.ts b/src/server/RouteStore.ts index de2553b2f..a310d0c95 100644 --- a/src/server/RouteStore.ts +++ b/src/server/RouteStore.ts @@ -39,6 +39,7 @@ export enum RouteStore { writeGoogleAccessToken = "/writeGoogleAccessToken", googlePhotosMediaUpload = "/googlePhotosMediaUpload", googlePhotosMediaDownload = "/googlePhotosMediaDownload", - googleDocsGet = "/googleDocsGet" + googleDocsGet = "/googleDocsGet", + checkGoogle = "/checkGoogleAuthentication" }
\ No newline at end of file diff --git a/src/server/apis/google/GoogleApiServerUtils.ts b/src/server/apis/google/GoogleApiServerUtils.ts index b9984649e..35a2541a9 100644 --- a/src/server/apis/google/GoogleApiServerUtils.ts +++ b/src/server/apis/google/GoogleApiServerUtils.ts @@ -2,27 +2,19 @@ import { google } from "googleapis"; import { readFile } from "fs"; import { OAuth2Client, Credentials, OAuth2ClientOptions } from "google-auth-library"; import { Opt } from "../../../new_fields/Doc"; -import { GlobalOptions } from "googleapis-common"; import { GaxiosResponse } from "gaxios"; import request = require('request-promise'); import * as qs from 'query-string'; import { Database } from "../../database"; -import path from "path"; +import * as path from "path"; /** - * + * Scopes give Google users fine granularity of control + * over the information they make accessible via the API. + * This is the somewhat overkill list of what Dash requests + * from the user. */ -const prefix = 'https://www.googleapis.com/auth/'; - -/** - * - */ -const refreshEndpoint = "https://oauth2.googleapis.com/token"; - -/** - * - */ -const SCOPES = [ +const scope = [ 'documents.readonly', 'documents', 'presentations', @@ -33,7 +25,7 @@ const SCOPES = [ 'photoslibrary.appendonly', 'photoslibrary.sharing', 'userinfo.profile' -]; +].map(relative => `https://www.googleapis.com/auth/${relative}`); /** * This namespace manages server side authentication for Google API queries, either @@ -42,33 +34,9 @@ const SCOPES = [ export namespace GoogleApiServerUtils { /** - * - */ - export interface CredentialsResult { - credentials: Credentials; - refreshed: boolean; - } - - /** - * - */ - export interface UserInfo { - at_hash: string; - aud: string; - azp: string; - exp: number; - family_name: string; - given_name: string; - iat: number; - iss: string; - locale: string; - name: string; - picture: string; - sub: string; - } - - /** - * + * As we expand out to more Google APIs that are accessible from + * the 'googleapis' module imported above, this enum will record + * the list and provide a unified string representation of each API. */ export enum Service { Documents = "Documents", @@ -76,15 +44,10 @@ export namespace GoogleApiServerUtils { } /** - * - */ - export interface CredentialInformation { - credentialsPath: string; - userId: string; - } - - /** - * + * Global credentials read once from a JSON file + * before the server is started that + * allow us to build OAuth2 clients with Dash's + * application specific credentials. */ let installed: OAuth2ClientOptions; @@ -99,27 +62,33 @@ export namespace GoogleApiServerUtils { let worker: OAuth2Client; /** - * + * A briefer format for the response from a 'googleapis' API request */ export type ApiResponse = Promise<GaxiosResponse>; /** - * + * A generic form for a handler that executes some request on the endpoint */ export type ApiRouter = (endpoint: Endpoint, parameters: any) => ApiResponse; /** - * + * A generic form for the asynchronous function that actually submits the + * request to the API and returns the corresporing response. Helpful when + * making an extensible endpoint definition. */ export type ApiHandler = (parameters: any, methodOptions?: any) => ApiResponse; /** - * + * A literal union type indicating the valid actions for these 'googleapis' + * requestions */ export type Action = "create" | "retrieve" | "update"; /** - * + * An interface defining any entity on which one can invoke + * anuy of the following handlers. All 'googleapis' wrappers + * such as google.docs().documents and google.slides().presentations + * satisfy this interface. */ export interface Endpoint { get: ApiHandler; @@ -128,15 +97,13 @@ export namespace GoogleApiServerUtils { } /** - * - */ - export type EndpointParameters = GlobalOptions & { version: "v1" }; - - /** - * + * This function is called once before the server is started, + * reading in Dash's project-specific credentials (client secret + * and client id) for later repeated access. It also sets up the + * global, intentionally unauthenticated worker OAuth2 client instance. */ - export const loadClientSecret = async () => { - return new Promise<void>((resolve, reject) => { + export async function loadClientSecret(): Promise<void> { + return new Promise((resolve, reject) => { readFile(path.join(__dirname, "../../credentials/google_docs_credentials.json"), async (err, projectCredentials) => { if (err) { reject(err); @@ -153,89 +120,111 @@ export namespace GoogleApiServerUtils { resolve(); }); }); - }; + } /** - * + * Maps the Dash user id of a given user to their single + * associated OAuth2 client, mitigating the creation + * of needless duplicate clients that would arise from + * making one new client instance per request. */ const authenticationClients = new Map<String, OAuth2Client>(); /** - * - * @param sector - * @param userId + * This function receives the target sector ("which G-Suite app's API am I interested in?") + * and the id of the Dash user making the request to the API. With this information, it generates + * an authenticated OAuth2 client and passes it into the relevant 'googleapis' wrapper. + * @param sector the particular desired G-Suite 'googleapis' API (docs, slides, etc.) + * @param userId the id of the Dash user making the request to the API + * @returns the relevant 'googleapis' wrapper, if any */ - export const GetEndpoint = (sector: string, userId: string) => { - return new Promise<Opt<Endpoint>>(resolve => { - retrieveOAuthClient(userId).then(auth => { - let routed: Opt<Endpoint>; - let parameters: EndpointParameters = { auth, version: "v1" }; - switch (sector) { - case Service.Documents: - routed = google.docs(parameters).documents; - break; - case Service.Slides: - routed = google.slides(parameters).presentations; - break; - } - resolve(routed); - }); + export async function GetEndpoint(sector: string, userId: string): Promise<Opt<Endpoint>> { + return new Promise(async resolve => { + const auth = await retrieveOAuthClient(userId); + if (!auth) { + return resolve(); + } + let routed: Opt<Endpoint>; + let parameters: any = { auth, version: "v1" }; + switch (sector) { + case Service.Documents: + routed = google.docs(parameters).documents; + break; + case Service.Slides: + routed = google.slides(parameters).presentations; + break; + } + resolve(routed); }); - }; + } /** - * - * @param userId + * Returns the lengthy string or access token that can be passed into + * the headers of an API request or into the constructor of the Photos + * client API wrapper. + * @param userId the Dash user id of the user requesting his/her associated + * access_token + * @returns the current access_token associated with the requesting + * Dash user. The access_token is valid for only an hour, and + * is then refreshed. */ - export const retrieveAccessToken = (userId: string): Promise<string> => { - return new Promise<string>((resolve, reject) => { - retrieveCredentials(userId).then( - ({ credentials }) => resolve(credentials.access_token!), - error => reject(`Error: unable to authenticate Google Photos API request.\n${error}`) - ); + export async function retrieveAccessToken(userId: string): Promise<string> { + return new Promise(async resolve => { + const { credentials } = await retrieveCredentials(userId); + if (!credentials) { + return resolve(); + } + resolve(credentials.access_token!); }); - }; + } /** - * - * @param userId + * Manipulates a mapping such that, in the limit, each Dash user has + * an associated authenticated OAuth2 client at their disposal. This + * function ensures that the client's credentials always remain up to date + * @param userId the Dash user id of the user requesting account integration + * @returns returns an initialized OAuth2 client instance, likely to be passed into Google's + * npm-installed API wrappers that use authenticated client instances rather than access codes for + * security. */ - export const retrieveOAuthClient = (userId: string): Promise<OAuth2Client> => { - return new Promise<OAuth2Client>((resolve, reject) => { - retrieveCredentials(userId).then( - ({ credentials, refreshed }) => { - let client = authenticationClients.get(userId); - if (!client) { - authenticationClients.set(userId, client = generateClient(credentials)); - } else if (refreshed) { - client.setCredentials(credentials); - } - resolve(client); - }, - error => reject(`Error: unable to instantiate and certify a new OAuth2 client.\n${error}`) - ); + export async function retrieveOAuthClient(userId: string): Promise<OAuth2Client> { + return new Promise(async resolve => { + const { credentials, refreshed } = await retrieveCredentials(userId); + if (!credentials) { + return resolve(); + } + let client = authenticationClients.get(userId); + if (!client) { + authenticationClients.set(userId, client = generateClient(credentials)); + } else if (refreshed) { + client.setCredentials(credentials); + } + resolve(client); }); - }; + } /** - * - * @param credentials + * Creates a new OAuth2Client instance, and if provided, sets + * the specific credentials on the client + * @param credentials if you have access to the credentials that you'll eventually set on + * the client, just pass them in at initialization + * @returns the newly created, potentially certified, OAuth2 client instance */ - function generateClient(credentials?: Credentials) { + function generateClient(credentials?: Credentials): OAuth2Client { const client = new google.auth.OAuth2(installed); credentials && client.setCredentials(credentials); return client; } /** - * + * Calls on the worker (which does not have and does not need + * any credentials) to produce a url to which the user can + * navigate to give Dash the necessary Google permissions. + * @returns the newly generated url to the authentication landing page */ - export const generateAuthenticationUrl = async () => { - return worker.generateAuthUrl({ - access_type: 'offline', - scope: SCOPES.map(relative => prefix + relative), - }); - }; + export function generateAuthenticationUrl(): string { + return worker.generateAuthUrl({ scope, access_type: 'offline' }); + } /** * This is what we return to the server in processNewUser(), after the @@ -255,32 +244,36 @@ export namespace GoogleApiServerUtils { * and the authentication code to fetch the full set of credentials that * we'll store in the database for each user. This is called once per * new account integration. - * @param userId The Dash user id of the user requesting account integration, used to associate the new credentials + * @param userId the Dash user id of the user requesting account integration, used to associate the new credentials * with a Dash user in the googleAuthentication table of the database. * @param authenticationCode the Google-provided authentication code that the user copied * from Google's permissions UI and pasted into the overlay. + * + * EXAMPLE CODE: 4/sgF2A5uGg4xASHf7VQDnLtdqo3mUlfQqLSce_HYz5qf1nFtHj9YTeGs + * * @returns the information necessary to authenticate a client side google photos request * and display basic user information in the overlay on successful authentication. * This can be expanded as needed by adding properties to the interface GoogleAuthenticationResult. */ - export const processNewUser = async (userId: string, authenticationCode: string): Promise<GoogleAuthenticationResult> => { - return new Promise<GoogleAuthenticationResult>((resolve, reject) => { + export async function processNewUser(userId: string, authenticationCode: string): Promise<GoogleAuthenticationResult> { + const credentials = await new Promise<Credentials>((resolve, reject) => { worker.getToken(authenticationCode, async (err, credentials) => { if (err || !credentials) { reject(err); - return console.error('Error retrieving access token', err); + return; } - const enriched = injectUserInfo(credentials); - await Database.Auxiliary.GoogleAuthenticationToken.Write(userId, enriched); - const { given_name, picture } = enriched.userInfo; - resolve({ - access_token: enriched.access_token!, - avatar: picture, - name: given_name - }); + resolve(credentials); }); }); - }; + const enriched = injectUserInfo(credentials); + await Database.Auxiliary.GoogleAuthenticationToken.Write(userId, enriched); + const { given_name, picture } = enriched.userInfo; + return { + access_token: enriched.access_token!, + avatar: picture, + name: given_name + }; + } /** * This type represents the union of the full set of OAuth2 credentials @@ -290,6 +283,26 @@ export namespace GoogleApiServerUtils { export type EnrichedCredentials = Credentials & { userInfo: UserInfo }; /** + * This interface defines all of the information we + * receive from parsing the base64 encoded info-token + * for a Google user. + */ + export interface UserInfo { + at_hash: string; + aud: string; + azp: string; + exp: number; + family_name: string; + given_name: string; + iat: number; + iss: string; + locale: string; + name: string; + picture: string; + sub: string; + } + + /** * It's pretty cool: the credentials id_token is split into thirds by periods. * The middle third contains a base64-encoded JSON string with all the * user info contained in the interface below. So, we isolate that middle third, @@ -299,34 +312,32 @@ export namespace GoogleApiServerUtils { * @returns the full set of credentials in the structure in which they'll be stored * in the database. */ - const injectUserInfo = (credentials: Credentials): EnrichedCredentials => { - const userInfo = JSON.parse(atob(credentials.id_token!.split(".")[1])); + function injectUserInfo(credentials: Credentials): EnrichedCredentials { + const userInfo: UserInfo = JSON.parse(atob(credentials.id_token!.split(".")[1])); return { ...credentials, userInfo }; - }; + } /** * Looks in the database for any credentials object with the given user id, * and returns them. If the credentials are found but expired, the function will * automatically refresh the credentials and then resolve with the updated values. - * @param userId the id of the Dash user requesting his/her credentials. Eventually - * might have multiple. - * @returns the credentials and whether or not they were updated in the process + * @param userId the id of the Dash user requesting his/her credentials. Eventually, each user might + * be associated with multiple different sets of Google credentials. + * @returns the credentials, or undefined if the user has no stored associated credentials, + * and a flag indicating whether or not they were refreshed during retrieval */ - const retrieveCredentials = async (userId: string): Promise<CredentialsResult> => { - return new Promise<CredentialsResult>((resolve, reject) => { - Database.Auxiliary.GoogleAuthenticationToken.Fetch(userId).then(credentials => { - if (!credentials) { - return reject(); - } - if (credentials.expiry_date! < new Date().getTime()) { - // Token has expired, so submitting a request for a refreshed access token - return refreshAccessToken(credentials, userId).then(resolve, reject); - } - // Authentication successful! - resolve({ credentials, refreshed: false }); - }); - }); - }; + async function retrieveCredentials(userId: string): Promise<{ credentials: Opt<Credentials>, refreshed: boolean }> { + let credentials: Opt<Credentials> = await Database.Auxiliary.GoogleAuthenticationToken.Fetch(userId); + let refreshed = false; + if (!credentials) { + return { credentials: undefined, refreshed }; + } + // check for token expiry + if (credentials.expiry_date! <= new Date().getTime()) { + credentials = await refreshAccessToken(credentials, userId); + } + return { credentials, refreshed }; + } /** * This function submits a request to OAuth with the local refresh token @@ -334,26 +345,28 @@ export namespace GoogleApiServerUtils { * the Dash user id passed in. In addition to returning the credentials, it * writes the diff to the database. * @param credentials the credentials - * @param userId + * @param userId the id of the Dash user implicitly requesting that + * his/her credentials be refreshed + * @returns the updated credentials */ - const refreshAccessToken = (credentials: Credentials, userId: string) => { - return new Promise<CredentialsResult>(resolve => { - let headerParameters = { headers: { 'Content-Type': 'application/x-www-form-urlencoded' } }; - let queryParameters = { - refreshToken: credentials.refresh_token, - grant_type: "refresh_token", - ...installed - }; - let url = `${refreshEndpoint}?${qs.stringify(queryParameters)}`; - request.post(url, headerParameters).then(async response => { - let { access_token, expires_in } = JSON.parse(response); - const expiry_date = new Date().getTime() + (expires_in * 1000); - await Database.Auxiliary.GoogleAuthenticationToken.Update(userId, access_token, expiry_date); - credentials.access_token = access_token; - credentials.expiry_date = expiry_date; - resolve({ credentials, refreshed: true }); - }); + async function refreshAccessToken(credentials: Credentials, userId: string): Promise<Credentials> { + let headerParameters = { headers: { 'Content-Type': 'application/x-www-form-urlencoded' } }; + let url = `https://oauth2.googleapis.com/token?${qs.stringify({ + refreshToken: credentials.refresh_token, + grant_type: "refresh_token", + ...installed + })}`; + const { access_token, expires_in } = await new Promise<any>(async resolve => { + const response = await request.post(url, headerParameters); + resolve(JSON.parse(response)); }); - }; + // expires_in is in seconds, but we're building the new expiry date in milliseconds + const expiry_date = new Date().getTime() + (expires_in * 1000); + await Database.Auxiliary.GoogleAuthenticationToken.Update(userId, access_token, expiry_date); + // update the relevant properties + credentials.access_token = access_token; + credentials.expiry_date = expiry_date; + return credentials; + } }
\ No newline at end of file diff --git a/src/server/apis/google/GooglePhotosUploadUtils.ts b/src/server/apis/google/GooglePhotosUploadUtils.ts index 172fa8d46..d8cf795b5 100644 --- a/src/server/apis/google/GooglePhotosUploadUtils.ts +++ b/src/server/apis/google/GooglePhotosUploadUtils.ts @@ -1,58 +1,113 @@ import request = require('request-promise'); -import { GoogleApiServerUtils } from './GoogleApiServerUtils'; import * as path from 'path'; -import { MediaItemCreationResult } from './SharedTypes'; +import { MediaItemCreationResult, NewMediaItemResult } from './SharedTypes'; import { NewMediaItem } from "../../index"; import { BatchedArray, TimeUnit } from 'array-batcher'; import { DashUploadUtils } from '../../DashUploadUtils'; +/** + * This namespace encompasses the logic + * necessary to upload images to Google's server, + * and then initialize / create those images in the Photos + * API given the upload tokens returned from the initial + * uploading process. + * + * https://developers.google.com/photos/library/reference/rest/v1/mediaItems/batchCreate + */ export namespace GooglePhotosUploadUtils { - export interface Paths { - uploadDirectory: string; - credentialsPath: string; - tokenPath: string; - } - - export interface MediaInput { + /** + * Specifies the structure of the object + * necessary to upload bytes to Google's servers. + * The url is streamed to access the image's bytes, + * and the description is what appears in Google Photos' + * description field. + */ + export interface UploadSource { url: string; description: string; } - const prepend = (extension: string) => `https://photoslibrary.googleapis.com/v1/${extension}`; - const headers = (type: string, token: string) => ({ - 'Content-Type': `application/${type}`, - 'Authorization': `Bearer ${token}`, - }); + /** + * A utility function to streamline making + * calls to the API's url - accentuates + * the relative path in the caller. + * @param extension the desired + * subset of the API + */ + function prepend(extension: string): string { + return `https://photoslibrary.googleapis.com/v1/${extension}`; + } + + /** + * Factors out the creation of the API request's + * authentication elements stored in the header. + * @param type the contents of the request + * @param token the user-specific Google access token + */ + function headers(type: string, token: string) { + return { + 'Content-Type': `application/${type}`, + 'Authorization': `Bearer ${token}`, + }; + } - export const DispatchGooglePhotosUpload = async (bearerToken: string, url: string) => { - if (!DashUploadUtils.imageFormats.includes(path.extname(url))) { + /** + * This is the first step in the remote image creation process. + * Here we upload the raw bytes of the image to Google's servers by + * setting authentication and other required header properties and including + * the raw bytes to the image, to be uploaded, in the body of the request. + * @param bearerToken the user-specific Google access token, specifies the account associated + * with the eventual image creation + * @param url the url of the image to upload + * @param filename an optional name associated with the uploaded image - if not specified + * defaults to the filename (basename) in the url + */ + export const DispatchGooglePhotosUpload = async (bearerToken: string, url: string, filename?: string): Promise<any> => { + // check if the url points to a non-image or an unsupported format + if (!DashUploadUtils.validateExtension(url)) { return undefined; } - const body = await request(url, { encoding: null }); const parameters = { method: 'POST', + uri: prepend('uploads'), headers: { ...headers('octet-stream', bearerToken), - 'X-Goog-Upload-File-Name': path.basename(url), + 'X-Goog-Upload-File-Name': filename || path.basename(url), 'X-Goog-Upload-Protocol': 'raw' }, - uri: prepend('uploads'), - body + body: await request(url, { encoding: null }) // returns a readable stream with the unencoded binary image data }; - return new Promise<any>((resolve, reject) => request(parameters, (error, _response, body) => { + return new Promise((resolve, reject) => request(parameters, (error, _response, body) => { if (error) { - console.log(error); + // on rejection, the server logs the error and the offending image return reject(error); } resolve(body); })); }; - export const CreateMediaItems = async (bearerToken: string, newMediaItems: NewMediaItem[], album?: { id: string }): Promise<MediaItemCreationResult> => { - const newMediaItemResults = await BatchedArray.from(newMediaItems, { batchSize: 50 }).batchedMapPatientInterval( + /** + * This is the second step in the remote image creation process: having uploaded + * the raw bytes of the image and received / stored pointers (upload tokens) to those + * bytes, we can now instruct the API to finalize the creation of those images by + * submitting a batch create request with the list of upload tokens and the description + * to be associated with reach resulting new image. + * @param bearerToken the user-specific Google access token, specifies the account associated + * with the eventual image creation + * @param newMediaItems a list of objects containing a description and, effectively, the + * pointer to the uploaded bytes + * @param album if included, will add all of the newly created remote images to the album + * with the specified id + */ + export const CreateMediaItems = async (bearerToken: string, newMediaItems: NewMediaItem[], album?: { id: string }): Promise<NewMediaItemResult[]> => { + // it's important to note that the API can't handle more than 50 items in each request and + // seems to need at least some latency between requests (spamming it synchronously has led to the server returning errors)... + const batched = BatchedArray.from(newMediaItems, { batchSize: 50 }); + // ...so we execute them in delayed batches and await the entire execution + return batched.batchedMapPatientInterval( { magnitude: 100, unit: TimeUnit.Milliseconds }, - async (batch: NewMediaItem[]) => { + async (batch: NewMediaItem[], collector) => { const parameters = { method: 'POST', headers: headers('json', bearerToken), @@ -60,19 +115,19 @@ export namespace GooglePhotosUploadUtils { body: { newMediaItems: batch } as any, json: true }; + // register the target album, if provided album && (parameters.body.albumId = album.id); - return (await new Promise<MediaItemCreationResult>((resolve, reject) => { + collector.push(...(await new Promise<NewMediaItemResult[]>((resolve, reject) => { request(parameters, (error, _response, body) => { if (error) { reject(error); } else { - resolve(body); + resolve(body.newMediaItemResults); } }); - })).newMediaItemResults; + }))); } ); - return { newMediaItemResults }; }; }
\ No newline at end of file diff --git a/src/server/database.ts b/src/server/database.ts index 79dd26b7d..b81fc03a4 100644 --- a/src/server/database.ts +++ b/src/server/database.ts @@ -60,6 +60,10 @@ export namespace Database { constructor() { this.MongoClient.connect(url, (_err, client) => { + if (!client) { + console.error("\nPlease start MongoDB by running 'mongod' in a terminal before continuing...\n"); + process.exit(0); + } this.db = client.db(); this.onConnect.forEach(fn => fn()); }); diff --git a/src/server/index.ts b/src/server/index.ts index eb19c71a9..25697e71f 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -573,18 +573,15 @@ function routeSetter(router: RouteManager) { onValidation: async ({ req, res, user }) => { let sector: GoogleApiServerUtils.Service = req.params.sector as GoogleApiServerUtils.Service; let action: GoogleApiServerUtils.Action = req.params.action as GoogleApiServerUtils.Action; - return GoogleApiServerUtils.GetEndpoint(GoogleApiServerUtils.Service[sector], user.id).then(endpoint => { - let handler = EndpointHandlerMap.get(action); - if (endpoint && handler) { - let execute = handler(endpoint, req.body).then( - response => res.send(response.data), - rejection => res.send(rejection) - ); - execute.catch(exception => res.send(exception)); - return; - } - res.send(undefined); - }); + const endpoint = await GoogleApiServerUtils.GetEndpoint(GoogleApiServerUtils.Service[sector], user.id); + let handler = EndpointHandlerMap.get(action); + if (endpoint && handler) { + handler(endpoint, req.body) + .then(response => res.send(response.data)) + .catch(exception => res.send(exception)); + return; + } + res.send(undefined); } }); @@ -593,11 +590,11 @@ function routeSetter(router: RouteManager) { subscription: RouteStore.readGoogleAccessToken, onValidation: async ({ user, res }) => { const userId = user.id; - const token = await Database.Auxiliary.GoogleAuthenticationToken.Fetch(userId); + const token = await GoogleApiServerUtils.retrieveAccessToken(userId); if (!token) { - return res.send(await GoogleApiServerUtils.generateAuthenticationUrl()); + return res.send(GoogleApiServerUtils.generateAuthenticationUrl()); } - return GoogleApiServerUtils.retrieveAccessToken(userId).then(token => res.send(token)); + return res.send(token); } }); @@ -609,8 +606,14 @@ function routeSetter(router: RouteManager) { } }); - const tokenError = "Unable to successfully upload bytes for all images!"; + const authenticationError = "Unable to authenticate Google credentials before uploading to Google Photos!"; const mediaError = "Unable to convert all uploaded bytes to media items!"; + interface GooglePhotosUploadFailure { + batch: number; + index: number; + url: string; + reason: string; + } router.addSupervisedRoute({ method: Method.POST, @@ -618,35 +621,40 @@ function routeSetter(router: RouteManager) { onValidation: async ({ user, req, res }) => { const { media } = req.body; - let failed: number[] = []; const token = await GoogleApiServerUtils.retrieveAccessToken(user.id); - const newMediaItems = await BatchedArray.from<GooglePhotosUploadUtils.MediaInput>(media, { batchSize: 25 }).batchedMapPatientInterval( + if (!token) { + return _error(res, authenticationError); + } + + let failed: GooglePhotosUploadFailure[] = []; + const batched = BatchedArray.from<GooglePhotosUploadUtils.UploadSource>(media, { batchSize: 25 }); + const newMediaItems = await batched.batchedMapPatientInterval<NewMediaItem>( { magnitude: 100, unit: TimeUnit.Milliseconds }, - async (batch: GooglePhotosUploadUtils.MediaInput[]) => { - const newMediaItems: NewMediaItem[] = []; + async (batch, collector, { completedBatches }) => { for (let index = 0; index < batch.length; index++) { - const element = batch[index]; - const uploadToken = await GooglePhotosUploadUtils.DispatchGooglePhotosUpload(token, element.url); + const { url, description } = batch[index]; + const fail = (reason: string) => failed.push({ reason, batch: completedBatches + 1, index, url }); + const uploadToken = await GooglePhotosUploadUtils.DispatchGooglePhotosUpload(token, url).catch(fail); if (!uploadToken) { - failed.push(index); + fail(`${path.extname(url)} is not an accepted extension`); } else { - newMediaItems.push({ - description: element.description, + collector.push({ + description, simpleMediaItem: { uploadToken } }); } } - return newMediaItems; } ); const failedCount = failed.length; if (failedCount) { console.error(`Unable to upload ${failedCount} image${failedCount === 1 ? "" : "s"} to Google's servers`); + console.log(failed.map(({ reason, batch, index, url }) => `@${batch}.${index}: ${url} failed:\n${reason}`).join('\n\n')); } return GooglePhotosUploadUtils.CreateMediaItems(token, newMediaItems, req.body.album).then( - result => _success(res, { results: result.newMediaItemResults, failed }), + results => _success(res, { results, failed }), error => _error(res, mediaError, error) ); } |