aboutsummaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to 'src')
-rw-r--r--src/client/Network.ts12
-rw-r--r--src/client/apis/GoogleAuthenticationManager.tsx39
-rw-r--r--src/client/apis/google_docs/GoogleApiClientUtils.ts8
-rw-r--r--src/client/apis/google_docs/GooglePhotosClientUtils.ts10
-rw-r--r--src/client/util/Import & Export/DirectoryImportBox.tsx8
-rw-r--r--src/client/util/Import & Export/ImageUtils.ts4
-rw-r--r--src/new_fields/RichTextUtils.ts4
-rw-r--r--src/server/DashUploadUtils.ts6
-rw-r--r--src/server/RouteManager.ts5
-rw-r--r--src/server/RouteStore.ts3
-rw-r--r--src/server/apis/google/GoogleApiServerUtils.ts361
-rw-r--r--src/server/apis/google/GooglePhotosUploadUtils.ts113
-rw-r--r--src/server/database.ts4
-rw-r--r--src/server/index.ts62
14 files changed, 362 insertions, 277 deletions
diff --git a/src/client/Network.ts b/src/client/Network.ts
index 75ccb5e99..f9ef27267 100644
--- a/src/client/Network.ts
+++ b/src/client/Network.ts
@@ -1,18 +1,16 @@
import { Utils } from "../Utils";
-import { CurrentUserUtils } from "../server/authentication/models/current_user_utils";
import requestPromise = require('request-promise');
-export namespace Identified {
+export namespace Networking {
export async function FetchFromServer(relativeRoute: string) {
- return (await fetch(relativeRoute, { headers: { userId: CurrentUserUtils.id } })).text();
+ return (await fetch(relativeRoute)).text();
}
export async function PostToServer(relativeRoute: string, body?: any) {
let options = {
uri: Utils.prepend(relativeRoute),
method: "POST",
- headers: { userId: CurrentUserUtils.id },
body,
json: true
};
@@ -22,12 +20,10 @@ export namespace Identified {
export async function PostFormDataToServer(relativeRoute: string, formData: FormData) {
const parameters = {
method: 'POST',
- headers: { userId: CurrentUserUtils.id },
- body: formData,
+ body: formData
};
const response = await fetch(relativeRoute, parameters);
- const text = await response.json();
- return text;
+ return response.json();
}
} \ No newline at end of file
diff --git a/src/client/apis/GoogleAuthenticationManager.tsx b/src/client/apis/GoogleAuthenticationManager.tsx
index 01dac3996..1ec9d8412 100644
--- a/src/client/apis/GoogleAuthenticationManager.tsx
+++ b/src/client/apis/GoogleAuthenticationManager.tsx
@@ -3,7 +3,7 @@ import { observer } from "mobx-react";
import * as React from "react";
import MainViewModal from "../views/MainViewModal";
import { Opt } from "../../new_fields/Doc";
-import { Identified } from "../Network";
+import { Networking } from "../Network";
import { RouteStore } from "../../server/RouteStore";
import "./GoogleAuthenticationManager.scss";
@@ -31,7 +31,7 @@ export default class GoogleAuthenticationManager extends React.Component<{}> {
}
public fetchOrGenerateAccessToken = async () => {
- let response = await Identified.FetchFromServer(RouteStore.readGoogleAccessToken);
+ let response = await Networking.FetchFromServer(RouteStore.readGoogleAccessToken);
// if this is an authentication url, activate the UI to register the new access token
if (new RegExp(AuthenticationUrl).test(response)) {
this.isOpen = true;
@@ -39,24 +39,25 @@ export default class GoogleAuthenticationManager extends React.Component<{}> {
return new Promise<string>(async resolve => {
const disposer = reaction(
() => this.authenticationCode,
- authenticationCode => {
- if (authenticationCode) {
- Identified.PostToServer(RouteStore.writeGoogleAccessToken, { authenticationCode }).then(
- ({ access_token, avatar, name }) => {
- runInAction(() => {
- this.avatar = avatar;
- this.username = name;
- });
- this.beginFadeout();
- disposer();
- resolve(access_token);
- },
- action(() => {
- this.hasBeenClicked = false;
- this.success = false;
- })
- );
+ async authenticationCode => {
+ if (!authenticationCode) {
+ return;
}
+ const { access_token, avatar, name } = await Networking.PostToServer(
+ RouteStore.writeGoogleAccessToken,
+ { authenticationCode }
+ );
+ runInAction(() => {
+ this.avatar = avatar;
+ this.username = name;
+ });
+ this.beginFadeout();
+ disposer();
+ resolve(access_token);
+ action(() => {
+ this.hasBeenClicked = false;
+ this.success = false;
+ });
}
);
});
diff --git a/src/client/apis/google_docs/GoogleApiClientUtils.ts b/src/client/apis/google_docs/GoogleApiClientUtils.ts
index 1cf01fc3d..183679317 100644
--- a/src/client/apis/google_docs/GoogleApiClientUtils.ts
+++ b/src/client/apis/google_docs/GoogleApiClientUtils.ts
@@ -3,7 +3,7 @@ import { RouteStore } from "../../../server/RouteStore";
import { Opt } from "../../../new_fields/Doc";
import { isArray } from "util";
import { EditorState } from "prosemirror-state";
-import { Identified } from "../../Network";
+import { Networking } from "../../Network";
export const Pulls = "googleDocsPullCount";
export const Pushes = "googleDocsPushCount";
@@ -84,7 +84,7 @@ export namespace GoogleApiClientUtils {
}
};
try {
- const schema: docs_v1.Schema$Document = await Identified.PostToServer(path, parameters);
+ const schema: docs_v1.Schema$Document = await Networking.PostToServer(path, parameters);
return schema.documentId;
} catch {
return undefined;
@@ -157,7 +157,7 @@ export namespace GoogleApiClientUtils {
const path = `${RouteStore.googleDocs}/Documents/${Actions.Retrieve}`;
try {
const parameters = { documentId: options.documentId };
- const schema: RetrievalResult = await Identified.PostToServer(path, parameters);
+ const schema: RetrievalResult = await Networking.PostToServer(path, parameters);
return schema;
} catch {
return undefined;
@@ -173,7 +173,7 @@ export namespace GoogleApiClientUtils {
}
};
try {
- const replies: UpdateResult = await Identified.PostToServer(path, parameters);
+ const replies: UpdateResult = await Networking.PostToServer(path, parameters);
return replies;
} catch {
return undefined;
diff --git a/src/client/apis/google_docs/GooglePhotosClientUtils.ts b/src/client/apis/google_docs/GooglePhotosClientUtils.ts
index e93fa6eb4..402fc64b5 100644
--- a/src/client/apis/google_docs/GooglePhotosClientUtils.ts
+++ b/src/client/apis/google_docs/GooglePhotosClientUtils.ts
@@ -13,7 +13,7 @@ import { Docs, DocumentOptions } from "../../documents/Documents";
import { NewMediaItemResult, MediaItem } from "../../../server/apis/google/SharedTypes";
import { AssertionError } from "assert";
import { DocumentView } from "../../views/nodes/DocumentView";
-import { Identified } from "../../Network";
+import { Networking } from "../../Network";
import GoogleAuthenticationManager from "../GoogleAuthenticationManager";
export namespace GooglePhotos {
@@ -78,6 +78,7 @@ export namespace GooglePhotos {
}
export const CollectionToAlbum = async (options: AlbumCreationOptions): Promise<Opt<AlbumCreationResult>> => {
+ await GoogleAuthenticationManager.Instance.fetchOrGenerateAccessToken();
const { collection, title, descriptionKey, tag } = options;
const dataDocument = Doc.GetProto(collection);
const images = ((await DocListCastAsync(dataDocument.data)) || []).filter(doc => Cast(doc.data, ImageField));
@@ -127,6 +128,7 @@ export namespace GooglePhotos {
export type CollectionConstructor = (data: Array<Doc>, options: DocumentOptions, ...args: any) => Doc;
export const CollectionFromSearch = async (constructor: CollectionConstructor, requested: Opt<Partial<Query.SearchOptions>>): Promise<Doc> => {
+ await GoogleAuthenticationManager.Instance.fetchOrGenerateAccessToken();
let response = await Query.ContentSearch(requested);
let uploads = await Transactions.WriteMediaItemsToServer(response);
const children = uploads.map((upload: Transactions.UploadInformation) => {
@@ -147,6 +149,7 @@ export namespace GooglePhotos {
const comparator = (a: string, b: string) => (a < b) ? -1 : (a > b ? 1 : 0);
export const TagChildImages = async (collection: Doc) => {
+ await GoogleAuthenticationManager.Instance.fetchOrGenerateAccessToken();
const idMapping = await Cast(collection.googlePhotosIdMapping, Doc);
if (!idMapping) {
throw new Error("Appending image metadata requires that the targeted collection have already been mapped to an album!");
@@ -304,7 +307,7 @@ export namespace GooglePhotos {
};
export const WriteMediaItemsToServer = async (body: { mediaItems: any[] }): Promise<UploadInformation[]> => {
- const uploads = await Identified.PostToServer(RouteStore.googlePhotosMediaDownload, body);
+ const uploads = await Networking.PostToServer(RouteStore.googlePhotosMediaDownload, body);
return uploads;
};
@@ -325,6 +328,7 @@ export namespace GooglePhotos {
}
export const UploadImages = async (sources: Doc[], album?: AlbumReference, descriptionKey = "caption"): Promise<Opt<ImageUploadResults>> => {
+ await GoogleAuthenticationManager.Instance.fetchOrGenerateAccessToken();
if (album && "title" in album) {
album = await Create.Album(album.title);
}
@@ -341,7 +345,7 @@ export namespace GooglePhotos {
media.push({ url, description });
}
if (media.length) {
- const results = await Identified.PostToServer(RouteStore.googlePhotosMediaUpload, { media, album });
+ const results = await Networking.PostToServer(RouteStore.googlePhotosMediaUpload, { media, album });
return results;
}
};
diff --git a/src/client/util/Import & Export/DirectoryImportBox.tsx b/src/client/util/Import & Export/DirectoryImportBox.tsx
index d74b51993..bdd59cb16 100644
--- a/src/client/util/Import & Export/DirectoryImportBox.tsx
+++ b/src/client/util/Import & Export/DirectoryImportBox.tsx
@@ -20,9 +20,8 @@ import { listSpec } from "../../../new_fields/Schema";
import { GooglePhotos } from "../../apis/google_docs/GooglePhotosClientUtils";
import { SchemaHeaderField } from "../../../new_fields/SchemaHeaderField";
import "./DirectoryImportBox.scss";
-import { Identified } from "../../Network";
+import { Networking } from "../../Network";
import { BatchedArray } from "array-batcher";
-import { ExifData } from "exif";
const unsupported = ["text/html", "text/plain"];
@@ -108,7 +107,7 @@ export default class DirectoryImportBox extends React.Component<FieldViewProps>
runInAction(() => this.phase = `Internal: uploading ${this.quota - this.completed} files to Dash...`);
- const uploads = await BatchedArray.from(validated, { batchSize: 15 }).batchedMapAsync(async batch => {
+ const uploads = await BatchedArray.from(validated, { batchSize: 15 }).batchedMapAsync<ImageUploadResponse>(async (batch, collector) => {
const formData = new FormData();
batch.forEach(file => {
@@ -117,9 +116,8 @@ export default class DirectoryImportBox extends React.Component<FieldViewProps>
formData.append(Utils.GenerateGuid(), file);
});
- const responses = await Identified.PostFormDataToServer(RouteStore.upload, formData);
+ collector.push(...(await Networking.PostFormDataToServer(RouteStore.upload, formData)));
runInAction(() => this.completed += batch.length);
- return responses as ImageUploadResponse[];
});
await Promise.all(uploads.map(async upload => {
diff --git a/src/client/util/Import & Export/ImageUtils.ts b/src/client/util/Import & Export/ImageUtils.ts
index c9abf38fa..914f4870a 100644
--- a/src/client/util/Import & Export/ImageUtils.ts
+++ b/src/client/util/Import & Export/ImageUtils.ts
@@ -3,7 +3,7 @@ import { ImageField } from "../../../new_fields/URLField";
import { Cast, StrCast } from "../../../new_fields/Types";
import { RouteStore } from "../../../server/RouteStore";
import { Docs } from "../../documents/Documents";
-import { Identified } from "../../Network";
+import { Networking } from "../../Network";
import { Id } from "../../../new_fields/FieldSymbols";
import { Utils } from "../../../Utils";
@@ -15,7 +15,7 @@ export namespace ImageUtils {
return false;
}
const source = field.url.href;
- const response = await Identified.PostToServer(RouteStore.inspectImage, { source });
+ const response = await Networking.PostToServer(RouteStore.inspectImage, { source });
const { error, data } = response.exifData;
document.exif = error || Docs.Get.DocumentHierarchyFromJson(data);
return data !== undefined;
diff --git a/src/new_fields/RichTextUtils.ts b/src/new_fields/RichTextUtils.ts
index 601939ed2..63d718ce8 100644
--- a/src/new_fields/RichTextUtils.ts
+++ b/src/new_fields/RichTextUtils.ts
@@ -17,7 +17,7 @@ import { Cast, StrCast } from "./Types";
import { Id } from "./FieldSymbols";
import { DocumentView } from "../client/views/nodes/DocumentView";
import { AssertionError } from "assert";
-import { Identified } from "../client/Network";
+import { Networking } from "../client/Network";
export namespace RichTextUtils {
@@ -129,7 +129,7 @@ export namespace RichTextUtils {
return { baseUrl, filename };
});
- const uploads = await Identified.PostToServer(RouteStore.googlePhotosMediaDownload, { mediaItems });
+ const uploads = await Networking.PostToServer(RouteStore.googlePhotosMediaDownload, { mediaItems });
if (uploads.length !== mediaItems.length) {
throw new AssertionError({ expected: mediaItems.length, actual: uploads.length, message: "Error with internally uploading inlineObjects!" });
diff --git a/src/server/DashUploadUtils.ts b/src/server/DashUploadUtils.ts
index 46d897339..9fddb466c 100644
--- a/src/server/DashUploadUtils.ts
+++ b/src/server/DashUploadUtils.ts
@@ -24,9 +24,13 @@ export namespace DashUploadUtils {
const gifs = [".gif"];
const pngs = [".png"];
const jpgs = [".jpg", ".jpeg"];
- export const imageFormats = [...pngs, ...jpgs, ...gifs];
+ const imageFormats = [...pngs, ...jpgs, ...gifs];
const videoFormats = [".mov", ".mp4"];
+ export function validateExtension(url: string) {
+ return imageFormats.includes(path.extname(url).toLowerCase());
+ }
+
const size = "content-length";
const type = "content-type";
diff --git a/src/server/RouteManager.ts b/src/server/RouteManager.ts
index 21ce9c9e4..c1d38327f 100644
--- a/src/server/RouteManager.ts
+++ b/src/server/RouteManager.ts
@@ -49,9 +49,9 @@ export default class RouteManager {
let supervised = async (req: express.Request, res: express.Response) => {
const { user, originalUrl: target } = req;
const core = { req, res, isRelease };
- const tryExecute = async (target: (args: any) => any | Promise<any>, args: any) => {
+ const tryExecute = async (toExecute: (args: any) => any | Promise<any>, args: any) => {
try {
- await target(args);
+ await toExecute(args);
} catch (e) {
if (onError) {
onError({ ...core, error: e });
@@ -114,6 +114,7 @@ export const STATUS = {
};
export function _error(res: express.Response, message: string, error?: any) {
+ console.error(message);
res.statusMessage = message;
res.status(STATUS.EXECUTION_ERROR).send(error);
}
diff --git a/src/server/RouteStore.ts b/src/server/RouteStore.ts
index de2553b2f..a310d0c95 100644
--- a/src/server/RouteStore.ts
+++ b/src/server/RouteStore.ts
@@ -39,6 +39,7 @@ export enum RouteStore {
writeGoogleAccessToken = "/writeGoogleAccessToken",
googlePhotosMediaUpload = "/googlePhotosMediaUpload",
googlePhotosMediaDownload = "/googlePhotosMediaDownload",
- googleDocsGet = "/googleDocsGet"
+ googleDocsGet = "/googleDocsGet",
+ checkGoogle = "/checkGoogleAuthentication"
} \ No newline at end of file
diff --git a/src/server/apis/google/GoogleApiServerUtils.ts b/src/server/apis/google/GoogleApiServerUtils.ts
index b9984649e..35a2541a9 100644
--- a/src/server/apis/google/GoogleApiServerUtils.ts
+++ b/src/server/apis/google/GoogleApiServerUtils.ts
@@ -2,27 +2,19 @@ import { google } from "googleapis";
import { readFile } from "fs";
import { OAuth2Client, Credentials, OAuth2ClientOptions } from "google-auth-library";
import { Opt } from "../../../new_fields/Doc";
-import { GlobalOptions } from "googleapis-common";
import { GaxiosResponse } from "gaxios";
import request = require('request-promise');
import * as qs from 'query-string';
import { Database } from "../../database";
-import path from "path";
+import * as path from "path";
/**
- *
+ * Scopes give Google users fine granularity of control
+ * over the information they make accessible via the API.
+ * This is the somewhat overkill list of what Dash requests
+ * from the user.
*/
-const prefix = 'https://www.googleapis.com/auth/';
-
-/**
- *
- */
-const refreshEndpoint = "https://oauth2.googleapis.com/token";
-
-/**
- *
- */
-const SCOPES = [
+const scope = [
'documents.readonly',
'documents',
'presentations',
@@ -33,7 +25,7 @@ const SCOPES = [
'photoslibrary.appendonly',
'photoslibrary.sharing',
'userinfo.profile'
-];
+].map(relative => `https://www.googleapis.com/auth/${relative}`);
/**
* This namespace manages server side authentication for Google API queries, either
@@ -42,33 +34,9 @@ const SCOPES = [
export namespace GoogleApiServerUtils {
/**
- *
- */
- export interface CredentialsResult {
- credentials: Credentials;
- refreshed: boolean;
- }
-
- /**
- *
- */
- export interface UserInfo {
- at_hash: string;
- aud: string;
- azp: string;
- exp: number;
- family_name: string;
- given_name: string;
- iat: number;
- iss: string;
- locale: string;
- name: string;
- picture: string;
- sub: string;
- }
-
- /**
- *
+ * As we expand out to more Google APIs that are accessible from
+ * the 'googleapis' module imported above, this enum will record
+ * the list and provide a unified string representation of each API.
*/
export enum Service {
Documents = "Documents",
@@ -76,15 +44,10 @@ export namespace GoogleApiServerUtils {
}
/**
- *
- */
- export interface CredentialInformation {
- credentialsPath: string;
- userId: string;
- }
-
- /**
- *
+ * Global credentials read once from a JSON file
+ * before the server is started that
+ * allow us to build OAuth2 clients with Dash's
+ * application specific credentials.
*/
let installed: OAuth2ClientOptions;
@@ -99,27 +62,33 @@ export namespace GoogleApiServerUtils {
let worker: OAuth2Client;
/**
- *
+ * A briefer format for the response from a 'googleapis' API request
*/
export type ApiResponse = Promise<GaxiosResponse>;
/**
- *
+ * A generic form for a handler that executes some request on the endpoint
*/
export type ApiRouter = (endpoint: Endpoint, parameters: any) => ApiResponse;
/**
- *
+ * A generic form for the asynchronous function that actually submits the
+ * request to the API and returns the corresporing response. Helpful when
+ * making an extensible endpoint definition.
*/
export type ApiHandler = (parameters: any, methodOptions?: any) => ApiResponse;
/**
- *
+ * A literal union type indicating the valid actions for these 'googleapis'
+ * requestions
*/
export type Action = "create" | "retrieve" | "update";
/**
- *
+ * An interface defining any entity on which one can invoke
+ * anuy of the following handlers. All 'googleapis' wrappers
+ * such as google.docs().documents and google.slides().presentations
+ * satisfy this interface.
*/
export interface Endpoint {
get: ApiHandler;
@@ -128,15 +97,13 @@ export namespace GoogleApiServerUtils {
}
/**
- *
- */
- export type EndpointParameters = GlobalOptions & { version: "v1" };
-
- /**
- *
+ * This function is called once before the server is started,
+ * reading in Dash's project-specific credentials (client secret
+ * and client id) for later repeated access. It also sets up the
+ * global, intentionally unauthenticated worker OAuth2 client instance.
*/
- export const loadClientSecret = async () => {
- return new Promise<void>((resolve, reject) => {
+ export async function loadClientSecret(): Promise<void> {
+ return new Promise((resolve, reject) => {
readFile(path.join(__dirname, "../../credentials/google_docs_credentials.json"), async (err, projectCredentials) => {
if (err) {
reject(err);
@@ -153,89 +120,111 @@ export namespace GoogleApiServerUtils {
resolve();
});
});
- };
+ }
/**
- *
+ * Maps the Dash user id of a given user to their single
+ * associated OAuth2 client, mitigating the creation
+ * of needless duplicate clients that would arise from
+ * making one new client instance per request.
*/
const authenticationClients = new Map<String, OAuth2Client>();
/**
- *
- * @param sector
- * @param userId
+ * This function receives the target sector ("which G-Suite app's API am I interested in?")
+ * and the id of the Dash user making the request to the API. With this information, it generates
+ * an authenticated OAuth2 client and passes it into the relevant 'googleapis' wrapper.
+ * @param sector the particular desired G-Suite 'googleapis' API (docs, slides, etc.)
+ * @param userId the id of the Dash user making the request to the API
+ * @returns the relevant 'googleapis' wrapper, if any
*/
- export const GetEndpoint = (sector: string, userId: string) => {
- return new Promise<Opt<Endpoint>>(resolve => {
- retrieveOAuthClient(userId).then(auth => {
- let routed: Opt<Endpoint>;
- let parameters: EndpointParameters = { auth, version: "v1" };
- switch (sector) {
- case Service.Documents:
- routed = google.docs(parameters).documents;
- break;
- case Service.Slides:
- routed = google.slides(parameters).presentations;
- break;
- }
- resolve(routed);
- });
+ export async function GetEndpoint(sector: string, userId: string): Promise<Opt<Endpoint>> {
+ return new Promise(async resolve => {
+ const auth = await retrieveOAuthClient(userId);
+ if (!auth) {
+ return resolve();
+ }
+ let routed: Opt<Endpoint>;
+ let parameters: any = { auth, version: "v1" };
+ switch (sector) {
+ case Service.Documents:
+ routed = google.docs(parameters).documents;
+ break;
+ case Service.Slides:
+ routed = google.slides(parameters).presentations;
+ break;
+ }
+ resolve(routed);
});
- };
+ }
/**
- *
- * @param userId
+ * Returns the lengthy string or access token that can be passed into
+ * the headers of an API request or into the constructor of the Photos
+ * client API wrapper.
+ * @param userId the Dash user id of the user requesting his/her associated
+ * access_token
+ * @returns the current access_token associated with the requesting
+ * Dash user. The access_token is valid for only an hour, and
+ * is then refreshed.
*/
- export const retrieveAccessToken = (userId: string): Promise<string> => {
- return new Promise<string>((resolve, reject) => {
- retrieveCredentials(userId).then(
- ({ credentials }) => resolve(credentials.access_token!),
- error => reject(`Error: unable to authenticate Google Photos API request.\n${error}`)
- );
+ export async function retrieveAccessToken(userId: string): Promise<string> {
+ return new Promise(async resolve => {
+ const { credentials } = await retrieveCredentials(userId);
+ if (!credentials) {
+ return resolve();
+ }
+ resolve(credentials.access_token!);
});
- };
+ }
/**
- *
- * @param userId
+ * Manipulates a mapping such that, in the limit, each Dash user has
+ * an associated authenticated OAuth2 client at their disposal. This
+ * function ensures that the client's credentials always remain up to date
+ * @param userId the Dash user id of the user requesting account integration
+ * @returns returns an initialized OAuth2 client instance, likely to be passed into Google's
+ * npm-installed API wrappers that use authenticated client instances rather than access codes for
+ * security.
*/
- export const retrieveOAuthClient = (userId: string): Promise<OAuth2Client> => {
- return new Promise<OAuth2Client>((resolve, reject) => {
- retrieveCredentials(userId).then(
- ({ credentials, refreshed }) => {
- let client = authenticationClients.get(userId);
- if (!client) {
- authenticationClients.set(userId, client = generateClient(credentials));
- } else if (refreshed) {
- client.setCredentials(credentials);
- }
- resolve(client);
- },
- error => reject(`Error: unable to instantiate and certify a new OAuth2 client.\n${error}`)
- );
+ export async function retrieveOAuthClient(userId: string): Promise<OAuth2Client> {
+ return new Promise(async resolve => {
+ const { credentials, refreshed } = await retrieveCredentials(userId);
+ if (!credentials) {
+ return resolve();
+ }
+ let client = authenticationClients.get(userId);
+ if (!client) {
+ authenticationClients.set(userId, client = generateClient(credentials));
+ } else if (refreshed) {
+ client.setCredentials(credentials);
+ }
+ resolve(client);
});
- };
+ }
/**
- *
- * @param credentials
+ * Creates a new OAuth2Client instance, and if provided, sets
+ * the specific credentials on the client
+ * @param credentials if you have access to the credentials that you'll eventually set on
+ * the client, just pass them in at initialization
+ * @returns the newly created, potentially certified, OAuth2 client instance
*/
- function generateClient(credentials?: Credentials) {
+ function generateClient(credentials?: Credentials): OAuth2Client {
const client = new google.auth.OAuth2(installed);
credentials && client.setCredentials(credentials);
return client;
}
/**
- *
+ * Calls on the worker (which does not have and does not need
+ * any credentials) to produce a url to which the user can
+ * navigate to give Dash the necessary Google permissions.
+ * @returns the newly generated url to the authentication landing page
*/
- export const generateAuthenticationUrl = async () => {
- return worker.generateAuthUrl({
- access_type: 'offline',
- scope: SCOPES.map(relative => prefix + relative),
- });
- };
+ export function generateAuthenticationUrl(): string {
+ return worker.generateAuthUrl({ scope, access_type: 'offline' });
+ }
/**
* This is what we return to the server in processNewUser(), after the
@@ -255,32 +244,36 @@ export namespace GoogleApiServerUtils {
* and the authentication code to fetch the full set of credentials that
* we'll store in the database for each user. This is called once per
* new account integration.
- * @param userId The Dash user id of the user requesting account integration, used to associate the new credentials
+ * @param userId the Dash user id of the user requesting account integration, used to associate the new credentials
* with a Dash user in the googleAuthentication table of the database.
* @param authenticationCode the Google-provided authentication code that the user copied
* from Google's permissions UI and pasted into the overlay.
+ *
+ * EXAMPLE CODE: 4/sgF2A5uGg4xASHf7VQDnLtdqo3mUlfQqLSce_HYz5qf1nFtHj9YTeGs
+ *
* @returns the information necessary to authenticate a client side google photos request
* and display basic user information in the overlay on successful authentication.
* This can be expanded as needed by adding properties to the interface GoogleAuthenticationResult.
*/
- export const processNewUser = async (userId: string, authenticationCode: string): Promise<GoogleAuthenticationResult> => {
- return new Promise<GoogleAuthenticationResult>((resolve, reject) => {
+ export async function processNewUser(userId: string, authenticationCode: string): Promise<GoogleAuthenticationResult> {
+ const credentials = await new Promise<Credentials>((resolve, reject) => {
worker.getToken(authenticationCode, async (err, credentials) => {
if (err || !credentials) {
reject(err);
- return console.error('Error retrieving access token', err);
+ return;
}
- const enriched = injectUserInfo(credentials);
- await Database.Auxiliary.GoogleAuthenticationToken.Write(userId, enriched);
- const { given_name, picture } = enriched.userInfo;
- resolve({
- access_token: enriched.access_token!,
- avatar: picture,
- name: given_name
- });
+ resolve(credentials);
});
});
- };
+ const enriched = injectUserInfo(credentials);
+ await Database.Auxiliary.GoogleAuthenticationToken.Write(userId, enriched);
+ const { given_name, picture } = enriched.userInfo;
+ return {
+ access_token: enriched.access_token!,
+ avatar: picture,
+ name: given_name
+ };
+ }
/**
* This type represents the union of the full set of OAuth2 credentials
@@ -290,6 +283,26 @@ export namespace GoogleApiServerUtils {
export type EnrichedCredentials = Credentials & { userInfo: UserInfo };
/**
+ * This interface defines all of the information we
+ * receive from parsing the base64 encoded info-token
+ * for a Google user.
+ */
+ export interface UserInfo {
+ at_hash: string;
+ aud: string;
+ azp: string;
+ exp: number;
+ family_name: string;
+ given_name: string;
+ iat: number;
+ iss: string;
+ locale: string;
+ name: string;
+ picture: string;
+ sub: string;
+ }
+
+ /**
* It's pretty cool: the credentials id_token is split into thirds by periods.
* The middle third contains a base64-encoded JSON string with all the
* user info contained in the interface below. So, we isolate that middle third,
@@ -299,34 +312,32 @@ export namespace GoogleApiServerUtils {
* @returns the full set of credentials in the structure in which they'll be stored
* in the database.
*/
- const injectUserInfo = (credentials: Credentials): EnrichedCredentials => {
- const userInfo = JSON.parse(atob(credentials.id_token!.split(".")[1]));
+ function injectUserInfo(credentials: Credentials): EnrichedCredentials {
+ const userInfo: UserInfo = JSON.parse(atob(credentials.id_token!.split(".")[1]));
return { ...credentials, userInfo };
- };
+ }
/**
* Looks in the database for any credentials object with the given user id,
* and returns them. If the credentials are found but expired, the function will
* automatically refresh the credentials and then resolve with the updated values.
- * @param userId the id of the Dash user requesting his/her credentials. Eventually
- * might have multiple.
- * @returns the credentials and whether or not they were updated in the process
+ * @param userId the id of the Dash user requesting his/her credentials. Eventually, each user might
+ * be associated with multiple different sets of Google credentials.
+ * @returns the credentials, or undefined if the user has no stored associated credentials,
+ * and a flag indicating whether or not they were refreshed during retrieval
*/
- const retrieveCredentials = async (userId: string): Promise<CredentialsResult> => {
- return new Promise<CredentialsResult>((resolve, reject) => {
- Database.Auxiliary.GoogleAuthenticationToken.Fetch(userId).then(credentials => {
- if (!credentials) {
- return reject();
- }
- if (credentials.expiry_date! < new Date().getTime()) {
- // Token has expired, so submitting a request for a refreshed access token
- return refreshAccessToken(credentials, userId).then(resolve, reject);
- }
- // Authentication successful!
- resolve({ credentials, refreshed: false });
- });
- });
- };
+ async function retrieveCredentials(userId: string): Promise<{ credentials: Opt<Credentials>, refreshed: boolean }> {
+ let credentials: Opt<Credentials> = await Database.Auxiliary.GoogleAuthenticationToken.Fetch(userId);
+ let refreshed = false;
+ if (!credentials) {
+ return { credentials: undefined, refreshed };
+ }
+ // check for token expiry
+ if (credentials.expiry_date! <= new Date().getTime()) {
+ credentials = await refreshAccessToken(credentials, userId);
+ }
+ return { credentials, refreshed };
+ }
/**
* This function submits a request to OAuth with the local refresh token
@@ -334,26 +345,28 @@ export namespace GoogleApiServerUtils {
* the Dash user id passed in. In addition to returning the credentials, it
* writes the diff to the database.
* @param credentials the credentials
- * @param userId
+ * @param userId the id of the Dash user implicitly requesting that
+ * his/her credentials be refreshed
+ * @returns the updated credentials
*/
- const refreshAccessToken = (credentials: Credentials, userId: string) => {
- return new Promise<CredentialsResult>(resolve => {
- let headerParameters = { headers: { 'Content-Type': 'application/x-www-form-urlencoded' } };
- let queryParameters = {
- refreshToken: credentials.refresh_token,
- grant_type: "refresh_token",
- ...installed
- };
- let url = `${refreshEndpoint}?${qs.stringify(queryParameters)}`;
- request.post(url, headerParameters).then(async response => {
- let { access_token, expires_in } = JSON.parse(response);
- const expiry_date = new Date().getTime() + (expires_in * 1000);
- await Database.Auxiliary.GoogleAuthenticationToken.Update(userId, access_token, expiry_date);
- credentials.access_token = access_token;
- credentials.expiry_date = expiry_date;
- resolve({ credentials, refreshed: true });
- });
+ async function refreshAccessToken(credentials: Credentials, userId: string): Promise<Credentials> {
+ let headerParameters = { headers: { 'Content-Type': 'application/x-www-form-urlencoded' } };
+ let url = `https://oauth2.googleapis.com/token?${qs.stringify({
+ refreshToken: credentials.refresh_token,
+ grant_type: "refresh_token",
+ ...installed
+ })}`;
+ const { access_token, expires_in } = await new Promise<any>(async resolve => {
+ const response = await request.post(url, headerParameters);
+ resolve(JSON.parse(response));
});
- };
+ // expires_in is in seconds, but we're building the new expiry date in milliseconds
+ const expiry_date = new Date().getTime() + (expires_in * 1000);
+ await Database.Auxiliary.GoogleAuthenticationToken.Update(userId, access_token, expiry_date);
+ // update the relevant properties
+ credentials.access_token = access_token;
+ credentials.expiry_date = expiry_date;
+ return credentials;
+ }
} \ No newline at end of file
diff --git a/src/server/apis/google/GooglePhotosUploadUtils.ts b/src/server/apis/google/GooglePhotosUploadUtils.ts
index 172fa8d46..d8cf795b5 100644
--- a/src/server/apis/google/GooglePhotosUploadUtils.ts
+++ b/src/server/apis/google/GooglePhotosUploadUtils.ts
@@ -1,58 +1,113 @@
import request = require('request-promise');
-import { GoogleApiServerUtils } from './GoogleApiServerUtils';
import * as path from 'path';
-import { MediaItemCreationResult } from './SharedTypes';
+import { MediaItemCreationResult, NewMediaItemResult } from './SharedTypes';
import { NewMediaItem } from "../../index";
import { BatchedArray, TimeUnit } from 'array-batcher';
import { DashUploadUtils } from '../../DashUploadUtils';
+/**
+ * This namespace encompasses the logic
+ * necessary to upload images to Google's server,
+ * and then initialize / create those images in the Photos
+ * API given the upload tokens returned from the initial
+ * uploading process.
+ *
+ * https://developers.google.com/photos/library/reference/rest/v1/mediaItems/batchCreate
+ */
export namespace GooglePhotosUploadUtils {
- export interface Paths {
- uploadDirectory: string;
- credentialsPath: string;
- tokenPath: string;
- }
-
- export interface MediaInput {
+ /**
+ * Specifies the structure of the object
+ * necessary to upload bytes to Google's servers.
+ * The url is streamed to access the image's bytes,
+ * and the description is what appears in Google Photos'
+ * description field.
+ */
+ export interface UploadSource {
url: string;
description: string;
}
- const prepend = (extension: string) => `https://photoslibrary.googleapis.com/v1/${extension}`;
- const headers = (type: string, token: string) => ({
- 'Content-Type': `application/${type}`,
- 'Authorization': `Bearer ${token}`,
- });
+ /**
+ * A utility function to streamline making
+ * calls to the API's url - accentuates
+ * the relative path in the caller.
+ * @param extension the desired
+ * subset of the API
+ */
+ function prepend(extension: string): string {
+ return `https://photoslibrary.googleapis.com/v1/${extension}`;
+ }
+
+ /**
+ * Factors out the creation of the API request's
+ * authentication elements stored in the header.
+ * @param type the contents of the request
+ * @param token the user-specific Google access token
+ */
+ function headers(type: string, token: string) {
+ return {
+ 'Content-Type': `application/${type}`,
+ 'Authorization': `Bearer ${token}`,
+ };
+ }
- export const DispatchGooglePhotosUpload = async (bearerToken: string, url: string) => {
- if (!DashUploadUtils.imageFormats.includes(path.extname(url))) {
+ /**
+ * This is the first step in the remote image creation process.
+ * Here we upload the raw bytes of the image to Google's servers by
+ * setting authentication and other required header properties and including
+ * the raw bytes to the image, to be uploaded, in the body of the request.
+ * @param bearerToken the user-specific Google access token, specifies the account associated
+ * with the eventual image creation
+ * @param url the url of the image to upload
+ * @param filename an optional name associated with the uploaded image - if not specified
+ * defaults to the filename (basename) in the url
+ */
+ export const DispatchGooglePhotosUpload = async (bearerToken: string, url: string, filename?: string): Promise<any> => {
+ // check if the url points to a non-image or an unsupported format
+ if (!DashUploadUtils.validateExtension(url)) {
return undefined;
}
- const body = await request(url, { encoding: null });
const parameters = {
method: 'POST',
+ uri: prepend('uploads'),
headers: {
...headers('octet-stream', bearerToken),
- 'X-Goog-Upload-File-Name': path.basename(url),
+ 'X-Goog-Upload-File-Name': filename || path.basename(url),
'X-Goog-Upload-Protocol': 'raw'
},
- uri: prepend('uploads'),
- body
+ body: await request(url, { encoding: null }) // returns a readable stream with the unencoded binary image data
};
- return new Promise<any>((resolve, reject) => request(parameters, (error, _response, body) => {
+ return new Promise((resolve, reject) => request(parameters, (error, _response, body) => {
if (error) {
- console.log(error);
+ // on rejection, the server logs the error and the offending image
return reject(error);
}
resolve(body);
}));
};
- export const CreateMediaItems = async (bearerToken: string, newMediaItems: NewMediaItem[], album?: { id: string }): Promise<MediaItemCreationResult> => {
- const newMediaItemResults = await BatchedArray.from(newMediaItems, { batchSize: 50 }).batchedMapPatientInterval(
+ /**
+ * This is the second step in the remote image creation process: having uploaded
+ * the raw bytes of the image and received / stored pointers (upload tokens) to those
+ * bytes, we can now instruct the API to finalize the creation of those images by
+ * submitting a batch create request with the list of upload tokens and the description
+ * to be associated with reach resulting new image.
+ * @param bearerToken the user-specific Google access token, specifies the account associated
+ * with the eventual image creation
+ * @param newMediaItems a list of objects containing a description and, effectively, the
+ * pointer to the uploaded bytes
+ * @param album if included, will add all of the newly created remote images to the album
+ * with the specified id
+ */
+ export const CreateMediaItems = async (bearerToken: string, newMediaItems: NewMediaItem[], album?: { id: string }): Promise<NewMediaItemResult[]> => {
+ // it's important to note that the API can't handle more than 50 items in each request and
+ // seems to need at least some latency between requests (spamming it synchronously has led to the server returning errors)...
+ const batched = BatchedArray.from(newMediaItems, { batchSize: 50 });
+ // ...so we execute them in delayed batches and await the entire execution
+ return batched.batchedMapPatientInterval(
{ magnitude: 100, unit: TimeUnit.Milliseconds },
- async (batch: NewMediaItem[]) => {
+ async (batch: NewMediaItem[], collector) => {
const parameters = {
method: 'POST',
headers: headers('json', bearerToken),
@@ -60,19 +115,19 @@ export namespace GooglePhotosUploadUtils {
body: { newMediaItems: batch } as any,
json: true
};
+ // register the target album, if provided
album && (parameters.body.albumId = album.id);
- return (await new Promise<MediaItemCreationResult>((resolve, reject) => {
+ collector.push(...(await new Promise<NewMediaItemResult[]>((resolve, reject) => {
request(parameters, (error, _response, body) => {
if (error) {
reject(error);
} else {
- resolve(body);
+ resolve(body.newMediaItemResults);
}
});
- })).newMediaItemResults;
+ })));
}
);
- return { newMediaItemResults };
};
} \ No newline at end of file
diff --git a/src/server/database.ts b/src/server/database.ts
index 79dd26b7d..b81fc03a4 100644
--- a/src/server/database.ts
+++ b/src/server/database.ts
@@ -60,6 +60,10 @@ export namespace Database {
constructor() {
this.MongoClient.connect(url, (_err, client) => {
+ if (!client) {
+ console.error("\nPlease start MongoDB by running 'mongod' in a terminal before continuing...\n");
+ process.exit(0);
+ }
this.db = client.db();
this.onConnect.forEach(fn => fn());
});
diff --git a/src/server/index.ts b/src/server/index.ts
index eb19c71a9..25697e71f 100644
--- a/src/server/index.ts
+++ b/src/server/index.ts
@@ -573,18 +573,15 @@ function routeSetter(router: RouteManager) {
onValidation: async ({ req, res, user }) => {
let sector: GoogleApiServerUtils.Service = req.params.sector as GoogleApiServerUtils.Service;
let action: GoogleApiServerUtils.Action = req.params.action as GoogleApiServerUtils.Action;
- return GoogleApiServerUtils.GetEndpoint(GoogleApiServerUtils.Service[sector], user.id).then(endpoint => {
- let handler = EndpointHandlerMap.get(action);
- if (endpoint && handler) {
- let execute = handler(endpoint, req.body).then(
- response => res.send(response.data),
- rejection => res.send(rejection)
- );
- execute.catch(exception => res.send(exception));
- return;
- }
- res.send(undefined);
- });
+ const endpoint = await GoogleApiServerUtils.GetEndpoint(GoogleApiServerUtils.Service[sector], user.id);
+ let handler = EndpointHandlerMap.get(action);
+ if (endpoint && handler) {
+ handler(endpoint, req.body)
+ .then(response => res.send(response.data))
+ .catch(exception => res.send(exception));
+ return;
+ }
+ res.send(undefined);
}
});
@@ -593,11 +590,11 @@ function routeSetter(router: RouteManager) {
subscription: RouteStore.readGoogleAccessToken,
onValidation: async ({ user, res }) => {
const userId = user.id;
- const token = await Database.Auxiliary.GoogleAuthenticationToken.Fetch(userId);
+ const token = await GoogleApiServerUtils.retrieveAccessToken(userId);
if (!token) {
- return res.send(await GoogleApiServerUtils.generateAuthenticationUrl());
+ return res.send(GoogleApiServerUtils.generateAuthenticationUrl());
}
- return GoogleApiServerUtils.retrieveAccessToken(userId).then(token => res.send(token));
+ return res.send(token);
}
});
@@ -609,8 +606,14 @@ function routeSetter(router: RouteManager) {
}
});
- const tokenError = "Unable to successfully upload bytes for all images!";
+ const authenticationError = "Unable to authenticate Google credentials before uploading to Google Photos!";
const mediaError = "Unable to convert all uploaded bytes to media items!";
+ interface GooglePhotosUploadFailure {
+ batch: number;
+ index: number;
+ url: string;
+ reason: string;
+ }
router.addSupervisedRoute({
method: Method.POST,
@@ -618,35 +621,40 @@ function routeSetter(router: RouteManager) {
onValidation: async ({ user, req, res }) => {
const { media } = req.body;
- let failed: number[] = [];
const token = await GoogleApiServerUtils.retrieveAccessToken(user.id);
- const newMediaItems = await BatchedArray.from<GooglePhotosUploadUtils.MediaInput>(media, { batchSize: 25 }).batchedMapPatientInterval(
+ if (!token) {
+ return _error(res, authenticationError);
+ }
+
+ let failed: GooglePhotosUploadFailure[] = [];
+ const batched = BatchedArray.from<GooglePhotosUploadUtils.UploadSource>(media, { batchSize: 25 });
+ const newMediaItems = await batched.batchedMapPatientInterval<NewMediaItem>(
{ magnitude: 100, unit: TimeUnit.Milliseconds },
- async (batch: GooglePhotosUploadUtils.MediaInput[]) => {
- const newMediaItems: NewMediaItem[] = [];
+ async (batch, collector, { completedBatches }) => {
for (let index = 0; index < batch.length; index++) {
- const element = batch[index];
- const uploadToken = await GooglePhotosUploadUtils.DispatchGooglePhotosUpload(token, element.url);
+ const { url, description } = batch[index];
+ const fail = (reason: string) => failed.push({ reason, batch: completedBatches + 1, index, url });
+ const uploadToken = await GooglePhotosUploadUtils.DispatchGooglePhotosUpload(token, url).catch(fail);
if (!uploadToken) {
- failed.push(index);
+ fail(`${path.extname(url)} is not an accepted extension`);
} else {
- newMediaItems.push({
- description: element.description,
+ collector.push({
+ description,
simpleMediaItem: { uploadToken }
});
}
}
- return newMediaItems;
}
);
const failedCount = failed.length;
if (failedCount) {
console.error(`Unable to upload ${failedCount} image${failedCount === 1 ? "" : "s"} to Google's servers`);
+ console.log(failed.map(({ reason, batch, index, url }) => `@${batch}.${index}: ${url} failed:\n${reason}`).join('\n\n'));
}
return GooglePhotosUploadUtils.CreateMediaItems(token, newMediaItems, req.body.album).then(
- result => _success(res, { results: result.newMediaItemResults, failed }),
+ results => _success(res, { results, failed }),
error => _error(res, mediaError, error)
);
}
generated by cgit v1.2.3-70-g09d2 (git 2.46.0) at 2025-10-07 04:23:45 +0000