From 77e08a4362ba8fab4cab361fcb472702c97edf15 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Sun, 1 Sep 2019 13:05:54 -0400 Subject: initial commit --- src/server/apis/google/GoogleApiServerUtils.ts | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) (limited to 'src/server/apis/google/GoogleApiServerUtils.ts') diff --git a/src/server/apis/google/GoogleApiServerUtils.ts b/src/server/apis/google/GoogleApiServerUtils.ts index 8785cd974..2fb44d9a2 100644 --- a/src/server/apis/google/GoogleApiServerUtils.ts +++ b/src/server/apis/google/GoogleApiServerUtils.ts @@ -5,6 +5,7 @@ import { OAuth2Client } from "google-auth-library"; import { Opt } from "../../../new_fields/Doc"; import { GlobalOptions } from "googleapis-common"; import { GaxiosResponse } from "gaxios"; +import Photos = require("googlephotos"); /** * Server side authentication for Google Api queries. @@ -20,16 +21,18 @@ export namespace GoogleApiServerUtils { 'presentations.readonly', 'drive', 'drive.file', + 'photoslibrary', + 'photoslibrary.sharing' ]; export const parseBuffer = (data: Buffer) => JSON.parse(data.toString()); export enum Service { Documents = "Documents", - Slides = "Slides" + Slides = "Slides", + Photos = "Photos" } - export interface CredentialPaths { credentials: string; token: string; @@ -50,7 +53,7 @@ export namespace GoogleApiServerUtils { reject(err); return console.log('Error loading client secret file:', err); } - return authorize(parseBuffer(credentials), paths.token).then(auth => { + return authorize(parseBuffer(credentials), paths.token).then(async auth => { let routed: Opt; let parameters: EndpointParameters = { auth, version: "v1" }; switch (sector) { @@ -60,6 +63,10 @@ export namespace GoogleApiServerUtils { case Service.Slides: routed = google.slides(parameters).presentations; break; + case Service.Photos: + const photos = new Photos(auth); + let response = await photos.albums.list(); + console.log("WE GOT SOMETHING!", response); } resolve(routed); }); -- cgit v1.2.3-70-g09d2 From 5e12b7d816f1778af112ce69f3029e2f4a72bb08 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Sun, 1 Sep 2019 14:03:32 -0400 Subject: authentication working --- src/server/apis/google/GoogleApiServerUtils.ts | 25 ++++++++-------- src/server/apis/google/GooglePhotosUtils.ts | 22 +++++++------- src/server/authentication/config/passport.ts | 13 -------- src/server/credentials/auth.json | 12 -------- .../credentials/google_photos_credentials.ts | 35 ---------------------- src/server/index.ts | 11 ++++--- 6 files changed, 29 insertions(+), 89 deletions(-) delete mode 100644 src/server/credentials/auth.json delete mode 100644 src/server/credentials/google_photos_credentials.ts (limited to 'src/server/apis/google/GoogleApiServerUtils.ts') diff --git a/src/server/apis/google/GoogleApiServerUtils.ts b/src/server/apis/google/GoogleApiServerUtils.ts index 2fb44d9a2..c1bd3300e 100644 --- a/src/server/apis/google/GoogleApiServerUtils.ts +++ b/src/server/apis/google/GoogleApiServerUtils.ts @@ -1,7 +1,7 @@ import { google, docs_v1, slides_v1 } from "googleapis"; import { createInterface } from "readline"; import { readFile, writeFile } from "fs"; -import { OAuth2Client } from "google-auth-library"; +import { OAuth2Client, Credentials } from "google-auth-library"; import { Opt } from "../../../new_fields/Doc"; import { GlobalOptions } from "googleapis-common"; import { GaxiosResponse } from "gaxios"; @@ -48,14 +48,14 @@ export namespace GoogleApiServerUtils { export const GetEndpoint = async (sector: string, paths: CredentialPaths) => { return new Promise>((resolve, reject) => { - readFile(paths.credentials, (err, credentials) => { + readFile(paths.credentials, async (err, credentials) => { if (err) { reject(err); return console.log('Error loading client secret file:', err); } - return authorize(parseBuffer(credentials), paths.token).then(async auth => { + authorize(parseBuffer(credentials), paths.token).then(async result => { let routed: Opt; - let parameters: EndpointParameters = { auth, version: "v1" }; + let parameters: EndpointParameters = { auth: result.client, version: "v1" }; switch (sector) { case Service.Documents: routed = google.docs(parameters).documents; @@ -64,7 +64,7 @@ export namespace GoogleApiServerUtils { routed = google.slides(parameters).presentations; break; case Service.Photos: - const photos = new Photos(auth); + const photos = new Photos(result.token.access_token); let response = await photos.albums.list(); console.log("WE GOT SOMETHING!", response); } @@ -74,24 +74,25 @@ export namespace GoogleApiServerUtils { }); }; - + type TokenResult = { token: Credentials, client: OAuth2Client }; /** * Create an OAuth2 client with the given credentials, and returns the promise resolving to the authenticated client * @param {Object} credentials The authorization client credentials. */ - export function authorize(credentials: any, token_path: string): Promise { + export function authorize(credentials: any, token_path: string): Promise { const { client_secret, client_id, redirect_uris } = credentials.installed; const oAuth2Client = new google.auth.OAuth2( client_id, client_secret, redirect_uris[0]); - return new Promise((resolve, reject) => { + return new Promise((resolve, reject) => { readFile(token_path, (err, token) => { // Check if we have previously stored a token. if (err) { return getNewToken(oAuth2Client, token_path).then(resolve, reject); } - oAuth2Client.setCredentials(parseBuffer(token)); - resolve(oAuth2Client); + let parsed = parseBuffer(token); + oAuth2Client.setCredentials(parsed); + resolve({ token: parsed, client: oAuth2Client }); }); }); } @@ -103,7 +104,7 @@ export namespace GoogleApiServerUtils { * @param {getEventsCallback} callback The callback for the authorized client. */ function getNewToken(oAuth2Client: OAuth2Client, token_path: string) { - return new Promise((resolve, reject) => { + return new Promise((resolve, reject) => { const authUrl = oAuth2Client.generateAuthUrl({ access_type: 'offline', scope: SCOPES.map(relative => prefix + relative), @@ -129,7 +130,7 @@ export namespace GoogleApiServerUtils { } console.log('Token stored to', token_path); }); - resolve(oAuth2Client); + resolve({ token, client: oAuth2Client }); }); }); }); diff --git a/src/server/apis/google/GooglePhotosUtils.ts b/src/server/apis/google/GooglePhotosUtils.ts index c33ad2dd9..7f9ffb6f3 100644 --- a/src/server/apis/google/GooglePhotosUtils.ts +++ b/src/server/apis/google/GooglePhotosUtils.ts @@ -1,12 +1,12 @@ -import request = require('request-promise'); -const key = require("../../credentials/auth.json"); +// import request = require('request-promise'); +// const key = require("../../credentials/auth.json"); -export const PhotosLibraryQuery = async (authToken: any, parameters: any) => { - let options = { - headers: { 'Content-Type': 'application/json' }, - json: parameters, - auth: { 'bearer': authToken }, - }; - const result = await request.post(config.apiEndpoint + '/v1/mediaItems:search', options); - return result; -}; \ No newline at end of file +// export const PhotosLibraryQuery = async (authToken: any, parameters: any) => { +// let options = { +// headers: { 'Content-Type': 'application/json' }, +// json: parameters, +// auth: { 'bearer': authToken }, +// }; +// const result = await request.post(config.apiEndpoint + '/v1/mediaItems:search', options); +// return result; +// }; \ No newline at end of file diff --git a/src/server/authentication/config/passport.ts b/src/server/authentication/config/passport.ts index 97ded8785..6e0e01b9e 100644 --- a/src/server/authentication/config/passport.ts +++ b/src/server/authentication/config/passport.ts @@ -5,7 +5,6 @@ import { default as User } from '../models/user_model'; import { Request, Response, NextFunction } from "express"; import { RouteStore } from '../../RouteStore'; import * as GoogleOAuth from "passport-google-oauth20"; -const config = require("../../credentials/google_photos_credentials"); const LocalStrategy = passportLocal.Strategy; const GoogleOAuthStrategy = GoogleOAuth.Strategy; @@ -34,18 +33,6 @@ passport.use(new LocalStrategy({ usernameField: 'email', passReqToCallback: true }); })); - -passport.use(new GoogleOAuthStrategy( - { - clientID: config.oAuthClientID, - clientSecret: config.oAuthclientSecret, - callbackURL: config.oAuthCallbackUrl, - // Set the correct profile URL that does not require any additional APIs - userProfileURL: 'https://www.googleapis.com/oauth2/v3/userinfo' - }, - (token, refreshToken, profile, done) => done(undefined, { profile, token }) -)); - export let isAuthenticated = (req: Request, res: Response, next: NextFunction) => { if (req.isAuthenticated()) { return next(); diff --git a/src/server/credentials/auth.json b/src/server/credentials/auth.json deleted file mode 100644 index 557eca4b6..000000000 --- a/src/server/credentials/auth.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "type": "service_account", - "project_id": "brown-dash", - "private_key_id": "ddf0473a9ac56956b5818e04a7ee406a64d5b0a6", - "private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCueRfxic2oL9nr\nnWSLgl7XR/BKikm4p2sib6szaoTO+q6itcJgt2TDleK/7Y4KW/KhvCfhWVet0Hz0\nIDyg4N/gc2yxuDA6/m8DPWU9kDj8VFR7LVFawOKgo1WbgLcC0Qu8qHzAffrlg8si\nhj3vGuoS/YDn/mz0krwFmCfIx+S0lJ9a7FUjJL5C+CIwAEEYiU7xnTW7pVVNXAm/\n/YKD17ToAjREOtlfVVYO7tZ7V5BiW0I0jJvxw+t1pgrZZe7WPBSBJg9KKGIl+mRi\ndtUMR9Hyt3nMKNZIrSm0OkAz82HxfcapRdSB3wkVjoyW63YaTVHKoBOqRElfMtoM\nqu8wbhhNAgMBAAECggEAA41wJ8kg8J8peQMZ/b7gZvzuPy+h0M/J3j2MrG3rY9qA\nrUv1oqoBSXvhuNDhtEN12oYIqtg6m+L+Sas8CMuOC2rWPafM2u/80IGoGtDhtCjp\nv8inBX8ew4YSiL7IxdbTU2/70Es7DVV5u0t6ndsmr88ibYwwPupGR/fhPCpDyssg\n7lFAEpOwnbKG9a7E7axHpXBRSIE54sh+ESyf6MHH/oyKOLhZ0v4PjRDKaKuMDRst\nMOClgNjD/4bzKpfWljuPYemXz1oIBQitBW5aXnCdsmdrmOLDQpz3qOgIo+RRiyki\nvVU5N54L65sj4WisLt1TT45wbhrkQUz+8GmhV5rHwQKBgQDow32/gSb/M0BKFk5y\n+pSeoLkYp/dwfBFWYT6CNBaKePARFVCdr3db8yOEQD4hrmTOU0EP9c4FSLcaa0Xy\n7n2crhhfZWFpIpRMyXhpeKpqdjQFimfBOK6cIdjnWrtJ6Ik3m4E9p7kKThIsInTc\n/TETwAyzFN+J2ADRdrUdCukOwQKBgQC/4+1Rk8a++Jr9Sznx+JH4vj/J2cGsu7uQ\n0nikcOAFO5HzG7+mt9Wv9/MiPtEYwmc7YziDXldKLpshT2m6wrS1uzzOXAnvXFAh\npiCXQsmVA3gmrVd53k+eZfqrZ1n/rL1kCewRS5LX8xIhM28VIkGqkVy4ZEifMotG\nZKSbH0b4jQKBgQCsJ6rh8Uw+hFGQel8be2pgyM8eBV1lvN213ca11oC1ei1U9Ubi\n2dyWDYa/UiSiFLJKSBlfDJaMIfQLfjwGKY6OS9WK+RjLAeBdysVcfPrOMw7W6j9D\nEgFTSVV8CAdt6qdSkZlNWLfrf0LBkdqNeFbMHMdHzLBo63HverUJ/f/SAQKBgHIk\n2t5T0T14FHnnbaiJ/ArC4J7pcVOWuJQFHs5ydk+mh8LdFrvNTsdF7tLIGwlnWpDx\nDITYcYQnBRBjdLkraONRZXI7PY2sk93wPCK+D7scPTSEmCxeGW5XqyyaZea4klAX\nttzy336lkHs/ZSxlHDqiDU2CGdDY+A//fgroKAdhAoGAA5FXfMzTQLGqxg4J2B1z\nFEXNbrqZZFGgKiveUhhZLm4zPiHXtZXvDtSLwGgcO8oGfTfYueTcHb/Eiar7mKv+\n+SqpAqkINJTthIFVIiD39S9jPFUXzBkf5ZJKPLKQArhzEGxen+SD6ZUO058fA94L\n9FblRGlMtr2o5z0NC7H5zaU=\n-----END PRIVATE KEY-----\n", - "client_email": "google-photos-api@brown-dash.iam.gserviceaccount.com", - "client_id": "112995422877175743408", - "auth_uri": "https://accounts.google.com/o/oauth2/auth", - "token_uri": "https://oauth2.googleapis.com/token", - "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", - "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/google-photos-api%40brown-dash.iam.gserviceaccount.com" -} \ No newline at end of file diff --git a/src/server/credentials/google_photos_credentials.ts b/src/server/credentials/google_photos_credentials.ts deleted file mode 100644 index 11c1c766c..000000000 --- a/src/server/credentials/google_photos_credentials.ts +++ /dev/null @@ -1,35 +0,0 @@ -const config: any = {}; - -// The OAuth client ID from the Google Developers console. -config.oAuthClientID = '1005546247619-l40012sl4idpq17b5emielcs1delffog.apps.googleusercontent.com'; - -// The OAuth client secret from the Google Developers console. -config.oAuthclientSecret = 'xEUJ0OBvhlCKA6SLt8TvWBs3'; - -// The callback to use for OAuth requests. This is the URL where the app is -// running. For testing and running it locally, use 127.0.0.1. -config.oAuthCallbackUrl = 'http://localhost:1050/auth/google/callback'; - -// The port where the app should listen for requests. -config.port = 1050; - -// The scopes to request. The app requires the photoslibrary.readonly and -// plus.me scopes. -config.scopes = [ - 'https://www.googleapis.com/auth/photoslibrary.readonly', - 'profile', -]; - -// The number of photos to load for search requests. -config.photosToLoad = 150; - -// The page size to use for search requests. 100 is reccommended. -config.searchPageSize = 100; - -// The page size to use for the listing albums request. 50 is reccommended. -config.albumPageSize = 50; - -// The API end point to use. Do not change. -config.apiEndpoint = 'https://photoslibrary.googleapis.com'; - -module.exports = config; \ No newline at end of file diff --git a/src/server/index.ts b/src/server/index.ts index 6105dedcc..1f105e9d2 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -29,7 +29,6 @@ import { RouteStore } from './RouteStore'; import v4 = require('uuid/v4'); const app = express(); const config = require('../../webpack.config'); -const OAuthConfig = require('../server/credentials/google_photos_credentials'); import { createCanvas, loadImage, Canvas } from "canvas"; const compiler = webpack(config); const port = 1050; // default port to listen @@ -43,11 +42,11 @@ var AdmZip = require('adm-zip'); import * as YoutubeApi from "./apis/youtube/youtubeApiSample"; import { Response } from 'express-serve-static-core'; import { GoogleApiServerUtils } from "./apis/google/GoogleApiServerUtils"; -import { GaxiosResponse } from 'gaxios'; -import { Opt } from '../new_fields/Doc'; -import { docs_v1 } from 'googleapis'; -import { Endpoint } from 'googleapis-common'; -import { PhotosLibraryQuery } from './apis/google/GooglePhotosUtils'; +// import { GaxiosResponse } from 'gaxios'; +// import { Opt } from '../new_fields/Doc'; +// import { docs_v1 } from 'googleapis'; +// import { Endpoint } from 'googleapis-common'; +// import { PhotosLibraryQuery } from './apis/google/GooglePhotosUtils'; const MongoStore = require('connect-mongo')(session); const mongoose = require('mongoose'); const probe = require("probe-image-size"); -- cgit v1.2.3-70-g09d2 From b8a6b72938804902a7e4478cde9c50339341f67d Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Sun, 1 Sep 2019 14:50:06 -0400 Subject: regenerated auth key --- src/server/apis/google/GoogleApiServerUtils.ts | 3 +-- src/server/credentials/google_docs_token.json | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) (limited to 'src/server/apis/google/GoogleApiServerUtils.ts') diff --git a/src/server/apis/google/GoogleApiServerUtils.ts b/src/server/apis/google/GoogleApiServerUtils.ts index c1bd3300e..bc9ae2726 100644 --- a/src/server/apis/google/GoogleApiServerUtils.ts +++ b/src/server/apis/google/GoogleApiServerUtils.ts @@ -65,8 +65,7 @@ export namespace GoogleApiServerUtils { break; case Service.Photos: const photos = new Photos(result.token.access_token); - let response = await photos.albums.list(); - console.log("WE GOT SOMETHING!", response); + console.log(await photos.albums.list()); } resolve(routed); }); diff --git a/src/server/credentials/google_docs_token.json b/src/server/credentials/google_docs_token.json index cea452f08..fcb5c8abc 100644 --- a/src/server/credentials/google_docs_token.json +++ b/src/server/credentials/google_docs_token.json @@ -1 +1 @@ -{"access_token":"ya29.Glt2B3lsrpWxZ9DMg1RcTksAFzfR8dVWhf7d7tAvbJ4UbcSVO0Q3aYNGtaMKPtmxR24rH88iQSiKCL8S328TQFEN6LtZgvizymednK5EW0jNCvG6ecdZQ-vwcypR","refresh_token":"1/6X3oGYz4A0p8UW2IgsZ-GqTgQUY43S6Ahsaf_GQhSs8","scope":"https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/documents.readonly https://www.googleapis.com/auth/photoslibrary.sharing https://www.googleapis.com/auth/photoslibrary","token_type":"Bearer","expiry_date":1567360444627} \ No newline at end of file +{"access_token":"ya29.Glt2ByOBCyO7DNKKXaiDbK5c5OMwRoprqiCksLCu93WKuAE-YQ0gDCZQCqP07WV6QH0gMpwn47ghico1h5Rkxh-ukJdY9ndRTv7rEKJY32To4__gZh4xKVhwfOvf","refresh_token":"1/ynmFZmA-yqA1sKU3wF-g6QxCx9wGSTIA2sOvIhC_Ps0","scope":"https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/photoslibrary https://www.googleapis.com/auth/photoslibrary.sharing https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/documents.readonly","token_type":"Bearer","expiry_date":1567366717827} \ No newline at end of file -- cgit v1.2.3-70-g09d2 From 769b4c0b9ac61729b94b32999d3713a2dce53627 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Sun, 1 Sep 2019 17:02:11 -0400 Subject: added typings for google photo album manipulations --- src/server/apis/google/GoogleApiServerUtils.ts | 17 ++- src/server/apis/google/GooglePhotosUtils.ts | 177 +++++++++++++++++++++++-- src/server/authentication/config/passport.ts | 2 - src/server/credentials/google_docs_token.json | 2 +- 4 files changed, 180 insertions(+), 18 deletions(-) (limited to 'src/server/apis/google/GoogleApiServerUtils.ts') diff --git a/src/server/apis/google/GoogleApiServerUtils.ts b/src/server/apis/google/GoogleApiServerUtils.ts index bc9ae2726..656984b8a 100644 --- a/src/server/apis/google/GoogleApiServerUtils.ts +++ b/src/server/apis/google/GoogleApiServerUtils.ts @@ -5,7 +5,8 @@ import { OAuth2Client, Credentials } from "google-auth-library"; import { Opt } from "../../../new_fields/Doc"; import { GlobalOptions } from "googleapis-common"; import { GaxiosResponse } from "gaxios"; -import Photos = require("googlephotos"); +import { GooglePhotos, CreateAlbum, Action } from "./GooglePhotosUtils"; +import { Utils } from "../../../Utils"; /** * Server side authentication for Google Api queries. @@ -64,8 +65,18 @@ export namespace GoogleApiServerUtils { routed = google.slides(parameters).presentations; break; case Service.Photos: - const photos = new Photos(result.token.access_token); - console.log(await photos.albums.list()); + let token = result.token.access_token; + if (token) { + let create: CreateAlbum = { + action: Action.Create, + body: { + album: { + title: "Sam's Bulk Export", + } + } + }; + console.log(await GooglePhotos.ExecuteQuery(token, create)); + } } resolve(routed); }); diff --git a/src/server/apis/google/GooglePhotosUtils.ts b/src/server/apis/google/GooglePhotosUtils.ts index 7f9ffb6f3..439a41cb6 100644 --- a/src/server/apis/google/GooglePhotosUtils.ts +++ b/src/server/apis/google/GooglePhotosUtils.ts @@ -1,12 +1,165 @@ -// import request = require('request-promise'); -// const key = require("../../credentials/auth.json"); - -// export const PhotosLibraryQuery = async (authToken: any, parameters: any) => { -// let options = { -// headers: { 'Content-Type': 'application/json' }, -// json: parameters, -// auth: { 'bearer': authToken }, -// }; -// const result = await request.post(config.apiEndpoint + '/v1/mediaItems:search', options); -// return result; -// }; \ No newline at end of file +import request = require('request-promise'); + +const apiEndpoint = "https://photoslibrary.googleapis.com"; + +export type GooglePhotosQuery = AlbumsQuery; + +export type AlbumsQuery = (AddEnrichment | BatchAddMediaItems | BatchRemoveMediaItems | CreateAlbum | GetAlbum | ListAlbum | ShareAlbum | UnshareAlbum) & { body: any }; + +export enum Action { + AddEnrichment, + BatchAddMediaItems, + BatchRemoveMediaItems, + Create, + Get, + List, + Share, + Unshare +} + +export interface AddEnrichment { + action: Action.AddEnrichment; + albumId: string; + body: { + newEnrichmentItem: NewEnrichmentItem; + albumPosition: MediaRelativeAlbumPosition; + }; +} + +export interface BatchAddMediaItems { + action: Action.BatchAddMediaItems; + albumId: string; + body: { + mediaItemIds: string[]; + }; +} + +export interface BatchRemoveMediaItems { + action: Action.BatchRemoveMediaItems; + albumId: string; + body: { + mediaItemIds: string[]; + }; +} + +export interface CreateAlbum { + action: Action.Create; + body: { + album: AlbumTemplate; + }; +} + +export interface GetAlbum { + action: Action.Get; + albumId: string; +} + +export interface ListAlbum { + action: Action.List; + parameters: { + pageSize: number, + pageToken: string, + excludeNonAppCreatedData: boolean + }; +} + +export interface ShareAlbum { + action: Action.Share; + albumId: string; + body: { + sharedAlbumOptions: SharedAlbumOptions; + }; +} + +export interface UnshareAlbum { + action: Action.Unshare; + albumId: string; +} + +export interface AlbumTemplate { + title: string; +} + +export interface Album { + id: string; + title: string; + productUrl: string; + isWriteable: boolean; + shareInfo: ShareInfo; + mediaItemsCount: string; + coverPhotoBaseUrl: string; + coverPhotoMediaItemId: string; +} + +export interface ShareInfo { + sharedAlbumOptions: SharedAlbumOptions; + shareableUrl: string; + shareToken: string; + isJoined: boolean; + isOwned: boolean; +} + +export interface SharedAlbumOptions { + isCollaborative: boolean; + isCommentable: boolean; +} + +export enum PositionType { + POSITION_TYPE_UNSPECIFIED, + FIRST_IN_ALBUM, + LAST_IN_ALBUM, + AFTER_MEDIA_ITEM, + AFTER_ENRICHMENT_ITEM +} + +export type AlbumPosition = GeneralAlbumPosition | MediaRelativeAlbumPosition | EnrichmentRelativeAlbumPosition; + +interface GeneralAlbumPosition { + position: PositionType.FIRST_IN_ALBUM | PositionType.LAST_IN_ALBUM | PositionType.POSITION_TYPE_UNSPECIFIED; +} + +interface MediaRelativeAlbumPosition { + position: PositionType.AFTER_MEDIA_ITEM; + relativeMediaItemId: string; +} + +interface EnrichmentRelativeAlbumPosition { + position: PositionType.AFTER_ENRICHMENT_ITEM; + relativeEnrichmentItemId: string; +} + +export interface Location { + locationName: string; + latlng: { + latitude: number, + longitude: number + }; +} + +export interface NewEnrichmentItem { + textEnrichment: { + text: string; + }; + locationEnrichment: { + location: Location + }; + mapEnrichment: { + origin: { location: Location }, + destination: { location: Location } + }; +} + +export namespace GooglePhotos { + + export const ExecuteQuery = async (authToken: string, query: AlbumsQuery) => { + let options = { + headers: { 'Content-Type': 'application/json' }, + auth: { 'bearer': authToken }, + body: query.body, + json: true + }; + const result = await request.post(apiEndpoint + '/v1/albums', options); + return result; + }; + +} diff --git a/src/server/authentication/config/passport.ts b/src/server/authentication/config/passport.ts index 6e0e01b9e..e5733cbb5 100644 --- a/src/server/authentication/config/passport.ts +++ b/src/server/authentication/config/passport.ts @@ -4,10 +4,8 @@ import _ from "lodash"; import { default as User } from '../models/user_model'; import { Request, Response, NextFunction } from "express"; import { RouteStore } from '../../RouteStore'; -import * as GoogleOAuth from "passport-google-oauth20"; const LocalStrategy = passportLocal.Strategy; -const GoogleOAuthStrategy = GoogleOAuth.Strategy; passport.serializeUser((user, done) => { done(undefined, user.id); diff --git a/src/server/credentials/google_docs_token.json b/src/server/credentials/google_docs_token.json index fcb5c8abc..61864512c 100644 --- a/src/server/credentials/google_docs_token.json +++ b/src/server/credentials/google_docs_token.json @@ -1 +1 @@ -{"access_token":"ya29.Glt2ByOBCyO7DNKKXaiDbK5c5OMwRoprqiCksLCu93WKuAE-YQ0gDCZQCqP07WV6QH0gMpwn47ghico1h5Rkxh-ukJdY9ndRTv7rEKJY32To4__gZh4xKVhwfOvf","refresh_token":"1/ynmFZmA-yqA1sKU3wF-g6QxCx9wGSTIA2sOvIhC_Ps0","scope":"https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/photoslibrary https://www.googleapis.com/auth/photoslibrary.sharing https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/documents.readonly","token_type":"Bearer","expiry_date":1567366717827} \ No newline at end of file +{"access_token":"ya29.Glt3B8HoVEda7Ab5TQMVrfvjPN2fFp4sFHtGoDs3TsBgFfw4G208q90JiFjkmQqwODjJi3sf4NCZd78VZTVL3aI0By7_ElZF7XaCvA0LJnfcAi2gi1P-2-boyjYO","refresh_token":"1/tJOVDbPZlADzd2B8Q2_j7jqignXlRwHsU7LbZkdbDBc","scope":"https://www.googleapis.com/auth/documents.readonly https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/photoslibrary.sharing https://www.googleapis.com/auth/photoslibrary","token_type":"Bearer","expiry_date":1567374969108} \ No newline at end of file -- cgit v1.2.3-70-g09d2 From d1960ed915e78014592567e16dd1de9545781a27 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Sun, 1 Sep 2019 17:12:45 -0400 Subject: factored out typings --- src/server/apis/google/GoogleApiServerUtils.ts | 7 +- src/server/apis/google/GooglePhotosUtils.ts | 152 +------------------------ src/server/apis/google/typings/albums.ts | 148 ++++++++++++++++++++++++ 3 files changed, 156 insertions(+), 151 deletions(-) create mode 100644 src/server/apis/google/typings/albums.ts (limited to 'src/server/apis/google/GoogleApiServerUtils.ts') diff --git a/src/server/apis/google/GoogleApiServerUtils.ts b/src/server/apis/google/GoogleApiServerUtils.ts index 656984b8a..c6d901577 100644 --- a/src/server/apis/google/GoogleApiServerUtils.ts +++ b/src/server/apis/google/GoogleApiServerUtils.ts @@ -5,8 +5,9 @@ import { OAuth2Client, Credentials } from "google-auth-library"; import { Opt } from "../../../new_fields/Doc"; import { GlobalOptions } from "googleapis-common"; import { GaxiosResponse } from "gaxios"; -import { GooglePhotos, CreateAlbum, Action } from "./GooglePhotosUtils"; +import { GooglePhotos } from "./GooglePhotosUtils"; import { Utils } from "../../../Utils"; +import { Album } from "./Typings/albums"; /** * Server side authentication for Google Api queries. @@ -67,8 +68,8 @@ export namespace GoogleApiServerUtils { case Service.Photos: let token = result.token.access_token; if (token) { - let create: CreateAlbum = { - action: Action.Create, + let create: Album.Create = { + action: Album.Action.Create, body: { album: { title: "Sam's Bulk Export", diff --git a/src/server/apis/google/GooglePhotosUtils.ts b/src/server/apis/google/GooglePhotosUtils.ts index 439a41cb6..d4f16fd5d 100644 --- a/src/server/apis/google/GooglePhotosUtils.ts +++ b/src/server/apis/google/GooglePhotosUtils.ts @@ -1,157 +1,13 @@ import request = require('request-promise'); +import { Album } from './Typings/albums'; const apiEndpoint = "https://photoslibrary.googleapis.com"; -export type GooglePhotosQuery = AlbumsQuery; - -export type AlbumsQuery = (AddEnrichment | BatchAddMediaItems | BatchRemoveMediaItems | CreateAlbum | GetAlbum | ListAlbum | ShareAlbum | UnshareAlbum) & { body: any }; - -export enum Action { - AddEnrichment, - BatchAddMediaItems, - BatchRemoveMediaItems, - Create, - Get, - List, - Share, - Unshare -} - -export interface AddEnrichment { - action: Action.AddEnrichment; - albumId: string; - body: { - newEnrichmentItem: NewEnrichmentItem; - albumPosition: MediaRelativeAlbumPosition; - }; -} - -export interface BatchAddMediaItems { - action: Action.BatchAddMediaItems; - albumId: string; - body: { - mediaItemIds: string[]; - }; -} - -export interface BatchRemoveMediaItems { - action: Action.BatchRemoveMediaItems; - albumId: string; - body: { - mediaItemIds: string[]; - }; -} - -export interface CreateAlbum { - action: Action.Create; - body: { - album: AlbumTemplate; - }; -} - -export interface GetAlbum { - action: Action.Get; - albumId: string; -} - -export interface ListAlbum { - action: Action.List; - parameters: { - pageSize: number, - pageToken: string, - excludeNonAppCreatedData: boolean - }; -} - -export interface ShareAlbum { - action: Action.Share; - albumId: string; - body: { - sharedAlbumOptions: SharedAlbumOptions; - }; -} - -export interface UnshareAlbum { - action: Action.Unshare; - albumId: string; -} - -export interface AlbumTemplate { - title: string; -} - -export interface Album { - id: string; - title: string; - productUrl: string; - isWriteable: boolean; - shareInfo: ShareInfo; - mediaItemsCount: string; - coverPhotoBaseUrl: string; - coverPhotoMediaItemId: string; -} - -export interface ShareInfo { - sharedAlbumOptions: SharedAlbumOptions; - shareableUrl: string; - shareToken: string; - isJoined: boolean; - isOwned: boolean; -} - -export interface SharedAlbumOptions { - isCollaborative: boolean; - isCommentable: boolean; -} - -export enum PositionType { - POSITION_TYPE_UNSPECIFIED, - FIRST_IN_ALBUM, - LAST_IN_ALBUM, - AFTER_MEDIA_ITEM, - AFTER_ENRICHMENT_ITEM -} - -export type AlbumPosition = GeneralAlbumPosition | MediaRelativeAlbumPosition | EnrichmentRelativeAlbumPosition; - -interface GeneralAlbumPosition { - position: PositionType.FIRST_IN_ALBUM | PositionType.LAST_IN_ALBUM | PositionType.POSITION_TYPE_UNSPECIFIED; -} - -interface MediaRelativeAlbumPosition { - position: PositionType.AFTER_MEDIA_ITEM; - relativeMediaItemId: string; -} - -interface EnrichmentRelativeAlbumPosition { - position: PositionType.AFTER_ENRICHMENT_ITEM; - relativeEnrichmentItemId: string; -} - -export interface Location { - locationName: string; - latlng: { - latitude: number, - longitude: number - }; -} - -export interface NewEnrichmentItem { - textEnrichment: { - text: string; - }; - locationEnrichment: { - location: Location - }; - mapEnrichment: { - origin: { location: Location }, - destination: { location: Location } - }; -} - export namespace GooglePhotos { - export const ExecuteQuery = async (authToken: string, query: AlbumsQuery) => { + export type Query = Album.Query; + + export const ExecuteQuery = async (authToken: string, query: GooglePhotos.Query) => { let options = { headers: { 'Content-Type': 'application/json' }, auth: { 'bearer': authToken }, diff --git a/src/server/apis/google/typings/albums.ts b/src/server/apis/google/typings/albums.ts new file mode 100644 index 000000000..1c9b379fe --- /dev/null +++ b/src/server/apis/google/typings/albums.ts @@ -0,0 +1,148 @@ +export namespace Album { + + export type Query = (AddEnrichment | BatchAddMediaItems | BatchRemoveMediaItems | Create | Get | List | Share | Unshare) & { body: any }; + + export enum Action { + AddEnrichment, + BatchAddMediaItems, + BatchRemoveMediaItems, + Create, + Get, + List, + Share, + Unshare + } + + export interface AddEnrichment { + action: Action.AddEnrichment; + albumId: string; + body: { + newEnrichmentItem: NewEnrichmentItem; + albumPosition: MediaRelativeAlbumPosition; + }; + } + + export interface BatchAddMediaItems { + action: Action.BatchAddMediaItems; + albumId: string; + body: { + mediaItemIds: string[]; + }; + } + + export interface BatchRemoveMediaItems { + action: Action.BatchRemoveMediaItems; + albumId: string; + body: { + mediaItemIds: string[]; + }; + } + + export interface Create { + action: Action.Create; + body: { + album: Template; + }; + } + + export interface Get { + action: Action.Get; + albumId: string; + } + + export interface List { + action: Action.List; + parameters: { + pageSize: number, + pageToken: string, + excludeNonAppCreatedData: boolean + }; + } + + export interface Share { + action: Action.Share; + albumId: string; + body: { + sharedAlbumOptions: SharedOptions; + }; + } + + export interface Unshare { + action: Action.Unshare; + albumId: string; + } + + export interface Template { + title: string; + } + + export interface Model { + id: string; + title: string; + productUrl: string; + isWriteable: boolean; + shareInfo: ShareInfo; + mediaItemsCount: string; + coverPhotoBaseUrl: string; + coverPhotoMediaItemId: string; + } + + export interface ShareInfo { + sharedAlbumOptions: SharedOptions; + shareableUrl: string; + shareToken: string; + isJoined: boolean; + isOwned: boolean; + } + + export interface SharedOptions { + isCollaborative: boolean; + isCommentable: boolean; + } + + export enum PositionType { + POSITION_TYPE_UNSPECIFIED, + FIRST_IN_ALBUM, + LAST_IN_ALBUM, + AFTER_MEDIA_ITEM, + AFTER_ENRICHMENT_ITEM + } + + export type Position = GeneralAlbumPosition | MediaRelativeAlbumPosition | EnrichmentRelativeAlbumPosition; + + interface GeneralAlbumPosition { + position: PositionType.FIRST_IN_ALBUM | PositionType.LAST_IN_ALBUM | PositionType.POSITION_TYPE_UNSPECIFIED; + } + + interface MediaRelativeAlbumPosition { + position: PositionType.AFTER_MEDIA_ITEM; + relativeMediaItemId: string; + } + + interface EnrichmentRelativeAlbumPosition { + position: PositionType.AFTER_ENRICHMENT_ITEM; + relativeEnrichmentItemId: string; + } + + export interface Location { + locationName: string; + latlng: { + latitude: number, + longitude: number + }; + } + + export interface NewEnrichmentItem { + textEnrichment: { + text: string; + }; + locationEnrichment: { + location: Location + }; + mapEnrichment: { + origin: { location: Location }, + destination: { location: Location } + }; + } + +} \ No newline at end of file -- cgit v1.2.3-70-g09d2 From 8b992ef2c152e86299fd3460112124d476393a60 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Sun, 1 Sep 2019 17:13:09 -0400 Subject: tweaks --- src/server/apis/google/GoogleApiServerUtils.ts | 2 +- src/server/apis/google/GooglePhotosUtils.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'src/server/apis/google/GoogleApiServerUtils.ts') diff --git a/src/server/apis/google/GoogleApiServerUtils.ts b/src/server/apis/google/GoogleApiServerUtils.ts index c6d901577..00e289b00 100644 --- a/src/server/apis/google/GoogleApiServerUtils.ts +++ b/src/server/apis/google/GoogleApiServerUtils.ts @@ -7,7 +7,7 @@ import { GlobalOptions } from "googleapis-common"; import { GaxiosResponse } from "gaxios"; import { GooglePhotos } from "./GooglePhotosUtils"; import { Utils } from "../../../Utils"; -import { Album } from "./Typings/albums"; +import { Album } from "./typings/albums"; /** * Server side authentication for Google Api queries. diff --git a/src/server/apis/google/GooglePhotosUtils.ts b/src/server/apis/google/GooglePhotosUtils.ts index d4f16fd5d..750630626 100644 --- a/src/server/apis/google/GooglePhotosUtils.ts +++ b/src/server/apis/google/GooglePhotosUtils.ts @@ -1,5 +1,5 @@ import request = require('request-promise'); -import { Album } from './Typings/albums'; +import { Album } from './typings/albums'; const apiEndpoint = "https://photoslibrary.googleapis.com"; -- cgit v1.2.3-70-g09d2 From 38176e5ba949b84dc410d29197180121d81e085c Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Tue, 3 Sep 2019 12:51:59 -0400 Subject: implemented refresh tokens and create, get, list --- .../apis/google_docs/GooglePhotosClientUtils.ts | 35 +++++++++ src/client/views/MainView.tsx | 10 ++- src/server/RouteStore.ts | 3 +- src/server/apis/google/GoogleApiServerUtils.ts | 90 ++++++++++++++-------- src/server/apis/google/GooglePhotosServerUtils.ts | 68 ++++++++++++++++ src/server/apis/google/GooglePhotosUploadUtils.ts | 42 ++++++++++ src/server/apis/google/GooglePhotosUtils.ts | 21 ----- src/server/apis/google/typings/albums.ts | 30 ++++---- src/server/credentials/google_docs_token.json | 2 +- src/server/index.ts | 54 ++++++------- 10 files changed, 252 insertions(+), 103 deletions(-) create mode 100644 src/client/apis/google_docs/GooglePhotosClientUtils.ts create mode 100644 src/server/apis/google/GooglePhotosServerUtils.ts create mode 100644 src/server/apis/google/GooglePhotosUploadUtils.ts delete mode 100644 src/server/apis/google/GooglePhotosUtils.ts (limited to 'src/server/apis/google/GoogleApiServerUtils.ts') diff --git a/src/client/apis/google_docs/GooglePhotosClientUtils.ts b/src/client/apis/google_docs/GooglePhotosClientUtils.ts new file mode 100644 index 000000000..67a282f48 --- /dev/null +++ b/src/client/apis/google_docs/GooglePhotosClientUtils.ts @@ -0,0 +1,35 @@ +import { Album } from "../../../server/apis/google/typings/albums"; +import { PostToServer } from "../../../Utils"; +import { RouteStore } from "../../../server/RouteStore"; + +export namespace GooglePhotosClientUtils { + + export const Create = async (title: string) => { + let parameters = { + action: Album.Action.Create, + body: { album: { title } } + } as Album.Create; + return PostToServer(RouteStore.googlePhotos, parameters); + }; + + export const List = async (options?: Partial) => { + let parameters = { + action: Album.Action.List, + parameters: { + pageSize: (options ? options.pageSize : 20) || 20, + pageToken: (options ? options.pageToken : undefined) || undefined, + excludeNonAppCreatedData: (options ? options.excludeNonAppCreatedData : false) || false, + } as Album.ListOptions + } as Album.List; + return PostToServer(RouteStore.googlePhotos, parameters); + }; + + export const Get = async (albumId: string) => { + let parameters = { + action: Album.Action.Get, + albumId + } as Album.Get; + return PostToServer(RouteStore.googlePhotos, parameters); + }; + +} \ No newline at end of file diff --git a/src/client/views/MainView.tsx b/src/client/views/MainView.tsx index df0718072..ece475c80 100644 --- a/src/client/views/MainView.tsx +++ b/src/client/views/MainView.tsx @@ -42,6 +42,8 @@ import PresModeMenu from './presentationview/PresentationModeMenu'; import { PresBox } from './nodes/PresBox'; import { GoogleApiClientUtils } from '../apis/google_docs/GoogleApiClientUtils'; import { docs_v1 } from 'googleapis'; +import { Album } from '../../server/apis/google/typings/albums'; +import { GooglePhotosClientUtils } from '../apis/google_docs/GooglePhotosClientUtils'; @observer export class MainView extends React.Component { @@ -128,7 +130,7 @@ export class MainView extends React.Component { window.removeEventListener("keydown", KeyManager.Instance.handle); window.addEventListener("keydown", KeyManager.Instance.handle); - PostToServer('/googleDocs/Photos/Test', {}); + this.executeGooglePhotosAlbumTestRoutine(); reaction(() => { let workspaces = CurrentUserUtils.UserDocument.workspaces; @@ -147,6 +149,12 @@ export class MainView extends React.Component { }, { fireImmediately: true }); } + executeGooglePhotosAlbumTestRoutine = async () => { + let title = "This is a generically created album!"; + console.log(await GooglePhotosClientUtils.Create(title)); + console.log(await GooglePhotosClientUtils.List({ pageSize: 50 })); + } + componentWillUnMount() { window.removeEventListener("keydown", KeyManager.Instance.handle); //close presentation diff --git a/src/server/RouteStore.ts b/src/server/RouteStore.ts index 014906054..fc5511f98 100644 --- a/src/server/RouteStore.ts +++ b/src/server/RouteStore.ts @@ -31,6 +31,7 @@ export enum RouteStore { // APIS cognitiveServices = "/cognitiveservices", - googleDocs = "/googleDocs" + googleDocs = "/googleDocs", + googlePhotos = "/googlePhotos" } \ No newline at end of file diff --git a/src/server/apis/google/GoogleApiServerUtils.ts b/src/server/apis/google/GoogleApiServerUtils.ts index 00e289b00..048ac4b21 100644 --- a/src/server/apis/google/GoogleApiServerUtils.ts +++ b/src/server/apis/google/GoogleApiServerUtils.ts @@ -1,13 +1,12 @@ -import { google, docs_v1, slides_v1 } from "googleapis"; +import { google } from "googleapis"; import { createInterface } from "readline"; import { readFile, writeFile } from "fs"; import { OAuth2Client, Credentials } from "google-auth-library"; import { Opt } from "../../../new_fields/Doc"; import { GlobalOptions } from "googleapis-common"; import { GaxiosResponse } from "gaxios"; -import { GooglePhotos } from "./GooglePhotosUtils"; -import { Utils } from "../../../Utils"; -import { Album } from "./typings/albums"; +import request = require('request-promise'); +import * as qs from 'query-string'; /** * Server side authentication for Google Api queries. @@ -31,8 +30,7 @@ export namespace GoogleApiServerUtils { export enum Service { Documents = "Documents", - Slides = "Slides", - Photos = "Photos" + Slides = "Slides" } export interface CredentialPaths { @@ -49,38 +47,31 @@ export namespace GoogleApiServerUtils { export type EndpointParameters = GlobalOptions & { version: "v1" }; export const GetEndpoint = async (sector: string, paths: CredentialPaths) => { - return new Promise>((resolve, reject) => { + return new Promise>(resolve => { + RetrieveAuthenticationInformation(paths).then(authentication => { + let routed: Opt; + let parameters: EndpointParameters = { auth: authentication.client, version: "v1" }; + switch (sector) { + case Service.Documents: + routed = google.docs(parameters).documents; + break; + case Service.Slides: + routed = google.slides(parameters).presentations; + break; + } + resolve(routed); + }); + }); + }; + + export const RetrieveAuthenticationInformation = async (paths: CredentialPaths) => { + return new Promise((resolve, reject) => { readFile(paths.credentials, async (err, credentials) => { if (err) { reject(err); return console.log('Error loading client secret file:', err); } - authorize(parseBuffer(credentials), paths.token).then(async result => { - let routed: Opt; - let parameters: EndpointParameters = { auth: result.client, version: "v1" }; - switch (sector) { - case Service.Documents: - routed = google.docs(parameters).documents; - break; - case Service.Slides: - routed = google.slides(parameters).presentations; - break; - case Service.Photos: - let token = result.token.access_token; - if (token) { - let create: Album.Create = { - action: Album.Action.Create, - body: { - album: { - title: "Sam's Bulk Export", - } - } - }; - console.log(await GooglePhotos.ExecuteQuery(token, create)); - } - } - resolve(routed); - }); + authorize(parseBuffer(credentials), paths.token).then(resolve, reject); }); }); }; @@ -101,13 +92,44 @@ export namespace GoogleApiServerUtils { if (err) { return getNewToken(oAuth2Client, token_path).then(resolve, reject); } - let parsed = parseBuffer(token); + let parsed: Credentials = parseBuffer(token); + if (parsed.expiry_date! < new Date().getTime()) { + return refreshToken(parsed, client_id, client_secret, oAuth2Client, token_path).then(resolve, reject); + } oAuth2Client.setCredentials(parsed); resolve({ token: parsed, client: oAuth2Client }); }); }); } + const refreshEndpoint = "https://oauth2.googleapis.com/token"; + const refreshToken = (credentials: Credentials, client_id: string, client_secret: string, oAuth2Client: OAuth2Client, token_path: string) => { + return new Promise((resolve, reject) => { + let headerParameters = { headers: { 'Content-Type': 'application/x-www-form-urlencoded' } }; + let queryParameters = { + refreshToken: credentials.refresh_token, + client_id, + client_secret, + grant_type: "refresh_token" + }; + let url = `${refreshEndpoint}?${qs.stringify(queryParameters)}`; + request.post(url, headerParameters).then(response => { + let parsed = JSON.parse(response); + credentials.access_token = parsed.access_token; + credentials.expiry_date = new Date().getTime() + parsed.expires_in; + writeFile(token_path, JSON.stringify(credentials), (err) => { + if (err) { + console.error(err); + reject(err); + } + console.log('Refreshed token stored to', token_path); + oAuth2Client.setCredentials(credentials); + resolve({ token: credentials, client: oAuth2Client }); + }); + }); + }); + }; + /** * Get and store new token after prompting for user authorization, and then * execute the given callback with the authorized OAuth2 client. diff --git a/src/server/apis/google/GooglePhotosServerUtils.ts b/src/server/apis/google/GooglePhotosServerUtils.ts new file mode 100644 index 000000000..cb5464abc --- /dev/null +++ b/src/server/apis/google/GooglePhotosServerUtils.ts @@ -0,0 +1,68 @@ +import request = require('request-promise'); +import { Album } from './typings/albums'; +import * as qs from 'query-string'; + +const apiEndpoint = "https://photoslibrary.googleapis.com/v1/"; + +export interface Authorization { + token: string; +} + +export namespace GooglePhotos { + + export type Query = Album.Query; + export type QueryParameters = { query: GooglePhotos.Query }; + interface DispatchParameters { + required: boolean; + method: "GET" | "POST"; + ignore?: boolean; + } + + export const ExecuteQuery = async (parameters: Authorization & QueryParameters): Promise => { + let action = parameters.query.action; + let dispatch = SuffixMap.get(action)!; + let suffix = Suffix(parameters, dispatch, action); + if (suffix) { + let query: any = parameters.query; + let options: any = { + headers: { 'Content-Type': 'application/json' }, + auth: { 'bearer': parameters.token }, + }; + if (query.body) { + options.body = query.body; + options.json = true; + } + let queryParameters = query.parameters; + if (queryParameters) { + suffix += `?${qs.stringify(queryParameters)}`; + } + let dispatcher = dispatch.method === "POST" ? request.post : request.get; + return dispatcher(apiEndpoint + suffix, options); + } + }; + + const Suffix = (parameters: QueryParameters, dispatch: DispatchParameters, action: Album.Action) => { + let query: any = parameters.query; + let id = query.albumId; + let suffix = 'albums'; + if (dispatch.required) { + if (!id) { + return undefined; + } + suffix += `/${id}${dispatch.ignore ? "" : `:${action}`}`; + } + return suffix; + }; + + const SuffixMap = new Map([ + [Album.Action.AddEnrichment, { required: true, method: "POST" }], + [Album.Action.BatchAddMediaItems, { required: true, method: "POST" }], + [Album.Action.BatchRemoveMediaItems, { required: true, method: "POST" }], + [Album.Action.Create, { required: false, method: "POST" }], + [Album.Action.Get, { required: true, ignore: true, method: "GET" }], + [Album.Action.List, { required: false, method: "GET" }], + [Album.Action.Share, { required: true, method: "POST" }], + [Album.Action.Unshare, { required: true, method: "POST" }] + ]); + +} diff --git a/src/server/apis/google/GooglePhotosUploadUtils.ts b/src/server/apis/google/GooglePhotosUploadUtils.ts new file mode 100644 index 000000000..2e1599aaf --- /dev/null +++ b/src/server/apis/google/GooglePhotosUploadUtils.ts @@ -0,0 +1,42 @@ +import request = require('request-promise'); +import { Authorization } from './GooglePhotosServerUtils'; + +export namespace GooglePhotosUploadUtils { + + interface UploadInformation { + title: string; + url: URL; + } + + const apiEndpoint = "https://photoslibrary.googleapis.com/v1/uploads"; + + export const SubmitUpload = async (parameters: Authorization & UploadInformation) => { + let MEDIA_BINARY_DATA = binary(parameters.url.href); + + let options = { + headers: { + 'Content-Type': 'application/octet-stream', + Authorization: { 'bearer': parameters.token }, + 'X-Goog-Upload-File-Name': parameters.title, + 'X-Goog-Upload-Protocol': 'raw' + }, + body: { MEDIA_BINARY_DATA }, + json: true + }; + const result = await request.post(apiEndpoint, options); + return result; + }; + + const binary = (source: string) => { + const image = document.createElement("img"); + image.src = source; + const canvas = document.createElement("canvas"); + canvas.width = image.width; + canvas.height = image.height; + const ctx = canvas.getContext("2d")!; + ctx.drawImage(image, 0, 0); + const dataUrl = canvas.toDataURL("image/png"); + return dataUrl.replace(/^data:image\/(png|jpg);base64,/, ""); + }; + +} \ No newline at end of file diff --git a/src/server/apis/google/GooglePhotosUtils.ts b/src/server/apis/google/GooglePhotosUtils.ts deleted file mode 100644 index 750630626..000000000 --- a/src/server/apis/google/GooglePhotosUtils.ts +++ /dev/null @@ -1,21 +0,0 @@ -import request = require('request-promise'); -import { Album } from './typings/albums'; - -const apiEndpoint = "https://photoslibrary.googleapis.com"; - -export namespace GooglePhotos { - - export type Query = Album.Query; - - export const ExecuteQuery = async (authToken: string, query: GooglePhotos.Query) => { - let options = { - headers: { 'Content-Type': 'application/json' }, - auth: { 'bearer': authToken }, - body: query.body, - json: true - }; - const result = await request.post(apiEndpoint + '/v1/albums', options); - return result; - }; - -} diff --git a/src/server/apis/google/typings/albums.ts b/src/server/apis/google/typings/albums.ts index 1c9b379fe..f3025567d 100644 --- a/src/server/apis/google/typings/albums.ts +++ b/src/server/apis/google/typings/albums.ts @@ -1,16 +1,16 @@ export namespace Album { - export type Query = (AddEnrichment | BatchAddMediaItems | BatchRemoveMediaItems | Create | Get | List | Share | Unshare) & { body: any }; + export type Query = (AddEnrichment | BatchAddMediaItems | BatchRemoveMediaItems | Create | Get | List | Share | Unshare); export enum Action { - AddEnrichment, - BatchAddMediaItems, - BatchRemoveMediaItems, - Create, - Get, - List, - Share, - Unshare + AddEnrichment = "addEnrichment", + BatchAddMediaItems = "batchAddMediaItems", + BatchRemoveMediaItems = "batchRemoveMediaItems", + Create = "create", + Get = "get", + List = "list", + Share = "share", + Unshare = "unshare" } export interface AddEnrichment { @@ -52,11 +52,13 @@ export namespace Album { export interface List { action: Action.List; - parameters: { - pageSize: number, - pageToken: string, - excludeNonAppCreatedData: boolean - }; + parameters: ListOptions; + } + + export interface ListOptions { + pageSize: number; + pageToken: string; + excludeNonAppCreatedData: boolean; } export interface Share { diff --git a/src/server/credentials/google_docs_token.json b/src/server/credentials/google_docs_token.json index 61864512c..39e067c86 100644 --- a/src/server/credentials/google_docs_token.json +++ b/src/server/credentials/google_docs_token.json @@ -1 +1 @@ -{"access_token":"ya29.Glt3B8HoVEda7Ab5TQMVrfvjPN2fFp4sFHtGoDs3TsBgFfw4G208q90JiFjkmQqwODjJi3sf4NCZd78VZTVL3aI0By7_ElZF7XaCvA0LJnfcAi2gi1P-2-boyjYO","refresh_token":"1/tJOVDbPZlADzd2B8Q2_j7jqignXlRwHsU7LbZkdbDBc","scope":"https://www.googleapis.com/auth/documents.readonly https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/photoslibrary.sharing https://www.googleapis.com/auth/photoslibrary","token_type":"Bearer","expiry_date":1567374969108} \ No newline at end of file +{"access_token":"ya29.Glx4B9p8tvFsmsD-AD-D4jygL_YZVCFpPewM9djtfOT3T4S6ROxN5r0WLAKTYVNnXQbUri3Gu_-vIb0NWq9wEy1TdFTLIM8azWD82X5-I5BQq2DSOsYiKugvgVoHLw","refresh_token":"1/kk_pPY7WBwT34JNPzx_HrSVoZlvfzys4EEVNjp7nqzg6aIbRrEKNMRTb0u0wr9GM","scope":"https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents.readonly https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/photoslibrary.sharing https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/photoslibrary","token_type":"Bearer","expiry_date":1567529401142} \ No newline at end of file diff --git a/src/server/index.ts b/src/server/index.ts index 1f105e9d2..54b954cfb 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -42,11 +42,7 @@ var AdmZip = require('adm-zip'); import * as YoutubeApi from "./apis/youtube/youtubeApiSample"; import { Response } from 'express-serve-static-core'; import { GoogleApiServerUtils } from "./apis/google/GoogleApiServerUtils"; -// import { GaxiosResponse } from 'gaxios'; -// import { Opt } from '../new_fields/Doc'; -// import { docs_v1 } from 'googleapis'; -// import { Endpoint } from 'googleapis-common'; -// import { PhotosLibraryQuery } from './apis/google/GooglePhotosUtils'; +import { GooglePhotos } from './apis/google/GooglePhotosServerUtils'; const MongoStore = require('connect-mongo')(session); const mongoose = require('mongoose'); const probe = require("probe-image-size"); @@ -195,32 +191,7 @@ app.get("/version", (req, res) => { // SEARCH const solrURL = "http://localhost:8983/solr/#/dash"; -// GETTERS - -// app.get('/auth/google', passport.authenticate('google', { -// scope: OAuthConfig.scopes, -// failureFlash: true, // Display errors to the user. -// session: true, -// })); - -// app.get("/failed", (req, res) => res.send("DIDN'T WORK!")); - -// app.get( -// '/auth/google/callback', -// passport.authenticate( -// 'google', { failureRedirect: '/failed', failureFlash: true, session: true }), -// (req, res) => { -// // User has logged in. -// console.log('OAUTH: user has logged in 1.'); -// PhotosLibraryQuery(req.user.token, {}); -// console.log('OAUTH: user has logged in 2.'); -// res.redirect('/'); -// }); - -// app.get('/GooglePhotos', (req, res) => { -// console.log("WORKING ON GOOGLE PHOTOS"); -// PhotosLibraryQuery(req.user.token, {}); -// }); +// GETTERSå app.get("/search", async (req, res) => { const solrQuery: any = {}; @@ -853,6 +824,27 @@ app.post(RouteStore.googleDocs + "/:sector/:action", (req, res) => { }); }); +app.post(RouteStore.googlePhotos, (req, res) => { + GoogleApiServerUtils.RetrieveAuthenticationInformation({ credentials, token }).then(authentication => { + let validated = authentication.token.access_token; + if (!validated) { + res.send("Error: unable to authenticate Google Photos API request."); + return; + } + GooglePhotos.ExecuteQuery({ token: validated, query: req.body }) + .then(response => { + if (response === undefined) { + res.send("Error: unable to build suffix for Google Photos API request"); + return; + } + res.send(response); + }) + .catch(error => { + res.send(`Error: an exception occurred in the execution of this Google Photos API request\n${error}`); + }); + }); +}); + const suffixMap: { [type: string]: (string | [string, string | ((json: any) => any)]) } = { "number": "_n", "string": "_t", -- cgit v1.2.3-70-g09d2 From 0724d62e2a6797c640e79b46b678bd1d3917147f Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Tue, 3 Sep 2019 13:18:23 -0400 Subject: fixed refresh token issue --- src/client/views/MainView.tsx | 8 -------- src/server/apis/google/GoogleApiServerUtils.ts | 2 +- src/server/credentials/google_docs_token.json | 2 +- 3 files changed, 2 insertions(+), 10 deletions(-) (limited to 'src/server/apis/google/GoogleApiServerUtils.ts') diff --git a/src/client/views/MainView.tsx b/src/client/views/MainView.tsx index ece475c80..3ee62ae86 100644 --- a/src/client/views/MainView.tsx +++ b/src/client/views/MainView.tsx @@ -130,8 +130,6 @@ export class MainView extends React.Component { window.removeEventListener("keydown", KeyManager.Instance.handle); window.addEventListener("keydown", KeyManager.Instance.handle); - this.executeGooglePhotosAlbumTestRoutine(); - reaction(() => { let workspaces = CurrentUserUtils.UserDocument.workspaces; let recent = CurrentUserUtils.UserDocument.recentlyClosed; @@ -149,12 +147,6 @@ export class MainView extends React.Component { }, { fireImmediately: true }); } - executeGooglePhotosAlbumTestRoutine = async () => { - let title = "This is a generically created album!"; - console.log(await GooglePhotosClientUtils.Create(title)); - console.log(await GooglePhotosClientUtils.List({ pageSize: 50 })); - } - componentWillUnMount() { window.removeEventListener("keydown", KeyManager.Instance.handle); //close presentation diff --git a/src/server/apis/google/GoogleApiServerUtils.ts b/src/server/apis/google/GoogleApiServerUtils.ts index 048ac4b21..edfd89de4 100644 --- a/src/server/apis/google/GoogleApiServerUtils.ts +++ b/src/server/apis/google/GoogleApiServerUtils.ts @@ -116,7 +116,7 @@ export namespace GoogleApiServerUtils { request.post(url, headerParameters).then(response => { let parsed = JSON.parse(response); credentials.access_token = parsed.access_token; - credentials.expiry_date = new Date().getTime() + parsed.expires_in; + credentials.expiry_date = new Date().getTime() + parsed.expires_in * 1000; writeFile(token_path, JSON.stringify(credentials), (err) => { if (err) { console.error(err); diff --git a/src/server/credentials/google_docs_token.json b/src/server/credentials/google_docs_token.json index 39e067c86..c83d82607 100644 --- a/src/server/credentials/google_docs_token.json +++ b/src/server/credentials/google_docs_token.json @@ -1 +1 @@ -{"access_token":"ya29.Glx4B9p8tvFsmsD-AD-D4jygL_YZVCFpPewM9djtfOT3T4S6ROxN5r0WLAKTYVNnXQbUri3Gu_-vIb0NWq9wEy1TdFTLIM8azWD82X5-I5BQq2DSOsYiKugvgVoHLw","refresh_token":"1/kk_pPY7WBwT34JNPzx_HrSVoZlvfzys4EEVNjp7nqzg6aIbRrEKNMRTb0u0wr9GM","scope":"https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents.readonly https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/photoslibrary.sharing https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/photoslibrary","token_type":"Bearer","expiry_date":1567529401142} \ No newline at end of file +{"access_token":"ya29.Glx4B4EQ5Q7WNw1hPrcqbZHssp0l1BtzszwNgmKGd783VBP9G3hLMe1AkbQw_Dl0amzgFeO29R4WP0ca-5U_Rf1tjsdtio6XjBGpJrY-S5wK7t71raKkwRPZFITw2Q","refresh_token":"1/kk_pPY7WBwT34JNPzx_HrSVoZlvfzys4EEVNjp7nqzg6aIbRrEKNMRTb0u0wr9GM","scope":"https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents.readonly https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/photoslibrary.sharing https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/photoslibrary","token_type":"Bearer","expiry_date":1567531020435} \ No newline at end of file -- cgit v1.2.3-70-g09d2 From 7e87aa4b7e0125482c87ab61f4a0de14e774558d Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Tue, 3 Sep 2019 16:23:42 -0400 Subject: working on media uploading --- .../apis/google_docs/GooglePhotosClientUtils.ts | 22 ++++++++-- src/client/views/MainView.tsx | 6 +++ src/server/RouteStore.ts | 3 +- src/server/apis/google/GoogleApiServerUtils.ts | 13 +++++- src/server/apis/google/GooglePhotosUploadUtils.ts | 20 ++------- src/server/credentials/google_docs_token.json | 2 +- src/server/index.ts | 50 ++++++++++++++-------- 7 files changed, 73 insertions(+), 43 deletions(-) (limited to 'src/server/apis/google/GoogleApiServerUtils.ts') diff --git a/src/client/apis/google_docs/GooglePhotosClientUtils.ts b/src/client/apis/google_docs/GooglePhotosClientUtils.ts index 67a282f48..0864ebdb1 100644 --- a/src/client/apis/google_docs/GooglePhotosClientUtils.ts +++ b/src/client/apis/google_docs/GooglePhotosClientUtils.ts @@ -1,6 +1,7 @@ import { Album } from "../../../server/apis/google/typings/albums"; import { PostToServer } from "../../../Utils"; import { RouteStore } from "../../../server/RouteStore"; +import { ImageField } from "../../../new_fields/URLField"; export namespace GooglePhotosClientUtils { @@ -9,7 +10,7 @@ export namespace GooglePhotosClientUtils { action: Album.Action.Create, body: { album: { title } } } as Album.Create; - return PostToServer(RouteStore.googlePhotos, parameters); + return PostToServer(RouteStore.googlePhotosQuery, parameters); }; export const List = async (options?: Partial) => { @@ -21,7 +22,7 @@ export namespace GooglePhotosClientUtils { excludeNonAppCreatedData: (options ? options.excludeNonAppCreatedData : false) || false, } as Album.ListOptions } as Album.List; - return PostToServer(RouteStore.googlePhotos, parameters); + return PostToServer(RouteStore.googlePhotosQuery, parameters); }; export const Get = async (albumId: string) => { @@ -29,7 +30,22 @@ export namespace GooglePhotosClientUtils { action: Album.Action.Get, albumId } as Album.Get; - return PostToServer(RouteStore.googlePhotos, parameters); + return PostToServer(RouteStore.googlePhotosQuery, parameters); + }; + + export const toDataURL = (field: ImageField | undefined) => { + if (!field) { + return undefined; + } + const image = document.createElement("img"); + image.src = field.url.href; + const canvas = document.createElement("canvas"); + canvas.width = image.width; + canvas.height = image.height; + const ctx = canvas.getContext("2d")!; + ctx.drawImage(image, 0, 0); + const dataUrl = canvas.toDataURL("image/png"); + return dataUrl.replace(/^data:image\/(png|jpg);base64,/, ""); }; } \ No newline at end of file diff --git a/src/client/views/MainView.tsx b/src/client/views/MainView.tsx index 3ee62ae86..590addaa3 100644 --- a/src/client/views/MainView.tsx +++ b/src/client/views/MainView.tsx @@ -44,6 +44,7 @@ import { GoogleApiClientUtils } from '../apis/google_docs/GoogleApiClientUtils'; import { docs_v1 } from 'googleapis'; import { Album } from '../../server/apis/google/typings/albums'; import { GooglePhotosClientUtils } from '../apis/google_docs/GooglePhotosClientUtils'; +import { ImageField } from '../../new_fields/URLField'; @observer export class MainView extends React.Component { @@ -130,6 +131,11 @@ export class MainView extends React.Component { window.removeEventListener("keydown", KeyManager.Instance.handle); window.addEventListener("keydown", KeyManager.Instance.handle); + let imgurl = "https://upload.wikimedia.org/wikipedia/commons/thumb/3/3a/Cat03.jpg/1200px-Cat03.jpg"; + let image = Docs.Create.ImageDocument(imgurl, { width: 200, title: "an image of a cat" }); + let parameters = { title: StrCast(image.title), MEDIA_BINARY_DATA: GooglePhotosClientUtils.toDataURL(Cast(image.data, ImageField)) }; + PostToServer(RouteStore.googlePhotosMediaUpload, parameters).then(console.log); + reaction(() => { let workspaces = CurrentUserUtils.UserDocument.workspaces; let recent = CurrentUserUtils.UserDocument.recentlyClosed; diff --git a/src/server/RouteStore.ts b/src/server/RouteStore.ts index fc5511f98..3b3fd9b4a 100644 --- a/src/server/RouteStore.ts +++ b/src/server/RouteStore.ts @@ -32,6 +32,7 @@ export enum RouteStore { // APIS cognitiveServices = "/cognitiveservices", googleDocs = "/googleDocs", - googlePhotos = "/googlePhotos" + googlePhotosQuery = "/googlePhotosQuery", + googlePhotosMediaUpload = "/googlePhotosMediaUpload" } \ No newline at end of file diff --git a/src/server/apis/google/GoogleApiServerUtils.ts b/src/server/apis/google/GoogleApiServerUtils.ts index edfd89de4..2c9085ebb 100644 --- a/src/server/apis/google/GoogleApiServerUtils.ts +++ b/src/server/apis/google/GoogleApiServerUtils.ts @@ -48,7 +48,7 @@ export namespace GoogleApiServerUtils { export const GetEndpoint = async (sector: string, paths: CredentialPaths) => { return new Promise>(resolve => { - RetrieveAuthenticationInformation(paths).then(authentication => { + RetrieveCredentials(paths).then(authentication => { let routed: Opt; let parameters: EndpointParameters = { auth: authentication.client, version: "v1" }; switch (sector) { @@ -64,7 +64,7 @@ export namespace GoogleApiServerUtils { }); }; - export const RetrieveAuthenticationInformation = async (paths: CredentialPaths) => { + export const RetrieveCredentials = async (paths: CredentialPaths) => { return new Promise((resolve, reject) => { readFile(paths.credentials, async (err, credentials) => { if (err) { @@ -76,6 +76,15 @@ export namespace GoogleApiServerUtils { }); }; + export const RetrieveAccessToken = async (paths: CredentialPaths) => { + return new Promise((resolve, reject) => { + RetrieveCredentials(paths).then( + credentials => resolve(credentials.token.access_token!), + error => reject(`Error: unable to authenticate Google Photos API request.\n${error}`) + ); + }); + }; + type TokenResult = { token: Credentials, client: OAuth2Client }; /** * Create an OAuth2 client with the given credentials, and returns the promise resolving to the authenticated client diff --git a/src/server/apis/google/GooglePhotosUploadUtils.ts b/src/server/apis/google/GooglePhotosUploadUtils.ts index 2e1599aaf..b358f9698 100644 --- a/src/server/apis/google/GooglePhotosUploadUtils.ts +++ b/src/server/apis/google/GooglePhotosUploadUtils.ts @@ -5,38 +5,24 @@ export namespace GooglePhotosUploadUtils { interface UploadInformation { title: string; - url: URL; + MEDIA_BINARY_DATA: string; } const apiEndpoint = "https://photoslibrary.googleapis.com/v1/uploads"; export const SubmitUpload = async (parameters: Authorization & UploadInformation) => { - let MEDIA_BINARY_DATA = binary(parameters.url.href); - let options = { headers: { 'Content-Type': 'application/octet-stream', - Authorization: { 'bearer': parameters.token }, + auth: { 'bearer': parameters.token }, 'X-Goog-Upload-File-Name': parameters.title, 'X-Goog-Upload-Protocol': 'raw' }, - body: { MEDIA_BINARY_DATA }, + body: { MEDIA_BINARY_DATA: parameters.MEDIA_BINARY_DATA }, json: true }; const result = await request.post(apiEndpoint, options); return result; }; - const binary = (source: string) => { - const image = document.createElement("img"); - image.src = source; - const canvas = document.createElement("canvas"); - canvas.width = image.width; - canvas.height = image.height; - const ctx = canvas.getContext("2d")!; - ctx.drawImage(image, 0, 0); - const dataUrl = canvas.toDataURL("image/png"); - return dataUrl.replace(/^data:image\/(png|jpg);base64,/, ""); - }; - } \ No newline at end of file diff --git a/src/server/credentials/google_docs_token.json b/src/server/credentials/google_docs_token.json index c83d82607..e4be9ff60 100644 --- a/src/server/credentials/google_docs_token.json +++ b/src/server/credentials/google_docs_token.json @@ -1 +1 @@ -{"access_token":"ya29.Glx4B4EQ5Q7WNw1hPrcqbZHssp0l1BtzszwNgmKGd783VBP9G3hLMe1AkbQw_Dl0amzgFeO29R4WP0ca-5U_Rf1tjsdtio6XjBGpJrY-S5wK7t71raKkwRPZFITw2Q","refresh_token":"1/kk_pPY7WBwT34JNPzx_HrSVoZlvfzys4EEVNjp7nqzg6aIbRrEKNMRTb0u0wr9GM","scope":"https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents.readonly https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/photoslibrary.sharing https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/photoslibrary","token_type":"Bearer","expiry_date":1567531020435} \ No newline at end of file +{"access_token":"ya29.Glx4B9gmH9fKJex9Je-k1KBNWEsJBPgQqoEWHhwk0TtTemCxONoyVMO38idAE7Vy9vdjcosjl0H4swnnH1s2QjTwZaspzKPeQr8Oh4sHkCJ-MbNd6naMZBdy1pccjQ","refresh_token":"1/kk_pPY7WBwT34JNPzx_HrSVoZlvfzys4EEVNjp7nqzg6aIbRrEKNMRTb0u0wr9GM","scope":"https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents.readonly https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/photoslibrary.sharing https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/photoslibrary","token_type":"Bearer","expiry_date":1567545365417} \ No newline at end of file diff --git a/src/server/index.ts b/src/server/index.ts index 54b954cfb..3e85b1ce0 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -43,6 +43,7 @@ import * as YoutubeApi from "./apis/youtube/youtubeApiSample"; import { Response } from 'express-serve-static-core'; import { GoogleApiServerUtils } from "./apis/google/GoogleApiServerUtils"; import { GooglePhotos } from './apis/google/GooglePhotosServerUtils'; +import { GooglePhotosUploadUtils } from './apis/google/GooglePhotosUploadUtils'; const MongoStore = require('connect-mongo')(session); const mongoose = require('mongoose'); const probe = require("probe-image-size"); @@ -824,25 +825,36 @@ app.post(RouteStore.googleDocs + "/:sector/:action", (req, res) => { }); }); -app.post(RouteStore.googlePhotos, (req, res) => { - GoogleApiServerUtils.RetrieveAuthenticationInformation({ credentials, token }).then(authentication => { - let validated = authentication.token.access_token; - if (!validated) { - res.send("Error: unable to authenticate Google Photos API request."); - return; - } - GooglePhotos.ExecuteQuery({ token: validated, query: req.body }) - .then(response => { - if (response === undefined) { - res.send("Error: unable to build suffix for Google Photos API request"); - return; - } - res.send(response); - }) - .catch(error => { - res.send(`Error: an exception occurred in the execution of this Google Photos API request\n${error}`); - }); - }); +app.post(RouteStore.googlePhotosQuery, (req, res) => { + GoogleApiServerUtils.RetrieveAccessToken({ credentials, token }).then( + token => { + GooglePhotos.ExecuteQuery({ token, query: req.body }) + .then(response => { + if (response === undefined) { + res.send("Error: unable to build suffix for Google Photos API request"); + return; + } + res.send(response); + }) + .catch(error => { + res.send(`Error: an exception occurred in the execution of this Google Photos API request\n${error}`); + }); + }, + error => res.send(error) + ); +}); + +app.post(RouteStore.googlePhotosMediaUpload, (req, res) => { + GoogleApiServerUtils.RetrieveAccessToken({ credentials, token }).then( + token => { + GooglePhotosUploadUtils.SubmitUpload({ token, ...req.body }) + .then(response => { + res.send(response); + }).catch(error => { + res.send(`Error: an exception occurred in uploading the given media\n${error}`); + }); + }, + error => res.send(error)); }); const suffixMap: { [type: string]: (string | [string, string | ((json: any) => any)]) } = { -- cgit v1.2.3-70-g09d2 From 0c02b2ff3a41697c43d0aed98f330bd0293ef761 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Tue, 3 Sep 2019 19:21:23 -0400 Subject: fixed upload authentication issue, need to fix image byte extraction --- src/server/apis/google/GoogleApiServerUtils.ts | 1 + src/server/apis/google/GooglePhotosUploadUtils.ts | 2 +- src/server/credentials/google_docs_token.json | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) (limited to 'src/server/apis/google/GoogleApiServerUtils.ts') diff --git a/src/server/apis/google/GoogleApiServerUtils.ts b/src/server/apis/google/GoogleApiServerUtils.ts index 2c9085ebb..b6330a609 100644 --- a/src/server/apis/google/GoogleApiServerUtils.ts +++ b/src/server/apis/google/GoogleApiServerUtils.ts @@ -23,6 +23,7 @@ export namespace GoogleApiServerUtils { 'drive', 'drive.file', 'photoslibrary', + 'photoslibrary.appendonly', 'photoslibrary.sharing' ]; diff --git a/src/server/apis/google/GooglePhotosUploadUtils.ts b/src/server/apis/google/GooglePhotosUploadUtils.ts index b358f9698..cd2a586eb 100644 --- a/src/server/apis/google/GooglePhotosUploadUtils.ts +++ b/src/server/apis/google/GooglePhotosUploadUtils.ts @@ -14,7 +14,7 @@ export namespace GooglePhotosUploadUtils { let options = { headers: { 'Content-Type': 'application/octet-stream', - auth: { 'bearer': parameters.token }, + Authorization: `Bearer ${parameters.token}`, 'X-Goog-Upload-File-Name': parameters.title, 'X-Goog-Upload-Protocol': 'raw' }, diff --git a/src/server/credentials/google_docs_token.json b/src/server/credentials/google_docs_token.json index e4be9ff60..2ac972ed8 100644 --- a/src/server/credentials/google_docs_token.json +++ b/src/server/credentials/google_docs_token.json @@ -1 +1 @@ -{"access_token":"ya29.Glx4B9gmH9fKJex9Je-k1KBNWEsJBPgQqoEWHhwk0TtTemCxONoyVMO38idAE7Vy9vdjcosjl0H4swnnH1s2QjTwZaspzKPeQr8Oh4sHkCJ-MbNd6naMZBdy1pccjQ","refresh_token":"1/kk_pPY7WBwT34JNPzx_HrSVoZlvfzys4EEVNjp7nqzg6aIbRrEKNMRTb0u0wr9GM","scope":"https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents.readonly https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/photoslibrary.sharing https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/photoslibrary","token_type":"Bearer","expiry_date":1567545365417} \ No newline at end of file +{"access_token":"ya29.Glt5ByifP30HMz6a1fEG77qZlz9fBEOCz4PQ1VA8t_Ck2ZTPJKoyr6xc3-GFZISAqrrw2U8XpMyZv02_URfPwUX0Z_tMdlIFqsygowR-uClukbgQPNtgxp2LS1oW","refresh_token":"1/wK1cUVLnt581ba_pYGPdlTvAa-OS64nB5m5XOXEBJ8Q","scope":"https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/documents.readonly https://www.googleapis.com/auth/photoslibrary.sharing https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/photoslibrary.appendonly https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/photoslibrary","token_type":"Bearer","expiry_date":1567556163894} \ No newline at end of file -- cgit v1.2.3-70-g09d2 From 5d59e9a379417140e10778cd43e8f87ecb816c37 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Fri, 6 Sep 2019 06:29:26 -0400 Subject: lightly tested functional export of any image doc (web url or stored in server public folder) to google photos, optionally stored in a given target album --- package.json | 2 +- .../apis/google_docs/GooglePhotosClientUtils.ts | 73 +++++----- src/client/views/MainView.tsx | 17 ++- src/client/views/nodes/FormattedTextBox.tsx | 1 + src/server/RouteStore.ts | 2 +- src/server/apis/google/GoogleApiServerUtils.ts | 28 ++-- src/server/apis/google/GooglePhotosServerUtils.ts | 68 ---------- src/server/apis/google/GooglePhotosUploadUtils.ts | 122 +++++++++++++++-- src/server/apis/google/typings/albums.ts | 150 --------------------- src/server/credentials/google_docs_token.json | 2 +- src/server/index.ts | 75 +++++------ 11 files changed, 206 insertions(+), 334 deletions(-) delete mode 100644 src/server/apis/google/GooglePhotosServerUtils.ts delete mode 100644 src/server/apis/google/typings/albums.ts (limited to 'src/server/apis/google/GoogleApiServerUtils.ts') diff --git a/package.json b/package.json index f0f2b467e..f56e34ce0 100644 --- a/package.json +++ b/package.json @@ -224,4 +224,4 @@ "xoauth2": "^1.2.0", "youtube": "^0.1.0" } -} +} \ No newline at end of file diff --git a/src/client/apis/google_docs/GooglePhotosClientUtils.ts b/src/client/apis/google_docs/GooglePhotosClientUtils.ts index b95cc98c9..2b72800a9 100644 --- a/src/client/apis/google_docs/GooglePhotosClientUtils.ts +++ b/src/client/apis/google_docs/GooglePhotosClientUtils.ts @@ -1,53 +1,42 @@ -import { Album } from "../../../server/apis/google/typings/albums"; -import { PostToServer } from "../../../Utils"; +import { PostToServer, Utils } from "../../../Utils"; import { RouteStore } from "../../../server/RouteStore"; import { ImageField } from "../../../new_fields/URLField"; +import { StrCast, Cast } from "../../../new_fields/Types"; +import { Doc, Opt } from "../../../new_fields/Doc"; +import { Id } from "../../../new_fields/FieldSymbols"; +import requestImageSize = require('../../util/request-image-size'); +import Photos = require('googlephotos'); export namespace GooglePhotosClientUtils { - export const Create = async (title: string) => { - let parameters = { - action: Album.Action.Create, - body: { album: { title } } - } as Album.Create; - return PostToServer(RouteStore.googlePhotosQuery, parameters); - }; + export type AlbumReference = { id: string } | { title: string }; + export const endpoint = () => fetch(Utils.prepend(RouteStore.googlePhotosAccessToken)).then(async response => new Photos(await response.text())); - export const List = async (options?: Partial) => { - let parameters = { - action: Album.Action.List, - parameters: { - pageSize: (options ? options.pageSize : 20) || 20, - pageToken: (options ? options.pageToken : undefined) || undefined, - excludeNonAppCreatedData: (options ? options.excludeNonAppCreatedData : false) || false, - } as Album.ListOptions - } as Album.List; - return PostToServer(RouteStore.googlePhotosQuery, parameters); - }; + export interface MediaInput { + description: string; + source: string; + } - export const Get = async (albumId: string) => { - let parameters = { - action: Album.Action.Get, - albumId - } as Album.Get; - return PostToServer(RouteStore.googlePhotosQuery, parameters); - }; - - export const toDataURL = (field: ImageField | undefined) => { - if (!field) { - return undefined; + export const UploadMedia = async (sources: Doc[], album?: AlbumReference) => { + if (album && "title" in album) { + album = (await endpoint()).albums.create(album.title); + } + const media: MediaInput[] = []; + sources.forEach(document => { + const data = Cast(Doc.GetProto(document).data, ImageField); + const description = StrCast(document.caption); + if (!data) { + return undefined; + } + media.push({ + source: data.url.href, + description, + } as MediaInput); + }); + if (media.length) { + return PostToServer(RouteStore.googlePhotosMediaUpload, { media, album }); } - const image = document.createElement("img"); - image.src = field.url.href; - image.width = 200; - image.height = 200; - const canvas = document.createElement("canvas"); - canvas.width = image.width; - canvas.height = image.height; - const ctx = canvas.getContext("2d")!; - ctx.drawImage(image, 0, 0); - const dataUrl = canvas.toDataURL("image/png"); - return dataUrl.replace(/^data:image\/(png|jpg);base64,/, ""); + return undefined; }; } \ No newline at end of file diff --git a/src/client/views/MainView.tsx b/src/client/views/MainView.tsx index c1c95fc88..6d366216e 100644 --- a/src/client/views/MainView.tsx +++ b/src/client/views/MainView.tsx @@ -40,11 +40,10 @@ import { PreviewCursor } from './PreviewCursor'; import { FilterBox } from './search/FilterBox'; import PresModeMenu from './presentationview/PresentationModeMenu'; import { PresBox } from './nodes/PresBox'; -import { docs_v1 } from 'googleapis'; -import { Album } from '../../server/apis/google/typings/albums'; import { GooglePhotosClientUtils } from '../apis/google_docs/GooglePhotosClientUtils'; import { ImageField } from '../../new_fields/URLField'; import { LinkFollowBox } from './linking/LinkFollowBox'; +import { DocumentManager } from '../util/DocumentManager'; @observer export class MainView extends React.Component { @@ -131,10 +130,7 @@ export class MainView extends React.Component { window.removeEventListener("keydown", KeyManager.Instance.handle); window.addEventListener("keydown", KeyManager.Instance.handle); - let imgurl = "https://upload.wikimedia.org/wikipedia/commons/thumb/3/3a/Cat03.jpg/1200px-Cat03.jpg"; - let image = Docs.Create.ImageDocument(imgurl, { width: 200, title: "an image of a cat" }); - let parameters = { title: StrCast(image.title), MEDIA_BINARY_DATA: GooglePhotosClientUtils.toDataURL(Cast(image.data, ImageField)) }; - // PostToServer(RouteStore.googlePhotosMediaUpload, parameters).then(console.log); + this.executeGooglePhotosRoutine(); reaction(() => { let workspaces = CurrentUserUtils.UserDocument.workspaces; @@ -153,6 +149,15 @@ export class MainView extends React.Component { }, { fireImmediately: true }); } + executeGooglePhotosRoutine = async () => { + let imgurl = "https://upload.wikimedia.org/wikipedia/commons/thumb/3/3a/Cat03.jpg/1200px-Cat03.jpg"; + let doc = Docs.Create.ImageDocument(imgurl, { width: 200, title: "an image of a cat" }); + doc.caption = "Well isn't this a nice cat image!"; + let photos = await GooglePhotosClientUtils.endpoint(); + let albumId = (await photos.albums.list(50)).albums.filter((album: any) => album.title === "This is a generically created album!")[0].id; + console.log(await GooglePhotosClientUtils.UploadMedia([doc], { id: albumId })); + } + componentWillUnMount() { window.removeEventListener("keydown", KeyManager.Instance.handle); //close presentation diff --git a/src/client/views/nodes/FormattedTextBox.tsx b/src/client/views/nodes/FormattedTextBox.tsx index b671d06ea..fda9ea33f 100644 --- a/src/client/views/nodes/FormattedTextBox.tsx +++ b/src/client/views/nodes/FormattedTextBox.tsx @@ -133,6 +133,7 @@ export class FormattedTextBox extends DocComponent<(FieldViewProps & FormattedTe if (this.props.isOverlay) { DragManager.StartDragFunctions.push(() => FormattedTextBox.InputBoxOverlay = undefined); } + FormattedTextBox.Instance = this; } public get CurrentDiv(): HTMLDivElement { return this._ref.current!; } diff --git a/src/server/RouteStore.ts b/src/server/RouteStore.ts index 3b3fd9b4a..b221b71bc 100644 --- a/src/server/RouteStore.ts +++ b/src/server/RouteStore.ts @@ -32,7 +32,7 @@ export enum RouteStore { // APIS cognitiveServices = "/cognitiveservices", googleDocs = "/googleDocs", - googlePhotosQuery = "/googlePhotosQuery", + googlePhotosAccessToken = "/googlePhotosAccessToken", googlePhotosMediaUpload = "/googlePhotosMediaUpload" } \ No newline at end of file diff --git a/src/server/apis/google/GoogleApiServerUtils.ts b/src/server/apis/google/GoogleApiServerUtils.ts index b6330a609..ac8023ce1 100644 --- a/src/server/apis/google/GoogleApiServerUtils.ts +++ b/src/server/apis/google/GoogleApiServerUtils.ts @@ -7,6 +7,7 @@ import { GlobalOptions } from "googleapis-common"; import { GaxiosResponse } from "gaxios"; import request = require('request-promise'); import * as qs from 'query-string'; +import Photos = require('googlephotos'); /** * Server side authentication for Google Api queries. @@ -35,19 +36,19 @@ export namespace GoogleApiServerUtils { } export interface CredentialPaths { - credentials: string; - token: string; + credentialsPath: string; + tokenPath: string; } export type ApiResponse = Promise; - export type ApiRouter = (endpoint: Endpoint, paramters: any) => ApiResponse; + export type ApiRouter = (endpoint: Endpoint, parameters: any) => ApiResponse; export type ApiHandler = (parameters: any) => ApiResponse; export type Action = "create" | "retrieve" | "update"; export type Endpoint = { get: ApiHandler, create: ApiHandler, batchUpdate: ApiHandler }; export type EndpointParameters = GlobalOptions & { version: "v1" }; - export const GetEndpoint = async (sector: string, paths: CredentialPaths) => { + export const GetEndpoint = (sector: string, paths: CredentialPaths) => { return new Promise>(resolve => { RetrieveCredentials(paths).then(authentication => { let routed: Opt; @@ -65,19 +66,19 @@ export namespace GoogleApiServerUtils { }); }; - export const RetrieveCredentials = async (paths: CredentialPaths) => { + export const RetrieveCredentials = (paths: CredentialPaths) => { return new Promise((resolve, reject) => { - readFile(paths.credentials, async (err, credentials) => { + readFile(paths.credentialsPath, async (err, credentials) => { if (err) { reject(err); return console.log('Error loading client secret file:', err); } - authorize(parseBuffer(credentials), paths.token).then(resolve, reject); + authorize(parseBuffer(credentials), paths.tokenPath).then(resolve, reject); }); }); }; - export const RetrieveAccessToken = async (paths: CredentialPaths) => { + export const RetrieveAccessToken = (paths: CredentialPaths) => { return new Promise((resolve, reject) => { RetrieveCredentials(paths).then( credentials => resolve(credentials.token.access_token!), @@ -86,6 +87,15 @@ export namespace GoogleApiServerUtils { }); }; + export const RetrievePhotosEndpoint = (paths: CredentialPaths) => { + return new Promise((resolve, reject) => { + RetrieveAccessToken(paths).then( + token => resolve(new Photos(token)), + reject + ); + }); + }; + type TokenResult = { token: Credentials, client: OAuth2Client }; /** * Create an OAuth2 client with the given credentials, and returns the promise resolving to the authenticated client @@ -126,7 +136,7 @@ export namespace GoogleApiServerUtils { request.post(url, headerParameters).then(response => { let parsed = JSON.parse(response); credentials.access_token = parsed.access_token; - credentials.expiry_date = new Date().getTime() + parsed.expires_in * 1000; + credentials.expiry_date = new Date().getTime() + (parsed.expires_in * 1000); writeFile(token_path, JSON.stringify(credentials), (err) => { if (err) { console.error(err); diff --git a/src/server/apis/google/GooglePhotosServerUtils.ts b/src/server/apis/google/GooglePhotosServerUtils.ts deleted file mode 100644 index cb5464abc..000000000 --- a/src/server/apis/google/GooglePhotosServerUtils.ts +++ /dev/null @@ -1,68 +0,0 @@ -import request = require('request-promise'); -import { Album } from './typings/albums'; -import * as qs from 'query-string'; - -const apiEndpoint = "https://photoslibrary.googleapis.com/v1/"; - -export interface Authorization { - token: string; -} - -export namespace GooglePhotos { - - export type Query = Album.Query; - export type QueryParameters = { query: GooglePhotos.Query }; - interface DispatchParameters { - required: boolean; - method: "GET" | "POST"; - ignore?: boolean; - } - - export const ExecuteQuery = async (parameters: Authorization & QueryParameters): Promise => { - let action = parameters.query.action; - let dispatch = SuffixMap.get(action)!; - let suffix = Suffix(parameters, dispatch, action); - if (suffix) { - let query: any = parameters.query; - let options: any = { - headers: { 'Content-Type': 'application/json' }, - auth: { 'bearer': parameters.token }, - }; - if (query.body) { - options.body = query.body; - options.json = true; - } - let queryParameters = query.parameters; - if (queryParameters) { - suffix += `?${qs.stringify(queryParameters)}`; - } - let dispatcher = dispatch.method === "POST" ? request.post : request.get; - return dispatcher(apiEndpoint + suffix, options); - } - }; - - const Suffix = (parameters: QueryParameters, dispatch: DispatchParameters, action: Album.Action) => { - let query: any = parameters.query; - let id = query.albumId; - let suffix = 'albums'; - if (dispatch.required) { - if (!id) { - return undefined; - } - suffix += `/${id}${dispatch.ignore ? "" : `:${action}`}`; - } - return suffix; - }; - - const SuffixMap = new Map([ - [Album.Action.AddEnrichment, { required: true, method: "POST" }], - [Album.Action.BatchAddMediaItems, { required: true, method: "POST" }], - [Album.Action.BatchRemoveMediaItems, { required: true, method: "POST" }], - [Album.Action.Create, { required: false, method: "POST" }], - [Album.Action.Get, { required: true, ignore: true, method: "GET" }], - [Album.Action.List, { required: false, method: "GET" }], - [Album.Action.Share, { required: true, method: "POST" }], - [Album.Action.Unshare, { required: true, method: "POST" }] - ]); - -} diff --git a/src/server/apis/google/GooglePhotosUploadUtils.ts b/src/server/apis/google/GooglePhotosUploadUtils.ts index cd2a586eb..3b513aaf1 100644 --- a/src/server/apis/google/GooglePhotosUploadUtils.ts +++ b/src/server/apis/google/GooglePhotosUploadUtils.ts @@ -1,28 +1,122 @@ import request = require('request-promise'); -import { Authorization } from './GooglePhotosServerUtils'; +import { GoogleApiServerUtils } from './GoogleApiServerUtils'; +import * as fs from 'fs'; +import { Utils } from '../../../Utils'; +import * as path from 'path'; +import { Opt } from '../../../new_fields/Doc'; export namespace GooglePhotosUploadUtils { - interface UploadInformation { - title: string; - MEDIA_BINARY_DATA: string; + export interface Paths { + uploadDirectory: string; + credentialsPath: string; + tokenPath: string; } - const apiEndpoint = "https://photoslibrary.googleapis.com/v1/uploads"; + export interface MediaInput { + description: string; + source: string; + } + + export interface DownloadInformation { + mediaPath: string; + contentType?: string; + contentSize?: string; + } + + const prepend = (extension: string) => `https://photoslibrary.googleapis.com/v1/${extension}`; + const headers = (type: string) => ({ + 'Content-Type': `application/${type}`, + 'Authorization': Bearer, + }); + + let Bearer: string; + let Paths: Paths; - export const SubmitUpload = async (parameters: Authorization & UploadInformation) => { - let options = { + export const initialize = async (paths: Paths) => { + Paths = paths; + const { tokenPath, credentialsPath } = paths; + const token = await GoogleApiServerUtils.RetrieveAccessToken({ tokenPath, credentialsPath }); + Bearer = `Bearer ${token}`; + }; + + export const DispatchGooglePhotosUpload = async (filename: string) => { + let body: Buffer; + if (filename.includes('upload_')) { + const mediaPath = Paths.uploadDirectory + filename; + body = await new Promise((resolve, reject) => { + fs.readFile(mediaPath, (error, data) => error ? reject(error) : resolve(data)); + }); + } else { + body = await request(filename, { encoding: null }); + } + const parameters = { + method: 'POST', headers: { - 'Content-Type': 'application/octet-stream', - Authorization: `Bearer ${parameters.token}`, - 'X-Goog-Upload-File-Name': parameters.title, + ...headers('octet-stream'), + 'X-Goog-Upload-File-Name': filename, 'X-Goog-Upload-Protocol': 'raw' }, - body: { MEDIA_BINARY_DATA: parameters.MEDIA_BINARY_DATA }, - json: true + uri: prepend('uploads'), + body }; - const result = await request.post(apiEndpoint, options); - return result; + return new Promise(resolve => request(parameters, (error, _response, body) => resolve(error ? undefined : body))); + }; + + export const CreateMediaItems = (newMediaItems: any[], album?: { id: string }) => { + return new Promise((resolve, reject) => { + const parameters = { + method: 'POST', + headers: headers('json'), + uri: prepend('mediaItems:batchCreate'), + body: { newMediaItems } as any, + json: true + }; + album && (parameters.body.albumId = album.id); + request(parameters, (error, _response, body) => { + if (error) { + reject(error); + } else { + resolve(body); + } + }); + }); }; + export namespace IOUtils { + + export const Download = async (url: string): Promise> => { + const filename = `temporary_upload_${Utils.GenerateGuid()}${path.extname(url).toLowerCase()}`; + const temporaryDirectory = Paths.uploadDirectory + "temporary/"; + const mediaPath = temporaryDirectory + filename; + + if (!(await createIfNotExists(temporaryDirectory))) { + return undefined; + } + + return new Promise((resolve, reject) => { + request.head(url, (error, res) => { + if (error) { + return reject(error); + } + const information: DownloadInformation = { + mediaPath, + contentType: res.headers['content-type'], + contentSize: res.headers['content-length'], + }; + request(url).pipe(fs.createWriteStream(mediaPath)).on('close', () => resolve(information)); + }); + }); + }; + + export const createIfNotExists = async (path: string) => { + if (await new Promise(resolve => fs.exists(path, resolve))) { + return true; + } + return new Promise(resolve => fs.mkdir(path, error => resolve(error === null))); + }; + + export const Destroy = (mediaPath: string) => new Promise(resolve => fs.unlink(mediaPath, error => resolve(error === null))); + } + } \ No newline at end of file diff --git a/src/server/apis/google/typings/albums.ts b/src/server/apis/google/typings/albums.ts deleted file mode 100644 index f3025567d..000000000 --- a/src/server/apis/google/typings/albums.ts +++ /dev/null @@ -1,150 +0,0 @@ -export namespace Album { - - export type Query = (AddEnrichment | BatchAddMediaItems | BatchRemoveMediaItems | Create | Get | List | Share | Unshare); - - export enum Action { - AddEnrichment = "addEnrichment", - BatchAddMediaItems = "batchAddMediaItems", - BatchRemoveMediaItems = "batchRemoveMediaItems", - Create = "create", - Get = "get", - List = "list", - Share = "share", - Unshare = "unshare" - } - - export interface AddEnrichment { - action: Action.AddEnrichment; - albumId: string; - body: { - newEnrichmentItem: NewEnrichmentItem; - albumPosition: MediaRelativeAlbumPosition; - }; - } - - export interface BatchAddMediaItems { - action: Action.BatchAddMediaItems; - albumId: string; - body: { - mediaItemIds: string[]; - }; - } - - export interface BatchRemoveMediaItems { - action: Action.BatchRemoveMediaItems; - albumId: string; - body: { - mediaItemIds: string[]; - }; - } - - export interface Create { - action: Action.Create; - body: { - album: Template; - }; - } - - export interface Get { - action: Action.Get; - albumId: string; - } - - export interface List { - action: Action.List; - parameters: ListOptions; - } - - export interface ListOptions { - pageSize: number; - pageToken: string; - excludeNonAppCreatedData: boolean; - } - - export interface Share { - action: Action.Share; - albumId: string; - body: { - sharedAlbumOptions: SharedOptions; - }; - } - - export interface Unshare { - action: Action.Unshare; - albumId: string; - } - - export interface Template { - title: string; - } - - export interface Model { - id: string; - title: string; - productUrl: string; - isWriteable: boolean; - shareInfo: ShareInfo; - mediaItemsCount: string; - coverPhotoBaseUrl: string; - coverPhotoMediaItemId: string; - } - - export interface ShareInfo { - sharedAlbumOptions: SharedOptions; - shareableUrl: string; - shareToken: string; - isJoined: boolean; - isOwned: boolean; - } - - export interface SharedOptions { - isCollaborative: boolean; - isCommentable: boolean; - } - - export enum PositionType { - POSITION_TYPE_UNSPECIFIED, - FIRST_IN_ALBUM, - LAST_IN_ALBUM, - AFTER_MEDIA_ITEM, - AFTER_ENRICHMENT_ITEM - } - - export type Position = GeneralAlbumPosition | MediaRelativeAlbumPosition | EnrichmentRelativeAlbumPosition; - - interface GeneralAlbumPosition { - position: PositionType.FIRST_IN_ALBUM | PositionType.LAST_IN_ALBUM | PositionType.POSITION_TYPE_UNSPECIFIED; - } - - interface MediaRelativeAlbumPosition { - position: PositionType.AFTER_MEDIA_ITEM; - relativeMediaItemId: string; - } - - interface EnrichmentRelativeAlbumPosition { - position: PositionType.AFTER_ENRICHMENT_ITEM; - relativeEnrichmentItemId: string; - } - - export interface Location { - locationName: string; - latlng: { - latitude: number, - longitude: number - }; - } - - export interface NewEnrichmentItem { - textEnrichment: { - text: string; - }; - locationEnrichment: { - location: Location - }; - mapEnrichment: { - origin: { location: Location }, - destination: { location: Location } - }; - } - -} \ No newline at end of file diff --git a/src/server/credentials/google_docs_token.json b/src/server/credentials/google_docs_token.json index 2ac972ed8..f3c8cf82a 100644 --- a/src/server/credentials/google_docs_token.json +++ b/src/server/credentials/google_docs_token.json @@ -1 +1 @@ -{"access_token":"ya29.Glt5ByifP30HMz6a1fEG77qZlz9fBEOCz4PQ1VA8t_Ck2ZTPJKoyr6xc3-GFZISAqrrw2U8XpMyZv02_URfPwUX0Z_tMdlIFqsygowR-uClukbgQPNtgxp2LS1oW","refresh_token":"1/wK1cUVLnt581ba_pYGPdlTvAa-OS64nB5m5XOXEBJ8Q","scope":"https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/documents.readonly https://www.googleapis.com/auth/photoslibrary.sharing https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/photoslibrary.appendonly https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/photoslibrary","token_type":"Bearer","expiry_date":1567556163894} \ No newline at end of file +{"access_token":"ya29.Glx7B9S6zCKDE0EgYk9xX9-RhcN8j4IwG9ONopTl1NkPX9FUOw0GI_81mY9bhaouuyOTnrc6FrZD5SDHolWwp3ABNT6l7TmhTLDILgGXIixZkWFRBPpF-xHC8lUd8A","refresh_token":"1/HTv_xFHszu2Nf3iiFrUTaeKzC_Vp2-6bpIB06xW_WHI","scope":"https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/documents.readonly https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/photoslibrary https://www.googleapis.com/auth/photoslibrary.appendonly https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/photoslibrary.sharing","token_type":"Bearer","expiry_date":1567765465073} \ No newline at end of file diff --git a/src/server/index.ts b/src/server/index.ts index 3940bbd58..fab00a02d 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -42,7 +42,6 @@ var AdmZip = require('adm-zip'); import * as YoutubeApi from "./apis/youtube/youtubeApiSample"; import { Response } from 'express-serve-static-core'; import { GoogleApiServerUtils } from "./apis/google/GoogleApiServerUtils"; -import { GooglePhotos } from './apis/google/GooglePhotosServerUtils'; import { GooglePhotosUploadUtils } from './apis/google/GooglePhotosUploadUtils'; const MongoStore = require('connect-mongo')(session); const mongoose = require('mongoose'); @@ -418,10 +417,10 @@ app.get("/thumbnail/:filename", (req, res) => { let filename = req.params.filename; let noExt = filename.substring(0, filename.length - ".png".length); let pagenumber = parseInt(noExt.split('-')[1]); - fs.exists(uploadDir + filename, (exists: boolean) => { - console.log(`${uploadDir + filename} ${exists ? "exists" : "does not exist"}`); + fs.exists(uploadDirectory + filename, (exists: boolean) => { + console.log(`${uploadDirectory + filename} ${exists ? "exists" : "does not exist"}`); if (exists) { - let input = fs.createReadStream(uploadDir + filename); + let input = fs.createReadStream(uploadDirectory + filename); probe(input, (err: any, result: any) => { if (err) { console.log(err); @@ -432,7 +431,7 @@ app.get("/thumbnail/:filename", (req, res) => { }); } else { - LoadPage(uploadDir + filename.substring(0, filename.length - noExt.split('-')[1].length - ".PNG".length - 1) + ".pdf", pagenumber, res); + LoadPage(uploadDirectory + filename.substring(0, filename.length - noExt.split('-')[1].length - ".PNG".length - 1) + ".pdf", pagenumber, res); } }); }); @@ -556,13 +555,13 @@ class NodeCanvasFactory { const pngTypes = [".png", ".PNG"]; const pdfTypes = [".pdf", ".PDF"]; const jpgTypes = [".jpg", ".JPG", ".jpeg", ".JPEG"]; -const uploadDir = __dirname + "/public/files/"; +const uploadDirectory = __dirname + "/public/files/"; // SETTERS app.post( RouteStore.upload, (req, res) => { let form = new formidable.IncomingForm(); - form.uploadDir = uploadDir; + form.uploadDir = uploadDirectory; form.keepExtensions = true; // let path = req.body.path; console.log("upload"); @@ -592,7 +591,7 @@ app.post( } if (isImage) { resizers.forEach(resizer => { - fs.createReadStream(uploadDir + file).pipe(resizer.resizer).pipe(fs.createWriteStream(uploadDir + file.substring(0, file.length - ext.length) + resizer.suffix + ext)); + fs.createReadStream(uploadDirectory + file).pipe(resizer.resizer).pipe(fs.createWriteStream(uploadDirectory + file.substring(0, file.length - ext.length) + resizer.suffix + ext)); }); } names.push(`/files/` + file); @@ -611,7 +610,7 @@ addSecureRoute( res.status(401).send("incorrect parameters specified"); return; } - imageDataUri.outputFile(uri, uploadDir + filename).then((savedName: string) => { + imageDataUri.outputFile(uri, uploadDirectory + filename).then((savedName: string) => { const ext = path.extname(savedName); let resizers = [ { resizer: sharp().resize(100, undefined, { withoutEnlargement: true }), suffix: "_s" }, @@ -632,7 +631,7 @@ addSecureRoute( } if (isImage) { resizers.forEach(resizer => { - fs.createReadStream(savedName).pipe(resizer.resizer).pipe(fs.createWriteStream(uploadDir + filename + resizer.suffix + ext)); + fs.createReadStream(savedName).pipe(resizer.resizer).pipe(fs.createWriteStream(uploadDirectory + filename + resizer.suffix + ext)); }); } res.send("/files/" + filename + ext); @@ -799,8 +798,8 @@ function HandleYoutubeQuery([query, callback]: [YoutubeQueryInput, (result?: any } } -const credentials = path.join(__dirname, "./credentials/google_docs_credentials.json"); -const token = path.join(__dirname, "./credentials/google_docs_token.json"); +const credentialsPath = path.join(__dirname, "./credentials/google_docs_credentials.json"); +const tokenPath = path.join(__dirname, "./credentials/google_docs_token.json"); const EndpointHandlerMap = new Map([ ["create", (api, params) => api.create(params)], @@ -811,7 +810,7 @@ const EndpointHandlerMap = new Map { let sector: any = req.params.sector; let action: any = req.params.action; - GoogleApiServerUtils.GetEndpoint(GoogleApiServerUtils.Service[sector], { credentials, token }).then(endpoint => { + GoogleApiServerUtils.GetEndpoint(GoogleApiServerUtils.Service[sector], { credentialsPath, tokenPath }).then(endpoint => { let handler = EndpointHandlerMap.get(action); if (endpoint && handler) { let execute = handler(endpoint, req.body).then( @@ -825,36 +824,28 @@ app.post(RouteStore.googleDocs + "/:sector/:action", (req, res) => { }); }); -app.post(RouteStore.googlePhotosQuery, (req, res) => { - GoogleApiServerUtils.RetrieveAccessToken({ credentials, token }).then( - token => { - GooglePhotos.ExecuteQuery({ token, query: req.body }) - .then(response => { - if (response === undefined) { - res.send("Error: unable to build suffix for Google Photos API request"); - return; - } - res.send(response); - }) - .catch(error => { - res.send(`Error: an exception occurred in the execution of this Google Photos API request\n${error}`); - }); - }, - error => res.send(error) - ); -}); +app.get(RouteStore.googlePhotosAccessToken, (req, res) => GoogleApiServerUtils.RetrieveAccessToken({ credentialsPath, tokenPath }).then(token => res.send(token))); -app.post(RouteStore.googlePhotosMediaUpload, (req, res) => { - GoogleApiServerUtils.RetrieveAccessToken({ credentials, token }).then( - token => { - GooglePhotosUploadUtils.SubmitUpload({ token, ...req.body }) - .then(response => { - res.send(response); - }).catch(error => { - res.send(`Error: an exception occurred in uploading the given media\n${error}`); - }); - }, - error => res.send(error)); +const tokenError = "Unable to successfully upload bytes for all images!"; +const mediaError = "Unable to convert all uploaded bytes to media items!"; + +app.post(RouteStore.googlePhotosMediaUpload, async (req, res) => { + const media: GooglePhotosUploadUtils.MediaInput[] = req.body.media; + await GooglePhotosUploadUtils.initialize({ uploadDirectory, credentialsPath, tokenPath }); + const newMediaItems = await Promise.all(media.map(async element => { + const uploadToken = await GooglePhotosUploadUtils.DispatchGooglePhotosUpload(element.source); + return !uploadToken ? undefined : { + description: element.description, + simpleMediaItem: { uploadToken } + }; + })); + if (!newMediaItems.every(item => item)) { + return res.send(tokenError); + } + GooglePhotosUploadUtils.CreateMediaItems(newMediaItems, req.body.album).then( + success => res.send(success), + () => res.send(mediaError) + ); }); const suffixMap: { [type: string]: (string | [string, string | ((json: any) => any)]) } = { -- cgit v1.2.3-70-g09d2 From b24c475d8cd36af860fc374b0c5621b0d096be1d Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Mon, 9 Sep 2019 20:47:34 -0400 Subject: nearly finished transferring images between text notes and google docs --- package.json | 2 +- .../apis/google_docs/GooglePhotosClientUtils.ts | 23 ++-- .../util/Import & Export/DirectoryImportBox.tsx | 2 +- src/client/views/MainView.tsx | 6 +- src/client/views/collections/CollectionSubView.tsx | 2 +- src/client/views/nodes/DocumentView.tsx | 2 +- src/new_fields/RichTextUtils.ts | 67 ++++++++++- src/server/RouteStore.ts | 3 +- src/server/apis/google/GoogleApiServerUtils.ts | 2 +- src/server/apis/google/GooglePhotosUploadUtils.ts | 5 + src/server/credentials/google_docs_token.json | 2 +- src/server/index.ts | 20 +++- src/server/updateSearch.ts | 123 --------------------- 13 files changed, 112 insertions(+), 147 deletions(-) delete mode 100644 src/server/updateSearch.ts (limited to 'src/server/apis/google/GoogleApiServerUtils.ts') diff --git a/package.json b/package.json index f56e34ce0..f0f2b467e 100644 --- a/package.json +++ b/package.json @@ -224,4 +224,4 @@ "xoauth2": "^1.2.0", "youtube": "^0.1.0" } -} \ No newline at end of file +} diff --git a/src/client/apis/google_docs/GooglePhotosClientUtils.ts b/src/client/apis/google_docs/GooglePhotosClientUtils.ts index 5f5b39b14..fddcf3aa5 100644 --- a/src/client/apis/google_docs/GooglePhotosClientUtils.ts +++ b/src/client/apis/google_docs/GooglePhotosClientUtils.ts @@ -51,17 +51,26 @@ export namespace GooglePhotosClientUtils { description: string; } - export const UploadImageDocuments = async (sources: Doc[], album?: AlbumReference, descriptionKey = "caption") => { + export const UploadImages = async (sources: (Doc | string)[], album?: AlbumReference, descriptionKey = "caption") => { if (album && "title" in album) { album = await (await endpoint()).albums.create(album.title); } const media: MediaInput[] = []; - sources.forEach(document => { - const data = Cast(Doc.GetProto(document).data, ImageField); - data && media.push({ - url: data.url.href, - description: parseDescription(document, descriptionKey), - }); + sources.forEach(source => { + let url: string; + let description: string; + if (source instanceof Doc) { + const data = Cast(Doc.GetProto(source).data, ImageField); + if (!data) { + return; + } + url = data.url.href; + description = parseDescription(source, descriptionKey); + } else { + url = source; + description = Utils.GenerateGuid(); + } + media.push({ url, description }); }); if (media.length) { return PostToServer(RouteStore.googlePhotosMediaUpload, { media, album }); diff --git a/src/client/util/Import & Export/DirectoryImportBox.tsx b/src/client/util/Import & Export/DirectoryImportBox.tsx index 7693a388f..a19fd39b7 100644 --- a/src/client/util/Import & Export/DirectoryImportBox.tsx +++ b/src/client/util/Import & Export/DirectoryImportBox.tsx @@ -117,7 +117,7 @@ export default class DirectoryImportBox extends React.Component console.log(`(${this.quota - this.remaining}/${this.quota}) ${upload.name}`); })); - await GooglePhotosClientUtils.UploadImageDocuments(docs, { title: directory }); + await GooglePhotosClientUtils.UploadImages(docs, { title: directory }); console.log("Finished upload!"); for (let i = 0; i < docs.length; i++) { diff --git a/src/client/views/MainView.tsx b/src/client/views/MainView.tsx index 326c13424..0c0ed9072 100644 --- a/src/client/views/MainView.tsx +++ b/src/client/views/MainView.tsx @@ -155,7 +155,7 @@ export class MainView extends React.Component { doc.caption = "Well isn't this a nice cat image!"; let photos = await GooglePhotosClientUtils.endpoint(); let albumId = (await photos.albums.list(50)).albums.filter((album: any) => album.title === "This is a generically created album!")[0].id; - console.log(await GooglePhotosClientUtils.UploadImageDocuments([doc], { id: albumId })); + console.log(await GooglePhotosClientUtils.UploadImages([doc], { id: albumId })); } componentWillUnMount() { @@ -470,7 +470,7 @@ export class MainView extends React.Component { // let youtubeurl = "https://www.youtube.com/embed/TqcApsGRzWw"; // let addYoutubeSearcher = action(() => Docs.Create.YoutubeDocument(youtubeurl, { width: 600, height: 600, title: "youtube search" })); - let googlePhotosSearch = () => GooglePhotosClientUtils.CollectionFromSearch(Docs.Create.MasonryDocument, { included: [GooglePhotosClientUtils.ContentCategories.LANDSCAPES] }); + // let googlePhotosSearch = () => GooglePhotosClientUtils.CollectionFromSearch(Docs.Create.MasonryDocument, { included: [GooglePhotosClientUtils.ContentCategories.LANDSCAPES] }); let btns: [React.RefObject, IconName, string, () => Doc | Promise][] = [ [React.createRef(), "object-group", "Add Collection", addColNode], @@ -478,7 +478,7 @@ export class MainView extends React.Component { [React.createRef(), "globe-asia", "Add Website", addWebNode], [React.createRef(), "bolt", "Add Button", addButtonDocument], [React.createRef(), "file", "Add Document Dragger", addDragboxNode], - [React.createRef(), "object-group", "Test Google Photos Search", googlePhotosSearch], + // [React.createRef(), "object-group", "Test Google Photos Search", googlePhotosSearch], [React.createRef(), "cloud-upload-alt", "Import Directory", addImportCollectionNode], //remove at some point in favor of addImportCollectionNode //[React.createRef(), "play", "Add Youtube Searcher", addYoutubeSearcher], ]; diff --git a/src/client/views/collections/CollectionSubView.tsx b/src/client/views/collections/CollectionSubView.tsx index 99e5ab7b3..5fc4f36a7 100644 --- a/src/client/views/collections/CollectionSubView.tsx +++ b/src/client/views/collections/CollectionSubView.tsx @@ -253,7 +253,7 @@ export function CollectionSubView(schemaCtor: (doc: Doc) => T) { }).then(async (res: Response) => { (await res.json()).map(action((file: any) => { let full = { ...options, nativeWidth: type.indexOf("video") !== -1 ? 600 : 300, width: 300, title: dropFileName }; - let path = Utils.prepend(file); + let path = Utils.prepend(file.path); Docs.Get.DocumentFromType(type, path, full).then(doc => doc && this.props.addDocument(doc)); })); }); diff --git a/src/client/views/nodes/DocumentView.tsx b/src/client/views/nodes/DocumentView.tsx index 4033ffd9c..cb9346a8b 100644 --- a/src/client/views/nodes/DocumentView.tsx +++ b/src/client/views/nodes/DocumentView.tsx @@ -591,7 +591,7 @@ export class DocumentView extends DocComponent(Docu subitems.push({ description: "Open Fields", event: this.fieldsClicked, icon: "layer-group" }); cm.addItem({ description: "Open...", subitems: subitems, icon: "external-link-alt" }); if (Cast(this.props.Document.data, ImageField)) { - cm.addItem({ description: "Export to Google Photos", event: () => GooglePhotosClientUtils.UploadImageDocuments([this.props.Document]), icon: "caret-square-right" }); + cm.addItem({ description: "Export to Google Photos", event: () => GooglePhotosClientUtils.UploadImages([this.props.Document]), icon: "caret-square-right" }); } let existingMake = ContextMenu.Instance.findByDescription("Make..."); let makes: ContextMenuProps[] = existingMake && "subitems" in existingMake ? existingMake.subitems : []; diff --git a/src/new_fields/RichTextUtils.ts b/src/new_fields/RichTextUtils.ts index 0aba50c0d..500b93676 100644 --- a/src/new_fields/RichTextUtils.ts +++ b/src/new_fields/RichTextUtils.ts @@ -7,6 +7,11 @@ import { FormattedTextBox } from "../client/views/nodes/FormattedTextBox"; import { Opt } from "./Doc"; import * as Color from "color"; import { sinkListItem } from "prosemirror-schema-list"; +import { Utils, PostToServer } from "../Utils"; +import { RouteStore } from "../server/RouteStore"; +import { Docs } from "../client/documents/Documents"; +import { schema } from "../client/util/RichTextSchema"; +import { GooglePhotosClientUtils } from "../client/apis/google_docs/GooglePhotosClientUtils"; export namespace RichTextUtils { @@ -86,19 +91,36 @@ export namespace RichTextUtils { export namespace GoogleDocs { export const Export = (state: EditorState): GoogleApiClientUtils.Docs.Content => { - let textNodes: Node[] = []; + let nodes: { [type: string]: Node[] } = { + text: [], + image: [] + }; let text = ToPlainText(state); let content = state.doc.content; - content.forEach(node => node.content.forEach(node => node.type.name === "text" && textNodes.push(node))); - let linkRequests = ExtractLinks(textNodes); + content.forEach(node => node.content.forEach(node => { + const type = node.type.name; + let existing = nodes[type]; + if (existing) { + existing.push(node); + } else { + nodes[type] = [node]; + } + })); + let linkRequests = ExtractLinks(nodes.text); + let imageRequests = ExtractImages(nodes.image); return { text, - requests: [...linkRequests] + requests: [...linkRequests, ...imageRequests] }; }; type BulletPosition = { value: number, sinks: number }; + interface MediaItem { + baseUrl: string; + filename: string; + width: number; + } export const Import = async (documentId: GoogleApiClientUtils.Docs.DocumentId): Promise> => { const document = await GoogleApiClientUtils.Docs.retrieve({ documentId }); if (!document) { @@ -109,6 +131,17 @@ export namespace RichTextUtils { const { text, paragraphs } = GoogleApiClientUtils.Docs.Utils.extractText(document); let state = FormattedTextBox.blankState(); let structured = parseLists(paragraphs); + const inline = document.inlineObjects; + let inlineUrls: MediaItem[] = []; + if (inline) { + inlineUrls = Object.keys(inline).map(key => { + const embedded = inline[key].inlineObjectProperties!.embeddedObject!; + const baseUrl = embedded.imageProperties!.contentUri!; + const filename = `upload_${Utils.GenerateGuid()}.png`; + const width = embedded.size!.width!.magnitude!; + return { baseUrl, filename, width }; + }); + } let position = 3; let lists: ListGroup[] = []; @@ -151,6 +184,12 @@ export namespace RichTextUtils { } } + const uploads = await PostToServer(RouteStore.googlePhotosMediaDownload, { mediaItems: inlineUrls }); + for (let i = 0; i < uploads.length; i++) { + const src = Utils.prepend(`/files/${uploads[i].fileNames.clean}`); + state = state.apply(state.tr.insert(0, schema.nodes.image.create({ src, width: inlineUrls[i].width }))); + } + return { title, text, state }; }; @@ -252,6 +291,26 @@ export namespace RichTextUtils { return links; }; + const ExtractImages = async (nodes: Node[]) => { + const images: docs_v1.Schema$Request[] = []; + let position = 1; + for (let node of nodes) { + const length = node.nodeSize; + const attrs = node.attrs; + const uri = attrs.src; + const result = (await GooglePhotosClientUtils.UploadImages([uri])).newMediaItemResults; + images.push({ + insertInlineImage: { + uri: result[0].mediaItem.productUrl, + objectSize: { width: { magnitude: parseFloat(attrs.width.replace("px", "")), unit: "PT" } }, + location: { index: position + length } + } + }); + position += length; + } + return images; + }; + const Encode = (information: LinkInformation) => { return { updateTextStyle: { diff --git a/src/server/RouteStore.ts b/src/server/RouteStore.ts index f65e6134c..ee9cd8a0e 100644 --- a/src/server/RouteStore.ts +++ b/src/server/RouteStore.ts @@ -34,6 +34,7 @@ export enum RouteStore { googleDocs = "/googleDocs", googlePhotosAccessToken = "/googlePhotosAccessToken", googlePhotosMediaUpload = "/googlePhotosMediaUpload", - googlePhotosMediaDownload = "/googlePhotosMediaDownload" + googlePhotosMediaDownload = "/googlePhotosMediaDownload", + googleDocsGet = "/googleDocsGet" } \ No newline at end of file diff --git a/src/server/apis/google/GoogleApiServerUtils.ts b/src/server/apis/google/GoogleApiServerUtils.ts index ac8023ce1..e0bd8a800 100644 --- a/src/server/apis/google/GoogleApiServerUtils.ts +++ b/src/server/apis/google/GoogleApiServerUtils.ts @@ -42,7 +42,7 @@ export namespace GoogleApiServerUtils { export type ApiResponse = Promise; export type ApiRouter = (endpoint: Endpoint, parameters: any) => ApiResponse; - export type ApiHandler = (parameters: any) => ApiResponse; + export type ApiHandler = (parameters: any, methodOptions?: any) => ApiResponse; export type Action = "create" | "retrieve" | "update"; export type Endpoint = { get: ApiHandler, create: ApiHandler, batchUpdate: ApiHandler }; diff --git a/src/server/apis/google/GooglePhotosUploadUtils.ts b/src/server/apis/google/GooglePhotosUploadUtils.ts index 35f986250..d1f1f81bd 100644 --- a/src/server/apis/google/GooglePhotosUploadUtils.ts +++ b/src/server/apis/google/GooglePhotosUploadUtils.ts @@ -151,6 +151,11 @@ export namespace DownloadUtils { .on('close', resolve) .on('error', reject); }); + if (!isLocal) { + await new Promise(resolve => { + stream(url).pipe(fs.createWriteStream(uploadDirectory + resolved)).on('close', resolve); + }); + } } resolve(information); }); diff --git a/src/server/credentials/google_docs_token.json b/src/server/credentials/google_docs_token.json index fabc18cfd..5c142fba1 100644 --- a/src/server/credentials/google_docs_token.json +++ b/src/server/credentials/google_docs_token.json @@ -1 +1 @@ -{"access_token":"ya29.Glx-BwgWcpQUukTNyuUqvSAYrDyxDNUhCLtrFDJAViROvicm0DrcRvCn4OaQdn2m2IZQYcG-19cvQYoOC3UJCtWXLRvKZzQCbZZSykpxYu_lflUyEnIGZOIHMbbEjA","refresh_token":"1/HTv_xFHszu2Nf3iiFrUTaeKzC_Vp2-6bpIB06xW_WHI","scope":"https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/documents.readonly https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/photoslibrary https://www.googleapis.com/auth/photoslibrary.appendonly https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/photoslibrary.sharing","token_type":"Bearer","expiry_date":1568008211814} \ No newline at end of file +{"access_token":"ya29.Glx_B6G7Q_FYs1LK5VcyV6P6Zg9JkoHO2aC_TsnN7AVxPYWHEpsBSC0WyWX7Ztr8HWhOUYA5JXqnZDkLrK1V3Hb-0GgtyApLRNtEPOWf1dJ7lOm_iKVw2tRvPe7XDQ","refresh_token":"1/HTv_xFHszu2Nf3iiFrUTaeKzC_Vp2-6bpIB06xW_WHI","scope":"https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/documents.readonly https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/photoslibrary https://www.googleapis.com/auth/photoslibrary.appendonly https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/photoslibrary.sharing","token_type":"Bearer","expiry_date":1568078116605} \ No newline at end of file diff --git a/src/server/index.ts b/src/server/index.ts index baef94a59..8469770d5 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -38,6 +38,7 @@ import flash = require('connect-flash'); import { Search } from './Search'; import _ = require('lodash'); import * as Archiver from 'archiver'; +import * as request_promise from 'request-promise'; var AdmZip = require('adm-zip'); import * as YoutubeApi from "./apis/youtube/youtubeApiSample"; import { Response } from 'express-serve-static-core'; @@ -576,9 +577,9 @@ app.post( form.parse(req, async (_err, _fields, files) => { let results: FileResponse[] = []; for (const key in files) { - const { name, type, path: location } = files[key]; + const { type, path: location, name } = files[key]; const filename = path.basename(location); - await UploadUtils.UploadImage(uploadDirectory + filename, path.basename(name)); + await UploadUtils.UploadImage(uploadDirectory + filename, filename); results.push({ name, type, path: `/files/${filename}` }); console.log(path.basename(name)); } @@ -790,10 +791,23 @@ const tokenPath = path.join(__dirname, "./credentials/google_docs_token.json"); const EndpointHandlerMap = new Map([ ["create", (api, params) => api.create(params)], - ["retrieve", (api, params) => api.get(params)], + ["retrieve", (api, params) => api.get(params, { params: "fields=inlineObjects" })], ["update", (api, params) => api.batchUpdate(params)], ]); +// app.post(RouteStore.googleDocsGet, async (req, res) => { +// const token = await GoogleApiServerUtils.RetrieveAccessToken({ credentialsPath, tokenPath }); +// request_promise.get({ +// uri: `https://docs.googleapis.com/v1/documents/${req.body.documentId}?fields=inlineObjects`, +// headers: { +// 'Authorization': `Bearer ${token}` +// } +// }).then(response => { +// console.log(response); +// res.send(response); +// }); +// }); + app.post(RouteStore.googleDocs + "/:sector/:action", (req, res) => { let sector: GoogleApiServerUtils.Service = req.params.sector; let action: GoogleApiServerUtils.Action = req.params.action; diff --git a/src/server/updateSearch.ts b/src/server/updateSearch.ts deleted file mode 100644 index 906b795f1..000000000 --- a/src/server/updateSearch.ts +++ /dev/null @@ -1,123 +0,0 @@ -import { Database } from "./database"; -import { Cursor } from "mongodb"; -import { Search } from "./Search"; -import pLimit from 'p-limit'; - -const suffixMap: { [type: string]: (string | [string, string | ((json: any) => any)]) } = { - "number": "_n", - "string": "_t", - "boolean": "_b", - // "image": ["_t", "url"], - "video": ["_t", "url"], - "pdf": ["_t", "url"], - "audio": ["_t", "url"], - "web": ["_t", "url"], - "date": ["_d", value => new Date(value.date).toISOString()], - "proxy": ["_i", "fieldId"], - "list": ["_l", list => { - const results = []; - for (const value of list.fields) { - const term = ToSearchTerm(value); - if (term) { - results.push(term.value); - } - } - return results.length ? results : null; - }] -}; - -function ToSearchTerm(val: any): { suffix: string, value: any } | undefined { - if (val === null || val === undefined) { - return; - } - const type = val.__type || typeof val; - let suffix = suffixMap[type]; - if (!suffix) { - return; - } - - if (Array.isArray(suffix)) { - const accessor = suffix[1]; - if (typeof accessor === "function") { - val = accessor(val); - } else { - val = val[accessor]; - } - suffix = suffix[0]; - } - - return { suffix, value: val }; -} - -function getSuffix(value: string | [string, any]): string { - return typeof value === "string" ? value : value[0]; -} - -const limit = pLimit(5); -async function update() { - // await new Promise(res => setTimeout(res, 5)); - console.log("update"); - await Search.Instance.clear(); - const cursor = await Database.Instance.query({}); - console.log("Cleared"); - const updates: any[] = []; - let numDocs = 0; - function updateDoc(doc: any) { - numDocs++; - if ((numDocs % 50) === 0) { - console.log("updateDoc " + numDocs); - } - // console.log("doc " + numDocs); - if (doc.__type !== "Doc") { - return; - } - const fields = doc.fields; - if (!fields) { - return; - } - const update: any = { id: doc._id }; - let dynfield = false; - for (const key in fields) { - const value = fields[key]; - const term = ToSearchTerm(value); - if (term !== undefined) { - let { suffix, value } = term; - update[key + suffix] = value; - dynfield = true; - } - } - if (dynfield) { - updates.push(update); - // console.log(updates.length); - } - } - await cursor.forEach(updateDoc); - console.log(`Updating ${updates.length} documents`); - const result = await Search.Instance.updateDocuments(updates); - try { - console.log(JSON.parse(result).responseHeader.status); - } catch { - console.log("Error:"); - // console.log(updates[i]); - console.log(result); - console.log("\n"); - } - // for (let i = 0; i < updates.length; i++) { - // console.log(i); - // const result = await Search.Instance.updateDocument(updates[i]); - // try { - // console.log(JSON.parse(result).responseHeader.status); - // } catch { - // console.log("Error:"); - // console.log(updates[i]); - // console.log(result); - // console.log("\n"); - // } - // } - // await Promise.all(updates.map(update => { - // return limit(() => Search.Instance.updateDocument(update)); - // })); - cursor.close(); -} - -update(); \ No newline at end of file -- cgit v1.2.3-70-g09d2 From 9b27f1ace4655f71a67ad68e1f6f6bba82f41e46 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Tue, 24 Sep 2019 19:09:33 -0400 Subject: now store cache in mongodb collection, fixed extension as --- src/Utils.ts | 6 +- .../apis/google_docs/GooglePhotosClientUtils.ts | 8 +- src/client/views/Main.tsx | 3 + src/extensions/ArrayExtensions.ts | 325 +---------------- src/extensions/Extensions.ts | 7 - src/extensions/General/Extensions.ts | 9 + src/extensions/General/ExtensionsTypings.ts | 8 + src/extensions/StringExtensions.ts | 11 +- src/server/DashUploadUtils.ts | 143 ++++++++ .../apis/google/CustomizedWrapper/filters.js | 46 --- src/server/apis/google/GoogleApiServerUtils.ts | 32 +- src/server/apis/google/GooglePhotosUploadUtils.ts | 147 +------- src/server/apis/google/existing_uploads.json | 0 src/server/credentials/google_docs_token.json | 8 +- src/server/database.ts | 402 ++++++++++++--------- src/server/index.ts | 49 +-- 16 files changed, 465 insertions(+), 739 deletions(-) delete mode 100644 src/extensions/Extensions.ts create mode 100644 src/extensions/General/Extensions.ts create mode 100644 src/extensions/General/ExtensionsTypings.ts create mode 100644 src/server/DashUploadUtils.ts delete mode 100644 src/server/apis/google/CustomizedWrapper/filters.js delete mode 100644 src/server/apis/google/existing_uploads.json (limited to 'src/server/apis/google/GoogleApiServerUtils.ts') diff --git a/src/Utils.ts b/src/Utils.ts index a842f5a20..aa2998971 100644 --- a/src/Utils.ts +++ b/src/Utils.ts @@ -4,6 +4,7 @@ import { Socket } from 'socket.io'; import { Message } from './server/Message'; import { RouteStore } from './server/RouteStore'; import requestPromise = require('request-promise'); +import { CurrentUserUtils } from './server/authentication/models/current_user_utils'; export class Utils { @@ -293,12 +294,13 @@ export namespace JSONUtils { } -export function PostToServer(relativeRoute: string, body: any) { +export function PostToServer(relativeRoute: string, body?: any) { + body = { userId: CurrentUserUtils.id, ...body }; let options = { method: "POST", uri: Utils.prepend(relativeRoute), json: true, - body: body + body }; return requestPromise.post(options); } \ No newline at end of file diff --git a/src/client/apis/google_docs/GooglePhotosClientUtils.ts b/src/client/apis/google_docs/GooglePhotosClientUtils.ts index 671d05421..0e09ad85b 100644 --- a/src/client/apis/google_docs/GooglePhotosClientUtils.ts +++ b/src/client/apis/google_docs/GooglePhotosClientUtils.ts @@ -12,17 +12,11 @@ import { FormattedTextBox } from "../../views/nodes/FormattedTextBox"; import { Docs, DocumentOptions } from "../../documents/Documents"; import { NewMediaItemResult, MediaItem } from "../../../server/apis/google/SharedTypes"; import { AssertionError } from "assert"; -import { List } from "../../../new_fields/List"; -import { listSpec } from "../../../new_fields/Schema"; import { DocumentView } from "../../views/nodes/DocumentView"; export namespace GooglePhotos { - const endpoint = async () => { - const getToken = Utils.prepend(RouteStore.googlePhotosAccessToken); - const token = await (await fetch(getToken)).text(); - return new Photos(token); - }; + const endpoint = async () => new Photos(await PostToServer(RouteStore.googlePhotosAccessToken)); export enum MediaType { ALL_MEDIA = 'ALL_MEDIA', diff --git a/src/client/views/Main.tsx b/src/client/views/Main.tsx index 70d2235e6..3bd898ac0 100644 --- a/src/client/views/Main.tsx +++ b/src/client/views/Main.tsx @@ -7,6 +7,9 @@ import { Cast } from "../../new_fields/Types"; import { Doc, DocListCastAsync } from "../../new_fields/Doc"; import { List } from "../../new_fields/List"; import { DocServer } from "../DocServer"; +import { AssignAllExtensions } from "../../extensions/General/Extensions"; + +AssignAllExtensions(); let swapDocs = async () => { let oldDoc = await Cast(CurrentUserUtils.UserDocument.linkManagerDoc, Doc); diff --git a/src/extensions/ArrayExtensions.ts b/src/extensions/ArrayExtensions.ts index ca407862b..422a10dbc 100644 --- a/src/extensions/ArrayExtensions.ts +++ b/src/extensions/ArrayExtensions.ts @@ -1,318 +1,13 @@ -interface Array { - lastElement(): T; -} - -// interface BatchContext { -// completedBatches: number; -// remainingBatches: number; -// } - -// interface ExecutorResult { -// updated: A; -// makeNextBatch: boolean; -// } - -// interface PredicateBatcherCommon { -// initial: A; -// persistAccumulator?: boolean; -// } - -// interface Interval { -// magnitude: number; -// unit: typeof module.exports.TimeUnit; -// } - -// type BatchConverterSync = (batch: I[], context: BatchContext) => O[]; -// type BatchConverterAsync = (batch: I[], context: BatchContext) => Promise; -// type BatchConverter = BatchConverterSync | BatchConverterAsync; - -// type BatchHandlerSync = (batch: I[], context: BatchContext) => void; -// type BatchHandlerAsync = (batch: I[], context: BatchContext) => Promise; -// type BatchHandler = BatchHandlerSync | BatchHandlerAsync; - -// type BatcherSync = FixedBatcher | PredicateBatcherSync; -// type BatcherAsync = PredicateBatcherAsync; -// type Batcher = BatcherSync | BatcherAsync; - -// type FixedBatcher = { batchSize: number } | { batchCount: number, mode?: typeof module.exports.Mode }; -// type PredicateBatcherSync = PredicateBatcherCommon & { executor: (element: I, accumulator: A) => ExecutorResult }; -// type PredicateBatcherAsync = PredicateBatcherCommon & { executorAsync: (element: I, accumulator: A) => Promise> }; - - -// module.exports.Mode = { -// Balanced: 0, -// Even: 1 -// }; - -// module.exports.TimeUnit = { -// Milliseconds: 0, -// Seconds: 1, -// Minutes: 2 -// }; - -// module.exports.Assign = function () { - -// Array.prototype.fixedBatch = function (batcher: FixedBatcher): T[][] { -// const batches: T[][] = []; -// const length = this.length; -// let i = 0; -// if ("batchSize" in batcher) { -// const { batchSize } = batcher; -// while (i < this.length) { -// const cap = Math.min(i + batchSize, length); -// batches.push(this.slice(i, i = cap)); -// } -// } else if ("batchCount" in batcher) { -// let { batchCount, mode } = batcher; -// const resolved = mode || module.exports.Mode.Balanced; -// if (batchCount < 1) { -// throw new Error("Batch count must be a positive integer!"); -// } -// if (batchCount === 1) { -// return [this]; -// } -// if (batchCount >= this.length) { -// return this.map((element: T) => [element]); -// } +function Assign() { -// let length = this.length; -// let size: number; + Array.prototype.lastElement = function () { + if (!this.length) { + return undefined; + } + const last: T = this[this.length - 1]; + return last; + }; -// if (length % batchCount === 0) { -// size = Math.floor(length / batchCount); -// while (i < length) { -// batches.push(this.slice(i, i += size)); -// } -// } else if (resolved === module.exports.Mode.Balanced) { -// while (i < length) { -// size = Math.ceil((length - i) / batchCount--); -// batches.push(this.slice(i, i += size)); -// } -// } else { -// batchCount--; -// size = Math.floor(length / batchCount); -// if (length % size === 0) { -// size--; -// } -// while (i < size * batchCount) { -// batches.push(this.slice(i, i += size)); -// } -// batches.push(this.slice(size * batchCount)); -// } -// } -// return batches; -// }; - -// Array.prototype.predicateBatch = function (batcher: PredicateBatcherSync): T[][] { -// const batches: T[][] = []; -// let batch: T[] = []; -// const { executor, initial, persistAccumulator } = batcher; -// let accumulator = initial; -// for (let element of this) { -// const { updated, makeNextBatch } = executor(element, accumulator); -// accumulator = updated; -// if (!makeNextBatch) { -// batch.push(element); -// } else { -// batches.push(batch); -// batch = [element]; -// if (!persistAccumulator) { -// accumulator = initial; -// } -// } -// } -// batches.push(batch); -// return batches; -// }; - -// Array.prototype.predicateBatchAsync = async function (batcher: BatcherAsync): Promise { -// const batches: T[][] = []; -// let batch: T[] = []; -// const { executorAsync, initial, persistAccumulator } = batcher; -// let accumulator: A = initial; -// for (let element of this) { -// const { updated, makeNextBatch } = await executorAsync(element, accumulator); -// accumulator = updated; -// if (!makeNextBatch) { -// batch.push(element); -// } else { -// batches.push(batch); -// batch = [element]; -// if (!persistAccumulator) { -// accumulator = initial; -// } -// } -// } -// batches.push(batch); -// return batches; -// }; - -// Array.prototype.batch = function (batcher: BatcherSync): T[][] { -// if ("executor" in batcher) { -// return this.predicateBatch(batcher); -// } else { -// return this.fixedBatch(batcher); -// } -// }; - -// Array.prototype.batchAsync = async function (batcher: Batcher): Promise { -// if ("executorAsync" in batcher) { -// return this.predicateBatchAsync(batcher); -// } else { -// return this.batch(batcher); -// } -// }; - -// Array.prototype.batchedForEach = function (batcher: BatcherSync, handler: BatchHandlerSync): void { -// if (this.length) { -// let completed = 0; -// const batches = this.batch(batcher); -// const quota = batches.length; -// for (let batch of batches) { -// const context: BatchContext = { -// completedBatches: completed, -// remainingBatches: quota - completed, -// }; -// handler(batch, context); -// completed++; -// } -// } -// }; - -// Array.prototype.batchedMap = function (batcher: BatcherSync, handler: BatchConverterSync): O[] { -// if (!this.length) { -// return []; -// } -// let collector: O[] = []; -// let completed = 0; -// const batches = this.batch(batcher); -// const quota = batches.length; -// for (let batch of batches) { -// const context: BatchContext = { -// completedBatches: completed, -// remainingBatches: quota - completed, -// }; -// collector.push(...handler(batch, context)); -// completed++; -// } -// return collector; -// }; - -// Array.prototype.batchedForEachAsync = async function (batcher: Batcher, handler: BatchHandler): Promise { -// if (this.length) { -// let completed = 0; -// const batches = await this.batchAsync(batcher); -// const quota = batches.length; -// for (let batch of batches) { -// const context: BatchContext = { -// completedBatches: completed, -// remainingBatches: quota - completed, -// }; -// await handler(batch, context); -// completed++; -// } -// } -// }; - -// Array.prototype.batchedMapAsync = async function (batcher: Batcher, handler: BatchConverter): Promise { -// if (!this.length) { -// return []; -// } -// let collector: O[] = []; -// let completed = 0; -// const batches = await this.batchAsync(batcher); -// const quota = batches.length; -// for (let batch of batches) { -// const context: BatchContext = { -// completedBatches: completed, -// remainingBatches: quota - completed, -// }; -// collector.push(...(await handler(batch, context))); -// completed++; -// } -// return collector; -// }; - -// Array.prototype.batchedForEachInterval = async function (batcher: Batcher, handler: BatchHandler, interval: Interval): Promise { -// if (!this.length) { -// return; -// } -// const batches = await this.batchAsync(batcher); -// const quota = batches.length; -// return new Promise(async resolve => { -// const iterator = batches[Symbol.iterator](); -// let completed = 0; -// while (true) { -// const next = iterator.next(); -// await new Promise(resolve => { -// setTimeout(async () => { -// const batch = next.value; -// const context: BatchContext = { -// completedBatches: completed, -// remainingBatches: quota - completed, -// }; -// await handler(batch, context); -// resolve(); -// }, convert(interval)); -// }); -// if (++completed === quota) { -// break; -// } -// } -// resolve(); -// }); -// }; - -// Array.prototype.batchedMapInterval = async function (batcher: Batcher, handler: BatchConverter, interval: Interval): Promise { -// if (!this.length) { -// return []; -// } -// let collector: O[] = []; -// const batches = await this.batchAsync(batcher); -// const quota = batches.length; -// return new Promise(async resolve => { -// const iterator = batches[Symbol.iterator](); -// let completed = 0; -// while (true) { -// const next = iterator.next(); -// await new Promise(resolve => { -// setTimeout(async () => { -// const batch = next.value; -// const context: BatchContext = { -// completedBatches: completed, -// remainingBatches: quota - completed, -// }; -// collector.push(...(await handler(batch, context))); -// resolve(); -// }, convert(interval)); -// }); -// if (++completed === quota) { -// resolve(collector); -// break; -// } -// } -// }); -// }; - -Array.prototype.lastElement = function () { - if (!this.length) { - return undefined; - } - const last: T = this[this.length - 1]; - return last; -}; - -// }; +} -// const convert = (interval: Interval) => { -// const { magnitude, unit } = interval; -// switch (unit) { -// default: -// case module.exports.TimeUnit.Milliseconds: -// return magnitude; -// case module.exports.TimeUnit.Seconds: -// return magnitude * 1000; -// case module.exports.TimeUnit.Minutes: -// return magnitude * 1000 * 60; -// } -// }; \ No newline at end of file +export { Assign }; \ No newline at end of file diff --git a/src/extensions/Extensions.ts b/src/extensions/Extensions.ts deleted file mode 100644 index 1391140b9..000000000 --- a/src/extensions/Extensions.ts +++ /dev/null @@ -1,7 +0,0 @@ -const ArrayExtensions = require("./ArrayExtensions"); -const StringExtensions = require("./StringExtensions"); - -module.exports.AssignExtensions = function () { - // ArrayExtensions.Assign(); - StringExtensions.Assign(); -}; \ No newline at end of file diff --git a/src/extensions/General/Extensions.ts b/src/extensions/General/Extensions.ts new file mode 100644 index 000000000..4b6d05d5f --- /dev/null +++ b/src/extensions/General/Extensions.ts @@ -0,0 +1,9 @@ +import { Assign as ArrayAssign } from "../ArrayExtensions"; +import { Assign as StringAssign } from "../StringExtensions"; + +function AssignAllExtensions() { + ArrayAssign(); + StringAssign(); +} + +export { AssignAllExtensions }; \ No newline at end of file diff --git a/src/extensions/General/ExtensionsTypings.ts b/src/extensions/General/ExtensionsTypings.ts new file mode 100644 index 000000000..370157ed0 --- /dev/null +++ b/src/extensions/General/ExtensionsTypings.ts @@ -0,0 +1,8 @@ +interface Array { + lastElement(): T; +} + +interface String { + removeTrailingNewlines(): string; + hasNewline(): boolean; +} \ No newline at end of file diff --git a/src/extensions/StringExtensions.ts b/src/extensions/StringExtensions.ts index 4cdbdebf7..2c76e56c8 100644 --- a/src/extensions/StringExtensions.ts +++ b/src/extensions/StringExtensions.ts @@ -1,9 +1,4 @@ -interface String { - removeTrailingNewlines(): string; - hasNewline(): boolean; -} - -module.exports.Assign = function () { +function Assign() { String.prototype.removeTrailingNewlines = function () { let sliced = this; @@ -17,4 +12,6 @@ module.exports.Assign = function () { return this.endsWith("\n"); }; -}; \ No newline at end of file +} + +export { Assign }; \ No newline at end of file diff --git a/src/server/DashUploadUtils.ts b/src/server/DashUploadUtils.ts new file mode 100644 index 000000000..66874e96c --- /dev/null +++ b/src/server/DashUploadUtils.ts @@ -0,0 +1,143 @@ +import * as fs from 'fs'; +import { Utils } from '../Utils'; +import * as path from 'path'; +import { Opt } from '../new_fields/Doc'; +import * as sharp from 'sharp'; +import request = require('request-promise'); + +const uploadDirectory = path.join(__dirname, './public/files/'); + +export namespace DashUploadUtils { + + export interface Size { + width: number; + suffix: string; + } + + export const Sizes: { [size: string]: Size } = { + SMALL: { width: 100, suffix: "_s" }, + MEDIUM: { width: 400, suffix: "_m" }, + LARGE: { width: 900, suffix: "_l" }, + }; + + const gifs = [".gif"]; + const pngs = [".png"]; + const jpgs = [".jpg", ".jpeg"]; + const imageFormats = [...pngs, ...jpgs, ...gifs]; + const videoFormats = [".mov", ".mp4"]; + + const size = "content-length"; + const type = "content-type"; + + export interface UploadInformation { + mediaPaths: string[]; + fileNames: { [key: string]: string }; + contentSize?: number; + contentType?: string; + } + + const generate = (prefix: string, url: string) => `${prefix}upload_${Utils.GenerateGuid()}${path.extname(url).toLowerCase()}`; + const sanitize = (filename: string) => filename.replace(/\s+/g, "_"); + + export interface InspectionResults { + isLocal: boolean; + stream: any; + normalizedUrl: string; + contentSize?: number; + contentType?: string; + } + + export const InspectImage = async (url: string): Promise => { + const { isLocal, stream, normalized: normalizedUrl } = classify(url); + const results = { + isLocal, + stream, + normalizedUrl + }; + if (isLocal) { + return results; + } + const metadata = (await new Promise((resolve, reject) => { + request.head(url, async (error, res) => { + if (error) { + return reject(error); + } + resolve(res); + }); + })).headers; + return { + contentSize: parseInt(metadata[size]), + contentType: metadata[type], + ...results + }; + }; + + export const UploadImage = async (metadata: InspectionResults, filename?: string, prefix = ""): Promise> => { + const { isLocal, stream, normalizedUrl, contentSize, contentType } = metadata; + const resolved = filename ? sanitize(filename) : generate(prefix, normalizedUrl); + let extension = path.extname(normalizedUrl) || path.extname(resolved); + extension && (extension = extension.toLowerCase()); + let information: UploadInformation = { + mediaPaths: [], + fileNames: { clean: resolved }, + contentSize, + contentType, + }; + return new Promise(async (resolve, reject) => { + const resizers = [ + { resizer: sharp().rotate(), suffix: "_o" }, + ...Object.values(Sizes).map(size => ({ + resizer: sharp().resize(size.width, undefined, { withoutEnlargement: true }).rotate(), + suffix: size.suffix + })) + ]; + let nonVisual = false; + if (pngs.includes(extension)) { + resizers.forEach(element => element.resizer = element.resizer.png()); + } else if (jpgs.includes(extension)) { + resizers.forEach(element => element.resizer = element.resizer.jpeg()); + } else if (![...imageFormats, ...videoFormats].includes(extension.toLowerCase())) { + nonVisual = true; + } + if (imageFormats.includes(extension)) { + for (let resizer of resizers) { + const suffix = resizer.suffix; + let mediaPath: string; + await new Promise(resolve => { + const filename = resolved.substring(0, resolved.length - extension.length) + suffix + extension; + information.mediaPaths.push(mediaPath = uploadDirectory + filename); + information.fileNames[suffix] = filename; + stream(normalizedUrl).pipe(resizer.resizer).pipe(fs.createWriteStream(mediaPath)) + .on('close', resolve) + .on('error', reject); + }); + } + } + if (!isLocal || nonVisual) { + await new Promise(resolve => { + stream(normalizedUrl).pipe(fs.createWriteStream(uploadDirectory + resolved)).on('close', resolve); + }); + } + resolve(information); + }); + }; + + const classify = (url: string) => { + const isLocal = /Dash-Web(\\|\/)src(\\|\/)server(\\|\/)public(\\|\/)files/g.test(url); + return { + isLocal, + stream: isLocal ? fs.createReadStream : request, + normalized: isLocal ? path.normalize(url) : url + }; + }; + + export const createIfNotExists = async (path: string) => { + if (await new Promise(resolve => fs.exists(path, resolve))) { + return true; + } + return new Promise(resolve => fs.mkdir(path, error => resolve(error === null))); + }; + + export const Destroy = (mediaPath: string) => new Promise(resolve => fs.unlink(mediaPath, error => resolve(error === null))); + +} \ No newline at end of file diff --git a/src/server/apis/google/CustomizedWrapper/filters.js b/src/server/apis/google/CustomizedWrapper/filters.js deleted file mode 100644 index 576a90b75..000000000 --- a/src/server/apis/google/CustomizedWrapper/filters.js +++ /dev/null @@ -1,46 +0,0 @@ -'use strict'; - -const DateFilter = require('../common/date_filter'); -const MediaTypeFilter = require('./media_type_filter'); -const ContentFilter = require('./content_filter'); - -class Filters { - constructor(includeArchivedMedia = false) { - this.includeArchivedMedia = includeArchivedMedia; - } - - setDateFilter(dateFilter) { - this.dateFilter = dateFilter; - return this; - } - - setContentFilter(contentFilter) { - this.contentFilter = contentFilter; - return this; - } - - setMediaTypeFilter(mediaTypeFilter) { - this.mediaTypeFilter = mediaTypeFilter; - return this; - } - - setIncludeArchivedMedia(includeArchivedMedia) { - this.includeArchivedMedia = includeArchivedMedia; - return this; - } - - toJSON() { - return { - dateFilter: this.dateFilter instanceof DateFilter ? this.dateFilter.toJSON() : this.dateFilter, - mediaTypeFilter: this.mediaTypeFilter instanceof MediaTypeFilter ? - this.mediaTypeFilter.toJSON() : - this.mediaTypeFilter, - contentFilter: this.contentFilter instanceof ContentFilter ? - this.contentFilter.toJSON() : - this.contentFilter, - includeArchivedMedia: this.includeArchivedMedia - }; - } -} - -module.exports = Filters; \ No newline at end of file diff --git a/src/server/apis/google/GoogleApiServerUtils.ts b/src/server/apis/google/GoogleApiServerUtils.ts index e0bd8a800..684a8081b 100644 --- a/src/server/apis/google/GoogleApiServerUtils.ts +++ b/src/server/apis/google/GoogleApiServerUtils.ts @@ -8,7 +8,7 @@ import { GaxiosResponse } from "gaxios"; import request = require('request-promise'); import * as qs from 'query-string'; import Photos = require('googlephotos'); - +import { Database } from "../../database"; /** * Server side authentication for Google Api queries. */ @@ -35,9 +35,9 @@ export namespace GoogleApiServerUtils { Slides = "Slides" } - export interface CredentialPaths { + export interface CredentialInformation { credentialsPath: string; - tokenPath: string; + userId: string; } export type ApiResponse = Promise; @@ -48,7 +48,7 @@ export namespace GoogleApiServerUtils { export type Endpoint = { get: ApiHandler, create: ApiHandler, batchUpdate: ApiHandler }; export type EndpointParameters = GlobalOptions & { version: "v1" }; - export const GetEndpoint = (sector: string, paths: CredentialPaths) => { + export const GetEndpoint = (sector: string, paths: CredentialInformation) => { return new Promise>(resolve => { RetrieveCredentials(paths).then(authentication => { let routed: Opt; @@ -66,28 +66,28 @@ export namespace GoogleApiServerUtils { }); }; - export const RetrieveCredentials = (paths: CredentialPaths) => { + export const RetrieveCredentials = (information: CredentialInformation) => { return new Promise((resolve, reject) => { - readFile(paths.credentialsPath, async (err, credentials) => { + readFile(information.credentialsPath, async (err, credentials) => { if (err) { reject(err); return console.log('Error loading client secret file:', err); } - authorize(parseBuffer(credentials), paths.tokenPath).then(resolve, reject); + authorize(parseBuffer(credentials), information.userId).then(resolve, reject); }); }); }; - export const RetrieveAccessToken = (paths: CredentialPaths) => { + export const RetrieveAccessToken = (information: CredentialInformation) => { return new Promise((resolve, reject) => { - RetrieveCredentials(paths).then( + RetrieveCredentials(information).then( credentials => resolve(credentials.token.access_token!), error => reject(`Error: unable to authenticate Google Photos API request.\n${error}`) ); }); }; - export const RetrievePhotosEndpoint = (paths: CredentialPaths) => { + export const RetrievePhotosEndpoint = (paths: CredentialInformation) => { return new Promise((resolve, reject) => { RetrieveAccessToken(paths).then( token => resolve(new Photos(token)), @@ -101,20 +101,20 @@ export namespace GoogleApiServerUtils { * Create an OAuth2 client with the given credentials, and returns the promise resolving to the authenticated client * @param {Object} credentials The authorization client credentials. */ - export function authorize(credentials: any, token_path: string): Promise { + export function authorize(credentials: any, userId: string): Promise { const { client_secret, client_id, redirect_uris } = credentials.installed; const oAuth2Client = new google.auth.OAuth2( client_id, client_secret, redirect_uris[0]); return new Promise((resolve, reject) => { - readFile(token_path, (err, token) => { - // Check if we have previously stored a token. - if (err) { - return getNewToken(oAuth2Client, token_path).then(resolve, reject); + Database.Auxiliary.FetchGoogleAuthenticationToken(userId).then(token => { + // Check if we have previously stored a token for this userId. + if (!token) { + return getNewToken(oAuth2Client, userId).then(resolve, reject); } let parsed: Credentials = parseBuffer(token); if (parsed.expiry_date! < new Date().getTime()) { - return refreshToken(parsed, client_id, client_secret, oAuth2Client, token_path).then(resolve, reject); + return refreshToken(parsed, client_id, client_secret, oAuth2Client, userId).then(resolve, reject); } oAuth2Client.setCredentials(parsed); resolve({ token: parsed, client: oAuth2Client }); diff --git a/src/server/apis/google/GooglePhotosUploadUtils.ts b/src/server/apis/google/GooglePhotosUploadUtils.ts index 4dc252577..507a868a3 100644 --- a/src/server/apis/google/GooglePhotosUploadUtils.ts +++ b/src/server/apis/google/GooglePhotosUploadUtils.ts @@ -1,16 +1,10 @@ import request = require('request-promise'); import { GoogleApiServerUtils } from './GoogleApiServerUtils'; -import * as fs from 'fs'; -import { Utils } from '../../../Utils'; import * as path from 'path'; -import { Opt } from '../../../new_fields/Doc'; -import * as sharp from 'sharp'; import { MediaItemCreationResult } from './SharedTypes'; import { NewMediaItem } from "../../index"; import BatchedArray, { FixedBatcher, TimeUnit, Interval } from "array-batcher"; -const uploadDirectory = path.join(__dirname, "../../public/files/"); - export namespace GooglePhotosUploadUtils { export interface Paths { @@ -31,12 +25,9 @@ export namespace GooglePhotosUploadUtils { }); let Bearer: string; - let Paths: Paths; - export const initialize = async (paths: Paths) => { - Paths = paths; - const { tokenPath, credentialsPath } = paths; - const token = await GoogleApiServerUtils.RetrieveAccessToken({ tokenPath, credentialsPath }); + export const initialize = async (information: GoogleApiServerUtils.CredentialInformation) => { + const token = await GoogleApiServerUtils.RetrieveAccessToken(information); Bearer = `Bearer ${token}`; }; @@ -87,138 +78,4 @@ export namespace GooglePhotosUploadUtils { return { newMediaItemResults }; }; -} - -export namespace DownloadUtils { - - export interface Size { - width: number; - suffix: string; - } - - export const Sizes: { [size: string]: Size } = { - SMALL: { width: 100, suffix: "_s" }, - MEDIUM: { width: 400, suffix: "_m" }, - LARGE: { width: 900, suffix: "_l" }, - }; - - const gifs = [".gif"]; - const pngs = [".png"]; - const jpgs = [".jpg", ".jpeg"]; - const imageFormats = [...pngs, ...jpgs, ...gifs]; - const videoFormats = [".mov", ".mp4"]; - - const size = "content-length"; - const type = "content-type"; - - export interface UploadInformation { - mediaPaths: string[]; - fileNames: { [key: string]: string }; - contentSize?: number; - contentType?: string; - } - - const generate = (prefix: string, url: string) => `${prefix}upload_${Utils.GenerateGuid()}${path.extname(url).toLowerCase()}`; - const sanitize = (filename: string) => filename.replace(/\s+/g, "_"); - - export interface InspectionResults { - isLocal: boolean; - stream: any; - normalizedUrl: string; - contentSize?: number; - contentType?: string; - } - - export const InspectImage = async (url: string): Promise => { - const { isLocal, stream, normalized: normalizedUrl } = classify(url); - const results = { - isLocal, - stream, - normalizedUrl - }; - if (isLocal) { - return results; - } - const metadata = (await new Promise((resolve, reject) => { - request.head(url, async (error, res) => { - if (error) { - return reject(error); - } - resolve(res); - }); - })).headers; - return { - contentSize: parseInt(metadata[size]), - contentType: metadata[type], - ...results - }; - }; - - export const UploadImage = async (metadata: InspectionResults, filename?: string, prefix = ""): Promise> => { - const { isLocal, stream, normalizedUrl, contentSize, contentType } = metadata; - const resolved = filename ? sanitize(filename) : generate(prefix, normalizedUrl); - let extension = path.extname(normalizedUrl) || path.extname(resolved); - extension && (extension = extension.toLowerCase()); - let information: UploadInformation = { - mediaPaths: [], - fileNames: { clean: resolved }, - contentSize, - contentType, - }; - return new Promise(async (resolve, reject) => { - const resizers = [ - { resizer: sharp().rotate(), suffix: "_o" }, - ...Object.values(Sizes).map(size => ({ - resizer: sharp().resize(size.width, undefined, { withoutEnlargement: true }).rotate(), - suffix: size.suffix - })) - ]; - let nonVisual = false; - if (pngs.includes(extension)) { - resizers.forEach(element => element.resizer = element.resizer.png()); - } else if (jpgs.includes(extension)) { - resizers.forEach(element => element.resizer = element.resizer.jpeg()); - } else if (![...imageFormats, ...videoFormats].includes(extension.toLowerCase())) { - nonVisual = true; - } - if (imageFormats.includes(extension)) { - for (let resizer of resizers) { - const suffix = resizer.suffix; - let mediaPath: string; - await new Promise(resolve => { - const filename = resolved.substring(0, resolved.length - extension.length) + suffix + extension; - information.mediaPaths.push(mediaPath = uploadDirectory + filename); - information.fileNames[suffix] = filename; - stream(normalizedUrl).pipe(resizer.resizer).pipe(fs.createWriteStream(mediaPath)) - .on('close', resolve) - .on('error', reject); - }); - } - } - if (!isLocal || nonVisual) { - await new Promise(resolve => { - stream(normalizedUrl).pipe(fs.createWriteStream(uploadDirectory + resolved)).on('close', resolve); - }); - } - resolve(information); - }); - }; - - const classify = (url: string) => { - const isLocal = /Dash-Web(\\|\/)src(\\|\/)server(\\|\/)public(\\|\/)files/g.test(url); - return { - isLocal, - stream: isLocal ? fs.createReadStream : request, - normalized: isLocal ? path.normalize(url) : url - }; - }; - - export const createIfNotExists = async (path: string) => { - if (await new Promise(resolve => fs.exists(path, resolve))) { - return true; - } - return new Promise(resolve => fs.mkdir(path, error => resolve(error === null))); - }; - - export const Destroy = (mediaPath: string) => new Promise(resolve => fs.unlink(mediaPath, error => resolve(error === null))); } \ No newline at end of file diff --git a/src/server/apis/google/existing_uploads.json b/src/server/apis/google/existing_uploads.json deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/server/credentials/google_docs_token.json b/src/server/credentials/google_docs_token.json index ee44c3f30..8bd62bdfa 100644 --- a/src/server/credentials/google_docs_token.json +++ b/src/server/credentials/google_docs_token.json @@ -1 +1,7 @@ -{"access_token":"ya29.GlyKBznz91v_qb8RYt4PT40Hp106N08Yk64UjMAKllBsIqJQEzBkxLbB5q5paydywHzguQYSNup5fT7ojJTDU4CMZdPbPKGcjQz17w_CospcG-8Buz94KZptvlQ_pQ","refresh_token":"1/HTv_xFHszu2Nf3iiFrUTaeKzC_Vp2-6bpIB06xW_WHI","scope":"https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/documents.readonly https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/photoslibrary https://www.googleapis.com/auth/photoslibrary.appendonly https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/photoslibrary.sharing","token_type":"Bearer","expiry_date":1569093749804} \ No newline at end of file +{ + "access_token": "ya29.ImCOBwXgckGbyHNLMX7r-13B5VDgxfzF5mQ7lFJ0FX5GF5EuAPBBN5_ijLnNLC4yw4xtFjJOkEtKiYr-60OIm4oOnowEJpZMyRGxFMy_Q8MTnzDpeN-7Di_baUzcu7m_KWM", + "refresh_token": "1/HTv_xFHszu2Nf3iiFrUTaeKzC_Vp2-6bpIB06xW_WHI", + "scope": "https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/documents.readonly https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/photoslibrary https://www.googleapis.com/auth/photoslibrary.appendonly https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/photoslibrary.sharing", + "token_type": "Bearer", + "expiry_date": 1569366907812 +} \ No newline at end of file diff --git a/src/server/database.ts b/src/server/database.ts index a7254fb0c..ce29478ad 100644 --- a/src/server/database.ts +++ b/src/server/database.ts @@ -1,209 +1,267 @@ import * as mongodb from 'mongodb'; import { Transferable } from './Message'; +import { Opt } from '../new_fields/Doc'; +import { Utils } from '../Utils'; +import { DashUploadUtils } from './DashUploadUtils'; -export class Database { - public static DocumentsCollection = 'documents'; - public static Instance = new Database(); - private MongoClient = mongodb.MongoClient; - private url = 'mongodb://localhost:27017/Dash'; - private currentWrites: { [id: string]: Promise } = {}; - private db?: mongodb.Db; - private onConnect: (() => void)[] = []; - - constructor() { - this.MongoClient.connect(this.url, (err, client) => { - this.db = client.db(); - this.onConnect.forEach(fn => fn()); - }); - } +export namespace Database { - public update(id: string, value: any, callback: (err: mongodb.MongoError, res: mongodb.UpdateWriteOpResult) => void, upsert = true, collectionName = Database.DocumentsCollection) { - if (this.db) { - let collection = this.db.collection(collectionName); - const prom = this.currentWrites[id]; - let newProm: Promise; - const run = (): Promise => { - return new Promise(resolve => { - collection.updateOne({ _id: id }, value, { upsert } - , (err, res) => { - if (this.currentWrites[id] === newProm) { - delete this.currentWrites[id]; - } - resolve(); - callback(err, res); - }); - }); - }; - newProm = prom ? prom.then(run) : run(); - this.currentWrites[id] = newProm; - } else { - this.onConnect.push(() => this.update(id, value, callback, upsert, collectionName)); - } - } + class Database { + public static DocumentsCollection = 'documents'; + private MongoClient = mongodb.MongoClient; + private url = 'mongodb://localhost:27017/Dash'; + private currentWrites: { [id: string]: Promise } = {}; + private db?: mongodb.Db; + private onConnect: (() => void)[] = []; - public replace(id: string, value: any, callback: (err: mongodb.MongoError, res: mongodb.UpdateWriteOpResult) => void, upsert = true, collectionName = Database.DocumentsCollection) { - if (this.db) { - let collection = this.db.collection(collectionName); - const prom = this.currentWrites[id]; - let newProm: Promise; - const run = (): Promise => { - return new Promise(resolve => { - collection.replaceOne({ _id: id }, value, { upsert } - , (err, res) => { - if (this.currentWrites[id] === newProm) { - delete this.currentWrites[id]; - } - resolve(); - callback(err, res); - }); - }); - }; - newProm = prom ? prom.then(run) : run(); - this.currentWrites[id] = newProm; - } else { - this.onConnect.push(() => this.replace(id, value, callback, upsert, collectionName)); + constructor() { + this.MongoClient.connect(this.url, (err, client) => { + this.db = client.db(); + this.onConnect.forEach(fn => fn()); + }); } - } - public delete(query: any, collectionName?: string): Promise; - public delete(id: string, collectionName?: string): Promise; - public delete(id: any, collectionName = Database.DocumentsCollection) { - if (typeof id === "string") { - id = { _id: id }; - } - if (this.db) { - const db = this.db; - return new Promise(res => db.collection(collectionName).deleteMany(id, (err, result) => res(result))); - } else { - return new Promise(res => this.onConnect.push(() => res(this.delete(id, collectionName)))); + public update(id: string, value: any, callback: (err: mongodb.MongoError, res: mongodb.UpdateWriteOpResult) => void, upsert = true, collectionName = Database.DocumentsCollection) { + if (this.db) { + let collection = this.db.collection(collectionName); + const prom = this.currentWrites[id]; + let newProm: Promise; + const run = (): Promise => { + return new Promise(resolve => { + collection.updateOne({ _id: id }, value, { upsert } + , (err, res) => { + if (this.currentWrites[id] === newProm) { + delete this.currentWrites[id]; + } + resolve(); + callback(err, res); + }); + }); + }; + newProm = prom ? prom.then(run) : run(); + this.currentWrites[id] = newProm; + } else { + this.onConnect.push(() => this.update(id, value, callback, upsert, collectionName)); + } } - } - public deleteAll(collectionName = Database.DocumentsCollection): Promise { - return new Promise(res => { + public replace(id: string, value: any, callback: (err: mongodb.MongoError, res: mongodb.UpdateWriteOpResult) => void, upsert = true, collectionName = Database.DocumentsCollection) { if (this.db) { - this.db.collection(collectionName).deleteMany({}, res); + let collection = this.db.collection(collectionName); + const prom = this.currentWrites[id]; + let newProm: Promise; + const run = (): Promise => { + return new Promise(resolve => { + collection.replaceOne({ _id: id }, value, { upsert } + , (err, res) => { + if (this.currentWrites[id] === newProm) { + delete this.currentWrites[id]; + } + resolve(); + callback(err, res); + }); + }); + }; + newProm = prom ? prom.then(run) : run(); + this.currentWrites[id] = newProm; } else { - this.onConnect.push(() => this.db && this.db.collection(collectionName).deleteMany({}, res)); + this.onConnect.push(() => this.replace(id, value, callback, upsert, collectionName)); } - }); - } + } - public insert(value: any, collectionName = Database.DocumentsCollection) { - if (this.db) { - if ("id" in value) { - value._id = value.id; - delete value.id; + public delete(query: any, collectionName?: string): Promise; + public delete(id: string, collectionName?: string): Promise; + public delete(id: any, collectionName = Database.DocumentsCollection) { + if (typeof id === "string") { + id = { _id: id }; + } + if (this.db) { + const db = this.db; + return new Promise(res => db.collection(collectionName).deleteMany(id, (err, result) => res(result))); + } else { + return new Promise(res => this.onConnect.push(() => res(this.delete(id, collectionName)))); } - const id = value._id; - const collection = this.db.collection(collectionName); - const prom = this.currentWrites[id]; - let newProm: Promise; - const run = (): Promise => { - return new Promise(resolve => { - collection.insertOne(value, (err, res) => { - if (this.currentWrites[id] === newProm) { - delete this.currentWrites[id]; - } - resolve(); - }); - }); - }; - newProm = prom ? prom.then(run) : run(); - this.currentWrites[id] = newProm; - } else { - this.onConnect.push(() => this.insert(value, collectionName)); } - } - public getDocument(id: string, fn: (result?: Transferable) => void, collectionName = Database.DocumentsCollection) { - if (this.db) { - this.db.collection(collectionName).findOne({ _id: id }, (err, result) => { - if (result) { - result.id = result._id; - delete result._id; - fn(result); + public async deleteAll(collectionName = Database.DocumentsCollection, persist = true): Promise { + return new Promise(resolve => { + const executor = async (database: mongodb.Db) => { + if (persist) { + await database.collection(collectionName).deleteMany({}); + } else { + await database.dropCollection(collectionName); + } + resolve(); + }; + if (this.db) { + executor(this.db); } else { - fn(undefined); + this.onConnect.push(() => this.db && executor(this.db)); } }); - } else { - this.onConnect.push(() => this.getDocument(id, fn, collectionName)); } - } - public getDocuments(ids: string[], fn: (result: Transferable[]) => void, collectionName = Database.DocumentsCollection) { - if (this.db) { - this.db.collection(collectionName).find({ _id: { "$in": ids } }).toArray((err, docs) => { - if (err) { - console.log(err.message); - console.log(err.errmsg); + public async insert(value: any, collectionName = Database.DocumentsCollection) { + if (this.db) { + if ("id" in value) { + value._id = value.id; + delete value.id; } - fn(docs.map(doc => { - doc.id = doc._id; - delete doc._id; - return doc; - })); - }); - } else { - this.onConnect.push(() => this.getDocuments(ids, fn, collectionName)); + const id = value._id; + const collection = this.db.collection(collectionName); + const prom = this.currentWrites[id]; + let newProm: Promise; + const run = (): Promise => { + return new Promise(resolve => { + collection.insertOne(value, (err, res) => { + if (this.currentWrites[id] === newProm) { + delete this.currentWrites[id]; + } + resolve(); + }); + }); + }; + newProm = prom ? prom.then(run) : run(); + this.currentWrites[id] = newProm; + return newProm; + } else { + this.onConnect.push(() => this.insert(value, collectionName)); + } } - } - public async visit(ids: string[], fn: (result: any) => string[], collectionName = "newDocuments"): Promise { - if (this.db) { - const visited = new Set(); - while (ids.length) { - const count = Math.min(ids.length, 1000); - const index = ids.length - count; - const fetchIds = ids.splice(index, count).filter(id => !visited.has(id)); - if (!fetchIds.length) { - continue; - } - const docs = await new Promise<{ [key: string]: any }[]>(res => Database.Instance.getDocuments(fetchIds, res, "newDocuments")); - for (const doc of docs) { - const id = doc.id; - visited.add(id); - ids.push(...fn(doc)); + public getDocument(id: string, fn: (result?: Transferable) => void, collectionName = Database.DocumentsCollection) { + if (this.db) { + this.db.collection(collectionName).findOne({ _id: id }, (err, result) => { + if (result) { + result.id = result._id; + delete result._id; + fn(result); + } else { + fn(undefined); + } + }); + } else { + this.onConnect.push(() => this.getDocument(id, fn, collectionName)); + } + } + + public getDocuments(ids: string[], fn: (result: Transferable[]) => void, collectionName = Database.DocumentsCollection) { + if (this.db) { + this.db.collection(collectionName).find({ _id: { "$in": ids } }).toArray((err, docs) => { + if (err) { + console.log(err.message); + console.log(err.errmsg); + } + fn(docs.map(doc => { + doc.id = doc._id; + delete doc._id; + return doc; + })); + }); + } else { + this.onConnect.push(() => this.getDocuments(ids, fn, collectionName)); + } + } + + public async visit(ids: string[], fn: (result: any) => string[], collectionName = "newDocuments"): Promise { + if (this.db) { + const visited = new Set(); + while (ids.length) { + const count = Math.min(ids.length, 1000); + const index = ids.length - count; + const fetchIds = ids.splice(index, count).filter(id => !visited.has(id)); + if (!fetchIds.length) { + continue; + } + const docs = await new Promise<{ [key: string]: any }[]>(res => Instance.getDocuments(fetchIds, res, "newDocuments")); + for (const doc of docs) { + const id = doc.id; + visited.add(id); + ids.push(...fn(doc)); + } } + + } else { + return new Promise(res => { + this.onConnect.push(() => { + this.visit(ids, fn, collectionName); + res(); + }); + }); } + } - } else { - return new Promise(res => { - this.onConnect.push(() => { - this.visit(ids, fn, collectionName); - res(); + public query(query: { [key: string]: any }, projection?: { [key: string]: 0 | 1 }, collectionName = "newDocuments"): Promise { + if (this.db) { + let cursor = this.db.collection(collectionName).find(query); + if (projection) { + cursor = cursor.project(projection); + } + return Promise.resolve(cursor); + } else { + return new Promise(res => { + this.onConnect.push(() => res(this.query(query, projection, collectionName))); }); - }); + } } - } - public query(query: { [key: string]: any }, projection?: { [key: string]: 0 | 1 }, collectionName = "newDocuments"): Promise { - if (this.db) { - let cursor = this.db.collection(collectionName).find(query); - if (projection) { - cursor = cursor.project(projection); + public updateMany(query: any, update: any, collectionName = "newDocuments") { + if (this.db) { + const db = this.db; + return new Promise(res => db.collection(collectionName).update(query, update, (_, result) => res(result))); + } else { + return new Promise(res => { + this.onConnect.push(() => this.updateMany(query, update, collectionName).then(res)); + }); } - return Promise.resolve(cursor); - } else { - return new Promise(res => { - this.onConnect.push(() => res(this.query(query, projection, collectionName))); - }); } - } - public updateMany(query: any, update: any, collectionName = "newDocuments") { - if (this.db) { - const db = this.db; - return new Promise(res => db.collection(collectionName).update(query, update, (_, result) => res(result))); - } else { - return new Promise(res => { - this.onConnect.push(() => this.updateMany(query, update, collectionName).then(res)); - }); + public print() { + console.log("db says hi!"); } } - public print() { - console.log("db says hi!"); + export const Instance = new Database(); + + export namespace Auxiliary { + + export enum AuxiliaryCollections { + GooglePhotosUploadHistory = "UploadedFromGooglePhotos" + } + + const GoogleAuthentication = "GoogleAuthentication"; + + const SanitizedSingletonQuery = async (query: { [key: string]: any }, collection: string) => { + const cursor = await Instance.query(query, undefined, collection); + const existing = (await cursor.toArray())[0]; + if (existing) { + delete existing._id; + } + return existing; + }; + + export const QueryUploadHistory = async (contentSize: number): Promise> => { + return SanitizedSingletonQuery({ contentSize }, AuxiliaryCollections.GooglePhotosUploadHistory); + }; + + export const LogUpload = async (information: DashUploadUtils.UploadInformation) => { + const bundle = { + _id: Utils.GenerateDeterministicGuid(String(information.contentSize!)), + ...information + }; + return Instance.insert(bundle, AuxiliaryCollections.GooglePhotosUploadHistory); + }; + + export const DeleteAll = async (persist = false) => { + const collectionNames = Object.values(AuxiliaryCollections); + const pendingDeletions = collectionNames.map(name => Instance.deleteAll(name, persist)); + return Promise.all(pendingDeletions); + }; + + export const FetchGoogleAuthenticationToken = async (userId: string) => { + return SanitizedSingletonQuery({ userId }, GoogleAuthentication); + }; + } -} + +} \ No newline at end of file diff --git a/src/server/index.ts b/src/server/index.ts index 4c4cb84d6..386ecce4d 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -41,13 +41,14 @@ var AdmZip = require('adm-zip'); import * as YoutubeApi from "./apis/youtube/youtubeApiSample"; import { Response } from 'express-serve-static-core'; import { GoogleApiServerUtils } from "./apis/google/GoogleApiServerUtils"; -import { GooglePhotosUploadUtils, DownloadUtils as UploadUtils } from './apis/google/GooglePhotosUploadUtils'; +import { GooglePhotosUploadUtils } from './apis/google/GooglePhotosUploadUtils'; const MongoStore = require('connect-mongo')(session); const mongoose = require('mongoose'); const probe = require("probe-image-size"); import * as qs from 'query-string'; import { Opt } from '../new_fields/Doc'; import BatchedArray, { TimeUnit } from "array-batcher"; +import { DashUploadUtils } from './DashUploadUtils'; const download = (url: string, dest: fs.PathLike) => request.get(url).pipe(fs.createWriteStream(dest)); let youtubeApiKey: string; @@ -581,8 +582,8 @@ app.post( for (const key in files) { const { type, path: location, name } = files[key]; const filename = path.basename(location); - const metadata = await UploadUtils.InspectImage(uploadDirectory + filename); - await UploadUtils.UploadImage(metadata, filename).catch(() => console.log(`Unable to process ${filename}`)); + const metadata = await DashUploadUtils.InspectImage(uploadDirectory + filename); + await DashUploadUtils.UploadImage(metadata, filename).catch(() => console.log(`Unable to process ${filename}`)); results.push({ name, type, path: `/files/${filename}` }); } _success(res, results); @@ -809,7 +810,7 @@ const EndpointHandlerMap = new Map { let sector: GoogleApiServerUtils.Service = req.params.sector as GoogleApiServerUtils.Service; let action: GoogleApiServerUtils.Action = req.params.action as GoogleApiServerUtils.Action; - GoogleApiServerUtils.GetEndpoint(GoogleApiServerUtils.Service[sector], { credentialsPath, tokenPath }).then(endpoint => { + GoogleApiServerUtils.GetEndpoint(GoogleApiServerUtils.Service[sector], { credentialsPath, userId: req.body.userId }).then(endpoint => { let handler = EndpointHandlerMap.get(action); if (endpoint && handler) { let execute = handler(endpoint, req.body).then( @@ -823,7 +824,7 @@ app.post(RouteStore.googleDocs + "/:sector/:action", (req, res) => { }); }); -app.get(RouteStore.googlePhotosAccessToken, (req, res) => GoogleApiServerUtils.RetrieveAccessToken({ credentialsPath, tokenPath }).then(token => res.send(token))); +app.get(RouteStore.googlePhotosAccessToken, (req, res) => GoogleApiServerUtils.RetrieveAccessToken({ credentialsPath, userId: req.body.userId }).then(token => res.send(token))); const tokenError = "Unable to successfully upload bytes for all images!"; const mediaError = "Unable to convert all uploaded bytes to media items!"; @@ -836,12 +837,13 @@ export interface NewMediaItem { } app.post(RouteStore.googlePhotosMediaUpload, async (req, res) => { - const mediaInput: GooglePhotosUploadUtils.MediaInput[] = req.body.media; - await GooglePhotosUploadUtils.initialize({ uploadDirectory, credentialsPath, tokenPath }); + const { userId, media } = req.body; + + await GooglePhotosUploadUtils.initialize({ credentialsPath, userId }); let failed = 0; - const newMediaItems = await BatchedArray.from(mediaInput, { batchSize: 25 }).batchedMapInterval( + const newMediaItems = await BatchedArray.from(media, { batchSize: 25 }).batchedMapInterval( async (batch: GooglePhotosUploadUtils.MediaInput[]) => { const newMediaItems: NewMediaItem[] = []; for (let element of batch) { @@ -879,31 +881,36 @@ const prefix = "google_photos_"; const downloadError = "Encountered an error while executing downloads."; const requestError = "Unable to execute download: the body's media items were malformed."; -app.get("/gapiCleanup", (req, res) => { - write_text_file(file, ""); +app.get("/deleteWithAux", async (req, res) => { + await Database.Auxiliary.DeleteAll(); res.redirect(RouteStore.delete); }); -const file = "./apis/google/existing_uploads.json"; +const UploadError = (count: number) => `Unable to upload ${count} images to Dash's server`; app.post(RouteStore.googlePhotosMediaDownload, async (req, res) => { const contents: { mediaItems: MediaItem[] } = req.body; + let failed = 0; if (contents) { - const completed: Opt[] = []; - const content = await read_text_file(file); - let existing = content.length ? JSON.parse(content) : {}; + const completed: Opt[] = []; for (let item of contents.mediaItems) { - const { contentSize, ...attributes } = await UploadUtils.InspectImage(item.baseUrl); - const found: UploadUtils.UploadInformation = existing[contentSize!]; + const { contentSize, ...attributes } = await DashUploadUtils.InspectImage(item.baseUrl); + const found: Opt = await Database.Auxiliary.QueryUploadHistory(contentSize!); if (!found) { - const upload = await UploadUtils.UploadImage({ contentSize, ...attributes }, item.filename, prefix).catch(error => _error(res, downloadError, error)); - upload && completed.push(existing[contentSize!] = upload); + const upload = await DashUploadUtils.UploadImage({ contentSize, ...attributes }, item.filename, prefix).catch(error => _error(res, downloadError, error)); + if (upload) { + completed.push(upload); + await Database.Auxiliary.LogUpload(upload); + } else { + failed++; + } } else { completed.push(found); } } - await write_text_file(file, JSON.stringify(existing)); - _success(res, completed); - return; + if (failed) { + return _error(res, UploadError(failed)); + } + return _success(res, completed); } _invalid(res, requestError); }); -- cgit v1.2.3-70-g09d2 From e95387732e1fbff49ec035c3bec4b03324d814c8 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Thu, 26 Sep 2019 18:11:10 -0400 Subject: beginning to read from and write to database --- src/Utils.ts | 92 ++++++++++------------ src/client/Network.ts | 25 ++++++ .../apis/google_docs/GoogleApiClientUtils.ts | 2 +- .../apis/google_docs/GooglePhotosClientUtils.ts | 3 +- .../util/Import & Export/DirectoryImportBox.tsx | 4 +- src/client/views/MainView.tsx | 2 +- src/new_fields/RichTextUtils.ts | 4 +- src/server/apis/google/GoogleApiServerUtils.ts | 51 +++++------- src/server/database.ts | 34 +++++--- src/server/index.ts | 12 ++- 10 files changed, 126 insertions(+), 103 deletions(-) create mode 100644 src/client/Network.ts (limited to 'src/server/apis/google/GoogleApiServerUtils.ts') diff --git a/src/Utils.ts b/src/Utils.ts index 5f06b5cec..ae8371f15 100644 --- a/src/Utils.ts +++ b/src/Utils.ts @@ -3,22 +3,20 @@ import v5 = require("uuid/v5"); import { Socket } from 'socket.io'; import { Message } from './server/Message'; import { RouteStore } from './server/RouteStore'; -import requestPromise = require('request-promise'); -import { CurrentUserUtils } from './server/authentication/models/current_user_utils'; -export class Utils { +export namespace Utils { - public static DRAG_THRESHOLD = 4; + export const DRAG_THRESHOLD = 4; - public static GenerateGuid(): string { + export function GenerateGuid(): string { return v4(); } - public static GenerateDeterministicGuid(seed: string): string { + export function GenerateDeterministicGuid(seed: string): string { return v5(seed, v5.URL); } - public static GetScreenTransform(ele?: HTMLElement): { scale: number, translateX: number, translateY: number } { + export function GetScreenTransform(ele?: HTMLElement): { scale: number, translateX: number, translateY: number } { if (!ele) { return { scale: 1, translateX: 1, translateY: 1 }; } @@ -35,23 +33,23 @@ export class Utils { * requested extension * @param extension the specified sub-path to append to the window origin */ - public static prepend(extension: string): string { + export function prepend(extension: string): string { return window.location.origin + extension; } - public static fileUrl(filename: string): string { - return this.prepend(`/files/${filename}`); + export function fileUrl(filename: string): string { + return prepend(`/files/${filename}`); } - public static shareUrl(documentId: string): string { - return this.prepend(`/doc/${documentId}?sharing=true`); + export function shareUrl(documentId: string): string { + return prepend(`/doc/${documentId}?sharing=true`); } - public static CorsProxy(url: string): string { - return this.prepend(RouteStore.corsProxy + "/") + encodeURIComponent(url); + export function CorsProxy(url: string): string { + return prepend(RouteStore.corsProxy + "/") + encodeURIComponent(url); } - public static CopyText(text: string) { + export function CopyText(text: string) { var textArea = document.createElement("textarea"); textArea.value = text; document.body.appendChild(textArea); @@ -63,7 +61,7 @@ export class Utils { document.body.removeChild(textArea); } - public static fromRGBAstr(rgba: string) { + export function fromRGBAstr(rgba: string) { let rm = rgba.match(/rgb[a]?\(([0-9]+)/); let r = rm ? Number(rm[1]) : 0; let gm = rgba.match(/rgb[a]?\([0-9]+,([0-9]+)/); @@ -74,11 +72,12 @@ export class Utils { let a = am ? Number(am[1]) : 0; return { r: r, g: g, b: b, a: a }; } - public static toRGBAstr(col: { r: number, g: number, b: number, a?: number }) { + + export function toRGBAstr(col: { r: number, g: number, b: number, a?: number }) { return "rgba(" + col.r + "," + col.g + "," + col.b + (col.a !== undefined ? "," + col.a : "") + ")"; } - public static HSLtoRGB(h: number, s: number, l: number) { + export function HSLtoRGB(h: number, s: number, l: number) { // Must be fractions of 1 // s /= 100; // l /= 100; @@ -108,7 +107,7 @@ export class Utils { return { r: r, g: g, b: b }; } - public static RGBToHSL(r: number, g: number, b: number) { + export function RGBToHSL(r: number, g: number, b: number) { // Make r, g, and b fractions of 1 r /= 255; g /= 255; @@ -150,7 +149,7 @@ export class Utils { } - public static GetClipboardText(): string { + export function GetClipboardText(): string { var textArea = document.createElement("textarea"); document.body.appendChild(textArea); textArea.focus(); @@ -163,51 +162,53 @@ export class Utils { return val; } - public static loggingEnabled: Boolean = false; - public static logFilter: number | undefined = undefined; - private static log(prefix: string, messageName: string, message: any, receiving: boolean) { - if (!this.loggingEnabled) { + export const loggingEnabled: Boolean = false; + export const logFilter: number | undefined = undefined; + + function log(prefix: string, messageName: string, message: any, receiving: boolean) { + if (!loggingEnabled) { return; } message = message || {}; - if (this.logFilter !== undefined && this.logFilter !== message.type) { + if (logFilter !== undefined && logFilter !== message.type) { return; } let idString = (message.id || "").padStart(36, ' '); prefix = prefix.padEnd(16, ' '); console.log(`${prefix}: ${idString}, ${receiving ? 'receiving' : 'sending'} ${messageName} with data ${JSON.stringify(message)}`); } - private static loggingCallback(prefix: string, func: (args: any) => any, messageName: string) { + + function loggingCallback(prefix: string, func: (args: any) => any, messageName: string) { return (args: any) => { - this.log(prefix, messageName, args, true); + log(prefix, messageName, args, true); func(args); }; } - public static Emit(socket: Socket | SocketIOClient.Socket, message: Message, args: T) { - this.log("Emit", message.Name, args, false); + export function Emit(socket: Socket | SocketIOClient.Socket, message: Message, args: T) { + log("Emit", message.Name, args, false); socket.emit(message.Message, args); } - public static EmitCallback(socket: Socket | SocketIOClient.Socket, message: Message, args: T): Promise; - public static EmitCallback(socket: Socket | SocketIOClient.Socket, message: Message, args: T, fn: (args: any) => any): void; - public static EmitCallback(socket: Socket | SocketIOClient.Socket, message: Message, args: T, fn?: (args: any) => any): void | Promise { - this.log("Emit", message.Name, args, false); + export function EmitCallback(socket: Socket | SocketIOClient.Socket, message: Message, args: T): Promise; + export function EmitCallback(socket: Socket | SocketIOClient.Socket, message: Message, args: T, fn: (args: any) => any): void; + export function EmitCallback(socket: Socket | SocketIOClient.Socket, message: Message, args: T, fn?: (args: any) => any): void | Promise { + log("Emit", message.Name, args, false); if (fn) { - socket.emit(message.Message, args, this.loggingCallback('Receiving', fn, message.Name)); + socket.emit(message.Message, args, loggingCallback('Receiving', fn, message.Name)); } else { - return new Promise(res => socket.emit(message.Message, args, this.loggingCallback('Receiving', res, message.Name))); + return new Promise(res => socket.emit(message.Message, args, loggingCallback('Receiving', res, message.Name))); } } - public static AddServerHandler(socket: Socket | SocketIOClient.Socket, message: Message, handler: (args: T) => any) { - socket.on(message.Message, this.loggingCallback('Incoming', handler, message.Name)); + export function AddServerHandler(socket: Socket | SocketIOClient.Socket, message: Message, handler: (args: T) => any) { + socket.on(message.Message, loggingCallback('Incoming', handler, message.Name)); } - public static AddServerHandlerCallback(socket: Socket, message: Message, handler: (args: [T, (res: any) => any]) => any) { + export function AddServerHandlerCallback(socket: Socket, message: Message, handler: (args: [T, (res: any) => any]) => any) { socket.on(message.Message, (arg: T, fn: (res: any) => any) => { - this.log('S receiving', message.Name, arg, true); - handler([arg, this.loggingCallback('S sending', fn, message.Name)]); + log('S receiving', message.Name, arg, true); + handler([arg, loggingCallback('S sending', fn, message.Name)]); }); } } @@ -291,15 +292,4 @@ export namespace JSONUtils { return results; } -} - -export function PostToServer(relativeRoute: string, body?: any) { - body = { userId: CurrentUserUtils.id, ...body }; - let options = { - method: "POST", - uri: Utils.prepend(relativeRoute), - json: true, - body - }; - return requestPromise.post(options); } \ No newline at end of file diff --git a/src/client/Network.ts b/src/client/Network.ts new file mode 100644 index 000000000..cb46105f8 --- /dev/null +++ b/src/client/Network.ts @@ -0,0 +1,25 @@ +import { Utils } from "../Utils"; +import { CurrentUserUtils } from "../server/authentication/models/current_user_utils"; +import requestPromise = require('request-promise'); + +export async function PostToServer(relativeRoute: string, body?: any) { + let options = { + uri: Utils.prepend(relativeRoute), + method: "POST", + headers: { userId: CurrentUserUtils.id }, + body, + json: true + }; + return requestPromise.post(options); +} + +export async function PostFormDataToServer(relativeRoute: string, formData: FormData) { + const parameters = { + method: 'POST', + headers: { userId: CurrentUserUtils.id }, + body: formData, + }; + const response = await fetch(relativeRoute, parameters); + const text = await response.json(); + return text; +} \ No newline at end of file diff --git a/src/client/apis/google_docs/GoogleApiClientUtils.ts b/src/client/apis/google_docs/GoogleApiClientUtils.ts index 2c84741db..0f0f81891 100644 --- a/src/client/apis/google_docs/GoogleApiClientUtils.ts +++ b/src/client/apis/google_docs/GoogleApiClientUtils.ts @@ -1,9 +1,9 @@ import { docs_v1, slides_v1 } from "googleapis"; -import { PostToServer } from "../../../Utils"; import { RouteStore } from "../../../server/RouteStore"; import { Opt } from "../../../new_fields/Doc"; import { isArray } from "util"; import { EditorState } from "prosemirror-state"; +import { PostToServer } from "../../Network"; export const Pulls = "googleDocsPullCount"; export const Pushes = "googleDocsPushCount"; diff --git a/src/client/apis/google_docs/GooglePhotosClientUtils.ts b/src/client/apis/google_docs/GooglePhotosClientUtils.ts index c45a49f1a..b1b29210a 100644 --- a/src/client/apis/google_docs/GooglePhotosClientUtils.ts +++ b/src/client/apis/google_docs/GooglePhotosClientUtils.ts @@ -1,4 +1,4 @@ -import { PostToServer, Utils } from "../../../Utils"; +import { Utils } from "../../../Utils"; import { RouteStore } from "../../../server/RouteStore"; import { ImageField } from "../../../new_fields/URLField"; import { Cast, StrCast } from "../../../new_fields/Types"; @@ -14,6 +14,7 @@ import { NewMediaItemResult, MediaItem } from "../../../server/apis/google/Share import { AssertionError } from "assert"; import { DocumentView } from "../../views/nodes/DocumentView"; import { DocumentManager } from "../../util/DocumentManager"; +import { PostToServer } from "../../Network"; export namespace GooglePhotos { diff --git a/src/client/util/Import & Export/DirectoryImportBox.tsx b/src/client/util/Import & Export/DirectoryImportBox.tsx index 6670f685e..d0291aec4 100644 --- a/src/client/util/Import & Export/DirectoryImportBox.tsx +++ b/src/client/util/Import & Export/DirectoryImportBox.tsx @@ -21,6 +21,7 @@ import { GooglePhotos } from "../../apis/google_docs/GooglePhotosClientUtils"; import { SchemaHeaderField } from "../../../new_fields/SchemaHeaderField"; import "./DirectoryImportBox.scss"; import BatchedArray from "array-batcher"; +import { PostFormDataToServer } from "../../Network"; const unsupported = ["text/html", "text/plain"]; interface FileResponse { @@ -106,7 +107,6 @@ export default class DirectoryImportBox extends React.Component const uploads = await BatchedArray.from(validated, { batchSize: 15 }).batchedMapAsync(async batch => { const formData = new FormData(); - const parameters = { method: 'POST', body: formData }; batch.forEach(file => { sizes.push(file.size); @@ -114,7 +114,7 @@ export default class DirectoryImportBox extends React.Component formData.append(Utils.GenerateGuid(), file); }); - const responses = await (await fetch(RouteStore.upload, parameters)).json(); + const responses = await PostFormDataToServer(RouteStore.upload, formData); runInAction(() => this.completed += batch.length); return responses as FileResponse[]; }); diff --git a/src/client/views/MainView.tsx b/src/client/views/MainView.tsx index b1f753635..296574a04 100644 --- a/src/client/views/MainView.tsx +++ b/src/client/views/MainView.tsx @@ -15,7 +15,7 @@ import { listSpec } from '../../new_fields/Schema'; import { BoolCast, Cast, FieldValue, StrCast, NumCast } from '../../new_fields/Types'; import { CurrentUserUtils } from '../../server/authentication/models/current_user_utils'; import { RouteStore } from '../../server/RouteStore'; -import { emptyFunction, returnOne, returnTrue, Utils, returnEmptyString, PostToServer } from '../../Utils'; +import { emptyFunction, returnOne, returnTrue, Utils, returnEmptyString } from '../../Utils'; import { DocServer } from '../DocServer'; import { ClientUtils } from '../util/ClientUtils'; import { DictationManager } from '../util/DictationManager'; diff --git a/src/new_fields/RichTextUtils.ts b/src/new_fields/RichTextUtils.ts index 02079e92c..f3208ce41 100644 --- a/src/new_fields/RichTextUtils.ts +++ b/src/new_fields/RichTextUtils.ts @@ -7,17 +7,17 @@ import { FormattedTextBox } from "../client/views/nodes/FormattedTextBox"; import { Opt, Doc } from "./Doc"; import Color = require('color'); import { sinkListItem } from "prosemirror-schema-list"; -import { Utils, PostToServer } from "../Utils"; +import { Utils } from "../Utils"; import { RouteStore } from "../server/RouteStore"; import { Docs } from "../client/documents/Documents"; import { schema } from "../client/util/RichTextSchema"; import { GooglePhotos } from "../client/apis/google_docs/GooglePhotosClientUtils"; -import { SchemaHeaderField } from "./SchemaHeaderField"; import { DocServer } from "../client/DocServer"; import { Cast, StrCast } from "./Types"; import { Id } from "./FieldSymbols"; import { DocumentView } from "../client/views/nodes/DocumentView"; import { AssertionError } from "assert"; +import { PostToServer } from "../client/Network"; export namespace RichTextUtils { diff --git a/src/server/apis/google/GoogleApiServerUtils.ts b/src/server/apis/google/GoogleApiServerUtils.ts index 684a8081b..665dcf862 100644 --- a/src/server/apis/google/GoogleApiServerUtils.ts +++ b/src/server/apis/google/GoogleApiServerUtils.ts @@ -66,6 +66,15 @@ export namespace GoogleApiServerUtils { }); }; + export const RetrieveAccessToken = (information: CredentialInformation) => { + return new Promise((resolve, reject) => { + RetrieveCredentials(information).then( + credentials => resolve(credentials.token.access_token!), + error => reject(`Error: unable to authenticate Google Photos API request.\n${error}`) + ); + }); + }; + export const RetrieveCredentials = (information: CredentialInformation) => { return new Promise((resolve, reject) => { readFile(information.credentialsPath, async (err, credentials) => { @@ -78,15 +87,6 @@ export namespace GoogleApiServerUtils { }); }; - export const RetrieveAccessToken = (information: CredentialInformation) => { - return new Promise((resolve, reject) => { - RetrieveCredentials(information).then( - credentials => resolve(credentials.token.access_token!), - error => reject(`Error: unable to authenticate Google Photos API request.\n${error}`) - ); - }); - }; - export const RetrievePhotosEndpoint = (paths: CredentialInformation) => { return new Promise((resolve, reject) => { RetrieveAccessToken(paths).then( @@ -107,7 +107,7 @@ export namespace GoogleApiServerUtils { client_id, client_secret, redirect_uris[0]); return new Promise((resolve, reject) => { - Database.Auxiliary.FetchGoogleAuthenticationToken(userId).then(token => { + Database.Auxiliary.GoogleAuthenticationToken.Fetch(userId).then(token => { // Check if we have previously stored a token for this userId. if (!token) { return getNewToken(oAuth2Client, userId).then(resolve, reject); @@ -123,8 +123,8 @@ export namespace GoogleApiServerUtils { } const refreshEndpoint = "https://oauth2.googleapis.com/token"; - const refreshToken = (credentials: Credentials, client_id: string, client_secret: string, oAuth2Client: OAuth2Client, token_path: string) => { - return new Promise((resolve, reject) => { + const refreshToken = (credentials: Credentials, client_id: string, client_secret: string, oAuth2Client: OAuth2Client, userId: string) => { + return new Promise(resolve => { let headerParameters = { headers: { 'Content-Type': 'application/x-www-form-urlencoded' } }; let queryParameters = { refreshToken: credentials.refresh_token, @@ -133,19 +133,13 @@ export namespace GoogleApiServerUtils { grant_type: "refresh_token" }; let url = `${refreshEndpoint}?${qs.stringify(queryParameters)}`; - request.post(url, headerParameters).then(response => { + request.post(url, headerParameters).then(async response => { let parsed = JSON.parse(response); credentials.access_token = parsed.access_token; credentials.expiry_date = new Date().getTime() + (parsed.expires_in * 1000); - writeFile(token_path, JSON.stringify(credentials), (err) => { - if (err) { - console.error(err); - reject(err); - } - console.log('Refreshed token stored to', token_path); - oAuth2Client.setCredentials(credentials); - resolve({ token: credentials, client: oAuth2Client }); - }); + oAuth2Client.setCredentials(credentials); + await Database.Auxiliary.GoogleAuthenticationToken.Write(userId, credentials); + resolve({ token: credentials, client: oAuth2Client }); }); }); }; @@ -156,7 +150,7 @@ export namespace GoogleApiServerUtils { * @param {google.auth.OAuth2} oAuth2Client The OAuth2 client to get token for. * @param {getEventsCallback} callback The callback for the authorized client. */ - function getNewToken(oAuth2Client: OAuth2Client, token_path: string) { + function getNewToken(oAuth2Client: OAuth2Client, userId: string) { return new Promise((resolve, reject) => { const authUrl = oAuth2Client.generateAuthUrl({ access_type: 'offline', @@ -169,20 +163,13 @@ export namespace GoogleApiServerUtils { }); rl.question('Enter the code from that page here: ', (code) => { rl.close(); - oAuth2Client.getToken(code, (err, token) => { + oAuth2Client.getToken(code, async (err, token) => { if (err || !token) { reject(err); return console.error('Error retrieving access token', err); } oAuth2Client.setCredentials(token); - // Store the token to disk for later program executions - writeFile(token_path, JSON.stringify(token), (err) => { - if (err) { - console.error(err); - reject(err); - } - console.log('Token stored to', token_path); - }); + await Database.Auxiliary.GoogleAuthenticationToken.Write(userId, token); resolve({ token, client: oAuth2Client }); }); }); diff --git a/src/server/database.ts b/src/server/database.ts index ce29478ad..890ac6b32 100644 --- a/src/server/database.ts +++ b/src/server/database.ts @@ -231,19 +231,37 @@ export namespace Database { const GoogleAuthentication = "GoogleAuthentication"; - const SanitizedSingletonQuery = async (query: { [key: string]: any }, collection: string) => { + const SanitizedCappedQuery = async (query: { [key: string]: any }, collection: string, cap: number) => { const cursor = await Instance.query(query, undefined, collection); - const existing = (await cursor.toArray())[0]; - if (existing) { - delete existing._id; - } - return existing; + const results = await cursor.toArray(); + const slice = results.slice(0, Math.min(cap, results.length)); + return slice.map(result => { + delete result._id; + return result; + }); + }; + + const SanitizedSingletonQuery = async (query: { [key: string]: any }, collection: string) => { + const results = await SanitizedCappedQuery(query, collection, 1); + return results.length ? results[0] : undefined; }; export const QueryUploadHistory = async (contentSize: number): Promise> => { return SanitizedSingletonQuery({ contentSize }, AuxiliaryCollections.GooglePhotosUploadHistory); }; + export namespace GoogleAuthenticationToken { + + export const Fetch = async (userId: string) => { + return SanitizedSingletonQuery({ userId }, GoogleAuthentication); + }; + + export const Write = async (userId: string, token: any) => { + return Instance.insert({ userId, ...token }, GoogleAuthentication); + }; + + } + export const LogUpload = async (information: DashUploadUtils.UploadInformation) => { const bundle = { _id: Utils.GenerateDeterministicGuid(String(information.contentSize!)), @@ -258,10 +276,6 @@ export namespace Database { return Promise.all(pendingDeletions); }; - export const FetchGoogleAuthenticationToken = async (userId: string) => { - return SanitizedSingletonQuery({ userId }, GoogleAuthentication); - }; - } } \ No newline at end of file diff --git a/src/server/index.ts b/src/server/index.ts index 386ecce4d..2ff63ab78 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -810,7 +810,7 @@ const EndpointHandlerMap = new Map { let sector: GoogleApiServerUtils.Service = req.params.sector as GoogleApiServerUtils.Service; let action: GoogleApiServerUtils.Action = req.params.action as GoogleApiServerUtils.Action; - GoogleApiServerUtils.GetEndpoint(GoogleApiServerUtils.Service[sector], { credentialsPath, userId: req.body.userId }).then(endpoint => { + GoogleApiServerUtils.GetEndpoint(GoogleApiServerUtils.Service[sector], { credentialsPath, userId: req.headers.userId as string }).then(endpoint => { let handler = EndpointHandlerMap.get(action); if (endpoint && handler) { let execute = handler(endpoint, req.body).then( @@ -824,10 +824,11 @@ app.post(RouteStore.googleDocs + "/:sector/:action", (req, res) => { }); }); -app.get(RouteStore.googlePhotosAccessToken, (req, res) => GoogleApiServerUtils.RetrieveAccessToken({ credentialsPath, userId: req.body.userId }).then(token => res.send(token))); +app.get(RouteStore.googlePhotosAccessToken, (req, res) => GoogleApiServerUtils.RetrieveAccessToken({ credentialsPath, userId: req.headers.userId as string }).then(token => res.send(token))); const tokenError = "Unable to successfully upload bytes for all images!"; const mediaError = "Unable to convert all uploaded bytes to media items!"; +const userIdError = "Unable to parse the identification of the user!"; export interface NewMediaItem { description: string; @@ -837,7 +838,12 @@ export interface NewMediaItem { } app.post(RouteStore.googlePhotosMediaUpload, async (req, res) => { - const { userId, media } = req.body; + const { media } = req.body; + const { userId } = req.headers; + + if (!userId || Array.isArray(userId)) { + return _error(res, userIdError); + } await GooglePhotosUploadUtils.initialize({ credentialsPath, userId }); -- cgit v1.2.3-70-g09d2 From 20540a35be82c34cc3962de4f957d1aa43f8a2b0 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Thu, 26 Sep 2019 22:17:15 -0400 Subject: finally transferred google credential management to database --- src/client/Network.ts | 46 +++++++++++++--------- .../apis/google_docs/GooglePhotosClientUtils.ts | 17 ++++---- src/client/documents/Documents.ts | 22 ++++++----- .../util/Import & Export/DirectoryImportBox.tsx | 5 +-- src/client/views/nodes/DocumentView.tsx | 7 ++-- src/server/apis/google/GoogleApiServerUtils.ts | 27 +++++++------ src/server/database.ts | 32 +++++++++------ src/server/index.ts | 6 +-- 8 files changed, 93 insertions(+), 69 deletions(-) (limited to 'src/server/apis/google/GoogleApiServerUtils.ts') diff --git a/src/client/Network.ts b/src/client/Network.ts index cb46105f8..75ccb5e99 100644 --- a/src/client/Network.ts +++ b/src/client/Network.ts @@ -2,24 +2,32 @@ import { Utils } from "../Utils"; import { CurrentUserUtils } from "../server/authentication/models/current_user_utils"; import requestPromise = require('request-promise'); -export async function PostToServer(relativeRoute: string, body?: any) { - let options = { - uri: Utils.prepend(relativeRoute), - method: "POST", - headers: { userId: CurrentUserUtils.id }, - body, - json: true - }; - return requestPromise.post(options); -} +export namespace Identified { + + export async function FetchFromServer(relativeRoute: string) { + return (await fetch(relativeRoute, { headers: { userId: CurrentUserUtils.id } })).text(); + } + + export async function PostToServer(relativeRoute: string, body?: any) { + let options = { + uri: Utils.prepend(relativeRoute), + method: "POST", + headers: { userId: CurrentUserUtils.id }, + body, + json: true + }; + return requestPromise.post(options); + } + + export async function PostFormDataToServer(relativeRoute: string, formData: FormData) { + const parameters = { + method: 'POST', + headers: { userId: CurrentUserUtils.id }, + body: formData, + }; + const response = await fetch(relativeRoute, parameters); + const text = await response.json(); + return text; + } -export async function PostFormDataToServer(relativeRoute: string, formData: FormData) { - const parameters = { - method: 'POST', - headers: { userId: CurrentUserUtils.id }, - body: formData, - }; - const response = await fetch(relativeRoute, parameters); - const text = await response.json(); - return text; } \ No newline at end of file diff --git a/src/client/apis/google_docs/GooglePhotosClientUtils.ts b/src/client/apis/google_docs/GooglePhotosClientUtils.ts index b1b29210a..29cc042b6 100644 --- a/src/client/apis/google_docs/GooglePhotosClientUtils.ts +++ b/src/client/apis/google_docs/GooglePhotosClientUtils.ts @@ -14,11 +14,14 @@ import { NewMediaItemResult, MediaItem } from "../../../server/apis/google/Share import { AssertionError } from "assert"; import { DocumentView } from "../../views/nodes/DocumentView"; import { DocumentManager } from "../../util/DocumentManager"; -import { PostToServer } from "../../Network"; +import { Identified } from "../../Network"; export namespace GooglePhotos { - const endpoint = async () => new Photos(await PostToServer(RouteStore.googlePhotosAccessToken)); + const endpoint = async () => { + const accessToken = await Identified.FetchFromServer(RouteStore.googlePhotosAccessToken); + return new Photos(accessToken); + }; export enum MediaType { ALL_MEDIA = 'ALL_MEDIA', @@ -296,7 +299,7 @@ export namespace GooglePhotos { }; export const WriteMediaItemsToServer = async (body: { mediaItems: any[] }): Promise => { - const uploads = await PostToServer(RouteStore.googlePhotosMediaDownload, body); + const uploads = await Identified.PostToServer(RouteStore.googlePhotosMediaDownload, body); return uploads; }; @@ -316,7 +319,7 @@ export namespace GooglePhotos { album = await Create.Album(album.title); } const media: MediaInput[] = []; - sources.forEach(source => { + for (let source of sources) { const data = Cast(Doc.GetProto(source).data, ImageField); if (!data) { return; @@ -324,11 +327,11 @@ export namespace GooglePhotos { const url = data.url.href; const target = Doc.MakeAlias(source); const description = parseDescription(target, descriptionKey); - DocumentView.makeCustomViewClicked(target, undefined); + await DocumentView.makeCustomViewClicked(target, undefined); media.push({ url, description }); - }); + } if (media.length) { - const uploads: NewMediaItemResult[] = await PostToServer(RouteStore.googlePhotosMediaUpload, { media, album }); + const uploads: NewMediaItemResult[] = await Identified.PostToServer(RouteStore.googlePhotosMediaUpload, { media, album }); return uploads; } }; diff --git a/src/client/documents/Documents.ts b/src/client/documents/Documents.ts index a7a006f47..0369e89a6 100644 --- a/src/client/documents/Documents.ts +++ b/src/client/documents/Documents.ts @@ -337,16 +337,18 @@ export namespace Docs { let extension = path.extname(target); target = `${target.substring(0, target.length - extension.length)}_o${extension}`; } - requestImageSize(target) - .then((size: any) => { - let aspect = size.height / size.width; - if (!inst.proto!.nativeWidth) { - inst.proto!.nativeWidth = size.width; - } - inst.proto!.nativeHeight = Number(inst.proto!.nativeWidth!) * aspect; - inst.proto!.height = NumCast(inst.proto!.width) * aspect; - }) - .catch((err: any) => console.log(err)); + if (target !== "http://www.cs.brown.edu/") { + requestImageSize(target) + .then((size: any) => { + let aspect = size.height / size.width; + if (!inst.proto!.nativeWidth) { + inst.proto!.nativeWidth = size.width; + } + inst.proto!.nativeHeight = Number(inst.proto!.nativeWidth!) * aspect; + inst.proto!.height = NumCast(inst.proto!.width) * aspect; + }) + .catch((err: any) => console.log(err)); + } return inst; } export function PresDocument(initial: List = new List(), options: DocumentOptions = {}) { diff --git a/src/client/util/Import & Export/DirectoryImportBox.tsx b/src/client/util/Import & Export/DirectoryImportBox.tsx index d0291aec4..de5287456 100644 --- a/src/client/util/Import & Export/DirectoryImportBox.tsx +++ b/src/client/util/Import & Export/DirectoryImportBox.tsx @@ -21,7 +21,7 @@ import { GooglePhotos } from "../../apis/google_docs/GooglePhotosClientUtils"; import { SchemaHeaderField } from "../../../new_fields/SchemaHeaderField"; import "./DirectoryImportBox.scss"; import BatchedArray from "array-batcher"; -import { PostFormDataToServer } from "../../Network"; +import { Identified } from "../../Network"; const unsupported = ["text/html", "text/plain"]; interface FileResponse { @@ -114,7 +114,7 @@ export default class DirectoryImportBox extends React.Component formData.append(Utils.GenerateGuid(), file); }); - const responses = await PostFormDataToServer(RouteStore.upload, formData); + const responses = await Identified.PostFormDataToServer(RouteStore.upload, formData); runInAction(() => this.completed += batch.length); return responses as FileResponse[]; }); @@ -279,7 +279,6 @@ export default class DirectoryImportBox extends React.Component }} />