From 77e08a4362ba8fab4cab361fcb472702c97edf15 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Sun, 1 Sep 2019 13:05:54 -0400 Subject: initial commit --- src/server/apis/google/GoogleApiServerUtils.ts | 13 ++++++++++--- src/server/apis/google/GooglePhotosUtils.ts | 12 ++++++++++++ 2 files changed, 22 insertions(+), 3 deletions(-) create mode 100644 src/server/apis/google/GooglePhotosUtils.ts (limited to 'src/server/apis') diff --git a/src/server/apis/google/GoogleApiServerUtils.ts b/src/server/apis/google/GoogleApiServerUtils.ts index 8785cd974..2fb44d9a2 100644 --- a/src/server/apis/google/GoogleApiServerUtils.ts +++ b/src/server/apis/google/GoogleApiServerUtils.ts @@ -5,6 +5,7 @@ import { OAuth2Client } from "google-auth-library"; import { Opt } from "../../../new_fields/Doc"; import { GlobalOptions } from "googleapis-common"; import { GaxiosResponse } from "gaxios"; +import Photos = require("googlephotos"); /** * Server side authentication for Google Api queries. @@ -20,16 +21,18 @@ export namespace GoogleApiServerUtils { 'presentations.readonly', 'drive', 'drive.file', + 'photoslibrary', + 'photoslibrary.sharing' ]; export const parseBuffer = (data: Buffer) => JSON.parse(data.toString()); export enum Service { Documents = "Documents", - Slides = "Slides" + Slides = "Slides", + Photos = "Photos" } - export interface CredentialPaths { credentials: string; token: string; @@ -50,7 +53,7 @@ export namespace GoogleApiServerUtils { reject(err); return console.log('Error loading client secret file:', err); } - return authorize(parseBuffer(credentials), paths.token).then(auth => { + return authorize(parseBuffer(credentials), paths.token).then(async auth => { let routed: Opt; let parameters: EndpointParameters = { auth, version: "v1" }; switch (sector) { @@ -60,6 +63,10 @@ export namespace GoogleApiServerUtils { case Service.Slides: routed = google.slides(parameters).presentations; break; + case Service.Photos: + const photos = new Photos(auth); + let response = await photos.albums.list(); + console.log("WE GOT SOMETHING!", response); } resolve(routed); }); diff --git a/src/server/apis/google/GooglePhotosUtils.ts b/src/server/apis/google/GooglePhotosUtils.ts new file mode 100644 index 000000000..c33ad2dd9 --- /dev/null +++ b/src/server/apis/google/GooglePhotosUtils.ts @@ -0,0 +1,12 @@ +import request = require('request-promise'); +const key = require("../../credentials/auth.json"); + +export const PhotosLibraryQuery = async (authToken: any, parameters: any) => { + let options = { + headers: { 'Content-Type': 'application/json' }, + json: parameters, + auth: { 'bearer': authToken }, + }; + const result = await request.post(config.apiEndpoint + '/v1/mediaItems:search', options); + return result; +}; \ No newline at end of file -- cgit v1.2.3-70-g09d2 From 5e12b7d816f1778af112ce69f3029e2f4a72bb08 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Sun, 1 Sep 2019 14:03:32 -0400 Subject: authentication working --- src/server/apis/google/GoogleApiServerUtils.ts | 25 ++++++++-------- src/server/apis/google/GooglePhotosUtils.ts | 22 +++++++------- src/server/authentication/config/passport.ts | 13 -------- src/server/credentials/auth.json | 12 -------- .../credentials/google_photos_credentials.ts | 35 ---------------------- src/server/index.ts | 11 ++++--- 6 files changed, 29 insertions(+), 89 deletions(-) delete mode 100644 src/server/credentials/auth.json delete mode 100644 src/server/credentials/google_photos_credentials.ts (limited to 'src/server/apis') diff --git a/src/server/apis/google/GoogleApiServerUtils.ts b/src/server/apis/google/GoogleApiServerUtils.ts index 2fb44d9a2..c1bd3300e 100644 --- a/src/server/apis/google/GoogleApiServerUtils.ts +++ b/src/server/apis/google/GoogleApiServerUtils.ts @@ -1,7 +1,7 @@ import { google, docs_v1, slides_v1 } from "googleapis"; import { createInterface } from "readline"; import { readFile, writeFile } from "fs"; -import { OAuth2Client } from "google-auth-library"; +import { OAuth2Client, Credentials } from "google-auth-library"; import { Opt } from "../../../new_fields/Doc"; import { GlobalOptions } from "googleapis-common"; import { GaxiosResponse } from "gaxios"; @@ -48,14 +48,14 @@ export namespace GoogleApiServerUtils { export const GetEndpoint = async (sector: string, paths: CredentialPaths) => { return new Promise>((resolve, reject) => { - readFile(paths.credentials, (err, credentials) => { + readFile(paths.credentials, async (err, credentials) => { if (err) { reject(err); return console.log('Error loading client secret file:', err); } - return authorize(parseBuffer(credentials), paths.token).then(async auth => { + authorize(parseBuffer(credentials), paths.token).then(async result => { let routed: Opt; - let parameters: EndpointParameters = { auth, version: "v1" }; + let parameters: EndpointParameters = { auth: result.client, version: "v1" }; switch (sector) { case Service.Documents: routed = google.docs(parameters).documents; @@ -64,7 +64,7 @@ export namespace GoogleApiServerUtils { routed = google.slides(parameters).presentations; break; case Service.Photos: - const photos = new Photos(auth); + const photos = new Photos(result.token.access_token); let response = await photos.albums.list(); console.log("WE GOT SOMETHING!", response); } @@ -74,24 +74,25 @@ export namespace GoogleApiServerUtils { }); }; - + type TokenResult = { token: Credentials, client: OAuth2Client }; /** * Create an OAuth2 client with the given credentials, and returns the promise resolving to the authenticated client * @param {Object} credentials The authorization client credentials. */ - export function authorize(credentials: any, token_path: string): Promise { + export function authorize(credentials: any, token_path: string): Promise { const { client_secret, client_id, redirect_uris } = credentials.installed; const oAuth2Client = new google.auth.OAuth2( client_id, client_secret, redirect_uris[0]); - return new Promise((resolve, reject) => { + return new Promise((resolve, reject) => { readFile(token_path, (err, token) => { // Check if we have previously stored a token. if (err) { return getNewToken(oAuth2Client, token_path).then(resolve, reject); } - oAuth2Client.setCredentials(parseBuffer(token)); - resolve(oAuth2Client); + let parsed = parseBuffer(token); + oAuth2Client.setCredentials(parsed); + resolve({ token: parsed, client: oAuth2Client }); }); }); } @@ -103,7 +104,7 @@ export namespace GoogleApiServerUtils { * @param {getEventsCallback} callback The callback for the authorized client. */ function getNewToken(oAuth2Client: OAuth2Client, token_path: string) { - return new Promise((resolve, reject) => { + return new Promise((resolve, reject) => { const authUrl = oAuth2Client.generateAuthUrl({ access_type: 'offline', scope: SCOPES.map(relative => prefix + relative), @@ -129,7 +130,7 @@ export namespace GoogleApiServerUtils { } console.log('Token stored to', token_path); }); - resolve(oAuth2Client); + resolve({ token, client: oAuth2Client }); }); }); }); diff --git a/src/server/apis/google/GooglePhotosUtils.ts b/src/server/apis/google/GooglePhotosUtils.ts index c33ad2dd9..7f9ffb6f3 100644 --- a/src/server/apis/google/GooglePhotosUtils.ts +++ b/src/server/apis/google/GooglePhotosUtils.ts @@ -1,12 +1,12 @@ -import request = require('request-promise'); -const key = require("../../credentials/auth.json"); +// import request = require('request-promise'); +// const key = require("../../credentials/auth.json"); -export const PhotosLibraryQuery = async (authToken: any, parameters: any) => { - let options = { - headers: { 'Content-Type': 'application/json' }, - json: parameters, - auth: { 'bearer': authToken }, - }; - const result = await request.post(config.apiEndpoint + '/v1/mediaItems:search', options); - return result; -}; \ No newline at end of file +// export const PhotosLibraryQuery = async (authToken: any, parameters: any) => { +// let options = { +// headers: { 'Content-Type': 'application/json' }, +// json: parameters, +// auth: { 'bearer': authToken }, +// }; +// const result = await request.post(config.apiEndpoint + '/v1/mediaItems:search', options); +// return result; +// }; \ No newline at end of file diff --git a/src/server/authentication/config/passport.ts b/src/server/authentication/config/passport.ts index 97ded8785..6e0e01b9e 100644 --- a/src/server/authentication/config/passport.ts +++ b/src/server/authentication/config/passport.ts @@ -5,7 +5,6 @@ import { default as User } from '../models/user_model'; import { Request, Response, NextFunction } from "express"; import { RouteStore } from '../../RouteStore'; import * as GoogleOAuth from "passport-google-oauth20"; -const config = require("../../credentials/google_photos_credentials"); const LocalStrategy = passportLocal.Strategy; const GoogleOAuthStrategy = GoogleOAuth.Strategy; @@ -34,18 +33,6 @@ passport.use(new LocalStrategy({ usernameField: 'email', passReqToCallback: true }); })); - -passport.use(new GoogleOAuthStrategy( - { - clientID: config.oAuthClientID, - clientSecret: config.oAuthclientSecret, - callbackURL: config.oAuthCallbackUrl, - // Set the correct profile URL that does not require any additional APIs - userProfileURL: 'https://www.googleapis.com/oauth2/v3/userinfo' - }, - (token, refreshToken, profile, done) => done(undefined, { profile, token }) -)); - export let isAuthenticated = (req: Request, res: Response, next: NextFunction) => { if (req.isAuthenticated()) { return next(); diff --git a/src/server/credentials/auth.json b/src/server/credentials/auth.json deleted file mode 100644 index 557eca4b6..000000000 --- a/src/server/credentials/auth.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "type": "service_account", - "project_id": "brown-dash", - "private_key_id": "ddf0473a9ac56956b5818e04a7ee406a64d5b0a6", - "private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCueRfxic2oL9nr\nnWSLgl7XR/BKikm4p2sib6szaoTO+q6itcJgt2TDleK/7Y4KW/KhvCfhWVet0Hz0\nIDyg4N/gc2yxuDA6/m8DPWU9kDj8VFR7LVFawOKgo1WbgLcC0Qu8qHzAffrlg8si\nhj3vGuoS/YDn/mz0krwFmCfIx+S0lJ9a7FUjJL5C+CIwAEEYiU7xnTW7pVVNXAm/\n/YKD17ToAjREOtlfVVYO7tZ7V5BiW0I0jJvxw+t1pgrZZe7WPBSBJg9KKGIl+mRi\ndtUMR9Hyt3nMKNZIrSm0OkAz82HxfcapRdSB3wkVjoyW63YaTVHKoBOqRElfMtoM\nqu8wbhhNAgMBAAECggEAA41wJ8kg8J8peQMZ/b7gZvzuPy+h0M/J3j2MrG3rY9qA\nrUv1oqoBSXvhuNDhtEN12oYIqtg6m+L+Sas8CMuOC2rWPafM2u/80IGoGtDhtCjp\nv8inBX8ew4YSiL7IxdbTU2/70Es7DVV5u0t6ndsmr88ibYwwPupGR/fhPCpDyssg\n7lFAEpOwnbKG9a7E7axHpXBRSIE54sh+ESyf6MHH/oyKOLhZ0v4PjRDKaKuMDRst\nMOClgNjD/4bzKpfWljuPYemXz1oIBQitBW5aXnCdsmdrmOLDQpz3qOgIo+RRiyki\nvVU5N54L65sj4WisLt1TT45wbhrkQUz+8GmhV5rHwQKBgQDow32/gSb/M0BKFk5y\n+pSeoLkYp/dwfBFWYT6CNBaKePARFVCdr3db8yOEQD4hrmTOU0EP9c4FSLcaa0Xy\n7n2crhhfZWFpIpRMyXhpeKpqdjQFimfBOK6cIdjnWrtJ6Ik3m4E9p7kKThIsInTc\n/TETwAyzFN+J2ADRdrUdCukOwQKBgQC/4+1Rk8a++Jr9Sznx+JH4vj/J2cGsu7uQ\n0nikcOAFO5HzG7+mt9Wv9/MiPtEYwmc7YziDXldKLpshT2m6wrS1uzzOXAnvXFAh\npiCXQsmVA3gmrVd53k+eZfqrZ1n/rL1kCewRS5LX8xIhM28VIkGqkVy4ZEifMotG\nZKSbH0b4jQKBgQCsJ6rh8Uw+hFGQel8be2pgyM8eBV1lvN213ca11oC1ei1U9Ubi\n2dyWDYa/UiSiFLJKSBlfDJaMIfQLfjwGKY6OS9WK+RjLAeBdysVcfPrOMw7W6j9D\nEgFTSVV8CAdt6qdSkZlNWLfrf0LBkdqNeFbMHMdHzLBo63HverUJ/f/SAQKBgHIk\n2t5T0T14FHnnbaiJ/ArC4J7pcVOWuJQFHs5ydk+mh8LdFrvNTsdF7tLIGwlnWpDx\nDITYcYQnBRBjdLkraONRZXI7PY2sk93wPCK+D7scPTSEmCxeGW5XqyyaZea4klAX\nttzy336lkHs/ZSxlHDqiDU2CGdDY+A//fgroKAdhAoGAA5FXfMzTQLGqxg4J2B1z\nFEXNbrqZZFGgKiveUhhZLm4zPiHXtZXvDtSLwGgcO8oGfTfYueTcHb/Eiar7mKv+\n+SqpAqkINJTthIFVIiD39S9jPFUXzBkf5ZJKPLKQArhzEGxen+SD6ZUO058fA94L\n9FblRGlMtr2o5z0NC7H5zaU=\n-----END PRIVATE KEY-----\n", - "client_email": "google-photos-api@brown-dash.iam.gserviceaccount.com", - "client_id": "112995422877175743408", - "auth_uri": "https://accounts.google.com/o/oauth2/auth", - "token_uri": "https://oauth2.googleapis.com/token", - "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", - "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/google-photos-api%40brown-dash.iam.gserviceaccount.com" -} \ No newline at end of file diff --git a/src/server/credentials/google_photos_credentials.ts b/src/server/credentials/google_photos_credentials.ts deleted file mode 100644 index 11c1c766c..000000000 --- a/src/server/credentials/google_photos_credentials.ts +++ /dev/null @@ -1,35 +0,0 @@ -const config: any = {}; - -// The OAuth client ID from the Google Developers console. -config.oAuthClientID = '1005546247619-l40012sl4idpq17b5emielcs1delffog.apps.googleusercontent.com'; - -// The OAuth client secret from the Google Developers console. -config.oAuthclientSecret = 'xEUJ0OBvhlCKA6SLt8TvWBs3'; - -// The callback to use for OAuth requests. This is the URL where the app is -// running. For testing and running it locally, use 127.0.0.1. -config.oAuthCallbackUrl = 'http://localhost:1050/auth/google/callback'; - -// The port where the app should listen for requests. -config.port = 1050; - -// The scopes to request. The app requires the photoslibrary.readonly and -// plus.me scopes. -config.scopes = [ - 'https://www.googleapis.com/auth/photoslibrary.readonly', - 'profile', -]; - -// The number of photos to load for search requests. -config.photosToLoad = 150; - -// The page size to use for search requests. 100 is reccommended. -config.searchPageSize = 100; - -// The page size to use for the listing albums request. 50 is reccommended. -config.albumPageSize = 50; - -// The API end point to use. Do not change. -config.apiEndpoint = 'https://photoslibrary.googleapis.com'; - -module.exports = config; \ No newline at end of file diff --git a/src/server/index.ts b/src/server/index.ts index 6105dedcc..1f105e9d2 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -29,7 +29,6 @@ import { RouteStore } from './RouteStore'; import v4 = require('uuid/v4'); const app = express(); const config = require('../../webpack.config'); -const OAuthConfig = require('../server/credentials/google_photos_credentials'); import { createCanvas, loadImage, Canvas } from "canvas"; const compiler = webpack(config); const port = 1050; // default port to listen @@ -43,11 +42,11 @@ var AdmZip = require('adm-zip'); import * as YoutubeApi from "./apis/youtube/youtubeApiSample"; import { Response } from 'express-serve-static-core'; import { GoogleApiServerUtils } from "./apis/google/GoogleApiServerUtils"; -import { GaxiosResponse } from 'gaxios'; -import { Opt } from '../new_fields/Doc'; -import { docs_v1 } from 'googleapis'; -import { Endpoint } from 'googleapis-common'; -import { PhotosLibraryQuery } from './apis/google/GooglePhotosUtils'; +// import { GaxiosResponse } from 'gaxios'; +// import { Opt } from '../new_fields/Doc'; +// import { docs_v1 } from 'googleapis'; +// import { Endpoint } from 'googleapis-common'; +// import { PhotosLibraryQuery } from './apis/google/GooglePhotosUtils'; const MongoStore = require('connect-mongo')(session); const mongoose = require('mongoose'); const probe = require("probe-image-size"); -- cgit v1.2.3-70-g09d2 From b8a6b72938804902a7e4478cde9c50339341f67d Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Sun, 1 Sep 2019 14:50:06 -0400 Subject: regenerated auth key --- src/server/apis/google/GoogleApiServerUtils.ts | 3 +-- src/server/credentials/google_docs_token.json | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) (limited to 'src/server/apis') diff --git a/src/server/apis/google/GoogleApiServerUtils.ts b/src/server/apis/google/GoogleApiServerUtils.ts index c1bd3300e..bc9ae2726 100644 --- a/src/server/apis/google/GoogleApiServerUtils.ts +++ b/src/server/apis/google/GoogleApiServerUtils.ts @@ -65,8 +65,7 @@ export namespace GoogleApiServerUtils { break; case Service.Photos: const photos = new Photos(result.token.access_token); - let response = await photos.albums.list(); - console.log("WE GOT SOMETHING!", response); + console.log(await photos.albums.list()); } resolve(routed); }); diff --git a/src/server/credentials/google_docs_token.json b/src/server/credentials/google_docs_token.json index cea452f08..fcb5c8abc 100644 --- a/src/server/credentials/google_docs_token.json +++ b/src/server/credentials/google_docs_token.json @@ -1 +1 @@ -{"access_token":"ya29.Glt2B3lsrpWxZ9DMg1RcTksAFzfR8dVWhf7d7tAvbJ4UbcSVO0Q3aYNGtaMKPtmxR24rH88iQSiKCL8S328TQFEN6LtZgvizymednK5EW0jNCvG6ecdZQ-vwcypR","refresh_token":"1/6X3oGYz4A0p8UW2IgsZ-GqTgQUY43S6Ahsaf_GQhSs8","scope":"https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/documents.readonly https://www.googleapis.com/auth/photoslibrary.sharing https://www.googleapis.com/auth/photoslibrary","token_type":"Bearer","expiry_date":1567360444627} \ No newline at end of file +{"access_token":"ya29.Glt2ByOBCyO7DNKKXaiDbK5c5OMwRoprqiCksLCu93WKuAE-YQ0gDCZQCqP07WV6QH0gMpwn47ghico1h5Rkxh-ukJdY9ndRTv7rEKJY32To4__gZh4xKVhwfOvf","refresh_token":"1/ynmFZmA-yqA1sKU3wF-g6QxCx9wGSTIA2sOvIhC_Ps0","scope":"https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/photoslibrary https://www.googleapis.com/auth/photoslibrary.sharing https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/documents.readonly","token_type":"Bearer","expiry_date":1567366717827} \ No newline at end of file -- cgit v1.2.3-70-g09d2 From 769b4c0b9ac61729b94b32999d3713a2dce53627 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Sun, 1 Sep 2019 17:02:11 -0400 Subject: added typings for google photo album manipulations --- src/server/apis/google/GoogleApiServerUtils.ts | 17 ++- src/server/apis/google/GooglePhotosUtils.ts | 177 +++++++++++++++++++++++-- src/server/authentication/config/passport.ts | 2 - src/server/credentials/google_docs_token.json | 2 +- 4 files changed, 180 insertions(+), 18 deletions(-) (limited to 'src/server/apis') diff --git a/src/server/apis/google/GoogleApiServerUtils.ts b/src/server/apis/google/GoogleApiServerUtils.ts index bc9ae2726..656984b8a 100644 --- a/src/server/apis/google/GoogleApiServerUtils.ts +++ b/src/server/apis/google/GoogleApiServerUtils.ts @@ -5,7 +5,8 @@ import { OAuth2Client, Credentials } from "google-auth-library"; import { Opt } from "../../../new_fields/Doc"; import { GlobalOptions } from "googleapis-common"; import { GaxiosResponse } from "gaxios"; -import Photos = require("googlephotos"); +import { GooglePhotos, CreateAlbum, Action } from "./GooglePhotosUtils"; +import { Utils } from "../../../Utils"; /** * Server side authentication for Google Api queries. @@ -64,8 +65,18 @@ export namespace GoogleApiServerUtils { routed = google.slides(parameters).presentations; break; case Service.Photos: - const photos = new Photos(result.token.access_token); - console.log(await photos.albums.list()); + let token = result.token.access_token; + if (token) { + let create: CreateAlbum = { + action: Action.Create, + body: { + album: { + title: "Sam's Bulk Export", + } + } + }; + console.log(await GooglePhotos.ExecuteQuery(token, create)); + } } resolve(routed); }); diff --git a/src/server/apis/google/GooglePhotosUtils.ts b/src/server/apis/google/GooglePhotosUtils.ts index 7f9ffb6f3..439a41cb6 100644 --- a/src/server/apis/google/GooglePhotosUtils.ts +++ b/src/server/apis/google/GooglePhotosUtils.ts @@ -1,12 +1,165 @@ -// import request = require('request-promise'); -// const key = require("../../credentials/auth.json"); - -// export const PhotosLibraryQuery = async (authToken: any, parameters: any) => { -// let options = { -// headers: { 'Content-Type': 'application/json' }, -// json: parameters, -// auth: { 'bearer': authToken }, -// }; -// const result = await request.post(config.apiEndpoint + '/v1/mediaItems:search', options); -// return result; -// }; \ No newline at end of file +import request = require('request-promise'); + +const apiEndpoint = "https://photoslibrary.googleapis.com"; + +export type GooglePhotosQuery = AlbumsQuery; + +export type AlbumsQuery = (AddEnrichment | BatchAddMediaItems | BatchRemoveMediaItems | CreateAlbum | GetAlbum | ListAlbum | ShareAlbum | UnshareAlbum) & { body: any }; + +export enum Action { + AddEnrichment, + BatchAddMediaItems, + BatchRemoveMediaItems, + Create, + Get, + List, + Share, + Unshare +} + +export interface AddEnrichment { + action: Action.AddEnrichment; + albumId: string; + body: { + newEnrichmentItem: NewEnrichmentItem; + albumPosition: MediaRelativeAlbumPosition; + }; +} + +export interface BatchAddMediaItems { + action: Action.BatchAddMediaItems; + albumId: string; + body: { + mediaItemIds: string[]; + }; +} + +export interface BatchRemoveMediaItems { + action: Action.BatchRemoveMediaItems; + albumId: string; + body: { + mediaItemIds: string[]; + }; +} + +export interface CreateAlbum { + action: Action.Create; + body: { + album: AlbumTemplate; + }; +} + +export interface GetAlbum { + action: Action.Get; + albumId: string; +} + +export interface ListAlbum { + action: Action.List; + parameters: { + pageSize: number, + pageToken: string, + excludeNonAppCreatedData: boolean + }; +} + +export interface ShareAlbum { + action: Action.Share; + albumId: string; + body: { + sharedAlbumOptions: SharedAlbumOptions; + }; +} + +export interface UnshareAlbum { + action: Action.Unshare; + albumId: string; +} + +export interface AlbumTemplate { + title: string; +} + +export interface Album { + id: string; + title: string; + productUrl: string; + isWriteable: boolean; + shareInfo: ShareInfo; + mediaItemsCount: string; + coverPhotoBaseUrl: string; + coverPhotoMediaItemId: string; +} + +export interface ShareInfo { + sharedAlbumOptions: SharedAlbumOptions; + shareableUrl: string; + shareToken: string; + isJoined: boolean; + isOwned: boolean; +} + +export interface SharedAlbumOptions { + isCollaborative: boolean; + isCommentable: boolean; +} + +export enum PositionType { + POSITION_TYPE_UNSPECIFIED, + FIRST_IN_ALBUM, + LAST_IN_ALBUM, + AFTER_MEDIA_ITEM, + AFTER_ENRICHMENT_ITEM +} + +export type AlbumPosition = GeneralAlbumPosition | MediaRelativeAlbumPosition | EnrichmentRelativeAlbumPosition; + +interface GeneralAlbumPosition { + position: PositionType.FIRST_IN_ALBUM | PositionType.LAST_IN_ALBUM | PositionType.POSITION_TYPE_UNSPECIFIED; +} + +interface MediaRelativeAlbumPosition { + position: PositionType.AFTER_MEDIA_ITEM; + relativeMediaItemId: string; +} + +interface EnrichmentRelativeAlbumPosition { + position: PositionType.AFTER_ENRICHMENT_ITEM; + relativeEnrichmentItemId: string; +} + +export interface Location { + locationName: string; + latlng: { + latitude: number, + longitude: number + }; +} + +export interface NewEnrichmentItem { + textEnrichment: { + text: string; + }; + locationEnrichment: { + location: Location + }; + mapEnrichment: { + origin: { location: Location }, + destination: { location: Location } + }; +} + +export namespace GooglePhotos { + + export const ExecuteQuery = async (authToken: string, query: AlbumsQuery) => { + let options = { + headers: { 'Content-Type': 'application/json' }, + auth: { 'bearer': authToken }, + body: query.body, + json: true + }; + const result = await request.post(apiEndpoint + '/v1/albums', options); + return result; + }; + +} diff --git a/src/server/authentication/config/passport.ts b/src/server/authentication/config/passport.ts index 6e0e01b9e..e5733cbb5 100644 --- a/src/server/authentication/config/passport.ts +++ b/src/server/authentication/config/passport.ts @@ -4,10 +4,8 @@ import _ from "lodash"; import { default as User } from '../models/user_model'; import { Request, Response, NextFunction } from "express"; import { RouteStore } from '../../RouteStore'; -import * as GoogleOAuth from "passport-google-oauth20"; const LocalStrategy = passportLocal.Strategy; -const GoogleOAuthStrategy = GoogleOAuth.Strategy; passport.serializeUser((user, done) => { done(undefined, user.id); diff --git a/src/server/credentials/google_docs_token.json b/src/server/credentials/google_docs_token.json index fcb5c8abc..61864512c 100644 --- a/src/server/credentials/google_docs_token.json +++ b/src/server/credentials/google_docs_token.json @@ -1 +1 @@ -{"access_token":"ya29.Glt2ByOBCyO7DNKKXaiDbK5c5OMwRoprqiCksLCu93WKuAE-YQ0gDCZQCqP07WV6QH0gMpwn47ghico1h5Rkxh-ukJdY9ndRTv7rEKJY32To4__gZh4xKVhwfOvf","refresh_token":"1/ynmFZmA-yqA1sKU3wF-g6QxCx9wGSTIA2sOvIhC_Ps0","scope":"https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/photoslibrary https://www.googleapis.com/auth/photoslibrary.sharing https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/documents.readonly","token_type":"Bearer","expiry_date":1567366717827} \ No newline at end of file +{"access_token":"ya29.Glt3B8HoVEda7Ab5TQMVrfvjPN2fFp4sFHtGoDs3TsBgFfw4G208q90JiFjkmQqwODjJi3sf4NCZd78VZTVL3aI0By7_ElZF7XaCvA0LJnfcAi2gi1P-2-boyjYO","refresh_token":"1/tJOVDbPZlADzd2B8Q2_j7jqignXlRwHsU7LbZkdbDBc","scope":"https://www.googleapis.com/auth/documents.readonly https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/photoslibrary.sharing https://www.googleapis.com/auth/photoslibrary","token_type":"Bearer","expiry_date":1567374969108} \ No newline at end of file -- cgit v1.2.3-70-g09d2 From d1960ed915e78014592567e16dd1de9545781a27 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Sun, 1 Sep 2019 17:12:45 -0400 Subject: factored out typings --- src/server/apis/google/GoogleApiServerUtils.ts | 7 +- src/server/apis/google/GooglePhotosUtils.ts | 152 +------------------------ src/server/apis/google/typings/albums.ts | 148 ++++++++++++++++++++++++ 3 files changed, 156 insertions(+), 151 deletions(-) create mode 100644 src/server/apis/google/typings/albums.ts (limited to 'src/server/apis') diff --git a/src/server/apis/google/GoogleApiServerUtils.ts b/src/server/apis/google/GoogleApiServerUtils.ts index 656984b8a..c6d901577 100644 --- a/src/server/apis/google/GoogleApiServerUtils.ts +++ b/src/server/apis/google/GoogleApiServerUtils.ts @@ -5,8 +5,9 @@ import { OAuth2Client, Credentials } from "google-auth-library"; import { Opt } from "../../../new_fields/Doc"; import { GlobalOptions } from "googleapis-common"; import { GaxiosResponse } from "gaxios"; -import { GooglePhotos, CreateAlbum, Action } from "./GooglePhotosUtils"; +import { GooglePhotos } from "./GooglePhotosUtils"; import { Utils } from "../../../Utils"; +import { Album } from "./Typings/albums"; /** * Server side authentication for Google Api queries. @@ -67,8 +68,8 @@ export namespace GoogleApiServerUtils { case Service.Photos: let token = result.token.access_token; if (token) { - let create: CreateAlbum = { - action: Action.Create, + let create: Album.Create = { + action: Album.Action.Create, body: { album: { title: "Sam's Bulk Export", diff --git a/src/server/apis/google/GooglePhotosUtils.ts b/src/server/apis/google/GooglePhotosUtils.ts index 439a41cb6..d4f16fd5d 100644 --- a/src/server/apis/google/GooglePhotosUtils.ts +++ b/src/server/apis/google/GooglePhotosUtils.ts @@ -1,157 +1,13 @@ import request = require('request-promise'); +import { Album } from './Typings/albums'; const apiEndpoint = "https://photoslibrary.googleapis.com"; -export type GooglePhotosQuery = AlbumsQuery; - -export type AlbumsQuery = (AddEnrichment | BatchAddMediaItems | BatchRemoveMediaItems | CreateAlbum | GetAlbum | ListAlbum | ShareAlbum | UnshareAlbum) & { body: any }; - -export enum Action { - AddEnrichment, - BatchAddMediaItems, - BatchRemoveMediaItems, - Create, - Get, - List, - Share, - Unshare -} - -export interface AddEnrichment { - action: Action.AddEnrichment; - albumId: string; - body: { - newEnrichmentItem: NewEnrichmentItem; - albumPosition: MediaRelativeAlbumPosition; - }; -} - -export interface BatchAddMediaItems { - action: Action.BatchAddMediaItems; - albumId: string; - body: { - mediaItemIds: string[]; - }; -} - -export interface BatchRemoveMediaItems { - action: Action.BatchRemoveMediaItems; - albumId: string; - body: { - mediaItemIds: string[]; - }; -} - -export interface CreateAlbum { - action: Action.Create; - body: { - album: AlbumTemplate; - }; -} - -export interface GetAlbum { - action: Action.Get; - albumId: string; -} - -export interface ListAlbum { - action: Action.List; - parameters: { - pageSize: number, - pageToken: string, - excludeNonAppCreatedData: boolean - }; -} - -export interface ShareAlbum { - action: Action.Share; - albumId: string; - body: { - sharedAlbumOptions: SharedAlbumOptions; - }; -} - -export interface UnshareAlbum { - action: Action.Unshare; - albumId: string; -} - -export interface AlbumTemplate { - title: string; -} - -export interface Album { - id: string; - title: string; - productUrl: string; - isWriteable: boolean; - shareInfo: ShareInfo; - mediaItemsCount: string; - coverPhotoBaseUrl: string; - coverPhotoMediaItemId: string; -} - -export interface ShareInfo { - sharedAlbumOptions: SharedAlbumOptions; - shareableUrl: string; - shareToken: string; - isJoined: boolean; - isOwned: boolean; -} - -export interface SharedAlbumOptions { - isCollaborative: boolean; - isCommentable: boolean; -} - -export enum PositionType { - POSITION_TYPE_UNSPECIFIED, - FIRST_IN_ALBUM, - LAST_IN_ALBUM, - AFTER_MEDIA_ITEM, - AFTER_ENRICHMENT_ITEM -} - -export type AlbumPosition = GeneralAlbumPosition | MediaRelativeAlbumPosition | EnrichmentRelativeAlbumPosition; - -interface GeneralAlbumPosition { - position: PositionType.FIRST_IN_ALBUM | PositionType.LAST_IN_ALBUM | PositionType.POSITION_TYPE_UNSPECIFIED; -} - -interface MediaRelativeAlbumPosition { - position: PositionType.AFTER_MEDIA_ITEM; - relativeMediaItemId: string; -} - -interface EnrichmentRelativeAlbumPosition { - position: PositionType.AFTER_ENRICHMENT_ITEM; - relativeEnrichmentItemId: string; -} - -export interface Location { - locationName: string; - latlng: { - latitude: number, - longitude: number - }; -} - -export interface NewEnrichmentItem { - textEnrichment: { - text: string; - }; - locationEnrichment: { - location: Location - }; - mapEnrichment: { - origin: { location: Location }, - destination: { location: Location } - }; -} - export namespace GooglePhotos { - export const ExecuteQuery = async (authToken: string, query: AlbumsQuery) => { + export type Query = Album.Query; + + export const ExecuteQuery = async (authToken: string, query: GooglePhotos.Query) => { let options = { headers: { 'Content-Type': 'application/json' }, auth: { 'bearer': authToken }, diff --git a/src/server/apis/google/typings/albums.ts b/src/server/apis/google/typings/albums.ts new file mode 100644 index 000000000..1c9b379fe --- /dev/null +++ b/src/server/apis/google/typings/albums.ts @@ -0,0 +1,148 @@ +export namespace Album { + + export type Query = (AddEnrichment | BatchAddMediaItems | BatchRemoveMediaItems | Create | Get | List | Share | Unshare) & { body: any }; + + export enum Action { + AddEnrichment, + BatchAddMediaItems, + BatchRemoveMediaItems, + Create, + Get, + List, + Share, + Unshare + } + + export interface AddEnrichment { + action: Action.AddEnrichment; + albumId: string; + body: { + newEnrichmentItem: NewEnrichmentItem; + albumPosition: MediaRelativeAlbumPosition; + }; + } + + export interface BatchAddMediaItems { + action: Action.BatchAddMediaItems; + albumId: string; + body: { + mediaItemIds: string[]; + }; + } + + export interface BatchRemoveMediaItems { + action: Action.BatchRemoveMediaItems; + albumId: string; + body: { + mediaItemIds: string[]; + }; + } + + export interface Create { + action: Action.Create; + body: { + album: Template; + }; + } + + export interface Get { + action: Action.Get; + albumId: string; + } + + export interface List { + action: Action.List; + parameters: { + pageSize: number, + pageToken: string, + excludeNonAppCreatedData: boolean + }; + } + + export interface Share { + action: Action.Share; + albumId: string; + body: { + sharedAlbumOptions: SharedOptions; + }; + } + + export interface Unshare { + action: Action.Unshare; + albumId: string; + } + + export interface Template { + title: string; + } + + export interface Model { + id: string; + title: string; + productUrl: string; + isWriteable: boolean; + shareInfo: ShareInfo; + mediaItemsCount: string; + coverPhotoBaseUrl: string; + coverPhotoMediaItemId: string; + } + + export interface ShareInfo { + sharedAlbumOptions: SharedOptions; + shareableUrl: string; + shareToken: string; + isJoined: boolean; + isOwned: boolean; + } + + export interface SharedOptions { + isCollaborative: boolean; + isCommentable: boolean; + } + + export enum PositionType { + POSITION_TYPE_UNSPECIFIED, + FIRST_IN_ALBUM, + LAST_IN_ALBUM, + AFTER_MEDIA_ITEM, + AFTER_ENRICHMENT_ITEM + } + + export type Position = GeneralAlbumPosition | MediaRelativeAlbumPosition | EnrichmentRelativeAlbumPosition; + + interface GeneralAlbumPosition { + position: PositionType.FIRST_IN_ALBUM | PositionType.LAST_IN_ALBUM | PositionType.POSITION_TYPE_UNSPECIFIED; + } + + interface MediaRelativeAlbumPosition { + position: PositionType.AFTER_MEDIA_ITEM; + relativeMediaItemId: string; + } + + interface EnrichmentRelativeAlbumPosition { + position: PositionType.AFTER_ENRICHMENT_ITEM; + relativeEnrichmentItemId: string; + } + + export interface Location { + locationName: string; + latlng: { + latitude: number, + longitude: number + }; + } + + export interface NewEnrichmentItem { + textEnrichment: { + text: string; + }; + locationEnrichment: { + location: Location + }; + mapEnrichment: { + origin: { location: Location }, + destination: { location: Location } + }; + } + +} \ No newline at end of file -- cgit v1.2.3-70-g09d2 From 8b992ef2c152e86299fd3460112124d476393a60 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Sun, 1 Sep 2019 17:13:09 -0400 Subject: tweaks --- src/server/apis/google/GoogleApiServerUtils.ts | 2 +- src/server/apis/google/GooglePhotosUtils.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'src/server/apis') diff --git a/src/server/apis/google/GoogleApiServerUtils.ts b/src/server/apis/google/GoogleApiServerUtils.ts index c6d901577..00e289b00 100644 --- a/src/server/apis/google/GoogleApiServerUtils.ts +++ b/src/server/apis/google/GoogleApiServerUtils.ts @@ -7,7 +7,7 @@ import { GlobalOptions } from "googleapis-common"; import { GaxiosResponse } from "gaxios"; import { GooglePhotos } from "./GooglePhotosUtils"; import { Utils } from "../../../Utils"; -import { Album } from "./Typings/albums"; +import { Album } from "./typings/albums"; /** * Server side authentication for Google Api queries. diff --git a/src/server/apis/google/GooglePhotosUtils.ts b/src/server/apis/google/GooglePhotosUtils.ts index d4f16fd5d..750630626 100644 --- a/src/server/apis/google/GooglePhotosUtils.ts +++ b/src/server/apis/google/GooglePhotosUtils.ts @@ -1,5 +1,5 @@ import request = require('request-promise'); -import { Album } from './Typings/albums'; +import { Album } from './typings/albums'; const apiEndpoint = "https://photoslibrary.googleapis.com"; -- cgit v1.2.3-70-g09d2 From 38176e5ba949b84dc410d29197180121d81e085c Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Tue, 3 Sep 2019 12:51:59 -0400 Subject: implemented refresh tokens and create, get, list --- .../apis/google_docs/GooglePhotosClientUtils.ts | 35 +++++++++ src/client/views/MainView.tsx | 10 ++- src/server/RouteStore.ts | 3 +- src/server/apis/google/GoogleApiServerUtils.ts | 90 ++++++++++++++-------- src/server/apis/google/GooglePhotosServerUtils.ts | 68 ++++++++++++++++ src/server/apis/google/GooglePhotosUploadUtils.ts | 42 ++++++++++ src/server/apis/google/GooglePhotosUtils.ts | 21 ----- src/server/apis/google/typings/albums.ts | 30 ++++---- src/server/credentials/google_docs_token.json | 2 +- src/server/index.ts | 54 ++++++------- 10 files changed, 252 insertions(+), 103 deletions(-) create mode 100644 src/client/apis/google_docs/GooglePhotosClientUtils.ts create mode 100644 src/server/apis/google/GooglePhotosServerUtils.ts create mode 100644 src/server/apis/google/GooglePhotosUploadUtils.ts delete mode 100644 src/server/apis/google/GooglePhotosUtils.ts (limited to 'src/server/apis') diff --git a/src/client/apis/google_docs/GooglePhotosClientUtils.ts b/src/client/apis/google_docs/GooglePhotosClientUtils.ts new file mode 100644 index 000000000..67a282f48 --- /dev/null +++ b/src/client/apis/google_docs/GooglePhotosClientUtils.ts @@ -0,0 +1,35 @@ +import { Album } from "../../../server/apis/google/typings/albums"; +import { PostToServer } from "../../../Utils"; +import { RouteStore } from "../../../server/RouteStore"; + +export namespace GooglePhotosClientUtils { + + export const Create = async (title: string) => { + let parameters = { + action: Album.Action.Create, + body: { album: { title } } + } as Album.Create; + return PostToServer(RouteStore.googlePhotos, parameters); + }; + + export const List = async (options?: Partial) => { + let parameters = { + action: Album.Action.List, + parameters: { + pageSize: (options ? options.pageSize : 20) || 20, + pageToken: (options ? options.pageToken : undefined) || undefined, + excludeNonAppCreatedData: (options ? options.excludeNonAppCreatedData : false) || false, + } as Album.ListOptions + } as Album.List; + return PostToServer(RouteStore.googlePhotos, parameters); + }; + + export const Get = async (albumId: string) => { + let parameters = { + action: Album.Action.Get, + albumId + } as Album.Get; + return PostToServer(RouteStore.googlePhotos, parameters); + }; + +} \ No newline at end of file diff --git a/src/client/views/MainView.tsx b/src/client/views/MainView.tsx index df0718072..ece475c80 100644 --- a/src/client/views/MainView.tsx +++ b/src/client/views/MainView.tsx @@ -42,6 +42,8 @@ import PresModeMenu from './presentationview/PresentationModeMenu'; import { PresBox } from './nodes/PresBox'; import { GoogleApiClientUtils } from '../apis/google_docs/GoogleApiClientUtils'; import { docs_v1 } from 'googleapis'; +import { Album } from '../../server/apis/google/typings/albums'; +import { GooglePhotosClientUtils } from '../apis/google_docs/GooglePhotosClientUtils'; @observer export class MainView extends React.Component { @@ -128,7 +130,7 @@ export class MainView extends React.Component { window.removeEventListener("keydown", KeyManager.Instance.handle); window.addEventListener("keydown", KeyManager.Instance.handle); - PostToServer('/googleDocs/Photos/Test', {}); + this.executeGooglePhotosAlbumTestRoutine(); reaction(() => { let workspaces = CurrentUserUtils.UserDocument.workspaces; @@ -147,6 +149,12 @@ export class MainView extends React.Component { }, { fireImmediately: true }); } + executeGooglePhotosAlbumTestRoutine = async () => { + let title = "This is a generically created album!"; + console.log(await GooglePhotosClientUtils.Create(title)); + console.log(await GooglePhotosClientUtils.List({ pageSize: 50 })); + } + componentWillUnMount() { window.removeEventListener("keydown", KeyManager.Instance.handle); //close presentation diff --git a/src/server/RouteStore.ts b/src/server/RouteStore.ts index 014906054..fc5511f98 100644 --- a/src/server/RouteStore.ts +++ b/src/server/RouteStore.ts @@ -31,6 +31,7 @@ export enum RouteStore { // APIS cognitiveServices = "/cognitiveservices", - googleDocs = "/googleDocs" + googleDocs = "/googleDocs", + googlePhotos = "/googlePhotos" } \ No newline at end of file diff --git a/src/server/apis/google/GoogleApiServerUtils.ts b/src/server/apis/google/GoogleApiServerUtils.ts index 00e289b00..048ac4b21 100644 --- a/src/server/apis/google/GoogleApiServerUtils.ts +++ b/src/server/apis/google/GoogleApiServerUtils.ts @@ -1,13 +1,12 @@ -import { google, docs_v1, slides_v1 } from "googleapis"; +import { google } from "googleapis"; import { createInterface } from "readline"; import { readFile, writeFile } from "fs"; import { OAuth2Client, Credentials } from "google-auth-library"; import { Opt } from "../../../new_fields/Doc"; import { GlobalOptions } from "googleapis-common"; import { GaxiosResponse } from "gaxios"; -import { GooglePhotos } from "./GooglePhotosUtils"; -import { Utils } from "../../../Utils"; -import { Album } from "./typings/albums"; +import request = require('request-promise'); +import * as qs from 'query-string'; /** * Server side authentication for Google Api queries. @@ -31,8 +30,7 @@ export namespace GoogleApiServerUtils { export enum Service { Documents = "Documents", - Slides = "Slides", - Photos = "Photos" + Slides = "Slides" } export interface CredentialPaths { @@ -49,38 +47,31 @@ export namespace GoogleApiServerUtils { export type EndpointParameters = GlobalOptions & { version: "v1" }; export const GetEndpoint = async (sector: string, paths: CredentialPaths) => { - return new Promise>((resolve, reject) => { + return new Promise>(resolve => { + RetrieveAuthenticationInformation(paths).then(authentication => { + let routed: Opt; + let parameters: EndpointParameters = { auth: authentication.client, version: "v1" }; + switch (sector) { + case Service.Documents: + routed = google.docs(parameters).documents; + break; + case Service.Slides: + routed = google.slides(parameters).presentations; + break; + } + resolve(routed); + }); + }); + }; + + export const RetrieveAuthenticationInformation = async (paths: CredentialPaths) => { + return new Promise((resolve, reject) => { readFile(paths.credentials, async (err, credentials) => { if (err) { reject(err); return console.log('Error loading client secret file:', err); } - authorize(parseBuffer(credentials), paths.token).then(async result => { - let routed: Opt; - let parameters: EndpointParameters = { auth: result.client, version: "v1" }; - switch (sector) { - case Service.Documents: - routed = google.docs(parameters).documents; - break; - case Service.Slides: - routed = google.slides(parameters).presentations; - break; - case Service.Photos: - let token = result.token.access_token; - if (token) { - let create: Album.Create = { - action: Album.Action.Create, - body: { - album: { - title: "Sam's Bulk Export", - } - } - }; - console.log(await GooglePhotos.ExecuteQuery(token, create)); - } - } - resolve(routed); - }); + authorize(parseBuffer(credentials), paths.token).then(resolve, reject); }); }); }; @@ -101,13 +92,44 @@ export namespace GoogleApiServerUtils { if (err) { return getNewToken(oAuth2Client, token_path).then(resolve, reject); } - let parsed = parseBuffer(token); + let parsed: Credentials = parseBuffer(token); + if (parsed.expiry_date! < new Date().getTime()) { + return refreshToken(parsed, client_id, client_secret, oAuth2Client, token_path).then(resolve, reject); + } oAuth2Client.setCredentials(parsed); resolve({ token: parsed, client: oAuth2Client }); }); }); } + const refreshEndpoint = "https://oauth2.googleapis.com/token"; + const refreshToken = (credentials: Credentials, client_id: string, client_secret: string, oAuth2Client: OAuth2Client, token_path: string) => { + return new Promise((resolve, reject) => { + let headerParameters = { headers: { 'Content-Type': 'application/x-www-form-urlencoded' } }; + let queryParameters = { + refreshToken: credentials.refresh_token, + client_id, + client_secret, + grant_type: "refresh_token" + }; + let url = `${refreshEndpoint}?${qs.stringify(queryParameters)}`; + request.post(url, headerParameters).then(response => { + let parsed = JSON.parse(response); + credentials.access_token = parsed.access_token; + credentials.expiry_date = new Date().getTime() + parsed.expires_in; + writeFile(token_path, JSON.stringify(credentials), (err) => { + if (err) { + console.error(err); + reject(err); + } + console.log('Refreshed token stored to', token_path); + oAuth2Client.setCredentials(credentials); + resolve({ token: credentials, client: oAuth2Client }); + }); + }); + }); + }; + /** * Get and store new token after prompting for user authorization, and then * execute the given callback with the authorized OAuth2 client. diff --git a/src/server/apis/google/GooglePhotosServerUtils.ts b/src/server/apis/google/GooglePhotosServerUtils.ts new file mode 100644 index 000000000..cb5464abc --- /dev/null +++ b/src/server/apis/google/GooglePhotosServerUtils.ts @@ -0,0 +1,68 @@ +import request = require('request-promise'); +import { Album } from './typings/albums'; +import * as qs from 'query-string'; + +const apiEndpoint = "https://photoslibrary.googleapis.com/v1/"; + +export interface Authorization { + token: string; +} + +export namespace GooglePhotos { + + export type Query = Album.Query; + export type QueryParameters = { query: GooglePhotos.Query }; + interface DispatchParameters { + required: boolean; + method: "GET" | "POST"; + ignore?: boolean; + } + + export const ExecuteQuery = async (parameters: Authorization & QueryParameters): Promise => { + let action = parameters.query.action; + let dispatch = SuffixMap.get(action)!; + let suffix = Suffix(parameters, dispatch, action); + if (suffix) { + let query: any = parameters.query; + let options: any = { + headers: { 'Content-Type': 'application/json' }, + auth: { 'bearer': parameters.token }, + }; + if (query.body) { + options.body = query.body; + options.json = true; + } + let queryParameters = query.parameters; + if (queryParameters) { + suffix += `?${qs.stringify(queryParameters)}`; + } + let dispatcher = dispatch.method === "POST" ? request.post : request.get; + return dispatcher(apiEndpoint + suffix, options); + } + }; + + const Suffix = (parameters: QueryParameters, dispatch: DispatchParameters, action: Album.Action) => { + let query: any = parameters.query; + let id = query.albumId; + let suffix = 'albums'; + if (dispatch.required) { + if (!id) { + return undefined; + } + suffix += `/${id}${dispatch.ignore ? "" : `:${action}`}`; + } + return suffix; + }; + + const SuffixMap = new Map([ + [Album.Action.AddEnrichment, { required: true, method: "POST" }], + [Album.Action.BatchAddMediaItems, { required: true, method: "POST" }], + [Album.Action.BatchRemoveMediaItems, { required: true, method: "POST" }], + [Album.Action.Create, { required: false, method: "POST" }], + [Album.Action.Get, { required: true, ignore: true, method: "GET" }], + [Album.Action.List, { required: false, method: "GET" }], + [Album.Action.Share, { required: true, method: "POST" }], + [Album.Action.Unshare, { required: true, method: "POST" }] + ]); + +} diff --git a/src/server/apis/google/GooglePhotosUploadUtils.ts b/src/server/apis/google/GooglePhotosUploadUtils.ts new file mode 100644 index 000000000..2e1599aaf --- /dev/null +++ b/src/server/apis/google/GooglePhotosUploadUtils.ts @@ -0,0 +1,42 @@ +import request = require('request-promise'); +import { Authorization } from './GooglePhotosServerUtils'; + +export namespace GooglePhotosUploadUtils { + + interface UploadInformation { + title: string; + url: URL; + } + + const apiEndpoint = "https://photoslibrary.googleapis.com/v1/uploads"; + + export const SubmitUpload = async (parameters: Authorization & UploadInformation) => { + let MEDIA_BINARY_DATA = binary(parameters.url.href); + + let options = { + headers: { + 'Content-Type': 'application/octet-stream', + Authorization: { 'bearer': parameters.token }, + 'X-Goog-Upload-File-Name': parameters.title, + 'X-Goog-Upload-Protocol': 'raw' + }, + body: { MEDIA_BINARY_DATA }, + json: true + }; + const result = await request.post(apiEndpoint, options); + return result; + }; + + const binary = (source: string) => { + const image = document.createElement("img"); + image.src = source; + const canvas = document.createElement("canvas"); + canvas.width = image.width; + canvas.height = image.height; + const ctx = canvas.getContext("2d")!; + ctx.drawImage(image, 0, 0); + const dataUrl = canvas.toDataURL("image/png"); + return dataUrl.replace(/^data:image\/(png|jpg);base64,/, ""); + }; + +} \ No newline at end of file diff --git a/src/server/apis/google/GooglePhotosUtils.ts b/src/server/apis/google/GooglePhotosUtils.ts deleted file mode 100644 index 750630626..000000000 --- a/src/server/apis/google/GooglePhotosUtils.ts +++ /dev/null @@ -1,21 +0,0 @@ -import request = require('request-promise'); -import { Album } from './typings/albums'; - -const apiEndpoint = "https://photoslibrary.googleapis.com"; - -export namespace GooglePhotos { - - export type Query = Album.Query; - - export const ExecuteQuery = async (authToken: string, query: GooglePhotos.Query) => { - let options = { - headers: { 'Content-Type': 'application/json' }, - auth: { 'bearer': authToken }, - body: query.body, - json: true - }; - const result = await request.post(apiEndpoint + '/v1/albums', options); - return result; - }; - -} diff --git a/src/server/apis/google/typings/albums.ts b/src/server/apis/google/typings/albums.ts index 1c9b379fe..f3025567d 100644 --- a/src/server/apis/google/typings/albums.ts +++ b/src/server/apis/google/typings/albums.ts @@ -1,16 +1,16 @@ export namespace Album { - export type Query = (AddEnrichment | BatchAddMediaItems | BatchRemoveMediaItems | Create | Get | List | Share | Unshare) & { body: any }; + export type Query = (AddEnrichment | BatchAddMediaItems | BatchRemoveMediaItems | Create | Get | List | Share | Unshare); export enum Action { - AddEnrichment, - BatchAddMediaItems, - BatchRemoveMediaItems, - Create, - Get, - List, - Share, - Unshare + AddEnrichment = "addEnrichment", + BatchAddMediaItems = "batchAddMediaItems", + BatchRemoveMediaItems = "batchRemoveMediaItems", + Create = "create", + Get = "get", + List = "list", + Share = "share", + Unshare = "unshare" } export interface AddEnrichment { @@ -52,11 +52,13 @@ export namespace Album { export interface List { action: Action.List; - parameters: { - pageSize: number, - pageToken: string, - excludeNonAppCreatedData: boolean - }; + parameters: ListOptions; + } + + export interface ListOptions { + pageSize: number; + pageToken: string; + excludeNonAppCreatedData: boolean; } export interface Share { diff --git a/src/server/credentials/google_docs_token.json b/src/server/credentials/google_docs_token.json index 61864512c..39e067c86 100644 --- a/src/server/credentials/google_docs_token.json +++ b/src/server/credentials/google_docs_token.json @@ -1 +1 @@ -{"access_token":"ya29.Glt3B8HoVEda7Ab5TQMVrfvjPN2fFp4sFHtGoDs3TsBgFfw4G208q90JiFjkmQqwODjJi3sf4NCZd78VZTVL3aI0By7_ElZF7XaCvA0LJnfcAi2gi1P-2-boyjYO","refresh_token":"1/tJOVDbPZlADzd2B8Q2_j7jqignXlRwHsU7LbZkdbDBc","scope":"https://www.googleapis.com/auth/documents.readonly https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/photoslibrary.sharing https://www.googleapis.com/auth/photoslibrary","token_type":"Bearer","expiry_date":1567374969108} \ No newline at end of file +{"access_token":"ya29.Glx4B9p8tvFsmsD-AD-D4jygL_YZVCFpPewM9djtfOT3T4S6ROxN5r0WLAKTYVNnXQbUri3Gu_-vIb0NWq9wEy1TdFTLIM8azWD82X5-I5BQq2DSOsYiKugvgVoHLw","refresh_token":"1/kk_pPY7WBwT34JNPzx_HrSVoZlvfzys4EEVNjp7nqzg6aIbRrEKNMRTb0u0wr9GM","scope":"https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents.readonly https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/photoslibrary.sharing https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/photoslibrary","token_type":"Bearer","expiry_date":1567529401142} \ No newline at end of file diff --git a/src/server/index.ts b/src/server/index.ts index 1f105e9d2..54b954cfb 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -42,11 +42,7 @@ var AdmZip = require('adm-zip'); import * as YoutubeApi from "./apis/youtube/youtubeApiSample"; import { Response } from 'express-serve-static-core'; import { GoogleApiServerUtils } from "./apis/google/GoogleApiServerUtils"; -// import { GaxiosResponse } from 'gaxios'; -// import { Opt } from '../new_fields/Doc'; -// import { docs_v1 } from 'googleapis'; -// import { Endpoint } from 'googleapis-common'; -// import { PhotosLibraryQuery } from './apis/google/GooglePhotosUtils'; +import { GooglePhotos } from './apis/google/GooglePhotosServerUtils'; const MongoStore = require('connect-mongo')(session); const mongoose = require('mongoose'); const probe = require("probe-image-size"); @@ -195,32 +191,7 @@ app.get("/version", (req, res) => { // SEARCH const solrURL = "http://localhost:8983/solr/#/dash"; -// GETTERS - -// app.get('/auth/google', passport.authenticate('google', { -// scope: OAuthConfig.scopes, -// failureFlash: true, // Display errors to the user. -// session: true, -// })); - -// app.get("/failed", (req, res) => res.send("DIDN'T WORK!")); - -// app.get( -// '/auth/google/callback', -// passport.authenticate( -// 'google', { failureRedirect: '/failed', failureFlash: true, session: true }), -// (req, res) => { -// // User has logged in. -// console.log('OAUTH: user has logged in 1.'); -// PhotosLibraryQuery(req.user.token, {}); -// console.log('OAUTH: user has logged in 2.'); -// res.redirect('/'); -// }); - -// app.get('/GooglePhotos', (req, res) => { -// console.log("WORKING ON GOOGLE PHOTOS"); -// PhotosLibraryQuery(req.user.token, {}); -// }); +// GETTERSå app.get("/search", async (req, res) => { const solrQuery: any = {}; @@ -853,6 +824,27 @@ app.post(RouteStore.googleDocs + "/:sector/:action", (req, res) => { }); }); +app.post(RouteStore.googlePhotos, (req, res) => { + GoogleApiServerUtils.RetrieveAuthenticationInformation({ credentials, token }).then(authentication => { + let validated = authentication.token.access_token; + if (!validated) { + res.send("Error: unable to authenticate Google Photos API request."); + return; + } + GooglePhotos.ExecuteQuery({ token: validated, query: req.body }) + .then(response => { + if (response === undefined) { + res.send("Error: unable to build suffix for Google Photos API request"); + return; + } + res.send(response); + }) + .catch(error => { + res.send(`Error: an exception occurred in the execution of this Google Photos API request\n${error}`); + }); + }); +}); + const suffixMap: { [type: string]: (string | [string, string | ((json: any) => any)]) } = { "number": "_n", "string": "_t", -- cgit v1.2.3-70-g09d2 From 0724d62e2a6797c640e79b46b678bd1d3917147f Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Tue, 3 Sep 2019 13:18:23 -0400 Subject: fixed refresh token issue --- src/client/views/MainView.tsx | 8 -------- src/server/apis/google/GoogleApiServerUtils.ts | 2 +- src/server/credentials/google_docs_token.json | 2 +- 3 files changed, 2 insertions(+), 10 deletions(-) (limited to 'src/server/apis') diff --git a/src/client/views/MainView.tsx b/src/client/views/MainView.tsx index ece475c80..3ee62ae86 100644 --- a/src/client/views/MainView.tsx +++ b/src/client/views/MainView.tsx @@ -130,8 +130,6 @@ export class MainView extends React.Component { window.removeEventListener("keydown", KeyManager.Instance.handle); window.addEventListener("keydown", KeyManager.Instance.handle); - this.executeGooglePhotosAlbumTestRoutine(); - reaction(() => { let workspaces = CurrentUserUtils.UserDocument.workspaces; let recent = CurrentUserUtils.UserDocument.recentlyClosed; @@ -149,12 +147,6 @@ export class MainView extends React.Component { }, { fireImmediately: true }); } - executeGooglePhotosAlbumTestRoutine = async () => { - let title = "This is a generically created album!"; - console.log(await GooglePhotosClientUtils.Create(title)); - console.log(await GooglePhotosClientUtils.List({ pageSize: 50 })); - } - componentWillUnMount() { window.removeEventListener("keydown", KeyManager.Instance.handle); //close presentation diff --git a/src/server/apis/google/GoogleApiServerUtils.ts b/src/server/apis/google/GoogleApiServerUtils.ts index 048ac4b21..edfd89de4 100644 --- a/src/server/apis/google/GoogleApiServerUtils.ts +++ b/src/server/apis/google/GoogleApiServerUtils.ts @@ -116,7 +116,7 @@ export namespace GoogleApiServerUtils { request.post(url, headerParameters).then(response => { let parsed = JSON.parse(response); credentials.access_token = parsed.access_token; - credentials.expiry_date = new Date().getTime() + parsed.expires_in; + credentials.expiry_date = new Date().getTime() + parsed.expires_in * 1000; writeFile(token_path, JSON.stringify(credentials), (err) => { if (err) { console.error(err); diff --git a/src/server/credentials/google_docs_token.json b/src/server/credentials/google_docs_token.json index 39e067c86..c83d82607 100644 --- a/src/server/credentials/google_docs_token.json +++ b/src/server/credentials/google_docs_token.json @@ -1 +1 @@ -{"access_token":"ya29.Glx4B9p8tvFsmsD-AD-D4jygL_YZVCFpPewM9djtfOT3T4S6ROxN5r0WLAKTYVNnXQbUri3Gu_-vIb0NWq9wEy1TdFTLIM8azWD82X5-I5BQq2DSOsYiKugvgVoHLw","refresh_token":"1/kk_pPY7WBwT34JNPzx_HrSVoZlvfzys4EEVNjp7nqzg6aIbRrEKNMRTb0u0wr9GM","scope":"https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents.readonly https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/photoslibrary.sharing https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/photoslibrary","token_type":"Bearer","expiry_date":1567529401142} \ No newline at end of file +{"access_token":"ya29.Glx4B4EQ5Q7WNw1hPrcqbZHssp0l1BtzszwNgmKGd783VBP9G3hLMe1AkbQw_Dl0amzgFeO29R4WP0ca-5U_Rf1tjsdtio6XjBGpJrY-S5wK7t71raKkwRPZFITw2Q","refresh_token":"1/kk_pPY7WBwT34JNPzx_HrSVoZlvfzys4EEVNjp7nqzg6aIbRrEKNMRTb0u0wr9GM","scope":"https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents.readonly https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/photoslibrary.sharing https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/photoslibrary","token_type":"Bearer","expiry_date":1567531020435} \ No newline at end of file -- cgit v1.2.3-70-g09d2 From 7e87aa4b7e0125482c87ab61f4a0de14e774558d Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Tue, 3 Sep 2019 16:23:42 -0400 Subject: working on media uploading --- .../apis/google_docs/GooglePhotosClientUtils.ts | 22 ++++++++-- src/client/views/MainView.tsx | 6 +++ src/server/RouteStore.ts | 3 +- src/server/apis/google/GoogleApiServerUtils.ts | 13 +++++- src/server/apis/google/GooglePhotosUploadUtils.ts | 20 ++------- src/server/credentials/google_docs_token.json | 2 +- src/server/index.ts | 50 ++++++++++++++-------- 7 files changed, 73 insertions(+), 43 deletions(-) (limited to 'src/server/apis') diff --git a/src/client/apis/google_docs/GooglePhotosClientUtils.ts b/src/client/apis/google_docs/GooglePhotosClientUtils.ts index 67a282f48..0864ebdb1 100644 --- a/src/client/apis/google_docs/GooglePhotosClientUtils.ts +++ b/src/client/apis/google_docs/GooglePhotosClientUtils.ts @@ -1,6 +1,7 @@ import { Album } from "../../../server/apis/google/typings/albums"; import { PostToServer } from "../../../Utils"; import { RouteStore } from "../../../server/RouteStore"; +import { ImageField } from "../../../new_fields/URLField"; export namespace GooglePhotosClientUtils { @@ -9,7 +10,7 @@ export namespace GooglePhotosClientUtils { action: Album.Action.Create, body: { album: { title } } } as Album.Create; - return PostToServer(RouteStore.googlePhotos, parameters); + return PostToServer(RouteStore.googlePhotosQuery, parameters); }; export const List = async (options?: Partial) => { @@ -21,7 +22,7 @@ export namespace GooglePhotosClientUtils { excludeNonAppCreatedData: (options ? options.excludeNonAppCreatedData : false) || false, } as Album.ListOptions } as Album.List; - return PostToServer(RouteStore.googlePhotos, parameters); + return PostToServer(RouteStore.googlePhotosQuery, parameters); }; export const Get = async (albumId: string) => { @@ -29,7 +30,22 @@ export namespace GooglePhotosClientUtils { action: Album.Action.Get, albumId } as Album.Get; - return PostToServer(RouteStore.googlePhotos, parameters); + return PostToServer(RouteStore.googlePhotosQuery, parameters); + }; + + export const toDataURL = (field: ImageField | undefined) => { + if (!field) { + return undefined; + } + const image = document.createElement("img"); + image.src = field.url.href; + const canvas = document.createElement("canvas"); + canvas.width = image.width; + canvas.height = image.height; + const ctx = canvas.getContext("2d")!; + ctx.drawImage(image, 0, 0); + const dataUrl = canvas.toDataURL("image/png"); + return dataUrl.replace(/^data:image\/(png|jpg);base64,/, ""); }; } \ No newline at end of file diff --git a/src/client/views/MainView.tsx b/src/client/views/MainView.tsx index 3ee62ae86..590addaa3 100644 --- a/src/client/views/MainView.tsx +++ b/src/client/views/MainView.tsx @@ -44,6 +44,7 @@ import { GoogleApiClientUtils } from '../apis/google_docs/GoogleApiClientUtils'; import { docs_v1 } from 'googleapis'; import { Album } from '../../server/apis/google/typings/albums'; import { GooglePhotosClientUtils } from '../apis/google_docs/GooglePhotosClientUtils'; +import { ImageField } from '../../new_fields/URLField'; @observer export class MainView extends React.Component { @@ -130,6 +131,11 @@ export class MainView extends React.Component { window.removeEventListener("keydown", KeyManager.Instance.handle); window.addEventListener("keydown", KeyManager.Instance.handle); + let imgurl = "https://upload.wikimedia.org/wikipedia/commons/thumb/3/3a/Cat03.jpg/1200px-Cat03.jpg"; + let image = Docs.Create.ImageDocument(imgurl, { width: 200, title: "an image of a cat" }); + let parameters = { title: StrCast(image.title), MEDIA_BINARY_DATA: GooglePhotosClientUtils.toDataURL(Cast(image.data, ImageField)) }; + PostToServer(RouteStore.googlePhotosMediaUpload, parameters).then(console.log); + reaction(() => { let workspaces = CurrentUserUtils.UserDocument.workspaces; let recent = CurrentUserUtils.UserDocument.recentlyClosed; diff --git a/src/server/RouteStore.ts b/src/server/RouteStore.ts index fc5511f98..3b3fd9b4a 100644 --- a/src/server/RouteStore.ts +++ b/src/server/RouteStore.ts @@ -32,6 +32,7 @@ export enum RouteStore { // APIS cognitiveServices = "/cognitiveservices", googleDocs = "/googleDocs", - googlePhotos = "/googlePhotos" + googlePhotosQuery = "/googlePhotosQuery", + googlePhotosMediaUpload = "/googlePhotosMediaUpload" } \ No newline at end of file diff --git a/src/server/apis/google/GoogleApiServerUtils.ts b/src/server/apis/google/GoogleApiServerUtils.ts index edfd89de4..2c9085ebb 100644 --- a/src/server/apis/google/GoogleApiServerUtils.ts +++ b/src/server/apis/google/GoogleApiServerUtils.ts @@ -48,7 +48,7 @@ export namespace GoogleApiServerUtils { export const GetEndpoint = async (sector: string, paths: CredentialPaths) => { return new Promise>(resolve => { - RetrieveAuthenticationInformation(paths).then(authentication => { + RetrieveCredentials(paths).then(authentication => { let routed: Opt; let parameters: EndpointParameters = { auth: authentication.client, version: "v1" }; switch (sector) { @@ -64,7 +64,7 @@ export namespace GoogleApiServerUtils { }); }; - export const RetrieveAuthenticationInformation = async (paths: CredentialPaths) => { + export const RetrieveCredentials = async (paths: CredentialPaths) => { return new Promise((resolve, reject) => { readFile(paths.credentials, async (err, credentials) => { if (err) { @@ -76,6 +76,15 @@ export namespace GoogleApiServerUtils { }); }; + export const RetrieveAccessToken = async (paths: CredentialPaths) => { + return new Promise((resolve, reject) => { + RetrieveCredentials(paths).then( + credentials => resolve(credentials.token.access_token!), + error => reject(`Error: unable to authenticate Google Photos API request.\n${error}`) + ); + }); + }; + type TokenResult = { token: Credentials, client: OAuth2Client }; /** * Create an OAuth2 client with the given credentials, and returns the promise resolving to the authenticated client diff --git a/src/server/apis/google/GooglePhotosUploadUtils.ts b/src/server/apis/google/GooglePhotosUploadUtils.ts index 2e1599aaf..b358f9698 100644 --- a/src/server/apis/google/GooglePhotosUploadUtils.ts +++ b/src/server/apis/google/GooglePhotosUploadUtils.ts @@ -5,38 +5,24 @@ export namespace GooglePhotosUploadUtils { interface UploadInformation { title: string; - url: URL; + MEDIA_BINARY_DATA: string; } const apiEndpoint = "https://photoslibrary.googleapis.com/v1/uploads"; export const SubmitUpload = async (parameters: Authorization & UploadInformation) => { - let MEDIA_BINARY_DATA = binary(parameters.url.href); - let options = { headers: { 'Content-Type': 'application/octet-stream', - Authorization: { 'bearer': parameters.token }, + auth: { 'bearer': parameters.token }, 'X-Goog-Upload-File-Name': parameters.title, 'X-Goog-Upload-Protocol': 'raw' }, - body: { MEDIA_BINARY_DATA }, + body: { MEDIA_BINARY_DATA: parameters.MEDIA_BINARY_DATA }, json: true }; const result = await request.post(apiEndpoint, options); return result; }; - const binary = (source: string) => { - const image = document.createElement("img"); - image.src = source; - const canvas = document.createElement("canvas"); - canvas.width = image.width; - canvas.height = image.height; - const ctx = canvas.getContext("2d")!; - ctx.drawImage(image, 0, 0); - const dataUrl = canvas.toDataURL("image/png"); - return dataUrl.replace(/^data:image\/(png|jpg);base64,/, ""); - }; - } \ No newline at end of file diff --git a/src/server/credentials/google_docs_token.json b/src/server/credentials/google_docs_token.json index c83d82607..e4be9ff60 100644 --- a/src/server/credentials/google_docs_token.json +++ b/src/server/credentials/google_docs_token.json @@ -1 +1 @@ -{"access_token":"ya29.Glx4B4EQ5Q7WNw1hPrcqbZHssp0l1BtzszwNgmKGd783VBP9G3hLMe1AkbQw_Dl0amzgFeO29R4WP0ca-5U_Rf1tjsdtio6XjBGpJrY-S5wK7t71raKkwRPZFITw2Q","refresh_token":"1/kk_pPY7WBwT34JNPzx_HrSVoZlvfzys4EEVNjp7nqzg6aIbRrEKNMRTb0u0wr9GM","scope":"https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents.readonly https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/photoslibrary.sharing https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/photoslibrary","token_type":"Bearer","expiry_date":1567531020435} \ No newline at end of file +{"access_token":"ya29.Glx4B9gmH9fKJex9Je-k1KBNWEsJBPgQqoEWHhwk0TtTemCxONoyVMO38idAE7Vy9vdjcosjl0H4swnnH1s2QjTwZaspzKPeQr8Oh4sHkCJ-MbNd6naMZBdy1pccjQ","refresh_token":"1/kk_pPY7WBwT34JNPzx_HrSVoZlvfzys4EEVNjp7nqzg6aIbRrEKNMRTb0u0wr9GM","scope":"https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents.readonly https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/photoslibrary.sharing https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/photoslibrary","token_type":"Bearer","expiry_date":1567545365417} \ No newline at end of file diff --git a/src/server/index.ts b/src/server/index.ts index 54b954cfb..3e85b1ce0 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -43,6 +43,7 @@ import * as YoutubeApi from "./apis/youtube/youtubeApiSample"; import { Response } from 'express-serve-static-core'; import { GoogleApiServerUtils } from "./apis/google/GoogleApiServerUtils"; import { GooglePhotos } from './apis/google/GooglePhotosServerUtils'; +import { GooglePhotosUploadUtils } from './apis/google/GooglePhotosUploadUtils'; const MongoStore = require('connect-mongo')(session); const mongoose = require('mongoose'); const probe = require("probe-image-size"); @@ -824,25 +825,36 @@ app.post(RouteStore.googleDocs + "/:sector/:action", (req, res) => { }); }); -app.post(RouteStore.googlePhotos, (req, res) => { - GoogleApiServerUtils.RetrieveAuthenticationInformation({ credentials, token }).then(authentication => { - let validated = authentication.token.access_token; - if (!validated) { - res.send("Error: unable to authenticate Google Photos API request."); - return; - } - GooglePhotos.ExecuteQuery({ token: validated, query: req.body }) - .then(response => { - if (response === undefined) { - res.send("Error: unable to build suffix for Google Photos API request"); - return; - } - res.send(response); - }) - .catch(error => { - res.send(`Error: an exception occurred in the execution of this Google Photos API request\n${error}`); - }); - }); +app.post(RouteStore.googlePhotosQuery, (req, res) => { + GoogleApiServerUtils.RetrieveAccessToken({ credentials, token }).then( + token => { + GooglePhotos.ExecuteQuery({ token, query: req.body }) + .then(response => { + if (response === undefined) { + res.send("Error: unable to build suffix for Google Photos API request"); + return; + } + res.send(response); + }) + .catch(error => { + res.send(`Error: an exception occurred in the execution of this Google Photos API request\n${error}`); + }); + }, + error => res.send(error) + ); +}); + +app.post(RouteStore.googlePhotosMediaUpload, (req, res) => { + GoogleApiServerUtils.RetrieveAccessToken({ credentials, token }).then( + token => { + GooglePhotosUploadUtils.SubmitUpload({ token, ...req.body }) + .then(response => { + res.send(response); + }).catch(error => { + res.send(`Error: an exception occurred in uploading the given media\n${error}`); + }); + }, + error => res.send(error)); }); const suffixMap: { [type: string]: (string | [string, string | ((json: any) => any)]) } = { -- cgit v1.2.3-70-g09d2 From 0c02b2ff3a41697c43d0aed98f330bd0293ef761 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Tue, 3 Sep 2019 19:21:23 -0400 Subject: fixed upload authentication issue, need to fix image byte extraction --- src/server/apis/google/GoogleApiServerUtils.ts | 1 + src/server/apis/google/GooglePhotosUploadUtils.ts | 2 +- src/server/credentials/google_docs_token.json | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) (limited to 'src/server/apis') diff --git a/src/server/apis/google/GoogleApiServerUtils.ts b/src/server/apis/google/GoogleApiServerUtils.ts index 2c9085ebb..b6330a609 100644 --- a/src/server/apis/google/GoogleApiServerUtils.ts +++ b/src/server/apis/google/GoogleApiServerUtils.ts @@ -23,6 +23,7 @@ export namespace GoogleApiServerUtils { 'drive', 'drive.file', 'photoslibrary', + 'photoslibrary.appendonly', 'photoslibrary.sharing' ]; diff --git a/src/server/apis/google/GooglePhotosUploadUtils.ts b/src/server/apis/google/GooglePhotosUploadUtils.ts index b358f9698..cd2a586eb 100644 --- a/src/server/apis/google/GooglePhotosUploadUtils.ts +++ b/src/server/apis/google/GooglePhotosUploadUtils.ts @@ -14,7 +14,7 @@ export namespace GooglePhotosUploadUtils { let options = { headers: { 'Content-Type': 'application/octet-stream', - auth: { 'bearer': parameters.token }, + Authorization: `Bearer ${parameters.token}`, 'X-Goog-Upload-File-Name': parameters.title, 'X-Goog-Upload-Protocol': 'raw' }, diff --git a/src/server/credentials/google_docs_token.json b/src/server/credentials/google_docs_token.json index e4be9ff60..2ac972ed8 100644 --- a/src/server/credentials/google_docs_token.json +++ b/src/server/credentials/google_docs_token.json @@ -1 +1 @@ -{"access_token":"ya29.Glx4B9gmH9fKJex9Je-k1KBNWEsJBPgQqoEWHhwk0TtTemCxONoyVMO38idAE7Vy9vdjcosjl0H4swnnH1s2QjTwZaspzKPeQr8Oh4sHkCJ-MbNd6naMZBdy1pccjQ","refresh_token":"1/kk_pPY7WBwT34JNPzx_HrSVoZlvfzys4EEVNjp7nqzg6aIbRrEKNMRTb0u0wr9GM","scope":"https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents.readonly https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/photoslibrary.sharing https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/photoslibrary","token_type":"Bearer","expiry_date":1567545365417} \ No newline at end of file +{"access_token":"ya29.Glt5ByifP30HMz6a1fEG77qZlz9fBEOCz4PQ1VA8t_Ck2ZTPJKoyr6xc3-GFZISAqrrw2U8XpMyZv02_URfPwUX0Z_tMdlIFqsygowR-uClukbgQPNtgxp2LS1oW","refresh_token":"1/wK1cUVLnt581ba_pYGPdlTvAa-OS64nB5m5XOXEBJ8Q","scope":"https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/documents.readonly https://www.googleapis.com/auth/photoslibrary.sharing https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/photoslibrary.appendonly https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/photoslibrary","token_type":"Bearer","expiry_date":1567556163894} \ No newline at end of file -- cgit v1.2.3-70-g09d2 From 5d59e9a379417140e10778cd43e8f87ecb816c37 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Fri, 6 Sep 2019 06:29:26 -0400 Subject: lightly tested functional export of any image doc (web url or stored in server public folder) to google photos, optionally stored in a given target album --- package.json | 2 +- .../apis/google_docs/GooglePhotosClientUtils.ts | 73 +++++----- src/client/views/MainView.tsx | 17 ++- src/client/views/nodes/FormattedTextBox.tsx | 1 + src/server/RouteStore.ts | 2 +- src/server/apis/google/GoogleApiServerUtils.ts | 28 ++-- src/server/apis/google/GooglePhotosServerUtils.ts | 68 ---------- src/server/apis/google/GooglePhotosUploadUtils.ts | 122 +++++++++++++++-- src/server/apis/google/typings/albums.ts | 150 --------------------- src/server/credentials/google_docs_token.json | 2 +- src/server/index.ts | 75 +++++------ 11 files changed, 206 insertions(+), 334 deletions(-) delete mode 100644 src/server/apis/google/GooglePhotosServerUtils.ts delete mode 100644 src/server/apis/google/typings/albums.ts (limited to 'src/server/apis') diff --git a/package.json b/package.json index f0f2b467e..f56e34ce0 100644 --- a/package.json +++ b/package.json @@ -224,4 +224,4 @@ "xoauth2": "^1.2.0", "youtube": "^0.1.0" } -} +} \ No newline at end of file diff --git a/src/client/apis/google_docs/GooglePhotosClientUtils.ts b/src/client/apis/google_docs/GooglePhotosClientUtils.ts index b95cc98c9..2b72800a9 100644 --- a/src/client/apis/google_docs/GooglePhotosClientUtils.ts +++ b/src/client/apis/google_docs/GooglePhotosClientUtils.ts @@ -1,53 +1,42 @@ -import { Album } from "../../../server/apis/google/typings/albums"; -import { PostToServer } from "../../../Utils"; +import { PostToServer, Utils } from "../../../Utils"; import { RouteStore } from "../../../server/RouteStore"; import { ImageField } from "../../../new_fields/URLField"; +import { StrCast, Cast } from "../../../new_fields/Types"; +import { Doc, Opt } from "../../../new_fields/Doc"; +import { Id } from "../../../new_fields/FieldSymbols"; +import requestImageSize = require('../../util/request-image-size'); +import Photos = require('googlephotos'); export namespace GooglePhotosClientUtils { - export const Create = async (title: string) => { - let parameters = { - action: Album.Action.Create, - body: { album: { title } } - } as Album.Create; - return PostToServer(RouteStore.googlePhotosQuery, parameters); - }; + export type AlbumReference = { id: string } | { title: string }; + export const endpoint = () => fetch(Utils.prepend(RouteStore.googlePhotosAccessToken)).then(async response => new Photos(await response.text())); - export const List = async (options?: Partial) => { - let parameters = { - action: Album.Action.List, - parameters: { - pageSize: (options ? options.pageSize : 20) || 20, - pageToken: (options ? options.pageToken : undefined) || undefined, - excludeNonAppCreatedData: (options ? options.excludeNonAppCreatedData : false) || false, - } as Album.ListOptions - } as Album.List; - return PostToServer(RouteStore.googlePhotosQuery, parameters); - }; + export interface MediaInput { + description: string; + source: string; + } - export const Get = async (albumId: string) => { - let parameters = { - action: Album.Action.Get, - albumId - } as Album.Get; - return PostToServer(RouteStore.googlePhotosQuery, parameters); - }; - - export const toDataURL = (field: ImageField | undefined) => { - if (!field) { - return undefined; + export const UploadMedia = async (sources: Doc[], album?: AlbumReference) => { + if (album && "title" in album) { + album = (await endpoint()).albums.create(album.title); + } + const media: MediaInput[] = []; + sources.forEach(document => { + const data = Cast(Doc.GetProto(document).data, ImageField); + const description = StrCast(document.caption); + if (!data) { + return undefined; + } + media.push({ + source: data.url.href, + description, + } as MediaInput); + }); + if (media.length) { + return PostToServer(RouteStore.googlePhotosMediaUpload, { media, album }); } - const image = document.createElement("img"); - image.src = field.url.href; - image.width = 200; - image.height = 200; - const canvas = document.createElement("canvas"); - canvas.width = image.width; - canvas.height = image.height; - const ctx = canvas.getContext("2d")!; - ctx.drawImage(image, 0, 0); - const dataUrl = canvas.toDataURL("image/png"); - return dataUrl.replace(/^data:image\/(png|jpg);base64,/, ""); + return undefined; }; } \ No newline at end of file diff --git a/src/client/views/MainView.tsx b/src/client/views/MainView.tsx index c1c95fc88..6d366216e 100644 --- a/src/client/views/MainView.tsx +++ b/src/client/views/MainView.tsx @@ -40,11 +40,10 @@ import { PreviewCursor } from './PreviewCursor'; import { FilterBox } from './search/FilterBox'; import PresModeMenu from './presentationview/PresentationModeMenu'; import { PresBox } from './nodes/PresBox'; -import { docs_v1 } from 'googleapis'; -import { Album } from '../../server/apis/google/typings/albums'; import { GooglePhotosClientUtils } from '../apis/google_docs/GooglePhotosClientUtils'; import { ImageField } from '../../new_fields/URLField'; import { LinkFollowBox } from './linking/LinkFollowBox'; +import { DocumentManager } from '../util/DocumentManager'; @observer export class MainView extends React.Component { @@ -131,10 +130,7 @@ export class MainView extends React.Component { window.removeEventListener("keydown", KeyManager.Instance.handle); window.addEventListener("keydown", KeyManager.Instance.handle); - let imgurl = "https://upload.wikimedia.org/wikipedia/commons/thumb/3/3a/Cat03.jpg/1200px-Cat03.jpg"; - let image = Docs.Create.ImageDocument(imgurl, { width: 200, title: "an image of a cat" }); - let parameters = { title: StrCast(image.title), MEDIA_BINARY_DATA: GooglePhotosClientUtils.toDataURL(Cast(image.data, ImageField)) }; - // PostToServer(RouteStore.googlePhotosMediaUpload, parameters).then(console.log); + this.executeGooglePhotosRoutine(); reaction(() => { let workspaces = CurrentUserUtils.UserDocument.workspaces; @@ -153,6 +149,15 @@ export class MainView extends React.Component { }, { fireImmediately: true }); } + executeGooglePhotosRoutine = async () => { + let imgurl = "https://upload.wikimedia.org/wikipedia/commons/thumb/3/3a/Cat03.jpg/1200px-Cat03.jpg"; + let doc = Docs.Create.ImageDocument(imgurl, { width: 200, title: "an image of a cat" }); + doc.caption = "Well isn't this a nice cat image!"; + let photos = await GooglePhotosClientUtils.endpoint(); + let albumId = (await photos.albums.list(50)).albums.filter((album: any) => album.title === "This is a generically created album!")[0].id; + console.log(await GooglePhotosClientUtils.UploadMedia([doc], { id: albumId })); + } + componentWillUnMount() { window.removeEventListener("keydown", KeyManager.Instance.handle); //close presentation diff --git a/src/client/views/nodes/FormattedTextBox.tsx b/src/client/views/nodes/FormattedTextBox.tsx index b671d06ea..fda9ea33f 100644 --- a/src/client/views/nodes/FormattedTextBox.tsx +++ b/src/client/views/nodes/FormattedTextBox.tsx @@ -133,6 +133,7 @@ export class FormattedTextBox extends DocComponent<(FieldViewProps & FormattedTe if (this.props.isOverlay) { DragManager.StartDragFunctions.push(() => FormattedTextBox.InputBoxOverlay = undefined); } + FormattedTextBox.Instance = this; } public get CurrentDiv(): HTMLDivElement { return this._ref.current!; } diff --git a/src/server/RouteStore.ts b/src/server/RouteStore.ts index 3b3fd9b4a..b221b71bc 100644 --- a/src/server/RouteStore.ts +++ b/src/server/RouteStore.ts @@ -32,7 +32,7 @@ export enum RouteStore { // APIS cognitiveServices = "/cognitiveservices", googleDocs = "/googleDocs", - googlePhotosQuery = "/googlePhotosQuery", + googlePhotosAccessToken = "/googlePhotosAccessToken", googlePhotosMediaUpload = "/googlePhotosMediaUpload" } \ No newline at end of file diff --git a/src/server/apis/google/GoogleApiServerUtils.ts b/src/server/apis/google/GoogleApiServerUtils.ts index b6330a609..ac8023ce1 100644 --- a/src/server/apis/google/GoogleApiServerUtils.ts +++ b/src/server/apis/google/GoogleApiServerUtils.ts @@ -7,6 +7,7 @@ import { GlobalOptions } from "googleapis-common"; import { GaxiosResponse } from "gaxios"; import request = require('request-promise'); import * as qs from 'query-string'; +import Photos = require('googlephotos'); /** * Server side authentication for Google Api queries. @@ -35,19 +36,19 @@ export namespace GoogleApiServerUtils { } export interface CredentialPaths { - credentials: string; - token: string; + credentialsPath: string; + tokenPath: string; } export type ApiResponse = Promise; - export type ApiRouter = (endpoint: Endpoint, paramters: any) => ApiResponse; + export type ApiRouter = (endpoint: Endpoint, parameters: any) => ApiResponse; export type ApiHandler = (parameters: any) => ApiResponse; export type Action = "create" | "retrieve" | "update"; export type Endpoint = { get: ApiHandler, create: ApiHandler, batchUpdate: ApiHandler }; export type EndpointParameters = GlobalOptions & { version: "v1" }; - export const GetEndpoint = async (sector: string, paths: CredentialPaths) => { + export const GetEndpoint = (sector: string, paths: CredentialPaths) => { return new Promise>(resolve => { RetrieveCredentials(paths).then(authentication => { let routed: Opt; @@ -65,19 +66,19 @@ export namespace GoogleApiServerUtils { }); }; - export const RetrieveCredentials = async (paths: CredentialPaths) => { + export const RetrieveCredentials = (paths: CredentialPaths) => { return new Promise((resolve, reject) => { - readFile(paths.credentials, async (err, credentials) => { + readFile(paths.credentialsPath, async (err, credentials) => { if (err) { reject(err); return console.log('Error loading client secret file:', err); } - authorize(parseBuffer(credentials), paths.token).then(resolve, reject); + authorize(parseBuffer(credentials), paths.tokenPath).then(resolve, reject); }); }); }; - export const RetrieveAccessToken = async (paths: CredentialPaths) => { + export const RetrieveAccessToken = (paths: CredentialPaths) => { return new Promise((resolve, reject) => { RetrieveCredentials(paths).then( credentials => resolve(credentials.token.access_token!), @@ -86,6 +87,15 @@ export namespace GoogleApiServerUtils { }); }; + export const RetrievePhotosEndpoint = (paths: CredentialPaths) => { + return new Promise((resolve, reject) => { + RetrieveAccessToken(paths).then( + token => resolve(new Photos(token)), + reject + ); + }); + }; + type TokenResult = { token: Credentials, client: OAuth2Client }; /** * Create an OAuth2 client with the given credentials, and returns the promise resolving to the authenticated client @@ -126,7 +136,7 @@ export namespace GoogleApiServerUtils { request.post(url, headerParameters).then(response => { let parsed = JSON.parse(response); credentials.access_token = parsed.access_token; - credentials.expiry_date = new Date().getTime() + parsed.expires_in * 1000; + credentials.expiry_date = new Date().getTime() + (parsed.expires_in * 1000); writeFile(token_path, JSON.stringify(credentials), (err) => { if (err) { console.error(err); diff --git a/src/server/apis/google/GooglePhotosServerUtils.ts b/src/server/apis/google/GooglePhotosServerUtils.ts deleted file mode 100644 index cb5464abc..000000000 --- a/src/server/apis/google/GooglePhotosServerUtils.ts +++ /dev/null @@ -1,68 +0,0 @@ -import request = require('request-promise'); -import { Album } from './typings/albums'; -import * as qs from 'query-string'; - -const apiEndpoint = "https://photoslibrary.googleapis.com/v1/"; - -export interface Authorization { - token: string; -} - -export namespace GooglePhotos { - - export type Query = Album.Query; - export type QueryParameters = { query: GooglePhotos.Query }; - interface DispatchParameters { - required: boolean; - method: "GET" | "POST"; - ignore?: boolean; - } - - export const ExecuteQuery = async (parameters: Authorization & QueryParameters): Promise => { - let action = parameters.query.action; - let dispatch = SuffixMap.get(action)!; - let suffix = Suffix(parameters, dispatch, action); - if (suffix) { - let query: any = parameters.query; - let options: any = { - headers: { 'Content-Type': 'application/json' }, - auth: { 'bearer': parameters.token }, - }; - if (query.body) { - options.body = query.body; - options.json = true; - } - let queryParameters = query.parameters; - if (queryParameters) { - suffix += `?${qs.stringify(queryParameters)}`; - } - let dispatcher = dispatch.method === "POST" ? request.post : request.get; - return dispatcher(apiEndpoint + suffix, options); - } - }; - - const Suffix = (parameters: QueryParameters, dispatch: DispatchParameters, action: Album.Action) => { - let query: any = parameters.query; - let id = query.albumId; - let suffix = 'albums'; - if (dispatch.required) { - if (!id) { - return undefined; - } - suffix += `/${id}${dispatch.ignore ? "" : `:${action}`}`; - } - return suffix; - }; - - const SuffixMap = new Map([ - [Album.Action.AddEnrichment, { required: true, method: "POST" }], - [Album.Action.BatchAddMediaItems, { required: true, method: "POST" }], - [Album.Action.BatchRemoveMediaItems, { required: true, method: "POST" }], - [Album.Action.Create, { required: false, method: "POST" }], - [Album.Action.Get, { required: true, ignore: true, method: "GET" }], - [Album.Action.List, { required: false, method: "GET" }], - [Album.Action.Share, { required: true, method: "POST" }], - [Album.Action.Unshare, { required: true, method: "POST" }] - ]); - -} diff --git a/src/server/apis/google/GooglePhotosUploadUtils.ts b/src/server/apis/google/GooglePhotosUploadUtils.ts index cd2a586eb..3b513aaf1 100644 --- a/src/server/apis/google/GooglePhotosUploadUtils.ts +++ b/src/server/apis/google/GooglePhotosUploadUtils.ts @@ -1,28 +1,122 @@ import request = require('request-promise'); -import { Authorization } from './GooglePhotosServerUtils'; +import { GoogleApiServerUtils } from './GoogleApiServerUtils'; +import * as fs from 'fs'; +import { Utils } from '../../../Utils'; +import * as path from 'path'; +import { Opt } from '../../../new_fields/Doc'; export namespace GooglePhotosUploadUtils { - interface UploadInformation { - title: string; - MEDIA_BINARY_DATA: string; + export interface Paths { + uploadDirectory: string; + credentialsPath: string; + tokenPath: string; } - const apiEndpoint = "https://photoslibrary.googleapis.com/v1/uploads"; + export interface MediaInput { + description: string; + source: string; + } + + export interface DownloadInformation { + mediaPath: string; + contentType?: string; + contentSize?: string; + } + + const prepend = (extension: string) => `https://photoslibrary.googleapis.com/v1/${extension}`; + const headers = (type: string) => ({ + 'Content-Type': `application/${type}`, + 'Authorization': Bearer, + }); + + let Bearer: string; + let Paths: Paths; - export const SubmitUpload = async (parameters: Authorization & UploadInformation) => { - let options = { + export const initialize = async (paths: Paths) => { + Paths = paths; + const { tokenPath, credentialsPath } = paths; + const token = await GoogleApiServerUtils.RetrieveAccessToken({ tokenPath, credentialsPath }); + Bearer = `Bearer ${token}`; + }; + + export const DispatchGooglePhotosUpload = async (filename: string) => { + let body: Buffer; + if (filename.includes('upload_')) { + const mediaPath = Paths.uploadDirectory + filename; + body = await new Promise((resolve, reject) => { + fs.readFile(mediaPath, (error, data) => error ? reject(error) : resolve(data)); + }); + } else { + body = await request(filename, { encoding: null }); + } + const parameters = { + method: 'POST', headers: { - 'Content-Type': 'application/octet-stream', - Authorization: `Bearer ${parameters.token}`, - 'X-Goog-Upload-File-Name': parameters.title, + ...headers('octet-stream'), + 'X-Goog-Upload-File-Name': filename, 'X-Goog-Upload-Protocol': 'raw' }, - body: { MEDIA_BINARY_DATA: parameters.MEDIA_BINARY_DATA }, - json: true + uri: prepend('uploads'), + body }; - const result = await request.post(apiEndpoint, options); - return result; + return new Promise(resolve => request(parameters, (error, _response, body) => resolve(error ? undefined : body))); + }; + + export const CreateMediaItems = (newMediaItems: any[], album?: { id: string }) => { + return new Promise((resolve, reject) => { + const parameters = { + method: 'POST', + headers: headers('json'), + uri: prepend('mediaItems:batchCreate'), + body: { newMediaItems } as any, + json: true + }; + album && (parameters.body.albumId = album.id); + request(parameters, (error, _response, body) => { + if (error) { + reject(error); + } else { + resolve(body); + } + }); + }); }; + export namespace IOUtils { + + export const Download = async (url: string): Promise> => { + const filename = `temporary_upload_${Utils.GenerateGuid()}${path.extname(url).toLowerCase()}`; + const temporaryDirectory = Paths.uploadDirectory + "temporary/"; + const mediaPath = temporaryDirectory + filename; + + if (!(await createIfNotExists(temporaryDirectory))) { + return undefined; + } + + return new Promise((resolve, reject) => { + request.head(url, (error, res) => { + if (error) { + return reject(error); + } + const information: DownloadInformation = { + mediaPath, + contentType: res.headers['content-type'], + contentSize: res.headers['content-length'], + }; + request(url).pipe(fs.createWriteStream(mediaPath)).on('close', () => resolve(information)); + }); + }); + }; + + export const createIfNotExists = async (path: string) => { + if (await new Promise(resolve => fs.exists(path, resolve))) { + return true; + } + return new Promise(resolve => fs.mkdir(path, error => resolve(error === null))); + }; + + export const Destroy = (mediaPath: string) => new Promise(resolve => fs.unlink(mediaPath, error => resolve(error === null))); + } + } \ No newline at end of file diff --git a/src/server/apis/google/typings/albums.ts b/src/server/apis/google/typings/albums.ts deleted file mode 100644 index f3025567d..000000000 --- a/src/server/apis/google/typings/albums.ts +++ /dev/null @@ -1,150 +0,0 @@ -export namespace Album { - - export type Query = (AddEnrichment | BatchAddMediaItems | BatchRemoveMediaItems | Create | Get | List | Share | Unshare); - - export enum Action { - AddEnrichment = "addEnrichment", - BatchAddMediaItems = "batchAddMediaItems", - BatchRemoveMediaItems = "batchRemoveMediaItems", - Create = "create", - Get = "get", - List = "list", - Share = "share", - Unshare = "unshare" - } - - export interface AddEnrichment { - action: Action.AddEnrichment; - albumId: string; - body: { - newEnrichmentItem: NewEnrichmentItem; - albumPosition: MediaRelativeAlbumPosition; - }; - } - - export interface BatchAddMediaItems { - action: Action.BatchAddMediaItems; - albumId: string; - body: { - mediaItemIds: string[]; - }; - } - - export interface BatchRemoveMediaItems { - action: Action.BatchRemoveMediaItems; - albumId: string; - body: { - mediaItemIds: string[]; - }; - } - - export interface Create { - action: Action.Create; - body: { - album: Template; - }; - } - - export interface Get { - action: Action.Get; - albumId: string; - } - - export interface List { - action: Action.List; - parameters: ListOptions; - } - - export interface ListOptions { - pageSize: number; - pageToken: string; - excludeNonAppCreatedData: boolean; - } - - export interface Share { - action: Action.Share; - albumId: string; - body: { - sharedAlbumOptions: SharedOptions; - }; - } - - export interface Unshare { - action: Action.Unshare; - albumId: string; - } - - export interface Template { - title: string; - } - - export interface Model { - id: string; - title: string; - productUrl: string; - isWriteable: boolean; - shareInfo: ShareInfo; - mediaItemsCount: string; - coverPhotoBaseUrl: string; - coverPhotoMediaItemId: string; - } - - export interface ShareInfo { - sharedAlbumOptions: SharedOptions; - shareableUrl: string; - shareToken: string; - isJoined: boolean; - isOwned: boolean; - } - - export interface SharedOptions { - isCollaborative: boolean; - isCommentable: boolean; - } - - export enum PositionType { - POSITION_TYPE_UNSPECIFIED, - FIRST_IN_ALBUM, - LAST_IN_ALBUM, - AFTER_MEDIA_ITEM, - AFTER_ENRICHMENT_ITEM - } - - export type Position = GeneralAlbumPosition | MediaRelativeAlbumPosition | EnrichmentRelativeAlbumPosition; - - interface GeneralAlbumPosition { - position: PositionType.FIRST_IN_ALBUM | PositionType.LAST_IN_ALBUM | PositionType.POSITION_TYPE_UNSPECIFIED; - } - - interface MediaRelativeAlbumPosition { - position: PositionType.AFTER_MEDIA_ITEM; - relativeMediaItemId: string; - } - - interface EnrichmentRelativeAlbumPosition { - position: PositionType.AFTER_ENRICHMENT_ITEM; - relativeEnrichmentItemId: string; - } - - export interface Location { - locationName: string; - latlng: { - latitude: number, - longitude: number - }; - } - - export interface NewEnrichmentItem { - textEnrichment: { - text: string; - }; - locationEnrichment: { - location: Location - }; - mapEnrichment: { - origin: { location: Location }, - destination: { location: Location } - }; - } - -} \ No newline at end of file diff --git a/src/server/credentials/google_docs_token.json b/src/server/credentials/google_docs_token.json index 2ac972ed8..f3c8cf82a 100644 --- a/src/server/credentials/google_docs_token.json +++ b/src/server/credentials/google_docs_token.json @@ -1 +1 @@ -{"access_token":"ya29.Glt5ByifP30HMz6a1fEG77qZlz9fBEOCz4PQ1VA8t_Ck2ZTPJKoyr6xc3-GFZISAqrrw2U8XpMyZv02_URfPwUX0Z_tMdlIFqsygowR-uClukbgQPNtgxp2LS1oW","refresh_token":"1/wK1cUVLnt581ba_pYGPdlTvAa-OS64nB5m5XOXEBJ8Q","scope":"https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/documents.readonly https://www.googleapis.com/auth/photoslibrary.sharing https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/photoslibrary.appendonly https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/photoslibrary","token_type":"Bearer","expiry_date":1567556163894} \ No newline at end of file +{"access_token":"ya29.Glx7B9S6zCKDE0EgYk9xX9-RhcN8j4IwG9ONopTl1NkPX9FUOw0GI_81mY9bhaouuyOTnrc6FrZD5SDHolWwp3ABNT6l7TmhTLDILgGXIixZkWFRBPpF-xHC8lUd8A","refresh_token":"1/HTv_xFHszu2Nf3iiFrUTaeKzC_Vp2-6bpIB06xW_WHI","scope":"https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/documents.readonly https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/photoslibrary https://www.googleapis.com/auth/photoslibrary.appendonly https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/photoslibrary.sharing","token_type":"Bearer","expiry_date":1567765465073} \ No newline at end of file diff --git a/src/server/index.ts b/src/server/index.ts index 3940bbd58..fab00a02d 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -42,7 +42,6 @@ var AdmZip = require('adm-zip'); import * as YoutubeApi from "./apis/youtube/youtubeApiSample"; import { Response } from 'express-serve-static-core'; import { GoogleApiServerUtils } from "./apis/google/GoogleApiServerUtils"; -import { GooglePhotos } from './apis/google/GooglePhotosServerUtils'; import { GooglePhotosUploadUtils } from './apis/google/GooglePhotosUploadUtils'; const MongoStore = require('connect-mongo')(session); const mongoose = require('mongoose'); @@ -418,10 +417,10 @@ app.get("/thumbnail/:filename", (req, res) => { let filename = req.params.filename; let noExt = filename.substring(0, filename.length - ".png".length); let pagenumber = parseInt(noExt.split('-')[1]); - fs.exists(uploadDir + filename, (exists: boolean) => { - console.log(`${uploadDir + filename} ${exists ? "exists" : "does not exist"}`); + fs.exists(uploadDirectory + filename, (exists: boolean) => { + console.log(`${uploadDirectory + filename} ${exists ? "exists" : "does not exist"}`); if (exists) { - let input = fs.createReadStream(uploadDir + filename); + let input = fs.createReadStream(uploadDirectory + filename); probe(input, (err: any, result: any) => { if (err) { console.log(err); @@ -432,7 +431,7 @@ app.get("/thumbnail/:filename", (req, res) => { }); } else { - LoadPage(uploadDir + filename.substring(0, filename.length - noExt.split('-')[1].length - ".PNG".length - 1) + ".pdf", pagenumber, res); + LoadPage(uploadDirectory + filename.substring(0, filename.length - noExt.split('-')[1].length - ".PNG".length - 1) + ".pdf", pagenumber, res); } }); }); @@ -556,13 +555,13 @@ class NodeCanvasFactory { const pngTypes = [".png", ".PNG"]; const pdfTypes = [".pdf", ".PDF"]; const jpgTypes = [".jpg", ".JPG", ".jpeg", ".JPEG"]; -const uploadDir = __dirname + "/public/files/"; +const uploadDirectory = __dirname + "/public/files/"; // SETTERS app.post( RouteStore.upload, (req, res) => { let form = new formidable.IncomingForm(); - form.uploadDir = uploadDir; + form.uploadDir = uploadDirectory; form.keepExtensions = true; // let path = req.body.path; console.log("upload"); @@ -592,7 +591,7 @@ app.post( } if (isImage) { resizers.forEach(resizer => { - fs.createReadStream(uploadDir + file).pipe(resizer.resizer).pipe(fs.createWriteStream(uploadDir + file.substring(0, file.length - ext.length) + resizer.suffix + ext)); + fs.createReadStream(uploadDirectory + file).pipe(resizer.resizer).pipe(fs.createWriteStream(uploadDirectory + file.substring(0, file.length - ext.length) + resizer.suffix + ext)); }); } names.push(`/files/` + file); @@ -611,7 +610,7 @@ addSecureRoute( res.status(401).send("incorrect parameters specified"); return; } - imageDataUri.outputFile(uri, uploadDir + filename).then((savedName: string) => { + imageDataUri.outputFile(uri, uploadDirectory + filename).then((savedName: string) => { const ext = path.extname(savedName); let resizers = [ { resizer: sharp().resize(100, undefined, { withoutEnlargement: true }), suffix: "_s" }, @@ -632,7 +631,7 @@ addSecureRoute( } if (isImage) { resizers.forEach(resizer => { - fs.createReadStream(savedName).pipe(resizer.resizer).pipe(fs.createWriteStream(uploadDir + filename + resizer.suffix + ext)); + fs.createReadStream(savedName).pipe(resizer.resizer).pipe(fs.createWriteStream(uploadDirectory + filename + resizer.suffix + ext)); }); } res.send("/files/" + filename + ext); @@ -799,8 +798,8 @@ function HandleYoutubeQuery([query, callback]: [YoutubeQueryInput, (result?: any } } -const credentials = path.join(__dirname, "./credentials/google_docs_credentials.json"); -const token = path.join(__dirname, "./credentials/google_docs_token.json"); +const credentialsPath = path.join(__dirname, "./credentials/google_docs_credentials.json"); +const tokenPath = path.join(__dirname, "./credentials/google_docs_token.json"); const EndpointHandlerMap = new Map([ ["create", (api, params) => api.create(params)], @@ -811,7 +810,7 @@ const EndpointHandlerMap = new Map { let sector: any = req.params.sector; let action: any = req.params.action; - GoogleApiServerUtils.GetEndpoint(GoogleApiServerUtils.Service[sector], { credentials, token }).then(endpoint => { + GoogleApiServerUtils.GetEndpoint(GoogleApiServerUtils.Service[sector], { credentialsPath, tokenPath }).then(endpoint => { let handler = EndpointHandlerMap.get(action); if (endpoint && handler) { let execute = handler(endpoint, req.body).then( @@ -825,36 +824,28 @@ app.post(RouteStore.googleDocs + "/:sector/:action", (req, res) => { }); }); -app.post(RouteStore.googlePhotosQuery, (req, res) => { - GoogleApiServerUtils.RetrieveAccessToken({ credentials, token }).then( - token => { - GooglePhotos.ExecuteQuery({ token, query: req.body }) - .then(response => { - if (response === undefined) { - res.send("Error: unable to build suffix for Google Photos API request"); - return; - } - res.send(response); - }) - .catch(error => { - res.send(`Error: an exception occurred in the execution of this Google Photos API request\n${error}`); - }); - }, - error => res.send(error) - ); -}); +app.get(RouteStore.googlePhotosAccessToken, (req, res) => GoogleApiServerUtils.RetrieveAccessToken({ credentialsPath, tokenPath }).then(token => res.send(token))); -app.post(RouteStore.googlePhotosMediaUpload, (req, res) => { - GoogleApiServerUtils.RetrieveAccessToken({ credentials, token }).then( - token => { - GooglePhotosUploadUtils.SubmitUpload({ token, ...req.body }) - .then(response => { - res.send(response); - }).catch(error => { - res.send(`Error: an exception occurred in uploading the given media\n${error}`); - }); - }, - error => res.send(error)); +const tokenError = "Unable to successfully upload bytes for all images!"; +const mediaError = "Unable to convert all uploaded bytes to media items!"; + +app.post(RouteStore.googlePhotosMediaUpload, async (req, res) => { + const media: GooglePhotosUploadUtils.MediaInput[] = req.body.media; + await GooglePhotosUploadUtils.initialize({ uploadDirectory, credentialsPath, tokenPath }); + const newMediaItems = await Promise.all(media.map(async element => { + const uploadToken = await GooglePhotosUploadUtils.DispatchGooglePhotosUpload(element.source); + return !uploadToken ? undefined : { + description: element.description, + simpleMediaItem: { uploadToken } + }; + })); + if (!newMediaItems.every(item => item)) { + return res.send(tokenError); + } + GooglePhotosUploadUtils.CreateMediaItems(newMediaItems, req.body.album).then( + success => res.send(success), + () => res.send(mediaError) + ); }); const suffixMap: { [type: string]: (string | [string, string | ((json: any) => any)]) } = { -- cgit v1.2.3-70-g09d2 From d94509864920b2bbe7f4af8837f3af3f721b7dad Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Sat, 7 Sep 2019 13:19:10 -0400 Subject: implemented via context menu --- src/client/apis/google_docs/GooglePhotosClientUtils.ts | 4 ++-- src/client/views/MainView.tsx | 2 +- src/client/views/nodes/DocumentView.tsx | 5 +++++ src/server/apis/google/GooglePhotosUploadUtils.ts | 16 ++++------------ src/server/credentials/google_docs_token.json | 2 +- src/server/index.ts | 8 ++++---- tsconfig.json | 2 +- 7 files changed, 18 insertions(+), 21 deletions(-) (limited to 'src/server/apis') diff --git a/src/client/apis/google_docs/GooglePhotosClientUtils.ts b/src/client/apis/google_docs/GooglePhotosClientUtils.ts index 2b72800a9..924362c03 100644 --- a/src/client/apis/google_docs/GooglePhotosClientUtils.ts +++ b/src/client/apis/google_docs/GooglePhotosClientUtils.ts @@ -13,8 +13,8 @@ export namespace GooglePhotosClientUtils { export const endpoint = () => fetch(Utils.prepend(RouteStore.googlePhotosAccessToken)).then(async response => new Photos(await response.text())); export interface MediaInput { + url: string; description: string; - source: string; } export const UploadMedia = async (sources: Doc[], album?: AlbumReference) => { @@ -29,7 +29,7 @@ export namespace GooglePhotosClientUtils { return undefined; } media.push({ - source: data.url.href, + url: data.url.href, description, } as MediaInput); }); diff --git a/src/client/views/MainView.tsx b/src/client/views/MainView.tsx index 6d366216e..7fe35494d 100644 --- a/src/client/views/MainView.tsx +++ b/src/client/views/MainView.tsx @@ -130,7 +130,7 @@ export class MainView extends React.Component { window.removeEventListener("keydown", KeyManager.Instance.handle); window.addEventListener("keydown", KeyManager.Instance.handle); - this.executeGooglePhotosRoutine(); + // this.executeGooglePhotosRoutine(); reaction(() => { let workspaces = CurrentUserUtils.UserDocument.workspaces; diff --git a/src/client/views/nodes/DocumentView.tsx b/src/client/views/nodes/DocumentView.tsx index b60730a6b..b8a034efc 100644 --- a/src/client/views/nodes/DocumentView.tsx +++ b/src/client/views/nodes/DocumentView.tsx @@ -41,6 +41,8 @@ import "./DocumentView.scss"; import { FormattedTextBox } from './FormattedTextBox'; import React = require("react"); import { DocumentType } from '../../documents/DocumentTypes'; +import { GooglePhotosClientUtils } from '../../apis/google_docs/GooglePhotosClientUtils'; +import { ImageField } from '../../../new_fields/URLField'; const JsxParser = require('react-jsx-parser').default; //TODO Why does this need to be imported like this? library.add(fa.faTrash); @@ -588,6 +590,9 @@ export class DocumentView extends DocComponent(Docu subitems.push({ description: "Open Right Alias", event: () => this.props.addDocTab && this.props.addDocTab(Doc.MakeAlias(this.props.Document), this.dataDoc, "onRight"), icon: "caret-square-right" }); subitems.push({ description: "Open Fields", event: this.fieldsClicked, icon: "layer-group" }); cm.addItem({ description: "Open...", subitems: subitems, icon: "external-link-alt" }); + if (Cast(this.props.Document.data, ImageField)) { + cm.addItem({ description: "Export to Google Photos", event: () => GooglePhotosClientUtils.UploadMedia([this.props.Document]), icon: "caret-square-right" }); + } let existingMake = ContextMenu.Instance.findByDescription("Make..."); let makes: ContextMenuProps[] = existingMake && "subitems" in existingMake ? existingMake.subitems : []; makes.push({ description: this.props.Document.isBackground ? "Remove Background" : "Into Background", event: this.makeBackground, icon: this.props.Document.lockedPosition ? "unlock" : "lock" }); diff --git a/src/server/apis/google/GooglePhotosUploadUtils.ts b/src/server/apis/google/GooglePhotosUploadUtils.ts index 3b513aaf1..13db1df03 100644 --- a/src/server/apis/google/GooglePhotosUploadUtils.ts +++ b/src/server/apis/google/GooglePhotosUploadUtils.ts @@ -14,8 +14,8 @@ export namespace GooglePhotosUploadUtils { } export interface MediaInput { + url: string; description: string; - source: string; } export interface DownloadInformation { @@ -40,21 +40,13 @@ export namespace GooglePhotosUploadUtils { Bearer = `Bearer ${token}`; }; - export const DispatchGooglePhotosUpload = async (filename: string) => { - let body: Buffer; - if (filename.includes('upload_')) { - const mediaPath = Paths.uploadDirectory + filename; - body = await new Promise((resolve, reject) => { - fs.readFile(mediaPath, (error, data) => error ? reject(error) : resolve(data)); - }); - } else { - body = await request(filename, { encoding: null }); - } + export const DispatchGooglePhotosUpload = async (url: string) => { + const body = await request(url, { encoding: null }); const parameters = { method: 'POST', headers: { ...headers('octet-stream'), - 'X-Goog-Upload-File-Name': filename, + 'X-Goog-Upload-File-Name': path.basename(url), 'X-Goog-Upload-Protocol': 'raw' }, uri: prepend('uploads'), diff --git a/src/server/credentials/google_docs_token.json b/src/server/credentials/google_docs_token.json index f3c8cf82a..e67c4b5ba 100644 --- a/src/server/credentials/google_docs_token.json +++ b/src/server/credentials/google_docs_token.json @@ -1 +1 @@ -{"access_token":"ya29.Glx7B9S6zCKDE0EgYk9xX9-RhcN8j4IwG9ONopTl1NkPX9FUOw0GI_81mY9bhaouuyOTnrc6FrZD5SDHolWwp3ABNT6l7TmhTLDILgGXIixZkWFRBPpF-xHC8lUd8A","refresh_token":"1/HTv_xFHszu2Nf3iiFrUTaeKzC_Vp2-6bpIB06xW_WHI","scope":"https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/documents.readonly https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/photoslibrary https://www.googleapis.com/auth/photoslibrary.appendonly https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/photoslibrary.sharing","token_type":"Bearer","expiry_date":1567765465073} \ No newline at end of file +{"access_token":"ya29.Glx8B81Wqa67aMtB6AwlIUcLO4k0bnsICbtkXJUkqXWPIZgnSw0SnCG0jiFAmwLGPg8ca-Qk3R0SqWt4JlgwfrzuOqt90I0P8tHH2x_4RXfgisVBg4Muf8Gz59AEkA","refresh_token":"1/HTv_xFHszu2Nf3iiFrUTaeKzC_Vp2-6bpIB06xW_WHI","scope":"https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/documents.readonly https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/photoslibrary https://www.googleapis.com/auth/photoslibrary.appendonly https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/photoslibrary.sharing","token_type":"Bearer","expiry_date":1567878663996} \ No newline at end of file diff --git a/src/server/index.ts b/src/server/index.ts index fab00a02d..99d8a02d4 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -808,9 +808,9 @@ const EndpointHandlerMap = new Map { - let sector: any = req.params.sector; - let action: any = req.params.action; - GoogleApiServerUtils.GetEndpoint(GoogleApiServerUtils.Service[sector], { credentialsPath, tokenPath }).then(endpoint => { + let sector: GoogleApiServerUtils.Service = req.params.sector; + let action: GoogleApiServerUtils.Action = req.params.action; + GoogleApiServerUtils.GetEndpoint(sector, { credentialsPath, tokenPath }).then(endpoint => { let handler = EndpointHandlerMap.get(action); if (endpoint && handler) { let execute = handler(endpoint, req.body).then( @@ -833,7 +833,7 @@ app.post(RouteStore.googlePhotosMediaUpload, async (req, res) => { const media: GooglePhotosUploadUtils.MediaInput[] = req.body.media; await GooglePhotosUploadUtils.initialize({ uploadDirectory, credentialsPath, tokenPath }); const newMediaItems = await Promise.all(media.map(async element => { - const uploadToken = await GooglePhotosUploadUtils.DispatchGooglePhotosUpload(element.source); + const uploadToken = await GooglePhotosUploadUtils.DispatchGooglePhotosUpload(element.url); return !uploadToken ? undefined : { description: element.description, simpleMediaItem: { uploadToken } diff --git a/tsconfig.json b/tsconfig.json index 9ea91ec49..75541abca 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -13,7 +13,7 @@ "lib": [ "dom", "es2015" - ], + ] }, // "exclude": [ // "node_modules", -- cgit v1.2.3-70-g09d2 From 32cd51e2bcc0a8cf498c0b31a5ead60802f672de Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Sat, 7 Sep 2019 17:08:49 -0400 Subject: working on import from google photos --- .../apis/google_docs/GooglePhotosClientUtils.ts | 123 +++++++++++++++++++-- src/client/views/MainView.tsx | 4 +- src/client/views/nodes/DocumentView.tsx | 2 +- src/server/RouteStore.ts | 3 +- src/server/apis/google/GooglePhotosUploadUtils.ts | 14 +-- src/server/credentials/google_docs_token.json | 2 +- src/server/index.ts | 16 ++- 7 files changed, 140 insertions(+), 24 deletions(-) (limited to 'src/server/apis') diff --git a/src/client/apis/google_docs/GooglePhotosClientUtils.ts b/src/client/apis/google_docs/GooglePhotosClientUtils.ts index 924362c03..e8daf3dd4 100644 --- a/src/client/apis/google_docs/GooglePhotosClientUtils.ts +++ b/src/client/apis/google_docs/GooglePhotosClientUtils.ts @@ -6,9 +6,42 @@ import { Doc, Opt } from "../../../new_fields/Doc"; import { Id } from "../../../new_fields/FieldSymbols"; import requestImageSize = require('../../util/request-image-size'); import Photos = require('googlephotos'); +import { RichTextField } from "../../../new_fields/RichTextField"; +import { RichTextUtils } from "../../../new_fields/RichTextUtils"; +import { EditorState } from "prosemirror-state"; +import { FormattedTextBox } from "../../views/nodes/FormattedTextBox"; export namespace GooglePhotosClientUtils { + export enum ContentCategories { + NONE = 'NONE', + LANDSCAPES = 'LANDSCAPES', + RECEIPTS = 'RECEIPTS', + CITYSCAPES = 'CITYSCAPES', + LANDMARKS = 'LANDMARKS', + SELFIES = 'SELFIES', + PEOPLE = 'PEOPLE', + PETS = 'PETS', + WEDDINGS = 'WEDDINGS', + BIRTHDAYS = 'BIRTHDAYS', + DOCUMENTS = 'DOCUMENTS', + TRAVEL = 'TRAVEL', + ANIMALS = 'ANIMALS', + FOOD = 'FOOD', + SPORT = 'SPORT', + NIGHT = 'NIGHT', + PERFORMANCES = 'PERFORMANCES', + WHITEBOARDS = 'WHITEBOARDS', + SCREENSHOTS = 'SCREENSHOTS', + UTILITY = 'UTILITY' + } + + export enum MediaType { + ALL_MEDIA = 'ALL_MEDIA', + PHOTO = 'PHOTO', + VIDEO = 'VIDEO' + } + export type AlbumReference = { id: string } | { title: string }; export const endpoint = () => fetch(Utils.prepend(RouteStore.googlePhotosAccessToken)).then(async response => new Photos(await response.text())); @@ -17,26 +50,96 @@ export namespace GooglePhotosClientUtils { description: string; } - export const UploadMedia = async (sources: Doc[], album?: AlbumReference) => { + export const UploadImageDocuments = async (sources: Doc[], album?: AlbumReference, descriptionKey = "caption") => { if (album && "title" in album) { - album = (await endpoint()).albums.create(album.title); + album = await (await endpoint()).albums.create(album.title); } const media: MediaInput[] = []; sources.forEach(document => { const data = Cast(Doc.GetProto(document).data, ImageField); - const description = StrCast(document.caption); - if (!data) { - return undefined; - } - media.push({ + data && media.push({ url: data.url.href, - description, - } as MediaInput); + description: parseDescription(document, descriptionKey), + }); }); if (media.length) { return PostToServer(RouteStore.googlePhotosMediaUpload, { media, album }); } - return undefined; + }; + + const parseDescription = (document: Doc, descriptionKey: string) => { + let description: string = Utils.prepend("/doc/" + document[Id]); + const target = document[descriptionKey]; + if (typeof target === "string") { + description = target; + } else if (target instanceof RichTextField) { + description = RichTextUtils.ToPlainText(EditorState.fromJSON(FormattedTextBox.Instance.config, JSON.parse(target.Data))); + } + return description; + }; + + export interface DateRange { + after: Date; + before: Date; + } + export interface SearchOptions { + pageSize: number; + included: ContentCategories[]; + excluded: ContentCategories[]; + date: Opt; + includeArchivedMedia: boolean; + type: MediaType; + } + + const DefaultSearchOptions: SearchOptions = { + pageSize: 20, + included: [], + excluded: [], + date: undefined, + includeArchivedMedia: true, + type: MediaType.ALL_MEDIA + }; + + export interface SearchResponse { + mediaItems: any[]; + nextPageToken: string; + } + + export const Search = async (requested: Opt>) => { + const options = requested || DefaultSearchOptions; + const photos = await endpoint(); + const filters = new photos.Filters(options.includeArchivedMedia === undefined ? true : options.includeArchivedMedia); + + const included = options.included || []; + const excluded = options.excluded || []; + const contentFilter = new photos.ContentFilter(); + included.length && included.forEach(category => contentFilter.addIncludedContentCategories(category)); + excluded.length && excluded.forEach(category => contentFilter.addExcludedContentCategories(category)); + filters.setContentFilter(contentFilter); + + const date = options.date; + if (date) { + const dateFilter = new photos.DateFilter(); + if (date instanceof Date) { + dateFilter.addDate(date); + } else { + dateFilter.addRange(date.after, date.before); + } + filters.setDateFilter(dateFilter); + } + + filters.setMediaTypeFilter(new photos.MediaTypeFilter(options.type || MediaType.ALL_MEDIA)); + + return new Promise((resolve, reject) => { + photos.mediaItems.search(filters, options.pageSize || 20).then(async (response: SearchResponse) => { + if (!response) { + return reject(); + } + let filenames = await PostToServer(RouteStore.googlePhotosMediaDownload, response); + console.log(filenames); + resolve(filenames); + }); + }); }; } \ No newline at end of file diff --git a/src/client/views/MainView.tsx b/src/client/views/MainView.tsx index 7fe35494d..ee58c684a 100644 --- a/src/client/views/MainView.tsx +++ b/src/client/views/MainView.tsx @@ -131,6 +131,8 @@ export class MainView extends React.Component { window.addEventListener("keydown", KeyManager.Instance.handle); // this.executeGooglePhotosRoutine(); + const imageTag = GooglePhotosClientUtils.ContentCategories; + GooglePhotosClientUtils.Search({ included: [imageTag.ANIMALS] }); reaction(() => { let workspaces = CurrentUserUtils.UserDocument.workspaces; @@ -155,7 +157,7 @@ export class MainView extends React.Component { doc.caption = "Well isn't this a nice cat image!"; let photos = await GooglePhotosClientUtils.endpoint(); let albumId = (await photos.albums.list(50)).albums.filter((album: any) => album.title === "This is a generically created album!")[0].id; - console.log(await GooglePhotosClientUtils.UploadMedia([doc], { id: albumId })); + console.log(await GooglePhotosClientUtils.UploadImageDocuments([doc], { id: albumId })); } componentWillUnMount() { diff --git a/src/client/views/nodes/DocumentView.tsx b/src/client/views/nodes/DocumentView.tsx index b8a034efc..4033ffd9c 100644 --- a/src/client/views/nodes/DocumentView.tsx +++ b/src/client/views/nodes/DocumentView.tsx @@ -591,7 +591,7 @@ export class DocumentView extends DocComponent(Docu subitems.push({ description: "Open Fields", event: this.fieldsClicked, icon: "layer-group" }); cm.addItem({ description: "Open...", subitems: subitems, icon: "external-link-alt" }); if (Cast(this.props.Document.data, ImageField)) { - cm.addItem({ description: "Export to Google Photos", event: () => GooglePhotosClientUtils.UploadMedia([this.props.Document]), icon: "caret-square-right" }); + cm.addItem({ description: "Export to Google Photos", event: () => GooglePhotosClientUtils.UploadImageDocuments([this.props.Document]), icon: "caret-square-right" }); } let existingMake = ContextMenu.Instance.findByDescription("Make..."); let makes: ContextMenuProps[] = existingMake && "subitems" in existingMake ? existingMake.subitems : []; diff --git a/src/server/RouteStore.ts b/src/server/RouteStore.ts index b221b71bc..f65e6134c 100644 --- a/src/server/RouteStore.ts +++ b/src/server/RouteStore.ts @@ -33,6 +33,7 @@ export enum RouteStore { cognitiveServices = "/cognitiveservices", googleDocs = "/googleDocs", googlePhotosAccessToken = "/googlePhotosAccessToken", - googlePhotosMediaUpload = "/googlePhotosMediaUpload" + googlePhotosMediaUpload = "/googlePhotosMediaUpload", + googlePhotosMediaDownload = "/googlePhotosMediaDownload" } \ No newline at end of file diff --git a/src/server/apis/google/GooglePhotosUploadUtils.ts b/src/server/apis/google/GooglePhotosUploadUtils.ts index 13db1df03..032bc2a2d 100644 --- a/src/server/apis/google/GooglePhotosUploadUtils.ts +++ b/src/server/apis/google/GooglePhotosUploadUtils.ts @@ -20,6 +20,7 @@ export namespace GooglePhotosUploadUtils { export interface DownloadInformation { mediaPath: string; + fileName: string; contentType?: string; contentSize?: string; } @@ -77,15 +78,9 @@ export namespace GooglePhotosUploadUtils { export namespace IOUtils { - export const Download = async (url: string): Promise> => { - const filename = `temporary_upload_${Utils.GenerateGuid()}${path.extname(url).toLowerCase()}`; - const temporaryDirectory = Paths.uploadDirectory + "temporary/"; - const mediaPath = temporaryDirectory + filename; - - if (!(await createIfNotExists(temporaryDirectory))) { - return undefined; - } - + export const Download = async (url: string, filename?: string): Promise> => { + const resolved = filename || `upload_${Utils.GenerateGuid()}${path.extname(url).toLowerCase()}`; + const mediaPath = Paths.uploadDirectory + resolved; return new Promise((resolve, reject) => { request.head(url, (error, res) => { if (error) { @@ -95,6 +90,7 @@ export namespace GooglePhotosUploadUtils { mediaPath, contentType: res.headers['content-type'], contentSize: res.headers['content-length'], + fileName: resolved }; request(url).pipe(fs.createWriteStream(mediaPath)).on('close', () => resolve(information)); }); diff --git a/src/server/credentials/google_docs_token.json b/src/server/credentials/google_docs_token.json index e67c4b5ba..88838e18a 100644 --- a/src/server/credentials/google_docs_token.json +++ b/src/server/credentials/google_docs_token.json @@ -1 +1 @@ -{"access_token":"ya29.Glx8B81Wqa67aMtB6AwlIUcLO4k0bnsICbtkXJUkqXWPIZgnSw0SnCG0jiFAmwLGPg8ca-Qk3R0SqWt4JlgwfrzuOqt90I0P8tHH2x_4RXfgisVBg4Muf8Gz59AEkA","refresh_token":"1/HTv_xFHszu2Nf3iiFrUTaeKzC_Vp2-6bpIB06xW_WHI","scope":"https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/documents.readonly https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/photoslibrary https://www.googleapis.com/auth/photoslibrary.appendonly https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/photoslibrary.sharing","token_type":"Bearer","expiry_date":1567878663996} \ No newline at end of file +{"access_token":"ya29.Glx8B266dydsOIEYhedUZYQ8sIsR9utSSxCBUex0O85zYrujZCSTbjVhrXF3Y4q41mLFghLwspgW-1w6zqnGnMtkZhuDGpBGArIwLZsJDyhUugEu3xvh7gY78WfePA","refresh_token":"1/HTv_xFHszu2Nf3iiFrUTaeKzC_Vp2-6bpIB06xW_WHI","scope":"https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/documents.readonly https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/photoslibrary https://www.googleapis.com/auth/photoslibrary.appendonly https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/photoslibrary.sharing","token_type":"Bearer","expiry_date":1567890805451} \ No newline at end of file diff --git a/src/server/index.ts b/src/server/index.ts index 99d8a02d4..aadadb11a 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -113,7 +113,7 @@ function addSecureRoute(method: Method, ) { let abstracted = (req: express.Request, res: express.Response) => { if (req.user) { - handler(req.user as any, res, req); + handler(req.user, res, req); } else { req.session!.target = req.originalUrl; onRejection(res, req); @@ -848,6 +848,20 @@ app.post(RouteStore.googlePhotosMediaUpload, async (req, res) => { ); }); +app.post(RouteStore.googlePhotosMediaDownload, async (req, res) => { + const contents = req.body; + if (!contents) { + return res.send(undefined); + } + await GooglePhotosUploadUtils.initialize({ uploadDirectory, credentialsPath, tokenPath }); + let bundles: GooglePhotosUploadUtils.DownloadInformation[] = []; + await Promise.all(contents.mediaItems.forEach(async (item: any) => { + const information = await GooglePhotosUploadUtils.IOUtils.Download(item.baseUrl, item.filename); + information && bundles.push(information); + })); + res.send(bundles); +}); + const suffixMap: { [type: string]: (string | [string, string | ((json: any) => any)]) } = { "number": "_n", "string": "_t", -- cgit v1.2.3-70-g09d2 From c24f5f29dff8dd22f1d4029a2722ee4d1a725aad Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Sat, 7 Sep 2019 23:15:32 -0400 Subject: collection of search --- src/client/apis/google_docs/GooglePhotosClientUtils.ts | 14 +++++++------- src/client/views/MainView.tsx | 7 ++++--- src/server/apis/google/GooglePhotosUploadUtils.ts | 10 +++++----- src/server/credentials/google_docs_token.json | 2 +- src/server/index.ts | 10 ++++------ 5 files changed, 21 insertions(+), 22 deletions(-) (limited to 'src/server/apis') diff --git a/src/client/apis/google_docs/GooglePhotosClientUtils.ts b/src/client/apis/google_docs/GooglePhotosClientUtils.ts index e8daf3dd4..bb5d23971 100644 --- a/src/client/apis/google_docs/GooglePhotosClientUtils.ts +++ b/src/client/apis/google_docs/GooglePhotosClientUtils.ts @@ -10,6 +10,7 @@ import { RichTextField } from "../../../new_fields/RichTextField"; import { RichTextUtils } from "../../../new_fields/RichTextUtils"; import { EditorState } from "prosemirror-state"; import { FormattedTextBox } from "../../views/nodes/FormattedTextBox"; +import { Docs } from "../../documents/Documents"; export namespace GooglePhotosClientUtils { @@ -130,14 +131,13 @@ export namespace GooglePhotosClientUtils { filters.setMediaTypeFilter(new photos.MediaTypeFilter(options.type || MediaType.ALL_MEDIA)); - return new Promise((resolve, reject) => { + return new Promise(resolve => { photos.mediaItems.search(filters, options.pageSize || 20).then(async (response: SearchResponse) => { - if (!response) { - return reject(); - } - let filenames = await PostToServer(RouteStore.googlePhotosMediaDownload, response); - console.log(filenames); - resolve(filenames); + response && resolve(Docs.Create.StackingDocument((await PostToServer(RouteStore.googlePhotosMediaDownload, response)).map((download: any) => { + let document = Docs.Create.ImageDocument(Utils.prepend(`/files/${download.fileName}`)); + document.contentSize = download.contentSize; + return document; + }), { width: 500, height: 500 })); }); }); }; diff --git a/src/client/views/MainView.tsx b/src/client/views/MainView.tsx index ee58c684a..b72df3715 100644 --- a/src/client/views/MainView.tsx +++ b/src/client/views/MainView.tsx @@ -131,8 +131,6 @@ export class MainView extends React.Component { window.addEventListener("keydown", KeyManager.Instance.handle); // this.executeGooglePhotosRoutine(); - const imageTag = GooglePhotosClientUtils.ContentCategories; - GooglePhotosClientUtils.Search({ included: [imageTag.ANIMALS] }); reaction(() => { let workspaces = CurrentUserUtils.UserDocument.workspaces; @@ -472,12 +470,15 @@ export class MainView extends React.Component { // let youtubeurl = "https://www.youtube.com/embed/TqcApsGRzWw"; // let addYoutubeSearcher = action(() => Docs.Create.YoutubeDocument(youtubeurl, { width: 600, height: 600, title: "youtube search" })); - let btns: [React.RefObject, IconName, string, () => Doc][] = [ + let googlePhotosSearch = () => GooglePhotosClientUtils.Search({ included: [GooglePhotosClientUtils.ContentCategories.ANIMALS] }); + + let btns: [React.RefObject, IconName, string, () => Doc | Promise][] = [ [React.createRef(), "object-group", "Add Collection", addColNode], [React.createRef(), "tv", "Add Presentation Trail", addPresNode], [React.createRef(), "globe-asia", "Add Website", addWebNode], [React.createRef(), "bolt", "Add Button", addButtonDocument], [React.createRef(), "file", "Add Document Dragger", addDragboxNode], + [React.createRef(), "object-group", "Test Google Photos Search", googlePhotosSearch], [React.createRef(), "cloud-upload-alt", "Import Directory", addImportCollectionNode], //remove at some point in favor of addImportCollectionNode //[React.createRef(), "play", "Add Youtube Searcher", addYoutubeSearcher], ]; diff --git a/src/server/apis/google/GooglePhotosUploadUtils.ts b/src/server/apis/google/GooglePhotosUploadUtils.ts index 032bc2a2d..9b3e68761 100644 --- a/src/server/apis/google/GooglePhotosUploadUtils.ts +++ b/src/server/apis/google/GooglePhotosUploadUtils.ts @@ -78,8 +78,8 @@ export namespace GooglePhotosUploadUtils { export namespace IOUtils { - export const Download = async (url: string, filename?: string): Promise> => { - const resolved = filename || `upload_${Utils.GenerateGuid()}${path.extname(url).toLowerCase()}`; + export const Download = async (url: string, filename?: string, prefix = ""): Promise> => { + const resolved = filename || `${prefix}upload_${Utils.GenerateGuid()}${path.extname(url).toLowerCase()}`; const mediaPath = Paths.uploadDirectory + resolved; return new Promise((resolve, reject) => { request.head(url, (error, res) => { @@ -87,10 +87,10 @@ export namespace GooglePhotosUploadUtils { return reject(error); } const information: DownloadInformation = { - mediaPath, - contentType: res.headers['content-type'], + fileName: resolved, contentSize: res.headers['content-length'], - fileName: resolved + contentType: res.headers['content-type'], + mediaPath }; request(url).pipe(fs.createWriteStream(mediaPath)).on('close', () => resolve(information)); }); diff --git a/src/server/credentials/google_docs_token.json b/src/server/credentials/google_docs_token.json index 88838e18a..a1c23ea35 100644 --- a/src/server/credentials/google_docs_token.json +++ b/src/server/credentials/google_docs_token.json @@ -1 +1 @@ -{"access_token":"ya29.Glx8B266dydsOIEYhedUZYQ8sIsR9utSSxCBUex0O85zYrujZCSTbjVhrXF3Y4q41mLFghLwspgW-1w6zqnGnMtkZhuDGpBGArIwLZsJDyhUugEu3xvh7gY78WfePA","refresh_token":"1/HTv_xFHszu2Nf3iiFrUTaeKzC_Vp2-6bpIB06xW_WHI","scope":"https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/documents.readonly https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/photoslibrary https://www.googleapis.com/auth/photoslibrary.appendonly https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/photoslibrary.sharing","token_type":"Bearer","expiry_date":1567890805451} \ No newline at end of file +{"access_token":"ya29.Glx9B3Fumh3qHpgasQvHNNrwNXtmTVWJR9XckFsnUjOswDOO91ccF3FhD4ko7Z-3rvxEljpP1Qj5BgNq305pt-pgIquoLPWYiaEtinHNF7IXGPz4s4raqJWEJPJxow","refresh_token":"1/HTv_xFHszu2Nf3iiFrUTaeKzC_Vp2-6bpIB06xW_WHI","scope":"https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/documents.readonly https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/photoslibrary https://www.googleapis.com/auth/photoslibrary.appendonly https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/photoslibrary.sharing","token_type":"Bearer","expiry_date":1567913435149} \ No newline at end of file diff --git a/src/server/index.ts b/src/server/index.ts index aadadb11a..49010e7e2 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -848,18 +848,16 @@ app.post(RouteStore.googlePhotosMediaUpload, async (req, res) => { ); }); +const prefix = "google_photos_"; app.post(RouteStore.googlePhotosMediaDownload, async (req, res) => { const contents = req.body; if (!contents) { return res.send(undefined); } await GooglePhotosUploadUtils.initialize({ uploadDirectory, credentialsPath, tokenPath }); - let bundles: GooglePhotosUploadUtils.DownloadInformation[] = []; - await Promise.all(contents.mediaItems.forEach(async (item: any) => { - const information = await GooglePhotosUploadUtils.IOUtils.Download(item.baseUrl, item.filename); - information && bundles.push(information); - })); - res.send(bundles); + res.send(await Promise.all(contents.mediaItems.map((item: any) => + GooglePhotosUploadUtils.IOUtils.Download(item.baseUrl, undefined, prefix))) + ); }); const suffixMap: { [type: string]: (string | [string, string | ((json: any) => any)]) } = { -- cgit v1.2.3-70-g09d2 From f18e2265e5d468f1cbf6e82dd5f01d5f5216b851 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Sun, 8 Sep 2019 04:16:04 -0400 Subject: factored out collection creation, sharp() resizers and image management --- .../apis/google_docs/GooglePhotosClientUtils.ts | 27 +++-- src/client/documents/Documents.ts | 9 +- src/client/views/MainView.tsx | 2 +- src/server/apis/google/GooglePhotosUploadUtils.ts | 119 +++++++++++++++------ src/server/credentials/google_docs_token.json | 2 +- src/server/index.ts | 27 +++-- 6 files changed, 131 insertions(+), 55 deletions(-) (limited to 'src/server/apis') diff --git a/src/client/apis/google_docs/GooglePhotosClientUtils.ts b/src/client/apis/google_docs/GooglePhotosClientUtils.ts index bb5d23971..5f5b39b14 100644 --- a/src/client/apis/google_docs/GooglePhotosClientUtils.ts +++ b/src/client/apis/google_docs/GooglePhotosClientUtils.ts @@ -1,16 +1,16 @@ import { PostToServer, Utils } from "../../../Utils"; import { RouteStore } from "../../../server/RouteStore"; import { ImageField } from "../../../new_fields/URLField"; -import { StrCast, Cast } from "../../../new_fields/Types"; +import { Cast } from "../../../new_fields/Types"; import { Doc, Opt } from "../../../new_fields/Doc"; import { Id } from "../../../new_fields/FieldSymbols"; -import requestImageSize = require('../../util/request-image-size'); import Photos = require('googlephotos'); import { RichTextField } from "../../../new_fields/RichTextField"; import { RichTextUtils } from "../../../new_fields/RichTextUtils"; import { EditorState } from "prosemirror-state"; import { FormattedTextBox } from "../../views/nodes/FormattedTextBox"; -import { Docs } from "../../documents/Documents"; +import { Docs, DocumentOptions } from "../../documents/Documents"; +import { type } from "os"; export namespace GooglePhotosClientUtils { @@ -98,7 +98,7 @@ export namespace GooglePhotosClientUtils { excluded: [], date: undefined, includeArchivedMedia: true, - type: MediaType.ALL_MEDIA + type: MediaType.ALL_MEDIA, }; export interface SearchResponse { @@ -106,7 +106,18 @@ export namespace GooglePhotosClientUtils { nextPageToken: string; } - export const Search = async (requested: Opt>) => { + export type CollectionConstructor = (data: Array, options: DocumentOptions, ...args: any) => Doc; + export const CollectionFromSearch = async (provider: CollectionConstructor, requested: Opt>): Promise => { + let downloads = await Search(requested); + return provider(downloads.map((download: any) => { + let document = Docs.Create.ImageDocument(Utils.prepend(`/files/${download.fileNames.clean}`)); + document.fillColumn = true; + document.contentSize = download.contentSize; + return document; + }), { width: 500, height: 500 }); + }; + + export const Search = async (requested: Opt>): Promise => { const options = requested || DefaultSearchOptions; const photos = await endpoint(); const filters = new photos.Filters(options.includeArchivedMedia === undefined ? true : options.includeArchivedMedia); @@ -133,11 +144,7 @@ export namespace GooglePhotosClientUtils { return new Promise(resolve => { photos.mediaItems.search(filters, options.pageSize || 20).then(async (response: SearchResponse) => { - response && resolve(Docs.Create.StackingDocument((await PostToServer(RouteStore.googlePhotosMediaDownload, response)).map((download: any) => { - let document = Docs.Create.ImageDocument(Utils.prepend(`/files/${download.fileName}`)); - document.contentSize = download.contentSize; - return document; - }), { width: 500, height: 500 })); + response && resolve(await PostToServer(RouteStore.googlePhotosMediaDownload, response)); }); }); }; diff --git a/src/client/documents/Documents.ts b/src/client/documents/Documents.ts index 4b7f1eeb6..9bac57d16 100644 --- a/src/client/documents/Documents.ts +++ b/src/client/documents/Documents.ts @@ -21,7 +21,7 @@ import { AggregateFunction } from "../northstar/model/idea/idea"; import { MINIMIZED_ICON_SIZE } from "../views/globalCssVariables.scss"; import { IconBox } from "../views/nodes/IconBox"; import { Field, Doc, Opt } from "../../new_fields/Doc"; -import { OmitKeys, JSONUtils } from "../../Utils"; +import { OmitKeys, JSONUtils, Utils } from "../../Utils"; import { ImageField, VideoField, AudioField, PdfField, WebField, YoutubeField } from "../../new_fields/URLField"; import { HtmlField } from "../../new_fields/HtmlField"; import { List } from "../../new_fields/List"; @@ -332,7 +332,12 @@ export namespace Docs { export function ImageDocument(url: string, options: DocumentOptions = {}) { let imgField = new ImageField(new URL(url)); let inst = InstanceFromProto(Prototypes.get(DocumentType.IMG), imgField, { title: path.basename(url), ...options }); - requestImageSize(imgField.url.href) + let target = imgField.url.href; + if (new RegExp(window.location.origin).test(target)) { + let extension = path.extname(target); + target = `${target.substring(0, target.length - extension.length)}_o${extension}`; + } + requestImageSize(target) .then((size: any) => { let aspect = size.height / size.width; if (!inst.proto!.nativeWidth) { diff --git a/src/client/views/MainView.tsx b/src/client/views/MainView.tsx index b72df3715..326c13424 100644 --- a/src/client/views/MainView.tsx +++ b/src/client/views/MainView.tsx @@ -470,7 +470,7 @@ export class MainView extends React.Component { // let youtubeurl = "https://www.youtube.com/embed/TqcApsGRzWw"; // let addYoutubeSearcher = action(() => Docs.Create.YoutubeDocument(youtubeurl, { width: 600, height: 600, title: "youtube search" })); - let googlePhotosSearch = () => GooglePhotosClientUtils.Search({ included: [GooglePhotosClientUtils.ContentCategories.ANIMALS] }); + let googlePhotosSearch = () => GooglePhotosClientUtils.CollectionFromSearch(Docs.Create.MasonryDocument, { included: [GooglePhotosClientUtils.ContentCategories.LANDSCAPES] }); let btns: [React.RefObject, IconName, string, () => Doc | Promise][] = [ [React.createRef(), "object-group", "Add Collection", addColNode], diff --git a/src/server/apis/google/GooglePhotosUploadUtils.ts b/src/server/apis/google/GooglePhotosUploadUtils.ts index 9b3e68761..5ac3eaef7 100644 --- a/src/server/apis/google/GooglePhotosUploadUtils.ts +++ b/src/server/apis/google/GooglePhotosUploadUtils.ts @@ -4,6 +4,9 @@ import * as fs from 'fs'; import { Utils } from '../../../Utils'; import * as path from 'path'; import { Opt } from '../../../new_fields/Doc'; +import * as sharp from 'sharp'; + +const uploadDirectory = path.join(__dirname, "../../public/files/"); export namespace GooglePhotosUploadUtils { @@ -18,13 +21,6 @@ export namespace GooglePhotosUploadUtils { description: string; } - export interface DownloadInformation { - mediaPath: string; - fileName: string; - contentType?: string; - contentSize?: string; - } - const prepend = (extension: string) => `https://photoslibrary.googleapis.com/v1/${extension}`; const headers = (type: string) => ({ 'Content-Type': `application/${type}`, @@ -76,35 +72,92 @@ export namespace GooglePhotosUploadUtils { }); }; - export namespace IOUtils { +} - export const Download = async (url: string, filename?: string, prefix = ""): Promise> => { - const resolved = filename || `${prefix}upload_${Utils.GenerateGuid()}${path.extname(url).toLowerCase()}`; - const mediaPath = Paths.uploadDirectory + resolved; - return new Promise((resolve, reject) => { - request.head(url, (error, res) => { - if (error) { - return reject(error); - } - const information: DownloadInformation = { - fileName: resolved, - contentSize: res.headers['content-length'], - contentType: res.headers['content-type'], - mediaPath - }; - request(url).pipe(fs.createWriteStream(mediaPath)).on('close', () => resolve(information)); - }); - }); - }; +export namespace DownloadUtils { - export const createIfNotExists = async (path: string) => { - if (await new Promise(resolve => fs.exists(path, resolve))) { - return true; - } - return new Promise(resolve => fs.mkdir(path, error => resolve(error === null))); - }; + export interface Size { + width: number; + suffix: string; + } - export const Destroy = (mediaPath: string) => new Promise(resolve => fs.unlink(mediaPath, error => resolve(error === null))); + export const Sizes: { [size: string]: Size } = { + SMALL: { width: 100, suffix: "_s" }, + MEDIUM: { width: 400, suffix: "_m" }, + LARGE: { width: 900, suffix: "_l" }, + }; + + const png = ".png"; + const pngs = [".png", ".PNG"]; + const jpg = [".jpg", ".JPG", ".jpeg", ".JPEG"]; + const size = "content-length"; + const type = "content-type"; + + export interface DownloadInformation { + mediaPaths: string[]; + fileNames: { [key: string]: string }; + contentSize?: string; + contentType?: string; } + const generate = (prefix: string, url: string) => `${prefix}upload_${Utils.GenerateGuid()}${path.extname(url).toLowerCase()}`; + const sanitize = (filename: string) => filename.replace(/\s+/g, "_"); + + export const Download = async (url: string, filename?: string, prefix = ""): Promise> => { + const resolved = filename ? sanitize(filename) : generate(prefix, url); + const extension = path.extname(url) || path.extname(resolved) || png; + return new Promise((resolve, reject) => { + request.head(url, async (error, res) => { + if (error) { + return reject(error); + } + const information: DownloadInformation = { + fileNames: { clean: resolved }, + contentSize: res.headers[size], + contentType: res.headers[type], + mediaPaths: [] + }; + const resizers = [ + { resizer: sharp().rotate(), suffix: "_o" }, + ...Object.values(Sizes).map(size => ({ + resizer: sharp().resize(size.width, undefined, { withoutEnlargement: true }).rotate(), + suffix: size.suffix + })) + ]; + let validated = true; + if (pngs.includes(extension)) { + resizers.forEach(element => element.resizer = element.resizer.png()); + } else if (jpg.includes(extension)) { + resizers.forEach(element => element.resizer = element.resizer.jpeg()); + } else { + validated = false; + } + if (validated) { + for (let resizer of resizers) { + const suffix = resizer.suffix; + let mediaPath: string; + await new Promise(resolve => { + const filename = resolved.substring(0, resolved.length - extension.length) + suffix + extension; + information.mediaPaths.push(mediaPath = uploadDirectory + filename); + information.fileNames[suffix] = filename; + request(url) + .pipe(resizer.resizer) + .pipe(fs.createWriteStream(mediaPath)) + .on('close', resolve); + }); + } + resolve(information); + } + }); + }); + }; + + export const createIfNotExists = async (path: string) => { + if (await new Promise(resolve => fs.exists(path, resolve))) { + return true; + } + return new Promise(resolve => fs.mkdir(path, error => resolve(error === null))); + }; + + export const Destroy = (mediaPath: string) => new Promise(resolve => fs.unlink(mediaPath, error => resolve(error === null))); } \ No newline at end of file diff --git a/src/server/credentials/google_docs_token.json b/src/server/credentials/google_docs_token.json index a1c23ea35..4f911f7e0 100644 --- a/src/server/credentials/google_docs_token.json +++ b/src/server/credentials/google_docs_token.json @@ -1 +1 @@ -{"access_token":"ya29.Glx9B3Fumh3qHpgasQvHNNrwNXtmTVWJR9XckFsnUjOswDOO91ccF3FhD4ko7Z-3rvxEljpP1Qj5BgNq305pt-pgIquoLPWYiaEtinHNF7IXGPz4s4raqJWEJPJxow","refresh_token":"1/HTv_xFHszu2Nf3iiFrUTaeKzC_Vp2-6bpIB06xW_WHI","scope":"https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/documents.readonly https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/photoslibrary https://www.googleapis.com/auth/photoslibrary.appendonly https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/photoslibrary.sharing","token_type":"Bearer","expiry_date":1567913435149} \ No newline at end of file +{"access_token":"ya29.Glx9B3_l5JbKNtvzx378Nsz917bP-OTKf6VZzc2K8QDBm-Y0j_-c8v7bL8LCEM3wF8d7JauF-5Z4Uq4v7wPwUQUlDO1uPyoHeSF6iz98xkgJr9OW4KzJo2Ij722gpQ","refresh_token":"1/HTv_xFHszu2Nf3iiFrUTaeKzC_Vp2-6bpIB06xW_WHI","scope":"https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/documents.readonly https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/photoslibrary https://www.googleapis.com/auth/photoslibrary.appendonly https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/photoslibrary.sharing","token_type":"Bearer","expiry_date":1567931641928} \ No newline at end of file diff --git a/src/server/index.ts b/src/server/index.ts index 49010e7e2..013345a76 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -42,7 +42,7 @@ var AdmZip = require('adm-zip'); import * as YoutubeApi from "./apis/youtube/youtubeApiSample"; import { Response } from 'express-serve-static-core'; import { GoogleApiServerUtils } from "./apis/google/GoogleApiServerUtils"; -import { GooglePhotosUploadUtils } from './apis/google/GooglePhotosUploadUtils'; +import { GooglePhotosUploadUtils, DownloadUtils } from './apis/google/GooglePhotosUploadUtils'; const MongoStore = require('connect-mongo')(session); const mongoose = require('mongoose'); const probe = require("probe-image-size"); @@ -154,6 +154,11 @@ app.get("/buxton", (req, res) => { command_line('python scraper.py', cwd).then(onResolved, tryPython3); }); +const STATUS = { + OK: 200, + BAD_REQUEST: 400 +}; + const command_line = (command: string, fromDirectory?: string) => { return new Promise((resolve, reject) => { let options: ExecOptions = {}; @@ -848,16 +853,22 @@ app.post(RouteStore.googlePhotosMediaUpload, async (req, res) => { ); }); +interface MediaItem { + baseUrl: string; + filename: string; +} const prefix = "google_photos_"; + app.post(RouteStore.googlePhotosMediaDownload, async (req, res) => { - const contents = req.body; - if (!contents) { - return res.send(undefined); + const contents: { mediaItems: MediaItem[] } = req.body; + if (contents) { + const downloads = contents.mediaItems.map(item => + DownloadUtils.Download(item.baseUrl, item.filename, prefix) + ); + res.status(STATUS.OK).send(await Promise.all(downloads)); + return; } - await GooglePhotosUploadUtils.initialize({ uploadDirectory, credentialsPath, tokenPath }); - res.send(await Promise.all(contents.mediaItems.map((item: any) => - GooglePhotosUploadUtils.IOUtils.Download(item.baseUrl, undefined, prefix))) - ); + res.status(STATUS.BAD_REQUEST).send(); }); const suffixMap: { [type: string]: (string | [string, string | ((json: any) => any)]) } = { -- cgit v1.2.3-70-g09d2 From 65e5366f59ef2933460aafdc98790f42611f149f Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Sun, 8 Sep 2019 15:24:53 -0400 Subject: refactor of uploader to handle local and remote images --- .../util/Import & Export/DirectoryImportBox.tsx | 2 +- src/server/apis/google/GooglePhotosUploadUtils.ts | 101 ++++++++++++--------- src/server/credentials/google_docs_token.json | 2 +- src/server/index.ts | 68 ++++++-------- 4 files changed, 89 insertions(+), 84 deletions(-) (limited to 'src/server/apis') diff --git a/src/client/util/Import & Export/DirectoryImportBox.tsx b/src/client/util/Import & Export/DirectoryImportBox.tsx index 75b0b52a7..35d6e3c60 100644 --- a/src/client/util/Import & Export/DirectoryImportBox.tsx +++ b/src/client/util/Import & Export/DirectoryImportBox.tsx @@ -313,7 +313,7 @@ export default class DirectoryImportBox extends React.Component style={{ pointerEvents: "none", position: "absolute", - right: isEditing ? 16.3 : 14.5, + right: isEditing ? 14 : 15, top: isEditing ? 15.4 : 16, opacity: uploading ? 0 : 1, transition: "0.4s opacity ease" diff --git a/src/server/apis/google/GooglePhotosUploadUtils.ts b/src/server/apis/google/GooglePhotosUploadUtils.ts index 5ac3eaef7..d4a2a2bb3 100644 --- a/src/server/apis/google/GooglePhotosUploadUtils.ts +++ b/src/server/apis/google/GooglePhotosUploadUtils.ts @@ -93,65 +93,78 @@ export namespace DownloadUtils { const size = "content-length"; const type = "content-type"; - export interface DownloadInformation { + export interface UploadInformation { mediaPaths: string[]; fileNames: { [key: string]: string }; - contentSize?: string; + contentSize?: number; contentType?: string; } const generate = (prefix: string, url: string) => `${prefix}upload_${Utils.GenerateGuid()}${path.extname(url).toLowerCase()}`; const sanitize = (filename: string) => filename.replace(/\s+/g, "_"); - export const Download = async (url: string, filename?: string, prefix = ""): Promise> => { + export const UploadImage = async (url: string, filename?: string, prefix = ""): Promise> => { const resolved = filename ? sanitize(filename) : generate(prefix, url); const extension = path.extname(url) || path.extname(resolved) || png; - return new Promise((resolve, reject) => { - request.head(url, async (error, res) => { - if (error) { - return reject(error); - } - const information: DownloadInformation = { - fileNames: { clean: resolved }, - contentSize: res.headers[size], - contentType: res.headers[type], - mediaPaths: [] - }; - const resizers = [ - { resizer: sharp().rotate(), suffix: "_o" }, - ...Object.values(Sizes).map(size => ({ - resizer: sharp().resize(size.width, undefined, { withoutEnlargement: true }).rotate(), - suffix: size.suffix - })) - ]; - let validated = true; - if (pngs.includes(extension)) { - resizers.forEach(element => element.resizer = element.resizer.png()); - } else if (jpg.includes(extension)) { - resizers.forEach(element => element.resizer = element.resizer.jpeg()); - } else { - validated = false; - } - if (validated) { - for (let resizer of resizers) { - const suffix = resizer.suffix; - let mediaPath: string; - await new Promise(resolve => { - const filename = resolved.substring(0, resolved.length - extension.length) + suffix + extension; - information.mediaPaths.push(mediaPath = uploadDirectory + filename); - information.fileNames[suffix] = filename; - request(url) - .pipe(resizer.resizer) - .pipe(fs.createWriteStream(mediaPath)) - .on('close', resolve); - }); + let information: UploadInformation = { + mediaPaths: [], + fileNames: { clean: resolved } + }; + const { isLocal, stream } = classify(url = path.normalize(url)); + if (!isLocal) { + const metadata = (await new Promise((resolve, reject) => { + request.head(url, async (error, res) => { + if (error) { + return reject(error); } - resolve(information); + resolve(res); + }); + })).headers; + information.contentSize = parseInt(metadata[size]); + information.contentType = metadata[type]; + } + return new Promise(async (resolve, reject) => { + const resizers = [ + { resizer: sharp().rotate(), suffix: "_o" }, + ...Object.values(Sizes).map(size => ({ + resizer: sharp().resize(size.width, undefined, { withoutEnlargement: true }).rotate(), + suffix: size.suffix + })) + ]; + let validated = true; + if (pngs.includes(extension)) { + resizers.forEach(element => element.resizer = element.resizer.png()); + } else if (jpg.includes(extension)) { + resizers.forEach(element => element.resizer = element.resizer.jpeg()); + } else { + validated = false; + } + if (validated) { + for (let resizer of resizers) { + const suffix = resizer.suffix; + let mediaPath: string; + await new Promise(resolve => { + const filename = resolved.substring(0, resolved.length - extension.length) + suffix + extension; + information.mediaPaths.push(mediaPath = uploadDirectory + filename); + information.fileNames[suffix] = filename; + stream(url).pipe(resizer.resizer).pipe(fs.createWriteStream(mediaPath)) + .on('close', resolve) + .on('error', reject); + }); } - }); + resolve(information); + } }); }; + const classify = (url: string) => { + const isLocal = /Dash-Web\\src\\server\\public\\files/g.test(url); + return { + isLocal, + stream: isLocal ? fs.createReadStream : request + }; + }; + export const createIfNotExists = async (path: string) => { if (await new Promise(resolve => fs.exists(path, resolve))) { return true; diff --git a/src/server/credentials/google_docs_token.json b/src/server/credentials/google_docs_token.json index 4f911f7e0..66668aaef 100644 --- a/src/server/credentials/google_docs_token.json +++ b/src/server/credentials/google_docs_token.json @@ -1 +1 @@ -{"access_token":"ya29.Glx9B3_l5JbKNtvzx378Nsz917bP-OTKf6VZzc2K8QDBm-Y0j_-c8v7bL8LCEM3wF8d7JauF-5Z4Uq4v7wPwUQUlDO1uPyoHeSF6iz98xkgJr9OW4KzJo2Ij722gpQ","refresh_token":"1/HTv_xFHszu2Nf3iiFrUTaeKzC_Vp2-6bpIB06xW_WHI","scope":"https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/documents.readonly https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/photoslibrary https://www.googleapis.com/auth/photoslibrary.appendonly https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/photoslibrary.sharing","token_type":"Bearer","expiry_date":1567931641928} \ No newline at end of file +{"access_token":"ya29.Glx9BwKT8sxbXNR78f2Ks3pAe2DfsxOhrYMj8SACNi13xwJ0MtLU4WYb4_cbHAj2X8imZW9eUBlAsY9RXoMEPOmVpMlhMjxVZKBo_0lwJ6xSTunSdrR1e8P7vkRV4Q","refresh_token":"1/HTv_xFHszu2Nf3iiFrUTaeKzC_Vp2-6bpIB06xW_WHI","scope":"https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/documents.readonly https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/photoslibrary https://www.googleapis.com/auth/photoslibrary.appendonly https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/photoslibrary.sharing","token_type":"Bearer","expiry_date":1567962258021} \ No newline at end of file diff --git a/src/server/index.ts b/src/server/index.ts index 013345a76..54525cd31 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -42,7 +42,7 @@ var AdmZip = require('adm-zip'); import * as YoutubeApi from "./apis/youtube/youtubeApiSample"; import { Response } from 'express-serve-static-core'; import { GoogleApiServerUtils } from "./apis/google/GoogleApiServerUtils"; -import { GooglePhotosUploadUtils, DownloadUtils } from './apis/google/GooglePhotosUploadUtils'; +import { GooglePhotosUploadUtils, DownloadUtils as UploadUtils } from './apis/google/GooglePhotosUploadUtils'; const MongoStore = require('connect-mongo')(session); const mongoose = require('mongoose'); const probe = require("probe-image-size"); @@ -156,7 +156,8 @@ app.get("/buxton", (req, res) => { const STATUS = { OK: 200, - BAD_REQUEST: 400 + BAD_REQUEST: 400, + EXECUTION_ERROR: 500 }; const command_line = (command: string, fromDirectory?: string) => { @@ -558,7 +559,6 @@ class NodeCanvasFactory { } const pngTypes = [".png", ".PNG"]; -const pdfTypes = [".pdf", ".PDF"]; const jpgTypes = [".jpg", ".JPG", ".jpeg", ".JPEG"]; const uploadDirectory = __dirname + "/public/files/"; // SETTERS @@ -568,37 +568,11 @@ app.post( let form = new formidable.IncomingForm(); form.uploadDir = uploadDirectory; form.keepExtensions = true; - // let path = req.body.path; - console.log("upload"); - form.parse(req, (err, fields, files) => { - console.log("parsing"); + form.parse(req, async (_err, _fields, files) => { let names: string[] = []; for (const name in files) { const file = path.basename(files[name].path); - const ext = path.extname(file); - let resizers = [ - { resizer: sharp().rotate(), suffix: "_o" }, - { resizer: sharp().resize(100, undefined, { withoutEnlargement: true }).rotate(), suffix: "_s" }, - { resizer: sharp().resize(400, undefined, { withoutEnlargement: true }).rotate(), suffix: "_m" }, - { resizer: sharp().resize(900, undefined, { withoutEnlargement: true }).rotate(), suffix: "_l" }, - ]; - let isImage = false; - if (pngTypes.includes(ext)) { - resizers.forEach(element => { - element.resizer = element.resizer.png(); - }); - isImage = true; - } else if (jpgTypes.includes(ext)) { - resizers.forEach(element => { - element.resizer = element.resizer.jpeg(); - }); - isImage = true; - } - if (isImage) { - resizers.forEach(resizer => { - fs.createReadStream(uploadDirectory + file).pipe(resizer.resizer).pipe(fs.createWriteStream(uploadDirectory + file.substring(0, file.length - ext.length) + resizer.suffix + ext)); - }); - } + await UploadUtils.UploadImage(uploadDirectory + file, file); names.push(`/files/` + file); } res.send(names); @@ -845,11 +819,11 @@ app.post(RouteStore.googlePhotosMediaUpload, async (req, res) => { }; })); if (!newMediaItems.every(item => item)) { - return res.send(tokenError); + return res.status(STATUS.EXECUTION_ERROR).send(tokenError); } GooglePhotosUploadUtils.CreateMediaItems(newMediaItems, req.body.album).then( - success => res.send(success), - () => res.send(mediaError) + success => res.status(STATUS.OK).send(success), + () => res.status(STATUS.EXECUTION_ERROR).send(mediaError) ); }); @@ -859,18 +833,36 @@ interface MediaItem { } const prefix = "google_photos_"; +const downloadError = "Encountered an error while executing downloads."; +const requestError = "Unable to execute download: the body's media items were malformed."; + app.post(RouteStore.googlePhotosMediaDownload, async (req, res) => { const contents: { mediaItems: MediaItem[] } = req.body; if (contents) { - const downloads = contents.mediaItems.map(item => - DownloadUtils.Download(item.baseUrl, item.filename, prefix) + const pending = contents.mediaItems.map(item => + UploadUtils.UploadImage(item.baseUrl, item.filename, prefix) ); - res.status(STATUS.OK).send(await Promise.all(downloads)); + const completed = await Promise.all(pending).catch(error => _error(res, downloadError, error)); + _success(res, completed); return; } - res.status(STATUS.BAD_REQUEST).send(); + _invalid(res, requestError); }); +const _error = (res: Response, message: string, error: any) => { + res.statusMessage = message; + res.status(STATUS.EXECUTION_ERROR).send(error); +}; + +const _success = (res: Response, body: any) => { + res.status(STATUS.OK).send(body); +}; + +const _invalid = (res: Response, message: string) => { + res.statusMessage = message; + res.status(STATUS.BAD_REQUEST).send(); +}; + const suffixMap: { [type: string]: (string | [string, string | ((json: any) => any)]) } = { "number": "_n", "string": "_t", -- cgit v1.2.3-70-g09d2 From 0c987a119fc6baa344cd6a8d229556c02af64898 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Mon, 9 Sep 2019 11:21:00 -0400 Subject: updates --- .../util/Import & Export/DirectoryImportBox.tsx | 48 ++++++++++++---------- src/server/apis/google/GooglePhotosUploadUtils.ts | 44 ++++++++++---------- src/server/credentials/google_docs_token.json | 2 +- src/server/index.ts | 22 ++++++---- 4 files changed, 64 insertions(+), 52 deletions(-) (limited to 'src/server/apis') diff --git a/src/client/util/Import & Export/DirectoryImportBox.tsx b/src/client/util/Import & Export/DirectoryImportBox.tsx index ab2801ee3..7693a388f 100644 --- a/src/client/util/Import & Export/DirectoryImportBox.tsx +++ b/src/client/util/Import & Export/DirectoryImportBox.tsx @@ -18,8 +18,14 @@ import { Id } from "../../../new_fields/FieldSymbols"; import { List } from "../../../new_fields/List"; import { Cast, BoolCast, NumCast } from "../../../new_fields/Types"; import { listSpec } from "../../../new_fields/Schema"; +import { GooglePhotosClientUtils } from "../../apis/google_docs/GooglePhotosClientUtils"; const unsupported = ["text/html", "text/plain"]; +interface FileResponse { + name: string; + path: string; + type: string; +} @observer export default class DirectoryImportBox extends React.Component { @@ -87,34 +93,32 @@ export default class DirectoryImportBox extends React.Component let sizes = []; let modifiedDates = []; + let formData = new FormData(); for (let uploaded_file of validated) { - let formData = new FormData(); - formData.append('file', uploaded_file); - let dropFileName = uploaded_file ? uploaded_file.name : "-empty-"; - let type = uploaded_file.type; - + formData.append(Utils.GenerateGuid(), uploaded_file); sizes.push(uploaded_file.size); modifiedDates.push(uploaded_file.lastModified); - runInAction(() => this.remaining++); - - let prom = fetch(Utils.prepend(RouteStore.upload), { - method: 'POST', - body: formData - }).then(async (res: Response) => { - let names = await res.json(); - console.log(names); - await Promise.all(names.map((file: any) => { - let docPromise = Docs.Get.DocumentFromType(type, Utils.prepend(file), { nativeWidth: 300, width: 300, title: dropFileName }); - docPromise.then(doc => { - doc && docs.push(doc) && runInAction(() => this.remaining--); - }); - })); - }); - promises.push(prom); } - await Promise.all(promises); + const parameters = { method: 'POST', body: formData }; + const uploads: FileResponse[] = await (await fetch(Utils.prepend(RouteStore.upload), parameters)).json(); + + await Promise.all(uploads.map(async upload => { + const type = upload.type; + const path = Utils.prepend(upload.path); + const options = { + nativeWidth: 300, + width: 300, + title: upload.name + }; + const document = await Docs.Get.DocumentFromType(type, path, options); + document && docs.push(document) && runInAction(() => this.remaining--); + console.log(`(${this.quota - this.remaining}/${this.quota}) ${upload.name}`); + })); + + await GooglePhotosClientUtils.UploadImageDocuments(docs, { title: directory }); + console.log("Finished upload!"); for (let i = 0; i < docs.length; i++) { let doc = docs[i]; diff --git a/src/server/apis/google/GooglePhotosUploadUtils.ts b/src/server/apis/google/GooglePhotosUploadUtils.ts index d4a2a2bb3..35f986250 100644 --- a/src/server/apis/google/GooglePhotosUploadUtils.ts +++ b/src/server/apis/google/GooglePhotosUploadUtils.ts @@ -89,7 +89,8 @@ export namespace DownloadUtils { const png = ".png"; const pngs = [".png", ".PNG"]; - const jpg = [".jpg", ".JPG", ".jpeg", ".JPEG"]; + const jpgs = [".jpg", ".JPG", ".jpeg", ".JPEG"]; + const formats = [".jpg", ".png", ".gif"]; const size = "content-length"; const type = "content-type"; @@ -110,7 +111,8 @@ export namespace DownloadUtils { mediaPaths: [], fileNames: { clean: resolved } }; - const { isLocal, stream } = classify(url = path.normalize(url)); + const { isLocal, stream, normalized } = classify(url); + url = normalized; if (!isLocal) { const metadata = (await new Promise((resolve, reject) => { request.head(url, async (error, res) => { @@ -131,37 +133,35 @@ export namespace DownloadUtils { suffix: size.suffix })) ]; - let validated = true; if (pngs.includes(extension)) { resizers.forEach(element => element.resizer = element.resizer.png()); - } else if (jpg.includes(extension)) { + } else if (jpgs.includes(extension)) { resizers.forEach(element => element.resizer = element.resizer.jpeg()); - } else { - validated = false; + } else if (!formats.includes(extension.toLowerCase())) { + return reject(); } - if (validated) { - for (let resizer of resizers) { - const suffix = resizer.suffix; - let mediaPath: string; - await new Promise(resolve => { - const filename = resolved.substring(0, resolved.length - extension.length) + suffix + extension; - information.mediaPaths.push(mediaPath = uploadDirectory + filename); - information.fileNames[suffix] = filename; - stream(url).pipe(resizer.resizer).pipe(fs.createWriteStream(mediaPath)) - .on('close', resolve) - .on('error', reject); - }); - } - resolve(information); + for (let resizer of resizers) { + const suffix = resizer.suffix; + let mediaPath: string; + await new Promise(resolve => { + const filename = resolved.substring(0, resolved.length - extension.length) + suffix + extension; + information.mediaPaths.push(mediaPath = uploadDirectory + filename); + information.fileNames[suffix] = filename; + stream(url).pipe(resizer.resizer).pipe(fs.createWriteStream(mediaPath)) + .on('close', resolve) + .on('error', reject); + }); } + resolve(information); }); }; const classify = (url: string) => { - const isLocal = /Dash-Web\\src\\server\\public\\files/g.test(url); + const isLocal = /Dash-Web(\\|\/)src(\\|\/)server(\\|\/)public(\\|\/)files/g.test(url); return { isLocal, - stream: isLocal ? fs.createReadStream : request + stream: isLocal ? fs.createReadStream : request, + normalized: isLocal ? path.normalize(url) : url }; }; diff --git a/src/server/credentials/google_docs_token.json b/src/server/credentials/google_docs_token.json index 66668aaef..fabc18cfd 100644 --- a/src/server/credentials/google_docs_token.json +++ b/src/server/credentials/google_docs_token.json @@ -1 +1 @@ -{"access_token":"ya29.Glx9BwKT8sxbXNR78f2Ks3pAe2DfsxOhrYMj8SACNi13xwJ0MtLU4WYb4_cbHAj2X8imZW9eUBlAsY9RXoMEPOmVpMlhMjxVZKBo_0lwJ6xSTunSdrR1e8P7vkRV4Q","refresh_token":"1/HTv_xFHszu2Nf3iiFrUTaeKzC_Vp2-6bpIB06xW_WHI","scope":"https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/documents.readonly https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/photoslibrary https://www.googleapis.com/auth/photoslibrary.appendonly https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/photoslibrary.sharing","token_type":"Bearer","expiry_date":1567962258021} \ No newline at end of file +{"access_token":"ya29.Glx-BwgWcpQUukTNyuUqvSAYrDyxDNUhCLtrFDJAViROvicm0DrcRvCn4OaQdn2m2IZQYcG-19cvQYoOC3UJCtWXLRvKZzQCbZZSykpxYu_lflUyEnIGZOIHMbbEjA","refresh_token":"1/HTv_xFHszu2Nf3iiFrUTaeKzC_Vp2-6bpIB06xW_WHI","scope":"https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/documents.readonly https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/photoslibrary https://www.googleapis.com/auth/photoslibrary.appendonly https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/photoslibrary.sharing","token_type":"Bearer","expiry_date":1568008211814} \ No newline at end of file diff --git a/src/server/index.ts b/src/server/index.ts index 54525cd31..baef94a59 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -561,6 +561,11 @@ class NodeCanvasFactory { const pngTypes = [".png", ".PNG"]; const jpgTypes = [".jpg", ".JPG", ".jpeg", ".JPEG"]; const uploadDirectory = __dirname + "/public/files/"; +interface FileResponse { + name: string; + path: string; + type: string; +} // SETTERS app.post( RouteStore.upload, @@ -569,13 +574,16 @@ app.post( form.uploadDir = uploadDirectory; form.keepExtensions = true; form.parse(req, async (_err, _fields, files) => { - let names: string[] = []; - for (const name in files) { - const file = path.basename(files[name].path); - await UploadUtils.UploadImage(uploadDirectory + file, file); - names.push(`/files/` + file); + let results: FileResponse[] = []; + for (const key in files) { + const { name, type, path: location } = files[key]; + const filename = path.basename(location); + await UploadUtils.UploadImage(uploadDirectory + filename, path.basename(name)); + results.push({ name, type, path: `/files/${filename}` }); + console.log(path.basename(name)); } - res.send(names); + console.log("All files traversed!"); + _success(res, results); }); } ); @@ -843,7 +851,7 @@ app.post(RouteStore.googlePhotosMediaDownload, async (req, res) => { UploadUtils.UploadImage(item.baseUrl, item.filename, prefix) ); const completed = await Promise.all(pending).catch(error => _error(res, downloadError, error)); - _success(res, completed); + Array.isArray(completed) && _success(res, completed); return; } _invalid(res, requestError); -- cgit v1.2.3-70-g09d2 From b24c475d8cd36af860fc374b0c5621b0d096be1d Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Mon, 9 Sep 2019 20:47:34 -0400 Subject: nearly finished transferring images between text notes and google docs --- package.json | 2 +- .../apis/google_docs/GooglePhotosClientUtils.ts | 23 ++-- .../util/Import & Export/DirectoryImportBox.tsx | 2 +- src/client/views/MainView.tsx | 6 +- src/client/views/collections/CollectionSubView.tsx | 2 +- src/client/views/nodes/DocumentView.tsx | 2 +- src/new_fields/RichTextUtils.ts | 67 ++++++++++- src/server/RouteStore.ts | 3 +- src/server/apis/google/GoogleApiServerUtils.ts | 2 +- src/server/apis/google/GooglePhotosUploadUtils.ts | 5 + src/server/credentials/google_docs_token.json | 2 +- src/server/index.ts | 20 +++- src/server/updateSearch.ts | 123 --------------------- 13 files changed, 112 insertions(+), 147 deletions(-) delete mode 100644 src/server/updateSearch.ts (limited to 'src/server/apis') diff --git a/package.json b/package.json index f56e34ce0..f0f2b467e 100644 --- a/package.json +++ b/package.json @@ -224,4 +224,4 @@ "xoauth2": "^1.2.0", "youtube": "^0.1.0" } -} \ No newline at end of file +} diff --git a/src/client/apis/google_docs/GooglePhotosClientUtils.ts b/src/client/apis/google_docs/GooglePhotosClientUtils.ts index 5f5b39b14..fddcf3aa5 100644 --- a/src/client/apis/google_docs/GooglePhotosClientUtils.ts +++ b/src/client/apis/google_docs/GooglePhotosClientUtils.ts @@ -51,17 +51,26 @@ export namespace GooglePhotosClientUtils { description: string; } - export const UploadImageDocuments = async (sources: Doc[], album?: AlbumReference, descriptionKey = "caption") => { + export const UploadImages = async (sources: (Doc | string)[], album?: AlbumReference, descriptionKey = "caption") => { if (album && "title" in album) { album = await (await endpoint()).albums.create(album.title); } const media: MediaInput[] = []; - sources.forEach(document => { - const data = Cast(Doc.GetProto(document).data, ImageField); - data && media.push({ - url: data.url.href, - description: parseDescription(document, descriptionKey), - }); + sources.forEach(source => { + let url: string; + let description: string; + if (source instanceof Doc) { + const data = Cast(Doc.GetProto(source).data, ImageField); + if (!data) { + return; + } + url = data.url.href; + description = parseDescription(source, descriptionKey); + } else { + url = source; + description = Utils.GenerateGuid(); + } + media.push({ url, description }); }); if (media.length) { return PostToServer(RouteStore.googlePhotosMediaUpload, { media, album }); diff --git a/src/client/util/Import & Export/DirectoryImportBox.tsx b/src/client/util/Import & Export/DirectoryImportBox.tsx index 7693a388f..a19fd39b7 100644 --- a/src/client/util/Import & Export/DirectoryImportBox.tsx +++ b/src/client/util/Import & Export/DirectoryImportBox.tsx @@ -117,7 +117,7 @@ export default class DirectoryImportBox extends React.Component console.log(`(${this.quota - this.remaining}/${this.quota}) ${upload.name}`); })); - await GooglePhotosClientUtils.UploadImageDocuments(docs, { title: directory }); + await GooglePhotosClientUtils.UploadImages(docs, { title: directory }); console.log("Finished upload!"); for (let i = 0; i < docs.length; i++) { diff --git a/src/client/views/MainView.tsx b/src/client/views/MainView.tsx index 326c13424..0c0ed9072 100644 --- a/src/client/views/MainView.tsx +++ b/src/client/views/MainView.tsx @@ -155,7 +155,7 @@ export class MainView extends React.Component { doc.caption = "Well isn't this a nice cat image!"; let photos = await GooglePhotosClientUtils.endpoint(); let albumId = (await photos.albums.list(50)).albums.filter((album: any) => album.title === "This is a generically created album!")[0].id; - console.log(await GooglePhotosClientUtils.UploadImageDocuments([doc], { id: albumId })); + console.log(await GooglePhotosClientUtils.UploadImages([doc], { id: albumId })); } componentWillUnMount() { @@ -470,7 +470,7 @@ export class MainView extends React.Component { // let youtubeurl = "https://www.youtube.com/embed/TqcApsGRzWw"; // let addYoutubeSearcher = action(() => Docs.Create.YoutubeDocument(youtubeurl, { width: 600, height: 600, title: "youtube search" })); - let googlePhotosSearch = () => GooglePhotosClientUtils.CollectionFromSearch(Docs.Create.MasonryDocument, { included: [GooglePhotosClientUtils.ContentCategories.LANDSCAPES] }); + // let googlePhotosSearch = () => GooglePhotosClientUtils.CollectionFromSearch(Docs.Create.MasonryDocument, { included: [GooglePhotosClientUtils.ContentCategories.LANDSCAPES] }); let btns: [React.RefObject, IconName, string, () => Doc | Promise][] = [ [React.createRef(), "object-group", "Add Collection", addColNode], @@ -478,7 +478,7 @@ export class MainView extends React.Component { [React.createRef(), "globe-asia", "Add Website", addWebNode], [React.createRef(), "bolt", "Add Button", addButtonDocument], [React.createRef(), "file", "Add Document Dragger", addDragboxNode], - [React.createRef(), "object-group", "Test Google Photos Search", googlePhotosSearch], + // [React.createRef(), "object-group", "Test Google Photos Search", googlePhotosSearch], [React.createRef(), "cloud-upload-alt", "Import Directory", addImportCollectionNode], //remove at some point in favor of addImportCollectionNode //[React.createRef(), "play", "Add Youtube Searcher", addYoutubeSearcher], ]; diff --git a/src/client/views/collections/CollectionSubView.tsx b/src/client/views/collections/CollectionSubView.tsx index 99e5ab7b3..5fc4f36a7 100644 --- a/src/client/views/collections/CollectionSubView.tsx +++ b/src/client/views/collections/CollectionSubView.tsx @@ -253,7 +253,7 @@ export function CollectionSubView(schemaCtor: (doc: Doc) => T) { }).then(async (res: Response) => { (await res.json()).map(action((file: any) => { let full = { ...options, nativeWidth: type.indexOf("video") !== -1 ? 600 : 300, width: 300, title: dropFileName }; - let path = Utils.prepend(file); + let path = Utils.prepend(file.path); Docs.Get.DocumentFromType(type, path, full).then(doc => doc && this.props.addDocument(doc)); })); }); diff --git a/src/client/views/nodes/DocumentView.tsx b/src/client/views/nodes/DocumentView.tsx index 4033ffd9c..cb9346a8b 100644 --- a/src/client/views/nodes/DocumentView.tsx +++ b/src/client/views/nodes/DocumentView.tsx @@ -591,7 +591,7 @@ export class DocumentView extends DocComponent(Docu subitems.push({ description: "Open Fields", event: this.fieldsClicked, icon: "layer-group" }); cm.addItem({ description: "Open...", subitems: subitems, icon: "external-link-alt" }); if (Cast(this.props.Document.data, ImageField)) { - cm.addItem({ description: "Export to Google Photos", event: () => GooglePhotosClientUtils.UploadImageDocuments([this.props.Document]), icon: "caret-square-right" }); + cm.addItem({ description: "Export to Google Photos", event: () => GooglePhotosClientUtils.UploadImages([this.props.Document]), icon: "caret-square-right" }); } let existingMake = ContextMenu.Instance.findByDescription("Make..."); let makes: ContextMenuProps[] = existingMake && "subitems" in existingMake ? existingMake.subitems : []; diff --git a/src/new_fields/RichTextUtils.ts b/src/new_fields/RichTextUtils.ts index 0aba50c0d..500b93676 100644 --- a/src/new_fields/RichTextUtils.ts +++ b/src/new_fields/RichTextUtils.ts @@ -7,6 +7,11 @@ import { FormattedTextBox } from "../client/views/nodes/FormattedTextBox"; import { Opt } from "./Doc"; import * as Color from "color"; import { sinkListItem } from "prosemirror-schema-list"; +import { Utils, PostToServer } from "../Utils"; +import { RouteStore } from "../server/RouteStore"; +import { Docs } from "../client/documents/Documents"; +import { schema } from "../client/util/RichTextSchema"; +import { GooglePhotosClientUtils } from "../client/apis/google_docs/GooglePhotosClientUtils"; export namespace RichTextUtils { @@ -86,19 +91,36 @@ export namespace RichTextUtils { export namespace GoogleDocs { export const Export = (state: EditorState): GoogleApiClientUtils.Docs.Content => { - let textNodes: Node[] = []; + let nodes: { [type: string]: Node[] } = { + text: [], + image: [] + }; let text = ToPlainText(state); let content = state.doc.content; - content.forEach(node => node.content.forEach(node => node.type.name === "text" && textNodes.push(node))); - let linkRequests = ExtractLinks(textNodes); + content.forEach(node => node.content.forEach(node => { + const type = node.type.name; + let existing = nodes[type]; + if (existing) { + existing.push(node); + } else { + nodes[type] = [node]; + } + })); + let linkRequests = ExtractLinks(nodes.text); + let imageRequests = ExtractImages(nodes.image); return { text, - requests: [...linkRequests] + requests: [...linkRequests, ...imageRequests] }; }; type BulletPosition = { value: number, sinks: number }; + interface MediaItem { + baseUrl: string; + filename: string; + width: number; + } export const Import = async (documentId: GoogleApiClientUtils.Docs.DocumentId): Promise> => { const document = await GoogleApiClientUtils.Docs.retrieve({ documentId }); if (!document) { @@ -109,6 +131,17 @@ export namespace RichTextUtils { const { text, paragraphs } = GoogleApiClientUtils.Docs.Utils.extractText(document); let state = FormattedTextBox.blankState(); let structured = parseLists(paragraphs); + const inline = document.inlineObjects; + let inlineUrls: MediaItem[] = []; + if (inline) { + inlineUrls = Object.keys(inline).map(key => { + const embedded = inline[key].inlineObjectProperties!.embeddedObject!; + const baseUrl = embedded.imageProperties!.contentUri!; + const filename = `upload_${Utils.GenerateGuid()}.png`; + const width = embedded.size!.width!.magnitude!; + return { baseUrl, filename, width }; + }); + } let position = 3; let lists: ListGroup[] = []; @@ -151,6 +184,12 @@ export namespace RichTextUtils { } } + const uploads = await PostToServer(RouteStore.googlePhotosMediaDownload, { mediaItems: inlineUrls }); + for (let i = 0; i < uploads.length; i++) { + const src = Utils.prepend(`/files/${uploads[i].fileNames.clean}`); + state = state.apply(state.tr.insert(0, schema.nodes.image.create({ src, width: inlineUrls[i].width }))); + } + return { title, text, state }; }; @@ -252,6 +291,26 @@ export namespace RichTextUtils { return links; }; + const ExtractImages = async (nodes: Node[]) => { + const images: docs_v1.Schema$Request[] = []; + let position = 1; + for (let node of nodes) { + const length = node.nodeSize; + const attrs = node.attrs; + const uri = attrs.src; + const result = (await GooglePhotosClientUtils.UploadImages([uri])).newMediaItemResults; + images.push({ + insertInlineImage: { + uri: result[0].mediaItem.productUrl, + objectSize: { width: { magnitude: parseFloat(attrs.width.replace("px", "")), unit: "PT" } }, + location: { index: position + length } + } + }); + position += length; + } + return images; + }; + const Encode = (information: LinkInformation) => { return { updateTextStyle: { diff --git a/src/server/RouteStore.ts b/src/server/RouteStore.ts index f65e6134c..ee9cd8a0e 100644 --- a/src/server/RouteStore.ts +++ b/src/server/RouteStore.ts @@ -34,6 +34,7 @@ export enum RouteStore { googleDocs = "/googleDocs", googlePhotosAccessToken = "/googlePhotosAccessToken", googlePhotosMediaUpload = "/googlePhotosMediaUpload", - googlePhotosMediaDownload = "/googlePhotosMediaDownload" + googlePhotosMediaDownload = "/googlePhotosMediaDownload", + googleDocsGet = "/googleDocsGet" } \ No newline at end of file diff --git a/src/server/apis/google/GoogleApiServerUtils.ts b/src/server/apis/google/GoogleApiServerUtils.ts index ac8023ce1..e0bd8a800 100644 --- a/src/server/apis/google/GoogleApiServerUtils.ts +++ b/src/server/apis/google/GoogleApiServerUtils.ts @@ -42,7 +42,7 @@ export namespace GoogleApiServerUtils { export type ApiResponse = Promise; export type ApiRouter = (endpoint: Endpoint, parameters: any) => ApiResponse; - export type ApiHandler = (parameters: any) => ApiResponse; + export type ApiHandler = (parameters: any, methodOptions?: any) => ApiResponse; export type Action = "create" | "retrieve" | "update"; export type Endpoint = { get: ApiHandler, create: ApiHandler, batchUpdate: ApiHandler }; diff --git a/src/server/apis/google/GooglePhotosUploadUtils.ts b/src/server/apis/google/GooglePhotosUploadUtils.ts index 35f986250..d1f1f81bd 100644 --- a/src/server/apis/google/GooglePhotosUploadUtils.ts +++ b/src/server/apis/google/GooglePhotosUploadUtils.ts @@ -151,6 +151,11 @@ export namespace DownloadUtils { .on('close', resolve) .on('error', reject); }); + if (!isLocal) { + await new Promise(resolve => { + stream(url).pipe(fs.createWriteStream(uploadDirectory + resolved)).on('close', resolve); + }); + } } resolve(information); }); diff --git a/src/server/credentials/google_docs_token.json b/src/server/credentials/google_docs_token.json index fabc18cfd..5c142fba1 100644 --- a/src/server/credentials/google_docs_token.json +++ b/src/server/credentials/google_docs_token.json @@ -1 +1 @@ -{"access_token":"ya29.Glx-BwgWcpQUukTNyuUqvSAYrDyxDNUhCLtrFDJAViROvicm0DrcRvCn4OaQdn2m2IZQYcG-19cvQYoOC3UJCtWXLRvKZzQCbZZSykpxYu_lflUyEnIGZOIHMbbEjA","refresh_token":"1/HTv_xFHszu2Nf3iiFrUTaeKzC_Vp2-6bpIB06xW_WHI","scope":"https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/documents.readonly https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/photoslibrary https://www.googleapis.com/auth/photoslibrary.appendonly https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/photoslibrary.sharing","token_type":"Bearer","expiry_date":1568008211814} \ No newline at end of file +{"access_token":"ya29.Glx_B6G7Q_FYs1LK5VcyV6P6Zg9JkoHO2aC_TsnN7AVxPYWHEpsBSC0WyWX7Ztr8HWhOUYA5JXqnZDkLrK1V3Hb-0GgtyApLRNtEPOWf1dJ7lOm_iKVw2tRvPe7XDQ","refresh_token":"1/HTv_xFHszu2Nf3iiFrUTaeKzC_Vp2-6bpIB06xW_WHI","scope":"https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/documents.readonly https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/photoslibrary https://www.googleapis.com/auth/photoslibrary.appendonly https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/photoslibrary.sharing","token_type":"Bearer","expiry_date":1568078116605} \ No newline at end of file diff --git a/src/server/index.ts b/src/server/index.ts index baef94a59..8469770d5 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -38,6 +38,7 @@ import flash = require('connect-flash'); import { Search } from './Search'; import _ = require('lodash'); import * as Archiver from 'archiver'; +import * as request_promise from 'request-promise'; var AdmZip = require('adm-zip'); import * as YoutubeApi from "./apis/youtube/youtubeApiSample"; import { Response } from 'express-serve-static-core'; @@ -576,9 +577,9 @@ app.post( form.parse(req, async (_err, _fields, files) => { let results: FileResponse[] = []; for (const key in files) { - const { name, type, path: location } = files[key]; + const { type, path: location, name } = files[key]; const filename = path.basename(location); - await UploadUtils.UploadImage(uploadDirectory + filename, path.basename(name)); + await UploadUtils.UploadImage(uploadDirectory + filename, filename); results.push({ name, type, path: `/files/${filename}` }); console.log(path.basename(name)); } @@ -790,10 +791,23 @@ const tokenPath = path.join(__dirname, "./credentials/google_docs_token.json"); const EndpointHandlerMap = new Map([ ["create", (api, params) => api.create(params)], - ["retrieve", (api, params) => api.get(params)], + ["retrieve", (api, params) => api.get(params, { params: "fields=inlineObjects" })], ["update", (api, params) => api.batchUpdate(params)], ]); +// app.post(RouteStore.googleDocsGet, async (req, res) => { +// const token = await GoogleApiServerUtils.RetrieveAccessToken({ credentialsPath, tokenPath }); +// request_promise.get({ +// uri: `https://docs.googleapis.com/v1/documents/${req.body.documentId}?fields=inlineObjects`, +// headers: { +// 'Authorization': `Bearer ${token}` +// } +// }).then(response => { +// console.log(response); +// res.send(response); +// }); +// }); + app.post(RouteStore.googleDocs + "/:sector/:action", (req, res) => { let sector: GoogleApiServerUtils.Service = req.params.sector; let action: GoogleApiServerUtils.Action = req.params.action; diff --git a/src/server/updateSearch.ts b/src/server/updateSearch.ts deleted file mode 100644 index 906b795f1..000000000 --- a/src/server/updateSearch.ts +++ /dev/null @@ -1,123 +0,0 @@ -import { Database } from "./database"; -import { Cursor } from "mongodb"; -import { Search } from "./Search"; -import pLimit from 'p-limit'; - -const suffixMap: { [type: string]: (string | [string, string | ((json: any) => any)]) } = { - "number": "_n", - "string": "_t", - "boolean": "_b", - // "image": ["_t", "url"], - "video": ["_t", "url"], - "pdf": ["_t", "url"], - "audio": ["_t", "url"], - "web": ["_t", "url"], - "date": ["_d", value => new Date(value.date).toISOString()], - "proxy": ["_i", "fieldId"], - "list": ["_l", list => { - const results = []; - for (const value of list.fields) { - const term = ToSearchTerm(value); - if (term) { - results.push(term.value); - } - } - return results.length ? results : null; - }] -}; - -function ToSearchTerm(val: any): { suffix: string, value: any } | undefined { - if (val === null || val === undefined) { - return; - } - const type = val.__type || typeof val; - let suffix = suffixMap[type]; - if (!suffix) { - return; - } - - if (Array.isArray(suffix)) { - const accessor = suffix[1]; - if (typeof accessor === "function") { - val = accessor(val); - } else { - val = val[accessor]; - } - suffix = suffix[0]; - } - - return { suffix, value: val }; -} - -function getSuffix(value: string | [string, any]): string { - return typeof value === "string" ? value : value[0]; -} - -const limit = pLimit(5); -async function update() { - // await new Promise(res => setTimeout(res, 5)); - console.log("update"); - await Search.Instance.clear(); - const cursor = await Database.Instance.query({}); - console.log("Cleared"); - const updates: any[] = []; - let numDocs = 0; - function updateDoc(doc: any) { - numDocs++; - if ((numDocs % 50) === 0) { - console.log("updateDoc " + numDocs); - } - // console.log("doc " + numDocs); - if (doc.__type !== "Doc") { - return; - } - const fields = doc.fields; - if (!fields) { - return; - } - const update: any = { id: doc._id }; - let dynfield = false; - for (const key in fields) { - const value = fields[key]; - const term = ToSearchTerm(value); - if (term !== undefined) { - let { suffix, value } = term; - update[key + suffix] = value; - dynfield = true; - } - } - if (dynfield) { - updates.push(update); - // console.log(updates.length); - } - } - await cursor.forEach(updateDoc); - console.log(`Updating ${updates.length} documents`); - const result = await Search.Instance.updateDocuments(updates); - try { - console.log(JSON.parse(result).responseHeader.status); - } catch { - console.log("Error:"); - // console.log(updates[i]); - console.log(result); - console.log("\n"); - } - // for (let i = 0; i < updates.length; i++) { - // console.log(i); - // const result = await Search.Instance.updateDocument(updates[i]); - // try { - // console.log(JSON.parse(result).responseHeader.status); - // } catch { - // console.log("Error:"); - // console.log(updates[i]); - // console.log(result); - // console.log("\n"); - // } - // } - // await Promise.all(updates.map(update => { - // return limit(() => Search.Instance.updateDocument(update)); - // })); - cursor.close(); -} - -update(); \ No newline at end of file -- cgit v1.2.3-70-g09d2 From 628eef55533118b1f2312b86b2ac5f7b64f7fc4a Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Tue, 10 Sep 2019 20:01:55 -0400 Subject: lots of refactoring, beginning autotagging --- src/Utils.ts | 5 + .../apis/google_docs/GooglePhotosClientUtils.ts | 339 ++++++++++++++------- .../util/Import & Export/DirectoryImportBox.tsx | 5 +- src/client/views/MainView.tsx | 18 +- src/client/views/nodes/DocumentView.tsx | 7 +- src/client/views/nodes/FormattedTextBox.tsx | 6 +- src/new_fields/RichTextUtils.ts | 13 +- src/server/apis/google/GooglePhotosUploadUtils.ts | 7 +- src/server/apis/google/SharedTypes.ts | 21 ++ src/server/credentials/google_docs_token.json | 2 +- src/server/index.ts | 21 +- 11 files changed, 284 insertions(+), 160 deletions(-) create mode 100644 src/server/apis/google/SharedTypes.ts (limited to 'src/server/apis') diff --git a/src/Utils.ts b/src/Utils.ts index 959b89fe5..71d88683a 100644 --- a/src/Utils.ts +++ b/src/Utils.ts @@ -37,6 +37,11 @@ export class Utils { public static prepend(extension: string): string { return window.location.origin + extension; } + + public static fileUrl(filename: string): string { + return this.prepend(`/file/${filename}`); + } + public static CorsProxy(url: string): string { return this.prepend(RouteStore.corsProxy + "/") + encodeURIComponent(url); } diff --git a/src/client/apis/google_docs/GooglePhotosClientUtils.ts b/src/client/apis/google_docs/GooglePhotosClientUtils.ts index fddcf3aa5..b1de24d1a 100644 --- a/src/client/apis/google_docs/GooglePhotosClientUtils.ts +++ b/src/client/apis/google_docs/GooglePhotosClientUtils.ts @@ -1,8 +1,8 @@ import { PostToServer, Utils } from "../../../Utils"; import { RouteStore } from "../../../server/RouteStore"; import { ImageField } from "../../../new_fields/URLField"; -import { Cast } from "../../../new_fields/Types"; -import { Doc, Opt } from "../../../new_fields/Doc"; +import { Cast, StrCast } from "../../../new_fields/Types"; +import { Doc, Opt, DocListCastAsync } from "../../../new_fields/Doc"; import { Id } from "../../../new_fields/FieldSymbols"; import Photos = require('googlephotos'); import { RichTextField } from "../../../new_fields/RichTextField"; @@ -10,32 +10,15 @@ import { RichTextUtils } from "../../../new_fields/RichTextUtils"; import { EditorState } from "prosemirror-state"; import { FormattedTextBox } from "../../views/nodes/FormattedTextBox"; import { Docs, DocumentOptions } from "../../documents/Documents"; -import { type } from "os"; - -export namespace GooglePhotosClientUtils { - - export enum ContentCategories { - NONE = 'NONE', - LANDSCAPES = 'LANDSCAPES', - RECEIPTS = 'RECEIPTS', - CITYSCAPES = 'CITYSCAPES', - LANDMARKS = 'LANDMARKS', - SELFIES = 'SELFIES', - PEOPLE = 'PEOPLE', - PETS = 'PETS', - WEDDINGS = 'WEDDINGS', - BIRTHDAYS = 'BIRTHDAYS', - DOCUMENTS = 'DOCUMENTS', - TRAVEL = 'TRAVEL', - ANIMALS = 'ANIMALS', - FOOD = 'FOOD', - SPORT = 'SPORT', - NIGHT = 'NIGHT', - PERFORMANCES = 'PERFORMANCES', - WHITEBOARDS = 'WHITEBOARDS', - SCREENSHOTS = 'SCREENSHOTS', - UTILITY = 'UTILITY' - } +import { MediaItemCreationResult, NewMediaItemResult, MediaItem } from "../../../server/apis/google/SharedTypes"; + +export namespace GooglePhotos { + + const endpoint = async () => { + const getToken = Utils.prepend(RouteStore.googlePhotosAccessToken); + const token = await (await fetch(getToken)).text(); + return new Photos(token); + }; export enum MediaType { ALL_MEDIA = 'ALL_MEDIA', @@ -44,118 +27,238 @@ export namespace GooglePhotosClientUtils { } export type AlbumReference = { id: string } | { title: string }; - export const endpoint = () => fetch(Utils.prepend(RouteStore.googlePhotosAccessToken)).then(async response => new Photos(await response.text())); export interface MediaInput { url: string; description: string; } - export const UploadImages = async (sources: (Doc | string)[], album?: AlbumReference, descriptionKey = "caption") => { - if (album && "title" in album) { - album = await (await endpoint()).albums.create(album.title); + export const ContentCategories = { + NONE: 'NONE', + LANDSCAPES: 'LANDSCAPES', + RECEIPTS: 'RECEIPTS', + CITYSCAPES: 'CITYSCAPES', + LANDMARKS: 'LANDMARKS', + SELFIES: 'SELFIES', + PEOPLE: 'PEOPLE', + PETS: 'PETS', + WEDDINGS: 'WEDDINGS', + BIRTHDAYS: 'BIRTHDAYS', + DOCUMENTS: 'DOCUMENTS', + TRAVEL: 'TRAVEL', + ANIMALS: 'ANIMALS', + FOOD: 'FOOD', + SPORT: 'SPORT', + NIGHT: 'NIGHT', + PERFORMANCES: 'PERFORMANCES', + WHITEBOARDS: 'WHITEBOARDS', + SCREENSHOTS: 'SCREENSHOTS', + UTILITY: 'UTILITY' + }; + + export namespace Export { + + export interface AlbumCreationResult { + albumId: string; + mediaItems: MediaItem[]; } - const media: MediaInput[] = []; - sources.forEach(source => { - let url: string; - let description: string; - if (source instanceof Doc) { - const data = Cast(Doc.GetProto(source).data, ImageField); - if (!data) { - return; + + export const CollectionToAlbum = async (collection: Doc, title?: string, descriptionKey?: string): Promise> => { + const dataDocument = Doc.GetProto(collection); + const images = ((await DocListCastAsync(dataDocument.data)) || []).filter(doc => Cast(doc.data, ImageField)); + if (!images || !images.length) { + return undefined; + } + const resolved = title ? title : (StrCast(collection.title) || `Dash Collection (${collection[Id]}`); + const { id } = await Create.Album(resolved); + const result = await Transactions.UploadImages(images, { id }, descriptionKey); + if (result) { + const mediaItems = result.newMediaItemResults.map(item => item.mediaItem); + return { albumId: id, mediaItems }; + } + }; + + } + + export namespace Import { + + export type CollectionConstructor = (data: Array, options: DocumentOptions, ...args: any) => Doc; + + export const CollectionFromSearch = async (constructor: CollectionConstructor, requested: Opt>): Promise => { + let response = await Query.Search(requested); + let uploads = await Transactions.WriteMediaItemsToServer(response); + const children = uploads.map((upload: Transactions.UploadInformation) => { + let document = Docs.Create.ImageDocument(Utils.fileUrl(upload.fileNames.clean)); + document.fillColumn = true; + document.contentSize = upload.contentSize; + return document; + }); + const options = { width: 500, height: 500 }; + return constructor(children, options); + }; + + } + + export namespace Query { + + export const AppendImageMetadata = (sources: (Doc | string)[]) => { + let keys = Object.keys(ContentCategories); + let included: string[] = []; + let excluded: string[] = []; + for (let i = 0; i < keys.length; i++) { + for (let j = 0; j < keys.length; j++) { + let value = ContentCategories[keys[i] as keyof typeof ContentCategories]; + if (j === i) { + included.push(value); + } else { + excluded.push(value); + } } - url = data.url.href; - description = parseDescription(source, descriptionKey); - } else { - url = source; - description = Utils.GenerateGuid(); + //... + included = excluded = []; } - media.push({ url, description }); - }); - if (media.length) { - return PostToServer(RouteStore.googlePhotosMediaUpload, { media, album }); + }; + + interface DateRange { + after: Date; + before: Date; } - }; - const parseDescription = (document: Doc, descriptionKey: string) => { - let description: string = Utils.prepend("/doc/" + document[Id]); - const target = document[descriptionKey]; - if (typeof target === "string") { - description = target; - } else if (target instanceof RichTextField) { - description = RichTextUtils.ToPlainText(EditorState.fromJSON(FormattedTextBox.Instance.config, JSON.parse(target.Data))); + const DefaultSearchOptions: SearchOptions = { + pageSize: 20, + included: [], + excluded: [], + date: undefined, + includeArchivedMedia: true, + type: MediaType.ALL_MEDIA, + }; + + export interface SearchOptions { + pageSize: number; + included: ContentCategories[]; + excluded: ContentCategories[]; + date: Opt; + includeArchivedMedia: boolean; + type: MediaType; } - return description; - }; - export interface DateRange { - after: Date; - before: Date; - } - export interface SearchOptions { - pageSize: number; - included: ContentCategories[]; - excluded: ContentCategories[]; - date: Opt; - includeArchivedMedia: boolean; - type: MediaType; + export interface SearchResponse { + mediaItems: any[]; + nextPageToken: string; + } + + export const Search = async (requested: Opt>): Promise => { + const options = requested || DefaultSearchOptions; + const photos = await endpoint(); + const filters = new photos.Filters(options.includeArchivedMedia === undefined ? true : options.includeArchivedMedia); + + const included = options.included || []; + const excluded = options.excluded || []; + const contentFilter = new photos.ContentFilter(); + included.length && included.forEach(category => contentFilter.addIncludedContentCategories(category)); + excluded.length && excluded.forEach(category => contentFilter.addExcludedContentCategories(category)); + filters.setContentFilter(contentFilter); + + const date = options.date; + if (date) { + const dateFilter = new photos.DateFilter(); + if (date instanceof Date) { + dateFilter.addDate(date); + } else { + dateFilter.addRange(date.after, date.before); + } + filters.setDateFilter(dateFilter); + } + + filters.setMediaTypeFilter(new photos.MediaTypeFilter(options.type || MediaType.ALL_MEDIA)); + + return new Promise(resolve => { + photos.mediaItems.search(filters, options.pageSize || 20).then(resolve); + }); + }; + + export const GetImage = async (mediaItemId: string): Promise => { + return (await endpoint()).mediaItems.get(mediaItemId); + }; + } - const DefaultSearchOptions: SearchOptions = { - pageSize: 20, - included: [], - excluded: [], - date: undefined, - includeArchivedMedia: true, - type: MediaType.ALL_MEDIA, - }; + export namespace Create { + + export const Album = async (title: string) => { + return (await endpoint()).albums.create(title); + }; - export interface SearchResponse { - mediaItems: any[]; - nextPageToken: string; } - export type CollectionConstructor = (data: Array, options: DocumentOptions, ...args: any) => Doc; - export const CollectionFromSearch = async (provider: CollectionConstructor, requested: Opt>): Promise => { - let downloads = await Search(requested); - return provider(downloads.map((download: any) => { - let document = Docs.Create.ImageDocument(Utils.prepend(`/files/${download.fileNames.clean}`)); - document.fillColumn = true; - document.contentSize = download.contentSize; - return document; - }), { width: 500, height: 500 }); - }; + export namespace Transactions { - export const Search = async (requested: Opt>): Promise => { - const options = requested || DefaultSearchOptions; - const photos = await endpoint(); - const filters = new photos.Filters(options.includeArchivedMedia === undefined ? true : options.includeArchivedMedia); - - const included = options.included || []; - const excluded = options.excluded || []; - const contentFilter = new photos.ContentFilter(); - included.length && included.forEach(category => contentFilter.addIncludedContentCategories(category)); - excluded.length && excluded.forEach(category => contentFilter.addExcludedContentCategories(category)); - filters.setContentFilter(contentFilter); - - const date = options.date; - if (date) { - const dateFilter = new photos.DateFilter(); - if (date instanceof Date) { - dateFilter.addDate(date); - } else { - dateFilter.addRange(date.after, date.before); - } - filters.setDateFilter(dateFilter); + export interface UploadInformation { + mediaPaths: string[]; + fileNames: { [key: string]: string }; + contentSize?: number; + contentType?: string; + } + + export interface MediaItem { + id: string; + filename: string; + baseUrl: string; } - filters.setMediaTypeFilter(new photos.MediaTypeFilter(options.type || MediaType.ALL_MEDIA)); + export const WriteMediaItemsToServer = async (body: { mediaItems: any[] }): Promise => { + const uploads = await PostToServer(RouteStore.googlePhotosMediaDownload, body); + return uploads; + }; - return new Promise(resolve => { - photos.mediaItems.search(filters, options.pageSize || 20).then(async (response: SearchResponse) => { - response && resolve(await PostToServer(RouteStore.googlePhotosMediaDownload, response)); + export const UploadThenFetch = async (sources: (Doc | string)[], album?: AlbumReference, descriptionKey = "caption") => { + const result = await UploadImages(sources, album, descriptionKey); + if (!result) { + return undefined; + } + const baseUrls: string[] = await Promise.all(result.newMediaItemResults.map((result: any) => { + return new Promise(resolve => Query.GetImage(result.mediaItem.id).then(item => resolve(item.baseUrl))); + })); + return baseUrls; + }; + + export const UploadImages = async (sources: (Doc | string)[], album?: AlbumReference, descriptionKey = "caption"): Promise> => { + if (album && "title" in album) { + album = await Create.Album(album.title); + } + const media: MediaInput[] = []; + sources.forEach(source => { + let url: string; + let description: string; + if (source instanceof Doc) { + const data = Cast(Doc.GetProto(source).data, ImageField); + if (!data) { + return; + } + url = data.url.href; + description = parseDescription(source, descriptionKey); + } else { + url = source; + description = Utils.GenerateGuid(); + } + media.push({ url, description }); }); - }); - }; + if (media.length) { + return PostToServer(RouteStore.googlePhotosMediaUpload, { media, album }); + } + }; + + const parseDescription = (document: Doc, descriptionKey: string) => { + let description: string = Utils.prepend("/doc/" + document[Id]); + const target = document[descriptionKey]; + if (typeof target === "string") { + description = target; + } else if (target instanceof RichTextField) { + description = RichTextUtils.ToPlainText(EditorState.fromJSON(FormattedTextBox.Instance.config, JSON.parse(target.Data))); + } + return description; + }; + + } } \ No newline at end of file diff --git a/src/client/util/Import & Export/DirectoryImportBox.tsx b/src/client/util/Import & Export/DirectoryImportBox.tsx index a19fd39b7..d58c02ce5 100644 --- a/src/client/util/Import & Export/DirectoryImportBox.tsx +++ b/src/client/util/Import & Export/DirectoryImportBox.tsx @@ -18,7 +18,7 @@ import { Id } from "../../../new_fields/FieldSymbols"; import { List } from "../../../new_fields/List"; import { Cast, BoolCast, NumCast } from "../../../new_fields/Types"; import { listSpec } from "../../../new_fields/Schema"; -import { GooglePhotosClientUtils } from "../../apis/google_docs/GooglePhotosClientUtils"; +import { GooglePhotos } from "../../apis/google_docs/GooglePhotosClientUtils"; const unsupported = ["text/html", "text/plain"]; interface FileResponse { @@ -117,8 +117,7 @@ export default class DirectoryImportBox extends React.Component console.log(`(${this.quota - this.remaining}/${this.quota}) ${upload.name}`); })); - await GooglePhotosClientUtils.UploadImages(docs, { title: directory }); - console.log("Finished upload!"); + await GooglePhotos.Transactions.UploadImages(docs, { title: directory }); for (let i = 0; i < docs.length; i++) { let doc = docs[i]; diff --git a/src/client/views/MainView.tsx b/src/client/views/MainView.tsx index 0c0ed9072..8d10a91ce 100644 --- a/src/client/views/MainView.tsx +++ b/src/client/views/MainView.tsx @@ -40,7 +40,7 @@ import { PreviewCursor } from './PreviewCursor'; import { FilterBox } from './search/FilterBox'; import PresModeMenu from './presentationview/PresentationModeMenu'; import { PresBox } from './nodes/PresBox'; -import { GooglePhotosClientUtils } from '../apis/google_docs/GooglePhotosClientUtils'; +import { GooglePhotos } from '../apis/google_docs/GooglePhotosClientUtils'; import { ImageField } from '../../new_fields/URLField'; import { LinkFollowBox } from './linking/LinkFollowBox'; import { DocumentManager } from '../util/DocumentManager'; @@ -149,14 +149,14 @@ export class MainView extends React.Component { }, { fireImmediately: true }); } - executeGooglePhotosRoutine = async () => { - let imgurl = "https://upload.wikimedia.org/wikipedia/commons/thumb/3/3a/Cat03.jpg/1200px-Cat03.jpg"; - let doc = Docs.Create.ImageDocument(imgurl, { width: 200, title: "an image of a cat" }); - doc.caption = "Well isn't this a nice cat image!"; - let photos = await GooglePhotosClientUtils.endpoint(); - let albumId = (await photos.albums.list(50)).albums.filter((album: any) => album.title === "This is a generically created album!")[0].id; - console.log(await GooglePhotosClientUtils.UploadImages([doc], { id: albumId })); - } + // executeGooglePhotosRoutine = async () => { + // let imgurl = "https://upload.wikimedia.org/wikipedia/commons/thumb/3/3a/Cat03.jpg/1200px-Cat03.jpg"; + // let doc = Docs.Create.ImageDocument(imgurl, { width: 200, title: "an image of a cat" }); + // doc.caption = "Well isn't this a nice cat image!"; + // let photos = await GooglePhotos.endpoint(); + // let albumId = (await photos.albums.list(50)).albums.filter((album: any) => album.title === "This is a generically created album!")[0].id; + // console.log(await GooglePhotos.UploadImages([doc], { id: albumId })); + // } componentWillUnMount() { window.removeEventListener("keydown", KeyManager.Instance.handle); diff --git a/src/client/views/nodes/DocumentView.tsx b/src/client/views/nodes/DocumentView.tsx index cb9346a8b..a51f783ad 100644 --- a/src/client/views/nodes/DocumentView.tsx +++ b/src/client/views/nodes/DocumentView.tsx @@ -41,7 +41,7 @@ import "./DocumentView.scss"; import { FormattedTextBox } from './FormattedTextBox'; import React = require("react"); import { DocumentType } from '../../documents/DocumentTypes'; -import { GooglePhotosClientUtils } from '../../apis/google_docs/GooglePhotosClientUtils'; +import { GooglePhotos } from '../../apis/google_docs/GooglePhotosClientUtils'; import { ImageField } from '../../../new_fields/URLField'; const JsxParser = require('react-jsx-parser').default; //TODO Why does this need to be imported like this? @@ -591,7 +591,10 @@ export class DocumentView extends DocComponent(Docu subitems.push({ description: "Open Fields", event: this.fieldsClicked, icon: "layer-group" }); cm.addItem({ description: "Open...", subitems: subitems, icon: "external-link-alt" }); if (Cast(this.props.Document.data, ImageField)) { - cm.addItem({ description: "Export to Google Photos", event: () => GooglePhotosClientUtils.UploadImages([this.props.Document]), icon: "caret-square-right" }); + cm.addItem({ description: "Export to Google Photos", event: () => GooglePhotos.Transactions.UploadImages([this.props.Document]), icon: "caret-square-right" }); + } + if (Cast(Doc.GetProto(this.props.Document).data, listSpec(Doc))) { + cm.addItem({ description: "Export to Google Photos Album", event: () => GooglePhotos.Export.CollectionToAlbum(this.props.Document).then(console.log), icon: "caret-square-right" }); } let existingMake = ContextMenu.Instance.findByDescription("Make..."); let makes: ContextMenuProps[] = existingMake && "subitems" in existingMake ? existingMake.subitems : []; diff --git a/src/client/views/nodes/FormattedTextBox.tsx b/src/client/views/nodes/FormattedTextBox.tsx index fda9ea33f..9d83fbd04 100644 --- a/src/client/views/nodes/FormattedTextBox.tsx +++ b/src/client/views/nodes/FormattedTextBox.tsx @@ -177,7 +177,7 @@ export class FormattedTextBox extends DocComponent<(FieldViewProps & FormattedTe syncNodeSelection(view: any, sel: any) { if (sel instanceof NodeSelection) { var desc = view.docView.descAt(sel.from); - if (desc != view.lastSelectedViewDesc) { + if (desc !== view.lastSelectedViewDesc) { if (view.lastSelectedViewDesc) { view.lastSelectedViewDesc.deselectNode(); view.lastSelectedViewDesc = null; @@ -463,7 +463,7 @@ export class FormattedTextBox extends DocComponent<(FieldViewProps & FormattedTe } let redo = async () => { if (this._editorView && reference) { - let content = RichTextUtils.GoogleDocs.Export(this._editorView.state); + let content = await RichTextUtils.GoogleDocs.Export(this._editorView.state); let response = await GoogleApiClientUtils.Docs.write({ reference, content, mode }); response && (this.dataDoc[GoogleRef] = response.documentId); let pushSuccess = response !== undefined && !("errors" in response); @@ -636,7 +636,7 @@ export class FormattedTextBox extends DocComponent<(FieldViewProps & FormattedTe image(node, view, getPos) { return new ImageResizeView(node, view, getPos); }, star(node, view, getPos) { return new SummarizedView(node, view, getPos); }, ordered_list(node, view, getPos) { return new OrderedListView(node, view, getPos); }, - footnote(node, view, getPos) { return new FootnoteView(node, view, getPos) } + footnote(node, view, getPos) { return new FootnoteView(node, view, getPos); } }, clipboardTextSerializer: this.clipboardTextSerializer, handlePaste: this.handlePaste, diff --git a/src/new_fields/RichTextUtils.ts b/src/new_fields/RichTextUtils.ts index 500b93676..27737782b 100644 --- a/src/new_fields/RichTextUtils.ts +++ b/src/new_fields/RichTextUtils.ts @@ -11,7 +11,7 @@ import { Utils, PostToServer } from "../Utils"; import { RouteStore } from "../server/RouteStore"; import { Docs } from "../client/documents/Documents"; import { schema } from "../client/util/RichTextSchema"; -import { GooglePhotosClientUtils } from "../client/apis/google_docs/GooglePhotosClientUtils"; +import { GooglePhotos } from "../client/apis/google_docs/GooglePhotosClientUtils"; export namespace RichTextUtils { @@ -90,7 +90,7 @@ export namespace RichTextUtils { export namespace GoogleDocs { - export const Export = (state: EditorState): GoogleApiClientUtils.Docs.Content => { + export const Export = async (state: EditorState): Promise => { let nodes: { [type: string]: Node[] } = { text: [], image: [] @@ -107,7 +107,7 @@ export namespace RichTextUtils { } })); let linkRequests = ExtractLinks(nodes.text); - let imageRequests = ExtractImages(nodes.image); + let imageRequests = await ExtractImages(nodes.image); return { text, requests: [...linkRequests, ...imageRequests] @@ -298,10 +298,13 @@ export namespace RichTextUtils { const length = node.nodeSize; const attrs = node.attrs; const uri = attrs.src; - const result = (await GooglePhotosClientUtils.UploadImages([uri])).newMediaItemResults; + const baseUrls = await GooglePhotos.Transactions.UploadThenFetch([uri]); + if (!baseUrls) { + continue; + } images.push({ insertInlineImage: { - uri: result[0].mediaItem.productUrl, + uri: baseUrls[0], objectSize: { width: { magnitude: parseFloat(attrs.width.replace("px", "")), unit: "PT" } }, location: { index: position + length } } diff --git a/src/server/apis/google/GooglePhotosUploadUtils.ts b/src/server/apis/google/GooglePhotosUploadUtils.ts index d1f1f81bd..447ed23ac 100644 --- a/src/server/apis/google/GooglePhotosUploadUtils.ts +++ b/src/server/apis/google/GooglePhotosUploadUtils.ts @@ -5,6 +5,7 @@ import { Utils } from '../../../Utils'; import * as path from 'path'; import { Opt } from '../../../new_fields/Doc'; import * as sharp from 'sharp'; +import { MediaItemCreationResult } from './SharedTypes'; const uploadDirectory = path.join(__dirname, "../../public/files/"); @@ -52,8 +53,10 @@ export namespace GooglePhotosUploadUtils { return new Promise(resolve => request(parameters, (error, _response, body) => resolve(error ? undefined : body))); }; - export const CreateMediaItems = (newMediaItems: any[], album?: { id: string }) => { - return new Promise((resolve, reject) => { + + + export const CreateMediaItems = (newMediaItems: any[], album?: { id: string }): Promise => { + return new Promise((resolve, reject) => { const parameters = { method: 'POST', headers: headers('json'), diff --git a/src/server/apis/google/SharedTypes.ts b/src/server/apis/google/SharedTypes.ts new file mode 100644 index 000000000..9ad6130b6 --- /dev/null +++ b/src/server/apis/google/SharedTypes.ts @@ -0,0 +1,21 @@ +export interface NewMediaItemResult { + uploadToken: string; + status: { code: number, message: string }; + mediaItem: MediaItem; +} + +export interface MediaItem { + id: string; + description: string; + productUrl: string; + baseUrl: string; + mimeType: string; + mediaMetadata: { + creationTime: string; + width: string; + height: string; + }; + filename: string; +} + +export type MediaItemCreationResult = { newMediaItemResults: NewMediaItemResult[] }; \ No newline at end of file diff --git a/src/server/credentials/google_docs_token.json b/src/server/credentials/google_docs_token.json index 5c142fba1..22d57d744 100644 --- a/src/server/credentials/google_docs_token.json +++ b/src/server/credentials/google_docs_token.json @@ -1 +1 @@ -{"access_token":"ya29.Glx_B6G7Q_FYs1LK5VcyV6P6Zg9JkoHO2aC_TsnN7AVxPYWHEpsBSC0WyWX7Ztr8HWhOUYA5JXqnZDkLrK1V3Hb-0GgtyApLRNtEPOWf1dJ7lOm_iKVw2tRvPe7XDQ","refresh_token":"1/HTv_xFHszu2Nf3iiFrUTaeKzC_Vp2-6bpIB06xW_WHI","scope":"https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/documents.readonly https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/photoslibrary https://www.googleapis.com/auth/photoslibrary.appendonly https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/photoslibrary.sharing","token_type":"Bearer","expiry_date":1568078116605} \ No newline at end of file +{"access_token":"ya29.GlyAB5T3dgJqWuYBcLaT94wQo7MZkmzJQZxDB2sSU95mdhW24E3diuFdLeNsUDVI57D3S765RweMnL98d-fdgu1dRxpzkV_J_3rLih99pZ8A4d6jVdm1354UT4py_w","refresh_token":"1/HTv_xFHszu2Nf3iiFrUTaeKzC_Vp2-6bpIB06xW_WHI","scope":"https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/documents.readonly https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/photoslibrary https://www.googleapis.com/auth/photoslibrary.appendonly https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/photoslibrary.sharing","token_type":"Bearer","expiry_date":1568161931458} \ No newline at end of file diff --git a/src/server/index.ts b/src/server/index.ts index 8469770d5..2c3e76c55 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -795,19 +795,6 @@ const EndpointHandlerMap = new Map api.batchUpdate(params)], ]); -// app.post(RouteStore.googleDocsGet, async (req, res) => { -// const token = await GoogleApiServerUtils.RetrieveAccessToken({ credentialsPath, tokenPath }); -// request_promise.get({ -// uri: `https://docs.googleapis.com/v1/documents/${req.body.documentId}?fields=inlineObjects`, -// headers: { -// 'Authorization': `Bearer ${token}` -// } -// }).then(response => { -// console.log(response); -// res.send(response); -// }); -// }); - app.post(RouteStore.googleDocs + "/:sector/:action", (req, res) => { let sector: GoogleApiServerUtils.Service = req.params.sector; let action: GoogleApiServerUtils.Action = req.params.action; @@ -841,11 +828,11 @@ app.post(RouteStore.googlePhotosMediaUpload, async (req, res) => { }; })); if (!newMediaItems.every(item => item)) { - return res.status(STATUS.EXECUTION_ERROR).send(tokenError); + return _error(res, tokenError); } GooglePhotosUploadUtils.CreateMediaItems(newMediaItems, req.body.album).then( - success => res.status(STATUS.OK).send(success), - () => res.status(STATUS.EXECUTION_ERROR).send(mediaError) + mediaItems => _success(res, mediaItems), + error => _error(res, mediaError, error) ); }); @@ -871,7 +858,7 @@ app.post(RouteStore.googlePhotosMediaDownload, async (req, res) => { _invalid(res, requestError); }); -const _error = (res: Response, message: string, error: any) => { +const _error = (res: Response, message: string, error?: any) => { res.statusMessage = message; res.status(STATUS.EXECUTION_ERROR).send(error); }; -- cgit v1.2.3-70-g09d2 From 5a0794a74ce62612435133907395482f494747f4 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Wed, 11 Sep 2019 12:19:30 -0400 Subject: now support auto tagging --- .../apis/google_docs/GooglePhotosClientUtils.ts | 126 ++++++++++++++------- .../util/Import & Export/DirectoryImportBox.tsx | 7 +- src/client/views/MainView.tsx | 19 ++-- src/client/views/nodes/DocumentView.tsx | 3 +- src/new_fields/RichTextUtils.ts | 2 +- .../apis/google/CustomizedWrapper/filters.js | 46 ++++++++ src/server/apis/google/GooglePhotosUploadUtils.ts | 34 ++++-- src/server/credentials/google_docs_token.json | 2 +- src/server/index.ts | 2 +- 9 files changed, 168 insertions(+), 73 deletions(-) create mode 100644 src/server/apis/google/CustomizedWrapper/filters.js (limited to 'src/server/apis') diff --git a/src/client/apis/google_docs/GooglePhotosClientUtils.ts b/src/client/apis/google_docs/GooglePhotosClientUtils.ts index b1de24d1a..a28b183d1 100644 --- a/src/client/apis/google_docs/GooglePhotosClientUtils.ts +++ b/src/client/apis/google_docs/GooglePhotosClientUtils.ts @@ -10,7 +10,10 @@ import { RichTextUtils } from "../../../new_fields/RichTextUtils"; import { EditorState } from "prosemirror-state"; import { FormattedTextBox } from "../../views/nodes/FormattedTextBox"; import { Docs, DocumentOptions } from "../../documents/Documents"; -import { MediaItemCreationResult, NewMediaItemResult, MediaItem } from "../../../server/apis/google/SharedTypes"; +import { NewMediaItemResult, MediaItem } from "../../../server/apis/google/SharedTypes"; +import { AssertionError } from "assert"; +import { List } from "../../../new_fields/List"; +import { listSpec } from "../../../new_fields/Schema"; export namespace GooglePhotos { @@ -53,7 +56,14 @@ export namespace GooglePhotos { PERFORMANCES: 'PERFORMANCES', WHITEBOARDS: 'WHITEBOARDS', SCREENSHOTS: 'SCREENSHOTS', - UTILITY: 'UTILITY' + UTILITY: 'UTILITY', + ARTS: 'ARTS', + CRAFTS: 'CRAFTS', + FASHION: 'FASHION', + HOUSES: 'HOUSES', + GARDENS: 'GARDENS', + FLOWERS: 'FLOWERS', + HOLIDAYS: 'HOLIDAYS' }; export namespace Export { @@ -63,7 +73,15 @@ export namespace GooglePhotos { mediaItems: MediaItem[]; } - export const CollectionToAlbum = async (collection: Doc, title?: string, descriptionKey?: string): Promise> => { + export interface AlbumCreationOptions { + collection: Doc; + title?: string; + descriptionKey?: string; + tag?: boolean; + } + + export const CollectionToAlbum = async (options: AlbumCreationOptions): Promise> => { + const { collection, title, descriptionKey, tag } = options; const dataDocument = Doc.GetProto(collection); const images = ((await DocListCastAsync(dataDocument.data)) || []).filter(doc => Cast(doc.data, ImageField)); if (!images || !images.length) { @@ -71,9 +89,24 @@ export namespace GooglePhotos { } const resolved = title ? title : (StrCast(collection.title) || `Dash Collection (${collection[Id]}`); const { id } = await Create.Album(resolved); - const result = await Transactions.UploadImages(images, { id }, descriptionKey); - if (result) { - const mediaItems = result.newMediaItemResults.map(item => item.mediaItem); + const newMediaItemResults = await Transactions.UploadImages(images, { id }, descriptionKey); + if (newMediaItemResults) { + const mediaItems = newMediaItemResults.map(item => item.mediaItem); + if (mediaItems.length !== images.length) { + throw new AssertionError({ actual: mediaItems.length, expected: images.length }); + } + const idMapping = new Doc; + for (let i = 0; i < images.length; i++) { + const image = images[i]; + const mediaItem = mediaItems[i]; + image.googlePhotosId = mediaItem.id; + image.googlePhotosUrl = mediaItem.baseUrl || mediaItem.productUrl; + idMapping[mediaItem.id] = image; + } + collection.googlePhotosIdMapping = idMapping; + if (tag) { + await Query.AppendImageMetadata(collection); + } return { albumId: id, mediaItems }; } }; @@ -101,21 +134,32 @@ export namespace GooglePhotos { export namespace Query { - export const AppendImageMetadata = (sources: (Doc | string)[]) => { - let keys = Object.keys(ContentCategories); - let included: string[] = []; - let excluded: string[] = []; - for (let i = 0; i < keys.length; i++) { - for (let j = 0; j < keys.length; j++) { - let value = ContentCategories[keys[i] as keyof typeof ContentCategories]; - if (j === i) { - included.push(value); - } else { - excluded.push(value); + export const AppendImageMetadata = async (collection: Doc) => { + const idMapping = await Cast(collection.googlePhotosIdMapping, Doc); + if (!idMapping) { + throw new Error("Appending image metadata requires that the targeted collection have already been mapped to an album!"); + } + const images = await DocListCastAsync(collection.data); + images && images.forEach(image => image.googlePhotosTags = new List()); + const values = Object.values(ContentCategories); + for (let value of values) { + console.log("Searching for ", value); + const results = await Search({ included: [value] }); + if (results.mediaItems) { + console.log(`${results.mediaItems.length} found!`); + const ids = results.mediaItems.map(item => item.id); + for (let id of ids) { + const image = await Cast(idMapping[id], Doc); + if (image) { + const tags = Cast(image.googlePhotosTags, listSpec("string"))!; + if (!tags.includes(value)) { + tags.push(value); + console.log(`${value}: ${id}`); + } + } } } - //... - included = excluded = []; + console.log(); } }; @@ -125,20 +169,22 @@ export namespace GooglePhotos { } const DefaultSearchOptions: SearchOptions = { - pageSize: 20, + pageSize: 50, included: [], excluded: [], date: undefined, includeArchivedMedia: true, + excludeNonAppCreatedData: false, type: MediaType.ALL_MEDIA, }; export interface SearchOptions { pageSize: number; - included: ContentCategories[]; - excluded: ContentCategories[]; + included: string[]; + excluded: string[]; date: Opt; includeArchivedMedia: boolean; + excludeNonAppCreatedData: boolean; type: MediaType; } @@ -173,7 +219,7 @@ export namespace GooglePhotos { filters.setMediaTypeFilter(new photos.MediaTypeFilter(options.type || MediaType.ALL_MEDIA)); return new Promise(resolve => { - photos.mediaItems.search(filters, options.pageSize || 20).then(resolve); + photos.mediaItems.search(filters, options.pageSize || 100).then(resolve); }); }; @@ -183,7 +229,7 @@ export namespace GooglePhotos { } - export namespace Create { + namespace Create { export const Album = async (title: string) => { return (await endpoint()).albums.create(title); @@ -211,40 +257,34 @@ export namespace GooglePhotos { return uploads; }; - export const UploadThenFetch = async (sources: (Doc | string)[], album?: AlbumReference, descriptionKey = "caption") => { - const result = await UploadImages(sources, album, descriptionKey); - if (!result) { + export const UploadThenFetch = async (sources: Doc[], album?: AlbumReference, descriptionKey = "caption") => { + const newMediaItems = await UploadImages(sources, album, descriptionKey); + if (!newMediaItems) { return undefined; } - const baseUrls: string[] = await Promise.all(result.newMediaItemResults.map((result: any) => { - return new Promise(resolve => Query.GetImage(result.mediaItem.id).then(item => resolve(item.baseUrl))); + const baseUrls: string[] = await Promise.all(newMediaItems.map(item => { + return new Promise(resolve => Query.GetImage(item.mediaItem.id).then(item => resolve(item.baseUrl))); })); return baseUrls; }; - export const UploadImages = async (sources: (Doc | string)[], album?: AlbumReference, descriptionKey = "caption"): Promise> => { + export const UploadImages = async (sources: Doc[], album?: AlbumReference, descriptionKey = "caption"): Promise> => { if (album && "title" in album) { album = await Create.Album(album.title); } const media: MediaInput[] = []; sources.forEach(source => { - let url: string; - let description: string; - if (source instanceof Doc) { - const data = Cast(Doc.GetProto(source).data, ImageField); - if (!data) { - return; - } - url = data.url.href; - description = parseDescription(source, descriptionKey); - } else { - url = source; - description = Utils.GenerateGuid(); + const data = Cast(Doc.GetProto(source).data, ImageField); + if (!data) { + return; } + const url = data.url.href; + const description = parseDescription(source, descriptionKey); media.push({ url, description }); }); if (media.length) { - return PostToServer(RouteStore.googlePhotosMediaUpload, { media, album }); + const uploads: NewMediaItemResult[] = await PostToServer(RouteStore.googlePhotosMediaUpload, { media, album }); + return uploads; } }; diff --git a/src/client/util/Import & Export/DirectoryImportBox.tsx b/src/client/util/Import & Export/DirectoryImportBox.tsx index d58c02ce5..348f216a5 100644 --- a/src/client/util/Import & Export/DirectoryImportBox.tsx +++ b/src/client/util/Import & Export/DirectoryImportBox.tsx @@ -74,13 +74,12 @@ export default class DirectoryImportBox extends React.Component handleSelection = async (e: React.ChangeEvent) => { runInAction(() => this.uploading = true); - let promises: Promise[] = []; let docs: Doc[] = []; let files = e.target.files; if (!files || files.length === 0) return; - let directory = (files.item(0) as any).webkitRelativePath.split("/", 1); + let directory = (files.item(0) as any).webkitRelativePath.split("/", 1)[0]; let validated: File[] = []; for (let i = 0; i < files.length; i++) { @@ -117,8 +116,6 @@ export default class DirectoryImportBox extends React.Component console.log(`(${this.quota - this.remaining}/${this.quota}) ${upload.name}`); })); - await GooglePhotos.Transactions.UploadImages(docs, { title: directory }); - for (let i = 0; i < docs.length; i++) { let doc = docs[i]; doc.size = sizes[i]; @@ -142,11 +139,11 @@ export default class DirectoryImportBox extends React.Component let parent = this.props.ContainingCollectionView; if (parent) { let importContainer = Docs.Create.StackingDocument(docs, options); + await GooglePhotos.Export.CollectionToAlbum({ collection: importContainer }); importContainer.singleColumn = false; Doc.AddDocToList(Doc.GetProto(parent.props.Document), "data", importContainer); !this.persistent && this.props.removeDocument && this.props.removeDocument(doc); DocumentManager.Instance.jumpToDocument(importContainer, true); - } runInAction(() => { diff --git a/src/client/views/MainView.tsx b/src/client/views/MainView.tsx index 8d10a91ce..28edf181b 100644 --- a/src/client/views/MainView.tsx +++ b/src/client/views/MainView.tsx @@ -130,7 +130,7 @@ export class MainView extends React.Component { window.removeEventListener("keydown", KeyManager.Instance.handle); window.addEventListener("keydown", KeyManager.Instance.handle); - // this.executeGooglePhotosRoutine(); + this.executeGooglePhotosRoutine(); reaction(() => { let workspaces = CurrentUserUtils.UserDocument.workspaces; @@ -149,14 +149,15 @@ export class MainView extends React.Component { }, { fireImmediately: true }); } - // executeGooglePhotosRoutine = async () => { - // let imgurl = "https://upload.wikimedia.org/wikipedia/commons/thumb/3/3a/Cat03.jpg/1200px-Cat03.jpg"; - // let doc = Docs.Create.ImageDocument(imgurl, { width: 200, title: "an image of a cat" }); - // doc.caption = "Well isn't this a nice cat image!"; - // let photos = await GooglePhotos.endpoint(); - // let albumId = (await photos.albums.list(50)).albums.filter((album: any) => album.title === "This is a generically created album!")[0].id; - // console.log(await GooglePhotos.UploadImages([doc], { id: albumId })); - // } + executeGooglePhotosRoutine = async () => { + // let imgurl = "https://upload.wikimedia.org/wikipedia/commons/thumb/3/3a/Cat03.jpg/1200px-Cat03.jpg"; + // let doc = Docs.Create.ImageDocument(imgurl, { width: 200, title: "an image of a cat" }); + // doc.caption = "Well isn't this a nice cat image!"; + // let photos = await GooglePhotos.endpoint(); + // let albumId = (await photos.albums.list(50)).albums.filter((album: any) => album.title === "This is a generically created album!")[0].id; + // console.log(await GooglePhotos.UploadImages([doc], { id: albumId })); + GooglePhotos.Query.Search({ included: [GooglePhotos.ContentCategories.ANIMALS] }).then(console.log); + } componentWillUnMount() { window.removeEventListener("keydown", KeyManager.Instance.handle); diff --git a/src/client/views/nodes/DocumentView.tsx b/src/client/views/nodes/DocumentView.tsx index a51f783ad..a38f42751 100644 --- a/src/client/views/nodes/DocumentView.tsx +++ b/src/client/views/nodes/DocumentView.tsx @@ -594,7 +594,8 @@ export class DocumentView extends DocComponent(Docu cm.addItem({ description: "Export to Google Photos", event: () => GooglePhotos.Transactions.UploadImages([this.props.Document]), icon: "caret-square-right" }); } if (Cast(Doc.GetProto(this.props.Document).data, listSpec(Doc))) { - cm.addItem({ description: "Export to Google Photos Album", event: () => GooglePhotos.Export.CollectionToAlbum(this.props.Document).then(console.log), icon: "caret-square-right" }); + cm.addItem({ description: "Export to Google Photos Album", event: () => GooglePhotos.Export.CollectionToAlbum({ collection: this.props.Document }).then(console.log), icon: "caret-square-right" }); + cm.addItem({ description: "Tag Child Images via Google Photos", event: () => GooglePhotos.Query.AppendImageMetadata(this.props.Document), icon: "caret-square-right" }); } let existingMake = ContextMenu.Instance.findByDescription("Make..."); let makes: ContextMenuProps[] = existingMake && "subitems" in existingMake ? existingMake.subitems : []; diff --git a/src/new_fields/RichTextUtils.ts b/src/new_fields/RichTextUtils.ts index 27737782b..6afe4ddfd 100644 --- a/src/new_fields/RichTextUtils.ts +++ b/src/new_fields/RichTextUtils.ts @@ -298,7 +298,7 @@ export namespace RichTextUtils { const length = node.nodeSize; const attrs = node.attrs; const uri = attrs.src; - const baseUrls = await GooglePhotos.Transactions.UploadThenFetch([uri]); + const baseUrls = await GooglePhotos.Transactions.UploadThenFetch([Docs.Create.ImageDocument(uri)]); if (!baseUrls) { continue; } diff --git a/src/server/apis/google/CustomizedWrapper/filters.js b/src/server/apis/google/CustomizedWrapper/filters.js new file mode 100644 index 000000000..576a90b75 --- /dev/null +++ b/src/server/apis/google/CustomizedWrapper/filters.js @@ -0,0 +1,46 @@ +'use strict'; + +const DateFilter = require('../common/date_filter'); +const MediaTypeFilter = require('./media_type_filter'); +const ContentFilter = require('./content_filter'); + +class Filters { + constructor(includeArchivedMedia = false) { + this.includeArchivedMedia = includeArchivedMedia; + } + + setDateFilter(dateFilter) { + this.dateFilter = dateFilter; + return this; + } + + setContentFilter(contentFilter) { + this.contentFilter = contentFilter; + return this; + } + + setMediaTypeFilter(mediaTypeFilter) { + this.mediaTypeFilter = mediaTypeFilter; + return this; + } + + setIncludeArchivedMedia(includeArchivedMedia) { + this.includeArchivedMedia = includeArchivedMedia; + return this; + } + + toJSON() { + return { + dateFilter: this.dateFilter instanceof DateFilter ? this.dateFilter.toJSON() : this.dateFilter, + mediaTypeFilter: this.mediaTypeFilter instanceof MediaTypeFilter ? + this.mediaTypeFilter.toJSON() : + this.mediaTypeFilter, + contentFilter: this.contentFilter instanceof ContentFilter ? + this.contentFilter.toJSON() : + this.contentFilter, + includeArchivedMedia: this.includeArchivedMedia + }; + } +} + +module.exports = Filters; \ No newline at end of file diff --git a/src/server/apis/google/GooglePhotosUploadUtils.ts b/src/server/apis/google/GooglePhotosUploadUtils.ts index 447ed23ac..1a8adc836 100644 --- a/src/server/apis/google/GooglePhotosUploadUtils.ts +++ b/src/server/apis/google/GooglePhotosUploadUtils.ts @@ -5,7 +5,7 @@ import { Utils } from '../../../Utils'; import * as path from 'path'; import { Opt } from '../../../new_fields/Doc'; import * as sharp from 'sharp'; -import { MediaItemCreationResult } from './SharedTypes'; +import { MediaItemCreationResult, NewMediaItemResult } from './SharedTypes'; const uploadDirectory = path.join(__dirname, "../../public/files/"); @@ -55,24 +55,34 @@ export namespace GooglePhotosUploadUtils { - export const CreateMediaItems = (newMediaItems: any[], album?: { id: string }): Promise => { - return new Promise((resolve, reject) => { + export const CreateMediaItems = async (newMediaItems: any[], album?: { id: string }): Promise => { + const quota = newMediaItems.length; + let handled = 0; + const newMediaItemResults: NewMediaItemResult[] = []; + while (handled < quota) { + const cap = Math.min(newMediaItems.length, handled + 50); + const batch = newMediaItems.slice(handled, cap); + console.log(batch.length); const parameters = { method: 'POST', headers: headers('json'), uri: prepend('mediaItems:batchCreate'), - body: { newMediaItems } as any, + body: { newMediaItems: batch } as any, json: true }; album && (parameters.body.albumId = album.id); - request(parameters, (error, _response, body) => { - if (error) { - reject(error); - } else { - resolve(body); - } - }); - }); + newMediaItemResults.push(...(await new Promise((resolve, reject) => { + request(parameters, (error, _response, body) => { + if (error) { + reject(error); + } else { + resolve(body); + } + }); + })).newMediaItemResults); + handled = cap; + } + return { newMediaItemResults }; }; } diff --git a/src/server/credentials/google_docs_token.json b/src/server/credentials/google_docs_token.json index 22d57d744..0c06f68b7 100644 --- a/src/server/credentials/google_docs_token.json +++ b/src/server/credentials/google_docs_token.json @@ -1 +1 @@ -{"access_token":"ya29.GlyAB5T3dgJqWuYBcLaT94wQo7MZkmzJQZxDB2sSU95mdhW24E3diuFdLeNsUDVI57D3S765RweMnL98d-fdgu1dRxpzkV_J_3rLih99pZ8A4d6jVdm1354UT4py_w","refresh_token":"1/HTv_xFHszu2Nf3iiFrUTaeKzC_Vp2-6bpIB06xW_WHI","scope":"https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/documents.readonly https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/photoslibrary https://www.googleapis.com/auth/photoslibrary.appendonly https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/photoslibrary.sharing","token_type":"Bearer","expiry_date":1568161931458} \ No newline at end of file +{"access_token":"ya29.GlyAB7VxfbK7fwV9-lqu9NZ1-p73aC8KaEXAYGHFOIIgAhx40CCUgS07vy485y7O0x9RwK-7FL6P547SscD5bVlTlJkclP-9uupKxDaeez7Tc7o2pJwt6bgJlbbw7w","refresh_token":"1/HTv_xFHszu2Nf3iiFrUTaeKzC_Vp2-6bpIB06xW_WHI","scope":"https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/documents.readonly https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/photoslibrary https://www.googleapis.com/auth/photoslibrary.appendonly https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/photoslibrary.sharing","token_type":"Bearer","expiry_date":1568220636395} \ No newline at end of file diff --git a/src/server/index.ts b/src/server/index.ts index 2c3e76c55..507463841 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -831,7 +831,7 @@ app.post(RouteStore.googlePhotosMediaUpload, async (req, res) => { return _error(res, tokenError); } GooglePhotosUploadUtils.CreateMediaItems(newMediaItems, req.body.album).then( - mediaItems => _success(res, mediaItems), + result => _success(res, result.newMediaItemResults), error => _error(res, mediaError, error) ); }); -- cgit v1.2.3-70-g09d2 From 5af7c8c709c8413239fe8642208891c2413dad62 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Wed, 11 Sep 2019 17:27:47 -0400 Subject: text enrichment and collections storing album id --- .../apis/google_docs/GooglePhotosClientUtils.ts | 14 +++++++++ src/client/views/nodes/DocumentView.tsx | 1 + src/server/apis/google/GooglePhotosUploadUtils.ts | 35 ++++++++++++---------- src/server/credentials/google_docs_token.json | 2 +- src/server/index.ts | 2 +- 5 files changed, 36 insertions(+), 18 deletions(-) (limited to 'src/server/apis') diff --git a/src/client/apis/google_docs/GooglePhotosClientUtils.ts b/src/client/apis/google_docs/GooglePhotosClientUtils.ts index 118462778..49eb5b354 100644 --- a/src/client/apis/google_docs/GooglePhotosClientUtils.ts +++ b/src/client/apis/google_docs/GooglePhotosClientUtils.ts @@ -107,6 +107,7 @@ export namespace GooglePhotos { if (tag) { await Query.TagChildImages(collection); } + collection.albumId = id; return { albumId: id, mediaItems }; } }; @@ -257,6 +258,19 @@ export namespace GooglePhotos { baseUrl: string; } + export const AddTextEnrichment = async (collection: Doc, content?: string) => { + const photos = await endpoint(); + const albumId = StrCast(collection.albumId); + if (albumId && albumId.length) { + const enrichment = new photos.TextEnrichment(content || Utils.prepend("/doc/" + collection[Id])); + const position = new photos.AlbumPosition(photos.AlbumPosition.POSITIONS.FIRST_IN_ALBUM); + const enrichmentItem = await photos.albums.addEnrichment(albumId, enrichment, position); + if (enrichmentItem) { + return enrichmentItem.id; + } + } + }; + export const WriteMediaItemsToServer = async (body: { mediaItems: any[] }): Promise => { const uploads = await PostToServer(RouteStore.googlePhotosMediaDownload, body); return uploads; diff --git a/src/client/views/nodes/DocumentView.tsx b/src/client/views/nodes/DocumentView.tsx index fdec84526..1e4216dbb 100644 --- a/src/client/views/nodes/DocumentView.tsx +++ b/src/client/views/nodes/DocumentView.tsx @@ -596,6 +596,7 @@ export class DocumentView extends DocComponent(Docu if (Cast(Doc.GetProto(this.props.Document).data, listSpec(Doc))) { cm.addItem({ description: "Export to Google Photos Album", event: () => GooglePhotos.Export.CollectionToAlbum({ collection: this.props.Document }).then(console.log), icon: "caret-square-right" }); cm.addItem({ description: "Tag Child Images via Google Photos", event: () => GooglePhotos.Query.TagChildImages(this.props.Document), icon: "caret-square-right" }); + cm.addItem({ description: "Write Back Link to Album", event: () => GooglePhotos.Transactions.AddTextEnrichment(this.props.Document), icon: "caret-square-right" }); } let existingMake = ContextMenu.Instance.findByDescription("Make..."); let makes: ContextMenuProps[] = existingMake && "subitems" in existingMake ? existingMake.subitems : []; diff --git a/src/server/apis/google/GooglePhotosUploadUtils.ts b/src/server/apis/google/GooglePhotosUploadUtils.ts index 1a8adc836..7f47259db 100644 --- a/src/server/apis/google/GooglePhotosUploadUtils.ts +++ b/src/server/apis/google/GooglePhotosUploadUtils.ts @@ -103,7 +103,8 @@ export namespace DownloadUtils { const png = ".png"; const pngs = [".png", ".PNG"]; const jpgs = [".jpg", ".JPG", ".jpeg", ".JPEG"]; - const formats = [".jpg", ".png", ".gif"]; + const imageFormats = [".jpg", ".png", ".gif"]; + const videoFormats = [".mov", ".mp4"]; const size = "content-length"; const type = "content-type"; @@ -150,26 +151,28 @@ export namespace DownloadUtils { resizers.forEach(element => element.resizer = element.resizer.png()); } else if (jpgs.includes(extension)) { resizers.forEach(element => element.resizer = element.resizer.jpeg()); - } else if (!formats.includes(extension.toLowerCase())) { - return reject(); + } else if (![...imageFormats, ...videoFormats].includes(extension.toLowerCase())) { + return resolve(undefined); } - for (let resizer of resizers) { - const suffix = resizer.suffix; - let mediaPath: string; - await new Promise(resolve => { - const filename = resolved.substring(0, resolved.length - extension.length) + suffix + extension; - information.mediaPaths.push(mediaPath = uploadDirectory + filename); - information.fileNames[suffix] = filename; - stream(url).pipe(resizer.resizer).pipe(fs.createWriteStream(mediaPath)) - .on('close', resolve) - .on('error', reject); - }); - if (!isLocal) { + if (imageFormats.includes(extension)) { + for (let resizer of resizers) { + const suffix = resizer.suffix; + let mediaPath: string; await new Promise(resolve => { - stream(url).pipe(fs.createWriteStream(uploadDirectory + resolved)).on('close', resolve); + const filename = resolved.substring(0, resolved.length - extension.length) + suffix + extension; + information.mediaPaths.push(mediaPath = uploadDirectory + filename); + information.fileNames[suffix] = filename; + stream(url).pipe(resizer.resizer).pipe(fs.createWriteStream(mediaPath)) + .on('close', resolve) + .on('error', reject); }); } } + if (!isLocal) { + await new Promise(resolve => { + stream(url).pipe(fs.createWriteStream(uploadDirectory + resolved)).on('close', resolve); + }); + } resolve(information); }); }; diff --git a/src/server/credentials/google_docs_token.json b/src/server/credentials/google_docs_token.json index c5026e60f..c58287bee 100644 --- a/src/server/credentials/google_docs_token.json +++ b/src/server/credentials/google_docs_token.json @@ -1 +1 @@ -{"access_token":"ya29.GlyAB0qt-0gbGYiOwleSnxXDKKHo8k5Djr5VOlbioTfUNbzHzRrguj4fHiauxPNEesgQjBssx5djYipTHtzheoLaRiR8uHZ9bcz8RHsQYIaAW4QpvTQkwnLjGwkG5w","refresh_token":"1/HTv_xFHszu2Nf3iiFrUTaeKzC_Vp2-6bpIB06xW_WHI","scope":"https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/documents.readonly https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/photoslibrary https://www.googleapis.com/auth/photoslibrary.appendonly https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/photoslibrary.sharing","token_type":"Bearer","expiry_date":1568235681891} \ No newline at end of file +{"access_token":"ya29.ImCBBwOPA7RqPIoh9RrZn90HLJnYAazRjts5R17yNQi9QLENQiChUUIUjcsTqbL-4cs_TK7UbEID6pR0w71gyTjVnA5uBcPJFcAaZ-GRPtheXx0PDU4oqSWHYoqlNQQKjn4","refresh_token":"1/HTv_xFHszu2Nf3iiFrUTaeKzC_Vp2-6bpIB06xW_WHI","scope":"https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/documents.readonly https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/photoslibrary https://www.googleapis.com/auth/photoslibrary.appendonly https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/photoslibrary.sharing","token_type":"Bearer","expiry_date":1568239483409} \ No newline at end of file diff --git a/src/server/index.ts b/src/server/index.ts index 507463841..388c8cd4d 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -579,7 +579,7 @@ app.post( for (const key in files) { const { type, path: location, name } = files[key]; const filename = path.basename(location); - await UploadUtils.UploadImage(uploadDirectory + filename, filename); + await UploadUtils.UploadImage(uploadDirectory + filename, filename).catch(() => console.log(`Unable to process ${filename}`)); results.push({ name, type, path: `/files/${filename}` }); console.log(path.basename(name)); } -- cgit v1.2.3-70-g09d2 From 2dd8b13fd3fa30fc390251ed75da3207efed4d5b Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Wed, 11 Sep 2019 21:49:56 -0400 Subject: restored labels to pivot viewer --- src/client/apis/google_docs/GooglePhotosClientUtils.ts | 17 +++++++++++------ .../collectionFreeForm/CollectionFreeFormView.tsx | 13 +++++++------ src/server/apis/google/GooglePhotosUploadUtils.ts | 8 +++++++- src/server/credentials/google_docs_token.json | 2 +- src/server/index.ts | 2 +- 5 files changed, 27 insertions(+), 15 deletions(-) (limited to 'src/server/apis') diff --git a/src/client/apis/google_docs/GooglePhotosClientUtils.ts b/src/client/apis/google_docs/GooglePhotosClientUtils.ts index 49eb5b354..700c0401a 100644 --- a/src/client/apis/google_docs/GooglePhotosClientUtils.ts +++ b/src/client/apis/google_docs/GooglePhotosClientUtils.ts @@ -135,13 +135,15 @@ export namespace GooglePhotos { export namespace Query { + const delimiter = ", "; export const TagChildImages = async (collection: Doc) => { const idMapping = await Cast(collection.googlePhotosIdMapping, Doc); if (!idMapping) { throw new Error("Appending image metadata requires that the targeted collection have already been mapped to an album!"); } + const tagMapping = new Map(); const images = await DocListCastAsync(collection.data); - images && images.forEach(image => image.googlePhotosTags = new List([ContentCategories.NONE])); + images && images.forEach(image => tagMapping.set(image[Id], ContentCategories.NONE)); const values = Object.values(ContentCategories); for (let value of values) { if (value !== ContentCategories.NONE) { @@ -151,9 +153,10 @@ export namespace GooglePhotos { for (let id of ids) { const image = await Cast(idMapping[id], Doc); if (image) { - const tags = Cast(image.googlePhotosTags, listSpec("string"))!; + const key = image[Id]; + const tags = tagMapping.get(key)!; if (!tags.includes(value)) { - tags.push(value); + tagMapping.set(key, tags + delimiter + value); } } } @@ -161,9 +164,11 @@ export namespace GooglePhotos { } } images && images.forEach(image => { - const tags = Cast(image.googlePhotosTags, listSpec("string"))!; - if (tags.includes(ContentCategories.NONE) && tags.length > 1) { - image.googlePhotosTags = new List(tags.splice(tags.indexOf(ContentCategories.NONE), 1)); + const concatenated = tagMapping.get(image[Id])!; + const tags = concatenated.split(delimiter); + if (tags.length > 1) { + const cleaned = concatenated.replace(ContentCategories.NONE + delimiter, ""); + image.googlePhotosTags = cleaned.split(delimiter).sort((a, b) => (a < b) ? -1 : (a > b ? 1 : 0)).join(delimiter); } }); diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx index a7acd9e91..1af534ecd 100644 --- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx +++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx @@ -98,7 +98,7 @@ export namespace PivotView { groups.forEach((val, key) => minSize = Math.min(minSize, val.length)); const numCols = NumCast(collection.pivotNumColumns) || Math.ceil(Math.sqrt(minSize)); - const fontSize = NumCast(collection.pivotFontSize); + const fontSize = NumCast(collection.pivotFontSize, 30); const docMap = new Map(); const groupNames: PivotData[] = []; @@ -113,7 +113,8 @@ export namespace PivotView { x, y: width + 50, width: width * 1.25 * numCols, - height: 100, fontSize: fontSize + height: 100, + fontSize }); for (const doc of val) { docMap.set(doc, { @@ -701,7 +702,7 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) { return result.result === undefined ? { x: Cast(doc.x, "number"), y: Cast(doc.y, "number"), z: Cast(doc.z, "number"), width: Cast(doc.width, "number"), height: Cast(doc.height, "number") } : result.result; } - viewDefsToJSX = (views: any[]) => { + viewDefsToJSX = (views: PivotView.PivotData[]) => { let elements: ViewDefResult[] = []; if (Array.isArray(views)) { elements = views.reduce((prev, ele) => { @@ -713,12 +714,12 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) { return elements; } - private viewDefToJSX(viewDef: any): Opt { + private viewDefToJSX(viewDef: PivotView.PivotData): Opt { if (viewDef.type === "text") { const text = Cast(viewDef.text, "string"); const x = Cast(viewDef.x, "number"); const y = Cast(viewDef.y, "number"); - const z = Cast(viewDef.z, "number"); + // const z = Cast(viewDef.z, "number"); const width = Cast(viewDef.width, "number"); const height = Cast(viewDef.height, "number"); const fontSize = Cast(viewDef.fontSize, "number"); @@ -730,7 +731,7 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) { ele:
{text}
, bounds: { x: x!, y: y!, z: z, width: width!, height: height! } + }}>{text}, bounds: { x: x!, y: y!, width: width!, height: height! } }; } } diff --git a/src/server/apis/google/GooglePhotosUploadUtils.ts b/src/server/apis/google/GooglePhotosUploadUtils.ts index 7f47259db..51642e345 100644 --- a/src/server/apis/google/GooglePhotosUploadUtils.ts +++ b/src/server/apis/google/GooglePhotosUploadUtils.ts @@ -6,6 +6,7 @@ import * as path from 'path'; import { Opt } from '../../../new_fields/Doc'; import * as sharp from 'sharp'; import { MediaItemCreationResult, NewMediaItemResult } from './SharedTypes'; +import { reject } from 'bluebird'; const uploadDirectory = path.join(__dirname, "../../public/files/"); @@ -50,7 +51,12 @@ export namespace GooglePhotosUploadUtils { uri: prepend('uploads'), body }; - return new Promise(resolve => request(parameters, (error, _response, body) => resolve(error ? undefined : body))); + return new Promise(resolve => request(parameters, (error, _response, body) => { + if (error) { + return reject(error); + } + resolve(body); + })); }; diff --git a/src/server/credentials/google_docs_token.json b/src/server/credentials/google_docs_token.json index c58287bee..7442c643a 100644 --- a/src/server/credentials/google_docs_token.json +++ b/src/server/credentials/google_docs_token.json @@ -1 +1 @@ -{"access_token":"ya29.ImCBBwOPA7RqPIoh9RrZn90HLJnYAazRjts5R17yNQi9QLENQiChUUIUjcsTqbL-4cs_TK7UbEID6pR0w71gyTjVnA5uBcPJFcAaZ-GRPtheXx0PDU4oqSWHYoqlNQQKjn4","refresh_token":"1/HTv_xFHszu2Nf3iiFrUTaeKzC_Vp2-6bpIB06xW_WHI","scope":"https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/documents.readonly https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/photoslibrary https://www.googleapis.com/auth/photoslibrary.appendonly https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/photoslibrary.sharing","token_type":"Bearer","expiry_date":1568239483409} \ No newline at end of file +{"access_token":"ya29.GlyBB9xlRimL3pw4tgNg7g7wcr73JWyQd4-XZbgOvngFM_sYUgsWP0YV7XCez5u6nytEfrOm228Sadj52wluJ46cJGhj2IwtSbW9GYzHHiiD-ts0i1phIV3n28wo5A","refresh_token":"1/HTv_xFHszu2Nf3iiFrUTaeKzC_Vp2-6bpIB06xW_WHI","scope":"https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/documents.readonly https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/photoslibrary https://www.googleapis.com/auth/photoslibrary.appendonly https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/photoslibrary.sharing","token_type":"Bearer","expiry_date":1568254634977} \ No newline at end of file diff --git a/src/server/index.ts b/src/server/index.ts index 388c8cd4d..9a2bd9a3a 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -821,7 +821,7 @@ app.post(RouteStore.googlePhotosMediaUpload, async (req, res) => { const media: GooglePhotosUploadUtils.MediaInput[] = req.body.media; await GooglePhotosUploadUtils.initialize({ uploadDirectory, credentialsPath, tokenPath }); const newMediaItems = await Promise.all(media.map(async element => { - const uploadToken = await GooglePhotosUploadUtils.DispatchGooglePhotosUpload(element.url); + const uploadToken = await GooglePhotosUploadUtils.DispatchGooglePhotosUpload(element.url).catch(error => _error(res, tokenError, error)); return !uploadToken ? undefined : { description: element.description, simpleMediaItem: { uploadToken } -- cgit v1.2.3-70-g09d2 From cbb016dd4bec4ce1367314717adf85640ae51c93 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Thu, 12 Sep 2019 05:21:29 -0400 Subject: sharing workflow supported --- .vscode/launch.json | 5 +- src/client/DocServer.ts | 10 +- .../apis/google_docs/GooglePhotosClientUtils.ts | 3 +- src/client/documents/Documents.ts | 1 - src/client/util/DictationManager.ts | 46 +- src/client/util/History.ts | 6 +- src/client/util/SharingManager.scss | 136 +++ src/client/util/SharingManager.tsx | 293 ++++++ src/client/views/GlobalKeyHandler.ts | 2 + src/client/views/InkingCanvas.scss | 2 +- src/client/views/Main.scss | 40 - src/client/views/Main.tsx | 16 +- src/client/views/MainView.tsx | 272 ++++-- src/client/views/MainViewModal.scss | 25 + src/client/views/MainViewModal.tsx | 44 + src/client/views/OverlayView.tsx | 3 + src/client/views/ScriptingRepl.scss | 1 + .../views/collections/CollectionDockingView.tsx | 33 +- .../collectionFreeForm/CollectionFreeFormView.tsx | 18 +- src/client/views/nodes/DocumentView.scss | 15 + src/client/views/nodes/DocumentView.tsx | 43 +- .../views/presentationview/PresentationView.tsx | 993 +++++++++++++++++++++ src/server/Message.ts | 7 +- src/server/apis/google/GooglePhotosUploadUtils.ts | 2 - .../authentication/models/current_user_utils.ts | 8 +- src/server/credentials/google_docs_token.json | 2 +- src/server/index.ts | 64 +- 27 files changed, 1835 insertions(+), 255 deletions(-) create mode 100644 src/client/util/SharingManager.scss create mode 100644 src/client/util/SharingManager.tsx create mode 100644 src/client/views/MainViewModal.scss create mode 100644 src/client/views/MainViewModal.tsx create mode 100644 src/client/views/presentationview/PresentationView.tsx (limited to 'src/server/apis') diff --git a/.vscode/launch.json b/.vscode/launch.json index d2c18d6f1..e1c5c6f94 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -3,14 +3,13 @@ // Hover to view descriptions of existing attributes. // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 "version": "0.2.0", - "configurations": [ - { + "configurations": [{ "type": "chrome", "request": "launch", "name": "Launch Chrome against localhost", "sourceMaps": true, "breakOnLoad": true, - "url": "http://localhost:1050/login", + "url": "http://localhost:1050/logout", "webRoot": "${workspaceFolder}", "runtimeArgs": [ "--experimental-modules" diff --git a/src/client/DocServer.ts b/src/client/DocServer.ts index 2cec1046b..4dea4f11c 100644 --- a/src/client/DocServer.ts +++ b/src/client/DocServer.ts @@ -144,7 +144,7 @@ export namespace DocServer { * the server if the document has not been cached. * @param id the id of the requested document */ - const _GetRefFieldImpl = (id: string): Promise> => { + const _GetRefFieldImpl = (id: string, mongoCollection?: string): Promise> => { // an initial pass through the cache to determine whether the document needs to be fetched, // is already in the process of being fetched or already exists in the // cache @@ -155,7 +155,7 @@ export namespace DocServer { // synchronously, we emit a single callback to the server requesting the serialized (i.e. represented by a string) // field for the given ids. This returns a promise, which, when resolved, indicates the the JSON serialized version of // the field has been returned from the server - const getSerializedField = Utils.EmitCallback(_socket, MessageStore.GetRefField, id); + const getSerializedField = Utils.EmitCallback(_socket, MessageStore.GetRefField, { id, mongoCollection }); // when the serialized RefField has been received, go head and begin deserializing it into an object. // Here, once deserialized, we also invoke .proto to 'load' the document's prototype, which ensures that all @@ -188,10 +188,10 @@ export namespace DocServer { } }; - let _GetRefField: (id: string) => Promise> = errorFunc; + let _GetRefField: (id: string, mongoCollection?: string) => Promise> = errorFunc; - export function GetRefField(id: string): Promise> { - return _GetRefField(id); + export function GetRefField(id: string, mongoCollection = "newDocuments"): Promise> { + return _GetRefField(id, mongoCollection); } export async function getYoutubeChannels() { diff --git a/src/client/apis/google_docs/GooglePhotosClientUtils.ts b/src/client/apis/google_docs/GooglePhotosClientUtils.ts index 700c0401a..b308cc9be 100644 --- a/src/client/apis/google_docs/GooglePhotosClientUtils.ts +++ b/src/client/apis/google_docs/GooglePhotosClientUtils.ts @@ -108,6 +108,7 @@ export namespace GooglePhotos { await Query.TagChildImages(collection); } collection.albumId = id; + Transactions.AddTextEnrichment(collection, `Find me at ${Utils.prepend(`/doc/${collection[Id]}?sharing=true`)}`); return { albumId: id, mediaItems }; } }; @@ -313,7 +314,7 @@ export namespace GooglePhotos { }; const parseDescription = (document: Doc, descriptionKey: string) => { - let description: string = Utils.prepend("/doc/" + document[Id]); + let description: string = Utils.prepend(`/doc/${document[Id]}?sharing=true`); const target = document[descriptionKey]; if (typeof target === "string") { description = target; diff --git a/src/client/documents/Documents.ts b/src/client/documents/Documents.ts index 5dd945c16..cfed2bf14 100644 --- a/src/client/documents/Documents.ts +++ b/src/client/documents/Documents.ts @@ -158,7 +158,6 @@ export namespace Docs { [DocumentType.LINKDOC, { data: new List(), layout: { view: EmptyBox }, - options: {} }], [DocumentType.YOUTUBE, { layout: { view: YoutubeBox } diff --git a/src/client/util/DictationManager.ts b/src/client/util/DictationManager.ts index fb3c15cea..0711effe6 100644 --- a/src/client/util/DictationManager.ts +++ b/src/client/util/DictationManager.ts @@ -3,7 +3,7 @@ import { DocumentView } from "../views/nodes/DocumentView"; import { UndoManager } from "./UndoManager"; import * as interpreter from "words-to-numbers"; import { DocumentType } from "../documents/DocumentTypes"; -import { Doc } from "../../new_fields/Doc"; +import { Doc, Opt } from "../../new_fields/Doc"; import { List } from "../../new_fields/List"; import { Docs } from "../documents/Documents"; import { CollectionViewType } from "../views/collections/CollectionBaseView"; @@ -40,12 +40,26 @@ export namespace DictationManager { webkitSpeechRecognition: any; } } - const { webkitSpeechRecognition }: CORE.IWindow = window as CORE.IWindow; + const { webkitSpeechRecognition }: CORE.IWindow = window as any as CORE.IWindow; export const placeholder = "Listening..."; export namespace Controls { export const Infringed = "unable to process: dictation manager still involved in previous session"; + const browser = (() => { + let identifier = navigator.userAgent.toLowerCase(); + if (identifier.indexOf("safari") >= 0) { + return "Safari"; + } + if (identifier.indexOf("chrome") >= 0) { + return "Chrome"; + } + if (identifier.indexOf("firefox") >= 0) { + return "Firefox"; + } + return "Unidentified Browser"; + })(); + const unsupported = `listening is not supported in ${browser}`; const intraSession = ". "; const interSession = " ... "; @@ -55,8 +69,7 @@ export namespace DictationManager { let current: string | undefined = undefined; let sessionResults: string[] = []; - const recognizer: SpeechRecognition = new webkitSpeechRecognition() || new SpeechRecognition(); - recognizer.onstart = () => console.log("initiating speech recognition session..."); + const recognizer: Opt = webkitSpeechRecognition ? new webkitSpeechRecognition() : undefined; export type InterimResultHandler = (results: string) => any; export type ContinuityArgs = { indefinite: boolean } | false; @@ -109,6 +122,10 @@ export namespace DictationManager { }; const listenImpl = (options?: Partial) => { + if (!recognizer) { + console.log(unsupported); + return unsupported; + } if (isListening) { return Infringed; } @@ -121,6 +138,7 @@ export namespace DictationManager { let intra = options && options.delimiters ? options.delimiters.intra : undefined; let inter = options && options.delimiters ? options.delimiters.inter : undefined; + recognizer.onstart = () => console.log("initiating speech recognition session..."); recognizer.interimResults = handler !== undefined; recognizer.continuous = continuous === undefined ? false : continuous !== false; recognizer.lang = language === undefined ? "en-US" : language; @@ -167,14 +185,20 @@ export namespace DictationManager { } else { resolve(current); } - reset(); + current = undefined; + sessionResults = []; + isListening = false; + isManuallyStopped = false; + recognizer.onresult = null; + recognizer.onerror = null; + recognizer.onend = null; }; }); }; export const stop = (salvageSession = true) => { - if (!isListening) { + if (!isListening || !recognizer) { return; } isManuallyStopped = true; @@ -197,16 +221,6 @@ export namespace DictationManager { return transcripts.join(delimiter || intraSession); }; - const reset = () => { - current = undefined; - sessionResults = []; - isListening = false; - isManuallyStopped = false; - recognizer.onresult = null; - recognizer.onerror = null; - recognizer.onend = null; - }; - } export namespace Commands { diff --git a/src/client/util/History.ts b/src/client/util/History.ts index e9ff21b22..c72ae05de 100644 --- a/src/client/util/History.ts +++ b/src/client/util/History.ts @@ -16,8 +16,10 @@ export namespace HistoryUtil { initializers?: { [docId: string]: DocInitializerList; }; + safe?: boolean; readonly?: boolean; nro?: boolean; + sharing?: boolean; } export type ParsedUrl = DocUrl; @@ -141,7 +143,7 @@ export namespace HistoryUtil { }; } - addParser("doc", {}, { readonly: true, initializers: true, nro: true }, (pathname, opts, current) => { + addParser("doc", {}, { readonly: true, initializers: true, nro: true, sharing: true }, (pathname, opts, current) => { if (pathname.length !== 2) return undefined; current.initializers = current.initializers || {}; @@ -156,7 +158,7 @@ export namespace HistoryUtil { export function parseUrl(location: Location | URL): ParsedUrl | undefined { const pathname = location.pathname.substring(1); const search = location.search; - const opts = qs.parse(search, { sort: false }); + const opts = search.length ? qs.parse(search, { sort: false }) : {}; let pathnameSplit = pathname.split("/"); const type = pathnameSplit[0]; diff --git a/src/client/util/SharingManager.scss b/src/client/util/SharingManager.scss new file mode 100644 index 000000000..9a4c5db30 --- /dev/null +++ b/src/client/util/SharingManager.scss @@ -0,0 +1,136 @@ +.sharing-interface { + display: flex; + flex-direction: column; + + p { + font-size: 20px; + text-align: left; + font-style: italic; + padding: 0; + margin: 0 0 20px 0; + } + + .hr-substitute { + border: solid black 0.5px; + margin-top: 20px; + } + + .people-with-container { + display: flex; + height: 25px; + + .people-with { + font-size: 14px; + margin: 0; + padding-top: 3px; + font-style: normal; + } + + .people-with-select { + width: 126px; + outline: none; + } + } + + .share-individual { + margin-top: 20px; + margin-bottom: 20px; + } + + .users-list { + font-style: italic; + background: white; + border: 1px solid black; + padding-left: 10px; + padding-right: 10px; + max-height: 200px; + overflow: scroll; + height: -webkit-fill-available; + text-align: left; + display: flex; + align-content: center; + align-items: center; + text-align: center; + justify-content: center; + color: red; + } + + .container { + display: block; + position: relative; + margin-top: 10px; + margin-bottom: 10px; + font-size: 22px; + -webkit-user-select: none; + -moz-user-select: none; + -ms-user-select: none; + user-select: none; + width: 700px; + min-width: 700px; + max-width: 700px; + text-align: left; + font-style: normal; + font-size: 15; + font-weight: normal; + padding: 0; + + .padding { + padding: 0 0 0 20px; + color: black; + } + + .permissions-dropdown { + outline: none; + } + } + + .no-users { + margin-top: 20px; + } + + .link-container { + display: flex; + flex-direction: row; + margin-bottom: 10px; + margin-left: auto; + margin-right: auto; + + .link-box, + .copy { + padding: 10px; + border-radius: 10px; + padding: 10px; + border: solid black 1px; + } + + .link-box { + background: white; + color: blue; + text-decoration: underline; + } + + .copy { + margin-left: 20px; + cursor: alias; + border-radius: 50%; + width: 42px; + height: 42px; + transition: 1.5s all ease; + padding-top: 12px; + } + } + + .close-button { + border-radius: 5px; + margin-top: 20px; + padding: 10px 0; + background: aliceblue; + transition: 0.5s ease all; + border: 1px solid; + border-color: aliceblue; + } + + .close-button:hover { + border-color: black; + } +} \ No newline at end of file diff --git a/src/client/util/SharingManager.tsx b/src/client/util/SharingManager.tsx new file mode 100644 index 000000000..72a4b4141 --- /dev/null +++ b/src/client/util/SharingManager.tsx @@ -0,0 +1,293 @@ +import { observable, runInAction, action, autorun } from "mobx"; +import * as React from "react"; +import MainViewModal from "../views/MainViewModal"; +import { CurrentUserUtils } from "../../server/authentication/models/current_user_utils"; +import { Doc, Opt } from "../../new_fields/Doc"; +import { DocServer } from "../DocServer"; +import { Cast, StrCast } from "../../new_fields/Types"; +import { listSpec } from "../../new_fields/Schema"; +import { List } from "../../new_fields/List"; +import { RouteStore } from "../../server/RouteStore"; +import * as RequestPromise from "request-promise"; +import { Utils } from "../../Utils"; +import "./SharingManager.scss"; +import { Id } from "../../new_fields/FieldSymbols"; +import { observer } from "mobx-react"; +import { MainView } from "../views/MainView"; +import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; +import { library } from '@fortawesome/fontawesome-svg-core'; +import * as fa from '@fortawesome/free-solid-svg-icons'; +import { DocumentView } from "../views/nodes/DocumentView"; +import { SelectionManager } from "./SelectionManager"; +import { DocumentManager } from "./DocumentManager"; +import { CollectionVideoView } from "../views/collections/CollectionVideoView"; +import { CollectionPDFView } from "../views/collections/CollectionPDFView"; +import { CollectionView } from "../views/collections/CollectionView"; + +library.add(fa.faCopy); + +export interface User { + email: string; + userDocumentId: string; +} + +export enum SharingPermissions { + None = "Not Shared", + View = "Can View", + Comment = "Can Comment", + Edit = "Can Edit" +} + +const ColorMapping = new Map([ + [SharingPermissions.None, "red"], + [SharingPermissions.View, "maroon"], + [SharingPermissions.Comment, "blue"], + [SharingPermissions.Edit, "green"] +]); + +const SharingKey = "sharingPermissions"; +const PublicKey = "publicLinkPermissions"; +const DefaultColor = "black"; + +@observer +export default class SharingManager extends React.Component<{}> { + public static Instance: SharingManager; + @observable private isOpen = false; + @observable private users: User[] = []; + @observable private targetDoc: Doc | undefined; + @observable private targetDocView: DocumentView | undefined; + @observable private copied = false; + @observable private dialogueBoxOpacity = 1; + @observable private overlayOpacity = 0.4; + + private get linkVisible() { + return this.sharingDoc ? this.sharingDoc[PublicKey] !== SharingPermissions.None : false; + } + + public open = (target: DocumentView) => { + SelectionManager.DeselectAll(); + this.populateUsers().then(action(() => { + this.targetDocView = target; + this.targetDoc = target.props.Document; + MainView.Instance.hasActiveModal = true; + this.isOpen = true; + if (!this.sharingDoc) { + this.sharingDoc = new Doc; + } + })); + } + + public close = action(() => { + this.isOpen = false; + setTimeout(action(() => { + this.copied = false; + MainView.Instance.hasActiveModal = false; + this.targetDoc = undefined; + }), 500); + }); + + private get sharingDoc() { + return this.targetDoc ? Cast(this.targetDoc[SharingKey], Doc) as Doc : undefined; + } + + private set sharingDoc(value: Doc | undefined) { + this.targetDoc && (this.targetDoc[SharingKey] = value); + } + + constructor(props: {}) { + super(props); + SharingManager.Instance = this; + } + + populateUsers = async () => { + let userList = await RequestPromise.get(Utils.prepend(RouteStore.getUsers)); + runInAction(() => { + this.users = (JSON.parse(userList) as User[]).filter(({ email }) => email !== CurrentUserUtils.email); + }); + } + + setInternalSharing = async (user: User, state: string) => { + if (!this.sharingDoc) { + console.log("SHARING ABORTED!"); + return; + } + let sharingDoc = await this.sharingDoc; + sharingDoc[user.userDocumentId] = state; + const userDocument = await DocServer.GetRefField(user.userDocumentId); + if (!(userDocument instanceof Doc)) { + console.log(`Couldn't get user document of user ${user.email}`); + return; + } + let target = this.targetDoc; + if (!target) { + console.log("SharingManager trying to share an undefined document!!"); + return; + } + const notifDoc = await Cast(userDocument.optionalRightCollection, Doc); + if (notifDoc instanceof Doc) { + const data = await Cast(notifDoc.data, listSpec(Doc)); + if (!data) { + console.log("UNABLE TO ACCESS NOTIFICATION DATA"); + return; + } + console.log(`Attempting to set permissions to ${state} for the document ${target[Id]}`); + if (state !== SharingPermissions.None) { + const sharedDoc = Doc.MakeAlias(target); + if (data) { + data.push(sharedDoc); + } else { + notifDoc.data = new List([sharedDoc]); + } + } else { + let dataDocs = (await Promise.all(data.map(doc => doc))).map(doc => Doc.GetProto(doc)); + if (dataDocs.includes(target)) { + console.log("Searching in ", dataDocs, "for", target); + dataDocs.splice(dataDocs.indexOf(target), 1); + console.log("SUCCESSFULLY UNSHARED DOC"); + } else { + console.log("DIDN'T THINK WE HAD IT, SO NOT SUCCESSFULLY UNSHARED"); + } + } + } + } + + private setExternalSharing = (state: string) => { + let sharingDoc = this.sharingDoc; + if (!sharingDoc) { + return; + } + sharingDoc[PublicKey] = state; + } + + private get sharingUrl() { + if (!this.targetDoc) { + return undefined; + } + let baseUrl = Utils.prepend("/doc/" + this.targetDoc[Id]); + return `${baseUrl}?sharing=true`; + } + + copy = action(() => { + if (this.sharingUrl) { + Utils.CopyText(this.sharingUrl); + this.copied = true; + } + }); + + private get sharingOptions() { + return Object.values(SharingPermissions).map(permission => { + return ( + + ); + }); + } + + private focusOn = (contents: string) => { + let title = this.targetDoc ? StrCast(this.targetDoc.title) : ""; + return ( + { + let context: Opt; + if (this.targetDoc && this.targetDocView && (context = this.targetDocView.props.ContainingCollectionView)) { + DocumentManager.Instance.jumpToDocument(this.targetDoc, true, undefined, undefined, undefined, context.props.Document); + } + }} + onPointerEnter={action(() => { + if (this.targetDoc) { + Doc.BrushDoc(this.targetDoc); + this.dialogueBoxOpacity = 0.1; + this.overlayOpacity = 0.1; + } + })} + onPointerLeave={action(() => { + this.targetDoc && Doc.UnBrushDoc(this.targetDoc); + this.dialogueBoxOpacity = 1; + this.overlayOpacity = 0.4; + })} + > + {contents} + + ); + } + + private get sharingInterface() { + return ( +
+

Manage the public link to {this.focusOn("this document...")}

+ {!this.linkVisible ? (null) : +
+
{this.sharingUrl}
+
+ +
+
+ } +
+ {!this.linkVisible ? (null) :

People with this link

} + +
+
+

Privately share {this.focusOn("this document")} with an individual...

+
+ {!this.users.length ? "There are no other users in your database." : + this.users.map(user => { + return ( +
+ + {user.email} +
+ ); + }) + } +
+
Done
+
+ ); + } + + render() { + return ( + + ); + } + +} \ No newline at end of file diff --git a/src/client/views/GlobalKeyHandler.ts b/src/client/views/GlobalKeyHandler.ts index d0464bd5f..0255ab78a 100644 --- a/src/client/views/GlobalKeyHandler.ts +++ b/src/client/views/GlobalKeyHandler.ts @@ -6,6 +6,7 @@ import { DragManager } from "../util/DragManager"; import { action, runInAction } from "mobx"; import { Doc } from "../../new_fields/Doc"; import { DictationManager } from "../util/DictationManager"; +import SharingManager from "../util/SharingManager"; const modifiers = ["control", "meta", "shift", "alt"]; type KeyHandler = (keycode: string, e: KeyboardEvent) => KeyControlInfo | Promise; @@ -72,6 +73,7 @@ export default class KeyManager { main.toggleColorPicker(true); SelectionManager.DeselectAll(); DictationManager.Controls.stop(); + SharingManager.Instance.close(); break; case "delete": case "backspace": diff --git a/src/client/views/InkingCanvas.scss b/src/client/views/InkingCanvas.scss index 5437b26d6..1365974dd 100644 --- a/src/client/views/InkingCanvas.scss +++ b/src/client/views/InkingCanvas.scss @@ -34,7 +34,7 @@ .inkingCanvas-noSelect { pointer-events: none; - cursor: "arrow"; + cursor: "crosshair"; } .inkingCanvas-paths-ink, diff --git a/src/client/views/Main.scss b/src/client/views/Main.scss index bc0975c86..04249506a 100644 --- a/src/client/views/Main.scss +++ b/src/client/views/Main.scss @@ -268,44 +268,4 @@ ul#add-options-list { height: 25%; position: relative; display: flex; -} - -.dictation-prompt { - position: absolute; - z-index: 1000; - text-align: center; - justify-content: center; - align-self: center; - align-content: center; - padding: 20px; - background: gainsboro; - border-radius: 10px; - border: 3px solid black; - box-shadow: #00000044 5px 5px 10px; - transform: translate(-50%, -50%); - top: 50%; - font-style: italic; - left: 50%; - transition: 0.5s all ease; - pointer-events: none; -} - -.dictation-prompt-overlay { - width: 100%; - height: 100%; - position: absolute; - z-index: 999; - transition: 0.5s all ease; - pointer-events: none; -} - -.webpage-input { - display: none; - height: 60px; - width: 600px; - position: absolute; - - .url-input { - width: 80%; - } } \ No newline at end of file diff --git a/src/client/views/Main.tsx b/src/client/views/Main.tsx index e35ba18e4..b623cab4e 100644 --- a/src/client/views/Main.tsx +++ b/src/client/views/Main.tsx @@ -52,12 +52,16 @@ let swapDocs = async () => { const info = await CurrentUserUtils.loadCurrentUser(); DocServer.init(window.location.protocol, window.location.hostname, 4321, info.email); await Docs.Prototypes.initialize(); - await CurrentUserUtils.loadUserDocument(info); - // updates old user documents to prevent chrome on tree view. - (await Cast(CurrentUserUtils.UserDocument.workspaces, Doc))!.chromeStatus = "disabled"; - (await Cast(CurrentUserUtils.UserDocument.recentlyClosed, Doc))!.chromeStatus = "disabled"; - (await Cast(CurrentUserUtils.UserDocument.sidebar, Doc))!.chromeStatus = "disabled"; - await swapDocs(); + if (info.id !== "__guest__") { + // a guest will not have an id registered + await CurrentUserUtils.loadUserDocument(info); + // updates old user documents to prevent chrome on tree view. + (await Cast(CurrentUserUtils.UserDocument.workspaces, Doc))!.chromeStatus = "disabled"; + (await Cast(CurrentUserUtils.UserDocument.recentlyClosed, Doc))!.chromeStatus = "disabled"; + (await Cast(CurrentUserUtils.UserDocument.sidebar, Doc))!.chromeStatus = "disabled"; + CurrentUserUtils.UserDocument.chromeStatus = "disabled"; + await swapDocs(); + } document.getElementById('root')!.addEventListener('wheel', event => { if (event.ctrlKey) { event.preventDefault(); diff --git a/src/client/views/MainView.tsx b/src/client/views/MainView.tsx index 28edf181b..85bf0344b 100644 --- a/src/client/views/MainView.tsx +++ b/src/client/views/MainView.tsx @@ -7,8 +7,8 @@ import "normalize.css"; import * as React from 'react'; import { SketchPicker } from 'react-color'; import Measure from 'react-measure'; -import { Doc, DocListCast, Opt, HeightSym } from '../../new_fields/Doc'; import { List } from '../../new_fields/List'; +import { Doc, DocListCast, Opt, HeightSym, FieldResult, Field } from '../../new_fields/Doc'; import { Id } from '../../new_fields/FieldSymbols'; import { InkTool } from '../../new_fields/InkField'; import { listSpec } from '../../new_fields/Schema'; @@ -17,14 +17,14 @@ import { CurrentUserUtils } from '../../server/authentication/models/current_use import { RouteStore } from '../../server/RouteStore'; import { emptyFunction, returnOne, returnTrue, Utils, returnEmptyString, PostToServer } from '../../Utils'; import { DocServer } from '../DocServer'; -import { Docs } from '../documents/Documents'; import { ClientUtils } from '../util/ClientUtils'; import { DictationManager } from '../util/DictationManager'; import { SetupDrag } from '../util/DragManager'; -import { HistoryUtil } from '../util/History'; import { Transform } from '../util/Transform'; import { UndoManager, undoBatch } from '../util/UndoManager'; -import { CollectionBaseView } from './collections/CollectionBaseView'; +import { Docs, DocumentOptions } from '../documents/Documents'; +import { HistoryUtil } from '../util/History'; +import { CollectionBaseView, CollectionViewType } from './collections/CollectionBaseView'; import { CollectionDockingView } from './collections/CollectionDockingView'; import { CollectionTreeView } from './collections/CollectionTreeView'; import { ContextMenu } from './ContextMenu'; @@ -44,6 +44,9 @@ import { GooglePhotos } from '../apis/google_docs/GooglePhotosClientUtils'; import { ImageField } from '../../new_fields/URLField'; import { LinkFollowBox } from './linking/LinkFollowBox'; import { DocumentManager } from '../util/DocumentManager'; +import { SchemaHeaderField, RandomPastel } from '../../new_fields/SchemaHeaderField'; +import MainViewModal from './MainViewModal'; +import SharingManager from '../util/SharingManager'; @observer export class MainView extends React.Component { @@ -57,6 +60,8 @@ export class MainView extends React.Component { @observable private dictationDisplayState = false; @observable private dictationListeningState: DictationManager.Controls.ListeningUIStatus = false; + public hasActiveModal = false; + public overlayTimeout: NodeJS.Timeout | undefined; public initiateDictationFade = () => { @@ -64,10 +69,17 @@ export class MainView extends React.Component { this.overlayTimeout = setTimeout(() => { this.dictationOverlayVisible = false; this.dictationSuccess = undefined; + this.hasActiveModal = false; setTimeout(() => this.dictatedPhrase = DictationManager.placeholder, 500); }, duration); } + private urlState: HistoryUtil.DocUrl; + + @computed private get userDoc() { + return CurrentUserUtils.UserDocument; + } + public cancelDictationFade = () => { if (this.overlayTimeout) { clearTimeout(this.overlayTimeout); @@ -76,7 +88,7 @@ export class MainView extends React.Component { } @computed private get mainContainer(): Opt { - return FieldValue(Cast(CurrentUserUtils.UserDocument.activeWorkspace, Doc)); + return this.userDoc ? FieldValue(Cast(this.userDoc.activeWorkspace, Doc)) : CurrentUserUtils.GuestWorkspace; } @computed get mainFreeform(): Opt { let docs = DocListCast(this.mainContainer!.data); @@ -85,7 +97,10 @@ export class MainView extends React.Component { public isPointerDown = false; private set mainContainer(doc: Opt) { if (doc) { - CurrentUserUtils.UserDocument.activeWorkspace = doc; + if (!("presentationView" in doc)) { + doc.presentationView = new List([Docs.Create.TreeDocument([], { title: "Presentation" })]); + } + this.userDoc ? (this.userDoc.activeWorkspace = doc) : (CurrentUserUtils.GuestWorkspace = doc); } } @@ -130,23 +145,23 @@ export class MainView extends React.Component { window.removeEventListener("keydown", KeyManager.Instance.handle); window.addEventListener("keydown", KeyManager.Instance.handle); - this.executeGooglePhotosRoutine(); - - reaction(() => { - let workspaces = CurrentUserUtils.UserDocument.workspaces; - let recent = CurrentUserUtils.UserDocument.recentlyClosed; - if (!(recent instanceof Doc)) return 0; - if (!(workspaces instanceof Doc)) return 0; - let workspacesDoc = workspaces; - let recentDoc = recent; - let libraryHeight = this.getPHeight() - workspacesDoc[HeightSym]() - recentDoc[HeightSym]() - 20 + CurrentUserUtils.UserDocument[HeightSym]() * 0.00001; - return libraryHeight; - }, (libraryHeight: number) => { - if (libraryHeight && Math.abs(CurrentUserUtils.UserDocument[HeightSym]() - libraryHeight) > 5) { - CurrentUserUtils.UserDocument.height = libraryHeight; - } - (Cast(CurrentUserUtils.UserDocument.recentlyClosed, Doc) as Doc).allowClear = true; - }, { fireImmediately: true }); + if (this.userDoc) { + reaction(() => { + let workspaces = this.userDoc.workspaces; + let recent = this.userDoc.recentlyClosed; + if (!(recent instanceof Doc)) return 0; + if (!(workspaces instanceof Doc)) return 0; + let workspacesDoc = workspaces; + let recentDoc = recent; + let libraryHeight = this.getPHeight() - workspacesDoc[HeightSym]() - recentDoc[HeightSym]() - 20 + this.userDoc[HeightSym]() * 0.00001; + return libraryHeight; + }, (libraryHeight: number) => { + if (libraryHeight && Math.abs(this.userDoc[HeightSym]() - libraryHeight) > 5) { + this.userDoc.height = libraryHeight; + } + (Cast(this.userDoc.recentlyClosed, Doc) as Doc).allowClear = true; + }, { fireImmediately: true }); + } } executeGooglePhotosRoutine = async () => { @@ -169,7 +184,7 @@ export class MainView extends React.Component { constructor(props: Readonly<{}>) { super(props); MainView.Instance = this; - + this.urlState = HistoryUtil.parseUrl(window.location) || {} as any; // causes errors to be generated when modifying an observable outside of an action configure({ enforceActions: "observed" }); if (window.location.pathname !== RouteStore.home) { @@ -178,6 +193,12 @@ export class MainView extends React.Component { let type = pathname[0]; if (type === "doc") { CurrentUserUtils.MainDocId = pathname[1]; + if (!this.userDoc) { + runInAction(() => this.flyoutWidth = 0); + DocServer.GetRefField(CurrentUserUtils.MainDocId).then(action(field => { + field instanceof Doc && (CurrentUserUtils.GuestTarget = field); + })); + } } } } @@ -234,68 +255,109 @@ export class MainView extends React.Component { initAuthenticationRouters = async () => { // Load the user's active workspace, or create a new one if initial session after signup - if (!CurrentUserUtils.MainDocId) { - const doc = await Cast(CurrentUserUtils.UserDocument.activeWorkspace, Doc); - if (doc) { + let received = CurrentUserUtils.MainDocId; + if (received && !this.userDoc) { + reaction( + () => CurrentUserUtils.GuestTarget, + target => target && this.createNewWorkspace(), + { fireImmediately: true } + ); + } else { + if (received && this.urlState.sharing) { + reaction( + () => { + let docking = CollectionDockingView.Instance; + return docking && docking.initialized; + }, + initialized => { + if (initialized && received) { + DocServer.GetRefField(received).then(field => { + if (field instanceof Doc && field.viewType !== CollectionViewType.Docking) { + const target = Doc.MakeAlias(field); + const artificialParent = Docs.Create.FreeformDocument([target], { title: `View of ${StrCast(field.title)}` }); + CollectionDockingView.Instance.AddRightSplit(artificialParent, undefined); + DocumentManager.Instance.jumpToDocument(target, true, undefined, undefined, undefined, artificialParent); + } + }); + } + }, + ); + } + let doc: Opt; + if (this.userDoc && (doc = await Cast(this.userDoc.activeWorkspace, Doc))) { this.openWorkspace(doc); } else { this.createNewWorkspace(); } - } else { - DocServer.GetRefField(CurrentUserUtils.MainDocId).then(field => - field instanceof Doc ? this.openWorkspace(field) : - this.createNewWorkspace(CurrentUserUtils.MainDocId)); } } - @action createNewWorkspace = async (id?: string) => { - let workspaces = Cast(CurrentUserUtils.UserDocument.workspaces, Doc); - if (!(workspaces instanceof Doc)) return; - const list = Cast((CurrentUserUtils.UserDocument.workspaces as Doc).data, listSpec(Doc)); - if (list) { - let freeformDoc = Docs.Create.FreeformDocument([], { x: 0, y: 400, width: this.pwidth * .7, height: this.pheight, title: `WS collection ${list.length + 1}` }); - var dockingLayout = { content: [{ type: 'row', content: [CollectionDockingView.makeDocumentConfig(freeformDoc, freeformDoc, 600)] }] }; - let mainDoc = Docs.Create.DockDocument([CurrentUserUtils.UserDocument, freeformDoc], JSON.stringify(dockingLayout), { title: `Workspace ${list.length + 1}` }, id); - if (!CurrentUserUtils.UserDocument.linkManagerDoc) { - let linkManagerDoc = new Doc(); - linkManagerDoc.allLinks = new List([]); - CurrentUserUtils.UserDocument.linkManagerDoc = linkManagerDoc; + let freeformOptions: DocumentOptions = { + x: 0, + y: 400, + width: this.pwidth * .7, + height: this.pheight, + title: CurrentUserUtils.GuestTarget ? `Guest View of ${StrCast(CurrentUserUtils.GuestTarget.title)}` : "My Blank Collection" + }; + let workspaces: FieldResult; + let freeformDoc = CurrentUserUtils.GuestTarget || Docs.Create.FreeformDocument([], freeformOptions); + var dockingLayout = { content: [{ type: 'row', content: [CollectionDockingView.makeDocumentConfig(freeformDoc, freeformDoc, 600)] }] }; + let mainDoc = Docs.Create.DockDocument([this.userDoc, freeformDoc], JSON.stringify(dockingLayout), {}, id); + if (this.userDoc && ((workspaces = Cast(this.userDoc.workspaces, Doc)) instanceof Doc)) { + const list = Cast((workspaces).data, listSpec(Doc)); + if (list) { + if (!this.userDoc.linkManagerDoc) { + let linkManagerDoc = new Doc(); + linkManagerDoc.allLinks = new List([]); + this.userDoc.linkManagerDoc = linkManagerDoc; + } + list.push(mainDoc); + mainDoc.title = `Workspace ${list.length}`; } - list.push(mainDoc); - // bcz: strangely, we need a timeout to prevent exceptions/issues initializing GoldenLayout (the rendering engine for Main Container) - setTimeout(() => { - this.openWorkspace(mainDoc); - // let pendingDocument = Docs.StackingDocument([], { title: "New Mobile Uploads" }); - // mainDoc.optionalRightCollection = pendingDocument; - }, 0); } + // bcz: strangely, we need a timeout to prevent exceptions/issues initializing GoldenLayout (the rendering engine for Main Container) + setTimeout(() => { + this.openWorkspace(mainDoc); + // let pendingDocument = Docs.StackingDocument([], { title: "New Mobile Uploads" }); + // mainDoc.optionalRightCollection = pendingDocument; + }, 0); } @action openWorkspace = async (doc: Doc, fromHistory = false) => { CurrentUserUtils.MainDocId = doc[Id]; this.mainContainer = doc; - const state = HistoryUtil.parseUrl(window.location) || {} as any; - fromHistory || HistoryUtil.pushState({ type: "doc", docId: doc[Id], readonly: state.readonly, nro: state.nro }); - if (state.readonly === true || state.readonly === null) { + let state = this.urlState; + if (state.sharing === true && !this.userDoc) { DocServer.Control.makeReadOnly(); - } else if (state.safe) { - if (!state.nro) { + } else { + fromHistory || HistoryUtil.pushState({ + type: "doc", + docId: doc[Id], + readonly: state.readonly, + nro: state.nro, + sharing: false, + }); + if (state.readonly === true || state.readonly === null) { + DocServer.Control.makeReadOnly(); + } else if (state.safe) { + if (!state.nro) { + DocServer.Control.makeReadOnly(); + } + CollectionBaseView.SetSafeMode(true); + } else if (state.nro || state.nro === null || state.readonly === false) { + } else if (BoolCast(doc.readOnly)) { DocServer.Control.makeReadOnly(); + } else { + DocServer.Control.makeEditable(); } - CollectionBaseView.SetSafeMode(true); - } else if (state.nro || state.nro === null || state.readonly === false) { - } else if (BoolCast(doc.readOnly)) { - DocServer.Control.makeReadOnly(); - } else { - DocServer.Control.makeEditable(); } - const col = await Cast(CurrentUserUtils.UserDocument.optionalRightCollection, Doc); + let col: Opt; // if there is a pending doc, and it has new data, show it (syip: we use a timeout to prevent collection docking view from being uninitialized) setTimeout(async () => { - if (col) { + if (this.userDoc && (col = await Cast(this.userDoc.optionalRightCollection, Doc))) { const l = Cast(col.data, listSpec(Doc)); if (l) { runInAction(() => CollectionTreeView.NotifsCol = col); @@ -389,11 +451,12 @@ export class MainView extends React.Component { } @computed get flyout() { - let sidebar = CurrentUserUtils.UserDocument.sidebar; - if (!(sidebar instanceof Doc)) return (null); - let sidebarDoc = sidebar; + let sidebar: FieldResult; + if (!this.userDoc || !((sidebar = this.userDoc.sidebar) instanceof Doc)) { + return (null); + } return + if (!this.userDoc) { + return
{this.dockingContent}
; + } + let sidebar = this.userDoc.sidebar; + if (!(sidebar instanceof Doc)) { + return (null); + } + return
@@ -448,14 +516,22 @@ export class MainView extends React.Component { } } - toggleLinkFollowBox = (shouldClose: boolean) => { - if (LinkFollowBox.Instance) { - let dvs = DocumentManager.Instance.getDocumentViews(LinkFollowBox.Instance.props.Document); - // if it already exisits, close it - LinkFollowBox.Instance.props.Document.isMinimized = (dvs.length > 0 && shouldClose); - } + setWriteMode = (mode: DocServer.WriteMode) => { + console.log(DocServer.WriteMode[mode]); + const mode1 = mode; + const mode2 = mode === DocServer.WriteMode.Default ? mode : DocServer.WriteMode.Playground; + DocServer.setFieldWriteMode("x", mode1); + DocServer.setFieldWriteMode("y", mode1); + DocServer.setFieldWriteMode("width", mode1); + DocServer.setFieldWriteMode("height", mode1); + + DocServer.setFieldWriteMode("panX", mode2); + DocServer.setFieldWriteMode("panY", mode2); + DocServer.setFieldWriteMode("scale", mode2); + DocServer.setFieldWriteMode("viewType", mode2); } + @observable private _colorPickerDisplay = false; /* for the expandable add nodes menu. Not included with the miscbuttons because once it expands it expands the whole div with it, making canvas interactions limited. */ nodesMenu() { @@ -501,7 +577,13 @@ export class MainView extends React.Component {
)} -
    • +
    • + {ClientUtils.RELEASE ? [] : [ +
    • , +
    • , +
    • , +
    • + ]}
  • ; + } else { + return ; + } + } + + //The function that starts or resets presentaton functionally, depending on status flag. + @action + startOrResetPres = async () => { + if (this.presStatus) { + this.resetPresentation(); + } else { + this.presStatus = true; + let startIndex = await this.findStartDocument(); + this.startPresentation(startIndex); + const current = NumCast(this.curPresentation.selectedDoc); + this.gotoDocument(startIndex, current); + } + this.curPresentation.presStatus = this.presStatus; + } + + /** + * This method is called to find the start document of presentation. So + * that when user presses on play, the correct presentation element will be + * selected. + */ + findStartDocument = async () => { + let docAtZero = await this.getDocAtIndex(0); + if (docAtZero === undefined) { + return 0; + } + let docAtZeroPresId = StrCast(docAtZero.presentId); + + if (this.groupMappings.has(docAtZeroPresId)) { + let group = this.groupMappings.get(docAtZeroPresId)!; + let lastDoc = group[group.length - 1]; + return this.childrenDocs.indexOf(lastDoc); + } else { + return 0; + } + } + + //The function that resets the presentation by removing every action done by it. It also + //stops the presentaton. + @action + resetPresentation = () => { + this.childrenDocs.forEach((doc: Doc) => { + doc.opacity = 1; + doc.viewScale = 1; + }); + this.curPresentation.selectedDoc = 0; + this.presStatus = false; + this.curPresentation.presStatus = this.presStatus; + if (this.childrenDocs.length === 0) { + return; + } + DocumentManager.Instance.zoomIntoScale(this.childrenDocs[0], 1); + } + + + //The function that starts the presentation, also checking if actions should be applied + //directly at start. + startPresentation = (startIndex: number) => { + let selectedButtons: boolean[]; + this.presElementsMappings.forEach((component: PresentationElement, doc: Doc) => { + selectedButtons = component.selected; + if (selectedButtons[buttonIndex.HideTillPressed]) { + if (this.childrenDocs.indexOf(doc) > startIndex) { + doc.opacity = 0; + } + + } + if (selectedButtons[buttonIndex.HideAfter]) { + if (this.childrenDocs.indexOf(doc) < startIndex) { + doc.opacity = 0; + } + } + if (selectedButtons[buttonIndex.FadeAfter]) { + if (this.childrenDocs.indexOf(doc) < startIndex) { + doc.opacity = 0.5; + } + } + + }); + + } + + /** + * The function that is called to add a new presentation to the presentationView. + * It sets up te mappings and local copies of it. Resets the groupings and presentation. + * Makes the new presentation current selected, and retrieve the back-Ups if present. + */ + @action + addNewPresentation = (presTitle: string) => { + //creating a new presentation doc + let newPresentationDoc = Docs.Create.TreeDocument([], { title: presTitle }); + this.props.Documents.push(newPresentationDoc); + + //setting that new doc as current + this.curPresentation = newPresentationDoc; + + //storing the doc in local copies for easier access + let newGuid = Utils.GenerateGuid(); + this.presentationsMapping.set(newGuid, newPresentationDoc); + this.presentationsKeyMapping.set(newPresentationDoc, newGuid); + + //resetting the previous presentation's actions so that new presentation can be loaded. + this.resetGroupIds(); + this.resetPresentation(); + this.presElementsMappings = new Map(); + this.currentSelectedPresValue = newGuid; + this.setPresentationBackUps(); + + } + + /** + * The function that is called to change the current selected presentation. + * Changes the presentation, also resetting groupings and presentation in process. + * Plus retrieving the backUps for the newly selected presentation. + */ + @action + getSelectedPresentation = (e: React.ChangeEvent) => { + //get the guid of the selected presentation + let selectedGuid = e.target.value; + //set that as current presentation + this.curPresentation = this.presentationsMapping.get(selectedGuid)!; + + //reset current Presentations local things so that new one can be loaded + this.resetGroupIds(); + this.resetPresentation(); + this.presElementsMappings = new Map(); + this.currentSelectedPresValue = selectedGuid; + this.setPresentationBackUps(); + + + } + + /** + * The function that is called to render either select for presentations, or title inputting. + */ + renderSelectOrPresSelection = () => { + let presentationList = DocListCast(this.props.Documents); + if (this.PresTitleInputOpen || this.PresTitleChangeOpen) { + return this.titleInputElement = e!} type="text" className="presentationView-title" placeholder="Enter Name!" onKeyDown={this.submitPresentationTitle} />; + } else { + return ; + } + } + + /** + * The function that is called on enter press of title input. It gives the + * new presentation the title user entered. If nothing is entered, gives a default title. + */ + @action + submitPresentationTitle = (e: React.KeyboardEvent) => { + if (e.keyCode === 13) { + let presTitle = this.titleInputElement!.value; + this.titleInputElement!.value = ""; + if (this.PresTitleInputOpen) { + if (presTitle === "") { + presTitle = "Presentation"; + } + this.PresTitleInputOpen = false; + this.addNewPresentation(presTitle); + } else if (this.PresTitleChangeOpen) { + this.PresTitleChangeOpen = false; + this.changePresentationTitle(presTitle); + } + } + } + + /** + * The function that is called to remove a presentation from all its copies, and the main Container's + * list. Sets up the next presentation as current. + */ + @action + removePresentation = async () => { + if (this.presentationsMapping.size !== 1) { + let presentationList = Cast(this.props.Documents, listSpec(Doc)); + let batch = UndoManager.StartBatch("presRemoval"); + + //getting the presentation that will be removed + let removedDoc = this.presentationsMapping.get(this.currentSelectedPresValue!); + //that presentation is removed + presentationList!.splice(presentationList!.indexOf(removedDoc!), 1); + + //its mappings are removed from local copies + this.presentationsKeyMapping.delete(removedDoc!); + this.presentationsMapping.delete(this.currentSelectedPresValue!); + + //the next presentation is set as current + let remainingPresentations = this.presentationsMapping.values(); + let nextDoc = remainingPresentations.next().value; + this.curPresentation = nextDoc; + + + //Storing these for being able to undo changes + let curGuid = this.currentSelectedPresValue!; + let curPresStatus = this.presStatus; + + //resetting the groups and presentation actions so that next presentation gets loaded + this.resetGroupIds(); + this.resetPresentation(); + this.currentSelectedPresValue = this.presentationsKeyMapping.get(nextDoc)!.toString(); + this.setPresentationBackUps(); + + //Storing for undo + let currentGroups = this.groupMappings; + let curPresElemMapping = this.presElementsMappings; + + //Event to undo actions that are not related to doc directly, aka. local things + UndoManager.AddEvent({ + undo: action(() => { + this.curPresentation = removedDoc!; + this.presentationsMapping.set(curGuid, removedDoc!); + this.presentationsKeyMapping.set(removedDoc!, curGuid); + this.currentSelectedPresValue = curGuid; + + this.presStatus = curPresStatus; + this.groupMappings = currentGroups; + this.presElementsMappings = curPresElemMapping; + this.setPresentationBackUps(); + + }), + redo: action(() => { + this.curPresentation = nextDoc; + this.presStatus = false; + this.presentationsKeyMapping.delete(removedDoc!); + this.presentationsMapping.delete(curGuid); + this.currentSelectedPresValue = this.presentationsKeyMapping.get(nextDoc)!.toString(); + this.setPresentationBackUps(); + + }), + }); + + batch.end(); + } + } + + /** + * The function that is called to change title of presentation to what user entered. + */ + @undoBatch + changePresentationTitle = (newTitle: string) => { + if (newTitle === "") { + return; + } + this.curPresentation.title = newTitle; + } + + /** + * On pointer down element that is catched on resizer of te + * presentation view. Sets up the event listeners to change the size with + * mouse move. + */ + _downsize = 0; + onPointerDown = (e: React.PointerEvent) => { + this._downsize = e.clientX; + document.removeEventListener("pointermove", this.onPointerMove); + document.removeEventListener("pointerup", this.onPointerUp); + document.addEventListener("pointermove", this.onPointerMove); + document.addEventListener("pointerup", this.onPointerUp); + e.stopPropagation(); + e.preventDefault(); + } + /** + * Changes the size of the presentation view, with mouse move. + * Minimum size is set to 300, so that every button is visible. + */ + @action + onPointerMove = (e: PointerEvent) => { + + this.curPresentation.width = Math.max(window.innerWidth - e.clientX, presMinWidth); + } + + /** + * The method that is called on pointer up event. It checks if the button is just + * clicked so that presentation view will be closed. The way it's done is to check + * for minimal pixel change like 4, and accept it as it's just a click on top of the dragger. + */ + @action + onPointerUp = (e: PointerEvent) => { + if (Math.abs(e.clientX - this._downsize) < 4) { + let presWidth = NumCast(this.curPresentation.width); + if (presWidth - presMinWidth !== 0) { + this.curPresentation.width = 0; + } + if (presWidth === 0) { + this.curPresentation.width = presMinWidth; + } + } + document.removeEventListener("pointermove", this.onPointerMove); + document.removeEventListener("pointerup", this.onPointerUp); + } + + /** + * This function is a setter that opens up the + * presentation mode, by setting it's render flag + * to true. It also closes the presentation view. + */ + @action + openPresMode = () => { + if (!this.presMode) { + this.curPresentation.width = 0; + this.presMode = true; + } + } + + /** + * This function closes the presentation mode by setting its + * render flag to false. It also opens up the presentation view. + * By setting it to it's minimum size. + */ + @action + closePresMode = () => { + if (this.presMode) { + this.presMode = false; + this.curPresentation.width = presMinWidth; + } + + } + + /** + * Function that is called to render the presentation mode, depending on its flag. + */ + renderPresMode = () => { + if (this.presMode) { + return ; + } else { + return (null); + } + + } + + render() { + + let width = NumCast(this.curPresentation.width); + + return ( +
    +
    !this.persistOpacity && (this.opacity = 1))} onPointerLeave={action(() => !this.persistOpacity && (this.opacity = 0.4))} style={{ width: width, overflowY: "scroll", overflowX: "hidden", opacity: this.opacity, transition: "0.7s opacity ease" }}> +
    + {this.renderSelectOrPresSelection()} + + + + + +
    +
    + + {this.renderPlayPauseButton()} + +
    + + this.presElementsMappings.clear()} + /> + ) => { + this.persistOpacity = e.target.checked; + this.opacity = this.persistOpacity ? 1 : 0.4; + })} + checked={this.persistOpacity} + style={{ position: "absolute", bottom: 5, left: 5 }} + onPointerEnter={action(() => this.labelOpacity = 1)} + onPointerLeave={action(() => this.labelOpacity = 0)} + /> +

    opacity {this.persistOpacity ? "persistent" : "on focus"}

    +
    +
    + +
    + {this.renderPresMode()} + +
    + ); + } +} diff --git a/src/server/Message.ts b/src/server/Message.ts index 4ec390ade..a5679797f 100644 --- a/src/server/Message.ts +++ b/src/server/Message.ts @@ -23,6 +23,7 @@ export interface Transferable { readonly id: string; readonly type: Types; readonly data?: any; + readonly mongoCollection?: string; } export enum YoutubeQueryTypes { @@ -43,6 +44,10 @@ export interface Diff extends Reference { readonly diff: any; } +export interface SourceSpecified extends Reference { + readonly mongoCollection?: string; +} + export namespace MessageStore { export const Foo = new Message("Foo"); export const Bar = new Message("Bar"); @@ -52,7 +57,7 @@ export namespace MessageStore { export const GetDocument = new Message("Get Document"); export const DeleteAll = new Message("Delete All"); - export const GetRefField = new Message("Get Ref Field"); + export const GetRefField = new Message("Get Ref Field"); export const GetRefFields = new Message("Get Ref Fields"); export const UpdateField = new Message("Update Ref Field"); export const CreateField = new Message("Create Ref Field"); diff --git a/src/server/apis/google/GooglePhotosUploadUtils.ts b/src/server/apis/google/GooglePhotosUploadUtils.ts index c2656cc1c..0215c533f 100644 --- a/src/server/apis/google/GooglePhotosUploadUtils.ts +++ b/src/server/apis/google/GooglePhotosUploadUtils.ts @@ -58,8 +58,6 @@ export namespace GooglePhotosUploadUtils { })); }; - - export const CreateMediaItems = async (newMediaItems: any[], album?: { id: string }): Promise => { const quota = newMediaItems.length; let handled = 0; diff --git a/src/server/authentication/models/current_user_utils.ts b/src/server/authentication/models/current_user_utils.ts index af5774ebe..050a71eb4 100644 --- a/src/server/authentication/models/current_user_utils.ts +++ b/src/server/authentication/models/current_user_utils.ts @@ -2,7 +2,6 @@ import { action, computed, observable, runInAction } from "mobx"; import * as rp from 'request-promise'; import { DocServer } from "../../../client/DocServer"; import { Docs } from "../../../client/documents/Documents"; -import { Gateway, NorthstarSettings } from "../../../client/northstar/manager/Gateway"; import { Attribute, AttributeGroup, Catalog, Schema } from "../../../client/northstar/model/idea/idea"; import { ArrayUtil } from "../../../client/northstar/utils/ArrayUtil"; import { CollectionViewType } from "../../../client/views/collections/CollectionBaseView"; @@ -24,6 +23,9 @@ export class CurrentUserUtils { public static get MainDocId() { return this.mainDocId; } public static set MainDocId(id: string | undefined) { this.mainDocId = id; } + @observable public static GuestTarget: Doc | undefined; + @observable public static GuestWorkspace: Doc | undefined; + private static createUserDocument(id: string): Doc { let doc = new Doc(id, true); doc.viewType = CollectionViewType.Tree; @@ -59,7 +61,7 @@ export class CurrentUserUtils { noteTypes.excludeFromLibrary = true; doc.noteTypes = noteTypes; } - PromiseValue(Cast(doc.noteTypes, Doc)).then(noteTypes => noteTypes && PromiseValue(noteTypes.data).then(vals => DocListCast(vals))); + PromiseValue(Cast(doc.noteTypes, Doc)).then(noteTypes => noteTypes && PromiseValue(noteTypes.data).then(DocListCast)); if (doc.recentlyClosed === undefined) { const recentlyClosed = Docs.Create.TreeDocument([], { title: "Recently Closed", height: 75 }); recentlyClosed.excludeFromLibrary = true; @@ -112,7 +114,7 @@ export class CurrentUserUtils { this.curr_id = id; Doc.CurrentUserEmail = email; await rp.get(Utils.prepend(RouteStore.getUserDocumentId)).then(id => { - if (id) { + if (id && id !== "guest") { return DocServer.GetRefField(id).then(async field => { if (field instanceof Doc) { await this.updateUserDocument(field); diff --git a/src/server/credentials/google_docs_token.json b/src/server/credentials/google_docs_token.json index fec1625f5..31763c2cf 100644 --- a/src/server/credentials/google_docs_token.json +++ b/src/server/credentials/google_docs_token.json @@ -1 +1 @@ -{"access_token":"ya29.GlyBB1MlCG7GL2pYFleLp9uUJoN6s0_PFBDLUIhyrKAY4kkVo7vbuaW_zmkJs1Fym0f7NVpaYvFsBK2dbN6Qn5P8bWNW2NsHNNGcwbyGIS8H52GUlyCsawNt6PTnOw","refresh_token":"1/HTv_xFHszu2Nf3iiFrUTaeKzC_Vp2-6bpIB06xW_WHI","scope":"https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/documents.readonly https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/photoslibrary https://www.googleapis.com/auth/photoslibrary.appendonly https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/photoslibrary.sharing","token_type":"Bearer","expiry_date":1568274162450} \ No newline at end of file +{"access_token":"ya29.GlyBB9YYhy7l9LZ9yDpItKvLpibt59SpmBQUMo_sX-3d4eN8W-9teuc_7Ca4YiOboy_gHTdcwaR1ArnpQEqZlzOsfNmV6dXZsldgxin3bVuDn1q4sCWvz01yuZduIA","refresh_token":"1/HTv_xFHszu2Nf3iiFrUTaeKzC_Vp2-6bpIB06xW_WHI","scope":"https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/documents.readonly https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/photoslibrary https://www.googleapis.com/auth/photoslibrary.appendonly https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/photoslibrary.sharing","token_type":"Bearer","expiry_date":1568281677559} \ No newline at end of file diff --git a/src/server/index.ts b/src/server/index.ts index 101a4f63f..62c3df8de 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -21,10 +21,10 @@ import * as wdm from 'webpack-dev-middleware'; import * as whm from 'webpack-hot-middleware'; import { Utils } from '../Utils'; import { getForgot, getLogin, getLogout, getReset, getSignup, postForgot, postLogin, postReset, postSignup } from './authentication/controllers/user_controller'; -import { DashUserModel } from './authentication/models/user_model'; +import User, { DashUserModel } from './authentication/models/user_model'; import { Client } from './Client'; import { Database } from './database'; -import { MessageStore, Transferable, Types, Diff, YoutubeQueryTypes as YoutubeQueryType, YoutubeQueryInput } from "./Message"; +import { MessageStore, Transferable, Types, Diff, YoutubeQueryTypes as YoutubeQueryType, YoutubeQueryInput, SourceSpecified } from "./Message"; import { RouteStore } from './RouteStore'; import v4 = require('uuid/v4'); const app = express(); @@ -36,9 +36,7 @@ const serverPort = 4321; import expressFlash = require('express-flash'); import flash = require('connect-flash'); import { Search } from './Search'; -import _ = require('lodash'); import * as Archiver from 'archiver'; -import * as request_promise from 'request-promise'; var AdmZip = require('adm-zip'); import * as YoutubeApi from "./apis/youtube/youtubeApiSample"; import { Response } from 'express-serve-static-core'; @@ -47,6 +45,7 @@ import { GooglePhotosUploadUtils, DownloadUtils as UploadUtils } from './apis/go const MongoStore = require('connect-mongo')(session); const mongoose = require('mongoose'); const probe = require("probe-image-size"); +import * as qs from 'query-string'; const download = (url: string, dest: fs.PathLike) => request.get(url).pipe(fs.createWriteStream(dest)); let youtubeApiKey: string; @@ -113,7 +112,9 @@ function addSecureRoute(method: Method, ...subscribers: string[] ) { let abstracted = (req: express.Request, res: express.Response) => { - if (req.user) { + let sharing = qs.parse(qs.extract(req.originalUrl), { sort: false }).sharing === "true"; + sharing = sharing && req.originalUrl.startsWith("/doc/"); + if (req.user || sharing) { handler(req.user, res, req); } else { req.session!.target = req.originalUrl; @@ -507,21 +508,20 @@ addSecureRoute( res.sendFile(path.join(__dirname, '../../deploy/' + filename)); }, undefined, - RouteStore.home, - RouteStore.openDocumentWithId + RouteStore.home, RouteStore.openDocumentWithId ); addSecureRoute( Method.GET, - (user, res) => res.send(user.userDocumentId || ""), - undefined, + (user, res) => res.send(user.userDocumentId), + (res) => res.send(undefined), RouteStore.getUserDocumentId, ); addSecureRoute( Method.GET, - (user, res) => res.send(JSON.stringify({ id: user.id, email: user.email })), - undefined, + (user, res) => { res.send(JSON.stringify({ id: user.id, email: user.email })); }, + (res) => res.send(JSON.stringify({ id: "__guest__", email: "" })), RouteStore.getCurrUser ); @@ -666,21 +666,31 @@ app.use(RouteStore.corsProxy, (req, res) => { }).pipe(res); }); -app.get(RouteStore.delete, (req, res) => { - if (release) { - res.send("no"); - return; - } - deleteFields().then(() => res.redirect(RouteStore.home)); -}); +addSecureRoute( + Method.GET, + (user, res, req) => { + if (release) { + res.send("no"); + return; + } + deleteFields().then(() => res.redirect(RouteStore.home)); + }, + undefined, + RouteStore.delete +); -app.get(RouteStore.deleteAll, (req, res) => { - if (release) { - res.send("no"); - return; - } - deleteAll().then(() => res.redirect(RouteStore.home)); -}); +addSecureRoute( + Method.GET, + (user, res, req) => { + if (release) { + res.send("no"); + return; + } + deleteAll().then(() => res.redirect(RouteStore.home)); + }, + undefined, + RouteStore.deleteAll +); app.use(wdm(compiler, { publicPath: config.output.publicPath })); @@ -766,8 +776,8 @@ function setField(socket: Socket, newValue: Transferable) { } } -function GetRefField([id, callback]: [string, (result?: Transferable) => void]) { - Database.Instance.getDocument(id, callback, "newDocuments"); +function GetRefField([args, callback]: [SourceSpecified, (result?: Transferable) => void]) { + Database.Instance.getDocument(args.id, callback, args.mongoCollection || "newDocuments"); } function GetRefFields([ids, callback]: [string[], (result?: Transferable[]) => void]) { -- cgit v1.2.3-70-g09d2 From 3c2b04f16ccfae103e2f3acdd852e337c5f974e1 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Fri, 13 Sep 2019 17:11:25 -0400 Subject: added batching, generically --- src/client/northstar/utils/Extensions.ts | 27 ++++++++++ .../util/Import & Export/DirectoryImportBox.tsx | 33 ++++++------ src/client/util/UtilExtensions.ts | 39 +++++++++++++++ src/client/views/Main.tsx | 8 --- src/server/apis/google/GooglePhotosUploadUtils.ts | 58 +++++++++++----------- src/server/credentials/google_docs_token.json | 2 +- src/server/index.ts | 53 ++++++++++++++++---- 7 files changed, 155 insertions(+), 65 deletions(-) create mode 100644 src/client/util/UtilExtensions.ts (limited to 'src/server/apis') diff --git a/src/client/northstar/utils/Extensions.ts b/src/client/northstar/utils/Extensions.ts index ab9384f1f..720f4a062 100644 --- a/src/client/northstar/utils/Extensions.ts +++ b/src/client/northstar/utils/Extensions.ts @@ -5,6 +5,8 @@ interface String { hasNewline(): boolean; } +const extensions = require(".././/.//../util/UtilExtensions"); + String.prototype.ReplaceAll = function (toReplace: string, replacement: string): string { var target = this; return target.split(toReplace).join(replacement); @@ -18,6 +20,31 @@ String.prototype.Truncate = function (length: number, replacement: string): Stri return target; }; +interface Action { + handler: (batch: T[]) => any; + interval?: number; +} + +interface BatchParameters { + size: number; + action?: Action; +} + +interface Array { + batch(parameters: BatchParameters): Promise; + lastElement(): T; +} + +Array.prototype.batch = extensions.Batch; + +Array.prototype.lastElement = function () { + if (!this.length) { + return undefined; + } + const last: T = this[this.length - 1]; + return last; +}; + interface Math { log10(val: number): number; } diff --git a/src/client/util/Import & Export/DirectoryImportBox.tsx b/src/client/util/Import & Export/DirectoryImportBox.tsx index 260c6a629..5915f3412 100644 --- a/src/client/util/Import & Export/DirectoryImportBox.tsx +++ b/src/client/util/Import & Export/DirectoryImportBox.tsx @@ -92,29 +92,28 @@ export default class DirectoryImportBox extends React.Component this.completed = 0; }); - let sizes = []; - let modifiedDates = []; + let sizes: number[] = []; + let modifiedDates: number[] = []; - let i = 0; const uploads: FileResponse[] = []; - const batchSize = 15; - while (i < validated.length) { - const cap = Math.min(validated.length, i + batchSize); - let formData = new FormData(); - const batch = validated.slice(i, cap); + await validated.batch({ + size: 15, + action: { + handler: async (batch: File[]) => { + sizes.push(...batch.map(file => file.size)); + modifiedDates.push(...batch.map(file => file.lastModified)); - sizes.push(...batch.map(file => file.size)); - modifiedDates.push(...batch.map(file => file.lastModified)); + let formData = new FormData(); + batch.forEach(file => formData.append(Utils.GenerateGuid(), file)); + const parameters = { method: 'POST', body: formData }; - batch.forEach(file => formData.append(Utils.GenerateGuid(), file)); - const parameters = { method: 'POST', body: formData }; - uploads.push(...(await (await fetch(Utils.prepend(RouteStore.upload), parameters)).json())); - - runInAction(() => this.completed += batch.length); - i = cap; - } + uploads.push(...(await (await fetch(Utils.prepend(RouteStore.upload), parameters)).json())); + runInAction(() => this.completed += batch.length); + } + } + }); await Promise.all(uploads.map(async upload => { const type = upload.type; diff --git a/src/client/util/UtilExtensions.ts b/src/client/util/UtilExtensions.ts new file mode 100644 index 000000000..1e277b242 --- /dev/null +++ b/src/client/util/UtilExtensions.ts @@ -0,0 +1,39 @@ +module.exports.Batch = async function (parameters: BatchParameters) { + const { size, action } = parameters; + const batches: T[][] = []; + let i = 0; + while (i < this.length) { + const cap = Math.min(i + size, this.length); + batches.push(this.slice(i, cap)); + i = cap; + } + console.log(`Beginning action on ${this.length} elements, split into ${batches.length} groups => ${batches.map(batch => batch.length).join(", ")}`); + if (action) { + const { handler, interval } = action; + if (!interval || batches.length === 1) { + for (let batch of batches) { + await handler(batch); + } + } else { + return new Promise(resolve => { + const iterator = batches[Symbol.iterator](); + const quota = batches.length; + let completed = 0; + const tag = setInterval(async () => { + const next = iterator.next(); + if (next.done) { + clearInterval(tag); + return; + } + const batch = next.value; + console.log(`Handling next batch with ${batch.length} elements`); + await handler(batch); + if (++completed === quota) { + resolve(batches); + } + }, interval); + }); + } + } + return batches; +}; \ No newline at end of file diff --git a/src/client/views/Main.tsx b/src/client/views/Main.tsx index b623cab4e..aa002cee9 100644 --- a/src/client/views/Main.tsx +++ b/src/client/views/Main.tsx @@ -20,14 +20,6 @@ String.prototype.hasNewline = function () { return this.endsWith("\n"); }; -(Array.prototype as any).lastElement = function (this: any[]) { - if (!this.length) { - return undefined; - } - return this[this.length - 1]; -}; - - let swapDocs = async () => { let oldDoc = await Cast(CurrentUserUtils.UserDocument.linkManagerDoc, Doc); // Docs.Prototypes.MainLinkDocument().allLinks = new List(); diff --git a/src/server/apis/google/GooglePhotosUploadUtils.ts b/src/server/apis/google/GooglePhotosUploadUtils.ts index e91f8352b..3989590c6 100644 --- a/src/server/apis/google/GooglePhotosUploadUtils.ts +++ b/src/server/apis/google/GooglePhotosUploadUtils.ts @@ -6,6 +6,7 @@ import * as path from 'path'; import { Opt } from '../../../new_fields/Doc'; import * as sharp from 'sharp'; import { MediaItemCreationResult, NewMediaItemResult } from './SharedTypes'; +import { NewMediaItem } from '../..'; const uploadDirectory = path.join(__dirname, "../../public/files/"); @@ -39,7 +40,7 @@ export namespace GooglePhotosUploadUtils { }; export const DispatchGooglePhotosUpload = async (url: string) => { - const body = await request(url, { encoding: null }); + const body = await request(url, { encoding: null }).catch(error => console.log("Error in streaming body!", error)); const parameters = { method: 'POST', headers: { @@ -56,36 +57,37 @@ export namespace GooglePhotosUploadUtils { return reject(error); } resolve(body); - })); + }).catch(error => console.log("Error in literal uploading process to Google's servers!", error))).catch(error => console.log("Error in literal uploading process to Google's servers!", error)); }; - export const CreateMediaItems = async (newMediaItems: any[], album?: { id: string }): Promise => { - const quota = newMediaItems.length; - let handled = 0; + export const CreateMediaItems = async (newMediaItems: NewMediaItem[], album?: { id: string }): Promise => { const newMediaItemResults: NewMediaItemResult[] = []; - while (handled < quota) { - const cap = Math.min(newMediaItems.length, handled + 50); - const batch = newMediaItems.slice(handled, cap); - console.log(batch.length); - const parameters = { - method: 'POST', - headers: headers('json'), - uri: prepend('mediaItems:batchCreate'), - body: { newMediaItems: batch } as any, - json: true - }; - album && (parameters.body.albumId = album.id); - newMediaItemResults.push(...(await new Promise((resolve, reject) => { - request(parameters, (error, _response, body) => { - if (error) { - reject(error); - } else { - resolve(body); - } - }); - })).newMediaItemResults); - handled = cap; - } + await newMediaItems.batch({ + size: 50, + action: { + handler: async (batch: NewMediaItem[]) => { + console.log(batch.length); + const parameters = { + method: 'POST', + headers: headers('json'), + uri: prepend('mediaItems:batchCreate'), + body: { newMediaItems: batch } as any, + json: true + }; + album && (parameters.body.albumId = album.id); + newMediaItemResults.push(...(await new Promise((resolve, reject) => { + request(parameters, (error, _response, body) => { + if (error) { + reject(error); + } else { + resolve(body); + } + }); + })).newMediaItemResults); + }, + interval: 1000 + } + }); return { newMediaItemResults }; }; diff --git a/src/server/credentials/google_docs_token.json b/src/server/credentials/google_docs_token.json index bdeca837b..d8e0eae21 100644 --- a/src/server/credentials/google_docs_token.json +++ b/src/server/credentials/google_docs_token.json @@ -1 +1 @@ -{"access_token":"ya29.ImCCBwLh8M4qd5ApvvhgMeCvbQidOUehUNU2fj3RH6Zx8D3rnCooiVgxoWbJ2ddS3a0_PGAQvCA7-GAeS70wUny80VKgCLjNbTlZkuxaRqpAd5yFGuWzcRljXrEIuA7EVu0","refresh_token":"1/HTv_xFHszu2Nf3iiFrUTaeKzC_Vp2-6bpIB06xW_WHI","scope":"https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/documents.readonly https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/photoslibrary https://www.googleapis.com/auth/photoslibrary.appendonly https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/photoslibrary.sharing","token_type":"Bearer","expiry_date":1568394019509} \ No newline at end of file +{"access_token":"ya29.GlyDB0wsV3-oS6q5TFuJSmH1YP_SPf_X6RHaJVmfqj0NTCtaPLFonZRxdT52kUkiHJgAoRizxZvlSIGptXKfnmG4BFouhgyo9ZKP0QtOH-kPR9b9x5WhGCd5NWqz0A","refresh_token":"1/HTv_xFHszu2Nf3iiFrUTaeKzC_Vp2-6bpIB06xW_WHI","scope":"https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/documents.readonly https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/photoslibrary https://www.googleapis.com/auth/photoslibrary.appendonly https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/photoslibrary.sharing","token_type":"Bearer","expiry_date":1568412547334} \ No newline at end of file diff --git a/src/server/index.ts b/src/server/index.ts index fdcc79b4d..542a4ea65 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -46,6 +46,7 @@ const MongoStore = require('connect-mongo')(session); const mongoose = require('mongoose'); const probe = require("probe-image-size"); import * as qs from 'query-string'; +const extensions = require("../client/util/UtilExtensions"); const download = (url: string, dest: fs.PathLike) => request.get(url).pipe(fs.createWriteStream(dest)); let youtubeApiKey: string; @@ -827,20 +828,50 @@ app.get(RouteStore.googlePhotosAccessToken, (req, res) => GoogleApiServerUtils.R const tokenError = "Unable to successfully upload bytes for all images!"; const mediaError = "Unable to convert all uploaded bytes to media items!"; +export interface NewMediaItem { + description: string; + simpleMediaItem: { + uploadToken: string; + }; +} + +Array.prototype.batch = extensions.Batch; + app.post(RouteStore.googlePhotosMediaUpload, async (req, res) => { - const media: GooglePhotosUploadUtils.MediaInput[] = req.body.media; + const mediaInput: GooglePhotosUploadUtils.MediaInput[] = req.body.media; await GooglePhotosUploadUtils.initialize({ uploadDirectory, credentialsPath, tokenPath }); - const newMediaItems = await Promise.all(media.map(async element => { - const uploadToken = await GooglePhotosUploadUtils.DispatchGooglePhotosUpload(element.url).catch(error => { - console.log("Dispatching upload error!"); - console.log(error); + + const newMediaItems: NewMediaItem[] = []; + let failed = 0; + const size = 25; + + try { + await mediaInput.batch({ + size, + action: { + handler: async (batch: GooglePhotosUploadUtils.MediaInput[]) => { + await Promise.all(batch.map(async element => { + console.log(`Uploading ${element.url} to Google's servers...`); + const uploadToken = await GooglePhotosUploadUtils.DispatchGooglePhotosUpload(element.url); + if (uploadToken) { + newMediaItems.push({ + description: element.description, + simpleMediaItem: { uploadToken } + }); + } else { + console.log("FAIL!", element.url, element.description); + failed++; + } + })); + }, + interval: 3000 + } }); - return !uploadToken ? undefined : { - description: element.description, - simpleMediaItem: { uploadToken } - }; - })); - if (!newMediaItems.every(item => item)) { + } catch (e) { + console.log("WHAT HAPPENED?"); + console.log(e); + } + if (failed) { return _error(res, tokenError); } GooglePhotosUploadUtils.CreateMediaItems(newMediaItems, req.body.album).then( -- cgit v1.2.3-70-g09d2 From dcbbfe6d34e89df49069a0ede64df0dc5adc6056 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Fri, 13 Sep 2019 20:17:19 -0400 Subject: fixed batching and refactor --- src/client/northstar/utils/Extensions.ts | 13 ++--- .../util/Import & Export/DirectoryImportBox.tsx | 33 ++++++------ src/client/util/UtilExtensions.ts | 62 +++++++++++----------- src/server/apis/google/GooglePhotosUploadUtils.ts | 51 ++++++++---------- src/server/credentials/google_docs_token.json | 2 +- src/server/index.ts | 50 +++++++---------- 6 files changed, 95 insertions(+), 116 deletions(-) (limited to 'src/server/apis') diff --git a/src/client/northstar/utils/Extensions.ts b/src/client/northstar/utils/Extensions.ts index 720f4a062..c866d1bc3 100644 --- a/src/client/northstar/utils/Extensions.ts +++ b/src/client/northstar/utils/Extensions.ts @@ -20,22 +20,17 @@ String.prototype.Truncate = function (length: number, replacement: string): Stri return target; }; -interface Action { - handler: (batch: T[]) => any; - interval?: number; -} -interface BatchParameters { - size: number; - action?: Action; -} +type BatchHandler = (batch: I[]) => O[] | Promise; interface Array { - batch(parameters: BatchParameters): Promise; + batch(batchSize: number): T[][]; + batchAction(batchSize: number, handler: BatchHandler, interval?: number): Promise; lastElement(): T; } Array.prototype.batch = extensions.Batch; +Array.prototype.batchAction = extensions.BatchAction; Array.prototype.lastElement = function () { if (!this.length) { diff --git a/src/client/util/Import & Export/DirectoryImportBox.tsx b/src/client/util/Import & Export/DirectoryImportBox.tsx index 5915f3412..1ae0e7525 100644 --- a/src/client/util/Import & Export/DirectoryImportBox.tsx +++ b/src/client/util/Import & Export/DirectoryImportBox.tsx @@ -95,25 +95,19 @@ export default class DirectoryImportBox extends React.Component let sizes: number[] = []; let modifiedDates: number[] = []; - const uploads: FileResponse[] = []; + const localUpload = async (batch: File[]) => { + sizes.push(...batch.map(file => file.size)); + modifiedDates.push(...batch.map(file => file.lastModified)); - await validated.batch({ - size: 15, - action: { - handler: async (batch: File[]) => { - sizes.push(...batch.map(file => file.size)); - modifiedDates.push(...batch.map(file => file.lastModified)); + let formData = new FormData(); + batch.forEach(file => formData.append(Utils.GenerateGuid(), file)); + const parameters = { method: 'POST', body: formData }; - let formData = new FormData(); - batch.forEach(file => formData.append(Utils.GenerateGuid(), file)); - const parameters = { method: 'POST', body: formData }; - - uploads.push(...(await (await fetch(Utils.prepend(RouteStore.upload), parameters)).json())); + runInAction(() => this.completed += batch.length); + return (await fetch(Utils.prepend(RouteStore.upload), parameters)).json(); + }; - runInAction(() => this.completed += batch.length); - } - } - }); + const uploads = await validated.batchAction(15, localUpload); await Promise.all(uploads.map(async upload => { const type = upload.type; @@ -149,7 +143,12 @@ export default class DirectoryImportBox extends React.Component }; let parent = this.props.ContainingCollectionView; if (parent) { - let importContainer = Docs.Create.MasonryDocument(docs, options); + let importContainer: Doc; + if (docs.length < 50) { + importContainer = Docs.Create.MasonryDocument(docs, options); + } else { + importContainer = Docs.Create.SchemaDocument([], docs, options); + } await GooglePhotos.Export.CollectionToAlbum({ collection: importContainer }); importContainer.singleColumn = false; Doc.AddDocToList(Doc.GetProto(parent.props.Document), "data", importContainer); diff --git a/src/client/util/UtilExtensions.ts b/src/client/util/UtilExtensions.ts index 1e277b242..0bf9f4e97 100644 --- a/src/client/util/UtilExtensions.ts +++ b/src/client/util/UtilExtensions.ts @@ -1,39 +1,41 @@ -module.exports.Batch = async function (parameters: BatchParameters) { - const { size, action } = parameters; +module.exports.Batch = function (batchSize: number): T[][] { const batches: T[][] = []; let i = 0; while (i < this.length) { - const cap = Math.min(i + size, this.length); + const cap = Math.min(i + batchSize, this.length); batches.push(this.slice(i, cap)); i = cap; } - console.log(`Beginning action on ${this.length} elements, split into ${batches.length} groups => ${batches.map(batch => batch.length).join(", ")}`); - if (action) { - const { handler, interval } = action; - if (!interval || batches.length === 1) { - for (let batch of batches) { - await handler(batch); - } - } else { - return new Promise(resolve => { - const iterator = batches[Symbol.iterator](); - const quota = batches.length; - let completed = 0; - const tag = setInterval(async () => { - const next = iterator.next(); - if (next.done) { - clearInterval(tag); - return; - } - const batch = next.value; - console.log(`Handling next batch with ${batch.length} elements`); - await handler(batch); - if (++completed === quota) { - resolve(batches); - } - }, interval); - }); + return batches; +}; + +module.exports.BatchAction = async function (batchSize: number, handler: BatchHandler, interval?: number): Promise { + if (!this.length) { + return []; + } + let collector: O[] = []; + const batches = this.batch(batchSize); + if (!interval || batches.length === 1) { + for (let batch of batches) { + collector.push(...(await handler(batch))); } + } else { + return new Promise(resolve => { + const iterator = batches[Symbol.iterator](); + let completed = 0; + const tag = setInterval(async () => { + const next = iterator.next(); + if (next.done) { + clearInterval(tag); + return; + } + const batch = next.value; + collector.push(...(await handler(batch))); + if (++completed === batches.length) { + resolve(collector); + } + }, interval); + }); } - return batches; + return collector; }; \ No newline at end of file diff --git a/src/server/apis/google/GooglePhotosUploadUtils.ts b/src/server/apis/google/GooglePhotosUploadUtils.ts index 3989590c6..e640f2a85 100644 --- a/src/server/apis/google/GooglePhotosUploadUtils.ts +++ b/src/server/apis/google/GooglePhotosUploadUtils.ts @@ -40,7 +40,7 @@ export namespace GooglePhotosUploadUtils { }; export const DispatchGooglePhotosUpload = async (url: string) => { - const body = await request(url, { encoding: null }).catch(error => console.log("Error in streaming body!", error)); + const body = await request(url, { encoding: null }); const parameters = { method: 'POST', headers: { @@ -57,37 +57,30 @@ export namespace GooglePhotosUploadUtils { return reject(error); } resolve(body); - }).catch(error => console.log("Error in literal uploading process to Google's servers!", error))).catch(error => console.log("Error in literal uploading process to Google's servers!", error)); + })); }; export const CreateMediaItems = async (newMediaItems: NewMediaItem[], album?: { id: string }): Promise => { - const newMediaItemResults: NewMediaItemResult[] = []; - await newMediaItems.batch({ - size: 50, - action: { - handler: async (batch: NewMediaItem[]) => { - console.log(batch.length); - const parameters = { - method: 'POST', - headers: headers('json'), - uri: prepend('mediaItems:batchCreate'), - body: { newMediaItems: batch } as any, - json: true - }; - album && (parameters.body.albumId = album.id); - newMediaItemResults.push(...(await new Promise((resolve, reject) => { - request(parameters, (error, _response, body) => { - if (error) { - reject(error); - } else { - resolve(body); - } - }); - })).newMediaItemResults); - }, - interval: 1000 - } - }); + const createFromUploadTokens = async (batch: NewMediaItem[]) => { + const parameters = { + method: 'POST', + headers: headers('json'), + uri: prepend('mediaItems:batchCreate'), + body: { newMediaItems: batch } as any, + json: true + }; + album && (parameters.body.albumId = album.id); + return (await new Promise((resolve, reject) => { + request(parameters, (error, _response, body) => { + if (error) { + reject(error); + } else { + resolve(body); + } + }); + })).newMediaItemResults; + }; + const newMediaItemResults = await newMediaItems.batchAction(50, createFromUploadTokens, 1000); return { newMediaItemResults }; }; diff --git a/src/server/credentials/google_docs_token.json b/src/server/credentials/google_docs_token.json index d8e0eae21..98d735acd 100644 --- a/src/server/credentials/google_docs_token.json +++ b/src/server/credentials/google_docs_token.json @@ -1 +1 @@ -{"access_token":"ya29.GlyDB0wsV3-oS6q5TFuJSmH1YP_SPf_X6RHaJVmfqj0NTCtaPLFonZRxdT52kUkiHJgAoRizxZvlSIGptXKfnmG4BFouhgyo9ZKP0QtOH-kPR9b9x5WhGCd5NWqz0A","refresh_token":"1/HTv_xFHszu2Nf3iiFrUTaeKzC_Vp2-6bpIB06xW_WHI","scope":"https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/documents.readonly https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/photoslibrary https://www.googleapis.com/auth/photoslibrary.appendonly https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/photoslibrary.sharing","token_type":"Bearer","expiry_date":1568412547334} \ No newline at end of file +{"access_token":"ya29.ImCDB7HE7-5O12GlhloCz2YWbDC5s8drlIs65oaPUVjAgL66RZMmIV8BptOs2X66ZWvQLCbRourz3ubcQooIuyzgpR8D1IVVm577RC5iyA2xB1Y1GNKZbHgpX3g8yGGmbS8","refresh_token":"1/HTv_xFHszu2Nf3iiFrUTaeKzC_Vp2-6bpIB06xW_WHI","scope":"https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/documents.readonly https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/photoslibrary https://www.googleapis.com/auth/photoslibrary.appendonly https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/photoslibrary.sharing","token_type":"Bearer","expiry_date":1568423265295} \ No newline at end of file diff --git a/src/server/index.ts b/src/server/index.ts index 542a4ea65..79e9155d2 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -582,9 +582,7 @@ app.post( const filename = path.basename(location); await UploadUtils.UploadImage(uploadDirectory + filename, filename).catch(() => console.log(`Unable to process ${filename}`)); results.push({ name, type, path: `/files/${filename}` }); - console.log(path.basename(name)); } - console.log("All files traversed!"); _success(res, results); }); } @@ -836,44 +834,36 @@ export interface NewMediaItem { } Array.prototype.batch = extensions.Batch; +Array.prototype.batchAction = extensions.BatchAction; app.post(RouteStore.googlePhotosMediaUpload, async (req, res) => { const mediaInput: GooglePhotosUploadUtils.MediaInput[] = req.body.media; await GooglePhotosUploadUtils.initialize({ uploadDirectory, credentialsPath, tokenPath }); - const newMediaItems: NewMediaItem[] = []; let failed = 0; - const size = 25; - - try { - await mediaInput.batch({ - size, - action: { - handler: async (batch: GooglePhotosUploadUtils.MediaInput[]) => { - await Promise.all(batch.map(async element => { - console.log(`Uploading ${element.url} to Google's servers...`); - const uploadToken = await GooglePhotosUploadUtils.DispatchGooglePhotosUpload(element.url); - if (uploadToken) { - newMediaItems.push({ - description: element.description, - simpleMediaItem: { uploadToken } - }); - } else { - console.log("FAIL!", element.url, element.description); - failed++; - } - })); - }, - interval: 3000 + + const dispatchUpload = async (batch: GooglePhotosUploadUtils.MediaInput[]) => { + const newMediaItems: NewMediaItem[] = []; + for (let element of batch) { + const uploadToken = await GooglePhotosUploadUtils.DispatchGooglePhotosUpload(element.url); + if (!uploadToken) { + failed++; + } else { + newMediaItems.push({ + description: element.description, + simpleMediaItem: { uploadToken } + }); } - }); - } catch (e) { - console.log("WHAT HAPPENED?"); - console.log(e); - } + } + return newMediaItems; + }; + + const newMediaItems = await mediaInput.batchAction(25, dispatchUpload, 3000); + if (failed) { return _error(res, tokenError); } + GooglePhotosUploadUtils.CreateMediaItems(newMediaItems, req.body.album).then( result => _success(res, result.newMediaItemResults), error => _error(res, mediaError, error) -- cgit v1.2.3-70-g09d2 From e4d6f6a643ca07516ec3c8eb4a542c69cfb7b1a2 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Fri, 13 Sep 2019 22:30:31 -0400 Subject: updates --- src/client/northstar/utils/Extensions.ts | 22 +++++- .../util/Import & Export/DirectoryImportBox.tsx | 4 +- src/client/util/UtilExtensions.ts | 92 +++++++++++++++++----- src/server/apis/google/GooglePhotosUploadUtils.ts | 2 +- src/server/index.ts | 9 ++- 5 files changed, 100 insertions(+), 29 deletions(-) (limited to 'src/server/apis') diff --git a/src/client/northstar/utils/Extensions.ts b/src/client/northstar/utils/Extensions.ts index c866d1bc3..00c1e113c 100644 --- a/src/client/northstar/utils/Extensions.ts +++ b/src/client/northstar/utils/Extensions.ts @@ -20,17 +20,31 @@ String.prototype.Truncate = function (length: number, replacement: string): Stri return target; }; - -type BatchHandler = (batch: I[]) => O[] | Promise; +type BatchConverterSync = (batch: I[]) => O[]; +type BatchHandlerSync = (batch: I[]) => void; +type BatchConverterAsync = (batch: I[]) => Promise; +type BatchHandlerAsync = (batch: I[]) => Promise; +type BatchConverter = BatchConverterSync | BatchConverterAsync; +type BatchHandler = BatchHandlerSync | BatchHandlerAsync; interface Array { batch(batchSize: number): T[][]; - batchAction(batchSize: number, handler: BatchHandler, interval?: number): Promise; + executeInBatches(batchSize: number, handler: BatchHandlerSync): void; + convertInBatches(batchSize: number, handler: BatchConverterSync): O[]; + executeInBatchesAsync(batchSize: number, handler: BatchHandler): Promise; + convertInBatchesAsync(batchSize: number, handler: BatchConverter): Promise; + executeInBatchesAtInterval(batchSize: number, handler: BatchHandler, interval: number): Promise; + convertInBatchesAtInterval(batchSize: number, handler: BatchConverter, interval: number): Promise; lastElement(): T; } Array.prototype.batch = extensions.Batch; -Array.prototype.batchAction = extensions.BatchAction; +Array.prototype.executeInBatches = extensions.ExecuteInBatches; +Array.prototype.convertInBatches = extensions.ConvertInBatches; +Array.prototype.executeInBatchesAsync = extensions.ExecuteInBatchesAsync; +Array.prototype.convertInBatchesAsync = extensions.ConvertInBatchesAsync; +Array.prototype.executeInBatchesAtInterval = extensions.ExecuteInBatchesAtInterval; +Array.prototype.convertInBatchesAtInterval = extensions.ConvertInBatchesAtInterval; Array.prototype.lastElement = function () { if (!this.length) { diff --git a/src/client/util/Import & Export/DirectoryImportBox.tsx b/src/client/util/Import & Export/DirectoryImportBox.tsx index 1ae0e7525..a625e06c0 100644 --- a/src/client/util/Import & Export/DirectoryImportBox.tsx +++ b/src/client/util/Import & Export/DirectoryImportBox.tsx @@ -95,7 +95,7 @@ export default class DirectoryImportBox extends React.Component let sizes: number[] = []; let modifiedDates: number[] = []; - const localUpload = async (batch: File[]) => { + const uploadLocally = async (batch: File[]) => { sizes.push(...batch.map(file => file.size)); modifiedDates.push(...batch.map(file => file.lastModified)); @@ -107,7 +107,7 @@ export default class DirectoryImportBox extends React.Component return (await fetch(Utils.prepend(RouteStore.upload), parameters)).json(); }; - const uploads = await validated.batchAction(15, localUpload); + const uploads = await validated.convertInBatchesAsync(15, uploadLocally); await Promise.all(uploads.map(async upload => { const type = upload.type; diff --git a/src/client/util/UtilExtensions.ts b/src/client/util/UtilExtensions.ts index 0bf9f4e97..3eeec6ca7 100644 --- a/src/client/util/UtilExtensions.ts +++ b/src/client/util/UtilExtensions.ts @@ -9,33 +9,85 @@ module.exports.Batch = function (batchSize: number): T[][] { return batches; }; -module.exports.BatchAction = async function (batchSize: number, handler: BatchHandler, interval?: number): Promise { +module.exports.ExecuteBatches = function (batchSize: number, handler: BatchHandlerSync): void { + if (this.length) { + for (let batch of this.batch(batchSize)) { + handler(batch); + } + } +}; + +module.exports.ConvertInBatches = function (batchSize: number, handler: BatchConverterSync): O[] { if (!this.length) { return []; } let collector: O[] = []; - const batches = this.batch(batchSize); - if (!interval || batches.length === 1) { - for (let batch of batches) { - collector.push(...(await handler(batch))); + for (let batch of this.batch(batchSize)) { + collector.push(...handler(batch)); + } + return collector; +}; + +module.exports.ExecuteInBatchesAsync = async function (batchSize: number, handler: BatchHandler): Promise { + if (this.length) { + for (let batch of this.batch(batchSize)) { + await handler(batch); } - } else { - return new Promise(resolve => { - const iterator = batches[Symbol.iterator](); - let completed = 0; - const tag = setInterval(async () => { - const next = iterator.next(); - if (next.done) { - clearInterval(tag); - return; + } +}; + +module.exports.ConvertInBatchesAsync = async function (batchSize: number, handler: BatchConverter): Promise { + if (!this.length) { + return []; + } + let collector: O[] = []; + for (let batch of this.batch(batchSize)) { + collector.push(...(await handler(batch))); + } + return collector; +}; + +module.exports.ExecuteInBatchesAtInterval = async function (batchSize: number, handler: BatchHandler, interval: number): Promise { + if (!this.length) { + return; + } + const batches = this.batch(batchSize); + return new Promise(resolve => { + const iterator = batches[Symbol.iterator](); + let completed = 0; + const tag = setInterval(async () => { + const next = iterator.next(); + if (next.done) { + clearInterval(tag); + } else { + await handler(next.value); + if (++completed === batches.length) { + resolve(); } - const batch = next.value; - collector.push(...(await handler(batch))); + } + }, interval * 1000); + }); +}; + +module.exports.ConvertInBatchesAtInterval = async function (batchSize: number, handler: BatchConverter, interval: number): Promise { + if (!this.length) { + return []; + } + let collector: O[] = []; + const batches = this.batch(batchSize); + return new Promise(resolve => { + const iterator = batches[Symbol.iterator](); + let completed = 0; + const tag = setInterval(async () => { + const next = iterator.next(); + if (next.done) { + clearInterval(tag); + } else { + collector.push(...(await handler(next.value))); if (++completed === batches.length) { resolve(collector); } - }, interval); - }); - } - return collector; + } + }, interval * 1000); + }); }; \ No newline at end of file diff --git a/src/server/apis/google/GooglePhotosUploadUtils.ts b/src/server/apis/google/GooglePhotosUploadUtils.ts index e640f2a85..e1478a097 100644 --- a/src/server/apis/google/GooglePhotosUploadUtils.ts +++ b/src/server/apis/google/GooglePhotosUploadUtils.ts @@ -80,7 +80,7 @@ export namespace GooglePhotosUploadUtils { }); })).newMediaItemResults; }; - const newMediaItemResults = await newMediaItems.batchAction(50, createFromUploadTokens, 1000); + const newMediaItemResults = await newMediaItems.convertInBatchesAtInterval(50, createFromUploadTokens, 1); return { newMediaItemResults }; }; diff --git a/src/server/index.ts b/src/server/index.ts index 79e9155d2..8767be17d 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -834,7 +834,12 @@ export interface NewMediaItem { } Array.prototype.batch = extensions.Batch; -Array.prototype.batchAction = extensions.BatchAction; +Array.prototype.executeInBatches = extensions.ExecuteInBatches; +Array.prototype.convertInBatches = extensions.ConvertInBatches; +Array.prototype.executeInBatchesAsync = extensions.ExecuteInBatchesAsync; +Array.prototype.convertInBatchesAsync = extensions.ConvertInBatchesAsync; +Array.prototype.executeInBatchesAtInterval = extensions.ExecuteInBatchesAtInterval; +Array.prototype.convertInBatchesAtInterval = extensions.ConvertInBatchesAtInterval; app.post(RouteStore.googlePhotosMediaUpload, async (req, res) => { const mediaInput: GooglePhotosUploadUtils.MediaInput[] = req.body.media; @@ -858,7 +863,7 @@ app.post(RouteStore.googlePhotosMediaUpload, async (req, res) => { return newMediaItems; }; - const newMediaItems = await mediaInput.batchAction(25, dispatchUpload, 3000); + const newMediaItems = await mediaInput.convertInBatchesAtInterval(25, dispatchUpload, 3); if (failed) { return _error(res, tokenError); -- cgit v1.2.3-70-g09d2 From 037098aca0993bee6f986b592c17aa54a7225905 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Sat, 14 Sep 2019 17:53:17 -0400 Subject: rich text google photos transfer improvments --- src/Utils.ts | 2 +- .../util/Import & Export/DirectoryImportBox.tsx | 6 +- .../CollectionFreeFormLinkView.scss | 2 +- .../CollectionFreeFormLinkView.tsx | 2 +- src/new_fields/RichTextUtils.ts | 121 +++++++++++++++------ src/server/apis/google/GooglePhotosUploadUtils.ts | 2 +- src/server/credentials/google_docs_token.json | 2 +- src/server/index.ts | 2 +- 8 files changed, 101 insertions(+), 38 deletions(-) (limited to 'src/server/apis') diff --git a/src/Utils.ts b/src/Utils.ts index ec2bee9bf..60f18eac2 100644 --- a/src/Utils.ts +++ b/src/Utils.ts @@ -39,7 +39,7 @@ export class Utils { } public static fileUrl(filename: string): string { - return this.prepend(`/file/${filename}`); + return this.prepend(`/files/${filename}`); } public static CorsProxy(url: string): string { diff --git a/src/client/util/Import & Export/DirectoryImportBox.tsx b/src/client/util/Import & Export/DirectoryImportBox.tsx index 87c187162..44075ecdd 100644 --- a/src/client/util/Import & Export/DirectoryImportBox.tsx +++ b/src/client/util/Import & Export/DirectoryImportBox.tsx @@ -156,7 +156,11 @@ export default class DirectoryImportBox extends React.Component if (docs.length < 50) { importContainer = Docs.Create.MasonryDocument(docs, options); } else { - const headers = ["title", "size"].map(key => new SchemaHeaderField(key)); + const headers = [ + new SchemaHeaderField("title", "yellow"), + new SchemaHeaderField("size", "blue"), + new SchemaHeaderField("googlePhotosTags", "green") + ]; importContainer = Docs.Create.SchemaDocument(headers, docs, options); } runInAction(() => this.phase = 'External: uploading files to Google Photos...'); diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormLinkView.scss b/src/client/views/collections/collectionFreeForm/CollectionFreeFormLinkView.scss index fc5212edd..2a64a7afb 100644 --- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormLinkView.scss +++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormLinkView.scss @@ -1,7 +1,7 @@ .collectionfreeformlinkview-linkLine { stroke: black; transform: translate(10000px,10000px); - opacity: 0.5; + // opacity: 0.5; pointer-events: all; } .collectionfreeformlinkview-linkCircle { diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormLinkView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormLinkView.tsx index 790c6694b..f19243bd6 100644 --- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormLinkView.tsx +++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormLinkView.tsx @@ -50,7 +50,7 @@ export class CollectionFreeFormLinkView extends React.Component {/* ([ + const StyleToMark = new Map([ ["bold", "strong"], ["italic", "em"], ["foregroundColor", "pFontColor"] ]); - const styleToMarks = (schema: any, textStyle?: docs_v1.Schema$TextStyle) => { + const MarkToStyle = new Map([ + ["strong", "bold"], + ["em", "italic"], + ["pFontColor", "foregroundColor"], + ["timesNewRoman", "weightedFontFamily"], + ["georgia", "weightedFontFamily"], + ["comicSans", "weightedFontFamily"], + ["tahoma", "weightedFontFamily"], + ["impact", "weightedFontFamily"] + ]); + + const FontFamilyMapping = new Map([ + ["timesNewRoman", "Times New Roman"], + ["arial", "Arial"], + ["georgia", "Georgia"], + ["comicSans", "Comic Sans MS"], + ["tahoma", "Tahoma"], + ["impact", "Impact"] + ]); + + const styleToMarks = (schema: any, textStyle?: docs_v1.Schema$TextStyle): Opt => { if (!textStyle) { return undefined; } @@ -236,12 +256,12 @@ export namespace RichTextUtils { let targeted = key as keyof docs_v1.Schema$TextStyle; if (value = textStyle[targeted]) { let attributes: any = {}; - let converted = MarkMapping.get(targeted) || targeted; + let converted = StyleToMark.get(targeted) || targeted; value.url && (attributes.href = value.url); if (value.color) { - let object: { [key: string]: number } = value.color.rgbColor; - attributes.color = Color.rgb(Object.values(object).map(value => value * 255)).hex(); + let object = value.color.rgbColor; + attributes.color = Color.rgb(["red", "green", "blue"].map(color => object[color] * 255 || 0)).hex(); } let mark = schema.mark(schema.marks[converted], attributes); @@ -251,45 +271,69 @@ export namespace RichTextUtils { return marks; }; - const marksToStyle = async (nodes: Node[]) => { + const ignored = ["user_mark"]; + + const marksToStyle = async (nodes: Node[]): Promise => { let requests: docs_v1.Schema$Request[] = []; let position = 1; for (let node of nodes) { - const length = node.nodeSize; - const marks = node.marks; - const attrs = node.attrs; + const { marks, attrs, nodeSize } = node; const textStyle: docs_v1.Schema$TextStyle = {}; const information: LinkInformation = { startIndex: position, - endIndex: position + length, + endIndex: position + nodeSize, textStyle }; if (marks.length) { - - const link = marks.find(mark => mark.type.name === "link"); - if (link) { - textStyle.link = { url: link.attrs.href }; - textStyle.foregroundColor = fromRgb(0, 0, 1); - textStyle.bold = true; + let mark: Mark; + const markMap = BuildMarkMap(marks); + Object.keys(schema.marks).map(markName => { + if (!ignored.includes(markName) && (mark = markMap[markName])) { + const converted = MarkToStyle.get(markName) || markName as keyof docs_v1.Schema$TextStyle; + let value: any = true; + if (converted) { + const { attrs } = mark; + switch (converted) { + case "link": + value = { url: attrs.href }; + textStyle.foregroundColor = fromRgb.blue; + textStyle.bold = true; + break; + case "fontSize": + value = attrs.fontSize; + break; + case "foregroundColor": + value = fromHex(attrs.color); + break; + case "weightedFontFamily": + value = { fontFamily: FontFamilyMapping.get(markName) }; + } + textStyle[converted] = value; + } + } + }); + if (Object.keys(textStyle).length) { + requests.push(EncodeStyleUpdate(information)); } - const bold = marks.find(mark => mark.type.name === "strong"); - bold && (textStyle.bold = true); - const foregroundColor = marks.find(mark => mark.type.name === "pFontColor"); - foregroundColor && (textStyle.foregroundColor = fromHex(foregroundColor.attrs.color)); } - requests.push(EncodeStyleUpdate(information)); if (node.type.name === "image") { requests.push(await EncodeImage({ - startIndex: position + length, + startIndex: position + nodeSize, uri: attrs.src, width: attrs.width })); } - position += length; + position += nodeSize; } return requests; }; + const BuildMarkMap = (marks: Mark[]) => { + const markMap: { [type: string]: Mark } = {}; + marks.forEach(mark => markMap[mark.type.name] = mark); + return markMap; + }; + interface LinkInformation { startIndex: number; endIndex: number; @@ -302,14 +346,29 @@ export namespace RichTextUtils { uri: string; } - const fromRgb = (red: number, green: number, blue: number): docs_v1.Schema$OptionalColor => { - return { color: { rgbColor: { red, green, blue } } }; - }; + namespace fromRgb { + + export const convert = (red: number, green: number, blue: number): docs_v1.Schema$OptionalColor => { + return { + color: { + rgbColor: { + red: red / 255, + green: green / 255, + blue: blue / 255 + } + } + }; + }; + + export const red = convert(255, 0, 0); + export const green = convert(0, 255, 0); + export const blue = convert(0, 0, 255); + + } const fromHex = (color: string): docs_v1.Schema$OptionalColor => { - const converted = new Color().hex(color).rgb(); - const { red, blue, green } = converted; - return fromRgb(red(), blue(), green()); + const c = Color(color); + return fromRgb.convert(c.red(), c.green(), c.blue()); }; const EncodeStyleUpdate = (information: LinkInformation): docs_v1.Schema$Request => { diff --git a/src/server/apis/google/GooglePhotosUploadUtils.ts b/src/server/apis/google/GooglePhotosUploadUtils.ts index e1478a097..f582cebd2 100644 --- a/src/server/apis/google/GooglePhotosUploadUtils.ts +++ b/src/server/apis/google/GooglePhotosUploadUtils.ts @@ -80,7 +80,7 @@ export namespace GooglePhotosUploadUtils { }); })).newMediaItemResults; }; - const newMediaItemResults = await newMediaItems.convertInBatchesAtInterval(50, createFromUploadTokens, 1); + const newMediaItemResults = await newMediaItems.convertInBatchesAtInterval(50, createFromUploadTokens, 0.1); return { newMediaItemResults }; }; diff --git a/src/server/credentials/google_docs_token.json b/src/server/credentials/google_docs_token.json index c8fd3bbf5..330d27141 100644 --- a/src/server/credentials/google_docs_token.json +++ b/src/server/credentials/google_docs_token.json @@ -1 +1 @@ -{"access_token":"ya29.GlyDBwCmpO9R1fAOMIzpdZiCWhEeaDHiJOPy7sNRAo-vAIqzIk7zy1DLdOhSFWaBQrbmewSOJZPvbBUAxqdDELc_aW_BsjwXFbxiTd4Us_N8IWkPDCtUeBmLAZjodA","refresh_token":"1/HTv_xFHszu2Nf3iiFrUTaeKzC_Vp2-6bpIB06xW_WHI","scope":"https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/documents.readonly https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/photoslibrary https://www.googleapis.com/auth/photoslibrary.appendonly https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/photoslibrary.sharing","token_type":"Bearer","expiry_date":1568484185591} \ No newline at end of file +{"access_token":"ya29.GlyEBxgqsCRjX9SAJGGss3EtfrPgwSjeMsfsuwJqTk7o4GRrBpwU0eQXXBNgPdAPRSrJzuVgAqWxap9kKrtkpf2tuHxk7Ml9Jblj48tU0BN2X0lMh66S2EIRhLnQnw","refresh_token":"1/HTv_xFHszu2Nf3iiFrUTaeKzC_Vp2-6bpIB06xW_WHI","scope":"https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/documents.readonly https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/photoslibrary https://www.googleapis.com/auth/photoslibrary.appendonly https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/photoslibrary.sharing","token_type":"Bearer","expiry_date":1568501067486} \ No newline at end of file diff --git a/src/server/index.ts b/src/server/index.ts index b18059053..2e60d9be7 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -863,7 +863,7 @@ app.post(RouteStore.googlePhotosMediaUpload, async (req, res) => { return newMediaItems; }; - const newMediaItems = await mediaInput.convertInBatchesAtInterval(25, dispatchUpload, 1); + const newMediaItems = await mediaInput.convertInBatchesAtInterval(25, dispatchUpload, 0.1); if (failed) { return _error(res, tokenError); -- cgit v1.2.3-70-g09d2 From d7012323e87c21a25c29d89d66a1c54b99c8b458 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Sun, 15 Sep 2019 19:28:10 -0400 Subject: syncing of images in imported google doc --- .../apis/google_docs/GoogleApiClientUtils.ts | 19 ++-- src/client/views/nodes/FormattedTextBox.tsx | 6 +- src/new_fields/RichTextUtils.ts | 115 +++++++++++++++------ src/server/apis/google/GooglePhotosUploadUtils.ts | 56 ++++++---- src/server/apis/google/existing_uploads.json | 1 + src/server/credentials/google_docs_token.json | 2 +- src/server/index.ts | 30 ++++-- 7 files changed, 162 insertions(+), 67 deletions(-) create mode 100644 src/server/apis/google/existing_uploads.json (limited to 'src/server/apis') diff --git a/src/client/apis/google_docs/GoogleApiClientUtils.ts b/src/client/apis/google_docs/GoogleApiClientUtils.ts index 828d4451a..cbc5da15b 100644 --- a/src/client/apis/google_docs/GoogleApiClientUtils.ts +++ b/src/client/apis/google_docs/GoogleApiClientUtils.ts @@ -97,25 +97,30 @@ export namespace GoogleApiClientUtils { export type ExtractResult = { text: string, paragraphs: DeconstructedParagraph[] }; export const extractText = (document: docs_v1.Schema$Document, removeNewlines = false): ExtractResult => { let paragraphs = extractParagraphs(document); - let text = paragraphs.map(paragraph => paragraph.runs.map(run => run.content).join("")).join(""); + let text = paragraphs.map(paragraph => paragraph.contents.filter(content => !("inlineObjectId" in content)).map(run => run as docs_v1.Schema$TextRun).join("")).join(""); text = text.substring(0, text.length - 1); removeNewlines && text.ReplaceAll("\n", ""); return { text, paragraphs }; }; - export type DeconstructedParagraph = { runs: docs_v1.Schema$TextRun[], bullet: Opt }; + export type ContentArray = (docs_v1.Schema$TextRun | docs_v1.Schema$InlineObjectElement)[]; + export type DeconstructedParagraph = { contents: ContentArray, bullet: Opt }; const extractParagraphs = (document: docs_v1.Schema$Document, filterEmpty = true): DeconstructedParagraph[] => { const fragments: DeconstructedParagraph[] = []; if (document.body && document.body.content) { for (const element of document.body.content) { - let runs: docs_v1.Schema$TextRun[] = []; + let runs: ContentArray = []; let bullet: Opt; if (element.paragraph) { if (element.paragraph.elements) { for (const inner of element.paragraph.elements) { - if (inner && inner.textRun) { - let run = inner.textRun; - (run.content || !filterEmpty) && runs.push(inner.textRun); + if (inner) { + if (inner.textRun) { + let run = inner.textRun; + (run.content || !filterEmpty) && runs.push(inner.textRun); + } else if (inner.inlineObjectElement) { + runs.push(inner.inlineObjectElement); + } } } } @@ -123,7 +128,7 @@ export namespace GoogleApiClientUtils { bullet = element.paragraph.bullet.nestingLevel || 0; } } - (runs.length || !filterEmpty) && fragments.push({ runs, bullet }); + (runs.length || !filterEmpty) && fragments.push({ contents: runs, bullet }); } } return fragments; diff --git a/src/client/views/nodes/FormattedTextBox.tsx b/src/client/views/nodes/FormattedTextBox.tsx index fc5b27220..8f0f142c4 100644 --- a/src/client/views/nodes/FormattedTextBox.tsx +++ b/src/client/views/nodes/FormattedTextBox.tsx @@ -193,8 +193,8 @@ export class FormattedTextBox extends DocComponent<(FieldViewProps & FormattedTe else DocUtils.MakeLink(this.dataDoc, this.dataDoc[key] as Doc, undefined, "Ref:" + value, undefined, undefined, id); }); }); - const link = this._editorView!.state.schema.marks.link.create({ href: `http://localhost:1050/doc/${id}`, location: "onRight", title: value }); - const mval = this._editorView!.state.schema.marks.metadataVal.create(); + const link = this._editorView.state.schema.marks.link.create({ href: `http://localhost:1050/doc/${id}`, location: "onRight", title: value }); + const mval = this._editorView.state.schema.marks.metadataVal.create(); let offset = (tx.selection.to === range!.end - 1 ? -1 : 0); tx = tx.addMark(textEndSelection - value.length + offset, textEndSelection, link).addMark(textEndSelection - value.length + offset, textEndSelection, mval); this.dataDoc[key] = value; @@ -506,7 +506,7 @@ export class FormattedTextBox extends DocComponent<(FieldViewProps & FormattedTe let documentId = StrCast(dataDoc[GoogleRef]); let exportState: Opt; if (documentId) { - exportState = await RichTextUtils.GoogleDocs.Import(documentId); + exportState = await RichTextUtils.GoogleDocs.Import(documentId, dataDoc); } UndoManager.RunInBatch(() => handler(exportState, dataDoc), Pulls); } diff --git a/src/new_fields/RichTextUtils.ts b/src/new_fields/RichTextUtils.ts index 555c41b67..ab5e677c8 100644 --- a/src/new_fields/RichTextUtils.ts +++ b/src/new_fields/RichTextUtils.ts @@ -14,9 +14,10 @@ import { schema } from "../client/util/RichTextSchema"; import { GooglePhotos } from "../client/apis/google_docs/GooglePhotosClientUtils"; import { SchemaHeaderField } from "./SchemaHeaderField"; import { DocServer } from "../client/DocServer"; -import { Cast } from "./Types"; +import { Cast, StrCast } from "./Types"; import { Id } from "./FieldSymbols"; import { DocumentView } from "../client/views/nodes/DocumentView"; +import { AssertionError } from "assert"; export namespace RichTextUtils { @@ -109,45 +110,78 @@ export namespace RichTextUtils { return { text, requests }; }; + interface ImageTemplate { + width: number; + title: string; + url: string; + } + + const parseInlineObjects = async (document: docs_v1.Schema$Document): Promise> => { + const inlineObjectMap = new Map(); + const inlineObjects = document.inlineObjects; + + if (inlineObjects) { + const objects = Object.keys(inlineObjects).map(objectId => inlineObjects[objectId]); + const mediaItems: MediaItem[] = objects.map(object => { + const embeddedObject = object.inlineObjectProperties!.embeddedObject!; + const baseUrl = embeddedObject.imageProperties!.contentUri!; + const filename = `upload_${Utils.GenerateGuid()}.png`; + return { baseUrl, filename }; + }); + + const uploads = await PostToServer(RouteStore.googlePhotosMediaDownload, { mediaItems }); + + if (uploads.length !== mediaItems.length) { + throw new AssertionError({ expected: mediaItems.length, actual: uploads.length, message: "Error with internally uploading inlineObjects!" }); + } + + for (let i = 0; i < objects.length; i++) { + const object = objects[i]; + const { fileNames } = uploads[i]; + const embeddedObject = object.inlineObjectProperties!.embeddedObject!; + const size = embeddedObject.size!; + const width = size.width!.magnitude!; + const url = Utils.fileUrl(fileNames.clean); + + inlineObjectMap.set(object.objectId!, { + title: embeddedObject.title || `Imported Image from ${document.title}`, + width, + url + }); + } + } + return inlineObjectMap; + }; + type BulletPosition = { value: number, sinks: number }; interface MediaItem { baseUrl: string; filename: string; - width: number; } - export const Import = async (documentId: GoogleApiClientUtils.Docs.DocumentId): Promise> => { + + export const Import = async (documentId: GoogleApiClientUtils.Docs.DocumentId, textNote: Doc): Promise> => { const document = await GoogleApiClientUtils.Docs.retrieve({ documentId }); if (!document) { return undefined; } - + const inlineObjectMap = await parseInlineObjects(document); const title = document.title!; const { text, paragraphs } = GoogleApiClientUtils.Docs.Utils.extractText(document); let state = FormattedTextBox.blankState(); let structured = parseLists(paragraphs); - const inline = document.inlineObjects; - let inlineUrls: MediaItem[] = []; - if (inline) { - inlineUrls = Object.keys(inline).map(key => { - const embedded = inline[key].inlineObjectProperties!.embeddedObject!; - const baseUrl = embedded.imageProperties!.contentUri!; - const filename = `upload_${Utils.GenerateGuid()}.png`; - const width = embedded.size!.width!.magnitude!; - return { baseUrl, filename, width }; - }); - } let position = 3; let lists: ListGroup[] = []; const indentMap = new Map(); let globalOffset = 0; - const nodes = structured.map(element => { + const nodes: Node[] = []; + for (let element of structured) { if (Array.isArray(element)) { lists.push(element); let positions: BulletPosition[] = []; let items = element.map(paragraph => { - let item = listItem(state.schema, paragraph.runs); + let item = listItem(state.schema, paragraph.contents); let sinks = paragraph.bullet!; positions.push({ value: position + globalOffset, @@ -158,13 +192,26 @@ export namespace RichTextUtils { return item; }); indentMap.set(element, positions); - return list(state.schema, items); + nodes.push(list(state.schema, items)); } else { - let paragraph = paragraphNode(state.schema, element.runs); - position += paragraph.nodeSize; - return paragraph; + if (element.contents.some(child => "inlineObjectId" in child)) { + let node: Node; + for (const child of element.contents) { + if ("inlineObjectId" in child) { + node = imageNode(state.schema, inlineObjectMap.get(child.inlineObjectId!)!, textNote); + } else { + node = paragraphNode(state.schema, [child]); + } + nodes.push(node); + position += node.nodeSize; + } + } else { + let paragraph = paragraphNode(state.schema, element.contents); + nodes.push(paragraph); + position += paragraph.nodeSize; + } } - }); + } state = state.apply(state.tr.replaceWith(0, 2, nodes)); let sink = sinkListItem(state.schema.nodes.list_item); @@ -179,14 +226,6 @@ export namespace RichTextUtils { } } - const uploads = await PostToServer(RouteStore.googlePhotosMediaDownload, { mediaItems: inlineUrls }); - for (let i = 0; i < uploads.length; i++) { - const src = Utils.fileUrl(uploads[i].fileNames.clean); - const width = inlineUrls[i].width; - const imageNode = schema.nodes.image.create({ src, width }); - state = state.apply(state.tr.insert(0, imageNode)); - } - return { title, text, state }; }; @@ -226,6 +265,22 @@ export namespace RichTextUtils { return schema.node("paragraph", null, fragment); }; + const imageNode = (schema: any, image: ImageTemplate, textNote: Doc) => { + const { url: src, width } = image; + let docid: string; + const guid = Utils.GenerateDeterministicGuid(src); + const backingDocId = StrCast(textNote[guid]); + if (!backingDocId) { + const backingDoc = Docs.Create.ImageDocument(src, { width: 300, height: 300 }); + DocumentView.makeCustomViewClicked(backingDoc); + docid = backingDoc[Id]; + textNote[guid] = docid; + } else { + docid = backingDocId; + } + return schema.node("image", { src, width, docid }); + }; + const textNode = (schema: any, run: docs_v1.Schema$TextRun) => { let text = run.content!.removeTrailingNewlines(); return text.length ? schema.text(text, styleToMarks(schema, run.textStyle)) : undefined; diff --git a/src/server/apis/google/GooglePhotosUploadUtils.ts b/src/server/apis/google/GooglePhotosUploadUtils.ts index f582cebd2..3ab9ba90f 100644 --- a/src/server/apis/google/GooglePhotosUploadUtils.ts +++ b/src/server/apis/google/GooglePhotosUploadUtils.ts @@ -118,28 +118,44 @@ export namespace DownloadUtils { const generate = (prefix: string, url: string) => `${prefix}upload_${Utils.GenerateGuid()}${path.extname(url).toLowerCase()}`; const sanitize = (filename: string) => filename.replace(/\s+/g, "_"); - export const UploadImage = async (url: string, filename?: string, prefix = ""): Promise> => { - const resolved = filename ? sanitize(filename) : generate(prefix, url); - let extension = path.extname(url) || path.extname(resolved); + export interface InspectionResults { + isLocal: boolean; + stream: any; + normalizedUrl: string; + contentSize: number; + contentType: string; + } + + export const InspectImage = async (url: string) => { + const { isLocal, stream, normalized: normalizedUrl } = classify(url); + const metadata = (await new Promise((resolve, reject) => { + request.head(url, async (error, res) => { + if (error) { + return reject(error); + } + resolve(res); + }); + })).headers; + return { + contentSize: parseInt(metadata[size]), + contentType: metadata[type], + isLocal, + stream, + normalizedUrl + }; + }; + + export const UploadImage = async (metadata: InspectionResults, filename?: string, prefix = ""): Promise> => { + const { isLocal, stream, normalizedUrl, contentSize, contentType } = metadata; + const resolved = filename ? sanitize(filename) : generate(prefix, normalizedUrl); + let extension = path.extname(normalizedUrl) || path.extname(resolved); extension && (extension = extension.toLowerCase()); let information: UploadInformation = { mediaPaths: [], - fileNames: { clean: resolved } + fileNames: { clean: resolved }, + contentSize, + contentType, }; - const { isLocal, stream, normalized } = classify(url); - url = normalized; - if (!isLocal) { - const metadata = (await new Promise((resolve, reject) => { - request.head(url, async (error, res) => { - if (error) { - return reject(error); - } - resolve(res); - }); - })).headers; - information.contentSize = parseInt(metadata[size]); - information.contentType = metadata[type]; - } return new Promise(async (resolve, reject) => { const resizers = [ { resizer: sharp().rotate(), suffix: "_o" }, @@ -164,7 +180,7 @@ export namespace DownloadUtils { const filename = resolved.substring(0, resolved.length - extension.length) + suffix + extension; information.mediaPaths.push(mediaPath = uploadDirectory + filename); information.fileNames[suffix] = filename; - stream(url).pipe(resizer.resizer).pipe(fs.createWriteStream(mediaPath)) + stream(normalizedUrl).pipe(resizer.resizer).pipe(fs.createWriteStream(mediaPath)) .on('close', resolve) .on('error', reject); }); @@ -172,7 +188,7 @@ export namespace DownloadUtils { } if (!isLocal || nonVisual) { await new Promise(resolve => { - stream(url).pipe(fs.createWriteStream(uploadDirectory + resolved)).on('close', resolve); + stream(normalizedUrl).pipe(fs.createWriteStream(uploadDirectory + resolved)).on('close', resolve); }); } resolve(information); diff --git a/src/server/apis/google/existing_uploads.json b/src/server/apis/google/existing_uploads.json new file mode 100644 index 000000000..05c20c33b --- /dev/null +++ b/src/server/apis/google/existing_uploads.json @@ -0,0 +1 @@ +{"23625":{"mediaPaths":["C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_7e2d5fef-860a-49a8-b9ec-b91f28073180_o.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_7e2d5fef-860a-49a8-b9ec-b91f28073180_s.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_7e2d5fef-860a-49a8-b9ec-b91f28073180_m.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_7e2d5fef-860a-49a8-b9ec-b91f28073180_l.png"],"fileNames":{"clean":"upload_7e2d5fef-860a-49a8-b9ec-b91f28073180.png","_o":"upload_7e2d5fef-860a-49a8-b9ec-b91f28073180_o.png","_s":"upload_7e2d5fef-860a-49a8-b9ec-b91f28073180_s.png","_m":"upload_7e2d5fef-860a-49a8-b9ec-b91f28073180_m.png","_l":"upload_7e2d5fef-860a-49a8-b9ec-b91f28073180_l.png"},"contentSize":23625,"contentType":"image/jpeg"}} \ No newline at end of file diff --git a/src/server/credentials/google_docs_token.json b/src/server/credentials/google_docs_token.json index 4f2fb0f9d..c10b0797f 100644 --- a/src/server/credentials/google_docs_token.json +++ b/src/server/credentials/google_docs_token.json @@ -1 +1 @@ -{"access_token":"ya29.GlyEB-6kaRm7dCD9x3j1b5AyujXvfpS5NWuJQwy6UKLO06KYXcF2e5XaCxvR7QJgH3Pn2iu3btjYrrJxNNaLffgEszcJHNsN_5IIWJBA4sdG6KLW63MmFwfV4U1hyQ","refresh_token":"1/HTv_xFHszu2Nf3iiFrUTaeKzC_Vp2-6bpIB06xW_WHI","scope":"https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/documents.readonly https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/photoslibrary https://www.googleapis.com/auth/photoslibrary.appendonly https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/photoslibrary.sharing","token_type":"Bearer","expiry_date":1568573667294} \ No newline at end of file +{"access_token":"ya29.ImCFB_ghOybVB6A4HvIIwIlyGyZw6wOymdwJyWJJECIpCmFTHNEzOAfP98KFzm5OUV2zZNS5Wx1iUT1xYWW35PY7NoZc7PWwjzmOaGkMzDm7_fxpsgjT0StdvEwTJprFIv0","refresh_token":"1/HTv_xFHszu2Nf3iiFrUTaeKzC_Vp2-6bpIB06xW_WHI","scope":"https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/documents.readonly https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/photoslibrary https://www.googleapis.com/auth/photoslibrary.appendonly https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/photoslibrary.sharing","token_type":"Bearer","expiry_date":1568590984976} \ No newline at end of file diff --git a/src/server/index.ts b/src/server/index.ts index 2e60d9be7..07ce4b6f0 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -46,6 +46,7 @@ const MongoStore = require('connect-mongo')(session); const mongoose = require('mongoose'); const probe = require("probe-image-size"); import * as qs from 'query-string'; +import { Opt } from '../new_fields/Doc'; const extensions = require("../client/util/UtilExtensions"); const download = (url: string, dest: fs.PathLike) => request.get(url).pipe(fs.createWriteStream(dest)); @@ -580,7 +581,8 @@ app.post( for (const key in files) { const { type, path: location, name } = files[key]; const filename = path.basename(location); - await UploadUtils.UploadImage(uploadDirectory + filename, filename).catch(() => console.log(`Unable to process ${filename}`)); + const metadata = await UploadUtils.InspectImage(uploadDirectory + filename); + await UploadUtils.UploadImage(metadata, filename).catch(() => console.log(`Unable to process ${filename}`)); results.push({ name, type, path: `/files/${filename}` }); } _success(res, results); @@ -884,14 +886,30 @@ const prefix = "google_photos_"; const downloadError = "Encountered an error while executing downloads."; const requestError = "Unable to execute download: the body's media items were malformed."; +app.get("/gapiCleanup", (req, res) => { + write_text_file(file, ""); + res.redirect(RouteStore.delete); +}); + +const file = "./apis/google/existing_uploads.json"; app.post(RouteStore.googlePhotosMediaDownload, async (req, res) => { const contents: { mediaItems: MediaItem[] } = req.body; if (contents) { - const pending = contents.mediaItems.map(item => - UploadUtils.UploadImage(item.baseUrl, item.filename, prefix) - ); - const completed = await Promise.all(pending).catch(error => _error(res, downloadError, error)); - Array.isArray(completed) && _success(res, completed); + const completed: Opt[] = []; + const content = await read_text_file(file); + let existing = content.length ? JSON.parse(content) : {}; + for (let item of contents.mediaItems) { + const { contentSize, ...attributes } = await UploadUtils.InspectImage(item.baseUrl); + const found: UploadUtils.UploadInformation = existing[contentSize]; + if (!found) { + const upload = await UploadUtils.UploadImage({ contentSize, ...attributes }, item.filename, prefix).catch(error => _error(res, downloadError, error)); + upload && completed.push(existing[contentSize] = upload); + } else { + completed.push(found); + } + } + await write_text_file(file, JSON.stringify(existing)); + _success(res, completed); return; } _invalid(res, requestError); -- cgit v1.2.3-70-g09d2 From fe2b302288d120a0b68a3fa9e078d14445de1251 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Mon, 16 Sep 2019 10:27:55 -0400 Subject: updates --- src/client/northstar/utils/Extensions.ts | 24 ++++---- .../util/Import & Export/DirectoryImportBox.tsx | 2 +- src/client/views/collections/CollectionSubView.tsx | 10 +++- .../collectionFreeForm/CollectionFreeFormView.tsx | 64 +++++++++++----------- src/new_fields/RichTextUtils.ts | 10 +++- src/server/apis/google/GooglePhotosUploadUtils.ts | 20 ++++--- src/server/apis/google/existing_uploads.json | 2 +- src/server/credentials/google_docs_token.json | 2 +- src/server/index.ts | 18 +++--- 9 files changed, 83 insertions(+), 69 deletions(-) (limited to 'src/server/apis') diff --git a/src/client/northstar/utils/Extensions.ts b/src/client/northstar/utils/Extensions.ts index f1fddf6c8..04af36731 100644 --- a/src/client/northstar/utils/Extensions.ts +++ b/src/client/northstar/utils/Extensions.ts @@ -29,22 +29,22 @@ type BatchHandler = BatchHandlerSync | BatchHandlerAsync; interface Array { batch(batchSize: number): T[][]; - executeInBatches(batchSize: number, handler: BatchHandlerSync): void; - convertInBatches(batchSize: number, handler: BatchConverterSync): O[]; - executeInBatchesAsync(batchSize: number, handler: BatchHandler): Promise; - convertInBatchesAsync(batchSize: number, handler: BatchConverter): Promise; - executeInBatchesAtInterval(batchSize: number, handler: BatchHandler, interval: number): Promise; - convertInBatchesAtInterval(batchSize: number, handler: BatchConverter, interval: number): Promise; + batchedForEach(batchSize: number, handler: BatchHandlerSync): void; + batchedMap(batchSize: number, handler: BatchConverterSync): O[]; + batchedForEachAsync(batchSize: number, handler: BatchHandler): Promise; + batchedMapAsync(batchSize: number, handler: BatchConverter): Promise; + batchedForEachInterval(batchSize: number, handler: BatchHandler, interval: number): Promise; + batchedMapInterval(batchSize: number, handler: BatchConverter, interval: number): Promise; lastElement(): T; } Array.prototype.batch = extensions.Batch; -Array.prototype.executeInBatches = extensions.ExecuteInBatches; -Array.prototype.convertInBatches = extensions.ConvertInBatches; -Array.prototype.executeInBatchesAsync = extensions.ExecuteInBatchesAsync; -Array.prototype.convertInBatchesAsync = extensions.ConvertInBatchesAsync; -Array.prototype.executeInBatchesAtInterval = extensions.ExecuteInBatchesAtInterval; -Array.prototype.convertInBatchesAtInterval = extensions.ConvertInBatchesAtInterval; +Array.prototype.batchedForEach = extensions.ExecuteInBatches; +Array.prototype.batchedMap = extensions.ConvertInBatches; +Array.prototype.batchedForEachAsync = extensions.ExecuteInBatchesAsync; +Array.prototype.batchedMapAsync = extensions.ConvertInBatchesAsync; +Array.prototype.batchedForEachInterval = extensions.ExecuteInBatchesAtInterval; +Array.prototype.batchedMapInterval = extensions.ConvertInBatchesAtInterval; Array.prototype.lastElement = function () { if (!this.length) { diff --git a/src/client/util/Import & Export/DirectoryImportBox.tsx b/src/client/util/Import & Export/DirectoryImportBox.tsx index 44075ecdd..d371766dd 100644 --- a/src/client/util/Import & Export/DirectoryImportBox.tsx +++ b/src/client/util/Import & Export/DirectoryImportBox.tsx @@ -103,7 +103,7 @@ export default class DirectoryImportBox extends React.Component runInAction(() => this.phase = `Internal: uploading ${this.quota - this.completed} files to Dash...`); - const uploads = await validated.convertInBatchesAsync(15, async (batch: File[]) => { + const uploads = await validated.batchedMapAsync(15, async (batch: File[]) => { const formData = new FormData(); const parameters = { method: 'POST', body: formData }; diff --git a/src/client/views/collections/CollectionSubView.tsx b/src/client/views/collections/CollectionSubView.tsx index 4d4f69b92..59fc11359 100644 --- a/src/client/views/collections/CollectionSubView.tsx +++ b/src/client/views/collections/CollectionSubView.tsx @@ -1,7 +1,7 @@ import { action, computed } from "mobx"; import * as rp from 'request-promise'; import CursorField from "../../../new_fields/CursorField"; -import { Doc, DocListCast } from "../../../new_fields/Doc"; +import { Doc, DocListCast, HeightSym } from "../../../new_fields/Doc"; import { Id } from "../../../new_fields/FieldSymbols"; import { List } from "../../../new_fields/List"; import { listSpec } from "../../../new_fields/Schema"; @@ -23,6 +23,7 @@ import { CollectionVideoView } from "./CollectionVideoView"; import { CollectionView } from "./CollectionView"; import React = require("react"); import { GooglePhotos } from "../../apis/google_docs/GooglePhotosClientUtils"; +import { CollectionDockingView } from "./CollectionDockingView"; export interface CollectionViewProps extends FieldViewProps { addDocument: (document: Doc, allowDuplicates?: boolean) => boolean; @@ -212,11 +213,14 @@ export function CollectionSubView(schemaCtor: (doc: Doc) => T) { if ((matches = /(https:\/\/)?docs\.google\.com\/document\/d\/([^\\]+)\/edit/g.exec(text)) !== null) { let newBox = Docs.Create.TextDocument({ ...options, width: 400, height: 200, title: "Awaiting title from Google Docs..." }); let proto = newBox.proto!; - proto.autoHeight = true; - proto[GoogleRef] = matches[2]; + const documentId = matches[2]; + proto[GoogleRef] = documentId; proto.data = "Please select this document and then click on its pull button to load its contents from from Google Docs..."; proto.backgroundColor = "#eeeeff"; this.props.addDocument(newBox); + // const parent = Docs.Create.StackingDocument([newBox], { title: `Google Doc Import (${documentId})` }); + // CollectionDockingView.Instance.AddRightSplit(parent, undefined); + // proto.height = parent[HeightSym](); return; } if ((matches = /(https:\/\/)?photos\.google\.com\/(u\/3\/)?album\/([^\\]+)/g.exec(text)) !== null) { diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx index b4ca6d797..9c7e8d22f 100644 --- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx +++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx @@ -523,38 +523,38 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) { let y = this.Document.panY || 0; let docs = this.childDocs || []; let [dx, dy] = this.getTransform().transformDirection(e.clientX - this._lastX, e.clientY - this._lastY); - if (!this.isAnnotationOverlay) { - PDFMenu.Instance.fadeOut(true); - let minx = docs.length ? NumCast(docs[0].x) : 0; - let maxx = docs.length ? NumCast(docs[0].width) + minx : minx; - let miny = docs.length ? NumCast(docs[0].y) : 0; - let maxy = docs.length ? NumCast(docs[0].height) + miny : miny; - let ranges = docs.filter(doc => doc).reduce((range, doc) => { - let x = NumCast(doc.x); - let xe = x + NumCast(doc.width); - let y = NumCast(doc.y); - let ye = y + NumCast(doc.height); - return [[range[0][0] > x ? x : range[0][0], range[0][1] < xe ? xe : range[0][1]], - [range[1][0] > y ? y : range[1][0], range[1][1] < ye ? ye : range[1][1]]]; - }, [[minx, maxx], [miny, maxy]]); - let ink = Cast(this.fieldExtensionDoc.ink, InkField); - if (ink && ink.inkData) { - ink.inkData.forEach((value: StrokeData, key: string) => { - let bounds = InkingCanvas.StrokeRect(value); - ranges[0] = [Math.min(ranges[0][0], bounds.left), Math.max(ranges[0][1], bounds.right)]; - ranges[1] = [Math.min(ranges[1][0], bounds.top), Math.max(ranges[1][1], bounds.bottom)]; - }); - } - - let panelDim = this.props.ScreenToLocalTransform().transformDirection(this._pwidth / this.zoomScaling(), - this._pheight / this.zoomScaling()); - let panelwidth = panelDim[0]; - let panelheight = panelDim[1]; - if (ranges[0][0] - dx > (this.panX() + panelwidth / 2)) x = ranges[0][1] + panelwidth / 2; - if (ranges[0][1] - dx < (this.panX() - panelwidth / 2)) x = ranges[0][0] - panelwidth / 2; - if (ranges[1][0] - dy > (this.panY() + panelheight / 2)) y = ranges[1][1] + panelheight / 2; - if (ranges[1][1] - dy < (this.panY() - panelheight / 2)) y = ranges[1][0] - panelheight / 2; - } + // if (!this.isAnnotationOverlay) { + // PDFMenu.Instance.fadeOut(true); + // let minx = docs.length ? NumCast(docs[0].x) : 0; + // let maxx = docs.length ? NumCast(docs[0].width) + minx : minx; + // let miny = docs.length ? NumCast(docs[0].y) : 0; + // let maxy = docs.length ? NumCast(docs[0].height) + miny : miny; + // let ranges = docs.filter(doc => doc).reduce((range, doc) => { + // let x = NumCast(doc.x); + // let xe = x + NumCast(doc.width); + // let y = NumCast(doc.y); + // let ye = y + NumCast(doc.height); + // return [[range[0][0] > x ? x : range[0][0], range[0][1] < xe ? xe : range[0][1]], + // [range[1][0] > y ? y : range[1][0], range[1][1] < ye ? ye : range[1][1]]]; + // }, [[minx, maxx], [miny, maxy]]); + // let ink = Cast(this.fieldExtensionDoc.ink, InkField); + // if (ink && ink.inkData) { + // ink.inkData.forEach((value: StrokeData, key: string) => { + // let bounds = InkingCanvas.StrokeRect(value); + // ranges[0] = [Math.min(ranges[0][0], bounds.left), Math.max(ranges[0][1], bounds.right)]; + // ranges[1] = [Math.min(ranges[1][0], bounds.top), Math.max(ranges[1][1], bounds.bottom)]; + // }); + // } + + // let panelDim = this.props.ScreenToLocalTransform().transformDirection(this._pwidth / this.zoomScaling(), + // this._pheight / this.zoomScaling()); + // let panelwidth = panelDim[0]; + // let panelheight = panelDim[1]; + // if (ranges[0][0] - dx > (this.panX() + panelwidth / 2)) x = ranges[0][1] + panelwidth / 2; + // if (ranges[0][1] - dx < (this.panX() - panelwidth / 2)) x = ranges[0][0] - panelwidth / 2; + // if (ranges[1][0] - dy > (this.panY() + panelheight / 2)) y = ranges[1][1] + panelheight / 2; + // if (ranges[1][1] - dy < (this.panY() - panelheight / 2)) y = ranges[1][0] - panelheight / 2; + // } this.setPan(x - dx, y - dy); this._lastX = e.pageX; this._lastY = e.pageY; diff --git a/src/new_fields/RichTextUtils.ts b/src/new_fields/RichTextUtils.ts index ab5e677c8..5b1da2669 100644 --- a/src/new_fields/RichTextUtils.ts +++ b/src/new_fields/RichTextUtils.ts @@ -278,7 +278,7 @@ export namespace RichTextUtils { } else { docid = backingDocId; } - return schema.node("image", { src, width, docid }); + return schema.node("image", { src, width, docid, float: null }); }; const textNode = (schema: any, run: docs_v1.Schema$TextRun) => { @@ -331,6 +331,7 @@ export namespace RichTextUtils { ["strong", "bold"], ["em", "italic"], ["pFontColor", "foregroundColor"], + ["pFontSize", "fontSize"], ["timesNewRoman", "weightedFontFamily"], ["georgia", "weightedFontFamily"], ["comicSans", "weightedFontFamily"], @@ -382,21 +383,24 @@ export namespace RichTextUtils { const delimiter = "/doc/"; const alreadyShared = "?sharing=true"; if (new RegExp(window.location.origin + delimiter).test(url) && !url.endsWith(alreadyShared)) { + alert("Reassigning alias!"); const linkDoc = await DocServer.GetRefField(url.split(delimiter)[1]); if (linkDoc instanceof Doc) { const target = (await Cast(linkDoc.anchor2, Doc))!; const exported = Doc.MakeAlias(target); DocumentView.makeCustomViewClicked(exported); - target && (url = Utils.shareUrl(exported[Id])); + const documentId = exported[Id]; + target && (url = Utils.shareUrl(documentId)); linkDoc.anchor2 = exported; } } + alert(`url: ${url}`); value = { url }; textStyle.foregroundColor = fromRgb.blue; textStyle.bold = true; break; case "fontSize": - value = attrs.fontSize; + value = { magnitude: attrs.fontSize, unit: "PT" }; break; case "foregroundColor": value = fromHex(attrs.color); diff --git a/src/server/apis/google/GooglePhotosUploadUtils.ts b/src/server/apis/google/GooglePhotosUploadUtils.ts index 3ab9ba90f..734d77fd7 100644 --- a/src/server/apis/google/GooglePhotosUploadUtils.ts +++ b/src/server/apis/google/GooglePhotosUploadUtils.ts @@ -80,7 +80,7 @@ export namespace GooglePhotosUploadUtils { }); })).newMediaItemResults; }; - const newMediaItemResults = await newMediaItems.convertInBatchesAtInterval(50, createFromUploadTokens, 0.1); + const newMediaItemResults = await newMediaItems.batchedMapInterval(50, createFromUploadTokens, 0.1); return { newMediaItemResults }; }; @@ -122,12 +122,20 @@ export namespace DownloadUtils { isLocal: boolean; stream: any; normalizedUrl: string; - contentSize: number; - contentType: string; + contentSize?: number; + contentType?: string; } - export const InspectImage = async (url: string) => { + export const InspectImage = async (url: string): Promise => { const { isLocal, stream, normalized: normalizedUrl } = classify(url); + const results = { + isLocal, + stream, + normalizedUrl + }; + if (isLocal) { + return results; + } const metadata = (await new Promise((resolve, reject) => { request.head(url, async (error, res) => { if (error) { @@ -139,9 +147,7 @@ export namespace DownloadUtils { return { contentSize: parseInt(metadata[size]), contentType: metadata[type], - isLocal, - stream, - normalizedUrl + ...results }; }; diff --git a/src/server/apis/google/existing_uploads.json b/src/server/apis/google/existing_uploads.json index 05c20c33b..4d723868e 100644 --- a/src/server/apis/google/existing_uploads.json +++ b/src/server/apis/google/existing_uploads.json @@ -1 +1 @@ -{"23625":{"mediaPaths":["C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_7e2d5fef-860a-49a8-b9ec-b91f28073180_o.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_7e2d5fef-860a-49a8-b9ec-b91f28073180_s.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_7e2d5fef-860a-49a8-b9ec-b91f28073180_m.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_7e2d5fef-860a-49a8-b9ec-b91f28073180_l.png"],"fileNames":{"clean":"upload_7e2d5fef-860a-49a8-b9ec-b91f28073180.png","_o":"upload_7e2d5fef-860a-49a8-b9ec-b91f28073180_o.png","_s":"upload_7e2d5fef-860a-49a8-b9ec-b91f28073180_s.png","_m":"upload_7e2d5fef-860a-49a8-b9ec-b91f28073180_m.png","_l":"upload_7e2d5fef-860a-49a8-b9ec-b91f28073180_l.png"},"contentSize":23625,"contentType":"image/jpeg"}} \ No newline at end of file +{"23394":{"mediaPaths":["C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_e6e6239c-a436-4f08-bea5-de29ad4e72f2_o.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_e6e6239c-a436-4f08-bea5-de29ad4e72f2_s.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_e6e6239c-a436-4f08-bea5-de29ad4e72f2_m.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_e6e6239c-a436-4f08-bea5-de29ad4e72f2_l.png"],"fileNames":{"clean":"upload_e6e6239c-a436-4f08-bea5-de29ad4e72f2.png","_o":"upload_e6e6239c-a436-4f08-bea5-de29ad4e72f2_o.png","_s":"upload_e6e6239c-a436-4f08-bea5-de29ad4e72f2_s.png","_m":"upload_e6e6239c-a436-4f08-bea5-de29ad4e72f2_m.png","_l":"upload_e6e6239c-a436-4f08-bea5-de29ad4e72f2_l.png"},"contentSize":23394,"contentType":"image/jpeg"},"23406":{"mediaPaths":["C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_2b310488-a12b-4ecf-8adf-38943282381a_o.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_2b310488-a12b-4ecf-8adf-38943282381a_s.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_2b310488-a12b-4ecf-8adf-38943282381a_m.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_2b310488-a12b-4ecf-8adf-38943282381a_l.png"],"fileNames":{"clean":"upload_2b310488-a12b-4ecf-8adf-38943282381a.png","_o":"upload_2b310488-a12b-4ecf-8adf-38943282381a_o.png","_s":"upload_2b310488-a12b-4ecf-8adf-38943282381a_s.png","_m":"upload_2b310488-a12b-4ecf-8adf-38943282381a_m.png","_l":"upload_2b310488-a12b-4ecf-8adf-38943282381a_l.png"},"contentSize":23406,"contentType":"image/jpeg"},"23408":{"mediaPaths":["C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_a35aef02-3dac-434f-a2d5-932ee3fc6b78_o.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_a35aef02-3dac-434f-a2d5-932ee3fc6b78_s.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_a35aef02-3dac-434f-a2d5-932ee3fc6b78_m.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_a35aef02-3dac-434f-a2d5-932ee3fc6b78_l.png"],"fileNames":{"clean":"upload_a35aef02-3dac-434f-a2d5-932ee3fc6b78.png","_o":"upload_a35aef02-3dac-434f-a2d5-932ee3fc6b78_o.png","_s":"upload_a35aef02-3dac-434f-a2d5-932ee3fc6b78_s.png","_m":"upload_a35aef02-3dac-434f-a2d5-932ee3fc6b78_m.png","_l":"upload_a35aef02-3dac-434f-a2d5-932ee3fc6b78_l.png"},"contentSize":23408,"contentType":"image/jpeg"},"23413":{"mediaPaths":["C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_8babdd04-c4af-4ebf-9e48-bcf5cb51ab6c_o.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_8babdd04-c4af-4ebf-9e48-bcf5cb51ab6c_s.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_8babdd04-c4af-4ebf-9e48-bcf5cb51ab6c_m.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_8babdd04-c4af-4ebf-9e48-bcf5cb51ab6c_l.png"],"fileNames":{"clean":"upload_8babdd04-c4af-4ebf-9e48-bcf5cb51ab6c.png","_o":"upload_8babdd04-c4af-4ebf-9e48-bcf5cb51ab6c_o.png","_s":"upload_8babdd04-c4af-4ebf-9e48-bcf5cb51ab6c_s.png","_m":"upload_8babdd04-c4af-4ebf-9e48-bcf5cb51ab6c_m.png","_l":"upload_8babdd04-c4af-4ebf-9e48-bcf5cb51ab6c_l.png"},"contentSize":23413,"contentType":"image/jpeg"},"23415":{"mediaPaths":["C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_2e51d02c-a113-4eec-8030-badb54d0f719_o.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_2e51d02c-a113-4eec-8030-badb54d0f719_s.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_2e51d02c-a113-4eec-8030-badb54d0f719_m.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_2e51d02c-a113-4eec-8030-badb54d0f719_l.png"],"fileNames":{"clean":"upload_2e51d02c-a113-4eec-8030-badb54d0f719.png","_o":"upload_2e51d02c-a113-4eec-8030-badb54d0f719_o.png","_s":"upload_2e51d02c-a113-4eec-8030-badb54d0f719_s.png","_m":"upload_2e51d02c-a113-4eec-8030-badb54d0f719_m.png","_l":"upload_2e51d02c-a113-4eec-8030-badb54d0f719_l.png"},"contentSize":23415,"contentType":"image/jpeg"},"23466":{"mediaPaths":["C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_ec4c15ea-9858-4886-bc15-03a5073b8f4f_o.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_ec4c15ea-9858-4886-bc15-03a5073b8f4f_s.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_ec4c15ea-9858-4886-bc15-03a5073b8f4f_m.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_ec4c15ea-9858-4886-bc15-03a5073b8f4f_l.png"],"fileNames":{"clean":"upload_ec4c15ea-9858-4886-bc15-03a5073b8f4f.png","_o":"upload_ec4c15ea-9858-4886-bc15-03a5073b8f4f_o.png","_s":"upload_ec4c15ea-9858-4886-bc15-03a5073b8f4f_s.png","_m":"upload_ec4c15ea-9858-4886-bc15-03a5073b8f4f_m.png","_l":"upload_ec4c15ea-9858-4886-bc15-03a5073b8f4f_l.png"},"contentSize":23466,"contentType":"image/jpeg"},"23625":{"mediaPaths":["C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_e1488792-46cb-4ff5-bb26-f62ea2ef91d1_o.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_e1488792-46cb-4ff5-bb26-f62ea2ef91d1_s.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_e1488792-46cb-4ff5-bb26-f62ea2ef91d1_m.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_e1488792-46cb-4ff5-bb26-f62ea2ef91d1_l.png"],"fileNames":{"clean":"upload_e1488792-46cb-4ff5-bb26-f62ea2ef91d1.png","_o":"upload_e1488792-46cb-4ff5-bb26-f62ea2ef91d1_o.png","_s":"upload_e1488792-46cb-4ff5-bb26-f62ea2ef91d1_s.png","_m":"upload_e1488792-46cb-4ff5-bb26-f62ea2ef91d1_m.png","_l":"upload_e1488792-46cb-4ff5-bb26-f62ea2ef91d1_l.png"},"contentSize":23625,"contentType":"image/jpeg"},"45184":{"mediaPaths":["C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_c0de990f-e3b5-4830-ab5d-17cfb4ddc402_o.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_c0de990f-e3b5-4830-ab5d-17cfb4ddc402_s.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_c0de990f-e3b5-4830-ab5d-17cfb4ddc402_m.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_c0de990f-e3b5-4830-ab5d-17cfb4ddc402_l.png"],"fileNames":{"clean":"upload_c0de990f-e3b5-4830-ab5d-17cfb4ddc402.png","_o":"upload_c0de990f-e3b5-4830-ab5d-17cfb4ddc402_o.png","_s":"upload_c0de990f-e3b5-4830-ab5d-17cfb4ddc402_s.png","_m":"upload_c0de990f-e3b5-4830-ab5d-17cfb4ddc402_m.png","_l":"upload_c0de990f-e3b5-4830-ab5d-17cfb4ddc402_l.png"},"contentSize":45184,"contentType":"image/jpeg"},"45211":{"mediaPaths":["C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_43ae4f14-b987-4a1a-9430-cc90bb24a9c8_o.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_43ae4f14-b987-4a1a-9430-cc90bb24a9c8_s.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_43ae4f14-b987-4a1a-9430-cc90bb24a9c8_m.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_43ae4f14-b987-4a1a-9430-cc90bb24a9c8_l.png"],"fileNames":{"clean":"upload_43ae4f14-b987-4a1a-9430-cc90bb24a9c8.png","_o":"upload_43ae4f14-b987-4a1a-9430-cc90bb24a9c8_o.png","_s":"upload_43ae4f14-b987-4a1a-9430-cc90bb24a9c8_s.png","_m":"upload_43ae4f14-b987-4a1a-9430-cc90bb24a9c8_m.png","_l":"upload_43ae4f14-b987-4a1a-9430-cc90bb24a9c8_l.png"},"contentSize":45211,"contentType":"image/jpeg"},"45228":{"mediaPaths":["C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_608688a6-53bc-4d1f-adfc-3aac153066fb_o.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_608688a6-53bc-4d1f-adfc-3aac153066fb_s.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_608688a6-53bc-4d1f-adfc-3aac153066fb_m.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_608688a6-53bc-4d1f-adfc-3aac153066fb_l.png"],"fileNames":{"clean":"upload_608688a6-53bc-4d1f-adfc-3aac153066fb.png","_o":"upload_608688a6-53bc-4d1f-adfc-3aac153066fb_o.png","_s":"upload_608688a6-53bc-4d1f-adfc-3aac153066fb_s.png","_m":"upload_608688a6-53bc-4d1f-adfc-3aac153066fb_m.png","_l":"upload_608688a6-53bc-4d1f-adfc-3aac153066fb_l.png"},"contentSize":45228,"contentType":"image/jpeg"},"45247":{"mediaPaths":["C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_1de378be-f389-41d7-a6bf-d680b2eee551_o.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_1de378be-f389-41d7-a6bf-d680b2eee551_s.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_1de378be-f389-41d7-a6bf-d680b2eee551_m.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_1de378be-f389-41d7-a6bf-d680b2eee551_l.png"],"fileNames":{"clean":"upload_1de378be-f389-41d7-a6bf-d680b2eee551.png","_o":"upload_1de378be-f389-41d7-a6bf-d680b2eee551_o.png","_s":"upload_1de378be-f389-41d7-a6bf-d680b2eee551_s.png","_m":"upload_1de378be-f389-41d7-a6bf-d680b2eee551_m.png","_l":"upload_1de378be-f389-41d7-a6bf-d680b2eee551_l.png"},"contentSize":45247,"contentType":"image/jpeg"},"45263":{"mediaPaths":["C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_984f220d-1409-418d-998b-44667879fde2_o.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_984f220d-1409-418d-998b-44667879fde2_s.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_984f220d-1409-418d-998b-44667879fde2_m.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_984f220d-1409-418d-998b-44667879fde2_l.png"],"fileNames":{"clean":"upload_984f220d-1409-418d-998b-44667879fde2.png","_o":"upload_984f220d-1409-418d-998b-44667879fde2_o.png","_s":"upload_984f220d-1409-418d-998b-44667879fde2_s.png","_m":"upload_984f220d-1409-418d-998b-44667879fde2_m.png","_l":"upload_984f220d-1409-418d-998b-44667879fde2_l.png"},"contentSize":45263,"contentType":"image/jpeg"},"45273":{"mediaPaths":["C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_86280e72-0c1e-40cf-a6c3-79291a7ec384_o.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_86280e72-0c1e-40cf-a6c3-79291a7ec384_s.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_86280e72-0c1e-40cf-a6c3-79291a7ec384_m.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_86280e72-0c1e-40cf-a6c3-79291a7ec384_l.png"],"fileNames":{"clean":"upload_86280e72-0c1e-40cf-a6c3-79291a7ec384.png","_o":"upload_86280e72-0c1e-40cf-a6c3-79291a7ec384_o.png","_s":"upload_86280e72-0c1e-40cf-a6c3-79291a7ec384_s.png","_m":"upload_86280e72-0c1e-40cf-a6c3-79291a7ec384_m.png","_l":"upload_86280e72-0c1e-40cf-a6c3-79291a7ec384_l.png"},"contentSize":45273,"contentType":"image/jpeg"},"45492":{"mediaPaths":["C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_a3070781-0b4d-43f7-80b8-5be8138d0930_o.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_a3070781-0b4d-43f7-80b8-5be8138d0930_s.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_a3070781-0b4d-43f7-80b8-5be8138d0930_m.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_a3070781-0b4d-43f7-80b8-5be8138d0930_l.png"],"fileNames":{"clean":"upload_a3070781-0b4d-43f7-80b8-5be8138d0930.png","_o":"upload_a3070781-0b4d-43f7-80b8-5be8138d0930_o.png","_s":"upload_a3070781-0b4d-43f7-80b8-5be8138d0930_s.png","_m":"upload_a3070781-0b4d-43f7-80b8-5be8138d0930_m.png","_l":"upload_a3070781-0b4d-43f7-80b8-5be8138d0930_l.png"},"contentSize":45492,"contentType":"image/jpeg"},"45510":{"mediaPaths":["C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_892f666b-ac98-4feb-9017-39448434b732_o.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_892f666b-ac98-4feb-9017-39448434b732_s.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_892f666b-ac98-4feb-9017-39448434b732_m.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_892f666b-ac98-4feb-9017-39448434b732_l.png"],"fileNames":{"clean":"upload_892f666b-ac98-4feb-9017-39448434b732.png","_o":"upload_892f666b-ac98-4feb-9017-39448434b732_o.png","_s":"upload_892f666b-ac98-4feb-9017-39448434b732_s.png","_m":"upload_892f666b-ac98-4feb-9017-39448434b732_m.png","_l":"upload_892f666b-ac98-4feb-9017-39448434b732_l.png"},"contentSize":45510,"contentType":"image/jpeg"},"45585":{"mediaPaths":["C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_f684c94c-7d37-47ed-91cb-bb458f9cff15_o.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_f684c94c-7d37-47ed-91cb-bb458f9cff15_s.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_f684c94c-7d37-47ed-91cb-bb458f9cff15_m.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_f684c94c-7d37-47ed-91cb-bb458f9cff15_l.png"],"fileNames":{"clean":"upload_f684c94c-7d37-47ed-91cb-bb458f9cff15.png","_o":"upload_f684c94c-7d37-47ed-91cb-bb458f9cff15_o.png","_s":"upload_f684c94c-7d37-47ed-91cb-bb458f9cff15_s.png","_m":"upload_f684c94c-7d37-47ed-91cb-bb458f9cff15_m.png","_l":"upload_f684c94c-7d37-47ed-91cb-bb458f9cff15_l.png"},"contentSize":45585,"contentType":"image/jpeg"},"74829":{"mediaPaths":["C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_cc43cdb6-b877-4b30-a702-b7f591a8bca5_o.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_cc43cdb6-b877-4b30-a702-b7f591a8bca5_s.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_cc43cdb6-b877-4b30-a702-b7f591a8bca5_m.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_cc43cdb6-b877-4b30-a702-b7f591a8bca5_l.png"],"fileNames":{"clean":"upload_cc43cdb6-b877-4b30-a702-b7f591a8bca5.png","_o":"upload_cc43cdb6-b877-4b30-a702-b7f591a8bca5_o.png","_s":"upload_cc43cdb6-b877-4b30-a702-b7f591a8bca5_s.png","_m":"upload_cc43cdb6-b877-4b30-a702-b7f591a8bca5_m.png","_l":"upload_cc43cdb6-b877-4b30-a702-b7f591a8bca5_l.png"},"contentSize":74829,"contentType":"image/jpeg"}} \ No newline at end of file diff --git a/src/server/credentials/google_docs_token.json b/src/server/credentials/google_docs_token.json index c10b0797f..5998ccfd8 100644 --- a/src/server/credentials/google_docs_token.json +++ b/src/server/credentials/google_docs_token.json @@ -1 +1 @@ -{"access_token":"ya29.ImCFB_ghOybVB6A4HvIIwIlyGyZw6wOymdwJyWJJECIpCmFTHNEzOAfP98KFzm5OUV2zZNS5Wx1iUT1xYWW35PY7NoZc7PWwjzmOaGkMzDm7_fxpsgjT0StdvEwTJprFIv0","refresh_token":"1/HTv_xFHszu2Nf3iiFrUTaeKzC_Vp2-6bpIB06xW_WHI","scope":"https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/documents.readonly https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/photoslibrary https://www.googleapis.com/auth/photoslibrary.appendonly https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/photoslibrary.sharing","token_type":"Bearer","expiry_date":1568590984976} \ No newline at end of file +{"access_token":"ya29.GlyFByFfWe7VNNjHImJwA58yoh2cAKDJUPhBKn5IDVUY9oPlXbpyhdJMjfGSRhDZpgEWM0QoSqONu9gBVWDV9aXsf7p8r9TDXK7jBfWs1Qnox4zPf54kSfCHsmV6iw","refresh_token":"1/HTv_xFHszu2Nf3iiFrUTaeKzC_Vp2-6bpIB06xW_WHI","scope":"https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/documents.readonly https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/photoslibrary https://www.googleapis.com/auth/photoslibrary.appendonly https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/photoslibrary.sharing","token_type":"Bearer","expiry_date":1568646012183} \ No newline at end of file diff --git a/src/server/index.ts b/src/server/index.ts index 07ce4b6f0..9da6a8b38 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -836,12 +836,12 @@ export interface NewMediaItem { } Array.prototype.batch = extensions.Batch; -Array.prototype.executeInBatches = extensions.ExecuteInBatches; -Array.prototype.convertInBatches = extensions.ConvertInBatches; -Array.prototype.executeInBatchesAsync = extensions.ExecuteInBatchesAsync; -Array.prototype.convertInBatchesAsync = extensions.ConvertInBatchesAsync; -Array.prototype.executeInBatchesAtInterval = extensions.ExecuteInBatchesAtInterval; -Array.prototype.convertInBatchesAtInterval = extensions.ConvertInBatchesAtInterval; +Array.prototype.batchedForEach = extensions.ExecuteInBatches; +Array.prototype.batchedMap = extensions.ConvertInBatches; +Array.prototype.batchedForEachAsync = extensions.ExecuteInBatchesAsync; +Array.prototype.batchedMapAsync = extensions.ConvertInBatchesAsync; +Array.prototype.batchedForEachInterval = extensions.ExecuteInBatchesAtInterval; +Array.prototype.batchedMapInterval = extensions.ConvertInBatchesAtInterval; app.post(RouteStore.googlePhotosMediaUpload, async (req, res) => { const mediaInput: GooglePhotosUploadUtils.MediaInput[] = req.body.media; @@ -865,7 +865,7 @@ app.post(RouteStore.googlePhotosMediaUpload, async (req, res) => { return newMediaItems; }; - const newMediaItems = await mediaInput.convertInBatchesAtInterval(25, dispatchUpload, 0.1); + const newMediaItems = await mediaInput.batchedMapInterval(25, dispatchUpload, 0.1); if (failed) { return _error(res, tokenError); @@ -900,10 +900,10 @@ app.post(RouteStore.googlePhotosMediaDownload, async (req, res) => { let existing = content.length ? JSON.parse(content) : {}; for (let item of contents.mediaItems) { const { contentSize, ...attributes } = await UploadUtils.InspectImage(item.baseUrl); - const found: UploadUtils.UploadInformation = existing[contentSize]; + const found: UploadUtils.UploadInformation = existing[contentSize!]; if (!found) { const upload = await UploadUtils.UploadImage({ contentSize, ...attributes }, item.filename, prefix).catch(error => _error(res, downloadError, error)); - upload && completed.push(existing[contentSize] = upload); + upload && completed.push(existing[contentSize!] = upload); } else { completed.push(found); } -- cgit v1.2.3-70-g09d2 From 36a4caa28a7a83823027e6f0af47ffb1878ae86b Mon Sep 17 00:00:00 2001 From: bob Date: Mon, 16 Sep 2019 12:19:17 -0400 Subject: fixes for converting between google and our doc formats --- src/client/util/RichTextSchema.tsx | 2 ++ src/client/views/nodes/DocumentView.tsx | 1 + src/new_fields/RichTextUtils.ts | 32 +++++++++++++++------------ src/server/apis/google/existing_uploads.json | 2 +- src/server/credentials/google_docs_token.json | 2 +- 5 files changed, 23 insertions(+), 16 deletions(-) (limited to 'src/server/apis') diff --git a/src/client/util/RichTextSchema.tsx b/src/client/util/RichTextSchema.tsx index f027a4bf7..2750203f2 100644 --- a/src/client/util/RichTextSchema.tsx +++ b/src/client/util/RichTextSchema.tsx @@ -135,6 +135,7 @@ export const nodes: { [index: string]: NodeSpec } = { alt: { default: null }, title: { default: null }, float: { default: "left" }, + location: { default: "onRight" }, docid: { default: "" } }, group: "inline", @@ -616,6 +617,7 @@ export class ImageResizeView { e.preventDefault(); e.stopPropagation(); DocServer.GetRefField(node.attrs.docid).then(async linkDoc => { + const location = node.attrs.location; if (linkDoc instanceof Doc) { let proto = Doc.GetProto(linkDoc); let targetContext = await Cast(proto.targetContext, Doc); diff --git a/src/client/views/nodes/DocumentView.tsx b/src/client/views/nodes/DocumentView.tsx index 81805af64..bf7e7df97 100644 --- a/src/client/views/nodes/DocumentView.tsx +++ b/src/client/views/nodes/DocumentView.tsx @@ -606,6 +606,7 @@ export class DocumentView extends DocComponent(Docu Doc.MakeTemplate(fieldTemplate, metaKey, proto); Doc.ApplyTemplateTo(docTemplate, document, undefined, false); + document.customLayout = document.layout; } }); }); diff --git a/src/new_fields/RichTextUtils.ts b/src/new_fields/RichTextUtils.ts index 44fdae638..ff28e6861 100644 --- a/src/new_fields/RichTextUtils.ts +++ b/src/new_fields/RichTextUtils.ts @@ -195,16 +195,19 @@ export namespace RichTextUtils { nodes.push(list(state.schema, items)); } else { if (element.contents.some(child => "inlineObjectId" in child)) { - let node: Node; - for (const child of element.contents) { + const group = element.contents; + group.forEach((child, i) => { + let node: Opt>; if ("inlineObjectId" in child) { node = imageNode(state.schema, inlineObjectMap.get(child.inlineObjectId!)!, textNote); - } else { + } else if ("content" in child && (i !== group.length - 1 || child.content!.removeTrailingNewlines().length)) { node = paragraphNode(state.schema, [child]); } - nodes.push(node); - position += node.nodeSize; - } + if (node) { + position += node.nodeSize; + nodes.push(node); + } + }); } else { let paragraph = paragraphNode(state.schema, element.contents); nodes.push(paragraph); @@ -278,7 +281,7 @@ export namespace RichTextUtils { } else { docid = backingDocId; } - return schema.node("image", { src, width, docid, float: null }); + return schema.node("image", { src, width, docid, float: null, location: "onRight" }); }; const textNode = (schema: any, run: docs_v1.Schema$TextRun) => { @@ -385,12 +388,13 @@ export namespace RichTextUtils { if (new RegExp(window.location.origin + delimiter).test(url) && !url.endsWith(alreadyShared)) { const linkDoc = await DocServer.GetRefField(url.split(delimiter)[1]); if (linkDoc instanceof Doc) { - const target = (await Cast(linkDoc.anchor2, Doc))!; - const exported = Doc.MakeAlias(target); - DocumentView.makeCustomViewClicked(exported); - const documentId = exported[Id]; - target && (url = Utils.shareUrl(documentId)); - linkDoc.anchor2 = exported; + let exported = (await Cast(linkDoc.anchor2, Doc))!; + if (!exported.customLayout) { + exported = Doc.MakeAlias(exported); + DocumentView.makeCustomViewClicked(exported); + linkDoc.anchor2 = exported; + } + url = Utils.shareUrl(exported[Id]); } } value = { url }; @@ -418,7 +422,7 @@ export namespace RichTextUtils { } if (node.type.name === "image") { requests.push(await EncodeImage({ - startIndex: position + nodeSize, + startIndex: position + nodeSize - 1, uri: attrs.src, width: attrs.width })); diff --git a/src/server/apis/google/existing_uploads.json b/src/server/apis/google/existing_uploads.json index 4d723868e..399c27672 100644 --- a/src/server/apis/google/existing_uploads.json +++ b/src/server/apis/google/existing_uploads.json @@ -1 +1 @@ -{"23394":{"mediaPaths":["C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_e6e6239c-a436-4f08-bea5-de29ad4e72f2_o.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_e6e6239c-a436-4f08-bea5-de29ad4e72f2_s.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_e6e6239c-a436-4f08-bea5-de29ad4e72f2_m.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_e6e6239c-a436-4f08-bea5-de29ad4e72f2_l.png"],"fileNames":{"clean":"upload_e6e6239c-a436-4f08-bea5-de29ad4e72f2.png","_o":"upload_e6e6239c-a436-4f08-bea5-de29ad4e72f2_o.png","_s":"upload_e6e6239c-a436-4f08-bea5-de29ad4e72f2_s.png","_m":"upload_e6e6239c-a436-4f08-bea5-de29ad4e72f2_m.png","_l":"upload_e6e6239c-a436-4f08-bea5-de29ad4e72f2_l.png"},"contentSize":23394,"contentType":"image/jpeg"},"23406":{"mediaPaths":["C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_2b310488-a12b-4ecf-8adf-38943282381a_o.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_2b310488-a12b-4ecf-8adf-38943282381a_s.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_2b310488-a12b-4ecf-8adf-38943282381a_m.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_2b310488-a12b-4ecf-8adf-38943282381a_l.png"],"fileNames":{"clean":"upload_2b310488-a12b-4ecf-8adf-38943282381a.png","_o":"upload_2b310488-a12b-4ecf-8adf-38943282381a_o.png","_s":"upload_2b310488-a12b-4ecf-8adf-38943282381a_s.png","_m":"upload_2b310488-a12b-4ecf-8adf-38943282381a_m.png","_l":"upload_2b310488-a12b-4ecf-8adf-38943282381a_l.png"},"contentSize":23406,"contentType":"image/jpeg"},"23408":{"mediaPaths":["C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_a35aef02-3dac-434f-a2d5-932ee3fc6b78_o.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_a35aef02-3dac-434f-a2d5-932ee3fc6b78_s.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_a35aef02-3dac-434f-a2d5-932ee3fc6b78_m.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_a35aef02-3dac-434f-a2d5-932ee3fc6b78_l.png"],"fileNames":{"clean":"upload_a35aef02-3dac-434f-a2d5-932ee3fc6b78.png","_o":"upload_a35aef02-3dac-434f-a2d5-932ee3fc6b78_o.png","_s":"upload_a35aef02-3dac-434f-a2d5-932ee3fc6b78_s.png","_m":"upload_a35aef02-3dac-434f-a2d5-932ee3fc6b78_m.png","_l":"upload_a35aef02-3dac-434f-a2d5-932ee3fc6b78_l.png"},"contentSize":23408,"contentType":"image/jpeg"},"23413":{"mediaPaths":["C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_8babdd04-c4af-4ebf-9e48-bcf5cb51ab6c_o.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_8babdd04-c4af-4ebf-9e48-bcf5cb51ab6c_s.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_8babdd04-c4af-4ebf-9e48-bcf5cb51ab6c_m.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_8babdd04-c4af-4ebf-9e48-bcf5cb51ab6c_l.png"],"fileNames":{"clean":"upload_8babdd04-c4af-4ebf-9e48-bcf5cb51ab6c.png","_o":"upload_8babdd04-c4af-4ebf-9e48-bcf5cb51ab6c_o.png","_s":"upload_8babdd04-c4af-4ebf-9e48-bcf5cb51ab6c_s.png","_m":"upload_8babdd04-c4af-4ebf-9e48-bcf5cb51ab6c_m.png","_l":"upload_8babdd04-c4af-4ebf-9e48-bcf5cb51ab6c_l.png"},"contentSize":23413,"contentType":"image/jpeg"},"23415":{"mediaPaths":["C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_2e51d02c-a113-4eec-8030-badb54d0f719_o.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_2e51d02c-a113-4eec-8030-badb54d0f719_s.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_2e51d02c-a113-4eec-8030-badb54d0f719_m.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_2e51d02c-a113-4eec-8030-badb54d0f719_l.png"],"fileNames":{"clean":"upload_2e51d02c-a113-4eec-8030-badb54d0f719.png","_o":"upload_2e51d02c-a113-4eec-8030-badb54d0f719_o.png","_s":"upload_2e51d02c-a113-4eec-8030-badb54d0f719_s.png","_m":"upload_2e51d02c-a113-4eec-8030-badb54d0f719_m.png","_l":"upload_2e51d02c-a113-4eec-8030-badb54d0f719_l.png"},"contentSize":23415,"contentType":"image/jpeg"},"23466":{"mediaPaths":["C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_ec4c15ea-9858-4886-bc15-03a5073b8f4f_o.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_ec4c15ea-9858-4886-bc15-03a5073b8f4f_s.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_ec4c15ea-9858-4886-bc15-03a5073b8f4f_m.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_ec4c15ea-9858-4886-bc15-03a5073b8f4f_l.png"],"fileNames":{"clean":"upload_ec4c15ea-9858-4886-bc15-03a5073b8f4f.png","_o":"upload_ec4c15ea-9858-4886-bc15-03a5073b8f4f_o.png","_s":"upload_ec4c15ea-9858-4886-bc15-03a5073b8f4f_s.png","_m":"upload_ec4c15ea-9858-4886-bc15-03a5073b8f4f_m.png","_l":"upload_ec4c15ea-9858-4886-bc15-03a5073b8f4f_l.png"},"contentSize":23466,"contentType":"image/jpeg"},"23625":{"mediaPaths":["C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_e1488792-46cb-4ff5-bb26-f62ea2ef91d1_o.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_e1488792-46cb-4ff5-bb26-f62ea2ef91d1_s.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_e1488792-46cb-4ff5-bb26-f62ea2ef91d1_m.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_e1488792-46cb-4ff5-bb26-f62ea2ef91d1_l.png"],"fileNames":{"clean":"upload_e1488792-46cb-4ff5-bb26-f62ea2ef91d1.png","_o":"upload_e1488792-46cb-4ff5-bb26-f62ea2ef91d1_o.png","_s":"upload_e1488792-46cb-4ff5-bb26-f62ea2ef91d1_s.png","_m":"upload_e1488792-46cb-4ff5-bb26-f62ea2ef91d1_m.png","_l":"upload_e1488792-46cb-4ff5-bb26-f62ea2ef91d1_l.png"},"contentSize":23625,"contentType":"image/jpeg"},"45184":{"mediaPaths":["C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_c0de990f-e3b5-4830-ab5d-17cfb4ddc402_o.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_c0de990f-e3b5-4830-ab5d-17cfb4ddc402_s.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_c0de990f-e3b5-4830-ab5d-17cfb4ddc402_m.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_c0de990f-e3b5-4830-ab5d-17cfb4ddc402_l.png"],"fileNames":{"clean":"upload_c0de990f-e3b5-4830-ab5d-17cfb4ddc402.png","_o":"upload_c0de990f-e3b5-4830-ab5d-17cfb4ddc402_o.png","_s":"upload_c0de990f-e3b5-4830-ab5d-17cfb4ddc402_s.png","_m":"upload_c0de990f-e3b5-4830-ab5d-17cfb4ddc402_m.png","_l":"upload_c0de990f-e3b5-4830-ab5d-17cfb4ddc402_l.png"},"contentSize":45184,"contentType":"image/jpeg"},"45211":{"mediaPaths":["C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_43ae4f14-b987-4a1a-9430-cc90bb24a9c8_o.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_43ae4f14-b987-4a1a-9430-cc90bb24a9c8_s.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_43ae4f14-b987-4a1a-9430-cc90bb24a9c8_m.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_43ae4f14-b987-4a1a-9430-cc90bb24a9c8_l.png"],"fileNames":{"clean":"upload_43ae4f14-b987-4a1a-9430-cc90bb24a9c8.png","_o":"upload_43ae4f14-b987-4a1a-9430-cc90bb24a9c8_o.png","_s":"upload_43ae4f14-b987-4a1a-9430-cc90bb24a9c8_s.png","_m":"upload_43ae4f14-b987-4a1a-9430-cc90bb24a9c8_m.png","_l":"upload_43ae4f14-b987-4a1a-9430-cc90bb24a9c8_l.png"},"contentSize":45211,"contentType":"image/jpeg"},"45228":{"mediaPaths":["C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_608688a6-53bc-4d1f-adfc-3aac153066fb_o.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_608688a6-53bc-4d1f-adfc-3aac153066fb_s.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_608688a6-53bc-4d1f-adfc-3aac153066fb_m.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_608688a6-53bc-4d1f-adfc-3aac153066fb_l.png"],"fileNames":{"clean":"upload_608688a6-53bc-4d1f-adfc-3aac153066fb.png","_o":"upload_608688a6-53bc-4d1f-adfc-3aac153066fb_o.png","_s":"upload_608688a6-53bc-4d1f-adfc-3aac153066fb_s.png","_m":"upload_608688a6-53bc-4d1f-adfc-3aac153066fb_m.png","_l":"upload_608688a6-53bc-4d1f-adfc-3aac153066fb_l.png"},"contentSize":45228,"contentType":"image/jpeg"},"45247":{"mediaPaths":["C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_1de378be-f389-41d7-a6bf-d680b2eee551_o.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_1de378be-f389-41d7-a6bf-d680b2eee551_s.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_1de378be-f389-41d7-a6bf-d680b2eee551_m.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_1de378be-f389-41d7-a6bf-d680b2eee551_l.png"],"fileNames":{"clean":"upload_1de378be-f389-41d7-a6bf-d680b2eee551.png","_o":"upload_1de378be-f389-41d7-a6bf-d680b2eee551_o.png","_s":"upload_1de378be-f389-41d7-a6bf-d680b2eee551_s.png","_m":"upload_1de378be-f389-41d7-a6bf-d680b2eee551_m.png","_l":"upload_1de378be-f389-41d7-a6bf-d680b2eee551_l.png"},"contentSize":45247,"contentType":"image/jpeg"},"45263":{"mediaPaths":["C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_984f220d-1409-418d-998b-44667879fde2_o.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_984f220d-1409-418d-998b-44667879fde2_s.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_984f220d-1409-418d-998b-44667879fde2_m.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_984f220d-1409-418d-998b-44667879fde2_l.png"],"fileNames":{"clean":"upload_984f220d-1409-418d-998b-44667879fde2.png","_o":"upload_984f220d-1409-418d-998b-44667879fde2_o.png","_s":"upload_984f220d-1409-418d-998b-44667879fde2_s.png","_m":"upload_984f220d-1409-418d-998b-44667879fde2_m.png","_l":"upload_984f220d-1409-418d-998b-44667879fde2_l.png"},"contentSize":45263,"contentType":"image/jpeg"},"45273":{"mediaPaths":["C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_86280e72-0c1e-40cf-a6c3-79291a7ec384_o.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_86280e72-0c1e-40cf-a6c3-79291a7ec384_s.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_86280e72-0c1e-40cf-a6c3-79291a7ec384_m.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_86280e72-0c1e-40cf-a6c3-79291a7ec384_l.png"],"fileNames":{"clean":"upload_86280e72-0c1e-40cf-a6c3-79291a7ec384.png","_o":"upload_86280e72-0c1e-40cf-a6c3-79291a7ec384_o.png","_s":"upload_86280e72-0c1e-40cf-a6c3-79291a7ec384_s.png","_m":"upload_86280e72-0c1e-40cf-a6c3-79291a7ec384_m.png","_l":"upload_86280e72-0c1e-40cf-a6c3-79291a7ec384_l.png"},"contentSize":45273,"contentType":"image/jpeg"},"45492":{"mediaPaths":["C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_a3070781-0b4d-43f7-80b8-5be8138d0930_o.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_a3070781-0b4d-43f7-80b8-5be8138d0930_s.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_a3070781-0b4d-43f7-80b8-5be8138d0930_m.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_a3070781-0b4d-43f7-80b8-5be8138d0930_l.png"],"fileNames":{"clean":"upload_a3070781-0b4d-43f7-80b8-5be8138d0930.png","_o":"upload_a3070781-0b4d-43f7-80b8-5be8138d0930_o.png","_s":"upload_a3070781-0b4d-43f7-80b8-5be8138d0930_s.png","_m":"upload_a3070781-0b4d-43f7-80b8-5be8138d0930_m.png","_l":"upload_a3070781-0b4d-43f7-80b8-5be8138d0930_l.png"},"contentSize":45492,"contentType":"image/jpeg"},"45510":{"mediaPaths":["C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_892f666b-ac98-4feb-9017-39448434b732_o.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_892f666b-ac98-4feb-9017-39448434b732_s.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_892f666b-ac98-4feb-9017-39448434b732_m.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_892f666b-ac98-4feb-9017-39448434b732_l.png"],"fileNames":{"clean":"upload_892f666b-ac98-4feb-9017-39448434b732.png","_o":"upload_892f666b-ac98-4feb-9017-39448434b732_o.png","_s":"upload_892f666b-ac98-4feb-9017-39448434b732_s.png","_m":"upload_892f666b-ac98-4feb-9017-39448434b732_m.png","_l":"upload_892f666b-ac98-4feb-9017-39448434b732_l.png"},"contentSize":45510,"contentType":"image/jpeg"},"45585":{"mediaPaths":["C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_f684c94c-7d37-47ed-91cb-bb458f9cff15_o.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_f684c94c-7d37-47ed-91cb-bb458f9cff15_s.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_f684c94c-7d37-47ed-91cb-bb458f9cff15_m.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_f684c94c-7d37-47ed-91cb-bb458f9cff15_l.png"],"fileNames":{"clean":"upload_f684c94c-7d37-47ed-91cb-bb458f9cff15.png","_o":"upload_f684c94c-7d37-47ed-91cb-bb458f9cff15_o.png","_s":"upload_f684c94c-7d37-47ed-91cb-bb458f9cff15_s.png","_m":"upload_f684c94c-7d37-47ed-91cb-bb458f9cff15_m.png","_l":"upload_f684c94c-7d37-47ed-91cb-bb458f9cff15_l.png"},"contentSize":45585,"contentType":"image/jpeg"},"74829":{"mediaPaths":["C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_cc43cdb6-b877-4b30-a702-b7f591a8bca5_o.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_cc43cdb6-b877-4b30-a702-b7f591a8bca5_s.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_cc43cdb6-b877-4b30-a702-b7f591a8bca5_m.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_cc43cdb6-b877-4b30-a702-b7f591a8bca5_l.png"],"fileNames":{"clean":"upload_cc43cdb6-b877-4b30-a702-b7f591a8bca5.png","_o":"upload_cc43cdb6-b877-4b30-a702-b7f591a8bca5_o.png","_s":"upload_cc43cdb6-b877-4b30-a702-b7f591a8bca5_s.png","_m":"upload_cc43cdb6-b877-4b30-a702-b7f591a8bca5_m.png","_l":"upload_cc43cdb6-b877-4b30-a702-b7f591a8bca5_l.png"},"contentSize":74829,"contentType":"image/jpeg"}} \ No newline at end of file +{"23394":{"mediaPaths":["C:\\gitstuff\\GitCode\\Dash-Web\\src\\server\\public\\files\\upload_33298c77-531a-4559-822c-46a4c43ec062_o.png","C:\\gitstuff\\GitCode\\Dash-Web\\src\\server\\public\\files\\upload_33298c77-531a-4559-822c-46a4c43ec062_s.png","C:\\gitstuff\\GitCode\\Dash-Web\\src\\server\\public\\files\\upload_33298c77-531a-4559-822c-46a4c43ec062_m.png","C:\\gitstuff\\GitCode\\Dash-Web\\src\\server\\public\\files\\upload_33298c77-531a-4559-822c-46a4c43ec062_l.png"],"fileNames":{"clean":"upload_33298c77-531a-4559-822c-46a4c43ec062.png","_o":"upload_33298c77-531a-4559-822c-46a4c43ec062_o.png","_s":"upload_33298c77-531a-4559-822c-46a4c43ec062_s.png","_m":"upload_33298c77-531a-4559-822c-46a4c43ec062_m.png","_l":"upload_33298c77-531a-4559-822c-46a4c43ec062_l.png"},"contentSize":23394,"contentType":"image/jpeg"},"23406":{"mediaPaths":["C:\\gitstuff\\GitCode\\Dash-Web\\src\\server\\public\\files\\upload_bd462cce-cf8d-4aa8-bd39-c6ad69bf1fbb_o.png","C:\\gitstuff\\GitCode\\Dash-Web\\src\\server\\public\\files\\upload_bd462cce-cf8d-4aa8-bd39-c6ad69bf1fbb_s.png","C:\\gitstuff\\GitCode\\Dash-Web\\src\\server\\public\\files\\upload_bd462cce-cf8d-4aa8-bd39-c6ad69bf1fbb_m.png","C:\\gitstuff\\GitCode\\Dash-Web\\src\\server\\public\\files\\upload_bd462cce-cf8d-4aa8-bd39-c6ad69bf1fbb_l.png"],"fileNames":{"clean":"upload_bd462cce-cf8d-4aa8-bd39-c6ad69bf1fbb.png","_o":"upload_bd462cce-cf8d-4aa8-bd39-c6ad69bf1fbb_o.png","_s":"upload_bd462cce-cf8d-4aa8-bd39-c6ad69bf1fbb_s.png","_m":"upload_bd462cce-cf8d-4aa8-bd39-c6ad69bf1fbb_m.png","_l":"upload_bd462cce-cf8d-4aa8-bd39-c6ad69bf1fbb_l.png"},"contentSize":23406,"contentType":"image/jpeg"}} \ No newline at end of file diff --git a/src/server/credentials/google_docs_token.json b/src/server/credentials/google_docs_token.json index 5998ccfd8..8900f3301 100644 --- a/src/server/credentials/google_docs_token.json +++ b/src/server/credentials/google_docs_token.json @@ -1 +1 @@ -{"access_token":"ya29.GlyFByFfWe7VNNjHImJwA58yoh2cAKDJUPhBKn5IDVUY9oPlXbpyhdJMjfGSRhDZpgEWM0QoSqONu9gBVWDV9aXsf7p8r9TDXK7jBfWs1Qnox4zPf54kSfCHsmV6iw","refresh_token":"1/HTv_xFHszu2Nf3iiFrUTaeKzC_Vp2-6bpIB06xW_WHI","scope":"https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/documents.readonly https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/photoslibrary https://www.googleapis.com/auth/photoslibrary.appendonly https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/photoslibrary.sharing","token_type":"Bearer","expiry_date":1568646012183} \ No newline at end of file +{"access_token":"ya29.GlyFB38lolkvmzKJYKWGKXTSSAkRH3HA1SrCMEp3YH8Gy_I06P7w5MN1C9zckoYXQPb-7OuO4vSCchAmiwfds19hhyXDFZhegUZ30y5WhBChJ0vS5X-086QWLgKbsg","refresh_token":"1/HTv_xFHszu2Nf3iiFrUTaeKzC_Vp2-6bpIB06xW_WHI","scope":"https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/documents.readonly https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/photoslibrary https://www.googleapis.com/auth/photoslibrary.appendonly https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/photoslibrary.sharing","token_type":"Bearer","expiry_date":1568653300717} \ No newline at end of file -- cgit v1.2.3-70-g09d2 From 97f1835d9ee5fba6aa6ebc5d792f6d8a4d979cfa Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Mon, 16 Sep 2019 19:34:49 -0400 Subject: directory import and document decorations tweaks --- src/client/apis/google_docs/GoogleApiClientUtils.ts | 1 - src/client/util/Import & Export/DirectoryImportBox.tsx | 6 +----- src/client/views/DocumentDecorations.scss | 4 ++-- src/client/views/DocumentDecorations.tsx | 4 +--- src/new_fields/RichTextUtils.ts | 17 +++++++++++++++-- src/server/apis/google/existing_uploads.json | 2 +- src/server/credentials/google_docs_token.json | 2 +- 7 files changed, 21 insertions(+), 15 deletions(-) (limited to 'src/server/apis') diff --git a/src/client/apis/google_docs/GoogleApiClientUtils.ts b/src/client/apis/google_docs/GoogleApiClientUtils.ts index cbc5da15b..2c84741db 100644 --- a/src/client/apis/google_docs/GoogleApiClientUtils.ts +++ b/src/client/apis/google_docs/GoogleApiClientUtils.ts @@ -4,7 +4,6 @@ import { RouteStore } from "../../../server/RouteStore"; import { Opt } from "../../../new_fields/Doc"; import { isArray } from "util"; import { EditorState } from "prosemirror-state"; -import { RichTextField } from "../../../new_fields/RichTextField"; export const Pulls = "googleDocsPullCount"; export const Pushes = "googleDocsPushCount"; diff --git a/src/client/util/Import & Export/DirectoryImportBox.tsx b/src/client/util/Import & Export/DirectoryImportBox.tsx index d371766dd..c9d34b594 100644 --- a/src/client/util/Import & Export/DirectoryImportBox.tsx +++ b/src/client/util/Import & Export/DirectoryImportBox.tsx @@ -156,11 +156,7 @@ export default class DirectoryImportBox extends React.Component if (docs.length < 50) { importContainer = Docs.Create.MasonryDocument(docs, options); } else { - const headers = [ - new SchemaHeaderField("title", "yellow"), - new SchemaHeaderField("size", "blue"), - new SchemaHeaderField("googlePhotosTags", "green") - ]; + const headers = [new SchemaHeaderField("title"), new SchemaHeaderField("size")]; importContainer = Docs.Create.SchemaDocument(headers, docs, options); } runInAction(() => this.phase = 'External: uploading files to Google Photos...'); diff --git a/src/client/views/DocumentDecorations.scss b/src/client/views/DocumentDecorations.scss index 39fc7031a..117e63a37 100644 --- a/src/client/views/DocumentDecorations.scss +++ b/src/client/views/DocumentDecorations.scss @@ -287,10 +287,10 @@ $linkGap : 3px; @keyframes shadow-pulse { 0% { - box-shadow: 0 0 0 0px rgba(0, 0, 0, 0.2); + box-shadow: 0 0 0 0px rgba(0, 0, 0, 0.8); } 100% { - box-shadow: 0 0 0 35px rgba(0, 0, 0, 0); + box-shadow: 0 0 0 10px rgba(0, 255, 0, 0); } } \ No newline at end of file diff --git a/src/client/views/DocumentDecorations.tsx b/src/client/views/DocumentDecorations.tsx index 0a971ed22..e8a1d08e4 100644 --- a/src/client/views/DocumentDecorations.tsx +++ b/src/client/views/DocumentDecorations.tsx @@ -727,9 +727,7 @@ export class DocumentDecorations extends React.Component<{}, { value: string }> return (
    { - if (!published) { - runInAction(() => this.isAnimatingPulse = true); - } + runInAction(() => this.isAnimatingPulse = true); DocumentDecorations.hasPushedHack = false; this.targetDoc[Pushes] = NumCast(this.targetDoc[Pushes]) + 1; }} style={{ animation: this.isAnimatingPulse ? "shadow-pulse 1s infinite" : "none" }}> diff --git a/src/new_fields/RichTextUtils.ts b/src/new_fields/RichTextUtils.ts index 9b6e55948..8f4ab58eb 100644 --- a/src/new_fields/RichTextUtils.ts +++ b/src/new_fields/RichTextUtils.ts @@ -317,6 +317,10 @@ export namespace RichTextUtils { attributes.fontSize = value.magnitude; } + if (converted === "weightedFontFamily") { + converted = ImportFontFamilyMapping.get(value.fontFamily) || "timesNewRoman"; + } + let mapped = schema.marks[converted]; if (!mapped) { alert(`No mapping found for ${converted}!`); @@ -342,7 +346,7 @@ export namespace RichTextUtils { ["impact", "weightedFontFamily"] ]); - const FontFamilyMapping = new Map([ + const ExportFontFamilyMapping = new Map([ ["timesNewRoman", "Times New Roman"], ["arial", "Arial"], ["georgia", "Georgia"], @@ -351,6 +355,15 @@ export namespace RichTextUtils { ["impact", "Impact"] ]); + const ImportFontFamilyMapping = new Map([ + ["Times New Roman", "timesNewRoman"], + ["Arial", "arial"], + ["Georgia", "georgia"], + ["Comic Sans MS", "comicSans"], + ["Tahoma", "tahoma"], + ["Impact", "impact"] + ]); + const ignored = ["user_mark"]; const marksToStyle = async (nodes: (Node | null)[]): Promise => { @@ -408,7 +421,7 @@ export namespace RichTextUtils { value = fromHex(attrs.color); break; case "weightedFontFamily": - value = { fontFamily: FontFamilyMapping.get(markName) }; + value = { fontFamily: ExportFontFamilyMapping.get(markName) }; } let matches: RegExpExecArray | null; if ((matches = /p(\d+)/g.exec(markName)) !== null) { diff --git a/src/server/apis/google/existing_uploads.json b/src/server/apis/google/existing_uploads.json index 399c27672..e3cca7a97 100644 --- a/src/server/apis/google/existing_uploads.json +++ b/src/server/apis/google/existing_uploads.json @@ -1 +1 @@ -{"23394":{"mediaPaths":["C:\\gitstuff\\GitCode\\Dash-Web\\src\\server\\public\\files\\upload_33298c77-531a-4559-822c-46a4c43ec062_o.png","C:\\gitstuff\\GitCode\\Dash-Web\\src\\server\\public\\files\\upload_33298c77-531a-4559-822c-46a4c43ec062_s.png","C:\\gitstuff\\GitCode\\Dash-Web\\src\\server\\public\\files\\upload_33298c77-531a-4559-822c-46a4c43ec062_m.png","C:\\gitstuff\\GitCode\\Dash-Web\\src\\server\\public\\files\\upload_33298c77-531a-4559-822c-46a4c43ec062_l.png"],"fileNames":{"clean":"upload_33298c77-531a-4559-822c-46a4c43ec062.png","_o":"upload_33298c77-531a-4559-822c-46a4c43ec062_o.png","_s":"upload_33298c77-531a-4559-822c-46a4c43ec062_s.png","_m":"upload_33298c77-531a-4559-822c-46a4c43ec062_m.png","_l":"upload_33298c77-531a-4559-822c-46a4c43ec062_l.png"},"contentSize":23394,"contentType":"image/jpeg"},"23406":{"mediaPaths":["C:\\gitstuff\\GitCode\\Dash-Web\\src\\server\\public\\files\\upload_bd462cce-cf8d-4aa8-bd39-c6ad69bf1fbb_o.png","C:\\gitstuff\\GitCode\\Dash-Web\\src\\server\\public\\files\\upload_bd462cce-cf8d-4aa8-bd39-c6ad69bf1fbb_s.png","C:\\gitstuff\\GitCode\\Dash-Web\\src\\server\\public\\files\\upload_bd462cce-cf8d-4aa8-bd39-c6ad69bf1fbb_m.png","C:\\gitstuff\\GitCode\\Dash-Web\\src\\server\\public\\files\\upload_bd462cce-cf8d-4aa8-bd39-c6ad69bf1fbb_l.png"],"fileNames":{"clean":"upload_bd462cce-cf8d-4aa8-bd39-c6ad69bf1fbb.png","_o":"upload_bd462cce-cf8d-4aa8-bd39-c6ad69bf1fbb_o.png","_s":"upload_bd462cce-cf8d-4aa8-bd39-c6ad69bf1fbb_s.png","_m":"upload_bd462cce-cf8d-4aa8-bd39-c6ad69bf1fbb_m.png","_l":"upload_bd462cce-cf8d-4aa8-bd39-c6ad69bf1fbb_l.png"},"contentSize":23406,"contentType":"image/jpeg"}} \ No newline at end of file +{"23394":{"mediaPaths":["C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_9d5ae803-8c10-4e85-8751-53d2fe71277f_o.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_9d5ae803-8c10-4e85-8751-53d2fe71277f_s.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_9d5ae803-8c10-4e85-8751-53d2fe71277f_m.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_9d5ae803-8c10-4e85-8751-53d2fe71277f_l.png"],"fileNames":{"clean":"upload_9d5ae803-8c10-4e85-8751-53d2fe71277f.png","_o":"upload_9d5ae803-8c10-4e85-8751-53d2fe71277f_o.png","_s":"upload_9d5ae803-8c10-4e85-8751-53d2fe71277f_s.png","_m":"upload_9d5ae803-8c10-4e85-8751-53d2fe71277f_m.png","_l":"upload_9d5ae803-8c10-4e85-8751-53d2fe71277f_l.png"},"contentSize":23394,"contentType":"image/jpeg"},"23406":{"mediaPaths":["C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_fd27edc4-99a0-4405-b1f6-4c70924667c8_o.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_fd27edc4-99a0-4405-b1f6-4c70924667c8_s.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_fd27edc4-99a0-4405-b1f6-4c70924667c8_m.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_fd27edc4-99a0-4405-b1f6-4c70924667c8_l.png"],"fileNames":{"clean":"upload_fd27edc4-99a0-4405-b1f6-4c70924667c8.png","_o":"upload_fd27edc4-99a0-4405-b1f6-4c70924667c8_o.png","_s":"upload_fd27edc4-99a0-4405-b1f6-4c70924667c8_s.png","_m":"upload_fd27edc4-99a0-4405-b1f6-4c70924667c8_m.png","_l":"upload_fd27edc4-99a0-4405-b1f6-4c70924667c8_l.png"},"contentSize":23406,"contentType":"image/jpeg"},"45210":{"mediaPaths":["C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_e46cf381-3841-48bc-833b-019a5c6157e3_o.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_e46cf381-3841-48bc-833b-019a5c6157e3_s.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_e46cf381-3841-48bc-833b-019a5c6157e3_m.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_e46cf381-3841-48bc-833b-019a5c6157e3_l.png"],"fileNames":{"clean":"upload_e46cf381-3841-48bc-833b-019a5c6157e3.png","_o":"upload_e46cf381-3841-48bc-833b-019a5c6157e3_o.png","_s":"upload_e46cf381-3841-48bc-833b-019a5c6157e3_s.png","_m":"upload_e46cf381-3841-48bc-833b-019a5c6157e3_m.png","_l":"upload_e46cf381-3841-48bc-833b-019a5c6157e3_l.png"},"contentSize":45210,"contentType":"image/jpeg"},"45229":{"mediaPaths":["C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_ba4f46ea-f2ab-4dcd-8da0-4f89916e665b_o.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_ba4f46ea-f2ab-4dcd-8da0-4f89916e665b_s.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_ba4f46ea-f2ab-4dcd-8da0-4f89916e665b_m.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_ba4f46ea-f2ab-4dcd-8da0-4f89916e665b_l.png"],"fileNames":{"clean":"upload_ba4f46ea-f2ab-4dcd-8da0-4f89916e665b.png","_o":"upload_ba4f46ea-f2ab-4dcd-8da0-4f89916e665b_o.png","_s":"upload_ba4f46ea-f2ab-4dcd-8da0-4f89916e665b_s.png","_m":"upload_ba4f46ea-f2ab-4dcd-8da0-4f89916e665b_m.png","_l":"upload_ba4f46ea-f2ab-4dcd-8da0-4f89916e665b_l.png"},"contentSize":45229,"contentType":"image/jpeg"},"45230":{"mediaPaths":["C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_2d39b115-4fcb-4bef-abbe-cdaa029c11f5_o.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_2d39b115-4fcb-4bef-abbe-cdaa029c11f5_s.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_2d39b115-4fcb-4bef-abbe-cdaa029c11f5_m.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_2d39b115-4fcb-4bef-abbe-cdaa029c11f5_l.png"],"fileNames":{"clean":"upload_2d39b115-4fcb-4bef-abbe-cdaa029c11f5.png","_o":"upload_2d39b115-4fcb-4bef-abbe-cdaa029c11f5_o.png","_s":"upload_2d39b115-4fcb-4bef-abbe-cdaa029c11f5_s.png","_m":"upload_2d39b115-4fcb-4bef-abbe-cdaa029c11f5_m.png","_l":"upload_2d39b115-4fcb-4bef-abbe-cdaa029c11f5_l.png"},"contentSize":45230,"contentType":"image/jpeg"},"45286":{"mediaPaths":["C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_3c726016-3c72-4a4a-8dfd-ce7a6977c8e0_o.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_3c726016-3c72-4a4a-8dfd-ce7a6977c8e0_s.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_3c726016-3c72-4a4a-8dfd-ce7a6977c8e0_m.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_3c726016-3c72-4a4a-8dfd-ce7a6977c8e0_l.png"],"fileNames":{"clean":"upload_3c726016-3c72-4a4a-8dfd-ce7a6977c8e0.png","_o":"upload_3c726016-3c72-4a4a-8dfd-ce7a6977c8e0_o.png","_s":"upload_3c726016-3c72-4a4a-8dfd-ce7a6977c8e0_s.png","_m":"upload_3c726016-3c72-4a4a-8dfd-ce7a6977c8e0_m.png","_l":"upload_3c726016-3c72-4a4a-8dfd-ce7a6977c8e0_l.png"},"contentSize":45286,"contentType":"image/jpeg"},"45585":{"mediaPaths":["C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_7e097c1d-1c8b-433b-ae3b-25beafe95a64_o.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_7e097c1d-1c8b-433b-ae3b-25beafe95a64_s.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_7e097c1d-1c8b-433b-ae3b-25beafe95a64_m.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_7e097c1d-1c8b-433b-ae3b-25beafe95a64_l.png"],"fileNames":{"clean":"upload_7e097c1d-1c8b-433b-ae3b-25beafe95a64.png","_o":"upload_7e097c1d-1c8b-433b-ae3b-25beafe95a64_o.png","_s":"upload_7e097c1d-1c8b-433b-ae3b-25beafe95a64_s.png","_m":"upload_7e097c1d-1c8b-433b-ae3b-25beafe95a64_m.png","_l":"upload_7e097c1d-1c8b-433b-ae3b-25beafe95a64_l.png"},"contentSize":45585,"contentType":"image/jpeg"},"74829":{"mediaPaths":["C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_8eacbcc4-b350-4767-9808-01d5b46e9b6a_o.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_8eacbcc4-b350-4767-9808-01d5b46e9b6a_s.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_8eacbcc4-b350-4767-9808-01d5b46e9b6a_m.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_8eacbcc4-b350-4767-9808-01d5b46e9b6a_l.png"],"fileNames":{"clean":"upload_8eacbcc4-b350-4767-9808-01d5b46e9b6a.png","_o":"upload_8eacbcc4-b350-4767-9808-01d5b46e9b6a_o.png","_s":"upload_8eacbcc4-b350-4767-9808-01d5b46e9b6a_s.png","_m":"upload_8eacbcc4-b350-4767-9808-01d5b46e9b6a_m.png","_l":"upload_8eacbcc4-b350-4767-9808-01d5b46e9b6a_l.png"},"contentSize":74829,"contentType":"image/jpeg"}} \ No newline at end of file diff --git a/src/server/credentials/google_docs_token.json b/src/server/credentials/google_docs_token.json index 64bd7a58d..a536d6c3d 100644 --- a/src/server/credentials/google_docs_token.json +++ b/src/server/credentials/google_docs_token.json @@ -1 +1 @@ -{"access_token":"ya29.GlyFBwglzOsVNH90uoePSgwPkCqGNDMfx_us2wVe8YyS-MOA54Zdo7F_iiGOTDm9kGsINkQVgLu4rBZE7OTMa5Qxm8BuZIbTG66PPdVI0vbH96nfSlHQL8fnX1WOMQ","refresh_token":"1/HTv_xFHszu2Nf3iiFrUTaeKzC_Vp2-6bpIB06xW_WHI","scope":"https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/documents.readonly https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/photoslibrary https://www.googleapis.com/auth/photoslibrary.appendonly https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/photoslibrary.sharing","token_type":"Bearer","expiry_date":1568657110414} \ No newline at end of file +{"access_token":"ya29.GlyGB1dc7fKtWD1FtSvh_6aL3eaDJMFAfiV2EGTDK20fCjinY2FNpzJKhDn8p_IN2NupjQ_fXwqM6orx-E6MUCyGN3YZdTmPOaSd-pQlqIl6TFN49pxuzoxguBL4Sw","refresh_token":"1/HTv_xFHszu2Nf3iiFrUTaeKzC_Vp2-6bpIB06xW_WHI","scope":"https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/documents.readonly https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/photoslibrary https://www.googleapis.com/auth/photoslibrary.appendonly https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/photoslibrary.sharing","token_type":"Bearer","expiry_date":1568679048543} \ No newline at end of file -- cgit v1.2.3-70-g09d2 From 960b5babede63780bf91ce8fe8658e8c41925ecb Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Wed, 18 Sep 2019 04:22:54 -0400 Subject: final batching expansion --- src/client/northstar/utils/Extensions.ts | 28 ++++- .../util/Import & Export/DirectoryImportBox.tsx | 2 +- src/client/util/UtilExtensions.ts | 135 +++++++++++++++++---- src/server/apis/google/GooglePhotosUploadUtils.ts | 2 +- src/server/apis/google/existing_uploads.json | 1 - src/server/credentials/google_docs_token.json | 2 +- src/server/index.ts | 7 +- 7 files changed, 146 insertions(+), 31 deletions(-) (limited to 'src/server/apis') diff --git a/src/client/northstar/utils/Extensions.ts b/src/client/northstar/utils/Extensions.ts index 65af7ea87..3722bfd52 100644 --- a/src/client/northstar/utils/Extensions.ts +++ b/src/client/northstar/utils/Extensions.ts @@ -23,7 +23,6 @@ String.prototype.Truncate = function (length: number, replacement: string): Stri interface BatchContext { completedBatches: number; remainingBatches: number; - isFullBatch: boolean; } type BatchConverterSync = (batch: I[], context: BatchContext) => O[]; type BatchHandlerSync = (batch: I[], context: BatchContext) => void; @@ -31,20 +30,43 @@ type BatchConverterAsync = (batch: I[], context: BatchContext) => Promise< type BatchHandlerAsync = (batch: I[], context: BatchContext) => Promise; type BatchConverter = BatchConverterSync | BatchConverterAsync; type BatchHandler = BatchHandlerSync | BatchHandlerAsync; -type Batcher = number | ((element: I, accumulator: A) => boolean | Promise); +type FixedBatcher = { batchSize: number } | { batchCount: number, mode?: Mode }; +interface PredicateBatcher { + executor: (element: I, accumulator: A | undefined) => A | undefined; + initial: A; +} +interface PredicateBatcherAsync { + executor: (element: I, accumulator: A | undefined) => Promise; + initial: A; +} +type Batcher = FixedBatcher | PredicateBatcher; +type BatcherAsync = Batcher | PredicateBatcherAsync; interface Array { - batch(batchSize: undefined): T[][]; + fixedBatch(batcher: FixedBatcher): T[][]; + predicateBatch(batcher: PredicateBatcher): T[][]; + predicateBatchAsync(batcher: PredicateBatcherAsync): Promise; + batch(batcher: Batcher): T[][]; + batchAsync(batcher: BatcherAsync): Promise; + batchedForEach(batcher: Batcher, handler: BatchHandlerSync): void; batchedMap(batcher: Batcher, handler: BatchConverterSync): O[]; + batchedForEachAsync(batcher: Batcher, handler: BatchHandler): Promise; batchedMapAsync(batcher: Batcher, handler: BatchConverter): Promise; + batchedForEachInterval(batcher: Batcher, handler: BatchHandler, interval: number): Promise; batchedMapInterval(batcher: Batcher, handler: BatchConverter, interval: number): Promise; + lastElement(): T; } +Array.prototype.fixedBatch = extensions.FixedBatch; +Array.prototype.predicateBatch = extensions.PredicateBatch; +Array.prototype.predicateBatchAsync = extensions.PredicateBatchAsync; Array.prototype.batch = extensions.Batch; +Array.prototype.batchAsync = extensions.BatchAsync; + Array.prototype.batchedForEach = extensions.ExecuteInBatches; Array.prototype.batchedMap = extensions.ConvertInBatches; Array.prototype.batchedForEachAsync = extensions.ExecuteInBatchesAsync; diff --git a/src/client/util/Import & Export/DirectoryImportBox.tsx b/src/client/util/Import & Export/DirectoryImportBox.tsx index a86ef9a31..e3958e3a4 100644 --- a/src/client/util/Import & Export/DirectoryImportBox.tsx +++ b/src/client/util/Import & Export/DirectoryImportBox.tsx @@ -103,7 +103,7 @@ export default class DirectoryImportBox extends React.Component runInAction(() => this.phase = `Internal: uploading ${this.quota - this.completed} files to Dash...`); - const uploads = await validated.batchedMapAsync(15, async batch => { + const uploads = await validated.batchedMapAsync({ batchSize: 15 }, async batch => { const formData = new FormData(); const parameters = { method: 'POST', body: formData }; diff --git a/src/client/util/UtilExtensions.ts b/src/client/util/UtilExtensions.ts index 792cd00a1..8bd8fd581 100644 --- a/src/client/util/UtilExtensions.ts +++ b/src/client/util/UtilExtensions.ts @@ -1,24 +1,118 @@ -module.exports.Batch = function (batchSize: number): T[][] { +module.exports.Batch = function (batcher: Batcher): T[][] { + if ("executor" in batcher) { + return this.predicateBatch(batcher); + } else { + return this.fixedBatch(batcher); + } +}; + +module.exports.BatchAsync = async function (batcher: BatcherAsync): Promise { + if ("executor" in batcher) { + return this.predicateBatchAsync(batcher); + } else { + return this.fixedBatch(batcher); + } +}; + +module.exports.PredicateBatch = function (batcher: PredicateBatcher): T[][] { + const batches: T[][] = []; + let batch: T[] = []; + const { executor, initial } = batcher; + let accumulator: A | undefined = initial; + for (let element of this) { + if ((accumulator = executor(element, accumulator)) !== undefined) { + batch.push(element); + } else { + batches.push(batch); + batch = [element]; + } + } + batches.push(batch); + return batches; +}; + +module.exports.PredicateBatchAsync = async function (batcher: PredicateBatcherAsync): Promise { + const batches: T[][] = []; + let batch: T[] = []; + const { executor, initial } = batcher; + let accumulator: A | undefined = initial; + for (let element of this) { + if ((accumulator = await executor(element, accumulator)) !== undefined) { + batch.push(element); + } else { + batches.push(batch); + batch = [element]; + } + } + batches.push(batch); + return batches; +}; + +enum Mode { + Balanced, + Even +} + +module.exports.FixedBatch = function (batcher: FixedBatcher): T[][] { const batches: T[][] = []; + const length = this.length; let i = 0; - while (i < this.length) { - const cap = Math.min(i + batchSize, this.length); - batches.push(this.slice(i, cap)); - i = cap; + if ("batchSize" in batcher) { + const { batchSize } = batcher; + while (i < this.length) { + const cap = Math.min(i + batchSize, length); + batches.push(this.slice(i, i = cap)); + } + } else if ("batchCount" in batcher) { + let { batchCount, mode } = batcher; + const resolved = mode || Mode.Balanced; + if (batchCount < 1) { + throw new Error("Batch count must be a positive integer!"); + } + if (batchCount === 1) { + return [this]; + } + if (batchCount >= this.length) { + return this.map((element: T) => [element]); + } + + let length = this.length; + let size: number; + + if (length % batchCount === 0) { + size = Math.floor(length / batchCount); + while (i < length) { + batches.push(this.slice(i, i += size)); + } + } else if (resolved === Mode.Balanced) { + while (i < length) { + size = Math.ceil((length - i) / batchCount--); + batches.push(this.slice(i, i += size)); + } + } else { + batchCount--; + size = Math.floor(length / batchCount); + if (length % size === 0) { + size--; + } + while (i < size * batchCount) { + batches.push(this.slice(i, i += size)); + } + batches.push(this.slice(size * batchCount)); + } } return batches; }; -module.exports.ExecuteBatches = function (batchSize: number, handler: BatchHandlerSync): void { +module.exports.ExecuteBatches = function (batcher: Batcher, handler: BatchHandlerSync): void { if (this.length) { let completed = 0; - const batches = this.batch(batchSize); + const batches = this.batch(batcher); const quota = batches.length; for (let batch of batches) { const context: BatchContext = { completedBatches: completed, remainingBatches: quota - completed, - isFullBatch: batch.length === batchSize }; handler(batch, context); completed++; @@ -26,19 +120,18 @@ module.exports.ExecuteBatches = function (batchSize: number, handler: BatchHa } }; -module.exports.ConvertInBatches = function (batchSize: number, handler: BatchConverterSync): O[] { +module.exports.ConvertInBatches = function (batcher: Batcher, handler: BatchConverterSync): O[] { if (!this.length) { return []; } let collector: O[] = []; let completed = 0; - const batches = this.batch(batchSize); + const batches = this.batch(batcher); const quota = batches.length; for (let batch of batches) { const context: BatchContext = { completedBatches: completed, remainingBatches: quota - completed, - isFullBatch: batch.length === batchSize }; collector.push(...handler(batch, context)); completed++; @@ -46,16 +139,15 @@ module.exports.ConvertInBatches = function (batchSize: number, handler: Ba return collector; }; -module.exports.ExecuteInBatchesAsync = async function (batchSize: number, handler: BatchHandler): Promise { +module.exports.ExecuteInBatchesAsync = async function (batcher: BatcherAsync, handler: BatchHandler): Promise { if (this.length) { let completed = 0; - const batches = this.batch(batchSize); + const batches = await this.batchAsync(batcher); const quota = batches.length; for (let batch of batches) { const context: BatchContext = { completedBatches: completed, remainingBatches: quota - completed, - isFullBatch: batch.length === batchSize }; await handler(batch, context); completed++; @@ -63,19 +155,18 @@ module.exports.ExecuteInBatchesAsync = async function (batchSize: number, han } }; -module.exports.ConvertInBatchesAsync = async function (batchSize: number, handler: BatchConverter): Promise { +module.exports.ConvertInBatchesAsync = async function (batcher: BatcherAsync, handler: BatchConverter): Promise { if (!this.length) { return []; } let collector: O[] = []; let completed = 0; - const batches = this.batch(batchSize); + const batches = await this.batchAsync(batcher); const quota = batches.length; for (let batch of batches) { const context: BatchContext = { completedBatches: completed, remainingBatches: quota - completed, - isFullBatch: batch.length === batchSize }; collector.push(...(await handler(batch, context))); completed++; @@ -83,11 +174,11 @@ module.exports.ConvertInBatchesAsync = async function (batchSize: number, return collector; }; -module.exports.ExecuteInBatchesAtInterval = async function (batchSize: number, handler: BatchHandler, interval: number): Promise { +module.exports.ExecuteInBatchesAtInterval = async function (batcher: BatcherAsync, handler: BatchHandler, interval: number): Promise { if (!this.length) { return; } - const batches = this.batch(batchSize); + const batches = await this.batchAsync(batcher); const quota = batches.length; return new Promise(async resolve => { const iterator = batches[Symbol.iterator](); @@ -100,7 +191,6 @@ module.exports.ExecuteInBatchesAtInterval = async function (batchSize: number const context: BatchContext = { completedBatches: completed, remainingBatches: quota - completed, - isFullBatch: batch.length === batchSize }; await handler(batch, context); resolve(); @@ -114,12 +204,12 @@ module.exports.ExecuteInBatchesAtInterval = async function (batchSize: number }); }; -module.exports.ConvertInBatchesAtInterval = async function (batchSize: number, handler: BatchConverter, interval: number): Promise { +module.exports.ConvertInBatchesAtInterval = async function (batcher: BatcherAsync, handler: BatchConverter, interval: number): Promise { if (!this.length) { return []; } let collector: O[] = []; - const batches = this.batch(batchSize); + const batches = await this.batchAsync(batcher); const quota = batches.length; return new Promise(async resolve => { const iterator = batches[Symbol.iterator](); @@ -132,7 +222,6 @@ module.exports.ConvertInBatchesAtInterval = async function (batchSize: num const context: BatchContext = { completedBatches: completed, remainingBatches: quota - completed, - isFullBatch: batch.length === batchSize }; collector.push(...(await handler(batch, context))); resolve(); diff --git a/src/server/apis/google/GooglePhotosUploadUtils.ts b/src/server/apis/google/GooglePhotosUploadUtils.ts index 734d77fd7..9733c7ec5 100644 --- a/src/server/apis/google/GooglePhotosUploadUtils.ts +++ b/src/server/apis/google/GooglePhotosUploadUtils.ts @@ -80,7 +80,7 @@ export namespace GooglePhotosUploadUtils { }); })).newMediaItemResults; }; - const newMediaItemResults = await newMediaItems.batchedMapInterval(50, createFromUploadTokens, 0.1); + const newMediaItemResults = await newMediaItems.batchedMapInterval({ batchSize: 50 }, createFromUploadTokens, 0.1); return { newMediaItemResults }; }; diff --git a/src/server/apis/google/existing_uploads.json b/src/server/apis/google/existing_uploads.json index e3cca7a97..e69de29bb 100644 --- a/src/server/apis/google/existing_uploads.json +++ b/src/server/apis/google/existing_uploads.json @@ -1 +0,0 @@ -{"23394":{"mediaPaths":["C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_9d5ae803-8c10-4e85-8751-53d2fe71277f_o.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_9d5ae803-8c10-4e85-8751-53d2fe71277f_s.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_9d5ae803-8c10-4e85-8751-53d2fe71277f_m.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_9d5ae803-8c10-4e85-8751-53d2fe71277f_l.png"],"fileNames":{"clean":"upload_9d5ae803-8c10-4e85-8751-53d2fe71277f.png","_o":"upload_9d5ae803-8c10-4e85-8751-53d2fe71277f_o.png","_s":"upload_9d5ae803-8c10-4e85-8751-53d2fe71277f_s.png","_m":"upload_9d5ae803-8c10-4e85-8751-53d2fe71277f_m.png","_l":"upload_9d5ae803-8c10-4e85-8751-53d2fe71277f_l.png"},"contentSize":23394,"contentType":"image/jpeg"},"23406":{"mediaPaths":["C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_fd27edc4-99a0-4405-b1f6-4c70924667c8_o.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_fd27edc4-99a0-4405-b1f6-4c70924667c8_s.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_fd27edc4-99a0-4405-b1f6-4c70924667c8_m.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_fd27edc4-99a0-4405-b1f6-4c70924667c8_l.png"],"fileNames":{"clean":"upload_fd27edc4-99a0-4405-b1f6-4c70924667c8.png","_o":"upload_fd27edc4-99a0-4405-b1f6-4c70924667c8_o.png","_s":"upload_fd27edc4-99a0-4405-b1f6-4c70924667c8_s.png","_m":"upload_fd27edc4-99a0-4405-b1f6-4c70924667c8_m.png","_l":"upload_fd27edc4-99a0-4405-b1f6-4c70924667c8_l.png"},"contentSize":23406,"contentType":"image/jpeg"},"45210":{"mediaPaths":["C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_e46cf381-3841-48bc-833b-019a5c6157e3_o.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_e46cf381-3841-48bc-833b-019a5c6157e3_s.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_e46cf381-3841-48bc-833b-019a5c6157e3_m.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_e46cf381-3841-48bc-833b-019a5c6157e3_l.png"],"fileNames":{"clean":"upload_e46cf381-3841-48bc-833b-019a5c6157e3.png","_o":"upload_e46cf381-3841-48bc-833b-019a5c6157e3_o.png","_s":"upload_e46cf381-3841-48bc-833b-019a5c6157e3_s.png","_m":"upload_e46cf381-3841-48bc-833b-019a5c6157e3_m.png","_l":"upload_e46cf381-3841-48bc-833b-019a5c6157e3_l.png"},"contentSize":45210,"contentType":"image/jpeg"},"45229":{"mediaPaths":["C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_ba4f46ea-f2ab-4dcd-8da0-4f89916e665b_o.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_ba4f46ea-f2ab-4dcd-8da0-4f89916e665b_s.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_ba4f46ea-f2ab-4dcd-8da0-4f89916e665b_m.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_ba4f46ea-f2ab-4dcd-8da0-4f89916e665b_l.png"],"fileNames":{"clean":"upload_ba4f46ea-f2ab-4dcd-8da0-4f89916e665b.png","_o":"upload_ba4f46ea-f2ab-4dcd-8da0-4f89916e665b_o.png","_s":"upload_ba4f46ea-f2ab-4dcd-8da0-4f89916e665b_s.png","_m":"upload_ba4f46ea-f2ab-4dcd-8da0-4f89916e665b_m.png","_l":"upload_ba4f46ea-f2ab-4dcd-8da0-4f89916e665b_l.png"},"contentSize":45229,"contentType":"image/jpeg"},"45230":{"mediaPaths":["C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_2d39b115-4fcb-4bef-abbe-cdaa029c11f5_o.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_2d39b115-4fcb-4bef-abbe-cdaa029c11f5_s.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_2d39b115-4fcb-4bef-abbe-cdaa029c11f5_m.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_2d39b115-4fcb-4bef-abbe-cdaa029c11f5_l.png"],"fileNames":{"clean":"upload_2d39b115-4fcb-4bef-abbe-cdaa029c11f5.png","_o":"upload_2d39b115-4fcb-4bef-abbe-cdaa029c11f5_o.png","_s":"upload_2d39b115-4fcb-4bef-abbe-cdaa029c11f5_s.png","_m":"upload_2d39b115-4fcb-4bef-abbe-cdaa029c11f5_m.png","_l":"upload_2d39b115-4fcb-4bef-abbe-cdaa029c11f5_l.png"},"contentSize":45230,"contentType":"image/jpeg"},"45286":{"mediaPaths":["C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_3c726016-3c72-4a4a-8dfd-ce7a6977c8e0_o.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_3c726016-3c72-4a4a-8dfd-ce7a6977c8e0_s.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_3c726016-3c72-4a4a-8dfd-ce7a6977c8e0_m.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_3c726016-3c72-4a4a-8dfd-ce7a6977c8e0_l.png"],"fileNames":{"clean":"upload_3c726016-3c72-4a4a-8dfd-ce7a6977c8e0.png","_o":"upload_3c726016-3c72-4a4a-8dfd-ce7a6977c8e0_o.png","_s":"upload_3c726016-3c72-4a4a-8dfd-ce7a6977c8e0_s.png","_m":"upload_3c726016-3c72-4a4a-8dfd-ce7a6977c8e0_m.png","_l":"upload_3c726016-3c72-4a4a-8dfd-ce7a6977c8e0_l.png"},"contentSize":45286,"contentType":"image/jpeg"},"45585":{"mediaPaths":["C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_7e097c1d-1c8b-433b-ae3b-25beafe95a64_o.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_7e097c1d-1c8b-433b-ae3b-25beafe95a64_s.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_7e097c1d-1c8b-433b-ae3b-25beafe95a64_m.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_7e097c1d-1c8b-433b-ae3b-25beafe95a64_l.png"],"fileNames":{"clean":"upload_7e097c1d-1c8b-433b-ae3b-25beafe95a64.png","_o":"upload_7e097c1d-1c8b-433b-ae3b-25beafe95a64_o.png","_s":"upload_7e097c1d-1c8b-433b-ae3b-25beafe95a64_s.png","_m":"upload_7e097c1d-1c8b-433b-ae3b-25beafe95a64_m.png","_l":"upload_7e097c1d-1c8b-433b-ae3b-25beafe95a64_l.png"},"contentSize":45585,"contentType":"image/jpeg"},"74829":{"mediaPaths":["C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_8eacbcc4-b350-4767-9808-01d5b46e9b6a_o.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_8eacbcc4-b350-4767-9808-01d5b46e9b6a_s.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_8eacbcc4-b350-4767-9808-01d5b46e9b6a_m.png","C:\\Users\\avd\\Desktop\\Sam\\Dash-Web\\src\\server\\public\\files\\upload_8eacbcc4-b350-4767-9808-01d5b46e9b6a_l.png"],"fileNames":{"clean":"upload_8eacbcc4-b350-4767-9808-01d5b46e9b6a.png","_o":"upload_8eacbcc4-b350-4767-9808-01d5b46e9b6a_o.png","_s":"upload_8eacbcc4-b350-4767-9808-01d5b46e9b6a_s.png","_m":"upload_8eacbcc4-b350-4767-9808-01d5b46e9b6a_m.png","_l":"upload_8eacbcc4-b350-4767-9808-01d5b46e9b6a_l.png"},"contentSize":74829,"contentType":"image/jpeg"}} \ No newline at end of file diff --git a/src/server/credentials/google_docs_token.json b/src/server/credentials/google_docs_token.json index 9765dd4e3..be50f4dd2 100644 --- a/src/server/credentials/google_docs_token.json +++ b/src/server/credentials/google_docs_token.json @@ -1 +1 @@ -{"access_token":"ya29.ImCGB4LCtQF4V2mY1S4ANuTSrclZ6R4QYDq5RGezUwTnk381Fg9b-oZYtJTdkW2zwryoFDjCRXaMGJqPvzIBYZvsFm1Zdwi_AH1by2bDAEOl0GqFlZukxa2W036ujGDv8tY","refresh_token":"1/HTv_xFHszu2Nf3iiFrUTaeKzC_Vp2-6bpIB06xW_WHI","scope":"https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/documents.readonly https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/photoslibrary https://www.googleapis.com/auth/photoslibrary.appendonly https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/photoslibrary.sharing","token_type":"Bearer","expiry_date":1568708539924} \ No newline at end of file +{"access_token":"ya29.ImCHB-QNh-blUK7Ajw1Dk1yNBZ7w5JAKcrgzW4xYE3sfNTLdq_gXEQOK0ZZeJBPL1_WcJbbX7EKuMxWN-fwWMuhAG6-IHk1TpkOG1KZpSGo3myT7xiZDnONxLfVuodT0Nlc","refresh_token":"1/HTv_xFHszu2Nf3iiFrUTaeKzC_Vp2-6bpIB06xW_WHI","scope":"https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/documents.readonly https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/photoslibrary https://www.googleapis.com/auth/photoslibrary.appendonly https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/photoslibrary.sharing","token_type":"Bearer","expiry_date":1568797587975} \ No newline at end of file diff --git a/src/server/index.ts b/src/server/index.ts index 9da6a8b38..7fe2ba132 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -835,7 +835,12 @@ export interface NewMediaItem { }; } +Array.prototype.fixedBatch = extensions.FixedBatch; +Array.prototype.predicateBatch = extensions.PredicateBatch; +Array.prototype.predicateBatchAsync = extensions.PredicateBatchAsync; Array.prototype.batch = extensions.Batch; +Array.prototype.batchAsync = extensions.BatchAsync; + Array.prototype.batchedForEach = extensions.ExecuteInBatches; Array.prototype.batchedMap = extensions.ConvertInBatches; Array.prototype.batchedForEachAsync = extensions.ExecuteInBatchesAsync; @@ -865,7 +870,7 @@ app.post(RouteStore.googlePhotosMediaUpload, async (req, res) => { return newMediaItems; }; - const newMediaItems = await mediaInput.batchedMapInterval(25, dispatchUpload, 0.1); + const newMediaItems = await mediaInput.batchedMapInterval({ batchSize: 25 }, dispatchUpload, 0.1); if (failed) { return _error(res, tokenError); -- cgit v1.2.3-70-g09d2 From 30ec493a6dae644b0907f6ee6e86696c06aa11b4 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Wed, 18 Sep 2019 04:31:53 -0400 Subject: additional timeunit --- src/client/northstar/utils/Extensions.ts | 15 +++++++++++++-- src/client/util/UtilExtensions.ts | 21 +++++++++++++++++---- src/server/apis/google/GooglePhotosUploadUtils.ts | 6 +++++- src/server/index.ts | 4 +++- 4 files changed, 38 insertions(+), 8 deletions(-) (limited to 'src/server/apis') diff --git a/src/client/northstar/utils/Extensions.ts b/src/client/northstar/utils/Extensions.ts index 3722bfd52..ad3e06806 100644 --- a/src/client/northstar/utils/Extensions.ts +++ b/src/client/northstar/utils/Extensions.ts @@ -42,6 +42,17 @@ interface PredicateBatcherAsync { type Batcher = FixedBatcher | PredicateBatcher; type BatcherAsync = Batcher | PredicateBatcherAsync; +enum TimeUnit { + Milliseconds, + Seconds, + Minutes +} + +interface Interval { + magnitude: number; + unit: TimeUnit; +} + interface Array { fixedBatch(batcher: FixedBatcher): T[][]; predicateBatch(batcher: PredicateBatcher): T[][]; @@ -55,8 +66,8 @@ interface Array { batchedForEachAsync(batcher: Batcher, handler: BatchHandler): Promise; batchedMapAsync(batcher: Batcher, handler: BatchConverter): Promise; - batchedForEachInterval(batcher: Batcher, handler: BatchHandler, interval: number): Promise; - batchedMapInterval(batcher: Batcher, handler: BatchConverter, interval: number): Promise; + batchedForEachInterval(batcher: Batcher, handler: BatchHandler, interval: Interval): Promise; + batchedMapInterval(batcher: Batcher, handler: BatchConverter, interval: Interval): Promise; lastElement(): T; } diff --git a/src/client/util/UtilExtensions.ts b/src/client/util/UtilExtensions.ts index 8bd8fd581..da47fc1bc 100644 --- a/src/client/util/UtilExtensions.ts +++ b/src/client/util/UtilExtensions.ts @@ -174,7 +174,20 @@ module.exports.ConvertInBatchesAsync = async function (batcher: Batcher return collector; }; -module.exports.ExecuteInBatchesAtInterval = async function (batcher: BatcherAsync, handler: BatchHandler, interval: number): Promise { +const convert = (interval: Interval) => { + const { magnitude, unit } = interval; + switch (unit) { + default: + case TimeUnit.Milliseconds: + return magnitude; + case TimeUnit.Seconds: + return magnitude * 1000; + case TimeUnit.Minutes: + return magnitude * 1000 * 60; + } +}; + +module.exports.ExecuteInBatchesAtInterval = async function (batcher: BatcherAsync, handler: BatchHandler, interval: Interval): Promise { if (!this.length) { return; } @@ -194,7 +207,7 @@ module.exports.ExecuteInBatchesAtInterval = async function (batcher: Batch }; await handler(batch, context); resolve(); - }, interval * 1000); + }, convert(interval)); }); if (++completed === quota) { break; @@ -204,7 +217,7 @@ module.exports.ExecuteInBatchesAtInterval = async function (batcher: Batch }); }; -module.exports.ConvertInBatchesAtInterval = async function (batcher: BatcherAsync, handler: BatchConverter, interval: number): Promise { +module.exports.ConvertInBatchesAtInterval = async function (batcher: BatcherAsync, handler: BatchConverter, interval: Interval): Promise { if (!this.length) { return []; } @@ -225,7 +238,7 @@ module.exports.ConvertInBatchesAtInterval = async function (batcher: Ba }; collector.push(...(await handler(batch, context))); resolve(); - }, interval * 1000); + }, convert(interval)); }); if (++completed === quota) { resolve(collector); diff --git a/src/server/apis/google/GooglePhotosUploadUtils.ts b/src/server/apis/google/GooglePhotosUploadUtils.ts index 9733c7ec5..9275bf125 100644 --- a/src/server/apis/google/GooglePhotosUploadUtils.ts +++ b/src/server/apis/google/GooglePhotosUploadUtils.ts @@ -80,7 +80,11 @@ export namespace GooglePhotosUploadUtils { }); })).newMediaItemResults; }; - const newMediaItemResults = await newMediaItems.batchedMapInterval({ batchSize: 50 }, createFromUploadTokens, 0.1); + const batcher = { batchSize: 50 }; + const interval = { magnitude: 100, unit: TimeUnit.Milliseconds }; + + const newMediaItemResults = await newMediaItems.batchedMapInterval(batcher, createFromUploadTokens, interval); + return { newMediaItemResults }; }; diff --git a/src/server/index.ts b/src/server/index.ts index 7fe2ba132..83bbe734b 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -869,8 +869,10 @@ app.post(RouteStore.googlePhotosMediaUpload, async (req, res) => { } return newMediaItems; }; + const batcher = { batchSize: 25 }; + const interval = { magnitude: 100, unit: TimeUnit.Milliseconds }; - const newMediaItems = await mediaInput.batchedMapInterval({ batchSize: 25 }, dispatchUpload, 0.1); + const newMediaItems = await mediaInput.batchedMapInterval(batcher, dispatchUpload, interval); if (failed) { return _error(res, tokenError); -- cgit v1.2.3-70-g09d2 From 6b6488be27a71d9dba0ae5959284ae9a18ae9230 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Thu, 19 Sep 2019 14:13:59 -0400 Subject: extensions fixes and tracking albums --- .../apis/google_docs/GooglePhotosClientUtils.ts | 4 +- src/client/northstar/utils/Extensions.ts | 83 ----- src/client/util/UtilExtensions.ts | 259 ---------------- src/client/views/Main.tsx | 13 +- src/extensions/ArrayExtensions.ts | 341 +++++++++++++++++++++ src/extensions/StringExtensions.ts | 20 ++ src/server/apis/google/GooglePhotosUploadUtils.ts | 1 + src/server/credentials/google_docs_token.json | 2 +- src/server/index.ts | 21 +- 9 files changed, 376 insertions(+), 368 deletions(-) delete mode 100644 src/client/util/UtilExtensions.ts create mode 100644 src/extensions/ArrayExtensions.ts create mode 100644 src/extensions/StringExtensions.ts (limited to 'src/server/apis') diff --git a/src/client/apis/google_docs/GooglePhotosClientUtils.ts b/src/client/apis/google_docs/GooglePhotosClientUtils.ts index a13d9dcd6..671d05421 100644 --- a/src/client/apis/google_docs/GooglePhotosClientUtils.ts +++ b/src/client/apis/google_docs/GooglePhotosClientUtils.ts @@ -89,7 +89,7 @@ export namespace GooglePhotos { return undefined; } const resolved = title ? title : (StrCast(collection.title) || `Dash Collection (${collection[Id]}`); - const { id } = await Create.Album(resolved); + const { id, productUrl } = await Create.Album(resolved); const newMediaItemResults = await Transactions.UploadImages(images, { id }, descriptionKey); if (newMediaItemResults) { const mediaItems = newMediaItemResults.map(item => item.mediaItem); @@ -101,9 +101,11 @@ export namespace GooglePhotos { const image = Doc.GetProto(images[i]); const mediaItem = mediaItems[i]; image.googlePhotosId = mediaItem.id; + image.googlePhotosAlbumUrl = productUrl; image.googlePhotosUrl = mediaItem.productUrl || mediaItem.baseUrl; idMapping[mediaItem.id] = image; } + collection.googlePhotosAlbumUrl = productUrl; collection.googlePhotosIdMapping = idMapping; if (tag) { await Query.TagChildImages(collection); diff --git a/src/client/northstar/utils/Extensions.ts b/src/client/northstar/utils/Extensions.ts index 8254c775c..df14d4da0 100644 --- a/src/client/northstar/utils/Extensions.ts +++ b/src/client/northstar/utils/Extensions.ts @@ -1,12 +1,8 @@ interface String { ReplaceAll(toReplace: string, replacement: string): string; Truncate(length: number, replacement: string): String; - removeTrailingNewlines(): string; - hasNewline(): boolean; } -const extensions = require(".././/.//../util/UtilExtensions"); - String.prototype.ReplaceAll = function (toReplace: string, replacement: string): string { var target = this; return target.split(toReplace).join(replacement); @@ -20,85 +16,6 @@ String.prototype.Truncate = function (length: number, replacement: string): Stri return target; }; -interface BatchContext { - completedBatches: number; - remainingBatches: number; -} -type BatchConverterSync = (batch: I[], context: BatchContext) => O[]; -type BatchHandlerSync = (batch: I[], context: BatchContext) => void; -type BatchConverterAsync = (batch: I[], context: BatchContext) => Promise; -type BatchHandlerAsync = (batch: I[], context: BatchContext) => Promise; -type BatchConverter = BatchConverterSync | BatchConverterAsync; -type BatchHandler = BatchHandlerSync | BatchHandlerAsync; -type FixedBatcher = { batchSize: number } | { batchCount: number, mode?: Mode }; -interface ExecutorResult { - updated: A; - makeNextBatch: boolean; -} -interface PredicateBatcher { - executor: (element: I, accumulator: A) => ExecutorResult; - initial: A; - persistAccumulator?: boolean; -} -interface PredicateBatcherAsync { - executor: (element: I, accumulator: A) => Promise>; - initial: A; - persistAccumulator?: boolean; -} -type Batcher = FixedBatcher | PredicateBatcher; -type BatcherAsync = Batcher | PredicateBatcherAsync; - -enum TimeUnit { - Milliseconds, - Seconds, - Minutes -} - -interface Interval { - magnitude: number; - unit: TimeUnit; -} - -interface Array { - fixedBatch(batcher: FixedBatcher): T[][]; - predicateBatch(batcher: PredicateBatcher): T[][]; - predicateBatchAsync(batcher: PredicateBatcherAsync): Promise; - batch(batcher: Batcher): T[][]; - batchAsync(batcher: BatcherAsync): Promise; - - batchedForEach(batcher: Batcher, handler: BatchHandlerSync): void; - batchedMap(batcher: Batcher, handler: BatchConverterSync): O[]; - - batchedForEachAsync(batcher: Batcher, handler: BatchHandler): Promise; - batchedMapAsync(batcher: Batcher, handler: BatchConverter): Promise; - - batchedForEachInterval(batcher: Batcher, handler: BatchHandler, interval: Interval): Promise; - batchedMapInterval(batcher: Batcher, handler: BatchConverter, interval: Interval): Promise; - - lastElement(): T; -} - -Array.prototype.fixedBatch = extensions.FixedBatch; -Array.prototype.predicateBatch = extensions.PredicateBatch; -Array.prototype.predicateBatchAsync = extensions.PredicateBatchAsync; -Array.prototype.batch = extensions.Batch; -Array.prototype.batchAsync = extensions.BatchAsync; - -Array.prototype.batchedForEach = extensions.ExecuteInBatches; -Array.prototype.batchedMap = extensions.ConvertInBatches; -Array.prototype.batchedForEachAsync = extensions.ExecuteInBatchesAsync; -Array.prototype.batchedMapAsync = extensions.ConvertInBatchesAsync; -Array.prototype.batchedForEachInterval = extensions.ExecuteInBatchesAtInterval; -Array.prototype.batchedMapInterval = extensions.ConvertInBatchesAtInterval; - -Array.prototype.lastElement = function () { - if (!this.length) { - return undefined; - } - const last: T = this[this.length - 1]; - return last; -}; - interface Math { log10(val: number): number; } diff --git a/src/client/util/UtilExtensions.ts b/src/client/util/UtilExtensions.ts deleted file mode 100644 index 1447e37cb..000000000 --- a/src/client/util/UtilExtensions.ts +++ /dev/null @@ -1,259 +0,0 @@ -module.exports.Batch = function (batcher: Batcher): T[][] { - if ("executor" in batcher) { - return this.predicateBatch(batcher); - } else { - return this.fixedBatch(batcher); - } -}; - -module.exports.BatchAsync = async function (batcher: BatcherAsync): Promise { - if ("executor" in batcher) { - return this.predicateBatchAsync(batcher); - } else { - return this.fixedBatch(batcher); - } -}; - -module.exports.PredicateBatch = function (batcher: PredicateBatcher): T[][] { - const batches: T[][] = []; - let batch: T[] = []; - const { executor, initial, persistAccumulator } = batcher; - let accumulator = initial; - for (let element of this) { - const { updated, makeNextBatch } = executor(element, accumulator); - accumulator = updated; - if (!makeNextBatch) { - batch.push(element); - } else { - batches.push(batch); - batch = [element]; - if (!persistAccumulator) { - accumulator = initial; - } - } - } - batches.push(batch); - return batches; -}; - -module.exports.PredicateBatchAsync = async function (batcher: PredicateBatcherAsync): Promise { - const batches: T[][] = []; - let batch: T[] = []; - const { executor, initial, persistAccumulator } = batcher; - let accumulator: A = initial; - for (let element of this) { - const { updated, makeNextBatch } = await executor(element, accumulator); - accumulator = updated; - if (!makeNextBatch) { - batch.push(element); - } else { - batches.push(batch); - batch = [element]; - if (!persistAccumulator) { - accumulator = initial; - } - } - } - batches.push(batch); - return batches; -}; - -enum Mode { - Balanced, - Even -} - -module.exports.FixedBatch = function (batcher: FixedBatcher): T[][] { - const batches: T[][] = []; - const length = this.length; - let i = 0; - if ("batchSize" in batcher) { - const { batchSize } = batcher; - while (i < this.length) { - const cap = Math.min(i + batchSize, length); - batches.push(this.slice(i, i = cap)); - } - } else if ("batchCount" in batcher) { - let { batchCount, mode } = batcher; - const resolved = mode || Mode.Balanced; - if (batchCount < 1) { - throw new Error("Batch count must be a positive integer!"); - } - if (batchCount === 1) { - return [this]; - } - if (batchCount >= this.length) { - return this.map((element: T) => [element]); - } - - let length = this.length; - let size: number; - - if (length % batchCount === 0) { - size = Math.floor(length / batchCount); - while (i < length) { - batches.push(this.slice(i, i += size)); - } - } else if (resolved === Mode.Balanced) { - while (i < length) { - size = Math.ceil((length - i) / batchCount--); - batches.push(this.slice(i, i += size)); - } - } else { - batchCount--; - size = Math.floor(length / batchCount); - if (length % size === 0) { - size--; - } - while (i < size * batchCount) { - batches.push(this.slice(i, i += size)); - } - batches.push(this.slice(size * batchCount)); - } - } - return batches; -}; - -module.exports.ExecuteBatches = function (batcher: Batcher, handler: BatchHandlerSync): void { - if (this.length) { - let completed = 0; - const batches = this.batch(batcher); - const quota = batches.length; - for (let batch of batches) { - const context: BatchContext = { - completedBatches: completed, - remainingBatches: quota - completed, - }; - handler(batch, context); - completed++; - } - } -}; - -module.exports.ConvertInBatches = function (batcher: Batcher, handler: BatchConverterSync): O[] { - if (!this.length) { - return []; - } - let collector: O[] = []; - let completed = 0; - const batches = this.batch(batcher); - const quota = batches.length; - for (let batch of batches) { - const context: BatchContext = { - completedBatches: completed, - remainingBatches: quota - completed, - }; - collector.push(...handler(batch, context)); - completed++; - } - return collector; -}; - -module.exports.ExecuteInBatchesAsync = async function (batcher: BatcherAsync, handler: BatchHandler): Promise { - if (this.length) { - let completed = 0; - const batches = await this.batchAsync(batcher); - const quota = batches.length; - for (let batch of batches) { - const context: BatchContext = { - completedBatches: completed, - remainingBatches: quota - completed, - }; - await handler(batch, context); - completed++; - } - } -}; - -module.exports.ConvertInBatchesAsync = async function (batcher: BatcherAsync, handler: BatchConverter): Promise { - if (!this.length) { - return []; - } - let collector: O[] = []; - let completed = 0; - const batches = await this.batchAsync(batcher); - const quota = batches.length; - for (let batch of batches) { - const context: BatchContext = { - completedBatches: completed, - remainingBatches: quota - completed, - }; - collector.push(...(await handler(batch, context))); - completed++; - } - return collector; -}; - -const convert = (interval: Interval) => { - const { magnitude, unit } = interval; - switch (unit) { - default: - case TimeUnit.Milliseconds: - return magnitude; - case TimeUnit.Seconds: - return magnitude * 1000; - case TimeUnit.Minutes: - return magnitude * 1000 * 60; - } -}; - -module.exports.ExecuteInBatchesAtInterval = async function (batcher: BatcherAsync, handler: BatchHandler, interval: Interval): Promise { - if (!this.length) { - return; - } - const batches = await this.batchAsync(batcher); - const quota = batches.length; - return new Promise(async resolve => { - const iterator = batches[Symbol.iterator](); - let completed = 0; - while (true) { - const next = iterator.next(); - await new Promise(resolve => { - setTimeout(async () => { - const batch = next.value; - const context: BatchContext = { - completedBatches: completed, - remainingBatches: quota - completed, - }; - await handler(batch, context); - resolve(); - }, convert(interval)); - }); - if (++completed === quota) { - break; - } - } - resolve(); - }); -}; - -module.exports.ConvertInBatchesAtInterval = async function (batcher: BatcherAsync, handler: BatchConverter, interval: Interval): Promise { - if (!this.length) { - return []; - } - let collector: O[] = []; - const batches = await this.batchAsync(batcher); - const quota = batches.length; - return new Promise(async resolve => { - const iterator = batches[Symbol.iterator](); - let completed = 0; - while (true) { - const next = iterator.next(); - await new Promise(resolve => { - setTimeout(async () => { - const batch = next.value; - const context: BatchContext = { - completedBatches: completed, - remainingBatches: quota - completed, - }; - collector.push(...(await handler(batch, context))); - resolve(); - }, convert(interval)); - }); - if (++completed === quota) { - resolve(collector); - break; - } - } - }); -}; \ No newline at end of file diff --git a/src/client/views/Main.tsx b/src/client/views/Main.tsx index aa002cee9..55fa138c8 100644 --- a/src/client/views/Main.tsx +++ b/src/client/views/Main.tsx @@ -7,18 +7,9 @@ import { Cast } from "../../new_fields/Types"; import { Doc, DocListCastAsync } from "../../new_fields/Doc"; import { List } from "../../new_fields/List"; import { DocServer } from "../DocServer"; +const ArrayExtensions = require("../../extensions/ArrayExtensions"); -String.prototype.removeTrailingNewlines = function () { - let sliced = this; - while (sliced.endsWith("\n")) { - sliced = sliced.substring(0, this.length - 1); - } - return sliced as string; -}; - -String.prototype.hasNewline = function () { - return this.endsWith("\n"); -}; +ArrayExtensions.AssignArrayExtensions(); let swapDocs = async () => { let oldDoc = await Cast(CurrentUserUtils.UserDocument.linkManagerDoc, Doc); diff --git a/src/extensions/ArrayExtensions.ts b/src/extensions/ArrayExtensions.ts new file mode 100644 index 000000000..1190aa48c --- /dev/null +++ b/src/extensions/ArrayExtensions.ts @@ -0,0 +1,341 @@ +interface BatchContext { + completedBatches: number; + remainingBatches: number; +} +type BatchConverterSync = (batch: I[], context: BatchContext) => O[]; +type BatchHandlerSync = (batch: I[], context: BatchContext) => void; +type BatchConverterAsync = (batch: I[], context: BatchContext) => Promise; +type BatchHandlerAsync = (batch: I[], context: BatchContext) => Promise; +type BatchConverter = BatchConverterSync | BatchConverterAsync; +type BatchHandler = BatchHandlerSync | BatchHandlerAsync; +type FixedBatcher = { batchSize: number } | { batchCount: number, mode?: Mode }; +interface ExecutorResult { + updated: A; + makeNextBatch: boolean; +} +interface PredicateBatcher { + executor: (element: I, accumulator: A) => ExecutorResult; + initial: A; + persistAccumulator?: boolean; +} +interface PredicateBatcherAsyncInterface { + executor: (element: I, accumulator: A) => Promise>; + initial: A; + persistAccumulator?: boolean; +} +type PredicateBatcherAsync = PredicateBatcher | PredicateBatcherAsyncInterface; +type Batcher = FixedBatcher | PredicateBatcher; +type BatcherAsync = Batcher | PredicateBatcherAsync; + +enum TimeUnit { + Milliseconds, + Seconds, + Minutes +} + +interface Interval { + magnitude: number; + unit: TimeUnit; +} + +enum Mode { + Balanced, + Even +} + +const convert = (interval: Interval) => { + const { magnitude, unit } = interval; + switch (unit) { + default: + case TimeUnit.Milliseconds: + return magnitude; + case TimeUnit.Seconds: + return magnitude * 1000; + case TimeUnit.Minutes: + return magnitude * 1000 * 60; + } +}; + +interface Array { + fixedBatch(batcher: FixedBatcher): T[][]; + predicateBatch(batcher: PredicateBatcher): T[][]; + predicateBatchAsync(batcher: PredicateBatcherAsync): Promise; + batch(batcher: Batcher): T[][]; + batchAsync(batcher: BatcherAsync): Promise; + + batchedForEach(batcher: Batcher, handler: BatchHandlerSync): void; + batchedMap(batcher: Batcher, handler: BatchConverterSync): O[]; + + batchedForEachAsync(batcher: Batcher, handler: BatchHandler): Promise; + batchedMapAsync(batcher: Batcher, handler: BatchConverter): Promise; + + batchedForEachInterval(batcher: Batcher, handler: BatchHandler, interval: Interval): Promise; + batchedMapInterval(batcher: Batcher, handler: BatchConverter, interval: Interval): Promise; + + lastElement(): T; +} + +module.exports.AssignArrayExtensions = function () { + Array.prototype.fixedBatch = module.exports.fixedBatch; + Array.prototype.predicateBatch = module.exports.predicateBatch; + Array.prototype.predicateBatchAsync = module.exports.predicateBatchAsync; + Array.prototype.batch = module.exports.batch; + Array.prototype.batchAsync = module.exports.batchAsync; + Array.prototype.batchedForEach = module.exports.batchedForEach; + Array.prototype.batchedMap = module.exports.batchedMap; + Array.prototype.batchedForEachAsync = module.exports.batchedForEachAsync; + Array.prototype.batchedMapAsync = module.exports.batchedMapAsync; + Array.prototype.batchedForEachInterval = module.exports.batchedForEachInterval; + Array.prototype.batchedMapInterval = module.exports.batchedMapInterval; + Array.prototype.lastElement = module.exports.lastElement; +}; + +module.exports.fixedBatch = function (batcher: FixedBatcher): T[][] { + const batches: T[][] = []; + const length = this.length; + let i = 0; + if ("batchSize" in batcher) { + const { batchSize } = batcher; + while (i < this.length) { + const cap = Math.min(i + batchSize, length); + batches.push(this.slice(i, i = cap)); + } + } else if ("batchCount" in batcher) { + let { batchCount, mode } = batcher; + const resolved = mode || Mode.Balanced; + if (batchCount < 1) { + throw new Error("Batch count must be a positive integer!"); + } + if (batchCount === 1) { + return [this]; + } + if (batchCount >= this.length) { + return this.map((element: T) => [element]); + } + + let length = this.length; + let size: number; + + if (length % batchCount === 0) { + size = Math.floor(length / batchCount); + while (i < length) { + batches.push(this.slice(i, i += size)); + } + } else if (resolved === Mode.Balanced) { + while (i < length) { + size = Math.ceil((length - i) / batchCount--); + batches.push(this.slice(i, i += size)); + } + } else { + batchCount--; + size = Math.floor(length / batchCount); + if (length % size === 0) { + size--; + } + while (i < size * batchCount) { + batches.push(this.slice(i, i += size)); + } + batches.push(this.slice(size * batchCount)); + } + } + return batches; +}; + +module.exports.predicateBatch = function (batcher: PredicateBatcher): T[][] { + const batches: T[][] = []; + let batch: T[] = []; + const { executor, initial, persistAccumulator } = batcher; + let accumulator = initial; + for (let element of this) { + const { updated, makeNextBatch } = executor(element, accumulator); + accumulator = updated; + if (!makeNextBatch) { + batch.push(element); + } else { + batches.push(batch); + batch = [element]; + if (!persistAccumulator) { + accumulator = initial; + } + } + } + batches.push(batch); + return batches; +}; + +module.exports.predicateBatchAsync = async function (batcher: PredicateBatcherAsync): Promise { + const batches: T[][] = []; + let batch: T[] = []; + const { executor, initial, persistAccumulator } = batcher; + let accumulator: A = initial; + for (let element of this) { + const { updated, makeNextBatch } = await executor(element, accumulator); + accumulator = updated; + if (!makeNextBatch) { + batch.push(element); + } else { + batches.push(batch); + batch = [element]; + if (!persistAccumulator) { + accumulator = initial; + } + } + } + batches.push(batch); + return batches; +}; + +module.exports.batch = function (batcher: Batcher): T[][] { + if ("executor" in batcher) { + return this.predicateBatch(batcher); + } else { + return this.fixedBatch(batcher); + } +}; + +module.exports.batchAsync = async function (batcher: BatcherAsync): Promise { + if ("executor" in batcher) { + return this.predicateBatchAsync(batcher); + } else { + return this.fixedBatch(batcher); + } +}; + +module.exports.batchedForEach = function (batcher: Batcher, handler: BatchHandlerSync): void { + if (this.length) { + let completed = 0; + const batches = this.batch(batcher); + const quota = batches.length; + for (let batch of batches) { + const context: BatchContext = { + completedBatches: completed, + remainingBatches: quota - completed, + }; + handler(batch, context); + completed++; + } + } +}; + +module.exports.batchedMap = function (batcher: Batcher, handler: BatchConverterSync): O[] { + if (!this.length) { + return []; + } + let collector: O[] = []; + let completed = 0; + const batches = this.batch(batcher); + const quota = batches.length; + for (let batch of batches) { + const context: BatchContext = { + completedBatches: completed, + remainingBatches: quota - completed, + }; + collector.push(...handler(batch, context)); + completed++; + } + return collector; +}; + +module.exports.batchedForEachAsync = async function (batcher: BatcherAsync, handler: BatchHandler): Promise { + if (this.length) { + let completed = 0; + const batches = await this.batchAsync(batcher); + const quota = batches.length; + for (let batch of batches) { + const context: BatchContext = { + completedBatches: completed, + remainingBatches: quota - completed, + }; + await handler(batch, context); + completed++; + } + } +}; + +module.exports.batchedMapAsync = async function (batcher: BatcherAsync, handler: BatchConverter): Promise { + if (!this.length) { + return []; + } + let collector: O[] = []; + let completed = 0; + const batches = await this.batchAsync(batcher); + const quota = batches.length; + for (let batch of batches) { + const context: BatchContext = { + completedBatches: completed, + remainingBatches: quota - completed, + }; + collector.push(...(await handler(batch, context))); + completed++; + } + return collector; +}; + +module.exports.batchedForEachInterval = async function (batcher: BatcherAsync, handler: BatchHandler, interval: Interval): Promise { + if (!this.length) { + return; + } + const batches = await this.batchAsync(batcher); + const quota = batches.length; + return new Promise(async resolve => { + const iterator = batches[Symbol.iterator](); + let completed = 0; + while (true) { + const next = iterator.next(); + await new Promise(resolve => { + setTimeout(async () => { + const batch = next.value; + const context: BatchContext = { + completedBatches: completed, + remainingBatches: quota - completed, + }; + await handler(batch, context); + resolve(); + }, convert(interval)); + }); + if (++completed === quota) { + break; + } + } + resolve(); + }); +}; + +module.exports.batchedMapInterval = async function (batcher: BatcherAsync, handler: BatchConverter, interval: Interval): Promise { + if (!this.length) { + return []; + } + let collector: O[] = []; + const batches = await this.batchAsync(batcher); + const quota = batches.length; + return new Promise(async resolve => { + const iterator = batches[Symbol.iterator](); + let completed = 0; + while (true) { + const next = iterator.next(); + await new Promise(resolve => { + setTimeout(async () => { + const batch = next.value; + const context: BatchContext = { + completedBatches: completed, + remainingBatches: quota - completed, + }; + collector.push(...(await handler(batch, context))); + resolve(); + }, convert(interval)); + }); + if (++completed === quota) { + resolve(collector); + break; + } + } + }); +}; + +module.exports.lastElement = function () { + if (!this.length) { + return undefined; + } + const last: T = this[this.length - 1]; + return last; +}; diff --git a/src/extensions/StringExtensions.ts b/src/extensions/StringExtensions.ts new file mode 100644 index 000000000..1168fdda8 --- /dev/null +++ b/src/extensions/StringExtensions.ts @@ -0,0 +1,20 @@ +interface String { + removeTrailingNewlines(): string; + hasNewline(): boolean; +} + +function AssignStringExtensions() { + + String.prototype.removeTrailingNewlines = function () { + let sliced = this; + while (sliced.endsWith("\n")) { + sliced = sliced.substring(0, this.length - 1); + } + return sliced as string; + }; + + String.prototype.hasNewline = function () { + return this.endsWith("\n"); + }; + +} \ No newline at end of file diff --git a/src/server/apis/google/GooglePhotosUploadUtils.ts b/src/server/apis/google/GooglePhotosUploadUtils.ts index 9275bf125..7eaf8a8b7 100644 --- a/src/server/apis/google/GooglePhotosUploadUtils.ts +++ b/src/server/apis/google/GooglePhotosUploadUtils.ts @@ -7,6 +7,7 @@ import { Opt } from '../../../new_fields/Doc'; import * as sharp from 'sharp'; import { MediaItemCreationResult, NewMediaItemResult } from './SharedTypes'; import { NewMediaItem } from '../..'; +import { TimeUnit } from "../../index"; const uploadDirectory = path.join(__dirname, "../../public/files/"); diff --git a/src/server/credentials/google_docs_token.json b/src/server/credentials/google_docs_token.json index be50f4dd2..a476498a8 100644 --- a/src/server/credentials/google_docs_token.json +++ b/src/server/credentials/google_docs_token.json @@ -1 +1 @@ -{"access_token":"ya29.ImCHB-QNh-blUK7Ajw1Dk1yNBZ7w5JAKcrgzW4xYE3sfNTLdq_gXEQOK0ZZeJBPL1_WcJbbX7EKuMxWN-fwWMuhAG6-IHk1TpkOG1KZpSGo3myT7xiZDnONxLfVuodT0Nlc","refresh_token":"1/HTv_xFHszu2Nf3iiFrUTaeKzC_Vp2-6bpIB06xW_WHI","scope":"https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/documents.readonly https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/photoslibrary https://www.googleapis.com/auth/photoslibrary.appendonly https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/photoslibrary.sharing","token_type":"Bearer","expiry_date":1568797587975} \ No newline at end of file +{"access_token":"ya29.ImCIBw01ZtZwR2NI608a-TfejTTGAzAWICqX9QdfNcLHo4upydH3tvpR7l5YmEbyuH2CHjHSQW2QKAPU_zXSpGAo_ZjQE5iRqsP_VdlSDVCS_NyabpHNL5m-0tmdyZJ8Qoc","refresh_token":"1/HTv_xFHszu2Nf3iiFrUTaeKzC_Vp2-6bpIB06xW_WHI","scope":"https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/documents.readonly https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/photoslibrary https://www.googleapis.com/auth/photoslibrary.appendonly https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/photoslibrary.sharing","token_type":"Bearer","expiry_date":1568919703546} \ No newline at end of file diff --git a/src/server/index.ts b/src/server/index.ts index 83bbe734b..6a06454ec 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -47,7 +47,7 @@ const mongoose = require('mongoose'); const probe = require("probe-image-size"); import * as qs from 'query-string'; import { Opt } from '../new_fields/Doc'; -const extensions = require("../client/util/UtilExtensions"); +const ArrayExtensions = require("../extensions/ArrayExtensions"); const download = (url: string, dest: fs.PathLike) => request.get(url).pipe(fs.createWriteStream(dest)); let youtubeApiKey: string; @@ -99,6 +99,8 @@ enum Method { POST } +ArrayExtensions.AssignArrayExtensions(); + /** * Please invoke this function when adding a new route to Dash's server. * It ensures that any requests leading to or containing user-sensitive information @@ -835,18 +837,11 @@ export interface NewMediaItem { }; } -Array.prototype.fixedBatch = extensions.FixedBatch; -Array.prototype.predicateBatch = extensions.PredicateBatch; -Array.prototype.predicateBatchAsync = extensions.PredicateBatchAsync; -Array.prototype.batch = extensions.Batch; -Array.prototype.batchAsync = extensions.BatchAsync; - -Array.prototype.batchedForEach = extensions.ExecuteInBatches; -Array.prototype.batchedMap = extensions.ConvertInBatches; -Array.prototype.batchedForEachAsync = extensions.ExecuteInBatchesAsync; -Array.prototype.batchedMapAsync = extensions.ConvertInBatchesAsync; -Array.prototype.batchedForEachInterval = extensions.ExecuteInBatchesAtInterval; -Array.prototype.batchedMapInterval = extensions.ConvertInBatchesAtInterval; +export enum TimeUnit { + Milliseconds, + Seconds, + Minutes +} app.post(RouteStore.googlePhotosMediaUpload, async (req, res) => { const mediaInput: GooglePhotosUploadUtils.MediaInput[] = req.body.media; -- cgit v1.2.3-70-g09d2 From 3fb212eb601dbaff28a61100fc0478b1c706a3b8 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Thu, 19 Sep 2019 16:30:14 -0400 Subject: organized and genericized --- src/extensions/ArrayExtensions.ts | 537 +++++++++++----------- src/extensions/Extensions.ts | 4 +- src/extensions/StringExtensions.ts | 2 +- src/server/apis/google/GooglePhotosUploadUtils.ts | 6 +- src/server/index.ts | 8 +- 5 files changed, 270 insertions(+), 287 deletions(-) (limited to 'src/server/apis') diff --git a/src/extensions/ArrayExtensions.ts b/src/extensions/ArrayExtensions.ts index 1190aa48c..70508f6df 100644 --- a/src/extensions/ArrayExtensions.ts +++ b/src/extensions/ArrayExtensions.ts @@ -1,61 +1,3 @@ -interface BatchContext { - completedBatches: number; - remainingBatches: number; -} -type BatchConverterSync = (batch: I[], context: BatchContext) => O[]; -type BatchHandlerSync = (batch: I[], context: BatchContext) => void; -type BatchConverterAsync = (batch: I[], context: BatchContext) => Promise; -type BatchHandlerAsync = (batch: I[], context: BatchContext) => Promise; -type BatchConverter = BatchConverterSync | BatchConverterAsync; -type BatchHandler = BatchHandlerSync | BatchHandlerAsync; -type FixedBatcher = { batchSize: number } | { batchCount: number, mode?: Mode }; -interface ExecutorResult { - updated: A; - makeNextBatch: boolean; -} -interface PredicateBatcher { - executor: (element: I, accumulator: A) => ExecutorResult; - initial: A; - persistAccumulator?: boolean; -} -interface PredicateBatcherAsyncInterface { - executor: (element: I, accumulator: A) => Promise>; - initial: A; - persistAccumulator?: boolean; -} -type PredicateBatcherAsync = PredicateBatcher | PredicateBatcherAsyncInterface; -type Batcher = FixedBatcher | PredicateBatcher; -type BatcherAsync = Batcher | PredicateBatcherAsync; - -enum TimeUnit { - Milliseconds, - Seconds, - Minutes -} - -interface Interval { - magnitude: number; - unit: TimeUnit; -} - -enum Mode { - Balanced, - Even -} - -const convert = (interval: Interval) => { - const { magnitude, unit } = interval; - switch (unit) { - default: - case TimeUnit.Milliseconds: - return magnitude; - case TimeUnit.Seconds: - return magnitude * 1000; - case TimeUnit.Minutes: - return magnitude * 1000 * 60; - } -}; - interface Array { fixedBatch(batcher: FixedBatcher): T[][]; predicateBatch(batcher: PredicateBatcher): T[][]; @@ -75,134 +17,185 @@ interface Array { lastElement(): T; } -module.exports.AssignArrayExtensions = function () { - Array.prototype.fixedBatch = module.exports.fixedBatch; - Array.prototype.predicateBatch = module.exports.predicateBatch; - Array.prototype.predicateBatchAsync = module.exports.predicateBatchAsync; - Array.prototype.batch = module.exports.batch; - Array.prototype.batchAsync = module.exports.batchAsync; - Array.prototype.batchedForEach = module.exports.batchedForEach; - Array.prototype.batchedMap = module.exports.batchedMap; - Array.prototype.batchedForEachAsync = module.exports.batchedForEachAsync; - Array.prototype.batchedMapAsync = module.exports.batchedMapAsync; - Array.prototype.batchedForEachInterval = module.exports.batchedForEachInterval; - Array.prototype.batchedMapInterval = module.exports.batchedMapInterval; - Array.prototype.lastElement = module.exports.lastElement; +interface BatchContext { + completedBatches: number; + remainingBatches: number; +} + +interface ExecutorResult { + updated: A; + makeNextBatch: boolean; +} + +interface PredicateBatcherCommon { + initial: A; + persistAccumulator?: boolean; +} + +interface Interval { + magnitude: number; + unit: typeof module.exports.TimeUnit; +} + +type BatchConverterSync = (batch: I[], context: BatchContext) => O[]; +type BatchHandlerSync = (batch: I[], context: BatchContext) => void; +type BatchConverterAsync = (batch: I[], context: BatchContext) => Promise; +type BatchHandlerAsync = (batch: I[], context: BatchContext) => Promise; +type BatchConverter = BatchConverterSync | BatchConverterAsync; +type BatchHandler = BatchHandlerSync | BatchHandlerAsync; + +type FixedBatcher = { batchSize: number } | { batchCount: number, mode?: typeof module.exports.Mode }; +type PredicateBatcher = PredicateBatcherCommon & { executor: (element: I, accumulator: A) => ExecutorResult }; +type PredicateBatcherAsync = PredicateBatcherCommon & { executor: (element: I, accumulator: A) => ExecutorResult | Promise> }; + +type Batcher = FixedBatcher | PredicateBatcher; +type BatcherAsync = Batcher | PredicateBatcherAsync; + +module.exports.Mode = { + Balanced: 0, + Even: 1 }; -module.exports.fixedBatch = function (batcher: FixedBatcher): T[][] { - const batches: T[][] = []; - const length = this.length; - let i = 0; - if ("batchSize" in batcher) { - const { batchSize } = batcher; - while (i < this.length) { - const cap = Math.min(i + batchSize, length); - batches.push(this.slice(i, i = cap)); - } - } else if ("batchCount" in batcher) { - let { batchCount, mode } = batcher; - const resolved = mode || Mode.Balanced; - if (batchCount < 1) { - throw new Error("Batch count must be a positive integer!"); - } - if (batchCount === 1) { - return [this]; - } - if (batchCount >= this.length) { - return this.map((element: T) => [element]); - } +module.exports.TimeUnit = { + Milliseconds: 0, + Seconds: 1, + Minutes: 2 +}; - let length = this.length; - let size: number; +module.exports.Assign = function () { - if (length % batchCount === 0) { - size = Math.floor(length / batchCount); - while (i < length) { - batches.push(this.slice(i, i += size)); + Array.prototype.fixedBatch = function (batcher: FixedBatcher): T[][] { + const batches: T[][] = []; + const length = this.length; + let i = 0; + if ("batchSize" in batcher) { + const { batchSize } = batcher; + while (i < this.length) { + const cap = Math.min(i + batchSize, length); + batches.push(this.slice(i, i = cap)); } - } else if (resolved === Mode.Balanced) { - while (i < length) { - size = Math.ceil((length - i) / batchCount--); - batches.push(this.slice(i, i += size)); + } else if ("batchCount" in batcher) { + let { batchCount, mode } = batcher; + const resolved = mode || module.exports.Mode.Balanced; + if (batchCount < 1) { + throw new Error("Batch count must be a positive integer!"); } - } else { - batchCount--; - size = Math.floor(length / batchCount); - if (length % size === 0) { - size--; + if (batchCount === 1) { + return [this]; } - while (i < size * batchCount) { - batches.push(this.slice(i, i += size)); + if (batchCount >= this.length) { + return this.map((element: T) => [element]); + } + + let length = this.length; + let size: number; + + if (length % batchCount === 0) { + size = Math.floor(length / batchCount); + while (i < length) { + batches.push(this.slice(i, i += size)); + } + } else if (resolved === module.exports.Mode.Balanced) { + while (i < length) { + size = Math.ceil((length - i) / batchCount--); + batches.push(this.slice(i, i += size)); + } + } else { + batchCount--; + size = Math.floor(length / batchCount); + if (length % size === 0) { + size--; + } + while (i < size * batchCount) { + batches.push(this.slice(i, i += size)); + } + batches.push(this.slice(size * batchCount)); } - batches.push(this.slice(size * batchCount)); } - } - return batches; -}; + return batches; + }; -module.exports.predicateBatch = function (batcher: PredicateBatcher): T[][] { - const batches: T[][] = []; - let batch: T[] = []; - const { executor, initial, persistAccumulator } = batcher; - let accumulator = initial; - for (let element of this) { - const { updated, makeNextBatch } = executor(element, accumulator); - accumulator = updated; - if (!makeNextBatch) { - batch.push(element); - } else { - batches.push(batch); - batch = [element]; - if (!persistAccumulator) { - accumulator = initial; + Array.prototype.predicateBatch = function (batcher: PredicateBatcher): T[][] { + const batches: T[][] = []; + let batch: T[] = []; + const { executor, initial, persistAccumulator } = batcher; + let accumulator = initial; + for (let element of this) { + const { updated, makeNextBatch } = executor(element, accumulator); + accumulator = updated; + if (!makeNextBatch) { + batch.push(element); + } else { + batches.push(batch); + batch = [element]; + if (!persistAccumulator) { + accumulator = initial; + } } } - } - batches.push(batch); - return batches; -}; + batches.push(batch); + return batches; + }; -module.exports.predicateBatchAsync = async function (batcher: PredicateBatcherAsync): Promise { - const batches: T[][] = []; - let batch: T[] = []; - const { executor, initial, persistAccumulator } = batcher; - let accumulator: A = initial; - for (let element of this) { - const { updated, makeNextBatch } = await executor(element, accumulator); - accumulator = updated; - if (!makeNextBatch) { - batch.push(element); - } else { - batches.push(batch); - batch = [element]; - if (!persistAccumulator) { - accumulator = initial; + Array.prototype.predicateBatchAsync = async function (batcher: PredicateBatcherAsync): Promise { + const batches: T[][] = []; + let batch: T[] = []; + const { executor, initial, persistAccumulator } = batcher; + let accumulator: A = initial; + for (let element of this) { + const { updated, makeNextBatch } = await executor(element, accumulator); + accumulator = updated; + if (!makeNextBatch) { + batch.push(element); + } else { + batches.push(batch); + batch = [element]; + if (!persistAccumulator) { + accumulator = initial; + } } } - } - batches.push(batch); - return batches; -}; + batches.push(batch); + return batches; + }; -module.exports.batch = function (batcher: Batcher): T[][] { - if ("executor" in batcher) { - return this.predicateBatch(batcher); - } else { - return this.fixedBatch(batcher); - } -}; + Array.prototype.batch = function (batcher: Batcher): T[][] { + if ("executor" in batcher) { + return this.predicateBatch(batcher); + } else { + return this.fixedBatch(batcher); + } + }; -module.exports.batchAsync = async function (batcher: BatcherAsync): Promise { - if ("executor" in batcher) { - return this.predicateBatchAsync(batcher); - } else { - return this.fixedBatch(batcher); - } -}; + Array.prototype.batchAsync = async function (batcher: BatcherAsync): Promise { + if ("executor" in batcher) { + return this.predicateBatchAsync(batcher); + } else { + return this.fixedBatch(batcher); + } + }; + + Array.prototype.batchedForEach = function (batcher: Batcher, handler: BatchHandlerSync): void { + if (this.length) { + let completed = 0; + const batches = this.batch(batcher); + const quota = batches.length; + for (let batch of batches) { + const context: BatchContext = { + completedBatches: completed, + remainingBatches: quota - completed, + }; + handler(batch, context); + completed++; + } + } + }; -module.exports.batchedForEach = function (batcher: Batcher, handler: BatchHandlerSync): void { - if (this.length) { + Array.prototype.batchedMap = function (batcher: Batcher, handler: BatchConverterSync): O[] { + if (!this.length) { + return []; + } + let collector: O[] = []; let completed = 0; const batches = this.batch(batcher); const quota = batches.length; @@ -211,33 +204,33 @@ module.exports.batchedForEach = function (batcher: Batcher, handler: completedBatches: completed, remainingBatches: quota - completed, }; - handler(batch, context); + collector.push(...handler(batch, context)); completed++; } - } -}; + return collector; + }; -module.exports.batchedMap = function (batcher: Batcher, handler: BatchConverterSync): O[] { - if (!this.length) { - return []; - } - let collector: O[] = []; - let completed = 0; - const batches = this.batch(batcher); - const quota = batches.length; - for (let batch of batches) { - const context: BatchContext = { - completedBatches: completed, - remainingBatches: quota - completed, - }; - collector.push(...handler(batch, context)); - completed++; - } - return collector; -}; + Array.prototype.batchedForEachAsync = async function (batcher: BatcherAsync, handler: BatchHandler): Promise { + if (this.length) { + let completed = 0; + const batches = await this.batchAsync(batcher); + const quota = batches.length; + for (let batch of batches) { + const context: BatchContext = { + completedBatches: completed, + remainingBatches: quota - completed, + }; + await handler(batch, context); + completed++; + } + } + }; -module.exports.batchedForEachAsync = async function (batcher: BatcherAsync, handler: BatchHandler): Promise { - if (this.length) { + Array.prototype.batchedMapAsync = async function (batcher: BatcherAsync, handler: BatchConverter): Promise { + if (!this.length) { + return []; + } + let collector: O[] = []; let completed = 0; const batches = await this.batchAsync(batcher); const quota = batches.length; @@ -246,96 +239,92 @@ module.exports.batchedForEachAsync = async function (batcher: BatcherAsync completedBatches: completed, remainingBatches: quota - completed, }; - await handler(batch, context); + collector.push(...(await handler(batch, context))); completed++; } - } -}; + return collector; + }; -module.exports.batchedMapAsync = async function (batcher: BatcherAsync, handler: BatchConverter): Promise { - if (!this.length) { - return []; - } - let collector: O[] = []; - let completed = 0; - const batches = await this.batchAsync(batcher); - const quota = batches.length; - for (let batch of batches) { - const context: BatchContext = { - completedBatches: completed, - remainingBatches: quota - completed, - }; - collector.push(...(await handler(batch, context))); - completed++; - } - return collector; -}; - -module.exports.batchedForEachInterval = async function (batcher: BatcherAsync, handler: BatchHandler, interval: Interval): Promise { - if (!this.length) { - return; - } - const batches = await this.batchAsync(batcher); - const quota = batches.length; - return new Promise(async resolve => { - const iterator = batches[Symbol.iterator](); - let completed = 0; - while (true) { - const next = iterator.next(); - await new Promise(resolve => { - setTimeout(async () => { - const batch = next.value; - const context: BatchContext = { - completedBatches: completed, - remainingBatches: quota - completed, - }; - await handler(batch, context); - resolve(); - }, convert(interval)); - }); - if (++completed === quota) { - break; - } + Array.prototype.batchedForEachInterval = async function (batcher: BatcherAsync, handler: BatchHandler, interval: Interval): Promise { + if (!this.length) { + return; } - resolve(); - }); -}; + const batches = await this.batchAsync(batcher); + const quota = batches.length; + return new Promise(async resolve => { + const iterator = batches[Symbol.iterator](); + let completed = 0; + while (true) { + const next = iterator.next(); + await new Promise(resolve => { + setTimeout(async () => { + const batch = next.value; + const context: BatchContext = { + completedBatches: completed, + remainingBatches: quota - completed, + }; + await handler(batch, context); + resolve(); + }, convert(interval)); + }); + if (++completed === quota) { + break; + } + } + resolve(); + }); + }; -module.exports.batchedMapInterval = async function (batcher: BatcherAsync, handler: BatchConverter, interval: Interval): Promise { - if (!this.length) { - return []; - } - let collector: O[] = []; - const batches = await this.batchAsync(batcher); - const quota = batches.length; - return new Promise(async resolve => { - const iterator = batches[Symbol.iterator](); - let completed = 0; - while (true) { - const next = iterator.next(); - await new Promise(resolve => { - setTimeout(async () => { - const batch = next.value; - const context: BatchContext = { - completedBatches: completed, - remainingBatches: quota - completed, - }; - collector.push(...(await handler(batch, context))); - resolve(); - }, convert(interval)); - }); - if (++completed === quota) { - resolve(collector); - break; + Array.prototype.batchedMapInterval = async function (batcher: BatcherAsync, handler: BatchConverter, interval: Interval): Promise { + if (!this.length) { + return []; + } + let collector: O[] = []; + const batches = await this.batchAsync(batcher); + const quota = batches.length; + return new Promise(async resolve => { + const iterator = batches[Symbol.iterator](); + let completed = 0; + while (true) { + const next = iterator.next(); + await new Promise(resolve => { + setTimeout(async () => { + const batch = next.value; + const context: BatchContext = { + completedBatches: completed, + remainingBatches: quota - completed, + }; + collector.push(...(await handler(batch, context))); + resolve(); + }, convert(interval)); + }); + if (++completed === quota) { + resolve(collector); + break; + } } + }); + }; + + Array.prototype.lastElement = function () { + if (!this.length) { + return undefined; } - }); + const last: T = this[this.length - 1]; + return last; + }; + }; -module.exports.lastElement = function () { - if (!this.length) { - return undefined; +const convert = (interval: Interval) => { + const { magnitude, unit } = interval; + switch (unit) { + default: + case module.exports.Mode.TimeUnit.Milliseconds: + return magnitude; + case module.exports.Mode.TimeUnit.Seconds: + return magnitude * 1000; + case module.exports.Mode.TimeUnit.Minutes: + return magnitude * 1000 * 60; } - const last: T = this[this.length - 1]; - return last; -}; +}; \ No newline at end of file diff --git a/src/extensions/Extensions.ts b/src/extensions/Extensions.ts index 774236ea4..1bcebd0e2 100644 --- a/src/extensions/Extensions.ts +++ b/src/extensions/Extensions.ts @@ -2,6 +2,6 @@ const ArrayExtensions = require("./ArrayExtensions"); const StringExtensions = require("./StringExtensions"); module.exports.AssignExtensions = function () { - ArrayExtensions.Assign; - StringExtensions.Assign; + ArrayExtensions.Assign(); + StringExtensions.Assign(); }; \ No newline at end of file diff --git a/src/extensions/StringExtensions.ts b/src/extensions/StringExtensions.ts index 2ef31ec84..4cdbdebf7 100644 --- a/src/extensions/StringExtensions.ts +++ b/src/extensions/StringExtensions.ts @@ -3,7 +3,7 @@ interface String { hasNewline(): boolean; } -module.exports.AssignStringExtensions = function () { +module.exports.Assign = function () { String.prototype.removeTrailingNewlines = function () { let sliced = this; diff --git a/src/server/apis/google/GooglePhotosUploadUtils.ts b/src/server/apis/google/GooglePhotosUploadUtils.ts index 7eaf8a8b7..fc6772ffd 100644 --- a/src/server/apis/google/GooglePhotosUploadUtils.ts +++ b/src/server/apis/google/GooglePhotosUploadUtils.ts @@ -5,9 +5,9 @@ import { Utils } from '../../../Utils'; import * as path from 'path'; import { Opt } from '../../../new_fields/Doc'; import * as sharp from 'sharp'; -import { MediaItemCreationResult, NewMediaItemResult } from './SharedTypes'; -import { NewMediaItem } from '../..'; -import { TimeUnit } from "../../index"; +import { MediaItemCreationResult } from './SharedTypes'; +import { NewMediaItem } from "../../index"; +const { TimeUnit } = require("../../../extensions/ArrayExtensions"); const uploadDirectory = path.join(__dirname, "../../public/files/"); diff --git a/src/server/index.ts b/src/server/index.ts index 2d6c99d8a..1f411ade2 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -837,12 +837,6 @@ export interface NewMediaItem { }; } -export enum TimeUnit { - Milliseconds, - Seconds, - Minutes -} - app.post(RouteStore.googlePhotosMediaUpload, async (req, res) => { const mediaInput: GooglePhotosUploadUtils.MediaInput[] = req.body.media; await GooglePhotosUploadUtils.initialize({ uploadDirectory, credentialsPath, tokenPath }); @@ -865,7 +859,7 @@ app.post(RouteStore.googlePhotosMediaUpload, async (req, res) => { return newMediaItems; }; const batcher = { batchSize: 25 }; - const interval = { magnitude: 100, unit: TimeUnit.Milliseconds }; + const interval = { magnitude: 100, unit: ArrayExtensions.TimeUnit.Milliseconds }; const newMediaItems = await mediaInput.batchedMapInterval(batcher, dispatchUpload, interval); -- cgit v1.2.3-70-g09d2 From d66b51213e448d5f4f37781389af488a3ac744c4 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Fri, 20 Sep 2019 05:10:21 -0400 Subject: factored out extensions into npm module --- package.json | 1 + .../util/Import & Export/DirectoryImportBox.tsx | 7 +- src/client/views/Main.tsx | 3 - src/extensions/ArrayExtensions.ts | 639 ++++++++++----------- src/extensions/Extensions.ts | 2 +- src/server/apis/google/GooglePhotosUploadUtils.ts | 8 +- src/server/credentials/google_docs_token.json | 2 +- src/server/index.ts | 10 +- 8 files changed, 326 insertions(+), 346 deletions(-) (limited to 'src/server/apis') diff --git a/package.json b/package.json index f869713ba..b20c31a7a 100644 --- a/package.json +++ b/package.json @@ -114,6 +114,7 @@ "@types/youtube": "0.0.38", "adm-zip": "^0.4.13", "archiver": "^3.0.3", + "array-batcher": "^1.0.2", "async": "^2.6.2", "babel-runtime": "^6.26.0", "bcrypt-nodejs": "0.0.3", diff --git a/src/client/util/Import & Export/DirectoryImportBox.tsx b/src/client/util/Import & Export/DirectoryImportBox.tsx index e3958e3a4..762302bc8 100644 --- a/src/client/util/Import & Export/DirectoryImportBox.tsx +++ b/src/client/util/Import & Export/DirectoryImportBox.tsx @@ -20,6 +20,7 @@ import { listSpec } from "../../../new_fields/Schema"; import { GooglePhotos } from "../../apis/google_docs/GooglePhotosClientUtils"; import { SchemaHeaderField } from "../../../new_fields/SchemaHeaderField"; import "./DirectoryImportBox.scss"; +import { batchedMapAsync } from "array-batcher"; const unsupported = ["text/html", "text/plain"]; interface FileResponse { @@ -103,7 +104,7 @@ export default class DirectoryImportBox extends React.Component runInAction(() => this.phase = `Internal: uploading ${this.quota - this.completed} files to Dash...`); - const uploads = await validated.batchedMapAsync({ batchSize: 15 }, async batch => { + const uploads = await batchedMapAsync(validated, { batchSize: 15 }, async batch => { const formData = new FormData(); const parameters = { method: 'POST', body: formData }; @@ -113,9 +114,9 @@ export default class DirectoryImportBox extends React.Component formData.append(Utils.GenerateGuid(), file); }); - const responses = (await fetch(RouteStore.upload, parameters)).json(); + const responses = await (await fetch(RouteStore.upload, parameters)).json(); runInAction(() => this.completed += batch.length); - return responses; + return responses as FileResponse[]; }); await Promise.all(uploads.map(async upload => { diff --git a/src/client/views/Main.tsx b/src/client/views/Main.tsx index 53912550c..70d2235e6 100644 --- a/src/client/views/Main.tsx +++ b/src/client/views/Main.tsx @@ -7,9 +7,6 @@ import { Cast } from "../../new_fields/Types"; import { Doc, DocListCastAsync } from "../../new_fields/Doc"; import { List } from "../../new_fields/List"; import { DocServer } from "../DocServer"; -const Extensions = require("../../extensions/Extensions"); - -Extensions.AssignExtensions(); let swapDocs = async () => { let oldDoc = await Cast(CurrentUserUtils.UserDocument.linkManagerDoc, Doc); diff --git a/src/extensions/ArrayExtensions.ts b/src/extensions/ArrayExtensions.ts index 872f107a7..ca407862b 100644 --- a/src/extensions/ArrayExtensions.ts +++ b/src/extensions/ArrayExtensions.ts @@ -1,333 +1,318 @@ interface Array { - fixedBatch(batcher: FixedBatcher): T[][]; - predicateBatch(batcher: PredicateBatcherSync): T[][]; - predicateBatchAsync(batcher: PredicateBatcherAsync): Promise; - batch(batcher: BatcherSync): T[][]; - batchAsync(batcher: Batcher): Promise; - - batchedForEach(batcher: BatcherSync, handler: BatchHandlerSync): void; - batchedMap(batcher: BatcherSync, handler: BatchConverterSync): O[]; - - batchedForEachAsync(batcher: Batcher, handler: BatchHandler): Promise; - batchedMapAsync(batcher: Batcher, handler: BatchConverter): Promise; - - batchedForEachInterval(batcher: Batcher, handler: BatchHandler, interval: Interval): Promise; - batchedMapInterval(batcher: Batcher, handler: BatchConverter, interval: Interval): Promise; - lastElement(): T; } -interface BatchContext { - completedBatches: number; - remainingBatches: number; -} - -interface ExecutorResult { - updated: A; - makeNextBatch: boolean; -} - -interface PredicateBatcherCommon { - initial: A; - persistAccumulator?: boolean; -} - -interface Interval { - magnitude: number; - unit: typeof module.exports.TimeUnit; -} - -type BatchConverterSync = (batch: I[], context: BatchContext) => O[]; -type BatchConverterAsync = (batch: I[], context: BatchContext) => Promise; -type BatchConverter = BatchConverterSync | BatchConverterAsync; - -type BatchHandlerSync = (batch: I[], context: BatchContext) => void; -type BatchHandlerAsync = (batch: I[], context: BatchContext) => Promise; -type BatchHandler = BatchHandlerSync | BatchHandlerAsync; - -type BatcherSync = FixedBatcher | PredicateBatcherSync; -type BatcherAsync = PredicateBatcherAsync; -type Batcher = BatcherSync | BatcherAsync; - -type FixedBatcher = { batchSize: number } | { batchCount: number, mode?: typeof module.exports.Mode }; -type PredicateBatcherSync = PredicateBatcherCommon & { executor: (element: I, accumulator: A) => ExecutorResult }; -type PredicateBatcherAsync = PredicateBatcherCommon & { executorAsync: (element: I, accumulator: A) => Promise> }; - - -module.exports.Mode = { - Balanced: 0, - Even: 1 -}; - -module.exports.TimeUnit = { - Milliseconds: 0, - Seconds: 1, - Minutes: 2 -}; - -module.exports.Assign = function () { - - Array.prototype.fixedBatch = function (batcher: FixedBatcher): T[][] { - const batches: T[][] = []; - const length = this.length; - let i = 0; - if ("batchSize" in batcher) { - const { batchSize } = batcher; - while (i < this.length) { - const cap = Math.min(i + batchSize, length); - batches.push(this.slice(i, i = cap)); - } - } else if ("batchCount" in batcher) { - let { batchCount, mode } = batcher; - const resolved = mode || module.exports.Mode.Balanced; - if (batchCount < 1) { - throw new Error("Batch count must be a positive integer!"); - } - if (batchCount === 1) { - return [this]; - } - if (batchCount >= this.length) { - return this.map((element: T) => [element]); - } - - let length = this.length; - let size: number; - - if (length % batchCount === 0) { - size = Math.floor(length / batchCount); - while (i < length) { - batches.push(this.slice(i, i += size)); - } - } else if (resolved === module.exports.Mode.Balanced) { - while (i < length) { - size = Math.ceil((length - i) / batchCount--); - batches.push(this.slice(i, i += size)); - } - } else { - batchCount--; - size = Math.floor(length / batchCount); - if (length % size === 0) { - size--; - } - while (i < size * batchCount) { - batches.push(this.slice(i, i += size)); - } - batches.push(this.slice(size * batchCount)); - } - } - return batches; - }; - - Array.prototype.predicateBatch = function (batcher: PredicateBatcherSync): T[][] { - const batches: T[][] = []; - let batch: T[] = []; - const { executor, initial, persistAccumulator } = batcher; - let accumulator = initial; - for (let element of this) { - const { updated, makeNextBatch } = executor(element, accumulator); - accumulator = updated; - if (!makeNextBatch) { - batch.push(element); - } else { - batches.push(batch); - batch = [element]; - if (!persistAccumulator) { - accumulator = initial; - } - } - } - batches.push(batch); - return batches; - }; - - Array.prototype.predicateBatchAsync = async function (batcher: BatcherAsync): Promise { - const batches: T[][] = []; - let batch: T[] = []; - const { executorAsync, initial, persistAccumulator } = batcher; - let accumulator: A = initial; - for (let element of this) { - const { updated, makeNextBatch } = await executorAsync(element, accumulator); - accumulator = updated; - if (!makeNextBatch) { - batch.push(element); - } else { - batches.push(batch); - batch = [element]; - if (!persistAccumulator) { - accumulator = initial; - } - } - } - batches.push(batch); - return batches; - }; - - Array.prototype.batch = function (batcher: BatcherSync): T[][] { - if ("executor" in batcher) { - return this.predicateBatch(batcher); - } else { - return this.fixedBatch(batcher); - } - }; - - Array.prototype.batchAsync = async function (batcher: Batcher): Promise { - if ("executorAsync" in batcher) { - return this.predicateBatchAsync(batcher); - } else { - return this.batch(batcher); - } - }; - - Array.prototype.batchedForEach = function (batcher: BatcherSync, handler: BatchHandlerSync): void { - if (this.length) { - let completed = 0; - const batches = this.batch(batcher); - const quota = batches.length; - for (let batch of batches) { - const context: BatchContext = { - completedBatches: completed, - remainingBatches: quota - completed, - }; - handler(batch, context); - completed++; - } - } - }; - - Array.prototype.batchedMap = function (batcher: BatcherSync, handler: BatchConverterSync): O[] { - if (!this.length) { - return []; - } - let collector: O[] = []; - let completed = 0; - const batches = this.batch(batcher); - const quota = batches.length; - for (let batch of batches) { - const context: BatchContext = { - completedBatches: completed, - remainingBatches: quota - completed, - }; - collector.push(...handler(batch, context)); - completed++; - } - return collector; - }; - - Array.prototype.batchedForEachAsync = async function (batcher: Batcher, handler: BatchHandler): Promise { - if (this.length) { - let completed = 0; - const batches = await this.batchAsync(batcher); - const quota = batches.length; - for (let batch of batches) { - const context: BatchContext = { - completedBatches: completed, - remainingBatches: quota - completed, - }; - await handler(batch, context); - completed++; - } - } - }; - - Array.prototype.batchedMapAsync = async function (batcher: Batcher, handler: BatchConverter): Promise { - if (!this.length) { - return []; - } - let collector: O[] = []; - let completed = 0; - const batches = await this.batchAsync(batcher); - const quota = batches.length; - for (let batch of batches) { - const context: BatchContext = { - completedBatches: completed, - remainingBatches: quota - completed, - }; - collector.push(...(await handler(batch, context))); - completed++; - } - return collector; - }; - - Array.prototype.batchedForEachInterval = async function (batcher: Batcher, handler: BatchHandler, interval: Interval): Promise { - if (!this.length) { - return; - } - const batches = await this.batchAsync(batcher); - const quota = batches.length; - return new Promise(async resolve => { - const iterator = batches[Symbol.iterator](); - let completed = 0; - while (true) { - const next = iterator.next(); - await new Promise(resolve => { - setTimeout(async () => { - const batch = next.value; - const context: BatchContext = { - completedBatches: completed, - remainingBatches: quota - completed, - }; - await handler(batch, context); - resolve(); - }, convert(interval)); - }); - if (++completed === quota) { - break; - } - } - resolve(); - }); - }; - - Array.prototype.batchedMapInterval = async function (batcher: Batcher, handler: BatchConverter, interval: Interval): Promise { - if (!this.length) { - return []; - } - let collector: O[] = []; - const batches = await this.batchAsync(batcher); - const quota = batches.length; - return new Promise(async resolve => { - const iterator = batches[Symbol.iterator](); - let completed = 0; - while (true) { - const next = iterator.next(); - await new Promise(resolve => { - setTimeout(async () => { - const batch = next.value; - const context: BatchContext = { - completedBatches: completed, - remainingBatches: quota - completed, - }; - collector.push(...(await handler(batch, context))); - resolve(); - }, convert(interval)); - }); - if (++completed === quota) { - resolve(collector); - break; - } - } - }); - }; - - Array.prototype.lastElement = function () { - if (!this.length) { - return undefined; - } - const last: T = this[this.length - 1]; - return last; - }; - +// interface BatchContext { +// completedBatches: number; +// remainingBatches: number; +// } + +// interface ExecutorResult { +// updated: A; +// makeNextBatch: boolean; +// } + +// interface PredicateBatcherCommon { +// initial: A; +// persistAccumulator?: boolean; +// } + +// interface Interval { +// magnitude: number; +// unit: typeof module.exports.TimeUnit; +// } + +// type BatchConverterSync = (batch: I[], context: BatchContext) => O[]; +// type BatchConverterAsync = (batch: I[], context: BatchContext) => Promise; +// type BatchConverter = BatchConverterSync | BatchConverterAsync; + +// type BatchHandlerSync = (batch: I[], context: BatchContext) => void; +// type BatchHandlerAsync = (batch: I[], context: BatchContext) => Promise; +// type BatchHandler = BatchHandlerSync | BatchHandlerAsync; + +// type BatcherSync = FixedBatcher | PredicateBatcherSync; +// type BatcherAsync = PredicateBatcherAsync; +// type Batcher = BatcherSync | BatcherAsync; + +// type FixedBatcher = { batchSize: number } | { batchCount: number, mode?: typeof module.exports.Mode }; +// type PredicateBatcherSync = PredicateBatcherCommon & { executor: (element: I, accumulator: A) => ExecutorResult }; +// type PredicateBatcherAsync = PredicateBatcherCommon & { executorAsync: (element: I, accumulator: A) => Promise> }; + + +// module.exports.Mode = { +// Balanced: 0, +// Even: 1 +// }; + +// module.exports.TimeUnit = { +// Milliseconds: 0, +// Seconds: 1, +// Minutes: 2 +// }; + +// module.exports.Assign = function () { + +// Array.prototype.fixedBatch = function (batcher: FixedBatcher): T[][] { +// const batches: T[][] = []; +// const length = this.length; +// let i = 0; +// if ("batchSize" in batcher) { +// const { batchSize } = batcher; +// while (i < this.length) { +// const cap = Math.min(i + batchSize, length); +// batches.push(this.slice(i, i = cap)); +// } +// } else if ("batchCount" in batcher) { +// let { batchCount, mode } = batcher; +// const resolved = mode || module.exports.Mode.Balanced; +// if (batchCount < 1) { +// throw new Error("Batch count must be a positive integer!"); +// } +// if (batchCount === 1) { +// return [this]; +// } +// if (batchCount >= this.length) { +// return this.map((element: T) => [element]); +// } + +// let length = this.length; +// let size: number; + +// if (length % batchCount === 0) { +// size = Math.floor(length / batchCount); +// while (i < length) { +// batches.push(this.slice(i, i += size)); +// } +// } else if (resolved === module.exports.Mode.Balanced) { +// while (i < length) { +// size = Math.ceil((length - i) / batchCount--); +// batches.push(this.slice(i, i += size)); +// } +// } else { +// batchCount--; +// size = Math.floor(length / batchCount); +// if (length % size === 0) { +// size--; +// } +// while (i < size * batchCount) { +// batches.push(this.slice(i, i += size)); +// } +// batches.push(this.slice(size * batchCount)); +// } +// } +// return batches; +// }; + +// Array.prototype.predicateBatch = function (batcher: PredicateBatcherSync): T[][] { +// const batches: T[][] = []; +// let batch: T[] = []; +// const { executor, initial, persistAccumulator } = batcher; +// let accumulator = initial; +// for (let element of this) { +// const { updated, makeNextBatch } = executor(element, accumulator); +// accumulator = updated; +// if (!makeNextBatch) { +// batch.push(element); +// } else { +// batches.push(batch); +// batch = [element]; +// if (!persistAccumulator) { +// accumulator = initial; +// } +// } +// } +// batches.push(batch); +// return batches; +// }; + +// Array.prototype.predicateBatchAsync = async function (batcher: BatcherAsync): Promise { +// const batches: T[][] = []; +// let batch: T[] = []; +// const { executorAsync, initial, persistAccumulator } = batcher; +// let accumulator: A = initial; +// for (let element of this) { +// const { updated, makeNextBatch } = await executorAsync(element, accumulator); +// accumulator = updated; +// if (!makeNextBatch) { +// batch.push(element); +// } else { +// batches.push(batch); +// batch = [element]; +// if (!persistAccumulator) { +// accumulator = initial; +// } +// } +// } +// batches.push(batch); +// return batches; +// }; + +// Array.prototype.batch = function (batcher: BatcherSync): T[][] { +// if ("executor" in batcher) { +// return this.predicateBatch(batcher); +// } else { +// return this.fixedBatch(batcher); +// } +// }; + +// Array.prototype.batchAsync = async function (batcher: Batcher): Promise { +// if ("executorAsync" in batcher) { +// return this.predicateBatchAsync(batcher); +// } else { +// return this.batch(batcher); +// } +// }; + +// Array.prototype.batchedForEach = function (batcher: BatcherSync, handler: BatchHandlerSync): void { +// if (this.length) { +// let completed = 0; +// const batches = this.batch(batcher); +// const quota = batches.length; +// for (let batch of batches) { +// const context: BatchContext = { +// completedBatches: completed, +// remainingBatches: quota - completed, +// }; +// handler(batch, context); +// completed++; +// } +// } +// }; + +// Array.prototype.batchedMap = function (batcher: BatcherSync, handler: BatchConverterSync): O[] { +// if (!this.length) { +// return []; +// } +// let collector: O[] = []; +// let completed = 0; +// const batches = this.batch(batcher); +// const quota = batches.length; +// for (let batch of batches) { +// const context: BatchContext = { +// completedBatches: completed, +// remainingBatches: quota - completed, +// }; +// collector.push(...handler(batch, context)); +// completed++; +// } +// return collector; +// }; + +// Array.prototype.batchedForEachAsync = async function (batcher: Batcher, handler: BatchHandler): Promise { +// if (this.length) { +// let completed = 0; +// const batches = await this.batchAsync(batcher); +// const quota = batches.length; +// for (let batch of batches) { +// const context: BatchContext = { +// completedBatches: completed, +// remainingBatches: quota - completed, +// }; +// await handler(batch, context); +// completed++; +// } +// } +// }; + +// Array.prototype.batchedMapAsync = async function (batcher: Batcher, handler: BatchConverter): Promise { +// if (!this.length) { +// return []; +// } +// let collector: O[] = []; +// let completed = 0; +// const batches = await this.batchAsync(batcher); +// const quota = batches.length; +// for (let batch of batches) { +// const context: BatchContext = { +// completedBatches: completed, +// remainingBatches: quota - completed, +// }; +// collector.push(...(await handler(batch, context))); +// completed++; +// } +// return collector; +// }; + +// Array.prototype.batchedForEachInterval = async function (batcher: Batcher, handler: BatchHandler, interval: Interval): Promise { +// if (!this.length) { +// return; +// } +// const batches = await this.batchAsync(batcher); +// const quota = batches.length; +// return new Promise(async resolve => { +// const iterator = batches[Symbol.iterator](); +// let completed = 0; +// while (true) { +// const next = iterator.next(); +// await new Promise(resolve => { +// setTimeout(async () => { +// const batch = next.value; +// const context: BatchContext = { +// completedBatches: completed, +// remainingBatches: quota - completed, +// }; +// await handler(batch, context); +// resolve(); +// }, convert(interval)); +// }); +// if (++completed === quota) { +// break; +// } +// } +// resolve(); +// }); +// }; + +// Array.prototype.batchedMapInterval = async function (batcher: Batcher, handler: BatchConverter, interval: Interval): Promise { +// if (!this.length) { +// return []; +// } +// let collector: O[] = []; +// const batches = await this.batchAsync(batcher); +// const quota = batches.length; +// return new Promise(async resolve => { +// const iterator = batches[Symbol.iterator](); +// let completed = 0; +// while (true) { +// const next = iterator.next(); +// await new Promise(resolve => { +// setTimeout(async () => { +// const batch = next.value; +// const context: BatchContext = { +// completedBatches: completed, +// remainingBatches: quota - completed, +// }; +// collector.push(...(await handler(batch, context))); +// resolve(); +// }, convert(interval)); +// }); +// if (++completed === quota) { +// resolve(collector); +// break; +// } +// } +// }); +// }; + +Array.prototype.lastElement = function () { + if (!this.length) { + return undefined; + } + const last: T = this[this.length - 1]; + return last; }; -const convert = (interval: Interval) => { - const { magnitude, unit } = interval; - switch (unit) { - default: - case module.exports.TimeUnit.Milliseconds: - return magnitude; - case module.exports.TimeUnit.Seconds: - return magnitude * 1000; - case module.exports.TimeUnit.Minutes: - return magnitude * 1000 * 60; - } -}; \ No newline at end of file +// }; + +// const convert = (interval: Interval) => { +// const { magnitude, unit } = interval; +// switch (unit) { +// default: +// case module.exports.TimeUnit.Milliseconds: +// return magnitude; +// case module.exports.TimeUnit.Seconds: +// return magnitude * 1000; +// case module.exports.TimeUnit.Minutes: +// return magnitude * 1000 * 60; +// } +// }; \ No newline at end of file diff --git a/src/extensions/Extensions.ts b/src/extensions/Extensions.ts index 1bcebd0e2..1391140b9 100644 --- a/src/extensions/Extensions.ts +++ b/src/extensions/Extensions.ts @@ -2,6 +2,6 @@ const ArrayExtensions = require("./ArrayExtensions"); const StringExtensions = require("./StringExtensions"); module.exports.AssignExtensions = function () { - ArrayExtensions.Assign(); + // ArrayExtensions.Assign(); StringExtensions.Assign(); }; \ No newline at end of file diff --git a/src/server/apis/google/GooglePhotosUploadUtils.ts b/src/server/apis/google/GooglePhotosUploadUtils.ts index fc6772ffd..29575763c 100644 --- a/src/server/apis/google/GooglePhotosUploadUtils.ts +++ b/src/server/apis/google/GooglePhotosUploadUtils.ts @@ -7,7 +7,7 @@ import { Opt } from '../../../new_fields/Doc'; import * as sharp from 'sharp'; import { MediaItemCreationResult } from './SharedTypes'; import { NewMediaItem } from "../../index"; -const { TimeUnit } = require("../../../extensions/ArrayExtensions"); +import { batchedMapInterval, FixedBatcher, TimeUnit, Interval } from "array-batcher"; const uploadDirectory = path.join(__dirname, "../../public/files/"); @@ -81,10 +81,10 @@ export namespace GooglePhotosUploadUtils { }); })).newMediaItemResults; }; - const batcher = { batchSize: 50 }; - const interval = { magnitude: 100, unit: TimeUnit.Milliseconds }; + const batcher: FixedBatcher = { batchSize: 50 }; + const interval: Interval = { magnitude: 100, unit: TimeUnit.Milliseconds }; - const newMediaItemResults = await newMediaItems.batchedMapInterval(batcher, createFromUploadTokens, interval); + const newMediaItemResults = await batchedMapInterval(newMediaItems, batcher, createFromUploadTokens, interval); return { newMediaItemResults }; }; diff --git a/src/server/credentials/google_docs_token.json b/src/server/credentials/google_docs_token.json index 7c49eed43..cdea139a3 100644 --- a/src/server/credentials/google_docs_token.json +++ b/src/server/credentials/google_docs_token.json @@ -1 +1 @@ -{"access_token":"ya29.ImCJB1Y8Z8vgUH4vyYA9xwqvLg281kOQKfA8_AGs_EqF1VKQVWfZsMoYkPJN3QwJmIUxlzTO1N-ehUGIxu0Jq3kKR-zzW7rQIMgeQu32OHogK4kvFxpM7l7RNYRw_9x22I0","refresh_token":"1/HTv_xFHszu2Nf3iiFrUTaeKzC_Vp2-6bpIB06xW_WHI","scope":"https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/documents.readonly https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/photoslibrary https://www.googleapis.com/auth/photoslibrary.appendonly https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/photoslibrary.sharing","token_type":"Bearer","expiry_date":1568935635717} \ No newline at end of file +{"access_token":"ya29.ImCJB_jd-XlGcIHAHgN2Zl3BWQ6sMHdeMMuRxU6sPCbAYIT8hXws-WDmQf65ZY1f-0d3y7HcCcuOxtZJ_0IcBb1-yIBxiOf3VJWPmvjGiJQq_mANGVSSmsBHhqpIaYkeQN0","refresh_token":"1/HTv_xFHszu2Nf3iiFrUTaeKzC_Vp2-6bpIB06xW_WHI","scope":"https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/documents.readonly https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/photoslibrary https://www.googleapis.com/auth/photoslibrary.appendonly https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/photoslibrary.sharing","token_type":"Bearer","expiry_date":1568973665276} \ No newline at end of file diff --git a/src/server/index.ts b/src/server/index.ts index e3e4221cf..e03079d66 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -47,9 +47,7 @@ const mongoose = require('mongoose'); const probe = require("probe-image-size"); import * as qs from 'query-string'; import { Opt } from '../new_fields/Doc'; -const Extensions = require("../extensions/Extensions"); -const ArrayExtensions = require("../extensions/ArrayExtensions"); - +import { batchedMapInterval, TimeUnit } from "array-batcher"; const download = (url: string, dest: fs.PathLike) => request.get(url).pipe(fs.createWriteStream(dest)); let youtubeApiKey: string; @@ -101,8 +99,6 @@ enum Method { POST } -Extensions.AssignExtensions(); - /** * Please invoke this function when adding a new route to Dash's server. * It ensures that any requests leading to or containing user-sensitive information @@ -861,9 +857,9 @@ app.post(RouteStore.googlePhotosMediaUpload, async (req, res) => { return newMediaItems; }; const batcher = { batchSize: 25 }; - const interval = { magnitude: 100, unit: ArrayExtensions.TimeUnit.Milliseconds }; + const interval = { magnitude: 100, unit: TimeUnit.Milliseconds }; - const newMediaItems = await mediaInput.batchedMapInterval(batcher, dispatchUpload, interval); + const newMediaItems = await batchedMapInterval(mediaInput, batcher, dispatchUpload, interval); if (failed) { return _error(res, tokenError); -- cgit v1.2.3-70-g09d2 From 8d9c9d0f51be91ee10b631be9f464af0822bb9be Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Fri, 20 Sep 2019 16:39:14 -0400 Subject: integrated with array batcher npm module --- package.json | 2 +- .../util/Import & Export/DirectoryImportBox.tsx | 31 +++++++------- src/server/apis/google/GooglePhotosUploadUtils.ts | 49 +++++++++++----------- src/server/credentials/google_docs_token.json | 2 +- src/server/index.ts | 36 ++++++++-------- 5 files changed, 61 insertions(+), 59 deletions(-) (limited to 'src/server/apis') diff --git a/package.json b/package.json index b20c31a7a..45babb358 100644 --- a/package.json +++ b/package.json @@ -114,7 +114,7 @@ "@types/youtube": "0.0.38", "adm-zip": "^0.4.13", "archiver": "^3.0.3", - "array-batcher": "^1.0.2", + "array-batcher": "^1.0.6", "async": "^2.6.2", "babel-runtime": "^6.26.0", "bcrypt-nodejs": "0.0.3", diff --git a/src/client/util/Import & Export/DirectoryImportBox.tsx b/src/client/util/Import & Export/DirectoryImportBox.tsx index 762302bc8..26a00dc7c 100644 --- a/src/client/util/Import & Export/DirectoryImportBox.tsx +++ b/src/client/util/Import & Export/DirectoryImportBox.tsx @@ -20,7 +20,7 @@ import { listSpec } from "../../../new_fields/Schema"; import { GooglePhotos } from "../../apis/google_docs/GooglePhotosClientUtils"; import { SchemaHeaderField } from "../../../new_fields/SchemaHeaderField"; import "./DirectoryImportBox.scss"; -import { batchedMapAsync } from "array-batcher"; +import BatchedArray from "array-batcher"; const unsupported = ["text/html", "text/plain"]; interface FileResponse { @@ -104,19 +104,22 @@ export default class DirectoryImportBox extends React.Component runInAction(() => this.phase = `Internal: uploading ${this.quota - this.completed} files to Dash...`); - const uploads = await batchedMapAsync(validated, { batchSize: 15 }, async batch => { - const formData = new FormData(); - const parameters = { method: 'POST', body: formData }; - - batch.forEach(file => { - sizes.push(file.size); - modifiedDates.push(file.lastModified); - formData.append(Utils.GenerateGuid(), file); - }); - - const responses = await (await fetch(RouteStore.upload, parameters)).json(); - runInAction(() => this.completed += batch.length); - return responses as FileResponse[]; + const uploads = await BatchedArray.from(validated).batchedMapAsync({ + batcher: { batchSize: 15 }, + converter: async batch => { + const formData = new FormData(); + const parameters = { method: 'POST', body: formData }; + + batch.forEach(file => { + sizes.push(file.size); + modifiedDates.push(file.lastModified); + formData.append(Utils.GenerateGuid(), file); + }); + + const responses = await (await fetch(RouteStore.upload, parameters)).json(); + runInAction(() => this.completed += batch.length); + return responses as FileResponse[]; + } }); await Promise.all(uploads.map(async upload => { diff --git a/src/server/apis/google/GooglePhotosUploadUtils.ts b/src/server/apis/google/GooglePhotosUploadUtils.ts index 29575763c..40564ff3b 100644 --- a/src/server/apis/google/GooglePhotosUploadUtils.ts +++ b/src/server/apis/google/GooglePhotosUploadUtils.ts @@ -7,7 +7,7 @@ import { Opt } from '../../../new_fields/Doc'; import * as sharp from 'sharp'; import { MediaItemCreationResult } from './SharedTypes'; import { NewMediaItem } from "../../index"; -import { batchedMapInterval, FixedBatcher, TimeUnit, Interval } from "array-batcher"; +import BatchedArray, { FixedBatcher, TimeUnit, Interval } from "array-batcher"; const uploadDirectory = path.join(__dirname, "../../public/files/"); @@ -62,30 +62,29 @@ export namespace GooglePhotosUploadUtils { }; export const CreateMediaItems = async (newMediaItems: NewMediaItem[], album?: { id: string }): Promise => { - const createFromUploadTokens = async (batch: NewMediaItem[]) => { - const parameters = { - method: 'POST', - headers: headers('json'), - uri: prepend('mediaItems:batchCreate'), - body: { newMediaItems: batch } as any, - json: true - }; - album && (parameters.body.albumId = album.id); - return (await new Promise((resolve, reject) => { - request(parameters, (error, _response, body) => { - if (error) { - reject(error); - } else { - resolve(body); - } - }); - })).newMediaItemResults; - }; - const batcher: FixedBatcher = { batchSize: 50 }; - const interval: Interval = { magnitude: 100, unit: TimeUnit.Milliseconds }; - - const newMediaItemResults = await batchedMapInterval(newMediaItems, batcher, createFromUploadTokens, interval); - + const newMediaItemResults = await BatchedArray.from(newMediaItems).batchedMapInterval({ + batcher: { batchSize: 50 }, + interval: { magnitude: 100, unit: TimeUnit.Milliseconds }, + converter: async (batch: NewMediaItem[]) => { + const parameters = { + method: 'POST', + headers: headers('json'), + uri: prepend('mediaItems:batchCreate'), + body: { newMediaItems: batch } as any, + json: true + }; + album && (parameters.body.albumId = album.id); + return (await new Promise((resolve, reject) => { + request(parameters, (error, _response, body) => { + if (error) { + reject(error); + } else { + resolve(body); + } + }); + })).newMediaItemResults; + } + }); return { newMediaItemResults }; }; diff --git a/src/server/credentials/google_docs_token.json b/src/server/credentials/google_docs_token.json index cdea139a3..9b0a649ac 100644 --- a/src/server/credentials/google_docs_token.json +++ b/src/server/credentials/google_docs_token.json @@ -1 +1 @@ -{"access_token":"ya29.ImCJB_jd-XlGcIHAHgN2Zl3BWQ6sMHdeMMuRxU6sPCbAYIT8hXws-WDmQf65ZY1f-0d3y7HcCcuOxtZJ_0IcBb1-yIBxiOf3VJWPmvjGiJQq_mANGVSSmsBHhqpIaYkeQN0","refresh_token":"1/HTv_xFHszu2Nf3iiFrUTaeKzC_Vp2-6bpIB06xW_WHI","scope":"https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/documents.readonly https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/photoslibrary https://www.googleapis.com/auth/photoslibrary.appendonly https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/photoslibrary.sharing","token_type":"Bearer","expiry_date":1568973665276} \ No newline at end of file +{"access_token":"ya29.ImCJB3_-mtKbCG8L7nQt3CDfn4HR2_xqUw8bUQkvaZZgxJZ8-WUlHe6UaSOmGtrvI4QabpYRGutYGSTUdc9bvIYooerZgAz80uk7-KEbnAxpwydRE-TK_1_VyWYZ2YW6Ht8","refresh_token":"1/HTv_xFHszu2Nf3iiFrUTaeKzC_Vp2-6bpIB06xW_WHI","scope":"https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/documents.readonly https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/photoslibrary https://www.googleapis.com/auth/photoslibrary.appendonly https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/photoslibrary.sharing","token_type":"Bearer","expiry_date":1569013995678} \ No newline at end of file diff --git a/src/server/index.ts b/src/server/index.ts index e03079d66..12ceb9886 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -47,7 +47,7 @@ const mongoose = require('mongoose'); const probe = require("probe-image-size"); import * as qs from 'query-string'; import { Opt } from '../new_fields/Doc'; -import { batchedMapInterval, TimeUnit } from "array-batcher"; +import BatchedArray, { TimeUnit } from "array-batcher"; const download = (url: string, dest: fs.PathLike) => request.get(url).pipe(fs.createWriteStream(dest)); let youtubeApiKey: string; @@ -841,25 +841,25 @@ app.post(RouteStore.googlePhotosMediaUpload, async (req, res) => { let failed = 0; - const dispatchUpload = async (batch: GooglePhotosUploadUtils.MediaInput[]) => { - const newMediaItems: NewMediaItem[] = []; - for (let element of batch) { - const uploadToken = await GooglePhotosUploadUtils.DispatchGooglePhotosUpload(element.url); - if (!uploadToken) { - failed++; - } else { - newMediaItems.push({ - description: element.description, - simpleMediaItem: { uploadToken } - }); + const newMediaItems = await BatchedArray.from(mediaInput).batchedMapInterval({ + batcher: { batchSize: 25 }, + interval: { magnitude: 100, unit: TimeUnit.Milliseconds }, + converter: async (batch: GooglePhotosUploadUtils.MediaInput[]) => { + const newMediaItems: NewMediaItem[] = []; + for (let element of batch) { + const uploadToken = await GooglePhotosUploadUtils.DispatchGooglePhotosUpload(element.url); + if (!uploadToken) { + failed++; + } else { + newMediaItems.push({ + description: element.description, + simpleMediaItem: { uploadToken } + }); + } } + return newMediaItems; } - return newMediaItems; - }; - const batcher = { batchSize: 25 }; - const interval = { magnitude: 100, unit: TimeUnit.Milliseconds }; - - const newMediaItems = await batchedMapInterval(mediaInput, batcher, dispatchUpload, interval); + }); if (failed) { return _error(res, tokenError); -- cgit v1.2.3-70-g09d2 From f529d7d22162689c08830418da67a89778111e16 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Sat, 21 Sep 2019 14:27:10 -0400 Subject: final batcher fixes --- package.json | 2 +- .../util/Import & Export/DirectoryImportBox.tsx | 29 ++++++++++------------ src/server/apis/google/GooglePhotosUploadUtils.ts | 11 ++++---- src/server/credentials/google_docs_token.json | 2 +- src/server/index.ts | 11 ++++---- 5 files changed, 25 insertions(+), 30 deletions(-) (limited to 'src/server/apis') diff --git a/package.json b/package.json index 45babb358..f049f8826 100644 --- a/package.json +++ b/package.json @@ -114,7 +114,7 @@ "@types/youtube": "0.0.38", "adm-zip": "^0.4.13", "archiver": "^3.0.3", - "array-batcher": "^1.0.6", + "array-batcher": "^1.0.7", "async": "^2.6.2", "babel-runtime": "^6.26.0", "bcrypt-nodejs": "0.0.3", diff --git a/src/client/util/Import & Export/DirectoryImportBox.tsx b/src/client/util/Import & Export/DirectoryImportBox.tsx index 26a00dc7c..8948b73f7 100644 --- a/src/client/util/Import & Export/DirectoryImportBox.tsx +++ b/src/client/util/Import & Export/DirectoryImportBox.tsx @@ -104,22 +104,19 @@ export default class DirectoryImportBox extends React.Component runInAction(() => this.phase = `Internal: uploading ${this.quota - this.completed} files to Dash...`); - const uploads = await BatchedArray.from(validated).batchedMapAsync({ - batcher: { batchSize: 15 }, - converter: async batch => { - const formData = new FormData(); - const parameters = { method: 'POST', body: formData }; - - batch.forEach(file => { - sizes.push(file.size); - modifiedDates.push(file.lastModified); - formData.append(Utils.GenerateGuid(), file); - }); - - const responses = await (await fetch(RouteStore.upload, parameters)).json(); - runInAction(() => this.completed += batch.length); - return responses as FileResponse[]; - } + const uploads = await BatchedArray.from(validated, { batchSize: 15 }).batchedMapAsync(async batch => { + const formData = new FormData(); + const parameters = { method: 'POST', body: formData }; + + batch.forEach(file => { + sizes.push(file.size); + modifiedDates.push(file.lastModified); + formData.append(Utils.GenerateGuid(), file); + }); + + const responses = await (await fetch(RouteStore.upload, parameters)).json(); + runInAction(() => this.completed += batch.length); + return responses as FileResponse[]; }); await Promise.all(uploads.map(async upload => { diff --git a/src/server/apis/google/GooglePhotosUploadUtils.ts b/src/server/apis/google/GooglePhotosUploadUtils.ts index 40564ff3b..4dc252577 100644 --- a/src/server/apis/google/GooglePhotosUploadUtils.ts +++ b/src/server/apis/google/GooglePhotosUploadUtils.ts @@ -62,10 +62,8 @@ export namespace GooglePhotosUploadUtils { }; export const CreateMediaItems = async (newMediaItems: NewMediaItem[], album?: { id: string }): Promise => { - const newMediaItemResults = await BatchedArray.from(newMediaItems).batchedMapInterval({ - batcher: { batchSize: 50 }, - interval: { magnitude: 100, unit: TimeUnit.Milliseconds }, - converter: async (batch: NewMediaItem[]) => { + const newMediaItemResults = await BatchedArray.from(newMediaItems, { batchSize: 50 }).batchedMapInterval( + async (batch: NewMediaItem[]) => { const parameters = { method: 'POST', headers: headers('json'), @@ -83,8 +81,9 @@ export namespace GooglePhotosUploadUtils { } }); })).newMediaItemResults; - } - }); + }, + { magnitude: 100, unit: TimeUnit.Milliseconds } + ); return { newMediaItemResults }; }; diff --git a/src/server/credentials/google_docs_token.json b/src/server/credentials/google_docs_token.json index 9b0a649ac..ee44c3f30 100644 --- a/src/server/credentials/google_docs_token.json +++ b/src/server/credentials/google_docs_token.json @@ -1 +1 @@ -{"access_token":"ya29.ImCJB3_-mtKbCG8L7nQt3CDfn4HR2_xqUw8bUQkvaZZgxJZ8-WUlHe6UaSOmGtrvI4QabpYRGutYGSTUdc9bvIYooerZgAz80uk7-KEbnAxpwydRE-TK_1_VyWYZ2YW6Ht8","refresh_token":"1/HTv_xFHszu2Nf3iiFrUTaeKzC_Vp2-6bpIB06xW_WHI","scope":"https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/documents.readonly https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/photoslibrary https://www.googleapis.com/auth/photoslibrary.appendonly https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/photoslibrary.sharing","token_type":"Bearer","expiry_date":1569013995678} \ No newline at end of file +{"access_token":"ya29.GlyKBznz91v_qb8RYt4PT40Hp106N08Yk64UjMAKllBsIqJQEzBkxLbB5q5paydywHzguQYSNup5fT7ojJTDU4CMZdPbPKGcjQz17w_CospcG-8Buz94KZptvlQ_pQ","refresh_token":"1/HTv_xFHszu2Nf3iiFrUTaeKzC_Vp2-6bpIB06xW_WHI","scope":"https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/documents.readonly https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/photoslibrary https://www.googleapis.com/auth/photoslibrary.appendonly https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/photoslibrary.sharing","token_type":"Bearer","expiry_date":1569093749804} \ No newline at end of file diff --git a/src/server/index.ts b/src/server/index.ts index 12ceb9886..4c4cb84d6 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -841,10 +841,8 @@ app.post(RouteStore.googlePhotosMediaUpload, async (req, res) => { let failed = 0; - const newMediaItems = await BatchedArray.from(mediaInput).batchedMapInterval({ - batcher: { batchSize: 25 }, - interval: { magnitude: 100, unit: TimeUnit.Milliseconds }, - converter: async (batch: GooglePhotosUploadUtils.MediaInput[]) => { + const newMediaItems = await BatchedArray.from(mediaInput, { batchSize: 25 }).batchedMapInterval( + async (batch: GooglePhotosUploadUtils.MediaInput[]) => { const newMediaItems: NewMediaItem[] = []; for (let element of batch) { const uploadToken = await GooglePhotosUploadUtils.DispatchGooglePhotosUpload(element.url); @@ -858,8 +856,9 @@ app.post(RouteStore.googlePhotosMediaUpload, async (req, res) => { } } return newMediaItems; - } - }); + }, + { magnitude: 100, unit: TimeUnit.Milliseconds } + ); if (failed) { return _error(res, tokenError); -- cgit v1.2.3-70-g09d2 From 9b27f1ace4655f71a67ad68e1f6f6bba82f41e46 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Tue, 24 Sep 2019 19:09:33 -0400 Subject: now store cache in mongodb collection, fixed extension as --- src/Utils.ts | 6 +- .../apis/google_docs/GooglePhotosClientUtils.ts | 8 +- src/client/views/Main.tsx | 3 + src/extensions/ArrayExtensions.ts | 325 +---------------- src/extensions/Extensions.ts | 7 - src/extensions/General/Extensions.ts | 9 + src/extensions/General/ExtensionsTypings.ts | 8 + src/extensions/StringExtensions.ts | 11 +- src/server/DashUploadUtils.ts | 143 ++++++++ .../apis/google/CustomizedWrapper/filters.js | 46 --- src/server/apis/google/GoogleApiServerUtils.ts | 32 +- src/server/apis/google/GooglePhotosUploadUtils.ts | 147 +------- src/server/apis/google/existing_uploads.json | 0 src/server/credentials/google_docs_token.json | 8 +- src/server/database.ts | 402 ++++++++++++--------- src/server/index.ts | 49 +-- 16 files changed, 465 insertions(+), 739 deletions(-) delete mode 100644 src/extensions/Extensions.ts create mode 100644 src/extensions/General/Extensions.ts create mode 100644 src/extensions/General/ExtensionsTypings.ts create mode 100644 src/server/DashUploadUtils.ts delete mode 100644 src/server/apis/google/CustomizedWrapper/filters.js delete mode 100644 src/server/apis/google/existing_uploads.json (limited to 'src/server/apis') diff --git a/src/Utils.ts b/src/Utils.ts index a842f5a20..aa2998971 100644 --- a/src/Utils.ts +++ b/src/Utils.ts @@ -4,6 +4,7 @@ import { Socket } from 'socket.io'; import { Message } from './server/Message'; import { RouteStore } from './server/RouteStore'; import requestPromise = require('request-promise'); +import { CurrentUserUtils } from './server/authentication/models/current_user_utils'; export class Utils { @@ -293,12 +294,13 @@ export namespace JSONUtils { } -export function PostToServer(relativeRoute: string, body: any) { +export function PostToServer(relativeRoute: string, body?: any) { + body = { userId: CurrentUserUtils.id, ...body }; let options = { method: "POST", uri: Utils.prepend(relativeRoute), json: true, - body: body + body }; return requestPromise.post(options); } \ No newline at end of file diff --git a/src/client/apis/google_docs/GooglePhotosClientUtils.ts b/src/client/apis/google_docs/GooglePhotosClientUtils.ts index 671d05421..0e09ad85b 100644 --- a/src/client/apis/google_docs/GooglePhotosClientUtils.ts +++ b/src/client/apis/google_docs/GooglePhotosClientUtils.ts @@ -12,17 +12,11 @@ import { FormattedTextBox } from "../../views/nodes/FormattedTextBox"; import { Docs, DocumentOptions } from "../../documents/Documents"; import { NewMediaItemResult, MediaItem } from "../../../server/apis/google/SharedTypes"; import { AssertionError } from "assert"; -import { List } from "../../../new_fields/List"; -import { listSpec } from "../../../new_fields/Schema"; import { DocumentView } from "../../views/nodes/DocumentView"; export namespace GooglePhotos { - const endpoint = async () => { - const getToken = Utils.prepend(RouteStore.googlePhotosAccessToken); - const token = await (await fetch(getToken)).text(); - return new Photos(token); - }; + const endpoint = async () => new Photos(await PostToServer(RouteStore.googlePhotosAccessToken)); export enum MediaType { ALL_MEDIA = 'ALL_MEDIA', diff --git a/src/client/views/Main.tsx b/src/client/views/Main.tsx index 70d2235e6..3bd898ac0 100644 --- a/src/client/views/Main.tsx +++ b/src/client/views/Main.tsx @@ -7,6 +7,9 @@ import { Cast } from "../../new_fields/Types"; import { Doc, DocListCastAsync } from "../../new_fields/Doc"; import { List } from "../../new_fields/List"; import { DocServer } from "../DocServer"; +import { AssignAllExtensions } from "../../extensions/General/Extensions"; + +AssignAllExtensions(); let swapDocs = async () => { let oldDoc = await Cast(CurrentUserUtils.UserDocument.linkManagerDoc, Doc); diff --git a/src/extensions/ArrayExtensions.ts b/src/extensions/ArrayExtensions.ts index ca407862b..422a10dbc 100644 --- a/src/extensions/ArrayExtensions.ts +++ b/src/extensions/ArrayExtensions.ts @@ -1,318 +1,13 @@ -interface Array { - lastElement(): T; -} - -// interface BatchContext { -// completedBatches: number; -// remainingBatches: number; -// } - -// interface ExecutorResult { -// updated: A; -// makeNextBatch: boolean; -// } - -// interface PredicateBatcherCommon { -// initial: A; -// persistAccumulator?: boolean; -// } - -// interface Interval { -// magnitude: number; -// unit: typeof module.exports.TimeUnit; -// } - -// type BatchConverterSync = (batch: I[], context: BatchContext) => O[]; -// type BatchConverterAsync = (batch: I[], context: BatchContext) => Promise; -// type BatchConverter = BatchConverterSync | BatchConverterAsync; - -// type BatchHandlerSync = (batch: I[], context: BatchContext) => void; -// type BatchHandlerAsync = (batch: I[], context: BatchContext) => Promise; -// type BatchHandler = BatchHandlerSync | BatchHandlerAsync; - -// type BatcherSync = FixedBatcher | PredicateBatcherSync; -// type BatcherAsync = PredicateBatcherAsync; -// type Batcher = BatcherSync | BatcherAsync; - -// type FixedBatcher = { batchSize: number } | { batchCount: number, mode?: typeof module.exports.Mode }; -// type PredicateBatcherSync = PredicateBatcherCommon & { executor: (element: I, accumulator: A) => ExecutorResult }; -// type PredicateBatcherAsync = PredicateBatcherCommon & { executorAsync: (element: I, accumulator: A) => Promise> }; - - -// module.exports.Mode = { -// Balanced: 0, -// Even: 1 -// }; - -// module.exports.TimeUnit = { -// Milliseconds: 0, -// Seconds: 1, -// Minutes: 2 -// }; - -// module.exports.Assign = function () { - -// Array.prototype.fixedBatch = function (batcher: FixedBatcher): T[][] { -// const batches: T[][] = []; -// const length = this.length; -// let i = 0; -// if ("batchSize" in batcher) { -// const { batchSize } = batcher; -// while (i < this.length) { -// const cap = Math.min(i + batchSize, length); -// batches.push(this.slice(i, i = cap)); -// } -// } else if ("batchCount" in batcher) { -// let { batchCount, mode } = batcher; -// const resolved = mode || module.exports.Mode.Balanced; -// if (batchCount < 1) { -// throw new Error("Batch count must be a positive integer!"); -// } -// if (batchCount === 1) { -// return [this]; -// } -// if (batchCount >= this.length) { -// return this.map((element: T) => [element]); -// } +function Assign() { -// let length = this.length; -// let size: number; + Array.prototype.lastElement = function () { + if (!this.length) { + return undefined; + } + const last: T = this[this.length - 1]; + return last; + }; -// if (length % batchCount === 0) { -// size = Math.floor(length / batchCount); -// while (i < length) { -// batches.push(this.slice(i, i += size)); -// } -// } else if (resolved === module.exports.Mode.Balanced) { -// while (i < length) { -// size = Math.ceil((length - i) / batchCount--); -// batches.push(this.slice(i, i += size)); -// } -// } else { -// batchCount--; -// size = Math.floor(length / batchCount); -// if (length % size === 0) { -// size--; -// } -// while (i < size * batchCount) { -// batches.push(this.slice(i, i += size)); -// } -// batches.push(this.slice(size * batchCount)); -// } -// } -// return batches; -// }; - -// Array.prototype.predicateBatch = function (batcher: PredicateBatcherSync): T[][] { -// const batches: T[][] = []; -// let batch: T[] = []; -// const { executor, initial, persistAccumulator } = batcher; -// let accumulator = initial; -// for (let element of this) { -// const { updated, makeNextBatch } = executor(element, accumulator); -// accumulator = updated; -// if (!makeNextBatch) { -// batch.push(element); -// } else { -// batches.push(batch); -// batch = [element]; -// if (!persistAccumulator) { -// accumulator = initial; -// } -// } -// } -// batches.push(batch); -// return batches; -// }; - -// Array.prototype.predicateBatchAsync = async function (batcher: BatcherAsync): Promise { -// const batches: T[][] = []; -// let batch: T[] = []; -// const { executorAsync, initial, persistAccumulator } = batcher; -// let accumulator: A = initial; -// for (let element of this) { -// const { updated, makeNextBatch } = await executorAsync(element, accumulator); -// accumulator = updated; -// if (!makeNextBatch) { -// batch.push(element); -// } else { -// batches.push(batch); -// batch = [element]; -// if (!persistAccumulator) { -// accumulator = initial; -// } -// } -// } -// batches.push(batch); -// return batches; -// }; - -// Array.prototype.batch = function (batcher: BatcherSync): T[][] { -// if ("executor" in batcher) { -// return this.predicateBatch(batcher); -// } else { -// return this.fixedBatch(batcher); -// } -// }; - -// Array.prototype.batchAsync = async function (batcher: Batcher): Promise { -// if ("executorAsync" in batcher) { -// return this.predicateBatchAsync(batcher); -// } else { -// return this.batch(batcher); -// } -// }; - -// Array.prototype.batchedForEach = function (batcher: BatcherSync, handler: BatchHandlerSync): void { -// if (this.length) { -// let completed = 0; -// const batches = this.batch(batcher); -// const quota = batches.length; -// for (let batch of batches) { -// const context: BatchContext = { -// completedBatches: completed, -// remainingBatches: quota - completed, -// }; -// handler(batch, context); -// completed++; -// } -// } -// }; - -// Array.prototype.batchedMap = function (batcher: BatcherSync, handler: BatchConverterSync): O[] { -// if (!this.length) { -// return []; -// } -// let collector: O[] = []; -// let completed = 0; -// const batches = this.batch(batcher); -// const quota = batches.length; -// for (let batch of batches) { -// const context: BatchContext = { -// completedBatches: completed, -// remainingBatches: quota - completed, -// }; -// collector.push(...handler(batch, context)); -// completed++; -// } -// return collector; -// }; - -// Array.prototype.batchedForEachAsync = async function (batcher: Batcher, handler: BatchHandler): Promise { -// if (this.length) { -// let completed = 0; -// const batches = await this.batchAsync(batcher); -// const quota = batches.length; -// for (let batch of batches) { -// const context: BatchContext = { -// completedBatches: completed, -// remainingBatches: quota - completed, -// }; -// await handler(batch, context); -// completed++; -// } -// } -// }; - -// Array.prototype.batchedMapAsync = async function (batcher: Batcher, handler: BatchConverter): Promise { -// if (!this.length) { -// return []; -// } -// let collector: O[] = []; -// let completed = 0; -// const batches = await this.batchAsync(batcher); -// const quota = batches.length; -// for (let batch of batches) { -// const context: BatchContext = { -// completedBatches: completed, -// remainingBatches: quota - completed, -// }; -// collector.push(...(await handler(batch, context))); -// completed++; -// } -// return collector; -// }; - -// Array.prototype.batchedForEachInterval = async function (batcher: Batcher, handler: BatchHandler, interval: Interval): Promise { -// if (!this.length) { -// return; -// } -// const batches = await this.batchAsync(batcher); -// const quota = batches.length; -// return new Promise(async resolve => { -// const iterator = batches[Symbol.iterator](); -// let completed = 0; -// while (true) { -// const next = iterator.next(); -// await new Promise(resolve => { -// setTimeout(async () => { -// const batch = next.value; -// const context: BatchContext = { -// completedBatches: completed, -// remainingBatches: quota - completed, -// }; -// await handler(batch, context); -// resolve(); -// }, convert(interval)); -// }); -// if (++completed === quota) { -// break; -// } -// } -// resolve(); -// }); -// }; - -// Array.prototype.batchedMapInterval = async function (batcher: Batcher, handler: BatchConverter, interval: Interval): Promise { -// if (!this.length) { -// return []; -// } -// let collector: O[] = []; -// const batches = await this.batchAsync(batcher); -// const quota = batches.length; -// return new Promise(async resolve => { -// const iterator = batches[Symbol.iterator](); -// let completed = 0; -// while (true) { -// const next = iterator.next(); -// await new Promise(resolve => { -// setTimeout(async () => { -// const batch = next.value; -// const context: BatchContext = { -// completedBatches: completed, -// remainingBatches: quota - completed, -// }; -// collector.push(...(await handler(batch, context))); -// resolve(); -// }, convert(interval)); -// }); -// if (++completed === quota) { -// resolve(collector); -// break; -// } -// } -// }); -// }; - -Array.prototype.lastElement = function () { - if (!this.length) { - return undefined; - } - const last: T = this[this.length - 1]; - return last; -}; - -// }; +} -// const convert = (interval: Interval) => { -// const { magnitude, unit } = interval; -// switch (unit) { -// default: -// case module.exports.TimeUnit.Milliseconds: -// return magnitude; -// case module.exports.TimeUnit.Seconds: -// return magnitude * 1000; -// case module.exports.TimeUnit.Minutes: -// return magnitude * 1000 * 60; -// } -// }; \ No newline at end of file +export { Assign }; \ No newline at end of file diff --git a/src/extensions/Extensions.ts b/src/extensions/Extensions.ts deleted file mode 100644 index 1391140b9..000000000 --- a/src/extensions/Extensions.ts +++ /dev/null @@ -1,7 +0,0 @@ -const ArrayExtensions = require("./ArrayExtensions"); -const StringExtensions = require("./StringExtensions"); - -module.exports.AssignExtensions = function () { - // ArrayExtensions.Assign(); - StringExtensions.Assign(); -}; \ No newline at end of file diff --git a/src/extensions/General/Extensions.ts b/src/extensions/General/Extensions.ts new file mode 100644 index 000000000..4b6d05d5f --- /dev/null +++ b/src/extensions/General/Extensions.ts @@ -0,0 +1,9 @@ +import { Assign as ArrayAssign } from "../ArrayExtensions"; +import { Assign as StringAssign } from "../StringExtensions"; + +function AssignAllExtensions() { + ArrayAssign(); + StringAssign(); +} + +export { AssignAllExtensions }; \ No newline at end of file diff --git a/src/extensions/General/ExtensionsTypings.ts b/src/extensions/General/ExtensionsTypings.ts new file mode 100644 index 000000000..370157ed0 --- /dev/null +++ b/src/extensions/General/ExtensionsTypings.ts @@ -0,0 +1,8 @@ +interface Array { + lastElement(): T; +} + +interface String { + removeTrailingNewlines(): string; + hasNewline(): boolean; +} \ No newline at end of file diff --git a/src/extensions/StringExtensions.ts b/src/extensions/StringExtensions.ts index 4cdbdebf7..2c76e56c8 100644 --- a/src/extensions/StringExtensions.ts +++ b/src/extensions/StringExtensions.ts @@ -1,9 +1,4 @@ -interface String { - removeTrailingNewlines(): string; - hasNewline(): boolean; -} - -module.exports.Assign = function () { +function Assign() { String.prototype.removeTrailingNewlines = function () { let sliced = this; @@ -17,4 +12,6 @@ module.exports.Assign = function () { return this.endsWith("\n"); }; -}; \ No newline at end of file +} + +export { Assign }; \ No newline at end of file diff --git a/src/server/DashUploadUtils.ts b/src/server/DashUploadUtils.ts new file mode 100644 index 000000000..66874e96c --- /dev/null +++ b/src/server/DashUploadUtils.ts @@ -0,0 +1,143 @@ +import * as fs from 'fs'; +import { Utils } from '../Utils'; +import * as path from 'path'; +import { Opt } from '../new_fields/Doc'; +import * as sharp from 'sharp'; +import request = require('request-promise'); + +const uploadDirectory = path.join(__dirname, './public/files/'); + +export namespace DashUploadUtils { + + export interface Size { + width: number; + suffix: string; + } + + export const Sizes: { [size: string]: Size } = { + SMALL: { width: 100, suffix: "_s" }, + MEDIUM: { width: 400, suffix: "_m" }, + LARGE: { width: 900, suffix: "_l" }, + }; + + const gifs = [".gif"]; + const pngs = [".png"]; + const jpgs = [".jpg", ".jpeg"]; + const imageFormats = [...pngs, ...jpgs, ...gifs]; + const videoFormats = [".mov", ".mp4"]; + + const size = "content-length"; + const type = "content-type"; + + export interface UploadInformation { + mediaPaths: string[]; + fileNames: { [key: string]: string }; + contentSize?: number; + contentType?: string; + } + + const generate = (prefix: string, url: string) => `${prefix}upload_${Utils.GenerateGuid()}${path.extname(url).toLowerCase()}`; + const sanitize = (filename: string) => filename.replace(/\s+/g, "_"); + + export interface InspectionResults { + isLocal: boolean; + stream: any; + normalizedUrl: string; + contentSize?: number; + contentType?: string; + } + + export const InspectImage = async (url: string): Promise => { + const { isLocal, stream, normalized: normalizedUrl } = classify(url); + const results = { + isLocal, + stream, + normalizedUrl + }; + if (isLocal) { + return results; + } + const metadata = (await new Promise((resolve, reject) => { + request.head(url, async (error, res) => { + if (error) { + return reject(error); + } + resolve(res); + }); + })).headers; + return { + contentSize: parseInt(metadata[size]), + contentType: metadata[type], + ...results + }; + }; + + export const UploadImage = async (metadata: InspectionResults, filename?: string, prefix = ""): Promise> => { + const { isLocal, stream, normalizedUrl, contentSize, contentType } = metadata; + const resolved = filename ? sanitize(filename) : generate(prefix, normalizedUrl); + let extension = path.extname(normalizedUrl) || path.extname(resolved); + extension && (extension = extension.toLowerCase()); + let information: UploadInformation = { + mediaPaths: [], + fileNames: { clean: resolved }, + contentSize, + contentType, + }; + return new Promise(async (resolve, reject) => { + const resizers = [ + { resizer: sharp().rotate(), suffix: "_o" }, + ...Object.values(Sizes).map(size => ({ + resizer: sharp().resize(size.width, undefined, { withoutEnlargement: true }).rotate(), + suffix: size.suffix + })) + ]; + let nonVisual = false; + if (pngs.includes(extension)) { + resizers.forEach(element => element.resizer = element.resizer.png()); + } else if (jpgs.includes(extension)) { + resizers.forEach(element => element.resizer = element.resizer.jpeg()); + } else if (![...imageFormats, ...videoFormats].includes(extension.toLowerCase())) { + nonVisual = true; + } + if (imageFormats.includes(extension)) { + for (let resizer of resizers) { + const suffix = resizer.suffix; + let mediaPath: string; + await new Promise(resolve => { + const filename = resolved.substring(0, resolved.length - extension.length) + suffix + extension; + information.mediaPaths.push(mediaPath = uploadDirectory + filename); + information.fileNames[suffix] = filename; + stream(normalizedUrl).pipe(resizer.resizer).pipe(fs.createWriteStream(mediaPath)) + .on('close', resolve) + .on('error', reject); + }); + } + } + if (!isLocal || nonVisual) { + await new Promise(resolve => { + stream(normalizedUrl).pipe(fs.createWriteStream(uploadDirectory + resolved)).on('close', resolve); + }); + } + resolve(information); + }); + }; + + const classify = (url: string) => { + const isLocal = /Dash-Web(\\|\/)src(\\|\/)server(\\|\/)public(\\|\/)files/g.test(url); + return { + isLocal, + stream: isLocal ? fs.createReadStream : request, + normalized: isLocal ? path.normalize(url) : url + }; + }; + + export const createIfNotExists = async (path: string) => { + if (await new Promise(resolve => fs.exists(path, resolve))) { + return true; + } + return new Promise(resolve => fs.mkdir(path, error => resolve(error === null))); + }; + + export const Destroy = (mediaPath: string) => new Promise(resolve => fs.unlink(mediaPath, error => resolve(error === null))); + +} \ No newline at end of file diff --git a/src/server/apis/google/CustomizedWrapper/filters.js b/src/server/apis/google/CustomizedWrapper/filters.js deleted file mode 100644 index 576a90b75..000000000 --- a/src/server/apis/google/CustomizedWrapper/filters.js +++ /dev/null @@ -1,46 +0,0 @@ -'use strict'; - -const DateFilter = require('../common/date_filter'); -const MediaTypeFilter = require('./media_type_filter'); -const ContentFilter = require('./content_filter'); - -class Filters { - constructor(includeArchivedMedia = false) { - this.includeArchivedMedia = includeArchivedMedia; - } - - setDateFilter(dateFilter) { - this.dateFilter = dateFilter; - return this; - } - - setContentFilter(contentFilter) { - this.contentFilter = contentFilter; - return this; - } - - setMediaTypeFilter(mediaTypeFilter) { - this.mediaTypeFilter = mediaTypeFilter; - return this; - } - - setIncludeArchivedMedia(includeArchivedMedia) { - this.includeArchivedMedia = includeArchivedMedia; - return this; - } - - toJSON() { - return { - dateFilter: this.dateFilter instanceof DateFilter ? this.dateFilter.toJSON() : this.dateFilter, - mediaTypeFilter: this.mediaTypeFilter instanceof MediaTypeFilter ? - this.mediaTypeFilter.toJSON() : - this.mediaTypeFilter, - contentFilter: this.contentFilter instanceof ContentFilter ? - this.contentFilter.toJSON() : - this.contentFilter, - includeArchivedMedia: this.includeArchivedMedia - }; - } -} - -module.exports = Filters; \ No newline at end of file diff --git a/src/server/apis/google/GoogleApiServerUtils.ts b/src/server/apis/google/GoogleApiServerUtils.ts index e0bd8a800..684a8081b 100644 --- a/src/server/apis/google/GoogleApiServerUtils.ts +++ b/src/server/apis/google/GoogleApiServerUtils.ts @@ -8,7 +8,7 @@ import { GaxiosResponse } from "gaxios"; import request = require('request-promise'); import * as qs from 'query-string'; import Photos = require('googlephotos'); - +import { Database } from "../../database"; /** * Server side authentication for Google Api queries. */ @@ -35,9 +35,9 @@ export namespace GoogleApiServerUtils { Slides = "Slides" } - export interface CredentialPaths { + export interface CredentialInformation { credentialsPath: string; - tokenPath: string; + userId: string; } export type ApiResponse = Promise; @@ -48,7 +48,7 @@ export namespace GoogleApiServerUtils { export type Endpoint = { get: ApiHandler, create: ApiHandler, batchUpdate: ApiHandler }; export type EndpointParameters = GlobalOptions & { version: "v1" }; - export const GetEndpoint = (sector: string, paths: CredentialPaths) => { + export const GetEndpoint = (sector: string, paths: CredentialInformation) => { return new Promise>(resolve => { RetrieveCredentials(paths).then(authentication => { let routed: Opt; @@ -66,28 +66,28 @@ export namespace GoogleApiServerUtils { }); }; - export const RetrieveCredentials = (paths: CredentialPaths) => { + export const RetrieveCredentials = (information: CredentialInformation) => { return new Promise((resolve, reject) => { - readFile(paths.credentialsPath, async (err, credentials) => { + readFile(information.credentialsPath, async (err, credentials) => { if (err) { reject(err); return console.log('Error loading client secret file:', err); } - authorize(parseBuffer(credentials), paths.tokenPath).then(resolve, reject); + authorize(parseBuffer(credentials), information.userId).then(resolve, reject); }); }); }; - export const RetrieveAccessToken = (paths: CredentialPaths) => { + export const RetrieveAccessToken = (information: CredentialInformation) => { return new Promise((resolve, reject) => { - RetrieveCredentials(paths).then( + RetrieveCredentials(information).then( credentials => resolve(credentials.token.access_token!), error => reject(`Error: unable to authenticate Google Photos API request.\n${error}`) ); }); }; - export const RetrievePhotosEndpoint = (paths: CredentialPaths) => { + export const RetrievePhotosEndpoint = (paths: CredentialInformation) => { return new Promise((resolve, reject) => { RetrieveAccessToken(paths).then( token => resolve(new Photos(token)), @@ -101,20 +101,20 @@ export namespace GoogleApiServerUtils { * Create an OAuth2 client with the given credentials, and returns the promise resolving to the authenticated client * @param {Object} credentials The authorization client credentials. */ - export function authorize(credentials: any, token_path: string): Promise { + export function authorize(credentials: any, userId: string): Promise { const { client_secret, client_id, redirect_uris } = credentials.installed; const oAuth2Client = new google.auth.OAuth2( client_id, client_secret, redirect_uris[0]); return new Promise((resolve, reject) => { - readFile(token_path, (err, token) => { - // Check if we have previously stored a token. - if (err) { - return getNewToken(oAuth2Client, token_path).then(resolve, reject); + Database.Auxiliary.FetchGoogleAuthenticationToken(userId).then(token => { + // Check if we have previously stored a token for this userId. + if (!token) { + return getNewToken(oAuth2Client, userId).then(resolve, reject); } let parsed: Credentials = parseBuffer(token); if (parsed.expiry_date! < new Date().getTime()) { - return refreshToken(parsed, client_id, client_secret, oAuth2Client, token_path).then(resolve, reject); + return refreshToken(parsed, client_id, client_secret, oAuth2Client, userId).then(resolve, reject); } oAuth2Client.setCredentials(parsed); resolve({ token: parsed, client: oAuth2Client }); diff --git a/src/server/apis/google/GooglePhotosUploadUtils.ts b/src/server/apis/google/GooglePhotosUploadUtils.ts index 4dc252577..507a868a3 100644 --- a/src/server/apis/google/GooglePhotosUploadUtils.ts +++ b/src/server/apis/google/GooglePhotosUploadUtils.ts @@ -1,16 +1,10 @@ import request = require('request-promise'); import { GoogleApiServerUtils } from './GoogleApiServerUtils'; -import * as fs from 'fs'; -import { Utils } from '../../../Utils'; import * as path from 'path'; -import { Opt } from '../../../new_fields/Doc'; -import * as sharp from 'sharp'; import { MediaItemCreationResult } from './SharedTypes'; import { NewMediaItem } from "../../index"; import BatchedArray, { FixedBatcher, TimeUnit, Interval } from "array-batcher"; -const uploadDirectory = path.join(__dirname, "../../public/files/"); - export namespace GooglePhotosUploadUtils { export interface Paths { @@ -31,12 +25,9 @@ export namespace GooglePhotosUploadUtils { }); let Bearer: string; - let Paths: Paths; - export const initialize = async (paths: Paths) => { - Paths = paths; - const { tokenPath, credentialsPath } = paths; - const token = await GoogleApiServerUtils.RetrieveAccessToken({ tokenPath, credentialsPath }); + export const initialize = async (information: GoogleApiServerUtils.CredentialInformation) => { + const token = await GoogleApiServerUtils.RetrieveAccessToken(information); Bearer = `Bearer ${token}`; }; @@ -87,138 +78,4 @@ export namespace GooglePhotosUploadUtils { return { newMediaItemResults }; }; -} - -export namespace DownloadUtils { - - export interface Size { - width: number; - suffix: string; - } - - export const Sizes: { [size: string]: Size } = { - SMALL: { width: 100, suffix: "_s" }, - MEDIUM: { width: 400, suffix: "_m" }, - LARGE: { width: 900, suffix: "_l" }, - }; - - const gifs = [".gif"]; - const pngs = [".png"]; - const jpgs = [".jpg", ".jpeg"]; - const imageFormats = [...pngs, ...jpgs, ...gifs]; - const videoFormats = [".mov", ".mp4"]; - - const size = "content-length"; - const type = "content-type"; - - export interface UploadInformation { - mediaPaths: string[]; - fileNames: { [key: string]: string }; - contentSize?: number; - contentType?: string; - } - - const generate = (prefix: string, url: string) => `${prefix}upload_${Utils.GenerateGuid()}${path.extname(url).toLowerCase()}`; - const sanitize = (filename: string) => filename.replace(/\s+/g, "_"); - - export interface InspectionResults { - isLocal: boolean; - stream: any; - normalizedUrl: string; - contentSize?: number; - contentType?: string; - } - - export const InspectImage = async (url: string): Promise => { - const { isLocal, stream, normalized: normalizedUrl } = classify(url); - const results = { - isLocal, - stream, - normalizedUrl - }; - if (isLocal) { - return results; - } - const metadata = (await new Promise((resolve, reject) => { - request.head(url, async (error, res) => { - if (error) { - return reject(error); - } - resolve(res); - }); - })).headers; - return { - contentSize: parseInt(metadata[size]), - contentType: metadata[type], - ...results - }; - }; - - export const UploadImage = async (metadata: InspectionResults, filename?: string, prefix = ""): Promise> => { - const { isLocal, stream, normalizedUrl, contentSize, contentType } = metadata; - const resolved = filename ? sanitize(filename) : generate(prefix, normalizedUrl); - let extension = path.extname(normalizedUrl) || path.extname(resolved); - extension && (extension = extension.toLowerCase()); - let information: UploadInformation = { - mediaPaths: [], - fileNames: { clean: resolved }, - contentSize, - contentType, - }; - return new Promise(async (resolve, reject) => { - const resizers = [ - { resizer: sharp().rotate(), suffix: "_o" }, - ...Object.values(Sizes).map(size => ({ - resizer: sharp().resize(size.width, undefined, { withoutEnlargement: true }).rotate(), - suffix: size.suffix - })) - ]; - let nonVisual = false; - if (pngs.includes(extension)) { - resizers.forEach(element => element.resizer = element.resizer.png()); - } else if (jpgs.includes(extension)) { - resizers.forEach(element => element.resizer = element.resizer.jpeg()); - } else if (![...imageFormats, ...videoFormats].includes(extension.toLowerCase())) { - nonVisual = true; - } - if (imageFormats.includes(extension)) { - for (let resizer of resizers) { - const suffix = resizer.suffix; - let mediaPath: string; - await new Promise(resolve => { - const filename = resolved.substring(0, resolved.length - extension.length) + suffix + extension; - information.mediaPaths.push(mediaPath = uploadDirectory + filename); - information.fileNames[suffix] = filename; - stream(normalizedUrl).pipe(resizer.resizer).pipe(fs.createWriteStream(mediaPath)) - .on('close', resolve) - .on('error', reject); - }); - } - } - if (!isLocal || nonVisual) { - await new Promise(resolve => { - stream(normalizedUrl).pipe(fs.createWriteStream(uploadDirectory + resolved)).on('close', resolve); - }); - } - resolve(information); - }); - }; - - const classify = (url: string) => { - const isLocal = /Dash-Web(\\|\/)src(\\|\/)server(\\|\/)public(\\|\/)files/g.test(url); - return { - isLocal, - stream: isLocal ? fs.createReadStream : request, - normalized: isLocal ? path.normalize(url) : url - }; - }; - - export const createIfNotExists = async (path: string) => { - if (await new Promise(resolve => fs.exists(path, resolve))) { - return true; - } - return new Promise(resolve => fs.mkdir(path, error => resolve(error === null))); - }; - - export const Destroy = (mediaPath: string) => new Promise(resolve => fs.unlink(mediaPath, error => resolve(error === null))); } \ No newline at end of file diff --git a/src/server/apis/google/existing_uploads.json b/src/server/apis/google/existing_uploads.json deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/server/credentials/google_docs_token.json b/src/server/credentials/google_docs_token.json index ee44c3f30..8bd62bdfa 100644 --- a/src/server/credentials/google_docs_token.json +++ b/src/server/credentials/google_docs_token.json @@ -1 +1,7 @@ -{"access_token":"ya29.GlyKBznz91v_qb8RYt4PT40Hp106N08Yk64UjMAKllBsIqJQEzBkxLbB5q5paydywHzguQYSNup5fT7ojJTDU4CMZdPbPKGcjQz17w_CospcG-8Buz94KZptvlQ_pQ","refresh_token":"1/HTv_xFHszu2Nf3iiFrUTaeKzC_Vp2-6bpIB06xW_WHI","scope":"https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/documents.readonly https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/photoslibrary https://www.googleapis.com/auth/photoslibrary.appendonly https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/photoslibrary.sharing","token_type":"Bearer","expiry_date":1569093749804} \ No newline at end of file +{ + "access_token": "ya29.ImCOBwXgckGbyHNLMX7r-13B5VDgxfzF5mQ7lFJ0FX5GF5EuAPBBN5_ijLnNLC4yw4xtFjJOkEtKiYr-60OIm4oOnowEJpZMyRGxFMy_Q8MTnzDpeN-7Di_baUzcu7m_KWM", + "refresh_token": "1/HTv_xFHszu2Nf3iiFrUTaeKzC_Vp2-6bpIB06xW_WHI", + "scope": "https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/documents.readonly https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/photoslibrary https://www.googleapis.com/auth/photoslibrary.appendonly https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/photoslibrary.sharing", + "token_type": "Bearer", + "expiry_date": 1569366907812 +} \ No newline at end of file diff --git a/src/server/database.ts b/src/server/database.ts index a7254fb0c..ce29478ad 100644 --- a/src/server/database.ts +++ b/src/server/database.ts @@ -1,209 +1,267 @@ import * as mongodb from 'mongodb'; import { Transferable } from './Message'; +import { Opt } from '../new_fields/Doc'; +import { Utils } from '../Utils'; +import { DashUploadUtils } from './DashUploadUtils'; -export class Database { - public static DocumentsCollection = 'documents'; - public static Instance = new Database(); - private MongoClient = mongodb.MongoClient; - private url = 'mongodb://localhost:27017/Dash'; - private currentWrites: { [id: string]: Promise } = {}; - private db?: mongodb.Db; - private onConnect: (() => void)[] = []; - - constructor() { - this.MongoClient.connect(this.url, (err, client) => { - this.db = client.db(); - this.onConnect.forEach(fn => fn()); - }); - } +export namespace Database { - public update(id: string, value: any, callback: (err: mongodb.MongoError, res: mongodb.UpdateWriteOpResult) => void, upsert = true, collectionName = Database.DocumentsCollection) { - if (this.db) { - let collection = this.db.collection(collectionName); - const prom = this.currentWrites[id]; - let newProm: Promise; - const run = (): Promise => { - return new Promise(resolve => { - collection.updateOne({ _id: id }, value, { upsert } - , (err, res) => { - if (this.currentWrites[id] === newProm) { - delete this.currentWrites[id]; - } - resolve(); - callback(err, res); - }); - }); - }; - newProm = prom ? prom.then(run) : run(); - this.currentWrites[id] = newProm; - } else { - this.onConnect.push(() => this.update(id, value, callback, upsert, collectionName)); - } - } + class Database { + public static DocumentsCollection = 'documents'; + private MongoClient = mongodb.MongoClient; + private url = 'mongodb://localhost:27017/Dash'; + private currentWrites: { [id: string]: Promise } = {}; + private db?: mongodb.Db; + private onConnect: (() => void)[] = []; - public replace(id: string, value: any, callback: (err: mongodb.MongoError, res: mongodb.UpdateWriteOpResult) => void, upsert = true, collectionName = Database.DocumentsCollection) { - if (this.db) { - let collection = this.db.collection(collectionName); - const prom = this.currentWrites[id]; - let newProm: Promise; - const run = (): Promise => { - return new Promise(resolve => { - collection.replaceOne({ _id: id }, value, { upsert } - , (err, res) => { - if (this.currentWrites[id] === newProm) { - delete this.currentWrites[id]; - } - resolve(); - callback(err, res); - }); - }); - }; - newProm = prom ? prom.then(run) : run(); - this.currentWrites[id] = newProm; - } else { - this.onConnect.push(() => this.replace(id, value, callback, upsert, collectionName)); + constructor() { + this.MongoClient.connect(this.url, (err, client) => { + this.db = client.db(); + this.onConnect.forEach(fn => fn()); + }); } - } - public delete(query: any, collectionName?: string): Promise; - public delete(id: string, collectionName?: string): Promise; - public delete(id: any, collectionName = Database.DocumentsCollection) { - if (typeof id === "string") { - id = { _id: id }; - } - if (this.db) { - const db = this.db; - return new Promise(res => db.collection(collectionName).deleteMany(id, (err, result) => res(result))); - } else { - return new Promise(res => this.onConnect.push(() => res(this.delete(id, collectionName)))); + public update(id: string, value: any, callback: (err: mongodb.MongoError, res: mongodb.UpdateWriteOpResult) => void, upsert = true, collectionName = Database.DocumentsCollection) { + if (this.db) { + let collection = this.db.collection(collectionName); + const prom = this.currentWrites[id]; + let newProm: Promise; + const run = (): Promise => { + return new Promise(resolve => { + collection.updateOne({ _id: id }, value, { upsert } + , (err, res) => { + if (this.currentWrites[id] === newProm) { + delete this.currentWrites[id]; + } + resolve(); + callback(err, res); + }); + }); + }; + newProm = prom ? prom.then(run) : run(); + this.currentWrites[id] = newProm; + } else { + this.onConnect.push(() => this.update(id, value, callback, upsert, collectionName)); + } } - } - public deleteAll(collectionName = Database.DocumentsCollection): Promise { - return new Promise(res => { + public replace(id: string, value: any, callback: (err: mongodb.MongoError, res: mongodb.UpdateWriteOpResult) => void, upsert = true, collectionName = Database.DocumentsCollection) { if (this.db) { - this.db.collection(collectionName).deleteMany({}, res); + let collection = this.db.collection(collectionName); + const prom = this.currentWrites[id]; + let newProm: Promise; + const run = (): Promise => { + return new Promise(resolve => { + collection.replaceOne({ _id: id }, value, { upsert } + , (err, res) => { + if (this.currentWrites[id] === newProm) { + delete this.currentWrites[id]; + } + resolve(); + callback(err, res); + }); + }); + }; + newProm = prom ? prom.then(run) : run(); + this.currentWrites[id] = newProm; } else { - this.onConnect.push(() => this.db && this.db.collection(collectionName).deleteMany({}, res)); + this.onConnect.push(() => this.replace(id, value, callback, upsert, collectionName)); } - }); - } + } - public insert(value: any, collectionName = Database.DocumentsCollection) { - if (this.db) { - if ("id" in value) { - value._id = value.id; - delete value.id; + public delete(query: any, collectionName?: string): Promise; + public delete(id: string, collectionName?: string): Promise; + public delete(id: any, collectionName = Database.DocumentsCollection) { + if (typeof id === "string") { + id = { _id: id }; + } + if (this.db) { + const db = this.db; + return new Promise(res => db.collection(collectionName).deleteMany(id, (err, result) => res(result))); + } else { + return new Promise(res => this.onConnect.push(() => res(this.delete(id, collectionName)))); } - const id = value._id; - const collection = this.db.collection(collectionName); - const prom = this.currentWrites[id]; - let newProm: Promise; - const run = (): Promise => { - return new Promise(resolve => { - collection.insertOne(value, (err, res) => { - if (this.currentWrites[id] === newProm) { - delete this.currentWrites[id]; - } - resolve(); - }); - }); - }; - newProm = prom ? prom.then(run) : run(); - this.currentWrites[id] = newProm; - } else { - this.onConnect.push(() => this.insert(value, collectionName)); } - } - public getDocument(id: string, fn: (result?: Transferable) => void, collectionName = Database.DocumentsCollection) { - if (this.db) { - this.db.collection(collectionName).findOne({ _id: id }, (err, result) => { - if (result) { - result.id = result._id; - delete result._id; - fn(result); + public async deleteAll(collectionName = Database.DocumentsCollection, persist = true): Promise { + return new Promise(resolve => { + const executor = async (database: mongodb.Db) => { + if (persist) { + await database.collection(collectionName).deleteMany({}); + } else { + await database.dropCollection(collectionName); + } + resolve(); + }; + if (this.db) { + executor(this.db); } else { - fn(undefined); + this.onConnect.push(() => this.db && executor(this.db)); } }); - } else { - this.onConnect.push(() => this.getDocument(id, fn, collectionName)); } - } - public getDocuments(ids: string[], fn: (result: Transferable[]) => void, collectionName = Database.DocumentsCollection) { - if (this.db) { - this.db.collection(collectionName).find({ _id: { "$in": ids } }).toArray((err, docs) => { - if (err) { - console.log(err.message); - console.log(err.errmsg); + public async insert(value: any, collectionName = Database.DocumentsCollection) { + if (this.db) { + if ("id" in value) { + value._id = value.id; + delete value.id; } - fn(docs.map(doc => { - doc.id = doc._id; - delete doc._id; - return doc; - })); - }); - } else { - this.onConnect.push(() => this.getDocuments(ids, fn, collectionName)); + const id = value._id; + const collection = this.db.collection(collectionName); + const prom = this.currentWrites[id]; + let newProm: Promise; + const run = (): Promise => { + return new Promise(resolve => { + collection.insertOne(value, (err, res) => { + if (this.currentWrites[id] === newProm) { + delete this.currentWrites[id]; + } + resolve(); + }); + }); + }; + newProm = prom ? prom.then(run) : run(); + this.currentWrites[id] = newProm; + return newProm; + } else { + this.onConnect.push(() => this.insert(value, collectionName)); + } } - } - public async visit(ids: string[], fn: (result: any) => string[], collectionName = "newDocuments"): Promise { - if (this.db) { - const visited = new Set(); - while (ids.length) { - const count = Math.min(ids.length, 1000); - const index = ids.length - count; - const fetchIds = ids.splice(index, count).filter(id => !visited.has(id)); - if (!fetchIds.length) { - continue; - } - const docs = await new Promise<{ [key: string]: any }[]>(res => Database.Instance.getDocuments(fetchIds, res, "newDocuments")); - for (const doc of docs) { - const id = doc.id; - visited.add(id); - ids.push(...fn(doc)); + public getDocument(id: string, fn: (result?: Transferable) => void, collectionName = Database.DocumentsCollection) { + if (this.db) { + this.db.collection(collectionName).findOne({ _id: id }, (err, result) => { + if (result) { + result.id = result._id; + delete result._id; + fn(result); + } else { + fn(undefined); + } + }); + } else { + this.onConnect.push(() => this.getDocument(id, fn, collectionName)); + } + } + + public getDocuments(ids: string[], fn: (result: Transferable[]) => void, collectionName = Database.DocumentsCollection) { + if (this.db) { + this.db.collection(collectionName).find({ _id: { "$in": ids } }).toArray((err, docs) => { + if (err) { + console.log(err.message); + console.log(err.errmsg); + } + fn(docs.map(doc => { + doc.id = doc._id; + delete doc._id; + return doc; + })); + }); + } else { + this.onConnect.push(() => this.getDocuments(ids, fn, collectionName)); + } + } + + public async visit(ids: string[], fn: (result: any) => string[], collectionName = "newDocuments"): Promise { + if (this.db) { + const visited = new Set(); + while (ids.length) { + const count = Math.min(ids.length, 1000); + const index = ids.length - count; + const fetchIds = ids.splice(index, count).filter(id => !visited.has(id)); + if (!fetchIds.length) { + continue; + } + const docs = await new Promise<{ [key: string]: any }[]>(res => Instance.getDocuments(fetchIds, res, "newDocuments")); + for (const doc of docs) { + const id = doc.id; + visited.add(id); + ids.push(...fn(doc)); + } } + + } else { + return new Promise(res => { + this.onConnect.push(() => { + this.visit(ids, fn, collectionName); + res(); + }); + }); } + } - } else { - return new Promise(res => { - this.onConnect.push(() => { - this.visit(ids, fn, collectionName); - res(); + public query(query: { [key: string]: any }, projection?: { [key: string]: 0 | 1 }, collectionName = "newDocuments"): Promise { + if (this.db) { + let cursor = this.db.collection(collectionName).find(query); + if (projection) { + cursor = cursor.project(projection); + } + return Promise.resolve(cursor); + } else { + return new Promise(res => { + this.onConnect.push(() => res(this.query(query, projection, collectionName))); }); - }); + } } - } - public query(query: { [key: string]: any }, projection?: { [key: string]: 0 | 1 }, collectionName = "newDocuments"): Promise { - if (this.db) { - let cursor = this.db.collection(collectionName).find(query); - if (projection) { - cursor = cursor.project(projection); + public updateMany(query: any, update: any, collectionName = "newDocuments") { + if (this.db) { + const db = this.db; + return new Promise(res => db.collection(collectionName).update(query, update, (_, result) => res(result))); + } else { + return new Promise(res => { + this.onConnect.push(() => this.updateMany(query, update, collectionName).then(res)); + }); } - return Promise.resolve(cursor); - } else { - return new Promise(res => { - this.onConnect.push(() => res(this.query(query, projection, collectionName))); - }); } - } - public updateMany(query: any, update: any, collectionName = "newDocuments") { - if (this.db) { - const db = this.db; - return new Promise(res => db.collection(collectionName).update(query, update, (_, result) => res(result))); - } else { - return new Promise(res => { - this.onConnect.push(() => this.updateMany(query, update, collectionName).then(res)); - }); + public print() { + console.log("db says hi!"); } } - public print() { - console.log("db says hi!"); + export const Instance = new Database(); + + export namespace Auxiliary { + + export enum AuxiliaryCollections { + GooglePhotosUploadHistory = "UploadedFromGooglePhotos" + } + + const GoogleAuthentication = "GoogleAuthentication"; + + const SanitizedSingletonQuery = async (query: { [key: string]: any }, collection: string) => { + const cursor = await Instance.query(query, undefined, collection); + const existing = (await cursor.toArray())[0]; + if (existing) { + delete existing._id; + } + return existing; + }; + + export const QueryUploadHistory = async (contentSize: number): Promise> => { + return SanitizedSingletonQuery({ contentSize }, AuxiliaryCollections.GooglePhotosUploadHistory); + }; + + export const LogUpload = async (information: DashUploadUtils.UploadInformation) => { + const bundle = { + _id: Utils.GenerateDeterministicGuid(String(information.contentSize!)), + ...information + }; + return Instance.insert(bundle, AuxiliaryCollections.GooglePhotosUploadHistory); + }; + + export const DeleteAll = async (persist = false) => { + const collectionNames = Object.values(AuxiliaryCollections); + const pendingDeletions = collectionNames.map(name => Instance.deleteAll(name, persist)); + return Promise.all(pendingDeletions); + }; + + export const FetchGoogleAuthenticationToken = async (userId: string) => { + return SanitizedSingletonQuery({ userId }, GoogleAuthentication); + }; + } -} + +} \ No newline at end of file diff --git a/src/server/index.ts b/src/server/index.ts index 4c4cb84d6..386ecce4d 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -41,13 +41,14 @@ var AdmZip = require('adm-zip'); import * as YoutubeApi from "./apis/youtube/youtubeApiSample"; import { Response } from 'express-serve-static-core'; import { GoogleApiServerUtils } from "./apis/google/GoogleApiServerUtils"; -import { GooglePhotosUploadUtils, DownloadUtils as UploadUtils } from './apis/google/GooglePhotosUploadUtils'; +import { GooglePhotosUploadUtils } from './apis/google/GooglePhotosUploadUtils'; const MongoStore = require('connect-mongo')(session); const mongoose = require('mongoose'); const probe = require("probe-image-size"); import * as qs from 'query-string'; import { Opt } from '../new_fields/Doc'; import BatchedArray, { TimeUnit } from "array-batcher"; +import { DashUploadUtils } from './DashUploadUtils'; const download = (url: string, dest: fs.PathLike) => request.get(url).pipe(fs.createWriteStream(dest)); let youtubeApiKey: string; @@ -581,8 +582,8 @@ app.post( for (const key in files) { const { type, path: location, name } = files[key]; const filename = path.basename(location); - const metadata = await UploadUtils.InspectImage(uploadDirectory + filename); - await UploadUtils.UploadImage(metadata, filename).catch(() => console.log(`Unable to process ${filename}`)); + const metadata = await DashUploadUtils.InspectImage(uploadDirectory + filename); + await DashUploadUtils.UploadImage(metadata, filename).catch(() => console.log(`Unable to process ${filename}`)); results.push({ name, type, path: `/files/${filename}` }); } _success(res, results); @@ -809,7 +810,7 @@ const EndpointHandlerMap = new Map { let sector: GoogleApiServerUtils.Service = req.params.sector as GoogleApiServerUtils.Service; let action: GoogleApiServerUtils.Action = req.params.action as GoogleApiServerUtils.Action; - GoogleApiServerUtils.GetEndpoint(GoogleApiServerUtils.Service[sector], { credentialsPath, tokenPath }).then(endpoint => { + GoogleApiServerUtils.GetEndpoint(GoogleApiServerUtils.Service[sector], { credentialsPath, userId: req.body.userId }).then(endpoint => { let handler = EndpointHandlerMap.get(action); if (endpoint && handler) { let execute = handler(endpoint, req.body).then( @@ -823,7 +824,7 @@ app.post(RouteStore.googleDocs + "/:sector/:action", (req, res) => { }); }); -app.get(RouteStore.googlePhotosAccessToken, (req, res) => GoogleApiServerUtils.RetrieveAccessToken({ credentialsPath, tokenPath }).then(token => res.send(token))); +app.get(RouteStore.googlePhotosAccessToken, (req, res) => GoogleApiServerUtils.RetrieveAccessToken({ credentialsPath, userId: req.body.userId }).then(token => res.send(token))); const tokenError = "Unable to successfully upload bytes for all images!"; const mediaError = "Unable to convert all uploaded bytes to media items!"; @@ -836,12 +837,13 @@ export interface NewMediaItem { } app.post(RouteStore.googlePhotosMediaUpload, async (req, res) => { - const mediaInput: GooglePhotosUploadUtils.MediaInput[] = req.body.media; - await GooglePhotosUploadUtils.initialize({ uploadDirectory, credentialsPath, tokenPath }); + const { userId, media } = req.body; + + await GooglePhotosUploadUtils.initialize({ credentialsPath, userId }); let failed = 0; - const newMediaItems = await BatchedArray.from(mediaInput, { batchSize: 25 }).batchedMapInterval( + const newMediaItems = await BatchedArray.from(media, { batchSize: 25 }).batchedMapInterval( async (batch: GooglePhotosUploadUtils.MediaInput[]) => { const newMediaItems: NewMediaItem[] = []; for (let element of batch) { @@ -879,31 +881,36 @@ const prefix = "google_photos_"; const downloadError = "Encountered an error while executing downloads."; const requestError = "Unable to execute download: the body's media items were malformed."; -app.get("/gapiCleanup", (req, res) => { - write_text_file(file, ""); +app.get("/deleteWithAux", async (req, res) => { + await Database.Auxiliary.DeleteAll(); res.redirect(RouteStore.delete); }); -const file = "./apis/google/existing_uploads.json"; +const UploadError = (count: number) => `Unable to upload ${count} images to Dash's server`; app.post(RouteStore.googlePhotosMediaDownload, async (req, res) => { const contents: { mediaItems: MediaItem[] } = req.body; + let failed = 0; if (contents) { - const completed: Opt[] = []; - const content = await read_text_file(file); - let existing = content.length ? JSON.parse(content) : {}; + const completed: Opt[] = []; for (let item of contents.mediaItems) { - const { contentSize, ...attributes } = await UploadUtils.InspectImage(item.baseUrl); - const found: UploadUtils.UploadInformation = existing[contentSize!]; + const { contentSize, ...attributes } = await DashUploadUtils.InspectImage(item.baseUrl); + const found: Opt = await Database.Auxiliary.QueryUploadHistory(contentSize!); if (!found) { - const upload = await UploadUtils.UploadImage({ contentSize, ...attributes }, item.filename, prefix).catch(error => _error(res, downloadError, error)); - upload && completed.push(existing[contentSize!] = upload); + const upload = await DashUploadUtils.UploadImage({ contentSize, ...attributes }, item.filename, prefix).catch(error => _error(res, downloadError, error)); + if (upload) { + completed.push(upload); + await Database.Auxiliary.LogUpload(upload); + } else { + failed++; + } } else { completed.push(found); } } - await write_text_file(file, JSON.stringify(existing)); - _success(res, completed); - return; + if (failed) { + return _error(res, UploadError(failed)); + } + return _success(res, completed); } _invalid(res, requestError); }); -- cgit v1.2.3-70-g09d2 From e95387732e1fbff49ec035c3bec4b03324d814c8 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Thu, 26 Sep 2019 18:11:10 -0400 Subject: beginning to read from and write to database --- src/Utils.ts | 92 ++++++++++------------ src/client/Network.ts | 25 ++++++ .../apis/google_docs/GoogleApiClientUtils.ts | 2 +- .../apis/google_docs/GooglePhotosClientUtils.ts | 3 +- .../util/Import & Export/DirectoryImportBox.tsx | 4 +- src/client/views/MainView.tsx | 2 +- src/new_fields/RichTextUtils.ts | 4 +- src/server/apis/google/GoogleApiServerUtils.ts | 51 +++++------- src/server/database.ts | 34 +++++--- src/server/index.ts | 12 ++- 10 files changed, 126 insertions(+), 103 deletions(-) create mode 100644 src/client/Network.ts (limited to 'src/server/apis') diff --git a/src/Utils.ts b/src/Utils.ts index 5f06b5cec..ae8371f15 100644 --- a/src/Utils.ts +++ b/src/Utils.ts @@ -3,22 +3,20 @@ import v5 = require("uuid/v5"); import { Socket } from 'socket.io'; import { Message } from './server/Message'; import { RouteStore } from './server/RouteStore'; -import requestPromise = require('request-promise'); -import { CurrentUserUtils } from './server/authentication/models/current_user_utils'; -export class Utils { +export namespace Utils { - public static DRAG_THRESHOLD = 4; + export const DRAG_THRESHOLD = 4; - public static GenerateGuid(): string { + export function GenerateGuid(): string { return v4(); } - public static GenerateDeterministicGuid(seed: string): string { + export function GenerateDeterministicGuid(seed: string): string { return v5(seed, v5.URL); } - public static GetScreenTransform(ele?: HTMLElement): { scale: number, translateX: number, translateY: number } { + export function GetScreenTransform(ele?: HTMLElement): { scale: number, translateX: number, translateY: number } { if (!ele) { return { scale: 1, translateX: 1, translateY: 1 }; } @@ -35,23 +33,23 @@ export class Utils { * requested extension * @param extension the specified sub-path to append to the window origin */ - public static prepend(extension: string): string { + export function prepend(extension: string): string { return window.location.origin + extension; } - public static fileUrl(filename: string): string { - return this.prepend(`/files/${filename}`); + export function fileUrl(filename: string): string { + return prepend(`/files/${filename}`); } - public static shareUrl(documentId: string): string { - return this.prepend(`/doc/${documentId}?sharing=true`); + export function shareUrl(documentId: string): string { + return prepend(`/doc/${documentId}?sharing=true`); } - public static CorsProxy(url: string): string { - return this.prepend(RouteStore.corsProxy + "/") + encodeURIComponent(url); + export function CorsProxy(url: string): string { + return prepend(RouteStore.corsProxy + "/") + encodeURIComponent(url); } - public static CopyText(text: string) { + export function CopyText(text: string) { var textArea = document.createElement("textarea"); textArea.value = text; document.body.appendChild(textArea); @@ -63,7 +61,7 @@ export class Utils { document.body.removeChild(textArea); } - public static fromRGBAstr(rgba: string) { + export function fromRGBAstr(rgba: string) { let rm = rgba.match(/rgb[a]?\(([0-9]+)/); let r = rm ? Number(rm[1]) : 0; let gm = rgba.match(/rgb[a]?\([0-9]+,([0-9]+)/); @@ -74,11 +72,12 @@ export class Utils { let a = am ? Number(am[1]) : 0; return { r: r, g: g, b: b, a: a }; } - public static toRGBAstr(col: { r: number, g: number, b: number, a?: number }) { + + export function toRGBAstr(col: { r: number, g: number, b: number, a?: number }) { return "rgba(" + col.r + "," + col.g + "," + col.b + (col.a !== undefined ? "," + col.a : "") + ")"; } - public static HSLtoRGB(h: number, s: number, l: number) { + export function HSLtoRGB(h: number, s: number, l: number) { // Must be fractions of 1 // s /= 100; // l /= 100; @@ -108,7 +107,7 @@ export class Utils { return { r: r, g: g, b: b }; } - public static RGBToHSL(r: number, g: number, b: number) { + export function RGBToHSL(r: number, g: number, b: number) { // Make r, g, and b fractions of 1 r /= 255; g /= 255; @@ -150,7 +149,7 @@ export class Utils { } - public static GetClipboardText(): string { + export function GetClipboardText(): string { var textArea = document.createElement("textarea"); document.body.appendChild(textArea); textArea.focus(); @@ -163,51 +162,53 @@ export class Utils { return val; } - public static loggingEnabled: Boolean = false; - public static logFilter: number | undefined = undefined; - private static log(prefix: string, messageName: string, message: any, receiving: boolean) { - if (!this.loggingEnabled) { + export const loggingEnabled: Boolean = false; + export const logFilter: number | undefined = undefined; + + function log(prefix: string, messageName: string, message: any, receiving: boolean) { + if (!loggingEnabled) { return; } message = message || {}; - if (this.logFilter !== undefined && this.logFilter !== message.type) { + if (logFilter !== undefined && logFilter !== message.type) { return; } let idString = (message.id || "").padStart(36, ' '); prefix = prefix.padEnd(16, ' '); console.log(`${prefix}: ${idString}, ${receiving ? 'receiving' : 'sending'} ${messageName} with data ${JSON.stringify(message)}`); } - private static loggingCallback(prefix: string, func: (args: any) => any, messageName: string) { + + function loggingCallback(prefix: string, func: (args: any) => any, messageName: string) { return (args: any) => { - this.log(prefix, messageName, args, true); + log(prefix, messageName, args, true); func(args); }; } - public static Emit(socket: Socket | SocketIOClient.Socket, message: Message, args: T) { - this.log("Emit", message.Name, args, false); + export function Emit(socket: Socket | SocketIOClient.Socket, message: Message, args: T) { + log("Emit", message.Name, args, false); socket.emit(message.Message, args); } - public static EmitCallback(socket: Socket | SocketIOClient.Socket, message: Message, args: T): Promise; - public static EmitCallback(socket: Socket | SocketIOClient.Socket, message: Message, args: T, fn: (args: any) => any): void; - public static EmitCallback(socket: Socket | SocketIOClient.Socket, message: Message, args: T, fn?: (args: any) => any): void | Promise { - this.log("Emit", message.Name, args, false); + export function EmitCallback(socket: Socket | SocketIOClient.Socket, message: Message, args: T): Promise; + export function EmitCallback(socket: Socket | SocketIOClient.Socket, message: Message, args: T, fn: (args: any) => any): void; + export function EmitCallback(socket: Socket | SocketIOClient.Socket, message: Message, args: T, fn?: (args: any) => any): void | Promise { + log("Emit", message.Name, args, false); if (fn) { - socket.emit(message.Message, args, this.loggingCallback('Receiving', fn, message.Name)); + socket.emit(message.Message, args, loggingCallback('Receiving', fn, message.Name)); } else { - return new Promise(res => socket.emit(message.Message, args, this.loggingCallback('Receiving', res, message.Name))); + return new Promise(res => socket.emit(message.Message, args, loggingCallback('Receiving', res, message.Name))); } } - public static AddServerHandler(socket: Socket | SocketIOClient.Socket, message: Message, handler: (args: T) => any) { - socket.on(message.Message, this.loggingCallback('Incoming', handler, message.Name)); + export function AddServerHandler(socket: Socket | SocketIOClient.Socket, message: Message, handler: (args: T) => any) { + socket.on(message.Message, loggingCallback('Incoming', handler, message.Name)); } - public static AddServerHandlerCallback(socket: Socket, message: Message, handler: (args: [T, (res: any) => any]) => any) { + export function AddServerHandlerCallback(socket: Socket, message: Message, handler: (args: [T, (res: any) => any]) => any) { socket.on(message.Message, (arg: T, fn: (res: any) => any) => { - this.log('S receiving', message.Name, arg, true); - handler([arg, this.loggingCallback('S sending', fn, message.Name)]); + log('S receiving', message.Name, arg, true); + handler([arg, loggingCallback('S sending', fn, message.Name)]); }); } } @@ -291,15 +292,4 @@ export namespace JSONUtils { return results; } -} - -export function PostToServer(relativeRoute: string, body?: any) { - body = { userId: CurrentUserUtils.id, ...body }; - let options = { - method: "POST", - uri: Utils.prepend(relativeRoute), - json: true, - body - }; - return requestPromise.post(options); } \ No newline at end of file diff --git a/src/client/Network.ts b/src/client/Network.ts new file mode 100644 index 000000000..cb46105f8 --- /dev/null +++ b/src/client/Network.ts @@ -0,0 +1,25 @@ +import { Utils } from "../Utils"; +import { CurrentUserUtils } from "../server/authentication/models/current_user_utils"; +import requestPromise = require('request-promise'); + +export async function PostToServer(relativeRoute: string, body?: any) { + let options = { + uri: Utils.prepend(relativeRoute), + method: "POST", + headers: { userId: CurrentUserUtils.id }, + body, + json: true + }; + return requestPromise.post(options); +} + +export async function PostFormDataToServer(relativeRoute: string, formData: FormData) { + const parameters = { + method: 'POST', + headers: { userId: CurrentUserUtils.id }, + body: formData, + }; + const response = await fetch(relativeRoute, parameters); + const text = await response.json(); + return text; +} \ No newline at end of file diff --git a/src/client/apis/google_docs/GoogleApiClientUtils.ts b/src/client/apis/google_docs/GoogleApiClientUtils.ts index 2c84741db..0f0f81891 100644 --- a/src/client/apis/google_docs/GoogleApiClientUtils.ts +++ b/src/client/apis/google_docs/GoogleApiClientUtils.ts @@ -1,9 +1,9 @@ import { docs_v1, slides_v1 } from "googleapis"; -import { PostToServer } from "../../../Utils"; import { RouteStore } from "../../../server/RouteStore"; import { Opt } from "../../../new_fields/Doc"; import { isArray } from "util"; import { EditorState } from "prosemirror-state"; +import { PostToServer } from "../../Network"; export const Pulls = "googleDocsPullCount"; export const Pushes = "googleDocsPushCount"; diff --git a/src/client/apis/google_docs/GooglePhotosClientUtils.ts b/src/client/apis/google_docs/GooglePhotosClientUtils.ts index c45a49f1a..b1b29210a 100644 --- a/src/client/apis/google_docs/GooglePhotosClientUtils.ts +++ b/src/client/apis/google_docs/GooglePhotosClientUtils.ts @@ -1,4 +1,4 @@ -import { PostToServer, Utils } from "../../../Utils"; +import { Utils } from "../../../Utils"; import { RouteStore } from "../../../server/RouteStore"; import { ImageField } from "../../../new_fields/URLField"; import { Cast, StrCast } from "../../../new_fields/Types"; @@ -14,6 +14,7 @@ import { NewMediaItemResult, MediaItem } from "../../../server/apis/google/Share import { AssertionError } from "assert"; import { DocumentView } from "../../views/nodes/DocumentView"; import { DocumentManager } from "../../util/DocumentManager"; +import { PostToServer } from "../../Network"; export namespace GooglePhotos { diff --git a/src/client/util/Import & Export/DirectoryImportBox.tsx b/src/client/util/Import & Export/DirectoryImportBox.tsx index 6670f685e..d0291aec4 100644 --- a/src/client/util/Import & Export/DirectoryImportBox.tsx +++ b/src/client/util/Import & Export/DirectoryImportBox.tsx @@ -21,6 +21,7 @@ import { GooglePhotos } from "../../apis/google_docs/GooglePhotosClientUtils"; import { SchemaHeaderField } from "../../../new_fields/SchemaHeaderField"; import "./DirectoryImportBox.scss"; import BatchedArray from "array-batcher"; +import { PostFormDataToServer } from "../../Network"; const unsupported = ["text/html", "text/plain"]; interface FileResponse { @@ -106,7 +107,6 @@ export default class DirectoryImportBox extends React.Component const uploads = await BatchedArray.from(validated, { batchSize: 15 }).batchedMapAsync(async batch => { const formData = new FormData(); - const parameters = { method: 'POST', body: formData }; batch.forEach(file => { sizes.push(file.size); @@ -114,7 +114,7 @@ export default class DirectoryImportBox extends React.Component formData.append(Utils.GenerateGuid(), file); }); - const responses = await (await fetch(RouteStore.upload, parameters)).json(); + const responses = await PostFormDataToServer(RouteStore.upload, formData); runInAction(() => this.completed += batch.length); return responses as FileResponse[]; }); diff --git a/src/client/views/MainView.tsx b/src/client/views/MainView.tsx index b1f753635..296574a04 100644 --- a/src/client/views/MainView.tsx +++ b/src/client/views/MainView.tsx @@ -15,7 +15,7 @@ import { listSpec } from '../../new_fields/Schema'; import { BoolCast, Cast, FieldValue, StrCast, NumCast } from '../../new_fields/Types'; import { CurrentUserUtils } from '../../server/authentication/models/current_user_utils'; import { RouteStore } from '../../server/RouteStore'; -import { emptyFunction, returnOne, returnTrue, Utils, returnEmptyString, PostToServer } from '../../Utils'; +import { emptyFunction, returnOne, returnTrue, Utils, returnEmptyString } from '../../Utils'; import { DocServer } from '../DocServer'; import { ClientUtils } from '../util/ClientUtils'; import { DictationManager } from '../util/DictationManager'; diff --git a/src/new_fields/RichTextUtils.ts b/src/new_fields/RichTextUtils.ts index 02079e92c..f3208ce41 100644 --- a/src/new_fields/RichTextUtils.ts +++ b/src/new_fields/RichTextUtils.ts @@ -7,17 +7,17 @@ import { FormattedTextBox } from "../client/views/nodes/FormattedTextBox"; import { Opt, Doc } from "./Doc"; import Color = require('color'); import { sinkListItem } from "prosemirror-schema-list"; -import { Utils, PostToServer } from "../Utils"; +import { Utils } from "../Utils"; import { RouteStore } from "../server/RouteStore"; import { Docs } from "../client/documents/Documents"; import { schema } from "../client/util/RichTextSchema"; import { GooglePhotos } from "../client/apis/google_docs/GooglePhotosClientUtils"; -import { SchemaHeaderField } from "./SchemaHeaderField"; import { DocServer } from "../client/DocServer"; import { Cast, StrCast } from "./Types"; import { Id } from "./FieldSymbols"; import { DocumentView } from "../client/views/nodes/DocumentView"; import { AssertionError } from "assert"; +import { PostToServer } from "../client/Network"; export namespace RichTextUtils { diff --git a/src/server/apis/google/GoogleApiServerUtils.ts b/src/server/apis/google/GoogleApiServerUtils.ts index 684a8081b..665dcf862 100644 --- a/src/server/apis/google/GoogleApiServerUtils.ts +++ b/src/server/apis/google/GoogleApiServerUtils.ts @@ -66,6 +66,15 @@ export namespace GoogleApiServerUtils { }); }; + export const RetrieveAccessToken = (information: CredentialInformation) => { + return new Promise((resolve, reject) => { + RetrieveCredentials(information).then( + credentials => resolve(credentials.token.access_token!), + error => reject(`Error: unable to authenticate Google Photos API request.\n${error}`) + ); + }); + }; + export const RetrieveCredentials = (information: CredentialInformation) => { return new Promise((resolve, reject) => { readFile(information.credentialsPath, async (err, credentials) => { @@ -78,15 +87,6 @@ export namespace GoogleApiServerUtils { }); }; - export const RetrieveAccessToken = (information: CredentialInformation) => { - return new Promise((resolve, reject) => { - RetrieveCredentials(information).then( - credentials => resolve(credentials.token.access_token!), - error => reject(`Error: unable to authenticate Google Photos API request.\n${error}`) - ); - }); - }; - export const RetrievePhotosEndpoint = (paths: CredentialInformation) => { return new Promise((resolve, reject) => { RetrieveAccessToken(paths).then( @@ -107,7 +107,7 @@ export namespace GoogleApiServerUtils { client_id, client_secret, redirect_uris[0]); return new Promise((resolve, reject) => { - Database.Auxiliary.FetchGoogleAuthenticationToken(userId).then(token => { + Database.Auxiliary.GoogleAuthenticationToken.Fetch(userId).then(token => { // Check if we have previously stored a token for this userId. if (!token) { return getNewToken(oAuth2Client, userId).then(resolve, reject); @@ -123,8 +123,8 @@ export namespace GoogleApiServerUtils { } const refreshEndpoint = "https://oauth2.googleapis.com/token"; - const refreshToken = (credentials: Credentials, client_id: string, client_secret: string, oAuth2Client: OAuth2Client, token_path: string) => { - return new Promise((resolve, reject) => { + const refreshToken = (credentials: Credentials, client_id: string, client_secret: string, oAuth2Client: OAuth2Client, userId: string) => { + return new Promise(resolve => { let headerParameters = { headers: { 'Content-Type': 'application/x-www-form-urlencoded' } }; let queryParameters = { refreshToken: credentials.refresh_token, @@ -133,19 +133,13 @@ export namespace GoogleApiServerUtils { grant_type: "refresh_token" }; let url = `${refreshEndpoint}?${qs.stringify(queryParameters)}`; - request.post(url, headerParameters).then(response => { + request.post(url, headerParameters).then(async response => { let parsed = JSON.parse(response); credentials.access_token = parsed.access_token; credentials.expiry_date = new Date().getTime() + (parsed.expires_in * 1000); - writeFile(token_path, JSON.stringify(credentials), (err) => { - if (err) { - console.error(err); - reject(err); - } - console.log('Refreshed token stored to', token_path); - oAuth2Client.setCredentials(credentials); - resolve({ token: credentials, client: oAuth2Client }); - }); + oAuth2Client.setCredentials(credentials); + await Database.Auxiliary.GoogleAuthenticationToken.Write(userId, credentials); + resolve({ token: credentials, client: oAuth2Client }); }); }); }; @@ -156,7 +150,7 @@ export namespace GoogleApiServerUtils { * @param {google.auth.OAuth2} oAuth2Client The OAuth2 client to get token for. * @param {getEventsCallback} callback The callback for the authorized client. */ - function getNewToken(oAuth2Client: OAuth2Client, token_path: string) { + function getNewToken(oAuth2Client: OAuth2Client, userId: string) { return new Promise((resolve, reject) => { const authUrl = oAuth2Client.generateAuthUrl({ access_type: 'offline', @@ -169,20 +163,13 @@ export namespace GoogleApiServerUtils { }); rl.question('Enter the code from that page here: ', (code) => { rl.close(); - oAuth2Client.getToken(code, (err, token) => { + oAuth2Client.getToken(code, async (err, token) => { if (err || !token) { reject(err); return console.error('Error retrieving access token', err); } oAuth2Client.setCredentials(token); - // Store the token to disk for later program executions - writeFile(token_path, JSON.stringify(token), (err) => { - if (err) { - console.error(err); - reject(err); - } - console.log('Token stored to', token_path); - }); + await Database.Auxiliary.GoogleAuthenticationToken.Write(userId, token); resolve({ token, client: oAuth2Client }); }); }); diff --git a/src/server/database.ts b/src/server/database.ts index ce29478ad..890ac6b32 100644 --- a/src/server/database.ts +++ b/src/server/database.ts @@ -231,19 +231,37 @@ export namespace Database { const GoogleAuthentication = "GoogleAuthentication"; - const SanitizedSingletonQuery = async (query: { [key: string]: any }, collection: string) => { + const SanitizedCappedQuery = async (query: { [key: string]: any }, collection: string, cap: number) => { const cursor = await Instance.query(query, undefined, collection); - const existing = (await cursor.toArray())[0]; - if (existing) { - delete existing._id; - } - return existing; + const results = await cursor.toArray(); + const slice = results.slice(0, Math.min(cap, results.length)); + return slice.map(result => { + delete result._id; + return result; + }); + }; + + const SanitizedSingletonQuery = async (query: { [key: string]: any }, collection: string) => { + const results = await SanitizedCappedQuery(query, collection, 1); + return results.length ? results[0] : undefined; }; export const QueryUploadHistory = async (contentSize: number): Promise> => { return SanitizedSingletonQuery({ contentSize }, AuxiliaryCollections.GooglePhotosUploadHistory); }; + export namespace GoogleAuthenticationToken { + + export const Fetch = async (userId: string) => { + return SanitizedSingletonQuery({ userId }, GoogleAuthentication); + }; + + export const Write = async (userId: string, token: any) => { + return Instance.insert({ userId, ...token }, GoogleAuthentication); + }; + + } + export const LogUpload = async (information: DashUploadUtils.UploadInformation) => { const bundle = { _id: Utils.GenerateDeterministicGuid(String(information.contentSize!)), @@ -258,10 +276,6 @@ export namespace Database { return Promise.all(pendingDeletions); }; - export const FetchGoogleAuthenticationToken = async (userId: string) => { - return SanitizedSingletonQuery({ userId }, GoogleAuthentication); - }; - } } \ No newline at end of file diff --git a/src/server/index.ts b/src/server/index.ts index 386ecce4d..2ff63ab78 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -810,7 +810,7 @@ const EndpointHandlerMap = new Map { let sector: GoogleApiServerUtils.Service = req.params.sector as GoogleApiServerUtils.Service; let action: GoogleApiServerUtils.Action = req.params.action as GoogleApiServerUtils.Action; - GoogleApiServerUtils.GetEndpoint(GoogleApiServerUtils.Service[sector], { credentialsPath, userId: req.body.userId }).then(endpoint => { + GoogleApiServerUtils.GetEndpoint(GoogleApiServerUtils.Service[sector], { credentialsPath, userId: req.headers.userId as string }).then(endpoint => { let handler = EndpointHandlerMap.get(action); if (endpoint && handler) { let execute = handler(endpoint, req.body).then( @@ -824,10 +824,11 @@ app.post(RouteStore.googleDocs + "/:sector/:action", (req, res) => { }); }); -app.get(RouteStore.googlePhotosAccessToken, (req, res) => GoogleApiServerUtils.RetrieveAccessToken({ credentialsPath, userId: req.body.userId }).then(token => res.send(token))); +app.get(RouteStore.googlePhotosAccessToken, (req, res) => GoogleApiServerUtils.RetrieveAccessToken({ credentialsPath, userId: req.headers.userId as string }).then(token => res.send(token))); const tokenError = "Unable to successfully upload bytes for all images!"; const mediaError = "Unable to convert all uploaded bytes to media items!"; +const userIdError = "Unable to parse the identification of the user!"; export interface NewMediaItem { description: string; @@ -837,7 +838,12 @@ export interface NewMediaItem { } app.post(RouteStore.googlePhotosMediaUpload, async (req, res) => { - const { userId, media } = req.body; + const { media } = req.body; + const { userId } = req.headers; + + if (!userId || Array.isArray(userId)) { + return _error(res, userIdError); + } await GooglePhotosUploadUtils.initialize({ credentialsPath, userId }); -- cgit v1.2.3-70-g09d2 From 20540a35be82c34cc3962de4f957d1aa43f8a2b0 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Thu, 26 Sep 2019 22:17:15 -0400 Subject: finally transferred google credential management to database --- src/client/Network.ts | 46 +++++++++++++--------- .../apis/google_docs/GooglePhotosClientUtils.ts | 17 ++++---- src/client/documents/Documents.ts | 22 ++++++----- .../util/Import & Export/DirectoryImportBox.tsx | 5 +-- src/client/views/nodes/DocumentView.tsx | 7 ++-- src/server/apis/google/GoogleApiServerUtils.ts | 27 +++++++------ src/server/database.ts | 32 +++++++++------ src/server/index.ts | 6 +-- 8 files changed, 93 insertions(+), 69 deletions(-) (limited to 'src/server/apis') diff --git a/src/client/Network.ts b/src/client/Network.ts index cb46105f8..75ccb5e99 100644 --- a/src/client/Network.ts +++ b/src/client/Network.ts @@ -2,24 +2,32 @@ import { Utils } from "../Utils"; import { CurrentUserUtils } from "../server/authentication/models/current_user_utils"; import requestPromise = require('request-promise'); -export async function PostToServer(relativeRoute: string, body?: any) { - let options = { - uri: Utils.prepend(relativeRoute), - method: "POST", - headers: { userId: CurrentUserUtils.id }, - body, - json: true - }; - return requestPromise.post(options); -} +export namespace Identified { + + export async function FetchFromServer(relativeRoute: string) { + return (await fetch(relativeRoute, { headers: { userId: CurrentUserUtils.id } })).text(); + } + + export async function PostToServer(relativeRoute: string, body?: any) { + let options = { + uri: Utils.prepend(relativeRoute), + method: "POST", + headers: { userId: CurrentUserUtils.id }, + body, + json: true + }; + return requestPromise.post(options); + } + + export async function PostFormDataToServer(relativeRoute: string, formData: FormData) { + const parameters = { + method: 'POST', + headers: { userId: CurrentUserUtils.id }, + body: formData, + }; + const response = await fetch(relativeRoute, parameters); + const text = await response.json(); + return text; + } -export async function PostFormDataToServer(relativeRoute: string, formData: FormData) { - const parameters = { - method: 'POST', - headers: { userId: CurrentUserUtils.id }, - body: formData, - }; - const response = await fetch(relativeRoute, parameters); - const text = await response.json(); - return text; } \ No newline at end of file diff --git a/src/client/apis/google_docs/GooglePhotosClientUtils.ts b/src/client/apis/google_docs/GooglePhotosClientUtils.ts index b1b29210a..29cc042b6 100644 --- a/src/client/apis/google_docs/GooglePhotosClientUtils.ts +++ b/src/client/apis/google_docs/GooglePhotosClientUtils.ts @@ -14,11 +14,14 @@ import { NewMediaItemResult, MediaItem } from "../../../server/apis/google/Share import { AssertionError } from "assert"; import { DocumentView } from "../../views/nodes/DocumentView"; import { DocumentManager } from "../../util/DocumentManager"; -import { PostToServer } from "../../Network"; +import { Identified } from "../../Network"; export namespace GooglePhotos { - const endpoint = async () => new Photos(await PostToServer(RouteStore.googlePhotosAccessToken)); + const endpoint = async () => { + const accessToken = await Identified.FetchFromServer(RouteStore.googlePhotosAccessToken); + return new Photos(accessToken); + }; export enum MediaType { ALL_MEDIA = 'ALL_MEDIA', @@ -296,7 +299,7 @@ export namespace GooglePhotos { }; export const WriteMediaItemsToServer = async (body: { mediaItems: any[] }): Promise => { - const uploads = await PostToServer(RouteStore.googlePhotosMediaDownload, body); + const uploads = await Identified.PostToServer(RouteStore.googlePhotosMediaDownload, body); return uploads; }; @@ -316,7 +319,7 @@ export namespace GooglePhotos { album = await Create.Album(album.title); } const media: MediaInput[] = []; - sources.forEach(source => { + for (let source of sources) { const data = Cast(Doc.GetProto(source).data, ImageField); if (!data) { return; @@ -324,11 +327,11 @@ export namespace GooglePhotos { const url = data.url.href; const target = Doc.MakeAlias(source); const description = parseDescription(target, descriptionKey); - DocumentView.makeCustomViewClicked(target, undefined); + await DocumentView.makeCustomViewClicked(target, undefined); media.push({ url, description }); - }); + } if (media.length) { - const uploads: NewMediaItemResult[] = await PostToServer(RouteStore.googlePhotosMediaUpload, { media, album }); + const uploads: NewMediaItemResult[] = await Identified.PostToServer(RouteStore.googlePhotosMediaUpload, { media, album }); return uploads; } }; diff --git a/src/client/documents/Documents.ts b/src/client/documents/Documents.ts index a7a006f47..0369e89a6 100644 --- a/src/client/documents/Documents.ts +++ b/src/client/documents/Documents.ts @@ -337,16 +337,18 @@ export namespace Docs { let extension = path.extname(target); target = `${target.substring(0, target.length - extension.length)}_o${extension}`; } - requestImageSize(target) - .then((size: any) => { - let aspect = size.height / size.width; - if (!inst.proto!.nativeWidth) { - inst.proto!.nativeWidth = size.width; - } - inst.proto!.nativeHeight = Number(inst.proto!.nativeWidth!) * aspect; - inst.proto!.height = NumCast(inst.proto!.width) * aspect; - }) - .catch((err: any) => console.log(err)); + if (target !== "http://www.cs.brown.edu/") { + requestImageSize(target) + .then((size: any) => { + let aspect = size.height / size.width; + if (!inst.proto!.nativeWidth) { + inst.proto!.nativeWidth = size.width; + } + inst.proto!.nativeHeight = Number(inst.proto!.nativeWidth!) * aspect; + inst.proto!.height = NumCast(inst.proto!.width) * aspect; + }) + .catch((err: any) => console.log(err)); + } return inst; } export function PresDocument(initial: List = new List(), options: DocumentOptions = {}) { diff --git a/src/client/util/Import & Export/DirectoryImportBox.tsx b/src/client/util/Import & Export/DirectoryImportBox.tsx index d0291aec4..de5287456 100644 --- a/src/client/util/Import & Export/DirectoryImportBox.tsx +++ b/src/client/util/Import & Export/DirectoryImportBox.tsx @@ -21,7 +21,7 @@ import { GooglePhotos } from "../../apis/google_docs/GooglePhotosClientUtils"; import { SchemaHeaderField } from "../../../new_fields/SchemaHeaderField"; import "./DirectoryImportBox.scss"; import BatchedArray from "array-batcher"; -import { PostFormDataToServer } from "../../Network"; +import { Identified } from "../../Network"; const unsupported = ["text/html", "text/plain"]; interface FileResponse { @@ -114,7 +114,7 @@ export default class DirectoryImportBox extends React.Component formData.append(Utils.GenerateGuid(), file); }); - const responses = await PostFormDataToServer(RouteStore.upload, formData); + const responses = await Identified.PostFormDataToServer(RouteStore.upload, formData); runInAction(() => this.completed += batch.length); return responses as FileResponse[]; }); @@ -279,7 +279,6 @@ export default class DirectoryImportBox extends React.Component }} />