From c9a7d747916c31730f71479d8e516ba0ed2c658f Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Fri, 30 Aug 2019 15:51:13 -0400 Subject: complete transfer except for list items --- package.json | 2 ++ 1 file changed, 2 insertions(+) (limited to 'package.json') diff --git a/package.json b/package.json index cd60b7b55..f98224469 100644 --- a/package.json +++ b/package.json @@ -59,6 +59,7 @@ "@types/bluebird": "^3.5.25", "@types/body-parser": "^1.17.0", "@types/classnames": "^2.2.8", + "@types/color": "^3.0.0", "@types/connect-flash": "0.0.34", "@types/cookie-parser": "^1.4.1", "@types/cookie-session": "^2.0.36", @@ -121,6 +122,7 @@ "canvas": "^2.5.0", "child_process": "^1.0.2", "class-transformer": "^0.2.0", + "color": "^3.1.2", "connect-flash": "^0.1.1", "connect-mongo": "^2.0.3", "cookie-parser": "^1.4.4", -- cgit v1.2.3-70-g09d2 From 77e08a4362ba8fab4cab361fcb472702c97edf15 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Sun, 1 Sep 2019 13:05:54 -0400 Subject: initial commit --- package.json | 3 + src/client/views/MainView.tsx | 4 +- src/server/apis/google/GoogleApiServerUtils.ts | 13 +- src/server/apis/google/GooglePhotosUtils.ts | 12 + src/server/authentication/config/passport.ts | 18 +- src/server/credentials/auth.json | 12 + .../credentials/google_docs_credentials.json | 12 +- src/server/credentials/google_docs_token.json | 2 +- .../credentials/google_photos_credentials.ts | 35 ++ src/server/index.ts | 27 + src/typings/index.d.ts | 632 +++++++++++---------- 11 files changed, 447 insertions(+), 323 deletions(-) create mode 100644 src/server/apis/google/GooglePhotosUtils.ts create mode 100644 src/server/credentials/auth.json create mode 100644 src/server/credentials/google_photos_credentials.ts (limited to 'package.json') diff --git a/package.json b/package.json index f98224469..f0f2b467e 100644 --- a/package.json +++ b/package.json @@ -81,6 +81,7 @@ "@types/node": "^10.12.30", "@types/nodemailer": "^4.6.6", "@types/passport": "^1.0.0", + "@types/passport-google-oauth20": "^2.0.2", "@types/passport-local": "^1.0.33", "@types/pdfjs-dist": "^2.0.0", "@types/prosemirror-commands": "^1.0.1", @@ -141,6 +142,7 @@ "golden-layout": "^1.5.9", "google-auth-library": "^4.2.4", "googleapis": "^40.0.0", + "googlephotos": "^0.2.1", "howler": "^2.1.2", "html-to-image": "^0.1.0", "i": "^0.3.6", @@ -166,6 +168,7 @@ "npm": "^6.10.3", "p-limit": "^2.2.0", "passport": "^0.4.0", + "passport-google-oauth20": "^2.0.0", "passport-local": "^1.0.0", "pdfjs-dist": "^2.0.943", "probe-image-size": "^4.0.0", diff --git a/src/client/views/MainView.tsx b/src/client/views/MainView.tsx index ab87f0c7b..df0718072 100644 --- a/src/client/views/MainView.tsx +++ b/src/client/views/MainView.tsx @@ -15,7 +15,7 @@ import { listSpec } from '../../new_fields/Schema'; import { BoolCast, Cast, FieldValue, StrCast } from '../../new_fields/Types'; import { CurrentUserUtils } from '../../server/authentication/models/current_user_utils'; import { RouteStore } from '../../server/RouteStore'; -import { emptyFunction, returnOne, returnTrue, Utils, returnEmptyString } from '../../Utils'; +import { emptyFunction, returnOne, returnTrue, Utils, returnEmptyString, PostToServer } from '../../Utils'; import { DocServer } from '../DocServer'; import { Docs } from '../documents/Documents'; import { ClientUtils } from '../util/ClientUtils'; @@ -128,6 +128,8 @@ export class MainView extends React.Component { window.removeEventListener("keydown", KeyManager.Instance.handle); window.addEventListener("keydown", KeyManager.Instance.handle); + PostToServer('/googleDocs/Photos/Test', {}); + reaction(() => { let workspaces = CurrentUserUtils.UserDocument.workspaces; let recent = CurrentUserUtils.UserDocument.recentlyClosed; diff --git a/src/server/apis/google/GoogleApiServerUtils.ts b/src/server/apis/google/GoogleApiServerUtils.ts index 8785cd974..2fb44d9a2 100644 --- a/src/server/apis/google/GoogleApiServerUtils.ts +++ b/src/server/apis/google/GoogleApiServerUtils.ts @@ -5,6 +5,7 @@ import { OAuth2Client } from "google-auth-library"; import { Opt } from "../../../new_fields/Doc"; import { GlobalOptions } from "googleapis-common"; import { GaxiosResponse } from "gaxios"; +import Photos = require("googlephotos"); /** * Server side authentication for Google Api queries. @@ -20,16 +21,18 @@ export namespace GoogleApiServerUtils { 'presentations.readonly', 'drive', 'drive.file', + 'photoslibrary', + 'photoslibrary.sharing' ]; export const parseBuffer = (data: Buffer) => JSON.parse(data.toString()); export enum Service { Documents = "Documents", - Slides = "Slides" + Slides = "Slides", + Photos = "Photos" } - export interface CredentialPaths { credentials: string; token: string; @@ -50,7 +53,7 @@ export namespace GoogleApiServerUtils { reject(err); return console.log('Error loading client secret file:', err); } - return authorize(parseBuffer(credentials), paths.token).then(auth => { + return authorize(parseBuffer(credentials), paths.token).then(async auth => { let routed: Opt; let parameters: EndpointParameters = { auth, version: "v1" }; switch (sector) { @@ -60,6 +63,10 @@ export namespace GoogleApiServerUtils { case Service.Slides: routed = google.slides(parameters).presentations; break; + case Service.Photos: + const photos = new Photos(auth); + let response = await photos.albums.list(); + console.log("WE GOT SOMETHING!", response); } resolve(routed); }); diff --git a/src/server/apis/google/GooglePhotosUtils.ts b/src/server/apis/google/GooglePhotosUtils.ts new file mode 100644 index 000000000..c33ad2dd9 --- /dev/null +++ b/src/server/apis/google/GooglePhotosUtils.ts @@ -0,0 +1,12 @@ +import request = require('request-promise'); +const key = require("../../credentials/auth.json"); + +export const PhotosLibraryQuery = async (authToken: any, parameters: any) => { + let options = { + headers: { 'Content-Type': 'application/json' }, + json: parameters, + auth: { 'bearer': authToken }, + }; + const result = await request.post(config.apiEndpoint + '/v1/mediaItems:search', options); + return result; +}; \ No newline at end of file diff --git a/src/server/authentication/config/passport.ts b/src/server/authentication/config/passport.ts index d42741410..97ded8785 100644 --- a/src/server/authentication/config/passport.ts +++ b/src/server/authentication/config/passport.ts @@ -1,12 +1,14 @@ import * as passport from 'passport'; import * as passportLocal from 'passport-local'; -import * as mongodb from 'mongodb'; -import * as _ from "lodash"; +import _ from "lodash"; import { default as User } from '../models/user_model'; import { Request, Response, NextFunction } from "express"; import { RouteStore } from '../../RouteStore'; +import * as GoogleOAuth from "passport-google-oauth20"; +const config = require("../../credentials/google_photos_credentials"); const LocalStrategy = passportLocal.Strategy; +const GoogleOAuthStrategy = GoogleOAuth.Strategy; passport.serializeUser((user, done) => { done(undefined, user.id); @@ -32,6 +34,18 @@ passport.use(new LocalStrategy({ usernameField: 'email', passReqToCallback: true }); })); + +passport.use(new GoogleOAuthStrategy( + { + clientID: config.oAuthClientID, + clientSecret: config.oAuthclientSecret, + callbackURL: config.oAuthCallbackUrl, + // Set the correct profile URL that does not require any additional APIs + userProfileURL: 'https://www.googleapis.com/oauth2/v3/userinfo' + }, + (token, refreshToken, profile, done) => done(undefined, { profile, token }) +)); + export let isAuthenticated = (req: Request, res: Response, next: NextFunction) => { if (req.isAuthenticated()) { return next(); diff --git a/src/server/credentials/auth.json b/src/server/credentials/auth.json new file mode 100644 index 000000000..557eca4b6 --- /dev/null +++ b/src/server/credentials/auth.json @@ -0,0 +1,12 @@ +{ + "type": "service_account", + "project_id": "brown-dash", + "private_key_id": "ddf0473a9ac56956b5818e04a7ee406a64d5b0a6", + "private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCueRfxic2oL9nr\nnWSLgl7XR/BKikm4p2sib6szaoTO+q6itcJgt2TDleK/7Y4KW/KhvCfhWVet0Hz0\nIDyg4N/gc2yxuDA6/m8DPWU9kDj8VFR7LVFawOKgo1WbgLcC0Qu8qHzAffrlg8si\nhj3vGuoS/YDn/mz0krwFmCfIx+S0lJ9a7FUjJL5C+CIwAEEYiU7xnTW7pVVNXAm/\n/YKD17ToAjREOtlfVVYO7tZ7V5BiW0I0jJvxw+t1pgrZZe7WPBSBJg9KKGIl+mRi\ndtUMR9Hyt3nMKNZIrSm0OkAz82HxfcapRdSB3wkVjoyW63YaTVHKoBOqRElfMtoM\nqu8wbhhNAgMBAAECggEAA41wJ8kg8J8peQMZ/b7gZvzuPy+h0M/J3j2MrG3rY9qA\nrUv1oqoBSXvhuNDhtEN12oYIqtg6m+L+Sas8CMuOC2rWPafM2u/80IGoGtDhtCjp\nv8inBX8ew4YSiL7IxdbTU2/70Es7DVV5u0t6ndsmr88ibYwwPupGR/fhPCpDyssg\n7lFAEpOwnbKG9a7E7axHpXBRSIE54sh+ESyf6MHH/oyKOLhZ0v4PjRDKaKuMDRst\nMOClgNjD/4bzKpfWljuPYemXz1oIBQitBW5aXnCdsmdrmOLDQpz3qOgIo+RRiyki\nvVU5N54L65sj4WisLt1TT45wbhrkQUz+8GmhV5rHwQKBgQDow32/gSb/M0BKFk5y\n+pSeoLkYp/dwfBFWYT6CNBaKePARFVCdr3db8yOEQD4hrmTOU0EP9c4FSLcaa0Xy\n7n2crhhfZWFpIpRMyXhpeKpqdjQFimfBOK6cIdjnWrtJ6Ik3m4E9p7kKThIsInTc\n/TETwAyzFN+J2ADRdrUdCukOwQKBgQC/4+1Rk8a++Jr9Sznx+JH4vj/J2cGsu7uQ\n0nikcOAFO5HzG7+mt9Wv9/MiPtEYwmc7YziDXldKLpshT2m6wrS1uzzOXAnvXFAh\npiCXQsmVA3gmrVd53k+eZfqrZ1n/rL1kCewRS5LX8xIhM28VIkGqkVy4ZEifMotG\nZKSbH0b4jQKBgQCsJ6rh8Uw+hFGQel8be2pgyM8eBV1lvN213ca11oC1ei1U9Ubi\n2dyWDYa/UiSiFLJKSBlfDJaMIfQLfjwGKY6OS9WK+RjLAeBdysVcfPrOMw7W6j9D\nEgFTSVV8CAdt6qdSkZlNWLfrf0LBkdqNeFbMHMdHzLBo63HverUJ/f/SAQKBgHIk\n2t5T0T14FHnnbaiJ/ArC4J7pcVOWuJQFHs5ydk+mh8LdFrvNTsdF7tLIGwlnWpDx\nDITYcYQnBRBjdLkraONRZXI7PY2sk93wPCK+D7scPTSEmCxeGW5XqyyaZea4klAX\nttzy336lkHs/ZSxlHDqiDU2CGdDY+A//fgroKAdhAoGAA5FXfMzTQLGqxg4J2B1z\nFEXNbrqZZFGgKiveUhhZLm4zPiHXtZXvDtSLwGgcO8oGfTfYueTcHb/Eiar7mKv+\n+SqpAqkINJTthIFVIiD39S9jPFUXzBkf5ZJKPLKQArhzEGxen+SD6ZUO058fA94L\n9FblRGlMtr2o5z0NC7H5zaU=\n-----END PRIVATE KEY-----\n", + "client_email": "google-photos-api@brown-dash.iam.gserviceaccount.com", + "client_id": "112995422877175743408", + "auth_uri": "https://accounts.google.com/o/oauth2/auth", + "token_uri": "https://oauth2.googleapis.com/token", + "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", + "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/google-photos-api%40brown-dash.iam.gserviceaccount.com" +} \ No newline at end of file diff --git a/src/server/credentials/google_docs_credentials.json b/src/server/credentials/google_docs_credentials.json index 8d097d363..955c5a3c1 100644 --- a/src/server/credentials/google_docs_credentials.json +++ b/src/server/credentials/google_docs_credentials.json @@ -1 +1,11 @@ -{"installed":{"client_id":"343179513178-ud6tvmh275r2fq93u9eesrnc66t6akh9.apps.googleusercontent.com","project_id":"quickstart-1565056383187","auth_uri":"https://accounts.google.com/o/oauth2/auth","token_uri":"https://oauth2.googleapis.com/token","auth_provider_x509_cert_url":"https://www.googleapis.com/oauth2/v1/certs","client_secret":"w8KIFSc0MQpmUYHed4qEzn8b","redirect_uris":["urn:ietf:wg:oauth:2.0:oob","http://localhost"]}} \ No newline at end of file +{ + "installed": { + "client_id": "343179513178-ud6tvmh275r2fq93u9eesrnc66t6akh9.apps.googleusercontent.com", + "project_id": "quickstart-1565056383187", + "auth_uri": "https://accounts.google.com/o/oauth2/auth", + "token_uri": "https://oauth2.googleapis.com/token", + "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", + "client_secret": "w8KIFSc0MQpmUYHed4qEzn8b", + "redirect_uris": ["urn:ietf:wg:oauth:2.0:oob", "http://localhost"] + } +} \ No newline at end of file diff --git a/src/server/credentials/google_docs_token.json b/src/server/credentials/google_docs_token.json index 07c02d56c..cea452f08 100644 --- a/src/server/credentials/google_docs_token.json +++ b/src/server/credentials/google_docs_token.json @@ -1 +1 @@ -{"access_token":"ya29.GltjB4-x03xFpd2NY2555cxg1xlT_ajqRi78M9osOfdOF2jTIjlPkn_UZL8cUwVP0DPC8rH3vhhg8RpspFe8Vewx92shAO3RPos_uMH0CUqEiCiZlaaB5I3Jq3Mv","refresh_token":"1/teUKUqGKMLjVqs-eed0L8omI02pzSxMUYaxGc2QxBw0","scope":"https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents.readonly","token_type":"Bearer","expiry_date":1565654175862} \ No newline at end of file +{"access_token":"ya29.Glt2B3lsrpWxZ9DMg1RcTksAFzfR8dVWhf7d7tAvbJ4UbcSVO0Q3aYNGtaMKPtmxR24rH88iQSiKCL8S328TQFEN6LtZgvizymednK5EW0jNCvG6ecdZQ-vwcypR","refresh_token":"1/6X3oGYz4A0p8UW2IgsZ-GqTgQUY43S6Ahsaf_GQhSs8","scope":"https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/documents.readonly https://www.googleapis.com/auth/photoslibrary.sharing https://www.googleapis.com/auth/photoslibrary","token_type":"Bearer","expiry_date":1567360444627} \ No newline at end of file diff --git a/src/server/credentials/google_photos_credentials.ts b/src/server/credentials/google_photos_credentials.ts new file mode 100644 index 000000000..11c1c766c --- /dev/null +++ b/src/server/credentials/google_photos_credentials.ts @@ -0,0 +1,35 @@ +const config: any = {}; + +// The OAuth client ID from the Google Developers console. +config.oAuthClientID = '1005546247619-l40012sl4idpq17b5emielcs1delffog.apps.googleusercontent.com'; + +// The OAuth client secret from the Google Developers console. +config.oAuthclientSecret = 'xEUJ0OBvhlCKA6SLt8TvWBs3'; + +// The callback to use for OAuth requests. This is the URL where the app is +// running. For testing and running it locally, use 127.0.0.1. +config.oAuthCallbackUrl = 'http://localhost:1050/auth/google/callback'; + +// The port where the app should listen for requests. +config.port = 1050; + +// The scopes to request. The app requires the photoslibrary.readonly and +// plus.me scopes. +config.scopes = [ + 'https://www.googleapis.com/auth/photoslibrary.readonly', + 'profile', +]; + +// The number of photos to load for search requests. +config.photosToLoad = 150; + +// The page size to use for search requests. 100 is reccommended. +config.searchPageSize = 100; + +// The page size to use for the listing albums request. 50 is reccommended. +config.albumPageSize = 50; + +// The API end point to use. Do not change. +config.apiEndpoint = 'https://photoslibrary.googleapis.com'; + +module.exports = config; \ No newline at end of file diff --git a/src/server/index.ts b/src/server/index.ts index 34a0a19f1..6105dedcc 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -29,6 +29,7 @@ import { RouteStore } from './RouteStore'; import v4 = require('uuid/v4'); const app = express(); const config = require('../../webpack.config'); +const OAuthConfig = require('../server/credentials/google_photos_credentials'); import { createCanvas, loadImage, Canvas } from "canvas"; const compiler = webpack(config); const port = 1050; // default port to listen @@ -46,6 +47,7 @@ import { GaxiosResponse } from 'gaxios'; import { Opt } from '../new_fields/Doc'; import { docs_v1 } from 'googleapis'; import { Endpoint } from 'googleapis-common'; +import { PhotosLibraryQuery } from './apis/google/GooglePhotosUtils'; const MongoStore = require('connect-mongo')(session); const mongoose = require('mongoose'); const probe = require("probe-image-size"); @@ -196,6 +198,31 @@ const solrURL = "http://localhost:8983/solr/#/dash"; // GETTERS +// app.get('/auth/google', passport.authenticate('google', { +// scope: OAuthConfig.scopes, +// failureFlash: true, // Display errors to the user. +// session: true, +// })); + +// app.get("/failed", (req, res) => res.send("DIDN'T WORK!")); + +// app.get( +// '/auth/google/callback', +// passport.authenticate( +// 'google', { failureRedirect: '/failed', failureFlash: true, session: true }), +// (req, res) => { +// // User has logged in. +// console.log('OAUTH: user has logged in 1.'); +// PhotosLibraryQuery(req.user.token, {}); +// console.log('OAUTH: user has logged in 2.'); +// res.redirect('/'); +// }); + +// app.get('/GooglePhotos', (req, res) => { +// console.log("WORKING ON GOOGLE PHOTOS"); +// PhotosLibraryQuery(req.user.token, {}); +// }); + app.get("/search", async (req, res) => { const solrQuery: any = {}; ["q", "fq", "start", "rows", "hl", "hl.fl"].forEach(key => solrQuery[key] = req.query[key]); diff --git a/src/typings/index.d.ts b/src/typings/index.d.ts index 7939ae8be..36d828fdb 100644 --- a/src/typings/index.d.ts +++ b/src/typings/index.d.ts @@ -1,322 +1,324 @@ /// -declare module '@react-pdf/renderer' { - import * as React from 'react'; - - namespace ReactPDF { - interface Style { - [property: string]: any; - } - interface Styles { - [key: string]: Style; - } - type Orientation = 'portrait' | 'landscape'; - - interface DocumentProps { - title?: string; - author?: string; - subject?: string; - keywords?: string; - creator?: string; - producer?: string; - onRender?: () => any; - } - - /** - * This component represent the PDF document itself. It must be the root - * of your tree element structure, and under no circumstances should it be - * used as children of another react-pdf component. In addition, it should - * only have childs of type . - */ - class Document extends React.Component { } - - interface NodeProps { - style?: Style | Style[]; - /** - * Render component in all wrapped pages. - * @see https://react-pdf.org/advanced#fixed-components - */ - fixed?: boolean; - /** - * Force the wrapping algorithm to start a new page when rendering the - * element. - * @see https://react-pdf.org/advanced#page-breaks - */ - break?: boolean; - } - - interface PageProps extends NodeProps { - /** - * Enable page wrapping for this page. - * @see https://react-pdf.org/components#page-wrapping - */ - wrap?: boolean; - debug?: boolean; - size?: string | [number, number] | { width: number; height: number }; - orientation?: Orientation; - ruler?: boolean; - rulerSteps?: number; - verticalRuler?: boolean; - verticalRulerSteps?: number; - horizontalRuler?: boolean; - horizontalRulerSteps?: number; - ref?: Page; - } - - /** - * Represents single page inside the PDF document, or a subset of them if - * using the wrapping feature. A can contain as many pages as - * you want, but ensure not rendering a page inside any component besides - * Document. - */ - class Page extends React.Component { } - - interface ViewProps extends NodeProps { - /** - * Enable/disable page wrapping for element. - * @see https://react-pdf.org/components#page-wrapping - */ - wrap?: boolean; - debug?: boolean; - render?: (props: { pageNumber: number }) => React.ReactNode; - children?: React.ReactNode; - } - - /** - * The most fundamental component for building a UI and is designed to be - * nested inside other views and can have 0 to many children. - */ - class View extends React.Component { } - - interface ImageProps extends NodeProps { - debug?: boolean; - src: string | { data: Buffer; format: 'png' | 'jpg' }; - cache?: boolean; - } - - /** - * A React component for displaying network or local (Node only) JPG or - * PNG images, as well as base64 encoded image strings. - */ - class Image extends React.Component { } - - interface TextProps extends NodeProps { - /** - * Enable/disable page wrapping for element. - * @see https://react-pdf.org/components#page-wrapping - */ - wrap?: boolean; - debug?: boolean; - render?: ( - props: { pageNumber: number; totalPages: number }, - ) => React.ReactNode; - children?: React.ReactNode; - /** - * How much hyphenated breaks should be avoided. - */ - hyphenationCallback?: number; - } - - /** - * A React component for displaying text. Text supports nesting of other - * Text or Link components to create inline styling. - */ - class Text extends React.Component { } - - interface LinkProps extends NodeProps { - /** - * Enable/disable page wrapping for element. - * @see https://react-pdf.org/components#page-wrapping - */ - wrap?: boolean; - debug?: boolean; - src: string; - children?: React.ReactNode; - } +declare module 'googlephotos'; - /** - * A React component for displaying an hyperlink. Link’s can be nested - * inside a Text component, or being inside any other valid primitive. - */ - class Link extends React.Component { } - - interface NoteProps extends NodeProps { - children: string; - } - - class Note extends React.Component { } - - interface BlobProviderParams { - blob: Blob | null; - url: string | null; - loading: boolean; - error: Error | null; - } - interface BlobProviderProps { - document: React.ReactElement; - children: (params: BlobProviderParams) => React.ReactNode; - } - - /** - * Easy and declarative way of getting document's blob data without - * showing it on screen. - * @see https://react-pdf.org/advanced#on-the-fly-rendering - * @platform web - */ - class BlobProvider extends React.Component { } - - interface PDFViewerProps { - width?: number; - height?: number; - style?: Style | Style[]; - className?: string; - children?: React.ReactElement; - } - - /** - * Iframe PDF viewer for client-side generated documents. - * @platform web - */ - class PDFViewer extends React.Component { } - - interface PDFDownloadLinkProps { - document: React.ReactElement; - fileName?: string; - style?: Style | Style[]; - className?: string; - children?: - | React.ReactNode - | ((params: BlobProviderParams) => React.ReactNode); +declare module '@react-pdf/renderer' { + import * as React from 'react'; + + namespace ReactPDF { + interface Style { + [property: string]: any; + } + interface Styles { + [key: string]: Style; + } + type Orientation = 'portrait' | 'landscape'; + + interface DocumentProps { + title?: string; + author?: string; + subject?: string; + keywords?: string; + creator?: string; + producer?: string; + onRender?: () => any; + } + + /** + * This component represent the PDF document itself. It must be the root + * of your tree element structure, and under no circumstances should it be + * used as children of another react-pdf component. In addition, it should + * only have childs of type . + */ + class Document extends React.Component { } + + interface NodeProps { + style?: Style | Style[]; + /** + * Render component in all wrapped pages. + * @see https://react-pdf.org/advanced#fixed-components + */ + fixed?: boolean; + /** + * Force the wrapping algorithm to start a new page when rendering the + * element. + * @see https://react-pdf.org/advanced#page-breaks + */ + break?: boolean; + } + + interface PageProps extends NodeProps { + /** + * Enable page wrapping for this page. + * @see https://react-pdf.org/components#page-wrapping + */ + wrap?: boolean; + debug?: boolean; + size?: string | [number, number] | { width: number; height: number }; + orientation?: Orientation; + ruler?: boolean; + rulerSteps?: number; + verticalRuler?: boolean; + verticalRulerSteps?: number; + horizontalRuler?: boolean; + horizontalRulerSteps?: number; + ref?: Page; + } + + /** + * Represents single page inside the PDF document, or a subset of them if + * using the wrapping feature. A can contain as many pages as + * you want, but ensure not rendering a page inside any component besides + * Document. + */ + class Page extends React.Component { } + + interface ViewProps extends NodeProps { + /** + * Enable/disable page wrapping for element. + * @see https://react-pdf.org/components#page-wrapping + */ + wrap?: boolean; + debug?: boolean; + render?: (props: { pageNumber: number }) => React.ReactNode; + children?: React.ReactNode; + } + + /** + * The most fundamental component for building a UI and is designed to be + * nested inside other views and can have 0 to many children. + */ + class View extends React.Component { } + + interface ImageProps extends NodeProps { + debug?: boolean; + src: string | { data: Buffer; format: 'png' | 'jpg' }; + cache?: boolean; + } + + /** + * A React component for displaying network or local (Node only) JPG or + * PNG images, as well as base64 encoded image strings. + */ + class Image extends React.Component { } + + interface TextProps extends NodeProps { + /** + * Enable/disable page wrapping for element. + * @see https://react-pdf.org/components#page-wrapping + */ + wrap?: boolean; + debug?: boolean; + render?: ( + props: { pageNumber: number; totalPages: number }, + ) => React.ReactNode; + children?: React.ReactNode; + /** + * How much hyphenated breaks should be avoided. + */ + hyphenationCallback?: number; + } + + /** + * A React component for displaying text. Text supports nesting of other + * Text or Link components to create inline styling. + */ + class Text extends React.Component { } + + interface LinkProps extends NodeProps { + /** + * Enable/disable page wrapping for element. + * @see https://react-pdf.org/components#page-wrapping + */ + wrap?: boolean; + debug?: boolean; + src: string; + children?: React.ReactNode; + } + + /** + * A React component for displaying an hyperlink. Link’s can be nested + * inside a Text component, or being inside any other valid primitive. + */ + class Link extends React.Component { } + + interface NoteProps extends NodeProps { + children: string; + } + + class Note extends React.Component { } + + interface BlobProviderParams { + blob: Blob | null; + url: string | null; + loading: boolean; + error: Error | null; + } + interface BlobProviderProps { + document: React.ReactElement; + children: (params: BlobProviderParams) => React.ReactNode; + } + + /** + * Easy and declarative way of getting document's blob data without + * showing it on screen. + * @see https://react-pdf.org/advanced#on-the-fly-rendering + * @platform web + */ + class BlobProvider extends React.Component { } + + interface PDFViewerProps { + width?: number; + height?: number; + style?: Style | Style[]; + className?: string; + children?: React.ReactElement; + } + + /** + * Iframe PDF viewer for client-side generated documents. + * @platform web + */ + class PDFViewer extends React.Component { } + + interface PDFDownloadLinkProps { + document: React.ReactElement; + fileName?: string; + style?: Style | Style[]; + className?: string; + children?: + | React.ReactNode + | ((params: BlobProviderParams) => React.ReactNode); + } + + /** + * Anchor tag to enable generate and download PDF documents on the fly. + * @see https://react-pdf.org/advanced#on-the-fly-rendering + * @platform web + */ + class PDFDownloadLink extends React.Component { } + + interface EmojiSource { + url: string; + format: string; + } + interface RegisteredFont { + src: string; + loaded: boolean; + loading: boolean; + data: any; + [key: string]: any; + } + type HyphenationCallback = ( + words: string[], + glyphString: { [key: string]: any }, + ) => string[]; + + const Font: { + register: ( + src: string, + options: { family: string;[key: string]: any }, + ) => void; + getEmojiSource: () => EmojiSource; + getRegisteredFonts: () => string[]; + registerEmojiSource: (emojiSource: EmojiSource) => void; + registerHyphenationCallback: ( + hyphenationCallback: HyphenationCallback, + ) => void; + getHyphenationCallback: () => HyphenationCallback; + getFont: (fontFamily: string) => RegisteredFont | undefined; + load: ( + fontFamily: string, + document: React.ReactElement, + ) => Promise; + clear: () => void; + reset: () => void; + }; + + const StyleSheet: { + hairlineWidth: number; + create: (styles: TStyles) => TStyles; + resolve: ( + style: Style, + container: { + width: number; + height: number; + orientation: Orientation; + }, + ) => Style; + flatten: (...styles: Style[]) => Style; + absoluteFillObject: { + position: 'absolute'; + left: 0; + right: 0; + top: 0; + bottom: 0; + }; + }; + + const version: any; + + const PDFRenderer: any; + + const createInstance: ( + element: { + type: string; + props: { [key: string]: any }; + }, + root?: any, + ) => any; + + const pdf: ( + document: React.ReactElement, + ) => { + isDirty: () => boolean; + updateContainer: (document: React.ReactElement) => void; + toBuffer: () => NodeJS.ReadableStream; + toBlob: () => Blob; + toString: () => string; + }; + + const renderToStream: ( + document: React.ReactElement, + ) => NodeJS.ReadableStream; + + const renderToFile: ( + document: React.ReactElement, + filePath: string, + callback?: (output: NodeJS.ReadableStream, filePath: string) => any, + ) => Promise; + + const render: typeof renderToFile; } - /** - * Anchor tag to enable generate and download PDF documents on the fly. - * @see https://react-pdf.org/advanced#on-the-fly-rendering - * @platform web - */ - class PDFDownloadLink extends React.Component { } - - interface EmojiSource { - url: string; - format: string; - } - interface RegisteredFont { - src: string; - loaded: boolean; - loading: boolean; - data: any; - [key: string]: any; - } - type HyphenationCallback = ( - words: string[], - glyphString: { [key: string]: any }, - ) => string[]; - - const Font: { - register: ( - src: string, - options: { family: string;[key: string]: any }, - ) => void; - getEmojiSource: () => EmojiSource; - getRegisteredFonts: () => string[]; - registerEmojiSource: (emojiSource: EmojiSource) => void; - registerHyphenationCallback: ( - hyphenationCallback: HyphenationCallback, - ) => void; - getHyphenationCallback: () => HyphenationCallback; - getFont: (fontFamily: string) => RegisteredFont | undefined; - load: ( - fontFamily: string, - document: React.ReactElement, - ) => Promise; - clear: () => void; - reset: () => void; + const Document: typeof ReactPDF.Document; + const Page: typeof ReactPDF.Page; + const View: typeof ReactPDF.View; + const Image: typeof ReactPDF.Image; + const Text: typeof ReactPDF.Text; + const Link: typeof ReactPDF.Link; + const Note: typeof ReactPDF.Note; + const Font: typeof ReactPDF.Font; + const StyleSheet: typeof ReactPDF.StyleSheet; + const createInstance: typeof ReactPDF.createInstance; + const PDFRenderer: typeof ReactPDF.PDFRenderer; + const version: typeof ReactPDF.version; + const pdf: typeof ReactPDF.pdf; + + export default ReactPDF; + export { + Document, + Page, + View, + Image, + Text, + Link, + Note, + Font, + StyleSheet, + createInstance, + PDFRenderer, + version, + pdf, }; - - const StyleSheet: { - hairlineWidth: number; - create: (styles: TStyles) => TStyles; - resolve: ( - style: Style, - container: { - width: number; - height: number; - orientation: Orientation; - }, - ) => Style; - flatten: (...styles: Style[]) => Style; - absoluteFillObject: { - position: 'absolute'; - left: 0; - right: 0; - top: 0; - bottom: 0; - }; - }; - - const version: any; - - const PDFRenderer: any; - - const createInstance: ( - element: { - type: string; - props: { [key: string]: any }; - }, - root?: any, - ) => any; - - const pdf: ( - document: React.ReactElement, - ) => { - isDirty: () => boolean; - updateContainer: (document: React.ReactElement) => void; - toBuffer: () => NodeJS.ReadableStream; - toBlob: () => Blob; - toString: () => string; - }; - - const renderToStream: ( - document: React.ReactElement, - ) => NodeJS.ReadableStream; - - const renderToFile: ( - document: React.ReactElement, - filePath: string, - callback?: (output: NodeJS.ReadableStream, filePath: string) => any, - ) => Promise; - - const render: typeof renderToFile; - } - - const Document: typeof ReactPDF.Document; - const Page: typeof ReactPDF.Page; - const View: typeof ReactPDF.View; - const Image: typeof ReactPDF.Image; - const Text: typeof ReactPDF.Text; - const Link: typeof ReactPDF.Link; - const Note: typeof ReactPDF.Note; - const Font: typeof ReactPDF.Font; - const StyleSheet: typeof ReactPDF.StyleSheet; - const createInstance: typeof ReactPDF.createInstance; - const PDFRenderer: typeof ReactPDF.PDFRenderer; - const version: typeof ReactPDF.version; - const pdf: typeof ReactPDF.pdf; - - export default ReactPDF; - export { - Document, - Page, - View, - Image, - Text, - Link, - Note, - Font, - StyleSheet, - createInstance, - PDFRenderer, - version, - pdf, - }; } \ No newline at end of file -- cgit v1.2.3-70-g09d2 From 5d59e9a379417140e10778cd43e8f87ecb816c37 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Fri, 6 Sep 2019 06:29:26 -0400 Subject: lightly tested functional export of any image doc (web url or stored in server public folder) to google photos, optionally stored in a given target album --- package.json | 2 +- .../apis/google_docs/GooglePhotosClientUtils.ts | 73 +++++----- src/client/views/MainView.tsx | 17 ++- src/client/views/nodes/FormattedTextBox.tsx | 1 + src/server/RouteStore.ts | 2 +- src/server/apis/google/GoogleApiServerUtils.ts | 28 ++-- src/server/apis/google/GooglePhotosServerUtils.ts | 68 ---------- src/server/apis/google/GooglePhotosUploadUtils.ts | 122 +++++++++++++++-- src/server/apis/google/typings/albums.ts | 150 --------------------- src/server/credentials/google_docs_token.json | 2 +- src/server/index.ts | 75 +++++------ 11 files changed, 206 insertions(+), 334 deletions(-) delete mode 100644 src/server/apis/google/GooglePhotosServerUtils.ts delete mode 100644 src/server/apis/google/typings/albums.ts (limited to 'package.json') diff --git a/package.json b/package.json index f0f2b467e..f56e34ce0 100644 --- a/package.json +++ b/package.json @@ -224,4 +224,4 @@ "xoauth2": "^1.2.0", "youtube": "^0.1.0" } -} +} \ No newline at end of file diff --git a/src/client/apis/google_docs/GooglePhotosClientUtils.ts b/src/client/apis/google_docs/GooglePhotosClientUtils.ts index b95cc98c9..2b72800a9 100644 --- a/src/client/apis/google_docs/GooglePhotosClientUtils.ts +++ b/src/client/apis/google_docs/GooglePhotosClientUtils.ts @@ -1,53 +1,42 @@ -import { Album } from "../../../server/apis/google/typings/albums"; -import { PostToServer } from "../../../Utils"; +import { PostToServer, Utils } from "../../../Utils"; import { RouteStore } from "../../../server/RouteStore"; import { ImageField } from "../../../new_fields/URLField"; +import { StrCast, Cast } from "../../../new_fields/Types"; +import { Doc, Opt } from "../../../new_fields/Doc"; +import { Id } from "../../../new_fields/FieldSymbols"; +import requestImageSize = require('../../util/request-image-size'); +import Photos = require('googlephotos'); export namespace GooglePhotosClientUtils { - export const Create = async (title: string) => { - let parameters = { - action: Album.Action.Create, - body: { album: { title } } - } as Album.Create; - return PostToServer(RouteStore.googlePhotosQuery, parameters); - }; + export type AlbumReference = { id: string } | { title: string }; + export const endpoint = () => fetch(Utils.prepend(RouteStore.googlePhotosAccessToken)).then(async response => new Photos(await response.text())); - export const List = async (options?: Partial) => { - let parameters = { - action: Album.Action.List, - parameters: { - pageSize: (options ? options.pageSize : 20) || 20, - pageToken: (options ? options.pageToken : undefined) || undefined, - excludeNonAppCreatedData: (options ? options.excludeNonAppCreatedData : false) || false, - } as Album.ListOptions - } as Album.List; - return PostToServer(RouteStore.googlePhotosQuery, parameters); - }; + export interface MediaInput { + description: string; + source: string; + } - export const Get = async (albumId: string) => { - let parameters = { - action: Album.Action.Get, - albumId - } as Album.Get; - return PostToServer(RouteStore.googlePhotosQuery, parameters); - }; - - export const toDataURL = (field: ImageField | undefined) => { - if (!field) { - return undefined; + export const UploadMedia = async (sources: Doc[], album?: AlbumReference) => { + if (album && "title" in album) { + album = (await endpoint()).albums.create(album.title); + } + const media: MediaInput[] = []; + sources.forEach(document => { + const data = Cast(Doc.GetProto(document).data, ImageField); + const description = StrCast(document.caption); + if (!data) { + return undefined; + } + media.push({ + source: data.url.href, + description, + } as MediaInput); + }); + if (media.length) { + return PostToServer(RouteStore.googlePhotosMediaUpload, { media, album }); } - const image = document.createElement("img"); - image.src = field.url.href; - image.width = 200; - image.height = 200; - const canvas = document.createElement("canvas"); - canvas.width = image.width; - canvas.height = image.height; - const ctx = canvas.getContext("2d")!; - ctx.drawImage(image, 0, 0); - const dataUrl = canvas.toDataURL("image/png"); - return dataUrl.replace(/^data:image\/(png|jpg);base64,/, ""); + return undefined; }; } \ No newline at end of file diff --git a/src/client/views/MainView.tsx b/src/client/views/MainView.tsx index c1c95fc88..6d366216e 100644 --- a/src/client/views/MainView.tsx +++ b/src/client/views/MainView.tsx @@ -40,11 +40,10 @@ import { PreviewCursor } from './PreviewCursor'; import { FilterBox } from './search/FilterBox'; import PresModeMenu from './presentationview/PresentationModeMenu'; import { PresBox } from './nodes/PresBox'; -import { docs_v1 } from 'googleapis'; -import { Album } from '../../server/apis/google/typings/albums'; import { GooglePhotosClientUtils } from '../apis/google_docs/GooglePhotosClientUtils'; import { ImageField } from '../../new_fields/URLField'; import { LinkFollowBox } from './linking/LinkFollowBox'; +import { DocumentManager } from '../util/DocumentManager'; @observer export class MainView extends React.Component { @@ -131,10 +130,7 @@ export class MainView extends React.Component { window.removeEventListener("keydown", KeyManager.Instance.handle); window.addEventListener("keydown", KeyManager.Instance.handle); - let imgurl = "https://upload.wikimedia.org/wikipedia/commons/thumb/3/3a/Cat03.jpg/1200px-Cat03.jpg"; - let image = Docs.Create.ImageDocument(imgurl, { width: 200, title: "an image of a cat" }); - let parameters = { title: StrCast(image.title), MEDIA_BINARY_DATA: GooglePhotosClientUtils.toDataURL(Cast(image.data, ImageField)) }; - // PostToServer(RouteStore.googlePhotosMediaUpload, parameters).then(console.log); + this.executeGooglePhotosRoutine(); reaction(() => { let workspaces = CurrentUserUtils.UserDocument.workspaces; @@ -153,6 +149,15 @@ export class MainView extends React.Component { }, { fireImmediately: true }); } + executeGooglePhotosRoutine = async () => { + let imgurl = "https://upload.wikimedia.org/wikipedia/commons/thumb/3/3a/Cat03.jpg/1200px-Cat03.jpg"; + let doc = Docs.Create.ImageDocument(imgurl, { width: 200, title: "an image of a cat" }); + doc.caption = "Well isn't this a nice cat image!"; + let photos = await GooglePhotosClientUtils.endpoint(); + let albumId = (await photos.albums.list(50)).albums.filter((album: any) => album.title === "This is a generically created album!")[0].id; + console.log(await GooglePhotosClientUtils.UploadMedia([doc], { id: albumId })); + } + componentWillUnMount() { window.removeEventListener("keydown", KeyManager.Instance.handle); //close presentation diff --git a/src/client/views/nodes/FormattedTextBox.tsx b/src/client/views/nodes/FormattedTextBox.tsx index b671d06ea..fda9ea33f 100644 --- a/src/client/views/nodes/FormattedTextBox.tsx +++ b/src/client/views/nodes/FormattedTextBox.tsx @@ -133,6 +133,7 @@ export class FormattedTextBox extends DocComponent<(FieldViewProps & FormattedTe if (this.props.isOverlay) { DragManager.StartDragFunctions.push(() => FormattedTextBox.InputBoxOverlay = undefined); } + FormattedTextBox.Instance = this; } public get CurrentDiv(): HTMLDivElement { return this._ref.current!; } diff --git a/src/server/RouteStore.ts b/src/server/RouteStore.ts index 3b3fd9b4a..b221b71bc 100644 --- a/src/server/RouteStore.ts +++ b/src/server/RouteStore.ts @@ -32,7 +32,7 @@ export enum RouteStore { // APIS cognitiveServices = "/cognitiveservices", googleDocs = "/googleDocs", - googlePhotosQuery = "/googlePhotosQuery", + googlePhotosAccessToken = "/googlePhotosAccessToken", googlePhotosMediaUpload = "/googlePhotosMediaUpload" } \ No newline at end of file diff --git a/src/server/apis/google/GoogleApiServerUtils.ts b/src/server/apis/google/GoogleApiServerUtils.ts index b6330a609..ac8023ce1 100644 --- a/src/server/apis/google/GoogleApiServerUtils.ts +++ b/src/server/apis/google/GoogleApiServerUtils.ts @@ -7,6 +7,7 @@ import { GlobalOptions } from "googleapis-common"; import { GaxiosResponse } from "gaxios"; import request = require('request-promise'); import * as qs from 'query-string'; +import Photos = require('googlephotos'); /** * Server side authentication for Google Api queries. @@ -35,19 +36,19 @@ export namespace GoogleApiServerUtils { } export interface CredentialPaths { - credentials: string; - token: string; + credentialsPath: string; + tokenPath: string; } export type ApiResponse = Promise; - export type ApiRouter = (endpoint: Endpoint, paramters: any) => ApiResponse; + export type ApiRouter = (endpoint: Endpoint, parameters: any) => ApiResponse; export type ApiHandler = (parameters: any) => ApiResponse; export type Action = "create" | "retrieve" | "update"; export type Endpoint = { get: ApiHandler, create: ApiHandler, batchUpdate: ApiHandler }; export type EndpointParameters = GlobalOptions & { version: "v1" }; - export const GetEndpoint = async (sector: string, paths: CredentialPaths) => { + export const GetEndpoint = (sector: string, paths: CredentialPaths) => { return new Promise>(resolve => { RetrieveCredentials(paths).then(authentication => { let routed: Opt; @@ -65,19 +66,19 @@ export namespace GoogleApiServerUtils { }); }; - export const RetrieveCredentials = async (paths: CredentialPaths) => { + export const RetrieveCredentials = (paths: CredentialPaths) => { return new Promise((resolve, reject) => { - readFile(paths.credentials, async (err, credentials) => { + readFile(paths.credentialsPath, async (err, credentials) => { if (err) { reject(err); return console.log('Error loading client secret file:', err); } - authorize(parseBuffer(credentials), paths.token).then(resolve, reject); + authorize(parseBuffer(credentials), paths.tokenPath).then(resolve, reject); }); }); }; - export const RetrieveAccessToken = async (paths: CredentialPaths) => { + export const RetrieveAccessToken = (paths: CredentialPaths) => { return new Promise((resolve, reject) => { RetrieveCredentials(paths).then( credentials => resolve(credentials.token.access_token!), @@ -86,6 +87,15 @@ export namespace GoogleApiServerUtils { }); }; + export const RetrievePhotosEndpoint = (paths: CredentialPaths) => { + return new Promise((resolve, reject) => { + RetrieveAccessToken(paths).then( + token => resolve(new Photos(token)), + reject + ); + }); + }; + type TokenResult = { token: Credentials, client: OAuth2Client }; /** * Create an OAuth2 client with the given credentials, and returns the promise resolving to the authenticated client @@ -126,7 +136,7 @@ export namespace GoogleApiServerUtils { request.post(url, headerParameters).then(response => { let parsed = JSON.parse(response); credentials.access_token = parsed.access_token; - credentials.expiry_date = new Date().getTime() + parsed.expires_in * 1000; + credentials.expiry_date = new Date().getTime() + (parsed.expires_in * 1000); writeFile(token_path, JSON.stringify(credentials), (err) => { if (err) { console.error(err); diff --git a/src/server/apis/google/GooglePhotosServerUtils.ts b/src/server/apis/google/GooglePhotosServerUtils.ts deleted file mode 100644 index cb5464abc..000000000 --- a/src/server/apis/google/GooglePhotosServerUtils.ts +++ /dev/null @@ -1,68 +0,0 @@ -import request = require('request-promise'); -import { Album } from './typings/albums'; -import * as qs from 'query-string'; - -const apiEndpoint = "https://photoslibrary.googleapis.com/v1/"; - -export interface Authorization { - token: string; -} - -export namespace GooglePhotos { - - export type Query = Album.Query; - export type QueryParameters = { query: GooglePhotos.Query }; - interface DispatchParameters { - required: boolean; - method: "GET" | "POST"; - ignore?: boolean; - } - - export const ExecuteQuery = async (parameters: Authorization & QueryParameters): Promise => { - let action = parameters.query.action; - let dispatch = SuffixMap.get(action)!; - let suffix = Suffix(parameters, dispatch, action); - if (suffix) { - let query: any = parameters.query; - let options: any = { - headers: { 'Content-Type': 'application/json' }, - auth: { 'bearer': parameters.token }, - }; - if (query.body) { - options.body = query.body; - options.json = true; - } - let queryParameters = query.parameters; - if (queryParameters) { - suffix += `?${qs.stringify(queryParameters)}`; - } - let dispatcher = dispatch.method === "POST" ? request.post : request.get; - return dispatcher(apiEndpoint + suffix, options); - } - }; - - const Suffix = (parameters: QueryParameters, dispatch: DispatchParameters, action: Album.Action) => { - let query: any = parameters.query; - let id = query.albumId; - let suffix = 'albums'; - if (dispatch.required) { - if (!id) { - return undefined; - } - suffix += `/${id}${dispatch.ignore ? "" : `:${action}`}`; - } - return suffix; - }; - - const SuffixMap = new Map([ - [Album.Action.AddEnrichment, { required: true, method: "POST" }], - [Album.Action.BatchAddMediaItems, { required: true, method: "POST" }], - [Album.Action.BatchRemoveMediaItems, { required: true, method: "POST" }], - [Album.Action.Create, { required: false, method: "POST" }], - [Album.Action.Get, { required: true, ignore: true, method: "GET" }], - [Album.Action.List, { required: false, method: "GET" }], - [Album.Action.Share, { required: true, method: "POST" }], - [Album.Action.Unshare, { required: true, method: "POST" }] - ]); - -} diff --git a/src/server/apis/google/GooglePhotosUploadUtils.ts b/src/server/apis/google/GooglePhotosUploadUtils.ts index cd2a586eb..3b513aaf1 100644 --- a/src/server/apis/google/GooglePhotosUploadUtils.ts +++ b/src/server/apis/google/GooglePhotosUploadUtils.ts @@ -1,28 +1,122 @@ import request = require('request-promise'); -import { Authorization } from './GooglePhotosServerUtils'; +import { GoogleApiServerUtils } from './GoogleApiServerUtils'; +import * as fs from 'fs'; +import { Utils } from '../../../Utils'; +import * as path from 'path'; +import { Opt } from '../../../new_fields/Doc'; export namespace GooglePhotosUploadUtils { - interface UploadInformation { - title: string; - MEDIA_BINARY_DATA: string; + export interface Paths { + uploadDirectory: string; + credentialsPath: string; + tokenPath: string; } - const apiEndpoint = "https://photoslibrary.googleapis.com/v1/uploads"; + export interface MediaInput { + description: string; + source: string; + } + + export interface DownloadInformation { + mediaPath: string; + contentType?: string; + contentSize?: string; + } + + const prepend = (extension: string) => `https://photoslibrary.googleapis.com/v1/${extension}`; + const headers = (type: string) => ({ + 'Content-Type': `application/${type}`, + 'Authorization': Bearer, + }); + + let Bearer: string; + let Paths: Paths; - export const SubmitUpload = async (parameters: Authorization & UploadInformation) => { - let options = { + export const initialize = async (paths: Paths) => { + Paths = paths; + const { tokenPath, credentialsPath } = paths; + const token = await GoogleApiServerUtils.RetrieveAccessToken({ tokenPath, credentialsPath }); + Bearer = `Bearer ${token}`; + }; + + export const DispatchGooglePhotosUpload = async (filename: string) => { + let body: Buffer; + if (filename.includes('upload_')) { + const mediaPath = Paths.uploadDirectory + filename; + body = await new Promise((resolve, reject) => { + fs.readFile(mediaPath, (error, data) => error ? reject(error) : resolve(data)); + }); + } else { + body = await request(filename, { encoding: null }); + } + const parameters = { + method: 'POST', headers: { - 'Content-Type': 'application/octet-stream', - Authorization: `Bearer ${parameters.token}`, - 'X-Goog-Upload-File-Name': parameters.title, + ...headers('octet-stream'), + 'X-Goog-Upload-File-Name': filename, 'X-Goog-Upload-Protocol': 'raw' }, - body: { MEDIA_BINARY_DATA: parameters.MEDIA_BINARY_DATA }, - json: true + uri: prepend('uploads'), + body }; - const result = await request.post(apiEndpoint, options); - return result; + return new Promise(resolve => request(parameters, (error, _response, body) => resolve(error ? undefined : body))); + }; + + export const CreateMediaItems = (newMediaItems: any[], album?: { id: string }) => { + return new Promise((resolve, reject) => { + const parameters = { + method: 'POST', + headers: headers('json'), + uri: prepend('mediaItems:batchCreate'), + body: { newMediaItems } as any, + json: true + }; + album && (parameters.body.albumId = album.id); + request(parameters, (error, _response, body) => { + if (error) { + reject(error); + } else { + resolve(body); + } + }); + }); }; + export namespace IOUtils { + + export const Download = async (url: string): Promise> => { + const filename = `temporary_upload_${Utils.GenerateGuid()}${path.extname(url).toLowerCase()}`; + const temporaryDirectory = Paths.uploadDirectory + "temporary/"; + const mediaPath = temporaryDirectory + filename; + + if (!(await createIfNotExists(temporaryDirectory))) { + return undefined; + } + + return new Promise((resolve, reject) => { + request.head(url, (error, res) => { + if (error) { + return reject(error); + } + const information: DownloadInformation = { + mediaPath, + contentType: res.headers['content-type'], + contentSize: res.headers['content-length'], + }; + request(url).pipe(fs.createWriteStream(mediaPath)).on('close', () => resolve(information)); + }); + }); + }; + + export const createIfNotExists = async (path: string) => { + if (await new Promise(resolve => fs.exists(path, resolve))) { + return true; + } + return new Promise(resolve => fs.mkdir(path, error => resolve(error === null))); + }; + + export const Destroy = (mediaPath: string) => new Promise(resolve => fs.unlink(mediaPath, error => resolve(error === null))); + } + } \ No newline at end of file diff --git a/src/server/apis/google/typings/albums.ts b/src/server/apis/google/typings/albums.ts deleted file mode 100644 index f3025567d..000000000 --- a/src/server/apis/google/typings/albums.ts +++ /dev/null @@ -1,150 +0,0 @@ -export namespace Album { - - export type Query = (AddEnrichment | BatchAddMediaItems | BatchRemoveMediaItems | Create | Get | List | Share | Unshare); - - export enum Action { - AddEnrichment = "addEnrichment", - BatchAddMediaItems = "batchAddMediaItems", - BatchRemoveMediaItems = "batchRemoveMediaItems", - Create = "create", - Get = "get", - List = "list", - Share = "share", - Unshare = "unshare" - } - - export interface AddEnrichment { - action: Action.AddEnrichment; - albumId: string; - body: { - newEnrichmentItem: NewEnrichmentItem; - albumPosition: MediaRelativeAlbumPosition; - }; - } - - export interface BatchAddMediaItems { - action: Action.BatchAddMediaItems; - albumId: string; - body: { - mediaItemIds: string[]; - }; - } - - export interface BatchRemoveMediaItems { - action: Action.BatchRemoveMediaItems; - albumId: string; - body: { - mediaItemIds: string[]; - }; - } - - export interface Create { - action: Action.Create; - body: { - album: Template; - }; - } - - export interface Get { - action: Action.Get; - albumId: string; - } - - export interface List { - action: Action.List; - parameters: ListOptions; - } - - export interface ListOptions { - pageSize: number; - pageToken: string; - excludeNonAppCreatedData: boolean; - } - - export interface Share { - action: Action.Share; - albumId: string; - body: { - sharedAlbumOptions: SharedOptions; - }; - } - - export interface Unshare { - action: Action.Unshare; - albumId: string; - } - - export interface Template { - title: string; - } - - export interface Model { - id: string; - title: string; - productUrl: string; - isWriteable: boolean; - shareInfo: ShareInfo; - mediaItemsCount: string; - coverPhotoBaseUrl: string; - coverPhotoMediaItemId: string; - } - - export interface ShareInfo { - sharedAlbumOptions: SharedOptions; - shareableUrl: string; - shareToken: string; - isJoined: boolean; - isOwned: boolean; - } - - export interface SharedOptions { - isCollaborative: boolean; - isCommentable: boolean; - } - - export enum PositionType { - POSITION_TYPE_UNSPECIFIED, - FIRST_IN_ALBUM, - LAST_IN_ALBUM, - AFTER_MEDIA_ITEM, - AFTER_ENRICHMENT_ITEM - } - - export type Position = GeneralAlbumPosition | MediaRelativeAlbumPosition | EnrichmentRelativeAlbumPosition; - - interface GeneralAlbumPosition { - position: PositionType.FIRST_IN_ALBUM | PositionType.LAST_IN_ALBUM | PositionType.POSITION_TYPE_UNSPECIFIED; - } - - interface MediaRelativeAlbumPosition { - position: PositionType.AFTER_MEDIA_ITEM; - relativeMediaItemId: string; - } - - interface EnrichmentRelativeAlbumPosition { - position: PositionType.AFTER_ENRICHMENT_ITEM; - relativeEnrichmentItemId: string; - } - - export interface Location { - locationName: string; - latlng: { - latitude: number, - longitude: number - }; - } - - export interface NewEnrichmentItem { - textEnrichment: { - text: string; - }; - locationEnrichment: { - location: Location - }; - mapEnrichment: { - origin: { location: Location }, - destination: { location: Location } - }; - } - -} \ No newline at end of file diff --git a/src/server/credentials/google_docs_token.json b/src/server/credentials/google_docs_token.json index 2ac972ed8..f3c8cf82a 100644 --- a/src/server/credentials/google_docs_token.json +++ b/src/server/credentials/google_docs_token.json @@ -1 +1 @@ -{"access_token":"ya29.Glt5ByifP30HMz6a1fEG77qZlz9fBEOCz4PQ1VA8t_Ck2ZTPJKoyr6xc3-GFZISAqrrw2U8XpMyZv02_URfPwUX0Z_tMdlIFqsygowR-uClukbgQPNtgxp2LS1oW","refresh_token":"1/wK1cUVLnt581ba_pYGPdlTvAa-OS64nB5m5XOXEBJ8Q","scope":"https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/documents.readonly https://www.googleapis.com/auth/photoslibrary.sharing https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/photoslibrary.appendonly https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/photoslibrary","token_type":"Bearer","expiry_date":1567556163894} \ No newline at end of file +{"access_token":"ya29.Glx7B9S6zCKDE0EgYk9xX9-RhcN8j4IwG9ONopTl1NkPX9FUOw0GI_81mY9bhaouuyOTnrc6FrZD5SDHolWwp3ABNT6l7TmhTLDILgGXIixZkWFRBPpF-xHC8lUd8A","refresh_token":"1/HTv_xFHszu2Nf3iiFrUTaeKzC_Vp2-6bpIB06xW_WHI","scope":"https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/documents.readonly https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/photoslibrary https://www.googleapis.com/auth/photoslibrary.appendonly https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/photoslibrary.sharing","token_type":"Bearer","expiry_date":1567765465073} \ No newline at end of file diff --git a/src/server/index.ts b/src/server/index.ts index 3940bbd58..fab00a02d 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -42,7 +42,6 @@ var AdmZip = require('adm-zip'); import * as YoutubeApi from "./apis/youtube/youtubeApiSample"; import { Response } from 'express-serve-static-core'; import { GoogleApiServerUtils } from "./apis/google/GoogleApiServerUtils"; -import { GooglePhotos } from './apis/google/GooglePhotosServerUtils'; import { GooglePhotosUploadUtils } from './apis/google/GooglePhotosUploadUtils'; const MongoStore = require('connect-mongo')(session); const mongoose = require('mongoose'); @@ -418,10 +417,10 @@ app.get("/thumbnail/:filename", (req, res) => { let filename = req.params.filename; let noExt = filename.substring(0, filename.length - ".png".length); let pagenumber = parseInt(noExt.split('-')[1]); - fs.exists(uploadDir + filename, (exists: boolean) => { - console.log(`${uploadDir + filename} ${exists ? "exists" : "does not exist"}`); + fs.exists(uploadDirectory + filename, (exists: boolean) => { + console.log(`${uploadDirectory + filename} ${exists ? "exists" : "does not exist"}`); if (exists) { - let input = fs.createReadStream(uploadDir + filename); + let input = fs.createReadStream(uploadDirectory + filename); probe(input, (err: any, result: any) => { if (err) { console.log(err); @@ -432,7 +431,7 @@ app.get("/thumbnail/:filename", (req, res) => { }); } else { - LoadPage(uploadDir + filename.substring(0, filename.length - noExt.split('-')[1].length - ".PNG".length - 1) + ".pdf", pagenumber, res); + LoadPage(uploadDirectory + filename.substring(0, filename.length - noExt.split('-')[1].length - ".PNG".length - 1) + ".pdf", pagenumber, res); } }); }); @@ -556,13 +555,13 @@ class NodeCanvasFactory { const pngTypes = [".png", ".PNG"]; const pdfTypes = [".pdf", ".PDF"]; const jpgTypes = [".jpg", ".JPG", ".jpeg", ".JPEG"]; -const uploadDir = __dirname + "/public/files/"; +const uploadDirectory = __dirname + "/public/files/"; // SETTERS app.post( RouteStore.upload, (req, res) => { let form = new formidable.IncomingForm(); - form.uploadDir = uploadDir; + form.uploadDir = uploadDirectory; form.keepExtensions = true; // let path = req.body.path; console.log("upload"); @@ -592,7 +591,7 @@ app.post( } if (isImage) { resizers.forEach(resizer => { - fs.createReadStream(uploadDir + file).pipe(resizer.resizer).pipe(fs.createWriteStream(uploadDir + file.substring(0, file.length - ext.length) + resizer.suffix + ext)); + fs.createReadStream(uploadDirectory + file).pipe(resizer.resizer).pipe(fs.createWriteStream(uploadDirectory + file.substring(0, file.length - ext.length) + resizer.suffix + ext)); }); } names.push(`/files/` + file); @@ -611,7 +610,7 @@ addSecureRoute( res.status(401).send("incorrect parameters specified"); return; } - imageDataUri.outputFile(uri, uploadDir + filename).then((savedName: string) => { + imageDataUri.outputFile(uri, uploadDirectory + filename).then((savedName: string) => { const ext = path.extname(savedName); let resizers = [ { resizer: sharp().resize(100, undefined, { withoutEnlargement: true }), suffix: "_s" }, @@ -632,7 +631,7 @@ addSecureRoute( } if (isImage) { resizers.forEach(resizer => { - fs.createReadStream(savedName).pipe(resizer.resizer).pipe(fs.createWriteStream(uploadDir + filename + resizer.suffix + ext)); + fs.createReadStream(savedName).pipe(resizer.resizer).pipe(fs.createWriteStream(uploadDirectory + filename + resizer.suffix + ext)); }); } res.send("/files/" + filename + ext); @@ -799,8 +798,8 @@ function HandleYoutubeQuery([query, callback]: [YoutubeQueryInput, (result?: any } } -const credentials = path.join(__dirname, "./credentials/google_docs_credentials.json"); -const token = path.join(__dirname, "./credentials/google_docs_token.json"); +const credentialsPath = path.join(__dirname, "./credentials/google_docs_credentials.json"); +const tokenPath = path.join(__dirname, "./credentials/google_docs_token.json"); const EndpointHandlerMap = new Map([ ["create", (api, params) => api.create(params)], @@ -811,7 +810,7 @@ const EndpointHandlerMap = new Map { let sector: any = req.params.sector; let action: any = req.params.action; - GoogleApiServerUtils.GetEndpoint(GoogleApiServerUtils.Service[sector], { credentials, token }).then(endpoint => { + GoogleApiServerUtils.GetEndpoint(GoogleApiServerUtils.Service[sector], { credentialsPath, tokenPath }).then(endpoint => { let handler = EndpointHandlerMap.get(action); if (endpoint && handler) { let execute = handler(endpoint, req.body).then( @@ -825,36 +824,28 @@ app.post(RouteStore.googleDocs + "/:sector/:action", (req, res) => { }); }); -app.post(RouteStore.googlePhotosQuery, (req, res) => { - GoogleApiServerUtils.RetrieveAccessToken({ credentials, token }).then( - token => { - GooglePhotos.ExecuteQuery({ token, query: req.body }) - .then(response => { - if (response === undefined) { - res.send("Error: unable to build suffix for Google Photos API request"); - return; - } - res.send(response); - }) - .catch(error => { - res.send(`Error: an exception occurred in the execution of this Google Photos API request\n${error}`); - }); - }, - error => res.send(error) - ); -}); +app.get(RouteStore.googlePhotosAccessToken, (req, res) => GoogleApiServerUtils.RetrieveAccessToken({ credentialsPath, tokenPath }).then(token => res.send(token))); -app.post(RouteStore.googlePhotosMediaUpload, (req, res) => { - GoogleApiServerUtils.RetrieveAccessToken({ credentials, token }).then( - token => { - GooglePhotosUploadUtils.SubmitUpload({ token, ...req.body }) - .then(response => { - res.send(response); - }).catch(error => { - res.send(`Error: an exception occurred in uploading the given media\n${error}`); - }); - }, - error => res.send(error)); +const tokenError = "Unable to successfully upload bytes for all images!"; +const mediaError = "Unable to convert all uploaded bytes to media items!"; + +app.post(RouteStore.googlePhotosMediaUpload, async (req, res) => { + const media: GooglePhotosUploadUtils.MediaInput[] = req.body.media; + await GooglePhotosUploadUtils.initialize({ uploadDirectory, credentialsPath, tokenPath }); + const newMediaItems = await Promise.all(media.map(async element => { + const uploadToken = await GooglePhotosUploadUtils.DispatchGooglePhotosUpload(element.source); + return !uploadToken ? undefined : { + description: element.description, + simpleMediaItem: { uploadToken } + }; + })); + if (!newMediaItems.every(item => item)) { + return res.send(tokenError); + } + GooglePhotosUploadUtils.CreateMediaItems(newMediaItems, req.body.album).then( + success => res.send(success), + () => res.send(mediaError) + ); }); const suffixMap: { [type: string]: (string | [string, string | ((json: any) => any)]) } = { -- cgit v1.2.3-70-g09d2 From b24c475d8cd36af860fc374b0c5621b0d096be1d Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Mon, 9 Sep 2019 20:47:34 -0400 Subject: nearly finished transferring images between text notes and google docs --- package.json | 2 +- .../apis/google_docs/GooglePhotosClientUtils.ts | 23 ++-- .../util/Import & Export/DirectoryImportBox.tsx | 2 +- src/client/views/MainView.tsx | 6 +- src/client/views/collections/CollectionSubView.tsx | 2 +- src/client/views/nodes/DocumentView.tsx | 2 +- src/new_fields/RichTextUtils.ts | 67 ++++++++++- src/server/RouteStore.ts | 3 +- src/server/apis/google/GoogleApiServerUtils.ts | 2 +- src/server/apis/google/GooglePhotosUploadUtils.ts | 5 + src/server/credentials/google_docs_token.json | 2 +- src/server/index.ts | 20 +++- src/server/updateSearch.ts | 123 --------------------- 13 files changed, 112 insertions(+), 147 deletions(-) delete mode 100644 src/server/updateSearch.ts (limited to 'package.json') diff --git a/package.json b/package.json index f56e34ce0..f0f2b467e 100644 --- a/package.json +++ b/package.json @@ -224,4 +224,4 @@ "xoauth2": "^1.2.0", "youtube": "^0.1.0" } -} \ No newline at end of file +} diff --git a/src/client/apis/google_docs/GooglePhotosClientUtils.ts b/src/client/apis/google_docs/GooglePhotosClientUtils.ts index 5f5b39b14..fddcf3aa5 100644 --- a/src/client/apis/google_docs/GooglePhotosClientUtils.ts +++ b/src/client/apis/google_docs/GooglePhotosClientUtils.ts @@ -51,17 +51,26 @@ export namespace GooglePhotosClientUtils { description: string; } - export const UploadImageDocuments = async (sources: Doc[], album?: AlbumReference, descriptionKey = "caption") => { + export const UploadImages = async (sources: (Doc | string)[], album?: AlbumReference, descriptionKey = "caption") => { if (album && "title" in album) { album = await (await endpoint()).albums.create(album.title); } const media: MediaInput[] = []; - sources.forEach(document => { - const data = Cast(Doc.GetProto(document).data, ImageField); - data && media.push({ - url: data.url.href, - description: parseDescription(document, descriptionKey), - }); + sources.forEach(source => { + let url: string; + let description: string; + if (source instanceof Doc) { + const data = Cast(Doc.GetProto(source).data, ImageField); + if (!data) { + return; + } + url = data.url.href; + description = parseDescription(source, descriptionKey); + } else { + url = source; + description = Utils.GenerateGuid(); + } + media.push({ url, description }); }); if (media.length) { return PostToServer(RouteStore.googlePhotosMediaUpload, { media, album }); diff --git a/src/client/util/Import & Export/DirectoryImportBox.tsx b/src/client/util/Import & Export/DirectoryImportBox.tsx index 7693a388f..a19fd39b7 100644 --- a/src/client/util/Import & Export/DirectoryImportBox.tsx +++ b/src/client/util/Import & Export/DirectoryImportBox.tsx @@ -117,7 +117,7 @@ export default class DirectoryImportBox extends React.Component console.log(`(${this.quota - this.remaining}/${this.quota}) ${upload.name}`); })); - await GooglePhotosClientUtils.UploadImageDocuments(docs, { title: directory }); + await GooglePhotosClientUtils.UploadImages(docs, { title: directory }); console.log("Finished upload!"); for (let i = 0; i < docs.length; i++) { diff --git a/src/client/views/MainView.tsx b/src/client/views/MainView.tsx index 326c13424..0c0ed9072 100644 --- a/src/client/views/MainView.tsx +++ b/src/client/views/MainView.tsx @@ -155,7 +155,7 @@ export class MainView extends React.Component { doc.caption = "Well isn't this a nice cat image!"; let photos = await GooglePhotosClientUtils.endpoint(); let albumId = (await photos.albums.list(50)).albums.filter((album: any) => album.title === "This is a generically created album!")[0].id; - console.log(await GooglePhotosClientUtils.UploadImageDocuments([doc], { id: albumId })); + console.log(await GooglePhotosClientUtils.UploadImages([doc], { id: albumId })); } componentWillUnMount() { @@ -470,7 +470,7 @@ export class MainView extends React.Component { // let youtubeurl = "https://www.youtube.com/embed/TqcApsGRzWw"; // let addYoutubeSearcher = action(() => Docs.Create.YoutubeDocument(youtubeurl, { width: 600, height: 600, title: "youtube search" })); - let googlePhotosSearch = () => GooglePhotosClientUtils.CollectionFromSearch(Docs.Create.MasonryDocument, { included: [GooglePhotosClientUtils.ContentCategories.LANDSCAPES] }); + // let googlePhotosSearch = () => GooglePhotosClientUtils.CollectionFromSearch(Docs.Create.MasonryDocument, { included: [GooglePhotosClientUtils.ContentCategories.LANDSCAPES] }); let btns: [React.RefObject, IconName, string, () => Doc | Promise][] = [ [React.createRef(), "object-group", "Add Collection", addColNode], @@ -478,7 +478,7 @@ export class MainView extends React.Component { [React.createRef(), "globe-asia", "Add Website", addWebNode], [React.createRef(), "bolt", "Add Button", addButtonDocument], [React.createRef(), "file", "Add Document Dragger", addDragboxNode], - [React.createRef(), "object-group", "Test Google Photos Search", googlePhotosSearch], + // [React.createRef(), "object-group", "Test Google Photos Search", googlePhotosSearch], [React.createRef(), "cloud-upload-alt", "Import Directory", addImportCollectionNode], //remove at some point in favor of addImportCollectionNode //[React.createRef(), "play", "Add Youtube Searcher", addYoutubeSearcher], ]; diff --git a/src/client/views/collections/CollectionSubView.tsx b/src/client/views/collections/CollectionSubView.tsx index 99e5ab7b3..5fc4f36a7 100644 --- a/src/client/views/collections/CollectionSubView.tsx +++ b/src/client/views/collections/CollectionSubView.tsx @@ -253,7 +253,7 @@ export function CollectionSubView(schemaCtor: (doc: Doc) => T) { }).then(async (res: Response) => { (await res.json()).map(action((file: any) => { let full = { ...options, nativeWidth: type.indexOf("video") !== -1 ? 600 : 300, width: 300, title: dropFileName }; - let path = Utils.prepend(file); + let path = Utils.prepend(file.path); Docs.Get.DocumentFromType(type, path, full).then(doc => doc && this.props.addDocument(doc)); })); }); diff --git a/src/client/views/nodes/DocumentView.tsx b/src/client/views/nodes/DocumentView.tsx index 4033ffd9c..cb9346a8b 100644 --- a/src/client/views/nodes/DocumentView.tsx +++ b/src/client/views/nodes/DocumentView.tsx @@ -591,7 +591,7 @@ export class DocumentView extends DocComponent(Docu subitems.push({ description: "Open Fields", event: this.fieldsClicked, icon: "layer-group" }); cm.addItem({ description: "Open...", subitems: subitems, icon: "external-link-alt" }); if (Cast(this.props.Document.data, ImageField)) { - cm.addItem({ description: "Export to Google Photos", event: () => GooglePhotosClientUtils.UploadImageDocuments([this.props.Document]), icon: "caret-square-right" }); + cm.addItem({ description: "Export to Google Photos", event: () => GooglePhotosClientUtils.UploadImages([this.props.Document]), icon: "caret-square-right" }); } let existingMake = ContextMenu.Instance.findByDescription("Make..."); let makes: ContextMenuProps[] = existingMake && "subitems" in existingMake ? existingMake.subitems : []; diff --git a/src/new_fields/RichTextUtils.ts b/src/new_fields/RichTextUtils.ts index 0aba50c0d..500b93676 100644 --- a/src/new_fields/RichTextUtils.ts +++ b/src/new_fields/RichTextUtils.ts @@ -7,6 +7,11 @@ import { FormattedTextBox } from "../client/views/nodes/FormattedTextBox"; import { Opt } from "./Doc"; import * as Color from "color"; import { sinkListItem } from "prosemirror-schema-list"; +import { Utils, PostToServer } from "../Utils"; +import { RouteStore } from "../server/RouteStore"; +import { Docs } from "../client/documents/Documents"; +import { schema } from "../client/util/RichTextSchema"; +import { GooglePhotosClientUtils } from "../client/apis/google_docs/GooglePhotosClientUtils"; export namespace RichTextUtils { @@ -86,19 +91,36 @@ export namespace RichTextUtils { export namespace GoogleDocs { export const Export = (state: EditorState): GoogleApiClientUtils.Docs.Content => { - let textNodes: Node[] = []; + let nodes: { [type: string]: Node[] } = { + text: [], + image: [] + }; let text = ToPlainText(state); let content = state.doc.content; - content.forEach(node => node.content.forEach(node => node.type.name === "text" && textNodes.push(node))); - let linkRequests = ExtractLinks(textNodes); + content.forEach(node => node.content.forEach(node => { + const type = node.type.name; + let existing = nodes[type]; + if (existing) { + existing.push(node); + } else { + nodes[type] = [node]; + } + })); + let linkRequests = ExtractLinks(nodes.text); + let imageRequests = ExtractImages(nodes.image); return { text, - requests: [...linkRequests] + requests: [...linkRequests, ...imageRequests] }; }; type BulletPosition = { value: number, sinks: number }; + interface MediaItem { + baseUrl: string; + filename: string; + width: number; + } export const Import = async (documentId: GoogleApiClientUtils.Docs.DocumentId): Promise> => { const document = await GoogleApiClientUtils.Docs.retrieve({ documentId }); if (!document) { @@ -109,6 +131,17 @@ export namespace RichTextUtils { const { text, paragraphs } = GoogleApiClientUtils.Docs.Utils.extractText(document); let state = FormattedTextBox.blankState(); let structured = parseLists(paragraphs); + const inline = document.inlineObjects; + let inlineUrls: MediaItem[] = []; + if (inline) { + inlineUrls = Object.keys(inline).map(key => { + const embedded = inline[key].inlineObjectProperties!.embeddedObject!; + const baseUrl = embedded.imageProperties!.contentUri!; + const filename = `upload_${Utils.GenerateGuid()}.png`; + const width = embedded.size!.width!.magnitude!; + return { baseUrl, filename, width }; + }); + } let position = 3; let lists: ListGroup[] = []; @@ -151,6 +184,12 @@ export namespace RichTextUtils { } } + const uploads = await PostToServer(RouteStore.googlePhotosMediaDownload, { mediaItems: inlineUrls }); + for (let i = 0; i < uploads.length; i++) { + const src = Utils.prepend(`/files/${uploads[i].fileNames.clean}`); + state = state.apply(state.tr.insert(0, schema.nodes.image.create({ src, width: inlineUrls[i].width }))); + } + return { title, text, state }; }; @@ -252,6 +291,26 @@ export namespace RichTextUtils { return links; }; + const ExtractImages = async (nodes: Node[]) => { + const images: docs_v1.Schema$Request[] = []; + let position = 1; + for (let node of nodes) { + const length = node.nodeSize; + const attrs = node.attrs; + const uri = attrs.src; + const result = (await GooglePhotosClientUtils.UploadImages([uri])).newMediaItemResults; + images.push({ + insertInlineImage: { + uri: result[0].mediaItem.productUrl, + objectSize: { width: { magnitude: parseFloat(attrs.width.replace("px", "")), unit: "PT" } }, + location: { index: position + length } + } + }); + position += length; + } + return images; + }; + const Encode = (information: LinkInformation) => { return { updateTextStyle: { diff --git a/src/server/RouteStore.ts b/src/server/RouteStore.ts index f65e6134c..ee9cd8a0e 100644 --- a/src/server/RouteStore.ts +++ b/src/server/RouteStore.ts @@ -34,6 +34,7 @@ export enum RouteStore { googleDocs = "/googleDocs", googlePhotosAccessToken = "/googlePhotosAccessToken", googlePhotosMediaUpload = "/googlePhotosMediaUpload", - googlePhotosMediaDownload = "/googlePhotosMediaDownload" + googlePhotosMediaDownload = "/googlePhotosMediaDownload", + googleDocsGet = "/googleDocsGet" } \ No newline at end of file diff --git a/src/server/apis/google/GoogleApiServerUtils.ts b/src/server/apis/google/GoogleApiServerUtils.ts index ac8023ce1..e0bd8a800 100644 --- a/src/server/apis/google/GoogleApiServerUtils.ts +++ b/src/server/apis/google/GoogleApiServerUtils.ts @@ -42,7 +42,7 @@ export namespace GoogleApiServerUtils { export type ApiResponse = Promise; export type ApiRouter = (endpoint: Endpoint, parameters: any) => ApiResponse; - export type ApiHandler = (parameters: any) => ApiResponse; + export type ApiHandler = (parameters: any, methodOptions?: any) => ApiResponse; export type Action = "create" | "retrieve" | "update"; export type Endpoint = { get: ApiHandler, create: ApiHandler, batchUpdate: ApiHandler }; diff --git a/src/server/apis/google/GooglePhotosUploadUtils.ts b/src/server/apis/google/GooglePhotosUploadUtils.ts index 35f986250..d1f1f81bd 100644 --- a/src/server/apis/google/GooglePhotosUploadUtils.ts +++ b/src/server/apis/google/GooglePhotosUploadUtils.ts @@ -151,6 +151,11 @@ export namespace DownloadUtils { .on('close', resolve) .on('error', reject); }); + if (!isLocal) { + await new Promise(resolve => { + stream(url).pipe(fs.createWriteStream(uploadDirectory + resolved)).on('close', resolve); + }); + } } resolve(information); }); diff --git a/src/server/credentials/google_docs_token.json b/src/server/credentials/google_docs_token.json index fabc18cfd..5c142fba1 100644 --- a/src/server/credentials/google_docs_token.json +++ b/src/server/credentials/google_docs_token.json @@ -1 +1 @@ -{"access_token":"ya29.Glx-BwgWcpQUukTNyuUqvSAYrDyxDNUhCLtrFDJAViROvicm0DrcRvCn4OaQdn2m2IZQYcG-19cvQYoOC3UJCtWXLRvKZzQCbZZSykpxYu_lflUyEnIGZOIHMbbEjA","refresh_token":"1/HTv_xFHszu2Nf3iiFrUTaeKzC_Vp2-6bpIB06xW_WHI","scope":"https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/documents.readonly https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/photoslibrary https://www.googleapis.com/auth/photoslibrary.appendonly https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/photoslibrary.sharing","token_type":"Bearer","expiry_date":1568008211814} \ No newline at end of file +{"access_token":"ya29.Glx_B6G7Q_FYs1LK5VcyV6P6Zg9JkoHO2aC_TsnN7AVxPYWHEpsBSC0WyWX7Ztr8HWhOUYA5JXqnZDkLrK1V3Hb-0GgtyApLRNtEPOWf1dJ7lOm_iKVw2tRvPe7XDQ","refresh_token":"1/HTv_xFHszu2Nf3iiFrUTaeKzC_Vp2-6bpIB06xW_WHI","scope":"https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/documents.readonly https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/photoslibrary https://www.googleapis.com/auth/photoslibrary.appendonly https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/photoslibrary.sharing","token_type":"Bearer","expiry_date":1568078116605} \ No newline at end of file diff --git a/src/server/index.ts b/src/server/index.ts index baef94a59..8469770d5 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -38,6 +38,7 @@ import flash = require('connect-flash'); import { Search } from './Search'; import _ = require('lodash'); import * as Archiver from 'archiver'; +import * as request_promise from 'request-promise'; var AdmZip = require('adm-zip'); import * as YoutubeApi from "./apis/youtube/youtubeApiSample"; import { Response } from 'express-serve-static-core'; @@ -576,9 +577,9 @@ app.post( form.parse(req, async (_err, _fields, files) => { let results: FileResponse[] = []; for (const key in files) { - const { name, type, path: location } = files[key]; + const { type, path: location, name } = files[key]; const filename = path.basename(location); - await UploadUtils.UploadImage(uploadDirectory + filename, path.basename(name)); + await UploadUtils.UploadImage(uploadDirectory + filename, filename); results.push({ name, type, path: `/files/${filename}` }); console.log(path.basename(name)); } @@ -790,10 +791,23 @@ const tokenPath = path.join(__dirname, "./credentials/google_docs_token.json"); const EndpointHandlerMap = new Map([ ["create", (api, params) => api.create(params)], - ["retrieve", (api, params) => api.get(params)], + ["retrieve", (api, params) => api.get(params, { params: "fields=inlineObjects" })], ["update", (api, params) => api.batchUpdate(params)], ]); +// app.post(RouteStore.googleDocsGet, async (req, res) => { +// const token = await GoogleApiServerUtils.RetrieveAccessToken({ credentialsPath, tokenPath }); +// request_promise.get({ +// uri: `https://docs.googleapis.com/v1/documents/${req.body.documentId}?fields=inlineObjects`, +// headers: { +// 'Authorization': `Bearer ${token}` +// } +// }).then(response => { +// console.log(response); +// res.send(response); +// }); +// }); + app.post(RouteStore.googleDocs + "/:sector/:action", (req, res) => { let sector: GoogleApiServerUtils.Service = req.params.sector; let action: GoogleApiServerUtils.Action = req.params.action; diff --git a/src/server/updateSearch.ts b/src/server/updateSearch.ts deleted file mode 100644 index 906b795f1..000000000 --- a/src/server/updateSearch.ts +++ /dev/null @@ -1,123 +0,0 @@ -import { Database } from "./database"; -import { Cursor } from "mongodb"; -import { Search } from "./Search"; -import pLimit from 'p-limit'; - -const suffixMap: { [type: string]: (string | [string, string | ((json: any) => any)]) } = { - "number": "_n", - "string": "_t", - "boolean": "_b", - // "image": ["_t", "url"], - "video": ["_t", "url"], - "pdf": ["_t", "url"], - "audio": ["_t", "url"], - "web": ["_t", "url"], - "date": ["_d", value => new Date(value.date).toISOString()], - "proxy": ["_i", "fieldId"], - "list": ["_l", list => { - const results = []; - for (const value of list.fields) { - const term = ToSearchTerm(value); - if (term) { - results.push(term.value); - } - } - return results.length ? results : null; - }] -}; - -function ToSearchTerm(val: any): { suffix: string, value: any } | undefined { - if (val === null || val === undefined) { - return; - } - const type = val.__type || typeof val; - let suffix = suffixMap[type]; - if (!suffix) { - return; - } - - if (Array.isArray(suffix)) { - const accessor = suffix[1]; - if (typeof accessor === "function") { - val = accessor(val); - } else { - val = val[accessor]; - } - suffix = suffix[0]; - } - - return { suffix, value: val }; -} - -function getSuffix(value: string | [string, any]): string { - return typeof value === "string" ? value : value[0]; -} - -const limit = pLimit(5); -async function update() { - // await new Promise(res => setTimeout(res, 5)); - console.log("update"); - await Search.Instance.clear(); - const cursor = await Database.Instance.query({}); - console.log("Cleared"); - const updates: any[] = []; - let numDocs = 0; - function updateDoc(doc: any) { - numDocs++; - if ((numDocs % 50) === 0) { - console.log("updateDoc " + numDocs); - } - // console.log("doc " + numDocs); - if (doc.__type !== "Doc") { - return; - } - const fields = doc.fields; - if (!fields) { - return; - } - const update: any = { id: doc._id }; - let dynfield = false; - for (const key in fields) { - const value = fields[key]; - const term = ToSearchTerm(value); - if (term !== undefined) { - let { suffix, value } = term; - update[key + suffix] = value; - dynfield = true; - } - } - if (dynfield) { - updates.push(update); - // console.log(updates.length); - } - } - await cursor.forEach(updateDoc); - console.log(`Updating ${updates.length} documents`); - const result = await Search.Instance.updateDocuments(updates); - try { - console.log(JSON.parse(result).responseHeader.status); - } catch { - console.log("Error:"); - // console.log(updates[i]); - console.log(result); - console.log("\n"); - } - // for (let i = 0; i < updates.length; i++) { - // console.log(i); - // const result = await Search.Instance.updateDocument(updates[i]); - // try { - // console.log(JSON.parse(result).responseHeader.status); - // } catch { - // console.log("Error:"); - // console.log(updates[i]); - // console.log(result); - // console.log("\n"); - // } - // } - // await Promise.all(updates.map(update => { - // return limit(() => Search.Instance.updateDocument(update)); - // })); - cursor.close(); -} - -update(); \ No newline at end of file -- cgit v1.2.3-70-g09d2 From d66b51213e448d5f4f37781389af488a3ac744c4 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Fri, 20 Sep 2019 05:10:21 -0400 Subject: factored out extensions into npm module --- package.json | 1 + .../util/Import & Export/DirectoryImportBox.tsx | 7 +- src/client/views/Main.tsx | 3 - src/extensions/ArrayExtensions.ts | 639 ++++++++++----------- src/extensions/Extensions.ts | 2 +- src/server/apis/google/GooglePhotosUploadUtils.ts | 8 +- src/server/credentials/google_docs_token.json | 2 +- src/server/index.ts | 10 +- 8 files changed, 326 insertions(+), 346 deletions(-) (limited to 'package.json') diff --git a/package.json b/package.json index f869713ba..b20c31a7a 100644 --- a/package.json +++ b/package.json @@ -114,6 +114,7 @@ "@types/youtube": "0.0.38", "adm-zip": "^0.4.13", "archiver": "^3.0.3", + "array-batcher": "^1.0.2", "async": "^2.6.2", "babel-runtime": "^6.26.0", "bcrypt-nodejs": "0.0.3", diff --git a/src/client/util/Import & Export/DirectoryImportBox.tsx b/src/client/util/Import & Export/DirectoryImportBox.tsx index e3958e3a4..762302bc8 100644 --- a/src/client/util/Import & Export/DirectoryImportBox.tsx +++ b/src/client/util/Import & Export/DirectoryImportBox.tsx @@ -20,6 +20,7 @@ import { listSpec } from "../../../new_fields/Schema"; import { GooglePhotos } from "../../apis/google_docs/GooglePhotosClientUtils"; import { SchemaHeaderField } from "../../../new_fields/SchemaHeaderField"; import "./DirectoryImportBox.scss"; +import { batchedMapAsync } from "array-batcher"; const unsupported = ["text/html", "text/plain"]; interface FileResponse { @@ -103,7 +104,7 @@ export default class DirectoryImportBox extends React.Component runInAction(() => this.phase = `Internal: uploading ${this.quota - this.completed} files to Dash...`); - const uploads = await validated.batchedMapAsync({ batchSize: 15 }, async batch => { + const uploads = await batchedMapAsync(validated, { batchSize: 15 }, async batch => { const formData = new FormData(); const parameters = { method: 'POST', body: formData }; @@ -113,9 +114,9 @@ export default class DirectoryImportBox extends React.Component formData.append(Utils.GenerateGuid(), file); }); - const responses = (await fetch(RouteStore.upload, parameters)).json(); + const responses = await (await fetch(RouteStore.upload, parameters)).json(); runInAction(() => this.completed += batch.length); - return responses; + return responses as FileResponse[]; }); await Promise.all(uploads.map(async upload => { diff --git a/src/client/views/Main.tsx b/src/client/views/Main.tsx index 53912550c..70d2235e6 100644 --- a/src/client/views/Main.tsx +++ b/src/client/views/Main.tsx @@ -7,9 +7,6 @@ import { Cast } from "../../new_fields/Types"; import { Doc, DocListCastAsync } from "../../new_fields/Doc"; import { List } from "../../new_fields/List"; import { DocServer } from "../DocServer"; -const Extensions = require("../../extensions/Extensions"); - -Extensions.AssignExtensions(); let swapDocs = async () => { let oldDoc = await Cast(CurrentUserUtils.UserDocument.linkManagerDoc, Doc); diff --git a/src/extensions/ArrayExtensions.ts b/src/extensions/ArrayExtensions.ts index 872f107a7..ca407862b 100644 --- a/src/extensions/ArrayExtensions.ts +++ b/src/extensions/ArrayExtensions.ts @@ -1,333 +1,318 @@ interface Array { - fixedBatch(batcher: FixedBatcher): T[][]; - predicateBatch(batcher: PredicateBatcherSync): T[][]; - predicateBatchAsync(batcher: PredicateBatcherAsync): Promise; - batch(batcher: BatcherSync): T[][]; - batchAsync(batcher: Batcher): Promise; - - batchedForEach(batcher: BatcherSync, handler: BatchHandlerSync): void; - batchedMap(batcher: BatcherSync, handler: BatchConverterSync): O[]; - - batchedForEachAsync(batcher: Batcher, handler: BatchHandler): Promise; - batchedMapAsync(batcher: Batcher, handler: BatchConverter): Promise; - - batchedForEachInterval(batcher: Batcher, handler: BatchHandler, interval: Interval): Promise; - batchedMapInterval(batcher: Batcher, handler: BatchConverter, interval: Interval): Promise; - lastElement(): T; } -interface BatchContext { - completedBatches: number; - remainingBatches: number; -} - -interface ExecutorResult { - updated: A; - makeNextBatch: boolean; -} - -interface PredicateBatcherCommon { - initial: A; - persistAccumulator?: boolean; -} - -interface Interval { - magnitude: number; - unit: typeof module.exports.TimeUnit; -} - -type BatchConverterSync = (batch: I[], context: BatchContext) => O[]; -type BatchConverterAsync = (batch: I[], context: BatchContext) => Promise; -type BatchConverter = BatchConverterSync | BatchConverterAsync; - -type BatchHandlerSync = (batch: I[], context: BatchContext) => void; -type BatchHandlerAsync = (batch: I[], context: BatchContext) => Promise; -type BatchHandler = BatchHandlerSync | BatchHandlerAsync; - -type BatcherSync = FixedBatcher | PredicateBatcherSync; -type BatcherAsync = PredicateBatcherAsync; -type Batcher = BatcherSync | BatcherAsync; - -type FixedBatcher = { batchSize: number } | { batchCount: number, mode?: typeof module.exports.Mode }; -type PredicateBatcherSync = PredicateBatcherCommon & { executor: (element: I, accumulator: A) => ExecutorResult }; -type PredicateBatcherAsync = PredicateBatcherCommon & { executorAsync: (element: I, accumulator: A) => Promise> }; - - -module.exports.Mode = { - Balanced: 0, - Even: 1 -}; - -module.exports.TimeUnit = { - Milliseconds: 0, - Seconds: 1, - Minutes: 2 -}; - -module.exports.Assign = function () { - - Array.prototype.fixedBatch = function (batcher: FixedBatcher): T[][] { - const batches: T[][] = []; - const length = this.length; - let i = 0; - if ("batchSize" in batcher) { - const { batchSize } = batcher; - while (i < this.length) { - const cap = Math.min(i + batchSize, length); - batches.push(this.slice(i, i = cap)); - } - } else if ("batchCount" in batcher) { - let { batchCount, mode } = batcher; - const resolved = mode || module.exports.Mode.Balanced; - if (batchCount < 1) { - throw new Error("Batch count must be a positive integer!"); - } - if (batchCount === 1) { - return [this]; - } - if (batchCount >= this.length) { - return this.map((element: T) => [element]); - } - - let length = this.length; - let size: number; - - if (length % batchCount === 0) { - size = Math.floor(length / batchCount); - while (i < length) { - batches.push(this.slice(i, i += size)); - } - } else if (resolved === module.exports.Mode.Balanced) { - while (i < length) { - size = Math.ceil((length - i) / batchCount--); - batches.push(this.slice(i, i += size)); - } - } else { - batchCount--; - size = Math.floor(length / batchCount); - if (length % size === 0) { - size--; - } - while (i < size * batchCount) { - batches.push(this.slice(i, i += size)); - } - batches.push(this.slice(size * batchCount)); - } - } - return batches; - }; - - Array.prototype.predicateBatch = function (batcher: PredicateBatcherSync): T[][] { - const batches: T[][] = []; - let batch: T[] = []; - const { executor, initial, persistAccumulator } = batcher; - let accumulator = initial; - for (let element of this) { - const { updated, makeNextBatch } = executor(element, accumulator); - accumulator = updated; - if (!makeNextBatch) { - batch.push(element); - } else { - batches.push(batch); - batch = [element]; - if (!persistAccumulator) { - accumulator = initial; - } - } - } - batches.push(batch); - return batches; - }; - - Array.prototype.predicateBatchAsync = async function (batcher: BatcherAsync): Promise { - const batches: T[][] = []; - let batch: T[] = []; - const { executorAsync, initial, persistAccumulator } = batcher; - let accumulator: A = initial; - for (let element of this) { - const { updated, makeNextBatch } = await executorAsync(element, accumulator); - accumulator = updated; - if (!makeNextBatch) { - batch.push(element); - } else { - batches.push(batch); - batch = [element]; - if (!persistAccumulator) { - accumulator = initial; - } - } - } - batches.push(batch); - return batches; - }; - - Array.prototype.batch = function (batcher: BatcherSync): T[][] { - if ("executor" in batcher) { - return this.predicateBatch(batcher); - } else { - return this.fixedBatch(batcher); - } - }; - - Array.prototype.batchAsync = async function (batcher: Batcher): Promise { - if ("executorAsync" in batcher) { - return this.predicateBatchAsync(batcher); - } else { - return this.batch(batcher); - } - }; - - Array.prototype.batchedForEach = function (batcher: BatcherSync, handler: BatchHandlerSync): void { - if (this.length) { - let completed = 0; - const batches = this.batch(batcher); - const quota = batches.length; - for (let batch of batches) { - const context: BatchContext = { - completedBatches: completed, - remainingBatches: quota - completed, - }; - handler(batch, context); - completed++; - } - } - }; - - Array.prototype.batchedMap = function (batcher: BatcherSync, handler: BatchConverterSync): O[] { - if (!this.length) { - return []; - } - let collector: O[] = []; - let completed = 0; - const batches = this.batch(batcher); - const quota = batches.length; - for (let batch of batches) { - const context: BatchContext = { - completedBatches: completed, - remainingBatches: quota - completed, - }; - collector.push(...handler(batch, context)); - completed++; - } - return collector; - }; - - Array.prototype.batchedForEachAsync = async function (batcher: Batcher, handler: BatchHandler): Promise { - if (this.length) { - let completed = 0; - const batches = await this.batchAsync(batcher); - const quota = batches.length; - for (let batch of batches) { - const context: BatchContext = { - completedBatches: completed, - remainingBatches: quota - completed, - }; - await handler(batch, context); - completed++; - } - } - }; - - Array.prototype.batchedMapAsync = async function (batcher: Batcher, handler: BatchConverter): Promise { - if (!this.length) { - return []; - } - let collector: O[] = []; - let completed = 0; - const batches = await this.batchAsync(batcher); - const quota = batches.length; - for (let batch of batches) { - const context: BatchContext = { - completedBatches: completed, - remainingBatches: quota - completed, - }; - collector.push(...(await handler(batch, context))); - completed++; - } - return collector; - }; - - Array.prototype.batchedForEachInterval = async function (batcher: Batcher, handler: BatchHandler, interval: Interval): Promise { - if (!this.length) { - return; - } - const batches = await this.batchAsync(batcher); - const quota = batches.length; - return new Promise(async resolve => { - const iterator = batches[Symbol.iterator](); - let completed = 0; - while (true) { - const next = iterator.next(); - await new Promise(resolve => { - setTimeout(async () => { - const batch = next.value; - const context: BatchContext = { - completedBatches: completed, - remainingBatches: quota - completed, - }; - await handler(batch, context); - resolve(); - }, convert(interval)); - }); - if (++completed === quota) { - break; - } - } - resolve(); - }); - }; - - Array.prototype.batchedMapInterval = async function (batcher: Batcher, handler: BatchConverter, interval: Interval): Promise { - if (!this.length) { - return []; - } - let collector: O[] = []; - const batches = await this.batchAsync(batcher); - const quota = batches.length; - return new Promise(async resolve => { - const iterator = batches[Symbol.iterator](); - let completed = 0; - while (true) { - const next = iterator.next(); - await new Promise(resolve => { - setTimeout(async () => { - const batch = next.value; - const context: BatchContext = { - completedBatches: completed, - remainingBatches: quota - completed, - }; - collector.push(...(await handler(batch, context))); - resolve(); - }, convert(interval)); - }); - if (++completed === quota) { - resolve(collector); - break; - } - } - }); - }; - - Array.prototype.lastElement = function () { - if (!this.length) { - return undefined; - } - const last: T = this[this.length - 1]; - return last; - }; - +// interface BatchContext { +// completedBatches: number; +// remainingBatches: number; +// } + +// interface ExecutorResult { +// updated: A; +// makeNextBatch: boolean; +// } + +// interface PredicateBatcherCommon { +// initial: A; +// persistAccumulator?: boolean; +// } + +// interface Interval { +// magnitude: number; +// unit: typeof module.exports.TimeUnit; +// } + +// type BatchConverterSync = (batch: I[], context: BatchContext) => O[]; +// type BatchConverterAsync = (batch: I[], context: BatchContext) => Promise; +// type BatchConverter = BatchConverterSync | BatchConverterAsync; + +// type BatchHandlerSync = (batch: I[], context: BatchContext) => void; +// type BatchHandlerAsync = (batch: I[], context: BatchContext) => Promise; +// type BatchHandler = BatchHandlerSync | BatchHandlerAsync; + +// type BatcherSync = FixedBatcher | PredicateBatcherSync; +// type BatcherAsync = PredicateBatcherAsync; +// type Batcher = BatcherSync | BatcherAsync; + +// type FixedBatcher = { batchSize: number } | { batchCount: number, mode?: typeof module.exports.Mode }; +// type PredicateBatcherSync = PredicateBatcherCommon & { executor: (element: I, accumulator: A) => ExecutorResult }; +// type PredicateBatcherAsync = PredicateBatcherCommon & { executorAsync: (element: I, accumulator: A) => Promise> }; + + +// module.exports.Mode = { +// Balanced: 0, +// Even: 1 +// }; + +// module.exports.TimeUnit = { +// Milliseconds: 0, +// Seconds: 1, +// Minutes: 2 +// }; + +// module.exports.Assign = function () { + +// Array.prototype.fixedBatch = function (batcher: FixedBatcher): T[][] { +// const batches: T[][] = []; +// const length = this.length; +// let i = 0; +// if ("batchSize" in batcher) { +// const { batchSize } = batcher; +// while (i < this.length) { +// const cap = Math.min(i + batchSize, length); +// batches.push(this.slice(i, i = cap)); +// } +// } else if ("batchCount" in batcher) { +// let { batchCount, mode } = batcher; +// const resolved = mode || module.exports.Mode.Balanced; +// if (batchCount < 1) { +// throw new Error("Batch count must be a positive integer!"); +// } +// if (batchCount === 1) { +// return [this]; +// } +// if (batchCount >= this.length) { +// return this.map((element: T) => [element]); +// } + +// let length = this.length; +// let size: number; + +// if (length % batchCount === 0) { +// size = Math.floor(length / batchCount); +// while (i < length) { +// batches.push(this.slice(i, i += size)); +// } +// } else if (resolved === module.exports.Mode.Balanced) { +// while (i < length) { +// size = Math.ceil((length - i) / batchCount--); +// batches.push(this.slice(i, i += size)); +// } +// } else { +// batchCount--; +// size = Math.floor(length / batchCount); +// if (length % size === 0) { +// size--; +// } +// while (i < size * batchCount) { +// batches.push(this.slice(i, i += size)); +// } +// batches.push(this.slice(size * batchCount)); +// } +// } +// return batches; +// }; + +// Array.prototype.predicateBatch = function (batcher: PredicateBatcherSync): T[][] { +// const batches: T[][] = []; +// let batch: T[] = []; +// const { executor, initial, persistAccumulator } = batcher; +// let accumulator = initial; +// for (let element of this) { +// const { updated, makeNextBatch } = executor(element, accumulator); +// accumulator = updated; +// if (!makeNextBatch) { +// batch.push(element); +// } else { +// batches.push(batch); +// batch = [element]; +// if (!persistAccumulator) { +// accumulator = initial; +// } +// } +// } +// batches.push(batch); +// return batches; +// }; + +// Array.prototype.predicateBatchAsync = async function (batcher: BatcherAsync): Promise { +// const batches: T[][] = []; +// let batch: T[] = []; +// const { executorAsync, initial, persistAccumulator } = batcher; +// let accumulator: A = initial; +// for (let element of this) { +// const { updated, makeNextBatch } = await executorAsync(element, accumulator); +// accumulator = updated; +// if (!makeNextBatch) { +// batch.push(element); +// } else { +// batches.push(batch); +// batch = [element]; +// if (!persistAccumulator) { +// accumulator = initial; +// } +// } +// } +// batches.push(batch); +// return batches; +// }; + +// Array.prototype.batch = function (batcher: BatcherSync): T[][] { +// if ("executor" in batcher) { +// return this.predicateBatch(batcher); +// } else { +// return this.fixedBatch(batcher); +// } +// }; + +// Array.prototype.batchAsync = async function (batcher: Batcher): Promise { +// if ("executorAsync" in batcher) { +// return this.predicateBatchAsync(batcher); +// } else { +// return this.batch(batcher); +// } +// }; + +// Array.prototype.batchedForEach = function (batcher: BatcherSync, handler: BatchHandlerSync): void { +// if (this.length) { +// let completed = 0; +// const batches = this.batch(batcher); +// const quota = batches.length; +// for (let batch of batches) { +// const context: BatchContext = { +// completedBatches: completed, +// remainingBatches: quota - completed, +// }; +// handler(batch, context); +// completed++; +// } +// } +// }; + +// Array.prototype.batchedMap = function (batcher: BatcherSync, handler: BatchConverterSync): O[] { +// if (!this.length) { +// return []; +// } +// let collector: O[] = []; +// let completed = 0; +// const batches = this.batch(batcher); +// const quota = batches.length; +// for (let batch of batches) { +// const context: BatchContext = { +// completedBatches: completed, +// remainingBatches: quota - completed, +// }; +// collector.push(...handler(batch, context)); +// completed++; +// } +// return collector; +// }; + +// Array.prototype.batchedForEachAsync = async function (batcher: Batcher, handler: BatchHandler): Promise { +// if (this.length) { +// let completed = 0; +// const batches = await this.batchAsync(batcher); +// const quota = batches.length; +// for (let batch of batches) { +// const context: BatchContext = { +// completedBatches: completed, +// remainingBatches: quota - completed, +// }; +// await handler(batch, context); +// completed++; +// } +// } +// }; + +// Array.prototype.batchedMapAsync = async function (batcher: Batcher, handler: BatchConverter): Promise { +// if (!this.length) { +// return []; +// } +// let collector: O[] = []; +// let completed = 0; +// const batches = await this.batchAsync(batcher); +// const quota = batches.length; +// for (let batch of batches) { +// const context: BatchContext = { +// completedBatches: completed, +// remainingBatches: quota - completed, +// }; +// collector.push(...(await handler(batch, context))); +// completed++; +// } +// return collector; +// }; + +// Array.prototype.batchedForEachInterval = async function (batcher: Batcher, handler: BatchHandler, interval: Interval): Promise { +// if (!this.length) { +// return; +// } +// const batches = await this.batchAsync(batcher); +// const quota = batches.length; +// return new Promise(async resolve => { +// const iterator = batches[Symbol.iterator](); +// let completed = 0; +// while (true) { +// const next = iterator.next(); +// await new Promise(resolve => { +// setTimeout(async () => { +// const batch = next.value; +// const context: BatchContext = { +// completedBatches: completed, +// remainingBatches: quota - completed, +// }; +// await handler(batch, context); +// resolve(); +// }, convert(interval)); +// }); +// if (++completed === quota) { +// break; +// } +// } +// resolve(); +// }); +// }; + +// Array.prototype.batchedMapInterval = async function (batcher: Batcher, handler: BatchConverter, interval: Interval): Promise { +// if (!this.length) { +// return []; +// } +// let collector: O[] = []; +// const batches = await this.batchAsync(batcher); +// const quota = batches.length; +// return new Promise(async resolve => { +// const iterator = batches[Symbol.iterator](); +// let completed = 0; +// while (true) { +// const next = iterator.next(); +// await new Promise(resolve => { +// setTimeout(async () => { +// const batch = next.value; +// const context: BatchContext = { +// completedBatches: completed, +// remainingBatches: quota - completed, +// }; +// collector.push(...(await handler(batch, context))); +// resolve(); +// }, convert(interval)); +// }); +// if (++completed === quota) { +// resolve(collector); +// break; +// } +// } +// }); +// }; + +Array.prototype.lastElement = function () { + if (!this.length) { + return undefined; + } + const last: T = this[this.length - 1]; + return last; }; -const convert = (interval: Interval) => { - const { magnitude, unit } = interval; - switch (unit) { - default: - case module.exports.TimeUnit.Milliseconds: - return magnitude; - case module.exports.TimeUnit.Seconds: - return magnitude * 1000; - case module.exports.TimeUnit.Minutes: - return magnitude * 1000 * 60; - } -}; \ No newline at end of file +// }; + +// const convert = (interval: Interval) => { +// const { magnitude, unit } = interval; +// switch (unit) { +// default: +// case module.exports.TimeUnit.Milliseconds: +// return magnitude; +// case module.exports.TimeUnit.Seconds: +// return magnitude * 1000; +// case module.exports.TimeUnit.Minutes: +// return magnitude * 1000 * 60; +// } +// }; \ No newline at end of file diff --git a/src/extensions/Extensions.ts b/src/extensions/Extensions.ts index 1bcebd0e2..1391140b9 100644 --- a/src/extensions/Extensions.ts +++ b/src/extensions/Extensions.ts @@ -2,6 +2,6 @@ const ArrayExtensions = require("./ArrayExtensions"); const StringExtensions = require("./StringExtensions"); module.exports.AssignExtensions = function () { - ArrayExtensions.Assign(); + // ArrayExtensions.Assign(); StringExtensions.Assign(); }; \ No newline at end of file diff --git a/src/server/apis/google/GooglePhotosUploadUtils.ts b/src/server/apis/google/GooglePhotosUploadUtils.ts index fc6772ffd..29575763c 100644 --- a/src/server/apis/google/GooglePhotosUploadUtils.ts +++ b/src/server/apis/google/GooglePhotosUploadUtils.ts @@ -7,7 +7,7 @@ import { Opt } from '../../../new_fields/Doc'; import * as sharp from 'sharp'; import { MediaItemCreationResult } from './SharedTypes'; import { NewMediaItem } from "../../index"; -const { TimeUnit } = require("../../../extensions/ArrayExtensions"); +import { batchedMapInterval, FixedBatcher, TimeUnit, Interval } from "array-batcher"; const uploadDirectory = path.join(__dirname, "../../public/files/"); @@ -81,10 +81,10 @@ export namespace GooglePhotosUploadUtils { }); })).newMediaItemResults; }; - const batcher = { batchSize: 50 }; - const interval = { magnitude: 100, unit: TimeUnit.Milliseconds }; + const batcher: FixedBatcher = { batchSize: 50 }; + const interval: Interval = { magnitude: 100, unit: TimeUnit.Milliseconds }; - const newMediaItemResults = await newMediaItems.batchedMapInterval(batcher, createFromUploadTokens, interval); + const newMediaItemResults = await batchedMapInterval(newMediaItems, batcher, createFromUploadTokens, interval); return { newMediaItemResults }; }; diff --git a/src/server/credentials/google_docs_token.json b/src/server/credentials/google_docs_token.json index 7c49eed43..cdea139a3 100644 --- a/src/server/credentials/google_docs_token.json +++ b/src/server/credentials/google_docs_token.json @@ -1 +1 @@ -{"access_token":"ya29.ImCJB1Y8Z8vgUH4vyYA9xwqvLg281kOQKfA8_AGs_EqF1VKQVWfZsMoYkPJN3QwJmIUxlzTO1N-ehUGIxu0Jq3kKR-zzW7rQIMgeQu32OHogK4kvFxpM7l7RNYRw_9x22I0","refresh_token":"1/HTv_xFHszu2Nf3iiFrUTaeKzC_Vp2-6bpIB06xW_WHI","scope":"https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/documents.readonly https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/photoslibrary https://www.googleapis.com/auth/photoslibrary.appendonly https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/photoslibrary.sharing","token_type":"Bearer","expiry_date":1568935635717} \ No newline at end of file +{"access_token":"ya29.ImCJB_jd-XlGcIHAHgN2Zl3BWQ6sMHdeMMuRxU6sPCbAYIT8hXws-WDmQf65ZY1f-0d3y7HcCcuOxtZJ_0IcBb1-yIBxiOf3VJWPmvjGiJQq_mANGVSSmsBHhqpIaYkeQN0","refresh_token":"1/HTv_xFHszu2Nf3iiFrUTaeKzC_Vp2-6bpIB06xW_WHI","scope":"https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/documents.readonly https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/photoslibrary https://www.googleapis.com/auth/photoslibrary.appendonly https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/photoslibrary.sharing","token_type":"Bearer","expiry_date":1568973665276} \ No newline at end of file diff --git a/src/server/index.ts b/src/server/index.ts index e3e4221cf..e03079d66 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -47,9 +47,7 @@ const mongoose = require('mongoose'); const probe = require("probe-image-size"); import * as qs from 'query-string'; import { Opt } from '../new_fields/Doc'; -const Extensions = require("../extensions/Extensions"); -const ArrayExtensions = require("../extensions/ArrayExtensions"); - +import { batchedMapInterval, TimeUnit } from "array-batcher"; const download = (url: string, dest: fs.PathLike) => request.get(url).pipe(fs.createWriteStream(dest)); let youtubeApiKey: string; @@ -101,8 +99,6 @@ enum Method { POST } -Extensions.AssignExtensions(); - /** * Please invoke this function when adding a new route to Dash's server. * It ensures that any requests leading to or containing user-sensitive information @@ -861,9 +857,9 @@ app.post(RouteStore.googlePhotosMediaUpload, async (req, res) => { return newMediaItems; }; const batcher = { batchSize: 25 }; - const interval = { magnitude: 100, unit: ArrayExtensions.TimeUnit.Milliseconds }; + const interval = { magnitude: 100, unit: TimeUnit.Milliseconds }; - const newMediaItems = await mediaInput.batchedMapInterval(batcher, dispatchUpload, interval); + const newMediaItems = await batchedMapInterval(mediaInput, batcher, dispatchUpload, interval); if (failed) { return _error(res, tokenError); -- cgit v1.2.3-70-g09d2 From 8d9c9d0f51be91ee10b631be9f464af0822bb9be Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Fri, 20 Sep 2019 16:39:14 -0400 Subject: integrated with array batcher npm module --- package.json | 2 +- .../util/Import & Export/DirectoryImportBox.tsx | 31 +++++++------- src/server/apis/google/GooglePhotosUploadUtils.ts | 49 +++++++++++----------- src/server/credentials/google_docs_token.json | 2 +- src/server/index.ts | 36 ++++++++-------- 5 files changed, 61 insertions(+), 59 deletions(-) (limited to 'package.json') diff --git a/package.json b/package.json index b20c31a7a..45babb358 100644 --- a/package.json +++ b/package.json @@ -114,7 +114,7 @@ "@types/youtube": "0.0.38", "adm-zip": "^0.4.13", "archiver": "^3.0.3", - "array-batcher": "^1.0.2", + "array-batcher": "^1.0.6", "async": "^2.6.2", "babel-runtime": "^6.26.0", "bcrypt-nodejs": "0.0.3", diff --git a/src/client/util/Import & Export/DirectoryImportBox.tsx b/src/client/util/Import & Export/DirectoryImportBox.tsx index 762302bc8..26a00dc7c 100644 --- a/src/client/util/Import & Export/DirectoryImportBox.tsx +++ b/src/client/util/Import & Export/DirectoryImportBox.tsx @@ -20,7 +20,7 @@ import { listSpec } from "../../../new_fields/Schema"; import { GooglePhotos } from "../../apis/google_docs/GooglePhotosClientUtils"; import { SchemaHeaderField } from "../../../new_fields/SchemaHeaderField"; import "./DirectoryImportBox.scss"; -import { batchedMapAsync } from "array-batcher"; +import BatchedArray from "array-batcher"; const unsupported = ["text/html", "text/plain"]; interface FileResponse { @@ -104,19 +104,22 @@ export default class DirectoryImportBox extends React.Component runInAction(() => this.phase = `Internal: uploading ${this.quota - this.completed} files to Dash...`); - const uploads = await batchedMapAsync(validated, { batchSize: 15 }, async batch => { - const formData = new FormData(); - const parameters = { method: 'POST', body: formData }; - - batch.forEach(file => { - sizes.push(file.size); - modifiedDates.push(file.lastModified); - formData.append(Utils.GenerateGuid(), file); - }); - - const responses = await (await fetch(RouteStore.upload, parameters)).json(); - runInAction(() => this.completed += batch.length); - return responses as FileResponse[]; + const uploads = await BatchedArray.from(validated).batchedMapAsync({ + batcher: { batchSize: 15 }, + converter: async batch => { + const formData = new FormData(); + const parameters = { method: 'POST', body: formData }; + + batch.forEach(file => { + sizes.push(file.size); + modifiedDates.push(file.lastModified); + formData.append(Utils.GenerateGuid(), file); + }); + + const responses = await (await fetch(RouteStore.upload, parameters)).json(); + runInAction(() => this.completed += batch.length); + return responses as FileResponse[]; + } }); await Promise.all(uploads.map(async upload => { diff --git a/src/server/apis/google/GooglePhotosUploadUtils.ts b/src/server/apis/google/GooglePhotosUploadUtils.ts index 29575763c..40564ff3b 100644 --- a/src/server/apis/google/GooglePhotosUploadUtils.ts +++ b/src/server/apis/google/GooglePhotosUploadUtils.ts @@ -7,7 +7,7 @@ import { Opt } from '../../../new_fields/Doc'; import * as sharp from 'sharp'; import { MediaItemCreationResult } from './SharedTypes'; import { NewMediaItem } from "../../index"; -import { batchedMapInterval, FixedBatcher, TimeUnit, Interval } from "array-batcher"; +import BatchedArray, { FixedBatcher, TimeUnit, Interval } from "array-batcher"; const uploadDirectory = path.join(__dirname, "../../public/files/"); @@ -62,30 +62,29 @@ export namespace GooglePhotosUploadUtils { }; export const CreateMediaItems = async (newMediaItems: NewMediaItem[], album?: { id: string }): Promise => { - const createFromUploadTokens = async (batch: NewMediaItem[]) => { - const parameters = { - method: 'POST', - headers: headers('json'), - uri: prepend('mediaItems:batchCreate'), - body: { newMediaItems: batch } as any, - json: true - }; - album && (parameters.body.albumId = album.id); - return (await new Promise((resolve, reject) => { - request(parameters, (error, _response, body) => { - if (error) { - reject(error); - } else { - resolve(body); - } - }); - })).newMediaItemResults; - }; - const batcher: FixedBatcher = { batchSize: 50 }; - const interval: Interval = { magnitude: 100, unit: TimeUnit.Milliseconds }; - - const newMediaItemResults = await batchedMapInterval(newMediaItems, batcher, createFromUploadTokens, interval); - + const newMediaItemResults = await BatchedArray.from(newMediaItems).batchedMapInterval({ + batcher: { batchSize: 50 }, + interval: { magnitude: 100, unit: TimeUnit.Milliseconds }, + converter: async (batch: NewMediaItem[]) => { + const parameters = { + method: 'POST', + headers: headers('json'), + uri: prepend('mediaItems:batchCreate'), + body: { newMediaItems: batch } as any, + json: true + }; + album && (parameters.body.albumId = album.id); + return (await new Promise((resolve, reject) => { + request(parameters, (error, _response, body) => { + if (error) { + reject(error); + } else { + resolve(body); + } + }); + })).newMediaItemResults; + } + }); return { newMediaItemResults }; }; diff --git a/src/server/credentials/google_docs_token.json b/src/server/credentials/google_docs_token.json index cdea139a3..9b0a649ac 100644 --- a/src/server/credentials/google_docs_token.json +++ b/src/server/credentials/google_docs_token.json @@ -1 +1 @@ -{"access_token":"ya29.ImCJB_jd-XlGcIHAHgN2Zl3BWQ6sMHdeMMuRxU6sPCbAYIT8hXws-WDmQf65ZY1f-0d3y7HcCcuOxtZJ_0IcBb1-yIBxiOf3VJWPmvjGiJQq_mANGVSSmsBHhqpIaYkeQN0","refresh_token":"1/HTv_xFHszu2Nf3iiFrUTaeKzC_Vp2-6bpIB06xW_WHI","scope":"https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/documents.readonly https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/photoslibrary https://www.googleapis.com/auth/photoslibrary.appendonly https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/photoslibrary.sharing","token_type":"Bearer","expiry_date":1568973665276} \ No newline at end of file +{"access_token":"ya29.ImCJB3_-mtKbCG8L7nQt3CDfn4HR2_xqUw8bUQkvaZZgxJZ8-WUlHe6UaSOmGtrvI4QabpYRGutYGSTUdc9bvIYooerZgAz80uk7-KEbnAxpwydRE-TK_1_VyWYZ2YW6Ht8","refresh_token":"1/HTv_xFHszu2Nf3iiFrUTaeKzC_Vp2-6bpIB06xW_WHI","scope":"https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/documents.readonly https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/photoslibrary https://www.googleapis.com/auth/photoslibrary.appendonly https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/photoslibrary.sharing","token_type":"Bearer","expiry_date":1569013995678} \ No newline at end of file diff --git a/src/server/index.ts b/src/server/index.ts index e03079d66..12ceb9886 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -47,7 +47,7 @@ const mongoose = require('mongoose'); const probe = require("probe-image-size"); import * as qs from 'query-string'; import { Opt } from '../new_fields/Doc'; -import { batchedMapInterval, TimeUnit } from "array-batcher"; +import BatchedArray, { TimeUnit } from "array-batcher"; const download = (url: string, dest: fs.PathLike) => request.get(url).pipe(fs.createWriteStream(dest)); let youtubeApiKey: string; @@ -841,25 +841,25 @@ app.post(RouteStore.googlePhotosMediaUpload, async (req, res) => { let failed = 0; - const dispatchUpload = async (batch: GooglePhotosUploadUtils.MediaInput[]) => { - const newMediaItems: NewMediaItem[] = []; - for (let element of batch) { - const uploadToken = await GooglePhotosUploadUtils.DispatchGooglePhotosUpload(element.url); - if (!uploadToken) { - failed++; - } else { - newMediaItems.push({ - description: element.description, - simpleMediaItem: { uploadToken } - }); + const newMediaItems = await BatchedArray.from(mediaInput).batchedMapInterval({ + batcher: { batchSize: 25 }, + interval: { magnitude: 100, unit: TimeUnit.Milliseconds }, + converter: async (batch: GooglePhotosUploadUtils.MediaInput[]) => { + const newMediaItems: NewMediaItem[] = []; + for (let element of batch) { + const uploadToken = await GooglePhotosUploadUtils.DispatchGooglePhotosUpload(element.url); + if (!uploadToken) { + failed++; + } else { + newMediaItems.push({ + description: element.description, + simpleMediaItem: { uploadToken } + }); + } } + return newMediaItems; } - return newMediaItems; - }; - const batcher = { batchSize: 25 }; - const interval = { magnitude: 100, unit: TimeUnit.Milliseconds }; - - const newMediaItems = await batchedMapInterval(mediaInput, batcher, dispatchUpload, interval); + }); if (failed) { return _error(res, tokenError); -- cgit v1.2.3-70-g09d2 From f529d7d22162689c08830418da67a89778111e16 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Sat, 21 Sep 2019 14:27:10 -0400 Subject: final batcher fixes --- package.json | 2 +- .../util/Import & Export/DirectoryImportBox.tsx | 29 ++++++++++------------ src/server/apis/google/GooglePhotosUploadUtils.ts | 11 ++++---- src/server/credentials/google_docs_token.json | 2 +- src/server/index.ts | 11 ++++---- 5 files changed, 25 insertions(+), 30 deletions(-) (limited to 'package.json') diff --git a/package.json b/package.json index 45babb358..f049f8826 100644 --- a/package.json +++ b/package.json @@ -114,7 +114,7 @@ "@types/youtube": "0.0.38", "adm-zip": "^0.4.13", "archiver": "^3.0.3", - "array-batcher": "^1.0.6", + "array-batcher": "^1.0.7", "async": "^2.6.2", "babel-runtime": "^6.26.0", "bcrypt-nodejs": "0.0.3", diff --git a/src/client/util/Import & Export/DirectoryImportBox.tsx b/src/client/util/Import & Export/DirectoryImportBox.tsx index 26a00dc7c..8948b73f7 100644 --- a/src/client/util/Import & Export/DirectoryImportBox.tsx +++ b/src/client/util/Import & Export/DirectoryImportBox.tsx @@ -104,22 +104,19 @@ export default class DirectoryImportBox extends React.Component runInAction(() => this.phase = `Internal: uploading ${this.quota - this.completed} files to Dash...`); - const uploads = await BatchedArray.from(validated).batchedMapAsync({ - batcher: { batchSize: 15 }, - converter: async batch => { - const formData = new FormData(); - const parameters = { method: 'POST', body: formData }; - - batch.forEach(file => { - sizes.push(file.size); - modifiedDates.push(file.lastModified); - formData.append(Utils.GenerateGuid(), file); - }); - - const responses = await (await fetch(RouteStore.upload, parameters)).json(); - runInAction(() => this.completed += batch.length); - return responses as FileResponse[]; - } + const uploads = await BatchedArray.from(validated, { batchSize: 15 }).batchedMapAsync(async batch => { + const formData = new FormData(); + const parameters = { method: 'POST', body: formData }; + + batch.forEach(file => { + sizes.push(file.size); + modifiedDates.push(file.lastModified); + formData.append(Utils.GenerateGuid(), file); + }); + + const responses = await (await fetch(RouteStore.upload, parameters)).json(); + runInAction(() => this.completed += batch.length); + return responses as FileResponse[]; }); await Promise.all(uploads.map(async upload => { diff --git a/src/server/apis/google/GooglePhotosUploadUtils.ts b/src/server/apis/google/GooglePhotosUploadUtils.ts index 40564ff3b..4dc252577 100644 --- a/src/server/apis/google/GooglePhotosUploadUtils.ts +++ b/src/server/apis/google/GooglePhotosUploadUtils.ts @@ -62,10 +62,8 @@ export namespace GooglePhotosUploadUtils { }; export const CreateMediaItems = async (newMediaItems: NewMediaItem[], album?: { id: string }): Promise => { - const newMediaItemResults = await BatchedArray.from(newMediaItems).batchedMapInterval({ - batcher: { batchSize: 50 }, - interval: { magnitude: 100, unit: TimeUnit.Milliseconds }, - converter: async (batch: NewMediaItem[]) => { + const newMediaItemResults = await BatchedArray.from(newMediaItems, { batchSize: 50 }).batchedMapInterval( + async (batch: NewMediaItem[]) => { const parameters = { method: 'POST', headers: headers('json'), @@ -83,8 +81,9 @@ export namespace GooglePhotosUploadUtils { } }); })).newMediaItemResults; - } - }); + }, + { magnitude: 100, unit: TimeUnit.Milliseconds } + ); return { newMediaItemResults }; }; diff --git a/src/server/credentials/google_docs_token.json b/src/server/credentials/google_docs_token.json index 9b0a649ac..ee44c3f30 100644 --- a/src/server/credentials/google_docs_token.json +++ b/src/server/credentials/google_docs_token.json @@ -1 +1 @@ -{"access_token":"ya29.ImCJB3_-mtKbCG8L7nQt3CDfn4HR2_xqUw8bUQkvaZZgxJZ8-WUlHe6UaSOmGtrvI4QabpYRGutYGSTUdc9bvIYooerZgAz80uk7-KEbnAxpwydRE-TK_1_VyWYZ2YW6Ht8","refresh_token":"1/HTv_xFHszu2Nf3iiFrUTaeKzC_Vp2-6bpIB06xW_WHI","scope":"https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/documents.readonly https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/photoslibrary https://www.googleapis.com/auth/photoslibrary.appendonly https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/photoslibrary.sharing","token_type":"Bearer","expiry_date":1569013995678} \ No newline at end of file +{"access_token":"ya29.GlyKBznz91v_qb8RYt4PT40Hp106N08Yk64UjMAKllBsIqJQEzBkxLbB5q5paydywHzguQYSNup5fT7ojJTDU4CMZdPbPKGcjQz17w_CospcG-8Buz94KZptvlQ_pQ","refresh_token":"1/HTv_xFHszu2Nf3iiFrUTaeKzC_Vp2-6bpIB06xW_WHI","scope":"https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/documents.readonly https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/photoslibrary https://www.googleapis.com/auth/photoslibrary.appendonly https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/photoslibrary.sharing","token_type":"Bearer","expiry_date":1569093749804} \ No newline at end of file diff --git a/src/server/index.ts b/src/server/index.ts index 12ceb9886..4c4cb84d6 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -841,10 +841,8 @@ app.post(RouteStore.googlePhotosMediaUpload, async (req, res) => { let failed = 0; - const newMediaItems = await BatchedArray.from(mediaInput).batchedMapInterval({ - batcher: { batchSize: 25 }, - interval: { magnitude: 100, unit: TimeUnit.Milliseconds }, - converter: async (batch: GooglePhotosUploadUtils.MediaInput[]) => { + const newMediaItems = await BatchedArray.from(mediaInput, { batchSize: 25 }).batchedMapInterval( + async (batch: GooglePhotosUploadUtils.MediaInput[]) => { const newMediaItems: NewMediaItem[] = []; for (let element of batch) { const uploadToken = await GooglePhotosUploadUtils.DispatchGooglePhotosUpload(element.url); @@ -858,8 +856,9 @@ app.post(RouteStore.googlePhotosMediaUpload, async (req, res) => { } } return newMediaItems; - } - }); + }, + { magnitude: 100, unit: TimeUnit.Milliseconds } + ); if (failed) { return _error(res, tokenError); -- cgit v1.2.3-70-g09d2 From db174b3adfc36678925b067aa398ecb0415ea0f8 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Fri, 27 Sep 2019 04:18:24 -0400 Subject: batcher updates --- package.json | 2 +- src/client/util/Import & Export/DirectoryImportBox.tsx | 3 ++- src/server/DashUploadUtils.ts | 4 ++-- src/server/apis/google/GooglePhotosUploadUtils.ts | 8 ++++---- src/server/index.ts | 8 ++++---- 5 files changed, 13 insertions(+), 12 deletions(-) (limited to 'package.json') diff --git a/package.json b/package.json index ea31b08a3..7a9e29f50 100644 --- a/package.json +++ b/package.json @@ -114,7 +114,7 @@ "@types/youtube": "0.0.38", "adm-zip": "^0.4.13", "archiver": "^3.0.3", - "array-batcher": "^1.0.7", + "array-batcher": "^1.1.3", "async": "^2.6.2", "babel-runtime": "^6.26.0", "bcrypt-nodejs": "0.0.3", diff --git a/src/client/util/Import & Export/DirectoryImportBox.tsx b/src/client/util/Import & Export/DirectoryImportBox.tsx index de5287456..d3f81b992 100644 --- a/src/client/util/Import & Export/DirectoryImportBox.tsx +++ b/src/client/util/Import & Export/DirectoryImportBox.tsx @@ -20,10 +20,11 @@ import { listSpec } from "../../../new_fields/Schema"; import { GooglePhotos } from "../../apis/google_docs/GooglePhotosClientUtils"; import { SchemaHeaderField } from "../../../new_fields/SchemaHeaderField"; import "./DirectoryImportBox.scss"; -import BatchedArray from "array-batcher"; import { Identified } from "../../Network"; +import { BatchedArray } from "array-batcher"; const unsupported = ["text/html", "text/plain"]; + interface FileResponse { name: string; path: string; diff --git a/src/server/DashUploadUtils.ts b/src/server/DashUploadUtils.ts index 04a3559dc..4230e9b17 100644 --- a/src/server/DashUploadUtils.ts +++ b/src/server/DashUploadUtils.ts @@ -50,10 +50,10 @@ export namespace DashUploadUtils { * event that @param filename is not specified * * @returns {UploadInformation} This method returns - * 1) the paths to the uploaded image + * 1) the paths to the uploaded images (plural due to resizing) * 2) the file name of each of the resized images * 3) the size of the image, in bytes (4432130) - * 4) the content type of the image (jpg | png | etc.) + * 4) the content type of the image, i.e. image/(jpeg | png | ...) */ export const UploadImage = async (source: string, filename?: string, prefix: string = ""): Promise => { const metadata = await InspectImage(source); diff --git a/src/server/apis/google/GooglePhotosUploadUtils.ts b/src/server/apis/google/GooglePhotosUploadUtils.ts index 507a868a3..16c4f6c3a 100644 --- a/src/server/apis/google/GooglePhotosUploadUtils.ts +++ b/src/server/apis/google/GooglePhotosUploadUtils.ts @@ -3,7 +3,7 @@ import { GoogleApiServerUtils } from './GoogleApiServerUtils'; import * as path from 'path'; import { MediaItemCreationResult } from './SharedTypes'; import { NewMediaItem } from "../../index"; -import BatchedArray, { FixedBatcher, TimeUnit, Interval } from "array-batcher"; +import { BatchedArray, TimeUnit } from 'array-batcher'; export namespace GooglePhotosUploadUtils { @@ -53,7 +53,8 @@ export namespace GooglePhotosUploadUtils { }; export const CreateMediaItems = async (newMediaItems: NewMediaItem[], album?: { id: string }): Promise => { - const newMediaItemResults = await BatchedArray.from(newMediaItems, { batchSize: 50 }).batchedMapInterval( + const newMediaItemResults = await BatchedArray.from(newMediaItems, { batchSize: 50 }).batchedMapPatientInterval( + { magnitude: 100, unit: TimeUnit.Milliseconds }, async (batch: NewMediaItem[]) => { const parameters = { method: 'POST', @@ -72,8 +73,7 @@ export namespace GooglePhotosUploadUtils { } }); })).newMediaItemResults; - }, - { magnitude: 100, unit: TimeUnit.Milliseconds } + } ); return { newMediaItemResults }; }; diff --git a/src/server/index.ts b/src/server/index.ts index 3da726cfa..9a778b88c 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -47,8 +47,8 @@ const mongoose = require('mongoose'); const probe = require("probe-image-size"); import * as qs from 'query-string'; import { Opt } from '../new_fields/Doc'; -import BatchedArray, { TimeUnit } from "array-batcher"; import { DashUploadUtils } from './DashUploadUtils'; +import { BatchedArray, TimeUnit } from 'array-batcher'; const download = (url: string, dest: fs.PathLike) => request.get(url).pipe(fs.createWriteStream(dest)); let youtubeApiKey: string; @@ -848,7 +848,8 @@ app.post(RouteStore.googlePhotosMediaUpload, async (req, res) => { let failed = 0; - const newMediaItems = await BatchedArray.from(media, { batchSize: 25 }).batchedMapInterval( + const newMediaItems = await BatchedArray.from(media, { batchSize: 25 }).batchedMapPatientInterval( + { magnitude: 100, unit: TimeUnit.Milliseconds }, async (batch: GooglePhotosUploadUtils.MediaInput[]) => { const newMediaItems: NewMediaItem[] = []; for (let element of batch) { @@ -863,8 +864,7 @@ app.post(RouteStore.googlePhotosMediaUpload, async (req, res) => { } } return newMediaItems; - }, - { magnitude: 100, unit: TimeUnit.Milliseconds } + } ); if (failed) { -- cgit v1.2.3-70-g09d2