From c9a7d747916c31730f71479d8e516ba0ed2c658f Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Fri, 30 Aug 2019 15:51:13 -0400 Subject: complete transfer except for list items --- package.json | 2 ++ 1 file changed, 2 insertions(+) (limited to 'package.json') diff --git a/package.json b/package.json index cd60b7b55..f98224469 100644 --- a/package.json +++ b/package.json @@ -59,6 +59,7 @@ "@types/bluebird": "^3.5.25", "@types/body-parser": "^1.17.0", "@types/classnames": "^2.2.8", + "@types/color": "^3.0.0", "@types/connect-flash": "0.0.34", "@types/cookie-parser": "^1.4.1", "@types/cookie-session": "^2.0.36", @@ -121,6 +122,7 @@ "canvas": "^2.5.0", "child_process": "^1.0.2", "class-transformer": "^0.2.0", + "color": "^3.1.2", "connect-flash": "^0.1.1", "connect-mongo": "^2.0.3", "cookie-parser": "^1.4.4", -- cgit v1.2.3-70-g09d2 From 77e08a4362ba8fab4cab361fcb472702c97edf15 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Sun, 1 Sep 2019 13:05:54 -0400 Subject: initial commit --- package.json | 3 + src/client/views/MainView.tsx | 4 +- src/server/apis/google/GoogleApiServerUtils.ts | 13 +- src/server/apis/google/GooglePhotosUtils.ts | 12 + src/server/authentication/config/passport.ts | 18 +- src/server/credentials/auth.json | 12 + .../credentials/google_docs_credentials.json | 12 +- src/server/credentials/google_docs_token.json | 2 +- .../credentials/google_photos_credentials.ts | 35 ++ src/server/index.ts | 27 + src/typings/index.d.ts | 632 +++++++++++---------- 11 files changed, 447 insertions(+), 323 deletions(-) create mode 100644 src/server/apis/google/GooglePhotosUtils.ts create mode 100644 src/server/credentials/auth.json create mode 100644 src/server/credentials/google_photos_credentials.ts (limited to 'package.json') diff --git a/package.json b/package.json index f98224469..f0f2b467e 100644 --- a/package.json +++ b/package.json @@ -81,6 +81,7 @@ "@types/node": "^10.12.30", "@types/nodemailer": "^4.6.6", "@types/passport": "^1.0.0", + "@types/passport-google-oauth20": "^2.0.2", "@types/passport-local": "^1.0.33", "@types/pdfjs-dist": "^2.0.0", "@types/prosemirror-commands": "^1.0.1", @@ -141,6 +142,7 @@ "golden-layout": "^1.5.9", "google-auth-library": "^4.2.4", "googleapis": "^40.0.0", + "googlephotos": "^0.2.1", "howler": "^2.1.2", "html-to-image": "^0.1.0", "i": "^0.3.6", @@ -166,6 +168,7 @@ "npm": "^6.10.3", "p-limit": "^2.2.0", "passport": "^0.4.0", + "passport-google-oauth20": "^2.0.0", "passport-local": "^1.0.0", "pdfjs-dist": "^2.0.943", "probe-image-size": "^4.0.0", diff --git a/src/client/views/MainView.tsx b/src/client/views/MainView.tsx index ab87f0c7b..df0718072 100644 --- a/src/client/views/MainView.tsx +++ b/src/client/views/MainView.tsx @@ -15,7 +15,7 @@ import { listSpec } from '../../new_fields/Schema'; import { BoolCast, Cast, FieldValue, StrCast } from '../../new_fields/Types'; import { CurrentUserUtils } from '../../server/authentication/models/current_user_utils'; import { RouteStore } from '../../server/RouteStore'; -import { emptyFunction, returnOne, returnTrue, Utils, returnEmptyString } from '../../Utils'; +import { emptyFunction, returnOne, returnTrue, Utils, returnEmptyString, PostToServer } from '../../Utils'; import { DocServer } from '../DocServer'; import { Docs } from '../documents/Documents'; import { ClientUtils } from '../util/ClientUtils'; @@ -128,6 +128,8 @@ export class MainView extends React.Component { window.removeEventListener("keydown", KeyManager.Instance.handle); window.addEventListener("keydown", KeyManager.Instance.handle); + PostToServer('/googleDocs/Photos/Test', {}); + reaction(() => { let workspaces = CurrentUserUtils.UserDocument.workspaces; let recent = CurrentUserUtils.UserDocument.recentlyClosed; diff --git a/src/server/apis/google/GoogleApiServerUtils.ts b/src/server/apis/google/GoogleApiServerUtils.ts index 8785cd974..2fb44d9a2 100644 --- a/src/server/apis/google/GoogleApiServerUtils.ts +++ b/src/server/apis/google/GoogleApiServerUtils.ts @@ -5,6 +5,7 @@ import { OAuth2Client } from "google-auth-library"; import { Opt } from "../../../new_fields/Doc"; import { GlobalOptions } from "googleapis-common"; import { GaxiosResponse } from "gaxios"; +import Photos = require("googlephotos"); /** * Server side authentication for Google Api queries. @@ -20,16 +21,18 @@ export namespace GoogleApiServerUtils { 'presentations.readonly', 'drive', 'drive.file', + 'photoslibrary', + 'photoslibrary.sharing' ]; export const parseBuffer = (data: Buffer) => JSON.parse(data.toString()); export enum Service { Documents = "Documents", - Slides = "Slides" + Slides = "Slides", + Photos = "Photos" } - export interface CredentialPaths { credentials: string; token: string; @@ -50,7 +53,7 @@ export namespace GoogleApiServerUtils { reject(err); return console.log('Error loading client secret file:', err); } - return authorize(parseBuffer(credentials), paths.token).then(auth => { + return authorize(parseBuffer(credentials), paths.token).then(async auth => { let routed: Opt; let parameters: EndpointParameters = { auth, version: "v1" }; switch (sector) { @@ -60,6 +63,10 @@ export namespace GoogleApiServerUtils { case Service.Slides: routed = google.slides(parameters).presentations; break; + case Service.Photos: + const photos = new Photos(auth); + let response = await photos.albums.list(); + console.log("WE GOT SOMETHING!", response); } resolve(routed); }); diff --git a/src/server/apis/google/GooglePhotosUtils.ts b/src/server/apis/google/GooglePhotosUtils.ts new file mode 100644 index 000000000..c33ad2dd9 --- /dev/null +++ b/src/server/apis/google/GooglePhotosUtils.ts @@ -0,0 +1,12 @@ +import request = require('request-promise'); +const key = require("../../credentials/auth.json"); + +export const PhotosLibraryQuery = async (authToken: any, parameters: any) => { + let options = { + headers: { 'Content-Type': 'application/json' }, + json: parameters, + auth: { 'bearer': authToken }, + }; + const result = await request.post(config.apiEndpoint + '/v1/mediaItems:search', options); + return result; +}; \ No newline at end of file diff --git a/src/server/authentication/config/passport.ts b/src/server/authentication/config/passport.ts index d42741410..97ded8785 100644 --- a/src/server/authentication/config/passport.ts +++ b/src/server/authentication/config/passport.ts @@ -1,12 +1,14 @@ import * as passport from 'passport'; import * as passportLocal from 'passport-local'; -import * as mongodb from 'mongodb'; -import * as _ from "lodash"; +import _ from "lodash"; import { default as User } from '../models/user_model'; import { Request, Response, NextFunction } from "express"; import { RouteStore } from '../../RouteStore'; +import * as GoogleOAuth from "passport-google-oauth20"; +const config = require("../../credentials/google_photos_credentials"); const LocalStrategy = passportLocal.Strategy; +const GoogleOAuthStrategy = GoogleOAuth.Strategy; passport.serializeUser((user, done) => { done(undefined, user.id); @@ -32,6 +34,18 @@ passport.use(new LocalStrategy({ usernameField: 'email', passReqToCallback: true }); })); + +passport.use(new GoogleOAuthStrategy( + { + clientID: config.oAuthClientID, + clientSecret: config.oAuthclientSecret, + callbackURL: config.oAuthCallbackUrl, + // Set the correct profile URL that does not require any additional APIs + userProfileURL: 'https://www.googleapis.com/oauth2/v3/userinfo' + }, + (token, refreshToken, profile, done) => done(undefined, { profile, token }) +)); + export let isAuthenticated = (req: Request, res: Response, next: NextFunction) => { if (req.isAuthenticated()) { return next(); diff --git a/src/server/credentials/auth.json b/src/server/credentials/auth.json new file mode 100644 index 000000000..557eca4b6 --- /dev/null +++ b/src/server/credentials/auth.json @@ -0,0 +1,12 @@ +{ + "type": "service_account", + "project_id": "brown-dash", + "private_key_id": "ddf0473a9ac56956b5818e04a7ee406a64d5b0a6", + "private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCueRfxic2oL9nr\nnWSLgl7XR/BKikm4p2sib6szaoTO+q6itcJgt2TDleK/7Y4KW/KhvCfhWVet0Hz0\nIDyg4N/gc2yxuDA6/m8DPWU9kDj8VFR7LVFawOKgo1WbgLcC0Qu8qHzAffrlg8si\nhj3vGuoS/YDn/mz0krwFmCfIx+S0lJ9a7FUjJL5C+CIwAEEYiU7xnTW7pVVNXAm/\n/YKD17ToAjREOtlfVVYO7tZ7V5BiW0I0jJvxw+t1pgrZZe7WPBSBJg9KKGIl+mRi\ndtUMR9Hyt3nMKNZIrSm0OkAz82HxfcapRdSB3wkVjoyW63YaTVHKoBOqRElfMtoM\nqu8wbhhNAgMBAAECggEAA41wJ8kg8J8peQMZ/b7gZvzuPy+h0M/J3j2MrG3rY9qA\nrUv1oqoBSXvhuNDhtEN12oYIqtg6m+L+Sas8CMuOC2rWPafM2u/80IGoGtDhtCjp\nv8inBX8ew4YSiL7IxdbTU2/70Es7DVV5u0t6ndsmr88ibYwwPupGR/fhPCpDyssg\n7lFAEpOwnbKG9a7E7axHpXBRSIE54sh+ESyf6MHH/oyKOLhZ0v4PjRDKaKuMDRst\nMOClgNjD/4bzKpfWljuPYemXz1oIBQitBW5aXnCdsmdrmOLDQpz3qOgIo+RRiyki\nvVU5N54L65sj4WisLt1TT45wbhrkQUz+8GmhV5rHwQKBgQDow32/gSb/M0BKFk5y\n+pSeoLkYp/dwfBFWYT6CNBaKePARFVCdr3db8yOEQD4hrmTOU0EP9c4FSLcaa0Xy\n7n2crhhfZWFpIpRMyXhpeKpqdjQFimfBOK6cIdjnWrtJ6Ik3m4E9p7kKThIsInTc\n/TETwAyzFN+J2ADRdrUdCukOwQKBgQC/4+1Rk8a++Jr9Sznx+JH4vj/J2cGsu7uQ\n0nikcOAFO5HzG7+mt9Wv9/MiPtEYwmc7YziDXldKLpshT2m6wrS1uzzOXAnvXFAh\npiCXQsmVA3gmrVd53k+eZfqrZ1n/rL1kCewRS5LX8xIhM28VIkGqkVy4ZEifMotG\nZKSbH0b4jQKBgQCsJ6rh8Uw+hFGQel8be2pgyM8eBV1lvN213ca11oC1ei1U9Ubi\n2dyWDYa/UiSiFLJKSBlfDJaMIfQLfjwGKY6OS9WK+RjLAeBdysVcfPrOMw7W6j9D\nEgFTSVV8CAdt6qdSkZlNWLfrf0LBkdqNeFbMHMdHzLBo63HverUJ/f/SAQKBgHIk\n2t5T0T14FHnnbaiJ/ArC4J7pcVOWuJQFHs5ydk+mh8LdFrvNTsdF7tLIGwlnWpDx\nDITYcYQnBRBjdLkraONRZXI7PY2sk93wPCK+D7scPTSEmCxeGW5XqyyaZea4klAX\nttzy336lkHs/ZSxlHDqiDU2CGdDY+A//fgroKAdhAoGAA5FXfMzTQLGqxg4J2B1z\nFEXNbrqZZFGgKiveUhhZLm4zPiHXtZXvDtSLwGgcO8oGfTfYueTcHb/Eiar7mKv+\n+SqpAqkINJTthIFVIiD39S9jPFUXzBkf5ZJKPLKQArhzEGxen+SD6ZUO058fA94L\n9FblRGlMtr2o5z0NC7H5zaU=\n-----END PRIVATE KEY-----\n", + "client_email": "google-photos-api@brown-dash.iam.gserviceaccount.com", + "client_id": "112995422877175743408", + "auth_uri": "https://accounts.google.com/o/oauth2/auth", + "token_uri": "https://oauth2.googleapis.com/token", + "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", + "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/google-photos-api%40brown-dash.iam.gserviceaccount.com" +} \ No newline at end of file diff --git a/src/server/credentials/google_docs_credentials.json b/src/server/credentials/google_docs_credentials.json index 8d097d363..955c5a3c1 100644 --- a/src/server/credentials/google_docs_credentials.json +++ b/src/server/credentials/google_docs_credentials.json @@ -1 +1,11 @@ -{"installed":{"client_id":"343179513178-ud6tvmh275r2fq93u9eesrnc66t6akh9.apps.googleusercontent.com","project_id":"quickstart-1565056383187","auth_uri":"https://accounts.google.com/o/oauth2/auth","token_uri":"https://oauth2.googleapis.com/token","auth_provider_x509_cert_url":"https://www.googleapis.com/oauth2/v1/certs","client_secret":"w8KIFSc0MQpmUYHed4qEzn8b","redirect_uris":["urn:ietf:wg:oauth:2.0:oob","http://localhost"]}} \ No newline at end of file +{ + "installed": { + "client_id": "343179513178-ud6tvmh275r2fq93u9eesrnc66t6akh9.apps.googleusercontent.com", + "project_id": "quickstart-1565056383187", + "auth_uri": "https://accounts.google.com/o/oauth2/auth", + "token_uri": "https://oauth2.googleapis.com/token", + "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", + "client_secret": "w8KIFSc0MQpmUYHed4qEzn8b", + "redirect_uris": ["urn:ietf:wg:oauth:2.0:oob", "http://localhost"] + } +} \ No newline at end of file diff --git a/src/server/credentials/google_docs_token.json b/src/server/credentials/google_docs_token.json index 07c02d56c..cea452f08 100644 --- a/src/server/credentials/google_docs_token.json +++ b/src/server/credentials/google_docs_token.json @@ -1 +1 @@ -{"access_token":"ya29.GltjB4-x03xFpd2NY2555cxg1xlT_ajqRi78M9osOfdOF2jTIjlPkn_UZL8cUwVP0DPC8rH3vhhg8RpspFe8Vewx92shAO3RPos_uMH0CUqEiCiZlaaB5I3Jq3Mv","refresh_token":"1/teUKUqGKMLjVqs-eed0L8omI02pzSxMUYaxGc2QxBw0","scope":"https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents.readonly","token_type":"Bearer","expiry_date":1565654175862} \ No newline at end of file +{"access_token":"ya29.Glt2B3lsrpWxZ9DMg1RcTksAFzfR8dVWhf7d7tAvbJ4UbcSVO0Q3aYNGtaMKPtmxR24rH88iQSiKCL8S328TQFEN6LtZgvizymednK5EW0jNCvG6ecdZQ-vwcypR","refresh_token":"1/6X3oGYz4A0p8UW2IgsZ-GqTgQUY43S6Ahsaf_GQhSs8","scope":"https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/documents.readonly https://www.googleapis.com/auth/photoslibrary.sharing https://www.googleapis.com/auth/photoslibrary","token_type":"Bearer","expiry_date":1567360444627} \ No newline at end of file diff --git a/src/server/credentials/google_photos_credentials.ts b/src/server/credentials/google_photos_credentials.ts new file mode 100644 index 000000000..11c1c766c --- /dev/null +++ b/src/server/credentials/google_photos_credentials.ts @@ -0,0 +1,35 @@ +const config: any = {}; + +// The OAuth client ID from the Google Developers console. +config.oAuthClientID = '1005546247619-l40012sl4idpq17b5emielcs1delffog.apps.googleusercontent.com'; + +// The OAuth client secret from the Google Developers console. +config.oAuthclientSecret = 'xEUJ0OBvhlCKA6SLt8TvWBs3'; + +// The callback to use for OAuth requests. This is the URL where the app is +// running. For testing and running it locally, use 127.0.0.1. +config.oAuthCallbackUrl = 'http://localhost:1050/auth/google/callback'; + +// The port where the app should listen for requests. +config.port = 1050; + +// The scopes to request. The app requires the photoslibrary.readonly and +// plus.me scopes. +config.scopes = [ + 'https://www.googleapis.com/auth/photoslibrary.readonly', + 'profile', +]; + +// The number of photos to load for search requests. +config.photosToLoad = 150; + +// The page size to use for search requests. 100 is reccommended. +config.searchPageSize = 100; + +// The page size to use for the listing albums request. 50 is reccommended. +config.albumPageSize = 50; + +// The API end point to use. Do not change. +config.apiEndpoint = 'https://photoslibrary.googleapis.com'; + +module.exports = config; \ No newline at end of file diff --git a/src/server/index.ts b/src/server/index.ts index 34a0a19f1..6105dedcc 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -29,6 +29,7 @@ import { RouteStore } from './RouteStore'; import v4 = require('uuid/v4'); const app = express(); const config = require('../../webpack.config'); +const OAuthConfig = require('../server/credentials/google_photos_credentials'); import { createCanvas, loadImage, Canvas } from "canvas"; const compiler = webpack(config); const port = 1050; // default port to listen @@ -46,6 +47,7 @@ import { GaxiosResponse } from 'gaxios'; import { Opt } from '../new_fields/Doc'; import { docs_v1 } from 'googleapis'; import { Endpoint } from 'googleapis-common'; +import { PhotosLibraryQuery } from './apis/google/GooglePhotosUtils'; const MongoStore = require('connect-mongo')(session); const mongoose = require('mongoose'); const probe = require("probe-image-size"); @@ -196,6 +198,31 @@ const solrURL = "http://localhost:8983/solr/#/dash"; // GETTERS +// app.get('/auth/google', passport.authenticate('google', { +// scope: OAuthConfig.scopes, +// failureFlash: true, // Display errors to the user. +// session: true, +// })); + +// app.get("/failed", (req, res) => res.send("DIDN'T WORK!")); + +// app.get( +// '/auth/google/callback', +// passport.authenticate( +// 'google', { failureRedirect: '/failed', failureFlash: true, session: true }), +// (req, res) => { +// // User has logged in. +// console.log('OAUTH: user has logged in 1.'); +// PhotosLibraryQuery(req.user.token, {}); +// console.log('OAUTH: user has logged in 2.'); +// res.redirect('/'); +// }); + +// app.get('/GooglePhotos', (req, res) => { +// console.log("WORKING ON GOOGLE PHOTOS"); +// PhotosLibraryQuery(req.user.token, {}); +// }); + app.get("/search", async (req, res) => { const solrQuery: any = {}; ["q", "fq", "start", "rows", "hl", "hl.fl"].forEach(key => solrQuery[key] = req.query[key]); diff --git a/src/typings/index.d.ts b/src/typings/index.d.ts index 7939ae8be..36d828fdb 100644 --- a/src/typings/index.d.ts +++ b/src/typings/index.d.ts @@ -1,322 +1,324 @@ /// -declare module '@react-pdf/renderer' { - import * as React from 'react'; - - namespace ReactPDF { - interface Style { - [property: string]: any; - } - interface Styles { - [key: string]: Style; - } - type Orientation = 'portrait' | 'landscape'; - - interface DocumentProps { - title?: string; - author?: string; - subject?: string; - keywords?: string; - creator?: string; - producer?: string; - onRender?: () => any; - } - - /** - * This component represent the PDF document itself. It must be the root - * of your tree element structure, and under no circumstances should it be - * used as children of another react-pdf component. In addition, it should - * only have childs of type . - */ - class Document extends React.Component { } - - interface NodeProps { - style?: Style | Style[]; - /** - * Render component in all wrapped pages. - * @see https://react-pdf.org/advanced#fixed-components - */ - fixed?: boolean; - /** - * Force the wrapping algorithm to start a new page when rendering the - * element. - * @see https://react-pdf.org/advanced#page-breaks - */ - break?: boolean; - } - - interface PageProps extends NodeProps { - /** - * Enable page wrapping for this page. - * @see https://react-pdf.org/components#page-wrapping - */ - wrap?: boolean; - debug?: boolean; - size?: string | [number, number] | { width: number; height: number }; - orientation?: Orientation; - ruler?: boolean; - rulerSteps?: number; - verticalRuler?: boolean; - verticalRulerSteps?: number; - horizontalRuler?: boolean; - horizontalRulerSteps?: number; - ref?: Page; - } - - /** - * Represents single page inside the PDF document, or a subset of them if - * using the wrapping feature. A can contain as many pages as - * you want, but ensure not rendering a page inside any component besides - * Document. - */ - class Page extends React.Component { } - - interface ViewProps extends NodeProps { - /** - * Enable/disable page wrapping for element. - * @see https://react-pdf.org/components#page-wrapping - */ - wrap?: boolean; - debug?: boolean; - render?: (props: { pageNumber: number }) => React.ReactNode; - children?: React.ReactNode; - } - - /** - * The most fundamental component for building a UI and is designed to be - * nested inside other views and can have 0 to many children. - */ - class View extends React.Component { } - - interface ImageProps extends NodeProps { - debug?: boolean; - src: string | { data: Buffer; format: 'png' | 'jpg' }; - cache?: boolean; - } - - /** - * A React component for displaying network or local (Node only) JPG or - * PNG images, as well as base64 encoded image strings. - */ - class Image extends React.Component { } - - interface TextProps extends NodeProps { - /** - * Enable/disable page wrapping for element. - * @see https://react-pdf.org/components#page-wrapping - */ - wrap?: boolean; - debug?: boolean; - render?: ( - props: { pageNumber: number; totalPages: number }, - ) => React.ReactNode; - children?: React.ReactNode; - /** - * How much hyphenated breaks should be avoided. - */ - hyphenationCallback?: number; - } - - /** - * A React component for displaying text. Text supports nesting of other - * Text or Link components to create inline styling. - */ - class Text extends React.Component { } - - interface LinkProps extends NodeProps { - /** - * Enable/disable page wrapping for element. - * @see https://react-pdf.org/components#page-wrapping - */ - wrap?: boolean; - debug?: boolean; - src: string; - children?: React.ReactNode; - } +declare module 'googlephotos'; - /** - * A React component for displaying an hyperlink. Link’s can be nested - * inside a Text component, or being inside any other valid primitive. - */ - class Link extends React.Component { } - - interface NoteProps extends NodeProps { - children: string; - } - - class Note extends React.Component { } - - interface BlobProviderParams { - blob: Blob | null; - url: string | null; - loading: boolean; - error: Error | null; - } - interface BlobProviderProps { - document: React.ReactElement; - children: (params: BlobProviderParams) => React.ReactNode; - } - - /** - * Easy and declarative way of getting document's blob data without - * showing it on screen. - * @see https://react-pdf.org/advanced#on-the-fly-rendering - * @platform web - */ - class BlobProvider extends React.Component { } - - interface PDFViewerProps { - width?: number; - height?: number; - style?: Style | Style[]; - className?: string; - children?: React.ReactElement; - } - - /** - * Iframe PDF viewer for client-side generated documents. - * @platform web - */ - class PDFViewer extends React.Component { } - - interface PDFDownloadLinkProps { - document: React.ReactElement; - fileName?: string; - style?: Style | Style[]; - className?: string; - children?: - | React.ReactNode - | ((params: BlobProviderParams) => React.ReactNode); +declare module '@react-pdf/renderer' { + import * as React from 'react'; + + namespace ReactPDF { + interface Style { + [property: string]: any; + } + interface Styles { + [key: string]: Style; + } + type Orientation = 'portrait' | 'landscape'; + + interface DocumentProps { + title?: string; + author?: string; + subject?: string; + keywords?: string; + creator?: string; + producer?: string; + onRender?: () => any; + } + + /** + * This component represent the PDF document itself. It must be the root + * of your tree element structure, and under no circumstances should it be + * used as children of another react-pdf component. In addition, it should + * only have childs of type . + */ + class Document extends React.Component { } + + interface NodeProps { + style?: Style | Style[]; + /** + * Render component in all wrapped pages. + * @see https://react-pdf.org/advanced#fixed-components + */ + fixed?: boolean; + /** + * Force the wrapping algorithm to start a new page when rendering the + * element. + * @see https://react-pdf.org/advanced#page-breaks + */ + break?: boolean; + } + + interface PageProps extends NodeProps { + /** + * Enable page wrapping for this page. + * @see https://react-pdf.org/components#page-wrapping + */ + wrap?: boolean; + debug?: boolean; + size?: string | [number, number] | { width: number; height: number }; + orientation?: Orientation; + ruler?: boolean; + rulerSteps?: number; + verticalRuler?: boolean; + verticalRulerSteps?: number; + horizontalRuler?: boolean; + horizontalRulerSteps?: number; + ref?: Page; + } + + /** + * Represents single page inside the PDF document, or a subset of them if + * using the wrapping feature. A can contain as many pages as + * you want, but ensure not rendering a page inside any component besides + * Document. + */ + class Page extends React.Component { } + + interface ViewProps extends NodeProps { + /** + * Enable/disable page wrapping for element. + * @see https://react-pdf.org/components#page-wrapping + */ + wrap?: boolean; + debug?: boolean; + render?: (props: { pageNumber: number }) => React.ReactNode; + children?: React.ReactNode; + } + + /** + * The most fundamental component for building a UI and is designed to be + * nested inside other views and can have 0 to many children. + */ + class View extends React.Component { } + + interface ImageProps extends NodeProps { + debug?: boolean; + src: string | { data: Buffer; format: 'png' | 'jpg' }; + cache?: boolean; + } + + /** + * A React component for displaying network or local (Node only) JPG or + * PNG images, as well as base64 encoded image strings. + */ + class Image extends React.Component { } + + interface TextProps extends NodeProps { + /** + * Enable/disable page wrapping for element. + * @see https://react-pdf.org/components#page-wrapping + */ + wrap?: boolean; + debug?: boolean; + render?: ( + props: { pageNumber: number; totalPages: number }, + ) => React.ReactNode; + children?: React.ReactNode; + /** + * How much hyphenated breaks should be avoided. + */ + hyphenationCallback?: number; + } + + /** + * A React component for displaying text. Text supports nesting of other + * Text or Link components to create inline styling. + */ + class Text extends React.Component { } + + interface LinkProps extends NodeProps { + /** + * Enable/disable page wrapping for element. + * @see https://react-pdf.org/components#page-wrapping + */ + wrap?: boolean; + debug?: boolean; + src: string; + children?: React.ReactNode; + } + + /** + * A React component for displaying an hyperlink. Link’s can be nested + * inside a Text component, or being inside any other valid primitive. + */ + class Link extends React.Component { } + + interface NoteProps extends NodeProps { + children: string; + } + + class Note extends React.Component { } + + interface BlobProviderParams { + blob: Blob | null; + url: string | null; + loading: boolean; + error: Error | null; + } + interface BlobProviderProps { + document: React.ReactElement; + children: (params: BlobProviderParams) => React.ReactNode; + } + + /** + * Easy and declarative way of getting document's blob data without + * showing it on screen. + * @see https://react-pdf.org/advanced#on-the-fly-rendering + * @platform web + */ + class BlobProvider extends React.Component { } + + interface PDFViewerProps { + width?: number; + height?: number; + style?: Style | Style[]; + className?: string; + children?: React.ReactElement; + } + + /** + * Iframe PDF viewer for client-side generated documents. + * @platform web + */ + class PDFViewer extends React.Component { } + + interface PDFDownloadLinkProps { + document: React.ReactElement; + fileName?: string; + style?: Style | Style[]; + className?: string; + children?: + | React.ReactNode + | ((params: BlobProviderParams) => React.ReactNode); + } + + /** + * Anchor tag to enable generate and download PDF documents on the fly. + * @see https://react-pdf.org/advanced#on-the-fly-rendering + * @platform web + */ + class PDFDownloadLink extends React.Component { } + + interface EmojiSource { + url: string; + format: string; + } + interface RegisteredFont { + src: string; + loaded: boolean; + loading: boolean; + data: any; + [key: string]: any; + } + type HyphenationCallback = ( + words: string[], + glyphString: { [key: string]: any }, + ) => string[]; + + const Font: { + register: ( + src: string, + options: { family: string;[key: string]: any }, + ) => void; + getEmojiSource: () => EmojiSource; + getRegisteredFonts: () => string[]; + registerEmojiSource: (emojiSource: EmojiSource) => void; + registerHyphenationCallback: ( + hyphenationCallback: HyphenationCallback, + ) => void; + getHyphenationCallback: () => HyphenationCallback; + getFont: (fontFamily: string) => RegisteredFont | undefined; + load: ( + fontFamily: string, + document: React.ReactElement, + ) => Promise; + clear: () => void; + reset: () => void; + }; + + const StyleSheet: { + hairlineWidth: number; + create: (styles: TStyles) => TStyles; + resolve: ( + style: Style, + container: { + width: number; + height: number; + orientation: Orientation; + }, + ) => Style; + flatten: (...styles: Style[]) => Style; + absoluteFillObject: { + position: 'absolute'; + left: 0; + right: 0; + top: 0; + bottom: 0; + }; + }; + + const version: any; + + const PDFRenderer: any; + + const createInstance: ( + element: { + type: string; + props: { [key: string]: any }; + }, + root?: any, + ) => any; + + const pdf: ( + document: React.ReactElement, + ) => { + isDirty: () => boolean; + updateContainer: (document: React.ReactElement) => void; + toBuffer: () => NodeJS.ReadableStream; + toBlob: () => Blob; + toString: () => string; + }; + + const renderToStream: ( + document: React.ReactElement, + ) => NodeJS.ReadableStream; + + const renderToFile: ( + document: React.ReactElement, + filePath: string, + callback?: (output: NodeJS.ReadableStream, filePath: string) => any, + ) => Promise; + + const render: typeof renderToFile; } - /** - * Anchor tag to enable generate and download PDF documents on the fly. - * @see https://react-pdf.org/advanced#on-the-fly-rendering - * @platform web - */ - class PDFDownloadLink extends React.Component { } - - interface EmojiSource { - url: string; - format: string; - } - interface RegisteredFont { - src: string; - loaded: boolean; - loading: boolean; - data: any; - [key: string]: any; - } - type HyphenationCallback = ( - words: string[], - glyphString: { [key: string]: any }, - ) => string[]; - - const Font: { - register: ( - src: string, - options: { family: string;[key: string]: any }, - ) => void; - getEmojiSource: () => EmojiSource; - getRegisteredFonts: () => string[]; - registerEmojiSource: (emojiSource: EmojiSource) => void; - registerHyphenationCallback: ( - hyphenationCallback: HyphenationCallback, - ) => void; - getHyphenationCallback: () => HyphenationCallback; - getFont: (fontFamily: string) => RegisteredFont | undefined; - load: ( - fontFamily: string, - document: React.ReactElement, - ) => Promise; - clear: () => void; - reset: () => void; + const Document: typeof ReactPDF.Document; + const Page: typeof ReactPDF.Page; + const View: typeof ReactPDF.View; + const Image: typeof ReactPDF.Image; + const Text: typeof ReactPDF.Text; + const Link: typeof ReactPDF.Link; + const Note: typeof ReactPDF.Note; + const Font: typeof ReactPDF.Font; + const StyleSheet: typeof ReactPDF.StyleSheet; + const createInstance: typeof ReactPDF.createInstance; + const PDFRenderer: typeof ReactPDF.PDFRenderer; + const version: typeof ReactPDF.version; + const pdf: typeof ReactPDF.pdf; + + export default ReactPDF; + export { + Document, + Page, + View, + Image, + Text, + Link, + Note, + Font, + StyleSheet, + createInstance, + PDFRenderer, + version, + pdf, }; - - const StyleSheet: { - hairlineWidth: number; - create: (styles: TStyles) => TStyles; - resolve: ( - style: Style, - container: { - width: number; - height: number; - orientation: Orientation; - }, - ) => Style; - flatten: (...styles: Style[]) => Style; - absoluteFillObject: { - position: 'absolute'; - left: 0; - right: 0; - top: 0; - bottom: 0; - }; - }; - - const version: any; - - const PDFRenderer: any; - - const createInstance: ( - element: { - type: string; - props: { [key: string]: any }; - }, - root?: any, - ) => any; - - const pdf: ( - document: React.ReactElement, - ) => { - isDirty: () => boolean; - updateContainer: (document: React.ReactElement) => void; - toBuffer: () => NodeJS.ReadableStream; - toBlob: () => Blob; - toString: () => string; - }; - - const renderToStream: ( - document: React.ReactElement, - ) => NodeJS.ReadableStream; - - const renderToFile: ( - document: React.ReactElement, - filePath: string, - callback?: (output: NodeJS.ReadableStream, filePath: string) => any, - ) => Promise; - - const render: typeof renderToFile; - } - - const Document: typeof ReactPDF.Document; - const Page: typeof ReactPDF.Page; - const View: typeof ReactPDF.View; - const Image: typeof ReactPDF.Image; - const Text: typeof ReactPDF.Text; - const Link: typeof ReactPDF.Link; - const Note: typeof ReactPDF.Note; - const Font: typeof ReactPDF.Font; - const StyleSheet: typeof ReactPDF.StyleSheet; - const createInstance: typeof ReactPDF.createInstance; - const PDFRenderer: typeof ReactPDF.PDFRenderer; - const version: typeof ReactPDF.version; - const pdf: typeof ReactPDF.pdf; - - export default ReactPDF; - export { - Document, - Page, - View, - Image, - Text, - Link, - Note, - Font, - StyleSheet, - createInstance, - PDFRenderer, - version, - pdf, - }; } \ No newline at end of file -- cgit v1.2.3-70-g09d2 From 5d59e9a379417140e10778cd43e8f87ecb816c37 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Fri, 6 Sep 2019 06:29:26 -0400 Subject: lightly tested functional export of any image doc (web url or stored in server public folder) to google photos, optionally stored in a given target album --- package.json | 2 +- .../apis/google_docs/GooglePhotosClientUtils.ts | 73 +++++----- src/client/views/MainView.tsx | 17 ++- src/client/views/nodes/FormattedTextBox.tsx | 1 + src/server/RouteStore.ts | 2 +- src/server/apis/google/GoogleApiServerUtils.ts | 28 ++-- src/server/apis/google/GooglePhotosServerUtils.ts | 68 ---------- src/server/apis/google/GooglePhotosUploadUtils.ts | 122 +++++++++++++++-- src/server/apis/google/typings/albums.ts | 150 --------------------- src/server/credentials/google_docs_token.json | 2 +- src/server/index.ts | 75 +++++------ 11 files changed, 206 insertions(+), 334 deletions(-) delete mode 100644 src/server/apis/google/GooglePhotosServerUtils.ts delete mode 100644 src/server/apis/google/typings/albums.ts (limited to 'package.json') diff --git a/package.json b/package.json index f0f2b467e..f56e34ce0 100644 --- a/package.json +++ b/package.json @@ -224,4 +224,4 @@ "xoauth2": "^1.2.0", "youtube": "^0.1.0" } -} +} \ No newline at end of file diff --git a/src/client/apis/google_docs/GooglePhotosClientUtils.ts b/src/client/apis/google_docs/GooglePhotosClientUtils.ts index b95cc98c9..2b72800a9 100644 --- a/src/client/apis/google_docs/GooglePhotosClientUtils.ts +++ b/src/client/apis/google_docs/GooglePhotosClientUtils.ts @@ -1,53 +1,42 @@ -import { Album } from "../../../server/apis/google/typings/albums"; -import { PostToServer } from "../../../Utils"; +import { PostToServer, Utils } from "../../../Utils"; import { RouteStore } from "../../../server/RouteStore"; import { ImageField } from "../../../new_fields/URLField"; +import { StrCast, Cast } from "../../../new_fields/Types"; +import { Doc, Opt } from "../../../new_fields/Doc"; +import { Id } from "../../../new_fields/FieldSymbols"; +import requestImageSize = require('../../util/request-image-size'); +import Photos = require('googlephotos'); export namespace GooglePhotosClientUtils { - export const Create = async (title: string) => { - let parameters = { - action: Album.Action.Create, - body: { album: { title } } - } as Album.Create; - return PostToServer(RouteStore.googlePhotosQuery, parameters); - }; + export type AlbumReference = { id: string } | { title: string }; + export const endpoint = () => fetch(Utils.prepend(RouteStore.googlePhotosAccessToken)).then(async response => new Photos(await response.text())); - export const List = async (options?: Partial) => { - let parameters = { - action: Album.Action.List, - parameters: { - pageSize: (options ? options.pageSize : 20) || 20, - pageToken: (options ? options.pageToken : undefined) || undefined, - excludeNonAppCreatedData: (options ? options.excludeNonAppCreatedData : false) || false, - } as Album.ListOptions - } as Album.List; - return PostToServer(RouteStore.googlePhotosQuery, parameters); - }; + export interface MediaInput { + description: string; + source: string; + } - export const Get = async (albumId: string) => { - let parameters = { - action: Album.Action.Get, - albumId - } as Album.Get; - return PostToServer(RouteStore.googlePhotosQuery, parameters); - }; - - export const toDataURL = (field: ImageField | undefined) => { - if (!field) { - return undefined; + export const UploadMedia = async (sources: Doc[], album?: AlbumReference) => { + if (album && "title" in album) { + album = (await endpoint()).albums.create(album.title); + } + const media: MediaInput[] = []; + sources.forEach(document => { + const data = Cast(Doc.GetProto(document).data, ImageField); + const description = StrCast(document.caption); + if (!data) { + return undefined; + } + media.push({ + source: data.url.href, + description, + } as MediaInput); + }); + if (media.length) { + return PostToServer(RouteStore.googlePhotosMediaUpload, { media, album }); } - const image = document.createElement("img"); - image.src = field.url.href; - image.width = 200; - image.height = 200; - const canvas = document.createElement("canvas"); - canvas.width = image.width; - canvas.height = image.height; - const ctx = canvas.getContext("2d")!; - ctx.drawImage(image, 0, 0); - const dataUrl = canvas.toDataURL("image/png"); - return dataUrl.replace(/^data:image\/(png|jpg);base64,/, ""); + return undefined; }; } \ No newline at end of file diff --git a/src/client/views/MainView.tsx b/src/client/views/MainView.tsx index c1c95fc88..6d366216e 100644 --- a/src/client/views/MainView.tsx +++ b/src/client/views/MainView.tsx @@ -40,11 +40,10 @@ import { PreviewCursor } from './PreviewCursor'; import { FilterBox } from './search/FilterBox'; import PresModeMenu from './presentationview/PresentationModeMenu'; import { PresBox } from './nodes/PresBox'; -import { docs_v1 } from 'googleapis'; -import { Album } from '../../server/apis/google/typings/albums'; import { GooglePhotosClientUtils } from '../apis/google_docs/GooglePhotosClientUtils'; import { ImageField } from '../../new_fields/URLField'; import { LinkFollowBox } from './linking/LinkFollowBox'; +import { DocumentManager } from '../util/DocumentManager'; @observer export class MainView extends React.Component { @@ -131,10 +130,7 @@ export class MainView extends React.Component { window.removeEventListener("keydown", KeyManager.Instance.handle); window.addEventListener("keydown", KeyManager.Instance.handle); - let imgurl = "https://upload.wikimedia.org/wikipedia/commons/thumb/3/3a/Cat03.jpg/1200px-Cat03.jpg"; - let image = Docs.Create.ImageDocument(imgurl, { width: 200, title: "an image of a cat" }); - let parameters = { title: StrCast(image.title), MEDIA_BINARY_DATA: GooglePhotosClientUtils.toDataURL(Cast(image.data, ImageField)) }; - // PostToServer(RouteStore.googlePhotosMediaUpload, parameters).then(console.log); + this.executeGooglePhotosRoutine(); reaction(() => { let workspaces = CurrentUserUtils.UserDocument.workspaces; @@ -153,6 +149,15 @@ export class MainView extends React.Component { }, { fireImmediately: true }); } + executeGooglePhotosRoutine = async () => { + let imgurl = "https://upload.wikimedia.org/wikipedia/commons/thumb/3/3a/Cat03.jpg/1200px-Cat03.jpg"; + let doc = Docs.Create.ImageDocument(imgurl, { width: 200, title: "an image of a cat" }); + doc.caption = "Well isn't this a nice cat image!"; + let photos = await GooglePhotosClientUtils.endpoint(); + let albumId = (await photos.albums.list(50)).albums.filter((album: any) => album.title === "This is a generically created album!")[0].id; + console.log(await GooglePhotosClientUtils.UploadMedia([doc], { id: albumId })); + } + componentWillUnMount() { window.removeEventListener("keydown", KeyManager.Instance.handle); //close presentation diff --git a/src/client/views/nodes/FormattedTextBox.tsx b/src/client/views/nodes/FormattedTextBox.tsx index b671d06ea..fda9ea33f 100644 --- a/src/client/views/nodes/FormattedTextBox.tsx +++ b/src/client/views/nodes/FormattedTextBox.tsx @@ -133,6 +133,7 @@ export class FormattedTextBox extends DocComponent<(FieldViewProps & FormattedTe if (this.props.isOverlay) { DragManager.StartDragFunctions.push(() => FormattedTextBox.InputBoxOverlay = undefined); } + FormattedTextBox.Instance = this; } public get CurrentDiv(): HTMLDivElement { return this._ref.current!; } diff --git a/src/server/RouteStore.ts b/src/server/RouteStore.ts index 3b3fd9b4a..b221b71bc 100644 --- a/src/server/RouteStore.ts +++ b/src/server/RouteStore.ts @@ -32,7 +32,7 @@ export enum RouteStore { // APIS cognitiveServices = "/cognitiveservices", googleDocs = "/googleDocs", - googlePhotosQuery = "/googlePhotosQuery", + googlePhotosAccessToken = "/googlePhotosAccessToken", googlePhotosMediaUpload = "/googlePhotosMediaUpload" } \ No newline at end of file diff --git a/src/server/apis/google/GoogleApiServerUtils.ts b/src/server/apis/google/GoogleApiServerUtils.ts index b6330a609..ac8023ce1 100644 --- a/src/server/apis/google/GoogleApiServerUtils.ts +++ b/src/server/apis/google/GoogleApiServerUtils.ts @@ -7,6 +7,7 @@ import { GlobalOptions } from "googleapis-common"; import { GaxiosResponse } from "gaxios"; import request = require('request-promise'); import * as qs from 'query-string'; +import Photos = require('googlephotos'); /** * Server side authentication for Google Api queries. @@ -35,19 +36,19 @@ export namespace GoogleApiServerUtils { } export interface CredentialPaths { - credentials: string; - token: string; + credentialsPath: string; + tokenPath: string; } export type ApiResponse = Promise; - export type ApiRouter = (endpoint: Endpoint, paramters: any) => ApiResponse; + export type ApiRouter = (endpoint: Endpoint, parameters: any) => ApiResponse; export type ApiHandler = (parameters: any) => ApiResponse; export type Action = "create" | "retrieve" | "update"; export type Endpoint = { get: ApiHandler, create: ApiHandler, batchUpdate: ApiHandler }; export type EndpointParameters = GlobalOptions & { version: "v1" }; - export const GetEndpoint = async (sector: string, paths: CredentialPaths) => { + export const GetEndpoint = (sector: string, paths: CredentialPaths) => { return new Promise>(resolve => { RetrieveCredentials(paths).then(authentication => { let routed: Opt; @@ -65,19 +66,19 @@ export namespace GoogleApiServerUtils { }); }; - export const RetrieveCredentials = async (paths: CredentialPaths) => { + export const RetrieveCredentials = (paths: CredentialPaths) => { return new Promise((resolve, reject) => { - readFile(paths.credentials, async (err, credentials) => { + readFile(paths.credentialsPath, async (err, credentials) => { if (err) { reject(err); return console.log('Error loading client secret file:', err); } - authorize(parseBuffer(credentials), paths.token).then(resolve, reject); + authorize(parseBuffer(credentials), paths.tokenPath).then(resolve, reject); }); }); }; - export const RetrieveAccessToken = async (paths: CredentialPaths) => { + export const RetrieveAccessToken = (paths: CredentialPaths) => { return new Promise((resolve, reject) => { RetrieveCredentials(paths).then( credentials => resolve(credentials.token.access_token!), @@ -86,6 +87,15 @@ export namespace GoogleApiServerUtils { }); }; + export const RetrievePhotosEndpoint = (paths: CredentialPaths) => { + return new Promise((resolve, reject) => { + RetrieveAccessToken(paths).then( + token => resolve(new Photos(token)), + reject + ); + }); + }; + type TokenResult = { token: Credentials, client: OAuth2Client }; /** * Create an OAuth2 client with the given credentials, and returns the promise resolving to the authenticated client @@ -126,7 +136,7 @@ export namespace GoogleApiServerUtils { request.post(url, headerParameters).then(response => { let parsed = JSON.parse(response); credentials.access_token = parsed.access_token; - credentials.expiry_date = new Date().getTime() + parsed.expires_in * 1000; + credentials.expiry_date = new Date().getTime() + (parsed.expires_in * 1000); writeFile(token_path, JSON.stringify(credentials), (err) => { if (err) { console.error(err); diff --git a/src/server/apis/google/GooglePhotosServerUtils.ts b/src/server/apis/google/GooglePhotosServerUtils.ts deleted file mode 100644 index cb5464abc..000000000 --- a/src/server/apis/google/GooglePhotosServerUtils.ts +++ /dev/null @@ -1,68 +0,0 @@ -import request = require('request-promise'); -import { Album } from './typings/albums'; -import * as qs from 'query-string'; - -const apiEndpoint = "https://photoslibrary.googleapis.com/v1/"; - -export interface Authorization { - token: string; -} - -export namespace GooglePhotos { - - export type Query = Album.Query; - export type QueryParameters = { query: GooglePhotos.Query }; - interface DispatchParameters { - required: boolean; - method: "GET" | "POST"; - ignore?: boolean; - } - - export const ExecuteQuery = async (parameters: Authorization & QueryParameters): Promise => { - let action = parameters.query.action; - let dispatch = SuffixMap.get(action)!; - let suffix = Suffix(parameters, dispatch, action); - if (suffix) { - let query: any = parameters.query; - let options: any = { - headers: { 'Content-Type': 'application/json' }, - auth: { 'bearer': parameters.token }, - }; - if (query.body) { - options.body = query.body; - options.json = true; - } - let queryParameters = query.parameters; - if (queryParameters) { - suffix += `?${qs.stringify(queryParameters)}`; - } - let dispatcher = dispatch.method === "POST" ? request.post : request.get; - return dispatcher(apiEndpoint + suffix, options); - } - }; - - const Suffix = (parameters: QueryParameters, dispatch: DispatchParameters, action: Album.Action) => { - let query: any = parameters.query; - let id = query.albumId; - let suffix = 'albums'; - if (dispatch.required) { - if (!id) { - return undefined; - } - suffix += `/${id}${dispatch.ignore ? "" : `:${action}`}`; - } - return suffix; - }; - - const SuffixMap = new Map([ - [Album.Action.AddEnrichment, { required: true, method: "POST" }], - [Album.Action.BatchAddMediaItems, { required: true, method: "POST" }], - [Album.Action.BatchRemoveMediaItems, { required: true, method: "POST" }], - [Album.Action.Create, { required: false, method: "POST" }], - [Album.Action.Get, { required: true, ignore: true, method: "GET" }], - [Album.Action.List, { required: false, method: "GET" }], - [Album.Action.Share, { required: true, method: "POST" }], - [Album.Action.Unshare, { required: true, method: "POST" }] - ]); - -} diff --git a/src/server/apis/google/GooglePhotosUploadUtils.ts b/src/server/apis/google/GooglePhotosUploadUtils.ts index cd2a586eb..3b513aaf1 100644 --- a/src/server/apis/google/GooglePhotosUploadUtils.ts +++ b/src/server/apis/google/GooglePhotosUploadUtils.ts @@ -1,28 +1,122 @@ import request = require('request-promise'); -import { Authorization } from './GooglePhotosServerUtils'; +import { GoogleApiServerUtils } from './GoogleApiServerUtils'; +import * as fs from 'fs'; +import { Utils } from '../../../Utils'; +import * as path from 'path'; +import { Opt } from '../../../new_fields/Doc'; export namespace GooglePhotosUploadUtils { - interface UploadInformation { - title: string; - MEDIA_BINARY_DATA: string; + export interface Paths { + uploadDirectory: string; + credentialsPath: string; + tokenPath: string; } - const apiEndpoint = "https://photoslibrary.googleapis.com/v1/uploads"; + export interface MediaInput { + description: string; + source: string; + } + + export interface DownloadInformation { + mediaPath: string; + contentType?: string; + contentSize?: string; + } + + const prepend = (extension: string) => `https://photoslibrary.googleapis.com/v1/${extension}`; + const headers = (type: string) => ({ + 'Content-Type': `application/${type}`, + 'Authorization': Bearer, + }); + + let Bearer: string; + let Paths: Paths; - export const SubmitUpload = async (parameters: Authorization & UploadInformation) => { - let options = { + export const initialize = async (paths: Paths) => { + Paths = paths; + const { tokenPath, credentialsPath } = paths; + const token = await GoogleApiServerUtils.RetrieveAccessToken({ tokenPath, credentialsPath }); + Bearer = `Bearer ${token}`; + }; + + export const DispatchGooglePhotosUpload = async (filename: string) => { + let body: Buffer; + if (filename.includes('upload_')) { + const mediaPath = Paths.uploadDirectory + filename; + body = await new Promise((resolve, reject) => { + fs.readFile(mediaPath, (error, data) => error ? reject(error) : resolve(data)); + }); + } else { + body = await request(filename, { encoding: null }); + } + const parameters = { + method: 'POST', headers: { - 'Content-Type': 'application/octet-stream', - Authorization: `Bearer ${parameters.token}`, - 'X-Goog-Upload-File-Name': parameters.title, + ...headers('octet-stream'), + 'X-Goog-Upload-File-Name': filename, 'X-Goog-Upload-Protocol': 'raw' }, - body: { MEDIA_BINARY_DATA: parameters.MEDIA_BINARY_DATA }, - json: true + uri: prepend('uploads'), + body }; - const result = await request.post(apiEndpoint, options); - return result; + return new Promise(resolve => request(parameters, (error, _response, body) => resolve(error ? undefined : body))); + }; + + export const CreateMediaItems = (newMediaItems: any[], album?: { id: string }) => { + return new Promise((resolve, reject) => { + const parameters = { + method: 'POST', + headers: headers('json'), + uri: prepend('mediaItems:batchCreate'), + body: { newMediaItems } as any, + json: true + }; + album && (parameters.body.albumId = album.id); + request(parameters, (error, _response, body) => { + if (error) { + reject(error); + } else { + resolve(body); + } + }); + }); }; + export namespace IOUtils { + + export const Download = async (url: string): Promise> => { + const filename = `temporary_upload_${Utils.GenerateGuid()}${path.extname(url).toLowerCase()}`; + const temporaryDirectory = Paths.uploadDirectory + "temporary/"; + const mediaPath = temporaryDirectory + filename; + + if (!(await createIfNotExists(temporaryDirectory))) { + return undefined; + } + + return new Promise((resolve, reject) => { + request.head(url, (error, res) => { + if (error) { + return reject(error); + } + const information: DownloadInformation = { + mediaPath, + contentType: res.headers['content-type'], + contentSize: res.headers['content-length'], + }; + request(url).pipe(fs.createWriteStream(mediaPath)).on('close', () => resolve(information)); + }); + }); + }; + + export const createIfNotExists = async (path: string) => { + if (await new Promise(resolve => fs.exists(path, resolve))) { + return true; + } + return new Promise(resolve => fs.mkdir(path, error => resolve(error === null))); + }; + + export const Destroy = (mediaPath: string) => new Promise(resolve => fs.unlink(mediaPath, error => resolve(error === null))); + } + } \ No newline at end of file diff --git a/src/server/apis/google/typings/albums.ts b/src/server/apis/google/typings/albums.ts deleted file mode 100644 index f3025567d..000000000 --- a/src/server/apis/google/typings/albums.ts +++ /dev/null @@ -1,150 +0,0 @@ -export namespace Album { - - export type Query = (AddEnrichment | BatchAddMediaItems | BatchRemoveMediaItems | Create | Get | List | Share | Unshare); - - export enum Action { - AddEnrichment = "addEnrichment", - BatchAddMediaItems = "batchAddMediaItems", - BatchRemoveMediaItems = "batchRemoveMediaItems", - Create = "create", - Get = "get", - List = "list", - Share = "share", - Unshare = "unshare" - } - - export interface AddEnrichment { - action: Action.AddEnrichment; - albumId: string; - body: { - newEnrichmentItem: NewEnrichmentItem; - albumPosition: MediaRelativeAlbumPosition; - }; - } - - export interface BatchAddMediaItems { - action: Action.BatchAddMediaItems; - albumId: string; - body: { - mediaItemIds: string[]; - }; - } - - export interface BatchRemoveMediaItems { - action: Action.BatchRemoveMediaItems; - albumId: string; - body: { - mediaItemIds: string[]; - }; - } - - export interface Create { - action: Action.Create; - body: { - album: Template; - }; - } - - export interface Get { - action: Action.Get; - albumId: string; - } - - export interface List { - action: Action.List; - parameters: ListOptions; - } - - export interface ListOptions { - pageSize: number; - pageToken: string; - excludeNonAppCreatedData: boolean; - } - - export interface Share { - action: Action.Share; - albumId: string; - body: { - sharedAlbumOptions: SharedOptions; - }; - } - - export interface Unshare { - action: Action.Unshare; - albumId: string; - } - - export interface Template { - title: string; - } - - export interface Model { - id: string; - title: string; - productUrl: string; - isWriteable: boolean; - shareInfo: ShareInfo; - mediaItemsCount: string; - coverPhotoBaseUrl: string; - coverPhotoMediaItemId: string; - } - - export interface ShareInfo { - sharedAlbumOptions: SharedOptions; - shareableUrl: string; - shareToken: string; - isJoined: boolean; - isOwned: boolean; - } - - export interface SharedOptions { - isCollaborative: boolean; - isCommentable: boolean; - } - - export enum PositionType { - POSITION_TYPE_UNSPECIFIED, - FIRST_IN_ALBUM, - LAST_IN_ALBUM, - AFTER_MEDIA_ITEM, - AFTER_ENRICHMENT_ITEM - } - - export type Position = GeneralAlbumPosition | MediaRelativeAlbumPosition | EnrichmentRelativeAlbumPosition; - - interface GeneralAlbumPosition { - position: PositionType.FIRST_IN_ALBUM | PositionType.LAST_IN_ALBUM | PositionType.POSITION_TYPE_UNSPECIFIED; - } - - interface MediaRelativeAlbumPosition { - position: PositionType.AFTER_MEDIA_ITEM; - relativeMediaItemId: string; - } - - interface EnrichmentRelativeAlbumPosition { - position: PositionType.AFTER_ENRICHMENT_ITEM; - relativeEnrichmentItemId: string; - } - - export interface Location { - locationName: string; - latlng: { - latitude: number, - longitude: number - }; - } - - export interface NewEnrichmentItem { - textEnrichment: { - text: string; - }; - locationEnrichment: { - location: Location - }; - mapEnrichment: { - origin: { location: Location }, - destination: { location: Location } - }; - } - -} \ No newline at end of file diff --git a/src/server/credentials/google_docs_token.json b/src/server/credentials/google_docs_token.json index 2ac972ed8..f3c8cf82a 100644 --- a/src/server/credentials/google_docs_token.json +++ b/src/server/credentials/google_docs_token.json @@ -1 +1 @@ -{"access_token":"ya29.Glt5ByifP30HMz6a1fEG77qZlz9fBEOCz4PQ1VA8t_Ck2ZTPJKoyr6xc3-GFZISAqrrw2U8XpMyZv02_URfPwUX0Z_tMdlIFqsygowR-uClukbgQPNtgxp2LS1oW","refresh_token":"1/wK1cUVLnt581ba_pYGPdlTvAa-OS64nB5m5XOXEBJ8Q","scope":"https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/documents.readonly https://www.googleapis.com/auth/photoslibrary.sharing https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/photoslibrary.appendonly https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/photoslibrary","token_type":"Bearer","expiry_date":1567556163894} \ No newline at end of file +{"access_token":"ya29.Glx7B9S6zCKDE0EgYk9xX9-RhcN8j4IwG9ONopTl1NkPX9FUOw0GI_81mY9bhaouuyOTnrc6FrZD5SDHolWwp3ABNT6l7TmhTLDILgGXIixZkWFRBPpF-xHC8lUd8A","refresh_token":"1/HTv_xFHszu2Nf3iiFrUTaeKzC_Vp2-6bpIB06xW_WHI","scope":"https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/documents.readonly https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/photoslibrary https://www.googleapis.com/auth/photoslibrary.appendonly https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/photoslibrary.sharing","token_type":"Bearer","expiry_date":1567765465073} \ No newline at end of file diff --git a/src/server/index.ts b/src/server/index.ts index 3940bbd58..fab00a02d 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -42,7 +42,6 @@ var AdmZip = require('adm-zip'); import * as YoutubeApi from "./apis/youtube/youtubeApiSample"; import { Response } from 'express-serve-static-core'; import { GoogleApiServerUtils } from "./apis/google/GoogleApiServerUtils"; -import { GooglePhotos } from './apis/google/GooglePhotosServerUtils'; import { GooglePhotosUploadUtils } from './apis/google/GooglePhotosUploadUtils'; const MongoStore = require('connect-mongo')(session); const mongoose = require('mongoose'); @@ -418,10 +417,10 @@ app.get("/thumbnail/:filename", (req, res) => { let filename = req.params.filename; let noExt = filename.substring(0, filename.length - ".png".length); let pagenumber = parseInt(noExt.split('-')[1]); - fs.exists(uploadDir + filename, (exists: boolean) => { - console.log(`${uploadDir + filename} ${exists ? "exists" : "does not exist"}`); + fs.exists(uploadDirectory + filename, (exists: boolean) => { + console.log(`${uploadDirectory + filename} ${exists ? "exists" : "does not exist"}`); if (exists) { - let input = fs.createReadStream(uploadDir + filename); + let input = fs.createReadStream(uploadDirectory + filename); probe(input, (err: any, result: any) => { if (err) { console.log(err); @@ -432,7 +431,7 @@ app.get("/thumbnail/:filename", (req, res) => { }); } else { - LoadPage(uploadDir + filename.substring(0, filename.length - noExt.split('-')[1].length - ".PNG".length - 1) + ".pdf", pagenumber, res); + LoadPage(uploadDirectory + filename.substring(0, filename.length - noExt.split('-')[1].length - ".PNG".length - 1) + ".pdf", pagenumber, res); } }); }); @@ -556,13 +555,13 @@ class NodeCanvasFactory { const pngTypes = [".png", ".PNG"]; const pdfTypes = [".pdf", ".PDF"]; const jpgTypes = [".jpg", ".JPG", ".jpeg", ".JPEG"]; -const uploadDir = __dirname + "/public/files/"; +const uploadDirectory = __dirname + "/public/files/"; // SETTERS app.post( RouteStore.upload, (req, res) => { let form = new formidable.IncomingForm(); - form.uploadDir = uploadDir; + form.uploadDir = uploadDirectory; form.keepExtensions = true; // let path = req.body.path; console.log("upload"); @@ -592,7 +591,7 @@ app.post( } if (isImage) { resizers.forEach(resizer => { - fs.createReadStream(uploadDir + file).pipe(resizer.resizer).pipe(fs.createWriteStream(uploadDir + file.substring(0, file.length - ext.length) + resizer.suffix + ext)); + fs.createReadStream(uploadDirectory + file).pipe(resizer.resizer).pipe(fs.createWriteStream(uploadDirectory + file.substring(0, file.length - ext.length) + resizer.suffix + ext)); }); } names.push(`/files/` + file); @@ -611,7 +610,7 @@ addSecureRoute( res.status(401).send("incorrect parameters specified"); return; } - imageDataUri.outputFile(uri, uploadDir + filename).then((savedName: string) => { + imageDataUri.outputFile(uri, uploadDirectory + filename).then((savedName: string) => { const ext = path.extname(savedName); let resizers = [ { resizer: sharp().resize(100, undefined, { withoutEnlargement: true }), suffix: "_s" }, @@ -632,7 +631,7 @@ addSecureRoute( } if (isImage) { resizers.forEach(resizer => { - fs.createReadStream(savedName).pipe(resizer.resizer).pipe(fs.createWriteStream(uploadDir + filename + resizer.suffix + ext)); + fs.createReadStream(savedName).pipe(resizer.resizer).pipe(fs.createWriteStream(uploadDirectory + filename + resizer.suffix + ext)); }); } res.send("/files/" + filename + ext); @@ -799,8 +798,8 @@ function HandleYoutubeQuery([query, callback]: [YoutubeQueryInput, (result?: any } } -const credentials = path.join(__dirname, "./credentials/google_docs_credentials.json"); -const token = path.join(__dirname, "./credentials/google_docs_token.json"); +const credentialsPath = path.join(__dirname, "./credentials/google_docs_credentials.json"); +const tokenPath = path.join(__dirname, "./credentials/google_docs_token.json"); const EndpointHandlerMap = new Map([ ["create", (api, params) => api.create(params)], @@ -811,7 +810,7 @@ const EndpointHandlerMap = new Map { let sector: any = req.params.sector; let action: any = req.params.action; - GoogleApiServerUtils.GetEndpoint(GoogleApiServerUtils.Service[sector], { credentials, token }).then(endpoint => { + GoogleApiServerUtils.GetEndpoint(GoogleApiServerUtils.Service[sector], { credentialsPath, tokenPath }).then(endpoint => { let handler = EndpointHandlerMap.get(action); if (endpoint && handler) { let execute = handler(endpoint, req.body).then( @@ -825,36 +824,28 @@ app.post(RouteStore.googleDocs + "/:sector/:action", (req, res) => { }); }); -app.post(RouteStore.googlePhotosQuery, (req, res) => { - GoogleApiServerUtils.RetrieveAccessToken({ credentials, token }).then( - token => { - GooglePhotos.ExecuteQuery({ token, query: req.body }) - .then(response => { - if (response === undefined) { - res.send("Error: unable to build suffix for Google Photos API request"); - return; - } - res.send(response); - }) - .catch(error => { - res.send(`Error: an exception occurred in the execution of this Google Photos API request\n${error}`); - }); - }, - error => res.send(error) - ); -}); +app.get(RouteStore.googlePhotosAccessToken, (req, res) => GoogleApiServerUtils.RetrieveAccessToken({ credentialsPath, tokenPath }).then(token => res.send(token))); -app.post(RouteStore.googlePhotosMediaUpload, (req, res) => { - GoogleApiServerUtils.RetrieveAccessToken({ credentials, token }).then( - token => { - GooglePhotosUploadUtils.SubmitUpload({ token, ...req.body }) - .then(response => { - res.send(response); - }).catch(error => { - res.send(`Error: an exception occurred in uploading the given media\n${error}`); - }); - }, - error => res.send(error)); +const tokenError = "Unable to successfully upload bytes for all images!"; +const mediaError = "Unable to convert all uploaded bytes to media items!"; + +app.post(RouteStore.googlePhotosMediaUpload, async (req, res) => { + const media: GooglePhotosUploadUtils.MediaInput[] = req.body.media; + await GooglePhotosUploadUtils.initialize({ uploadDirectory, credentialsPath, tokenPath }); + const newMediaItems = await Promise.all(media.map(async element => { + const uploadToken = await GooglePhotosUploadUtils.DispatchGooglePhotosUpload(element.source); + return !uploadToken ? undefined : { + description: element.description, + simpleMediaItem: { uploadToken } + }; + })); + if (!newMediaItems.every(item => item)) { + return res.send(tokenError); + } + GooglePhotosUploadUtils.CreateMediaItems(newMediaItems, req.body.album).then( + success => res.send(success), + () => res.send(mediaError) + ); }); const suffixMap: { [type: string]: (string | [string, string | ((json: any) => any)]) } = { -- cgit v1.2.3-70-g09d2 From b24c475d8cd36af860fc374b0c5621b0d096be1d Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Mon, 9 Sep 2019 20:47:34 -0400 Subject: nearly finished transferring images between text notes and google docs --- package.json | 2 +- .../apis/google_docs/GooglePhotosClientUtils.ts | 23 ++-- .../util/Import & Export/DirectoryImportBox.tsx | 2 +- src/client/views/MainView.tsx | 6 +- src/client/views/collections/CollectionSubView.tsx | 2 +- src/client/views/nodes/DocumentView.tsx | 2 +- src/new_fields/RichTextUtils.ts | 67 ++++++++++- src/server/RouteStore.ts | 3 +- src/server/apis/google/GoogleApiServerUtils.ts | 2 +- src/server/apis/google/GooglePhotosUploadUtils.ts | 5 + src/server/credentials/google_docs_token.json | 2 +- src/server/index.ts | 20 +++- src/server/updateSearch.ts | 123 --------------------- 13 files changed, 112 insertions(+), 147 deletions(-) delete mode 100644 src/server/updateSearch.ts (limited to 'package.json') diff --git a/package.json b/package.json index f56e34ce0..f0f2b467e 100644 --- a/package.json +++ b/package.json @@ -224,4 +224,4 @@ "xoauth2": "^1.2.0", "youtube": "^0.1.0" } -} \ No newline at end of file +} diff --git a/src/client/apis/google_docs/GooglePhotosClientUtils.ts b/src/client/apis/google_docs/GooglePhotosClientUtils.ts index 5f5b39b14..fddcf3aa5 100644 --- a/src/client/apis/google_docs/GooglePhotosClientUtils.ts +++ b/src/client/apis/google_docs/GooglePhotosClientUtils.ts @@ -51,17 +51,26 @@ export namespace GooglePhotosClientUtils { description: string; } - export const UploadImageDocuments = async (sources: Doc[], album?: AlbumReference, descriptionKey = "caption") => { + export const UploadImages = async (sources: (Doc | string)[], album?: AlbumReference, descriptionKey = "caption") => { if (album && "title" in album) { album = await (await endpoint()).albums.create(album.title); } const media: MediaInput[] = []; - sources.forEach(document => { - const data = Cast(Doc.GetProto(document).data, ImageField); - data && media.push({ - url: data.url.href, - description: parseDescription(document, descriptionKey), - }); + sources.forEach(source => { + let url: string; + let description: string; + if (source instanceof Doc) { + const data = Cast(Doc.GetProto(source).data, ImageField); + if (!data) { + return; + } + url = data.url.href; + description = parseDescription(source, descriptionKey); + } else { + url = source; + description = Utils.GenerateGuid(); + } + media.push({ url, description }); }); if (media.length) { return PostToServer(RouteStore.googlePhotosMediaUpload, { media, album }); diff --git a/src/client/util/Import & Export/DirectoryImportBox.tsx b/src/client/util/Import & Export/DirectoryImportBox.tsx index 7693a388f..a19fd39b7 100644 --- a/src/client/util/Import & Export/DirectoryImportBox.tsx +++ b/src/client/util/Import & Export/DirectoryImportBox.tsx @@ -117,7 +117,7 @@ export default class DirectoryImportBox extends React.Component console.log(`(${this.quota - this.remaining}/${this.quota}) ${upload.name}`); })); - await GooglePhotosClientUtils.UploadImageDocuments(docs, { title: directory }); + await GooglePhotosClientUtils.UploadImages(docs, { title: directory }); console.log("Finished upload!"); for (let i = 0; i < docs.length; i++) { diff --git a/src/client/views/MainView.tsx b/src/client/views/MainView.tsx index 326c13424..0c0ed9072 100644 --- a/src/client/views/MainView.tsx +++ b/src/client/views/MainView.tsx @@ -155,7 +155,7 @@ export class MainView extends React.Component { doc.caption = "Well isn't this a nice cat image!"; let photos = await GooglePhotosClientUtils.endpoint(); let albumId = (await photos.albums.list(50)).albums.filter((album: any) => album.title === "This is a generically created album!")[0].id; - console.log(await GooglePhotosClientUtils.UploadImageDocuments([doc], { id: albumId })); + console.log(await GooglePhotosClientUtils.UploadImages([doc], { id: albumId })); } componentWillUnMount() { @@ -470,7 +470,7 @@ export class MainView extends React.Component { // let youtubeurl = "https://www.youtube.com/embed/TqcApsGRzWw"; // let addYoutubeSearcher = action(() => Docs.Create.YoutubeDocument(youtubeurl, { width: 600, height: 600, title: "youtube search" })); - let googlePhotosSearch = () => GooglePhotosClientUtils.CollectionFromSearch(Docs.Create.MasonryDocument, { included: [GooglePhotosClientUtils.ContentCategories.LANDSCAPES] }); + // let googlePhotosSearch = () => GooglePhotosClientUtils.CollectionFromSearch(Docs.Create.MasonryDocument, { included: [GooglePhotosClientUtils.ContentCategories.LANDSCAPES] }); let btns: [React.RefObject, IconName, string, () => Doc | Promise][] = [ [React.createRef(), "object-group", "Add Collection", addColNode], @@ -478,7 +478,7 @@ export class MainView extends React.Component { [React.createRef(), "globe-asia", "Add Website", addWebNode], [React.createRef(), "bolt", "Add Button", addButtonDocument], [React.createRef(), "file", "Add Document Dragger", addDragboxNode], - [React.createRef(), "object-group", "Test Google Photos Search", googlePhotosSearch], + // [React.createRef(), "object-group", "Test Google Photos Search", googlePhotosSearch], [React.createRef(), "cloud-upload-alt", "Import Directory", addImportCollectionNode], //remove at some point in favor of addImportCollectionNode //[React.createRef(), "play", "Add Youtube Searcher", addYoutubeSearcher], ]; diff --git a/src/client/views/collections/CollectionSubView.tsx b/src/client/views/collections/CollectionSubView.tsx index 99e5ab7b3..5fc4f36a7 100644 --- a/src/client/views/collections/CollectionSubView.tsx +++ b/src/client/views/collections/CollectionSubView.tsx @@ -253,7 +253,7 @@ export function CollectionSubView(schemaCtor: (doc: Doc) => T) { }).then(async (res: Response) => { (await res.json()).map(action((file: any) => { let full = { ...options, nativeWidth: type.indexOf("video") !== -1 ? 600 : 300, width: 300, title: dropFileName }; - let path = Utils.prepend(file); + let path = Utils.prepend(file.path); Docs.Get.DocumentFromType(type, path, full).then(doc => doc && this.props.addDocument(doc)); })); }); diff --git a/src/client/views/nodes/DocumentView.tsx b/src/client/views/nodes/DocumentView.tsx index 4033ffd9c..cb9346a8b 100644 --- a/src/client/views/nodes/DocumentView.tsx +++ b/src/client/views/nodes/DocumentView.tsx @@ -591,7 +591,7 @@ export class DocumentView extends DocComponent(Docu subitems.push({ description: "Open Fields", event: this.fieldsClicked, icon: "layer-group" }); cm.addItem({ description: "Open...", subitems: subitems, icon: "external-link-alt" }); if (Cast(this.props.Document.data, ImageField)) { - cm.addItem({ description: "Export to Google Photos", event: () => GooglePhotosClientUtils.UploadImageDocuments([this.props.Document]), icon: "caret-square-right" }); + cm.addItem({ description: "Export to Google Photos", event: () => GooglePhotosClientUtils.UploadImages([this.props.Document]), icon: "caret-square-right" }); } let existingMake = ContextMenu.Instance.findByDescription("Make..."); let makes: ContextMenuProps[] = existingMake && "subitems" in existingMake ? existingMake.subitems : []; diff --git a/src/new_fields/RichTextUtils.ts b/src/new_fields/RichTextUtils.ts index 0aba50c0d..500b93676 100644 --- a/src/new_fields/RichTextUtils.ts +++ b/src/new_fields/RichTextUtils.ts @@ -7,6 +7,11 @@ import { FormattedTextBox } from "../client/views/nodes/FormattedTextBox"; import { Opt } from "./Doc"; import * as Color from "color"; import { sinkListItem } from "prosemirror-schema-list"; +import { Utils, PostToServer } from "../Utils"; +import { RouteStore } from "../server/RouteStore"; +import { Docs } from "../client/documents/Documents"; +import { schema } from "../client/util/RichTextSchema"; +import { GooglePhotosClientUtils } from "../client/apis/google_docs/GooglePhotosClientUtils"; export namespace RichTextUtils { @@ -86,19 +91,36 @@ export namespace RichTextUtils { export namespace GoogleDocs { export const Export = (state: EditorState): GoogleApiClientUtils.Docs.Content => { - let textNodes: Node[] = []; + let nodes: { [type: string]: Node[] } = { + text: [], + image: [] + }; let text = ToPlainText(state); let content = state.doc.content; - content.forEach(node => node.content.forEach(node => node.type.name === "text" && textNodes.push(node))); - let linkRequests = ExtractLinks(textNodes); + content.forEach(node => node.content.forEach(node => { + const type = node.type.name; + let existing = nodes[type]; + if (existing) { + existing.push(node); + } else { + nodes[type] = [node]; + } + })); + let linkRequests = ExtractLinks(nodes.text); + let imageRequests = ExtractImages(nodes.image); return { text, - requests: [...linkRequests] + requests: [...linkRequests, ...imageRequests] }; }; type BulletPosition = { value: number, sinks: number }; + interface MediaItem { + baseUrl: string; + filename: string; + width: number; + } export const Import = async (documentId: GoogleApiClientUtils.Docs.DocumentId): Promise> => { const document = await GoogleApiClientUtils.Docs.retrieve({ documentId }); if (!document) { @@ -109,6 +131,17 @@ export namespace RichTextUtils { const { text, paragraphs } = GoogleApiClientUtils.Docs.Utils.extractText(document); let state = FormattedTextBox.blankState(); let structured = parseLists(paragraphs); + const inline = document.inlineObjects; + let inlineUrls: MediaItem[] = []; + if (inline) { + inlineUrls = Object.keys(inline).map(key => { + const embedded = inline[key].inlineObjectProperties!.embeddedObject!; + const baseUrl = embedded.imageProperties!.contentUri!; + const filename = `upload_${Utils.GenerateGuid()}.png`; + const width = embedded.size!.width!.magnitude!; + return { baseUrl, filename, width }; + }); + } let position = 3; let lists: ListGroup[] = []; @@ -151,6 +184,12 @@ export namespace RichTextUtils { } } + const uploads = await PostToServer(RouteStore.googlePhotosMediaDownload, { mediaItems: inlineUrls }); + for (let i = 0; i < uploads.length; i++) { + const src = Utils.prepend(`/files/${uploads[i].fileNames.clean}`); + state = state.apply(state.tr.insert(0, schema.nodes.image.create({ src, width: inlineUrls[i].width }))); + } + return { title, text, state }; }; @@ -252,6 +291,26 @@ export namespace RichTextUtils { return links; }; + const ExtractImages = async (nodes: Node[]) => { + const images: docs_v1.Schema$Request[] = []; + let position = 1; + for (let node of nodes) { + const length = node.nodeSize; + const attrs = node.attrs; + const uri = attrs.src; + const result = (await GooglePhotosClientUtils.UploadImages([uri])).newMediaItemResults; + images.push({ + insertInlineImage: { + uri: result[0].mediaItem.productUrl, + objectSize: { width: { magnitude: parseFloat(attrs.width.replace("px", "")), unit: "PT" } }, + location: { index: position + length } + } + }); + position += length; + } + return images; + }; + const Encode = (information: LinkInformation) => { return { updateTextStyle: { diff --git a/src/server/RouteStore.ts b/src/server/RouteStore.ts index f65e6134c..ee9cd8a0e 100644 --- a/src/server/RouteStore.ts +++ b/src/server/RouteStore.ts @@ -34,6 +34,7 @@ export enum RouteStore { googleDocs = "/googleDocs", googlePhotosAccessToken = "/googlePhotosAccessToken", googlePhotosMediaUpload = "/googlePhotosMediaUpload", - googlePhotosMediaDownload = "/googlePhotosMediaDownload" + googlePhotosMediaDownload = "/googlePhotosMediaDownload", + googleDocsGet = "/googleDocsGet" } \ No newline at end of file diff --git a/src/server/apis/google/GoogleApiServerUtils.ts b/src/server/apis/google/GoogleApiServerUtils.ts index ac8023ce1..e0bd8a800 100644 --- a/src/server/apis/google/GoogleApiServerUtils.ts +++ b/src/server/apis/google/GoogleApiServerUtils.ts @@ -42,7 +42,7 @@ export namespace GoogleApiServerUtils { export type ApiResponse = Promise; export type ApiRouter = (endpoint: Endpoint, parameters: any) => ApiResponse; - export type ApiHandler = (parameters: any) => ApiResponse; + export type ApiHandler = (parameters: any, methodOptions?: any) => ApiResponse; export type Action = "create" | "retrieve" | "update"; export type Endpoint = { get: ApiHandler, create: ApiHandler, batchUpdate: ApiHandler }; diff --git a/src/server/apis/google/GooglePhotosUploadUtils.ts b/src/server/apis/google/GooglePhotosUploadUtils.ts index 35f986250..d1f1f81bd 100644 --- a/src/server/apis/google/GooglePhotosUploadUtils.ts +++ b/src/server/apis/google/GooglePhotosUploadUtils.ts @@ -151,6 +151,11 @@ export namespace DownloadUtils { .on('close', resolve) .on('error', reject); }); + if (!isLocal) { + await new Promise(resolve => { + stream(url).pipe(fs.createWriteStream(uploadDirectory + resolved)).on('close', resolve); + }); + } } resolve(information); }); diff --git a/src/server/credentials/google_docs_token.json b/src/server/credentials/google_docs_token.json index fabc18cfd..5c142fba1 100644 --- a/src/server/credentials/google_docs_token.json +++ b/src/server/credentials/google_docs_token.json @@ -1 +1 @@ -{"access_token":"ya29.Glx-BwgWcpQUukTNyuUqvSAYrDyxDNUhCLtrFDJAViROvicm0DrcRvCn4OaQdn2m2IZQYcG-19cvQYoOC3UJCtWXLRvKZzQCbZZSykpxYu_lflUyEnIGZOIHMbbEjA","refresh_token":"1/HTv_xFHszu2Nf3iiFrUTaeKzC_Vp2-6bpIB06xW_WHI","scope":"https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/documents.readonly https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/photoslibrary https://www.googleapis.com/auth/photoslibrary.appendonly https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/photoslibrary.sharing","token_type":"Bearer","expiry_date":1568008211814} \ No newline at end of file +{"access_token":"ya29.Glx_B6G7Q_FYs1LK5VcyV6P6Zg9JkoHO2aC_TsnN7AVxPYWHEpsBSC0WyWX7Ztr8HWhOUYA5JXqnZDkLrK1V3Hb-0GgtyApLRNtEPOWf1dJ7lOm_iKVw2tRvPe7XDQ","refresh_token":"1/HTv_xFHszu2Nf3iiFrUTaeKzC_Vp2-6bpIB06xW_WHI","scope":"https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/documents.readonly https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/photoslibrary https://www.googleapis.com/auth/photoslibrary.appendonly https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/photoslibrary.sharing","token_type":"Bearer","expiry_date":1568078116605} \ No newline at end of file diff --git a/src/server/index.ts b/src/server/index.ts index baef94a59..8469770d5 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -38,6 +38,7 @@ import flash = require('connect-flash'); import { Search } from './Search'; import _ = require('lodash'); import * as Archiver from 'archiver'; +import * as request_promise from 'request-promise'; var AdmZip = require('adm-zip'); import * as YoutubeApi from "./apis/youtube/youtubeApiSample"; import { Response } from 'express-serve-static-core'; @@ -576,9 +577,9 @@ app.post( form.parse(req, async (_err, _fields, files) => { let results: FileResponse[] = []; for (const key in files) { - const { name, type, path: location } = files[key]; + const { type, path: location, name } = files[key]; const filename = path.basename(location); - await UploadUtils.UploadImage(uploadDirectory + filename, path.basename(name)); + await UploadUtils.UploadImage(uploadDirectory + filename, filename); results.push({ name, type, path: `/files/${filename}` }); console.log(path.basename(name)); } @@ -790,10 +791,23 @@ const tokenPath = path.join(__dirname, "./credentials/google_docs_token.json"); const EndpointHandlerMap = new Map([ ["create", (api, params) => api.create(params)], - ["retrieve", (api, params) => api.get(params)], + ["retrieve", (api, params) => api.get(params, { params: "fields=inlineObjects" })], ["update", (api, params) => api.batchUpdate(params)], ]); +// app.post(RouteStore.googleDocsGet, async (req, res) => { +// const token = await GoogleApiServerUtils.RetrieveAccessToken({ credentialsPath, tokenPath }); +// request_promise.get({ +// uri: `https://docs.googleapis.com/v1/documents/${req.body.documentId}?fields=inlineObjects`, +// headers: { +// 'Authorization': `Bearer ${token}` +// } +// }).then(response => { +// console.log(response); +// res.send(response); +// }); +// }); + app.post(RouteStore.googleDocs + "/:sector/:action", (req, res) => { let sector: GoogleApiServerUtils.Service = req.params.sector; let action: GoogleApiServerUtils.Action = req.params.action; diff --git a/src/server/updateSearch.ts b/src/server/updateSearch.ts deleted file mode 100644 index 906b795f1..000000000 --- a/src/server/updateSearch.ts +++ /dev/null @@ -1,123 +0,0 @@ -import { Database } from "./database"; -import { Cursor } from "mongodb"; -import { Search } from "./Search"; -import pLimit from 'p-limit'; - -const suffixMap: { [type: string]: (string | [string, string | ((json: any) => any)]) } = { - "number": "_n", - "string": "_t", - "boolean": "_b", - // "image": ["_t", "url"], - "video": ["_t", "url"], - "pdf": ["_t", "url"], - "audio": ["_t", "url"], - "web": ["_t", "url"], - "date": ["_d", value => new Date(value.date).toISOString()], - "proxy": ["_i", "fieldId"], - "list": ["_l", list => { - const results = []; - for (const value of list.fields) { - const term = ToSearchTerm(value); - if (term) { - results.push(term.value); - } - } - return results.length ? results : null; - }] -}; - -function ToSearchTerm(val: any): { suffix: string, value: any } | undefined { - if (val === null || val === undefined) { - return; - } - const type = val.__type || typeof val; - let suffix = suffixMap[type]; - if (!suffix) { - return; - } - - if (Array.isArray(suffix)) { - const accessor = suffix[1]; - if (typeof accessor === "function") { - val = accessor(val); - } else { - val = val[accessor]; - } - suffix = suffix[0]; - } - - return { suffix, value: val }; -} - -function getSuffix(value: string | [string, any]): string { - return typeof value === "string" ? value : value[0]; -} - -const limit = pLimit(5); -async function update() { - // await new Promise(res => setTimeout(res, 5)); - console.log("update"); - await Search.Instance.clear(); - const cursor = await Database.Instance.query({}); - console.log("Cleared"); - const updates: any[] = []; - let numDocs = 0; - function updateDoc(doc: any) { - numDocs++; - if ((numDocs % 50) === 0) { - console.log("updateDoc " + numDocs); - } - // console.log("doc " + numDocs); - if (doc.__type !== "Doc") { - return; - } - const fields = doc.fields; - if (!fields) { - return; - } - const update: any = { id: doc._id }; - let dynfield = false; - for (const key in fields) { - const value = fields[key]; - const term = ToSearchTerm(value); - if (term !== undefined) { - let { suffix, value } = term; - update[key + suffix] = value; - dynfield = true; - } - } - if (dynfield) { - updates.push(update); - // console.log(updates.length); - } - } - await cursor.forEach(updateDoc); - console.log(`Updating ${updates.length} documents`); - const result = await Search.Instance.updateDocuments(updates); - try { - console.log(JSON.parse(result).responseHeader.status); - } catch { - console.log("Error:"); - // console.log(updates[i]); - console.log(result); - console.log("\n"); - } - // for (let i = 0; i < updates.length; i++) { - // console.log(i); - // const result = await Search.Instance.updateDocument(updates[i]); - // try { - // console.log(JSON.parse(result).responseHeader.status); - // } catch { - // console.log("Error:"); - // console.log(updates[i]); - // console.log(result); - // console.log("\n"); - // } - // } - // await Promise.all(updates.map(update => { - // return limit(() => Search.Instance.updateDocument(update)); - // })); - cursor.close(); -} - -update(); \ No newline at end of file -- cgit v1.2.3-70-g09d2 From d66b51213e448d5f4f37781389af488a3ac744c4 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Fri, 20 Sep 2019 05:10:21 -0400 Subject: factored out extensions into npm module --- package.json | 1 + .../util/Import & Export/DirectoryImportBox.tsx | 7 +- src/client/views/Main.tsx | 3 - src/extensions/ArrayExtensions.ts | 639 ++++++++++----------- src/extensions/Extensions.ts | 2 +- src/server/apis/google/GooglePhotosUploadUtils.ts | 8 +- src/server/credentials/google_docs_token.json | 2 +- src/server/index.ts | 10 +- 8 files changed, 326 insertions(+), 346 deletions(-) (limited to 'package.json') diff --git a/package.json b/package.json index f869713ba..b20c31a7a 100644 --- a/package.json +++ b/package.json @@ -114,6 +114,7 @@ "@types/youtube": "0.0.38", "adm-zip": "^0.4.13", "archiver": "^3.0.3", + "array-batcher": "^1.0.2", "async": "^2.6.2", "babel-runtime": "^6.26.0", "bcrypt-nodejs": "0.0.3", diff --git a/src/client/util/Import & Export/DirectoryImportBox.tsx b/src/client/util/Import & Export/DirectoryImportBox.tsx index e3958e3a4..762302bc8 100644 --- a/src/client/util/Import & Export/DirectoryImportBox.tsx +++ b/src/client/util/Import & Export/DirectoryImportBox.tsx @@ -20,6 +20,7 @@ import { listSpec } from "../../../new_fields/Schema"; import { GooglePhotos } from "../../apis/google_docs/GooglePhotosClientUtils"; import { SchemaHeaderField } from "../../../new_fields/SchemaHeaderField"; import "./DirectoryImportBox.scss"; +import { batchedMapAsync } from "array-batcher"; const unsupported = ["text/html", "text/plain"]; interface FileResponse { @@ -103,7 +104,7 @@ export default class DirectoryImportBox extends React.Component runInAction(() => this.phase = `Internal: uploading ${this.quota - this.completed} files to Dash...`); - const uploads = await validated.batchedMapAsync({ batchSize: 15 }, async batch => { + const uploads = await batchedMapAsync(validated, { batchSize: 15 }, async batch => { const formData = new FormData(); const parameters = { method: 'POST', body: formData }; @@ -113,9 +114,9 @@ export default class DirectoryImportBox extends React.Component formData.append(Utils.GenerateGuid(), file); }); - const responses = (await fetch(RouteStore.upload, parameters)).json(); + const responses = await (await fetch(RouteStore.upload, parameters)).json(); runInAction(() => this.completed += batch.length); - return responses; + return responses as FileResponse[]; }); await Promise.all(uploads.map(async upload => { diff --git a/src/client/views/Main.tsx b/src/client/views/Main.tsx index 53912550c..70d2235e6 100644 --- a/src/client/views/Main.tsx +++ b/src/client/views/Main.tsx @@ -7,9 +7,6 @@ import { Cast } from "../../new_fields/Types"; import { Doc, DocListCastAsync } from "../../new_fields/Doc"; import { List } from "../../new_fields/List"; import { DocServer } from "../DocServer"; -const Extensions = require("../../extensions/Extensions"); - -Extensions.AssignExtensions(); let swapDocs = async () => { let oldDoc = await Cast(CurrentUserUtils.UserDocument.linkManagerDoc, Doc); diff --git a/src/extensions/ArrayExtensions.ts b/src/extensions/ArrayExtensions.ts index 872f107a7..ca407862b 100644 --- a/src/extensions/ArrayExtensions.ts +++ b/src/extensions/ArrayExtensions.ts @@ -1,333 +1,318 @@ interface Array { - fixedBatch(batcher: FixedBatcher): T[][]; - predicateBatch(batcher: PredicateBatcherSync): T[][]; - predicateBatchAsync(batcher: PredicateBatcherAsync): Promise; - batch(batcher: BatcherSync): T[][]; - batchAsync(batcher: Batcher): Promise; - - batchedForEach(batcher: BatcherSync, handler: BatchHandlerSync): void; - batchedMap(batcher: BatcherSync, handler: BatchConverterSync): O[]; - - batchedForEachAsync(batcher: Batcher, handler: BatchHandler): Promise; - batchedMapAsync(batcher: Batcher, handler: BatchConverter): Promise; - - batchedForEachInterval(batcher: Batcher, handler: BatchHandler, interval: Interval): Promise; - batchedMapInterval(batcher: Batcher, handler: BatchConverter, interval: Interval): Promise; - lastElement(): T; } -interface BatchContext { - completedBatches: number; - remainingBatches: number; -} - -interface ExecutorResult { - updated: A; - makeNextBatch: boolean; -} - -interface PredicateBatcherCommon { - initial: A; - persistAccumulator?: boolean; -} - -interface Interval { - magnitude: number; - unit: typeof module.exports.TimeUnit; -} - -type BatchConverterSync = (batch: I[], context: BatchContext) => O[]; -type BatchConverterAsync = (batch: I[], context: BatchContext) => Promise; -type BatchConverter = BatchConverterSync | BatchConverterAsync; - -type BatchHandlerSync = (batch: I[], context: BatchContext) => void; -type BatchHandlerAsync = (batch: I[], context: BatchContext) => Promise; -type BatchHandler = BatchHandlerSync | BatchHandlerAsync; - -type BatcherSync = FixedBatcher | PredicateBatcherSync; -type BatcherAsync = PredicateBatcherAsync; -type Batcher = BatcherSync | BatcherAsync; - -type FixedBatcher = { batchSize: number } | { batchCount: number, mode?: typeof module.exports.Mode }; -type PredicateBatcherSync = PredicateBatcherCommon & { executor: (element: I, accumulator: A) => ExecutorResult }; -type PredicateBatcherAsync = PredicateBatcherCommon & { executorAsync: (element: I, accumulator: A) => Promise> }; - - -module.exports.Mode = { - Balanced: 0, - Even: 1 -}; - -module.exports.TimeUnit = { - Milliseconds: 0, - Seconds: 1, - Minutes: 2 -}; - -module.exports.Assign = function () { - - Array.prototype.fixedBatch = function (batcher: FixedBatcher): T[][] { - const batches: T[][] = []; - const length = this.length; - let i = 0; - if ("batchSize" in batcher) { - const { batchSize } = batcher; - while (i < this.length) { - const cap = Math.min(i + batchSize, length); - batches.push(this.slice(i, i = cap)); - } - } else if ("batchCount" in batcher) { - let { batchCount, mode } = batcher; - const resolved = mode || module.exports.Mode.Balanced; - if (batchCount < 1) { - throw new Error("Batch count must be a positive integer!"); - } - if (batchCount === 1) { - return [this]; - } - if (batchCount >= this.length) { - return this.map((element: T) => [element]); - } - - let length = this.length; - let size: number; - - if (length % batchCount === 0) { - size = Math.floor(length / batchCount); - while (i < length) { - batches.push(this.slice(i, i += size)); - } - } else if (resolved === module.exports.Mode.Balanced) { - while (i < length) { - size = Math.ceil((length - i) / batchCount--); - batches.push(this.slice(i, i += size)); - } - } else { - batchCount--; - size = Math.floor(length / batchCount); - if (length % size === 0) { - size--; - } - while (i < size * batchCount) { - batches.push(this.slice(i, i += size)); - } - batches.push(this.slice(size * batchCount)); - } - } - return batches; - }; - - Array.prototype.predicateBatch = function (batcher: PredicateBatcherSync): T[][] { - const batches: T[][] = []; - let batch: T[] = []; - const { executor, initial, persistAccumulator } = batcher; - let accumulator = initial; - for (let element of this) { - const { updated, makeNextBatch } = executor(element, accumulator); - accumulator = updated; - if (!makeNextBatch) { - batch.push(element); - } else { - batches.push(batch); - batch = [element]; - if (!persistAccumulator) { - accumulator = initial; - } - } - } - batches.push(batch); - return batches; - }; - - Array.prototype.predicateBatchAsync = async function (batcher: BatcherAsync): Promise { - const batches: T[][] = []; - let batch: T[] = []; - const { executorAsync, initial, persistAccumulator } = batcher; - let accumulator: A = initial; - for (let element of this) { - const { updated, makeNextBatch } = await executorAsync(element, accumulator); - accumulator = updated; - if (!makeNextBatch) { - batch.push(element); - } else { - batches.push(batch); - batch = [element]; - if (!persistAccumulator) { - accumulator = initial; - } - } - } - batches.push(batch); - return batches; - }; - - Array.prototype.batch = function (batcher: BatcherSync): T[][] { - if ("executor" in batcher) { - return this.predicateBatch(batcher); - } else { - return this.fixedBatch(batcher); - } - }; - - Array.prototype.batchAsync = async function (batcher: Batcher): Promise { - if ("executorAsync" in batcher) { - return this.predicateBatchAsync(batcher); - } else { - return this.batch(batcher); - } - }; - - Array.prototype.batchedForEach = function (batcher: BatcherSync, handler: BatchHandlerSync): void { - if (this.length) { - let completed = 0; - const batches = this.batch(batcher); - const quota = batches.length; - for (let batch of batches) { - const context: BatchContext = { - completedBatches: completed, - remainingBatches: quota - completed, - }; - handler(batch, context); - completed++; - } - } - }; - - Array.prototype.batchedMap = function (batcher: BatcherSync, handler: BatchConverterSync): O[] { - if (!this.length) { - return []; - } - let collector: O[] = []; - let completed = 0; - const batches = this.batch(batcher); - const quota = batches.length; - for (let batch of batches) { - const context: BatchContext = { - completedBatches: completed, - remainingBatches: quota - completed, - }; - collector.push(...handler(batch, context)); - completed++; - } - return collector; - }; - - Array.prototype.batchedForEachAsync = async function (batcher: Batcher, handler: BatchHandler): Promise { - if (this.length) { - let completed = 0; - const batches = await this.batchAsync(batcher); - const quota = batches.length; - for (let batch of batches) { - const context: BatchContext = { - completedBatches: completed, - remainingBatches: quota - completed, - }; - await handler(batch, context); - completed++; - } - } - }; - - Array.prototype.batchedMapAsync = async function (batcher: Batcher, handler: BatchConverter): Promise { - if (!this.length) { - return []; - } - let collector: O[] = []; - let completed = 0; - const batches = await this.batchAsync(batcher); - const quota = batches.length; - for (let batch of batches) { - const context: BatchContext = { - completedBatches: completed, - remainingBatches: quota - completed, - }; - collector.push(...(await handler(batch, context))); - completed++; - } - return collector; - }; - - Array.prototype.batchedForEachInterval = async function (batcher: Batcher, handler: BatchHandler, interval: Interval): Promise { - if (!this.length) { - return; - } - const batches = await this.batchAsync(batcher); - const quota = batches.length; - return new Promise(async resolve => { - const iterator = batches[Symbol.iterator](); - let completed = 0; - while (true) { - const next = iterator.next(); - await new Promise(resolve => { - setTimeout(async () => { - const batch = next.value; - const context: BatchContext = { - completedBatches: completed, - remainingBatches: quota - completed, - }; - await handler(batch, context); - resolve(); - }, convert(interval)); - }); - if (++completed === quota) { - break; - } - } - resolve(); - }); - }; - - Array.prototype.batchedMapInterval = async function (batcher: Batcher, handler: BatchConverter, interval: Interval): Promise { - if (!this.length) { - return []; - } - let collector: O[] = []; - const batches = await this.batchAsync(batcher); - const quota = batches.length; - return new Promise(async resolve => { - const iterator = batches[Symbol.iterator](); - let completed = 0; - while (true) { - const next = iterator.next(); - await new Promise(resolve => { - setTimeout(async () => { - const batch = next.value; - const context: BatchContext = { - completedBatches: completed, - remainingBatches: quota - completed, - }; - collector.push(...(await handler(batch, context))); - resolve(); - }, convert(interval)); - }); - if (++completed === quota) { - resolve(collector); - break; - } - } - }); - }; - - Array.prototype.lastElement = function () { - if (!this.length) { - return undefined; - } - const last: T = this[this.length - 1]; - return last; - }; - +// interface BatchContext { +// completedBatches: number; +// remainingBatches: number; +// } + +// interface ExecutorResult { +// updated: A; +// makeNextBatch: boolean; +// } + +// interface PredicateBatcherCommon { +// initial: A; +// persistAccumulator?: boolean; +// } + +// interface Interval { +// magnitude: number; +// unit: typeof module.exports.TimeUnit; +// } + +// type BatchConverterSync = (batch: I[], context: BatchContext) => O[]; +// type BatchConverterAsync = (batch: I[], context: BatchContext) => Promise; +// type BatchConverter = BatchConverterSync | BatchConverterAsync; + +// type BatchHandlerSync = (batch: I[], context: BatchContext) => void; +// type BatchHandlerAsync = (batch: I[], context: BatchContext) => Promise; +// type BatchHandler = BatchHandlerSync | BatchHandlerAsync; + +// type BatcherSync = FixedBatcher | PredicateBatcherSync; +// type BatcherAsync = PredicateBatcherAsync; +// type Batcher = BatcherSync | BatcherAsync; + +// type FixedBatcher = { batchSize: number } | { batchCount: number, mode?: typeof module.exports.Mode }; +// type PredicateBatcherSync = PredicateBatcherCommon & { executor: (element: I, accumulator: A) => ExecutorResult }; +// type PredicateBatcherAsync = PredicateBatcherCommon & { executorAsync: (element: I, accumulator: A) => Promise> }; + + +// module.exports.Mode = { +// Balanced: 0, +// Even: 1 +// }; + +// module.exports.TimeUnit = { +// Milliseconds: 0, +// Seconds: 1, +// Minutes: 2 +// }; + +// module.exports.Assign = function () { + +// Array.prototype.fixedBatch = function (batcher: FixedBatcher): T[][] { +// const batches: T[][] = []; +// const length = this.length; +// let i = 0; +// if ("batchSize" in batcher) { +// const { batchSize } = batcher; +// while (i < this.length) { +// const cap = Math.min(i + batchSize, length); +// batches.push(this.slice(i, i = cap)); +// } +// } else if ("batchCount" in batcher) { +// let { batchCount, mode } = batcher; +// const resolved = mode || module.exports.Mode.Balanced; +// if (batchCount < 1) { +// throw new Error("Batch count must be a positive integer!"); +// } +// if (batchCount === 1) { +// return [this]; +// } +// if (batchCount >= this.length) { +// return this.map((element: T) => [element]); +// } + +// let length = this.length; +// let size: number; + +// if (length % batchCount === 0) { +// size = Math.floor(length / batchCount); +// while (i < length) { +// batches.push(this.slice(i, i += size)); +// } +// } else if (resolved === module.exports.Mode.Balanced) { +// while (i < length) { +// size = Math.ceil((length - i) / batchCount--); +// batches.push(this.slice(i, i += size)); +// } +// } else { +// batchCount--; +// size = Math.floor(length / batchCount); +// if (length % size === 0) { +// size--; +// } +// while (i < size * batchCount) { +// batches.push(this.slice(i, i += size)); +// } +// batches.push(this.slice(size * batchCount)); +// } +// } +// return batches; +// }; + +// Array.prototype.predicateBatch = function (batcher: PredicateBatcherSync): T[][] { +// const batches: T[][] = []; +// let batch: T[] = []; +// const { executor, initial, persistAccumulator } = batcher; +// let accumulator = initial; +// for (let element of this) { +// const { updated, makeNextBatch } = executor(element, accumulator); +// accumulator = updated; +// if (!makeNextBatch) { +// batch.push(element); +// } else { +// batches.push(batch); +// batch = [element]; +// if (!persistAccumulator) { +// accumulator = initial; +// } +// } +// } +// batches.push(batch); +// return batches; +// }; + +// Array.prototype.predicateBatchAsync = async function (batcher: BatcherAsync): Promise { +// const batches: T[][] = []; +// let batch: T[] = []; +// const { executorAsync, initial, persistAccumulator } = batcher; +// let accumulator: A = initial; +// for (let element of this) { +// const { updated, makeNextBatch } = await executorAsync(element, accumulator); +// accumulator = updated; +// if (!makeNextBatch) { +// batch.push(element); +// } else { +// batches.push(batch); +// batch = [element]; +// if (!persistAccumulator) { +// accumulator = initial; +// } +// } +// } +// batches.push(batch); +// return batches; +// }; + +// Array.prototype.batch = function (batcher: BatcherSync): T[][] { +// if ("executor" in batcher) { +// return this.predicateBatch(batcher); +// } else { +// return this.fixedBatch(batcher); +// } +// }; + +// Array.prototype.batchAsync = async function (batcher: Batcher): Promise { +// if ("executorAsync" in batcher) { +// return this.predicateBatchAsync(batcher); +// } else { +// return this.batch(batcher); +// } +// }; + +// Array.prototype.batchedForEach = function (batcher: BatcherSync, handler: BatchHandlerSync): void { +// if (this.length) { +// let completed = 0; +// const batches = this.batch(batcher); +// const quota = batches.length; +// for (let batch of batches) { +// const context: BatchContext = { +// completedBatches: completed, +// remainingBatches: quota - completed, +// }; +// handler(batch, context); +// completed++; +// } +// } +// }; + +// Array.prototype.batchedMap = function (batcher: BatcherSync, handler: BatchConverterSync): O[] { +// if (!this.length) { +// return []; +// } +// let collector: O[] = []; +// let completed = 0; +// const batches = this.batch(batcher); +// const quota = batches.length; +// for (let batch of batches) { +// const context: BatchContext = { +// completedBatches: completed, +// remainingBatches: quota - completed, +// }; +// collector.push(...handler(batch, context)); +// completed++; +// } +// return collector; +// }; + +// Array.prototype.batchedForEachAsync = async function (batcher: Batcher, handler: BatchHandler): Promise { +// if (this.length) { +// let completed = 0; +// const batches = await this.batchAsync(batcher); +// const quota = batches.length; +// for (let batch of batches) { +// const context: BatchContext = { +// completedBatches: completed, +// remainingBatches: quota - completed, +// }; +// await handler(batch, context); +// completed++; +// } +// } +// }; + +// Array.prototype.batchedMapAsync = async function (batcher: Batcher, handler: BatchConverter): Promise { +// if (!this.length) { +// return []; +// } +// let collector: O[] = []; +// let completed = 0; +// const batches = await this.batchAsync(batcher); +// const quota = batches.length; +// for (let batch of batches) { +// const context: BatchContext = { +// completedBatches: completed, +// remainingBatches: quota - completed, +// }; +// collector.push(...(await handler(batch, context))); +// completed++; +// } +// return collector; +// }; + +// Array.prototype.batchedForEachInterval = async function (batcher: Batcher, handler: BatchHandler, interval: Interval): Promise { +// if (!this.length) { +// return; +// } +// const batches = await this.batchAsync(batcher); +// const quota = batches.length; +// return new Promise(async resolve => { +// const iterator = batches[Symbol.iterator](); +// let completed = 0; +// while (true) { +// const next = iterator.next(); +// await new Promise(resolve => { +// setTimeout(async () => { +// const batch = next.value; +// const context: BatchContext = { +// completedBatches: completed, +// remainingBatches: quota - completed, +// }; +// await handler(batch, context); +// resolve(); +// }, convert(interval)); +// }); +// if (++completed === quota) { +// break; +// } +// } +// resolve(); +// }); +// }; + +// Array.prototype.batchedMapInterval = async function (batcher: Batcher, handler: BatchConverter, interval: Interval): Promise { +// if (!this.length) { +// return []; +// } +// let collector: O[] = []; +// const batches = await this.batchAsync(batcher); +// const quota = batches.length; +// return new Promise(async resolve => { +// const iterator = batches[Symbol.iterator](); +// let completed = 0; +// while (true) { +// const next = iterator.next(); +// await new Promise(resolve => { +// setTimeout(async () => { +// const batch = next.value; +// const context: BatchContext = { +// completedBatches: completed, +// remainingBatches: quota - completed, +// }; +// collector.push(...(await handler(batch, context))); +// resolve(); +// }, convert(interval)); +// }); +// if (++completed === quota) { +// resolve(collector); +// break; +// } +// } +// }); +// }; + +Array.prototype.lastElement = function () { + if (!this.length) { + return undefined; + } + const last: T = this[this.length - 1]; + return last; }; -const convert = (interval: Interval) => { - const { magnitude, unit } = interval; - switch (unit) { - default: - case module.exports.TimeUnit.Milliseconds: - return magnitude; - case module.exports.TimeUnit.Seconds: - return magnitude * 1000; - case module.exports.TimeUnit.Minutes: - return magnitude * 1000 * 60; - } -}; \ No newline at end of file +// }; + +// const convert = (interval: Interval) => { +// const { magnitude, unit } = interval; +// switch (unit) { +// default: +// case module.exports.TimeUnit.Milliseconds: +// return magnitude; +// case module.exports.TimeUnit.Seconds: +// return magnitude * 1000; +// case module.exports.TimeUnit.Minutes: +// return magnitude * 1000 * 60; +// } +// }; \ No newline at end of file diff --git a/src/extensions/Extensions.ts b/src/extensions/Extensions.ts index 1bcebd0e2..1391140b9 100644 --- a/src/extensions/Extensions.ts +++ b/src/extensions/Extensions.ts @@ -2,6 +2,6 @@ const ArrayExtensions = require("./ArrayExtensions"); const StringExtensions = require("./StringExtensions"); module.exports.AssignExtensions = function () { - ArrayExtensions.Assign(); + // ArrayExtensions.Assign(); StringExtensions.Assign(); }; \ No newline at end of file diff --git a/src/server/apis/google/GooglePhotosUploadUtils.ts b/src/server/apis/google/GooglePhotosUploadUtils.ts index fc6772ffd..29575763c 100644 --- a/src/server/apis/google/GooglePhotosUploadUtils.ts +++ b/src/server/apis/google/GooglePhotosUploadUtils.ts @@ -7,7 +7,7 @@ import { Opt } from '../../../new_fields/Doc'; import * as sharp from 'sharp'; import { MediaItemCreationResult } from './SharedTypes'; import { NewMediaItem } from "../../index"; -const { TimeUnit } = require("../../../extensions/ArrayExtensions"); +import { batchedMapInterval, FixedBatcher, TimeUnit, Interval } from "array-batcher"; const uploadDirectory = path.join(__dirname, "../../public/files/"); @@ -81,10 +81,10 @@ export namespace GooglePhotosUploadUtils { }); })).newMediaItemResults; }; - const batcher = { batchSize: 50 }; - const interval = { magnitude: 100, unit: TimeUnit.Milliseconds }; + const batcher: FixedBatcher = { batchSize: 50 }; + const interval: Interval = { magnitude: 100, unit: TimeUnit.Milliseconds }; - const newMediaItemResults = await newMediaItems.batchedMapInterval(batcher, createFromUploadTokens, interval); + const newMediaItemResults = await batchedMapInterval(newMediaItems, batcher, createFromUploadTokens, interval); return { newMediaItemResults }; }; diff --git a/src/server/credentials/google_docs_token.json b/src/server/credentials/google_docs_token.json index 7c49eed43..cdea139a3 100644 --- a/src/server/credentials/google_docs_token.json +++ b/src/server/credentials/google_docs_token.json @@ -1 +1 @@ -{"access_token":"ya29.ImCJB1Y8Z8vgUH4vyYA9xwqvLg281kOQKfA8_AGs_EqF1VKQVWfZsMoYkPJN3QwJmIUxlzTO1N-ehUGIxu0Jq3kKR-zzW7rQIMgeQu32OHogK4kvFxpM7l7RNYRw_9x22I0","refresh_token":"1/HTv_xFHszu2Nf3iiFrUTaeKzC_Vp2-6bpIB06xW_WHI","scope":"https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/documents.readonly https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/photoslibrary https://www.googleapis.com/auth/photoslibrary.appendonly https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/photoslibrary.sharing","token_type":"Bearer","expiry_date":1568935635717} \ No newline at end of file +{"access_token":"ya29.ImCJB_jd-XlGcIHAHgN2Zl3BWQ6sMHdeMMuRxU6sPCbAYIT8hXws-WDmQf65ZY1f-0d3y7HcCcuOxtZJ_0IcBb1-yIBxiOf3VJWPmvjGiJQq_mANGVSSmsBHhqpIaYkeQN0","refresh_token":"1/HTv_xFHszu2Nf3iiFrUTaeKzC_Vp2-6bpIB06xW_WHI","scope":"https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/documents.readonly https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/photoslibrary https://www.googleapis.com/auth/photoslibrary.appendonly https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/photoslibrary.sharing","token_type":"Bearer","expiry_date":1568973665276} \ No newline at end of file diff --git a/src/server/index.ts b/src/server/index.ts index e3e4221cf..e03079d66 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -47,9 +47,7 @@ const mongoose = require('mongoose'); const probe = require("probe-image-size"); import * as qs from 'query-string'; import { Opt } from '../new_fields/Doc'; -const Extensions = require("../extensions/Extensions"); -const ArrayExtensions = require("../extensions/ArrayExtensions"); - +import { batchedMapInterval, TimeUnit } from "array-batcher"; const download = (url: string, dest: fs.PathLike) => request.get(url).pipe(fs.createWriteStream(dest)); let youtubeApiKey: string; @@ -101,8 +99,6 @@ enum Method { POST } -Extensions.AssignExtensions(); - /** * Please invoke this function when adding a new route to Dash's server. * It ensures that any requests leading to or containing user-sensitive information @@ -861,9 +857,9 @@ app.post(RouteStore.googlePhotosMediaUpload, async (req, res) => { return newMediaItems; }; const batcher = { batchSize: 25 }; - const interval = { magnitude: 100, unit: ArrayExtensions.TimeUnit.Milliseconds }; + const interval = { magnitude: 100, unit: TimeUnit.Milliseconds }; - const newMediaItems = await mediaInput.batchedMapInterval(batcher, dispatchUpload, interval); + const newMediaItems = await batchedMapInterval(mediaInput, batcher, dispatchUpload, interval); if (failed) { return _error(res, tokenError); -- cgit v1.2.3-70-g09d2 From 8d9c9d0f51be91ee10b631be9f464af0822bb9be Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Fri, 20 Sep 2019 16:39:14 -0400 Subject: integrated with array batcher npm module --- package.json | 2 +- .../util/Import & Export/DirectoryImportBox.tsx | 31 +++++++------- src/server/apis/google/GooglePhotosUploadUtils.ts | 49 +++++++++++----------- src/server/credentials/google_docs_token.json | 2 +- src/server/index.ts | 36 ++++++++-------- 5 files changed, 61 insertions(+), 59 deletions(-) (limited to 'package.json') diff --git a/package.json b/package.json index b20c31a7a..45babb358 100644 --- a/package.json +++ b/package.json @@ -114,7 +114,7 @@ "@types/youtube": "0.0.38", "adm-zip": "^0.4.13", "archiver": "^3.0.3", - "array-batcher": "^1.0.2", + "array-batcher": "^1.0.6", "async": "^2.6.2", "babel-runtime": "^6.26.0", "bcrypt-nodejs": "0.0.3", diff --git a/src/client/util/Import & Export/DirectoryImportBox.tsx b/src/client/util/Import & Export/DirectoryImportBox.tsx index 762302bc8..26a00dc7c 100644 --- a/src/client/util/Import & Export/DirectoryImportBox.tsx +++ b/src/client/util/Import & Export/DirectoryImportBox.tsx @@ -20,7 +20,7 @@ import { listSpec } from "../../../new_fields/Schema"; import { GooglePhotos } from "../../apis/google_docs/GooglePhotosClientUtils"; import { SchemaHeaderField } from "../../../new_fields/SchemaHeaderField"; import "./DirectoryImportBox.scss"; -import { batchedMapAsync } from "array-batcher"; +import BatchedArray from "array-batcher"; const unsupported = ["text/html", "text/plain"]; interface FileResponse { @@ -104,19 +104,22 @@ export default class DirectoryImportBox extends React.Component runInAction(() => this.phase = `Internal: uploading ${this.quota - this.completed} files to Dash...`); - const uploads = await batchedMapAsync(validated, { batchSize: 15 }, async batch => { - const formData = new FormData(); - const parameters = { method: 'POST', body: formData }; - - batch.forEach(file => { - sizes.push(file.size); - modifiedDates.push(file.lastModified); - formData.append(Utils.GenerateGuid(), file); - }); - - const responses = await (await fetch(RouteStore.upload, parameters)).json(); - runInAction(() => this.completed += batch.length); - return responses as FileResponse[]; + const uploads = await BatchedArray.from(validated).batchedMapAsync({ + batcher: { batchSize: 15 }, + converter: async batch => { + const formData = new FormData(); + const parameters = { method: 'POST', body: formData }; + + batch.forEach(file => { + sizes.push(file.size); + modifiedDates.push(file.lastModified); + formData.append(Utils.GenerateGuid(), file); + }); + + const responses = await (await fetch(RouteStore.upload, parameters)).json(); + runInAction(() => this.completed += batch.length); + return responses as FileResponse[]; + } }); await Promise.all(uploads.map(async upload => { diff --git a/src/server/apis/google/GooglePhotosUploadUtils.ts b/src/server/apis/google/GooglePhotosUploadUtils.ts index 29575763c..40564ff3b 100644 --- a/src/server/apis/google/GooglePhotosUploadUtils.ts +++ b/src/server/apis/google/GooglePhotosUploadUtils.ts @@ -7,7 +7,7 @@ import { Opt } from '../../../new_fields/Doc'; import * as sharp from 'sharp'; import { MediaItemCreationResult } from './SharedTypes'; import { NewMediaItem } from "../../index"; -import { batchedMapInterval, FixedBatcher, TimeUnit, Interval } from "array-batcher"; +import BatchedArray, { FixedBatcher, TimeUnit, Interval } from "array-batcher"; const uploadDirectory = path.join(__dirname, "../../public/files/"); @@ -62,30 +62,29 @@ export namespace GooglePhotosUploadUtils { }; export const CreateMediaItems = async (newMediaItems: NewMediaItem[], album?: { id: string }): Promise => { - const createFromUploadTokens = async (batch: NewMediaItem[]) => { - const parameters = { - method: 'POST', - headers: headers('json'), - uri: prepend('mediaItems:batchCreate'), - body: { newMediaItems: batch } as any, - json: true - }; - album && (parameters.body.albumId = album.id); - return (await new Promise((resolve, reject) => { - request(parameters, (error, _response, body) => { - if (error) { - reject(error); - } else { - resolve(body); - } - }); - })).newMediaItemResults; - }; - const batcher: FixedBatcher = { batchSize: 50 }; - const interval: Interval = { magnitude: 100, unit: TimeUnit.Milliseconds }; - - const newMediaItemResults = await batchedMapInterval(newMediaItems, batcher, createFromUploadTokens, interval); - + const newMediaItemResults = await BatchedArray.from(newMediaItems).batchedMapInterval({ + batcher: { batchSize: 50 }, + interval: { magnitude: 100, unit: TimeUnit.Milliseconds }, + converter: async (batch: NewMediaItem[]) => { + const parameters = { + method: 'POST', + headers: headers('json'), + uri: prepend('mediaItems:batchCreate'), + body: { newMediaItems: batch } as any, + json: true + }; + album && (parameters.body.albumId = album.id); + return (await new Promise((resolve, reject) => { + request(parameters, (error, _response, body) => { + if (error) { + reject(error); + } else { + resolve(body); + } + }); + })).newMediaItemResults; + } + }); return { newMediaItemResults }; }; diff --git a/src/server/credentials/google_docs_token.json b/src/server/credentials/google_docs_token.json index cdea139a3..9b0a649ac 100644 --- a/src/server/credentials/google_docs_token.json +++ b/src/server/credentials/google_docs_token.json @@ -1 +1 @@ -{"access_token":"ya29.ImCJB_jd-XlGcIHAHgN2Zl3BWQ6sMHdeMMuRxU6sPCbAYIT8hXws-WDmQf65ZY1f-0d3y7HcCcuOxtZJ_0IcBb1-yIBxiOf3VJWPmvjGiJQq_mANGVSSmsBHhqpIaYkeQN0","refresh_token":"1/HTv_xFHszu2Nf3iiFrUTaeKzC_Vp2-6bpIB06xW_WHI","scope":"https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/documents.readonly https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/photoslibrary https://www.googleapis.com/auth/photoslibrary.appendonly https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/photoslibrary.sharing","token_type":"Bearer","expiry_date":1568973665276} \ No newline at end of file +{"access_token":"ya29.ImCJB3_-mtKbCG8L7nQt3CDfn4HR2_xqUw8bUQkvaZZgxJZ8-WUlHe6UaSOmGtrvI4QabpYRGutYGSTUdc9bvIYooerZgAz80uk7-KEbnAxpwydRE-TK_1_VyWYZ2YW6Ht8","refresh_token":"1/HTv_xFHszu2Nf3iiFrUTaeKzC_Vp2-6bpIB06xW_WHI","scope":"https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/documents.readonly https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/photoslibrary https://www.googleapis.com/auth/photoslibrary.appendonly https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/photoslibrary.sharing","token_type":"Bearer","expiry_date":1569013995678} \ No newline at end of file diff --git a/src/server/index.ts b/src/server/index.ts index e03079d66..12ceb9886 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -47,7 +47,7 @@ const mongoose = require('mongoose'); const probe = require("probe-image-size"); import * as qs from 'query-string'; import { Opt } from '../new_fields/Doc'; -import { batchedMapInterval, TimeUnit } from "array-batcher"; +import BatchedArray, { TimeUnit } from "array-batcher"; const download = (url: string, dest: fs.PathLike) => request.get(url).pipe(fs.createWriteStream(dest)); let youtubeApiKey: string; @@ -841,25 +841,25 @@ app.post(RouteStore.googlePhotosMediaUpload, async (req, res) => { let failed = 0; - const dispatchUpload = async (batch: GooglePhotosUploadUtils.MediaInput[]) => { - const newMediaItems: NewMediaItem[] = []; - for (let element of batch) { - const uploadToken = await GooglePhotosUploadUtils.DispatchGooglePhotosUpload(element.url); - if (!uploadToken) { - failed++; - } else { - newMediaItems.push({ - description: element.description, - simpleMediaItem: { uploadToken } - }); + const newMediaItems = await BatchedArray.from(mediaInput).batchedMapInterval({ + batcher: { batchSize: 25 }, + interval: { magnitude: 100, unit: TimeUnit.Milliseconds }, + converter: async (batch: GooglePhotosUploadUtils.MediaInput[]) => { + const newMediaItems: NewMediaItem[] = []; + for (let element of batch) { + const uploadToken = await GooglePhotosUploadUtils.DispatchGooglePhotosUpload(element.url); + if (!uploadToken) { + failed++; + } else { + newMediaItems.push({ + description: element.description, + simpleMediaItem: { uploadToken } + }); + } } + return newMediaItems; } - return newMediaItems; - }; - const batcher = { batchSize: 25 }; - const interval = { magnitude: 100, unit: TimeUnit.Milliseconds }; - - const newMediaItems = await batchedMapInterval(mediaInput, batcher, dispatchUpload, interval); + }); if (failed) { return _error(res, tokenError); -- cgit v1.2.3-70-g09d2 From f529d7d22162689c08830418da67a89778111e16 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Sat, 21 Sep 2019 14:27:10 -0400 Subject: final batcher fixes --- package.json | 2 +- .../util/Import & Export/DirectoryImportBox.tsx | 29 ++++++++++------------ src/server/apis/google/GooglePhotosUploadUtils.ts | 11 ++++---- src/server/credentials/google_docs_token.json | 2 +- src/server/index.ts | 11 ++++---- 5 files changed, 25 insertions(+), 30 deletions(-) (limited to 'package.json') diff --git a/package.json b/package.json index 45babb358..f049f8826 100644 --- a/package.json +++ b/package.json @@ -114,7 +114,7 @@ "@types/youtube": "0.0.38", "adm-zip": "^0.4.13", "archiver": "^3.0.3", - "array-batcher": "^1.0.6", + "array-batcher": "^1.0.7", "async": "^2.6.2", "babel-runtime": "^6.26.0", "bcrypt-nodejs": "0.0.3", diff --git a/src/client/util/Import & Export/DirectoryImportBox.tsx b/src/client/util/Import & Export/DirectoryImportBox.tsx index 26a00dc7c..8948b73f7 100644 --- a/src/client/util/Import & Export/DirectoryImportBox.tsx +++ b/src/client/util/Import & Export/DirectoryImportBox.tsx @@ -104,22 +104,19 @@ export default class DirectoryImportBox extends React.Component runInAction(() => this.phase = `Internal: uploading ${this.quota - this.completed} files to Dash...`); - const uploads = await BatchedArray.from(validated).batchedMapAsync({ - batcher: { batchSize: 15 }, - converter: async batch => { - const formData = new FormData(); - const parameters = { method: 'POST', body: formData }; - - batch.forEach(file => { - sizes.push(file.size); - modifiedDates.push(file.lastModified); - formData.append(Utils.GenerateGuid(), file); - }); - - const responses = await (await fetch(RouteStore.upload, parameters)).json(); - runInAction(() => this.completed += batch.length); - return responses as FileResponse[]; - } + const uploads = await BatchedArray.from(validated, { batchSize: 15 }).batchedMapAsync(async batch => { + const formData = new FormData(); + const parameters = { method: 'POST', body: formData }; + + batch.forEach(file => { + sizes.push(file.size); + modifiedDates.push(file.lastModified); + formData.append(Utils.GenerateGuid(), file); + }); + + const responses = await (await fetch(RouteStore.upload, parameters)).json(); + runInAction(() => this.completed += batch.length); + return responses as FileResponse[]; }); await Promise.all(uploads.map(async upload => { diff --git a/src/server/apis/google/GooglePhotosUploadUtils.ts b/src/server/apis/google/GooglePhotosUploadUtils.ts index 40564ff3b..4dc252577 100644 --- a/src/server/apis/google/GooglePhotosUploadUtils.ts +++ b/src/server/apis/google/GooglePhotosUploadUtils.ts @@ -62,10 +62,8 @@ export namespace GooglePhotosUploadUtils { }; export const CreateMediaItems = async (newMediaItems: NewMediaItem[], album?: { id: string }): Promise => { - const newMediaItemResults = await BatchedArray.from(newMediaItems).batchedMapInterval({ - batcher: { batchSize: 50 }, - interval: { magnitude: 100, unit: TimeUnit.Milliseconds }, - converter: async (batch: NewMediaItem[]) => { + const newMediaItemResults = await BatchedArray.from(newMediaItems, { batchSize: 50 }).batchedMapInterval( + async (batch: NewMediaItem[]) => { const parameters = { method: 'POST', headers: headers('json'), @@ -83,8 +81,9 @@ export namespace GooglePhotosUploadUtils { } }); })).newMediaItemResults; - } - }); + }, + { magnitude: 100, unit: TimeUnit.Milliseconds } + ); return { newMediaItemResults }; }; diff --git a/src/server/credentials/google_docs_token.json b/src/server/credentials/google_docs_token.json index 9b0a649ac..ee44c3f30 100644 --- a/src/server/credentials/google_docs_token.json +++ b/src/server/credentials/google_docs_token.json @@ -1 +1 @@ -{"access_token":"ya29.ImCJB3_-mtKbCG8L7nQt3CDfn4HR2_xqUw8bUQkvaZZgxJZ8-WUlHe6UaSOmGtrvI4QabpYRGutYGSTUdc9bvIYooerZgAz80uk7-KEbnAxpwydRE-TK_1_VyWYZ2YW6Ht8","refresh_token":"1/HTv_xFHszu2Nf3iiFrUTaeKzC_Vp2-6bpIB06xW_WHI","scope":"https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/documents.readonly https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/photoslibrary https://www.googleapis.com/auth/photoslibrary.appendonly https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/photoslibrary.sharing","token_type":"Bearer","expiry_date":1569013995678} \ No newline at end of file +{"access_token":"ya29.GlyKBznz91v_qb8RYt4PT40Hp106N08Yk64UjMAKllBsIqJQEzBkxLbB5q5paydywHzguQYSNup5fT7ojJTDU4CMZdPbPKGcjQz17w_CospcG-8Buz94KZptvlQ_pQ","refresh_token":"1/HTv_xFHszu2Nf3iiFrUTaeKzC_Vp2-6bpIB06xW_WHI","scope":"https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/documents.readonly https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/photoslibrary https://www.googleapis.com/auth/photoslibrary.appendonly https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/photoslibrary.sharing","token_type":"Bearer","expiry_date":1569093749804} \ No newline at end of file diff --git a/src/server/index.ts b/src/server/index.ts index 12ceb9886..4c4cb84d6 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -841,10 +841,8 @@ app.post(RouteStore.googlePhotosMediaUpload, async (req, res) => { let failed = 0; - const newMediaItems = await BatchedArray.from(mediaInput).batchedMapInterval({ - batcher: { batchSize: 25 }, - interval: { magnitude: 100, unit: TimeUnit.Milliseconds }, - converter: async (batch: GooglePhotosUploadUtils.MediaInput[]) => { + const newMediaItems = await BatchedArray.from(mediaInput, { batchSize: 25 }).batchedMapInterval( + async (batch: GooglePhotosUploadUtils.MediaInput[]) => { const newMediaItems: NewMediaItem[] = []; for (let element of batch) { const uploadToken = await GooglePhotosUploadUtils.DispatchGooglePhotosUpload(element.url); @@ -858,8 +856,9 @@ app.post(RouteStore.googlePhotosMediaUpload, async (req, res) => { } } return newMediaItems; - } - }); + }, + { magnitude: 100, unit: TimeUnit.Milliseconds } + ); if (failed) { return _error(res, tokenError); -- cgit v1.2.3-70-g09d2 From db174b3adfc36678925b067aa398ecb0415ea0f8 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Fri, 27 Sep 2019 04:18:24 -0400 Subject: batcher updates --- package.json | 2 +- src/client/util/Import & Export/DirectoryImportBox.tsx | 3 ++- src/server/DashUploadUtils.ts | 4 ++-- src/server/apis/google/GooglePhotosUploadUtils.ts | 8 ++++---- src/server/index.ts | 8 ++++---- 5 files changed, 13 insertions(+), 12 deletions(-) (limited to 'package.json') diff --git a/package.json b/package.json index ea31b08a3..7a9e29f50 100644 --- a/package.json +++ b/package.json @@ -114,7 +114,7 @@ "@types/youtube": "0.0.38", "adm-zip": "^0.4.13", "archiver": "^3.0.3", - "array-batcher": "^1.0.7", + "array-batcher": "^1.1.3", "async": "^2.6.2", "babel-runtime": "^6.26.0", "bcrypt-nodejs": "0.0.3", diff --git a/src/client/util/Import & Export/DirectoryImportBox.tsx b/src/client/util/Import & Export/DirectoryImportBox.tsx index de5287456..d3f81b992 100644 --- a/src/client/util/Import & Export/DirectoryImportBox.tsx +++ b/src/client/util/Import & Export/DirectoryImportBox.tsx @@ -20,10 +20,11 @@ import { listSpec } from "../../../new_fields/Schema"; import { GooglePhotos } from "../../apis/google_docs/GooglePhotosClientUtils"; import { SchemaHeaderField } from "../../../new_fields/SchemaHeaderField"; import "./DirectoryImportBox.scss"; -import BatchedArray from "array-batcher"; import { Identified } from "../../Network"; +import { BatchedArray } from "array-batcher"; const unsupported = ["text/html", "text/plain"]; + interface FileResponse { name: string; path: string; diff --git a/src/server/DashUploadUtils.ts b/src/server/DashUploadUtils.ts index 04a3559dc..4230e9b17 100644 --- a/src/server/DashUploadUtils.ts +++ b/src/server/DashUploadUtils.ts @@ -50,10 +50,10 @@ export namespace DashUploadUtils { * event that @param filename is not specified * * @returns {UploadInformation} This method returns - * 1) the paths to the uploaded image + * 1) the paths to the uploaded images (plural due to resizing) * 2) the file name of each of the resized images * 3) the size of the image, in bytes (4432130) - * 4) the content type of the image (jpg | png | etc.) + * 4) the content type of the image, i.e. image/(jpeg | png | ...) */ export const UploadImage = async (source: string, filename?: string, prefix: string = ""): Promise => { const metadata = await InspectImage(source); diff --git a/src/server/apis/google/GooglePhotosUploadUtils.ts b/src/server/apis/google/GooglePhotosUploadUtils.ts index 507a868a3..16c4f6c3a 100644 --- a/src/server/apis/google/GooglePhotosUploadUtils.ts +++ b/src/server/apis/google/GooglePhotosUploadUtils.ts @@ -3,7 +3,7 @@ import { GoogleApiServerUtils } from './GoogleApiServerUtils'; import * as path from 'path'; import { MediaItemCreationResult } from './SharedTypes'; import { NewMediaItem } from "../../index"; -import BatchedArray, { FixedBatcher, TimeUnit, Interval } from "array-batcher"; +import { BatchedArray, TimeUnit } from 'array-batcher'; export namespace GooglePhotosUploadUtils { @@ -53,7 +53,8 @@ export namespace GooglePhotosUploadUtils { }; export const CreateMediaItems = async (newMediaItems: NewMediaItem[], album?: { id: string }): Promise => { - const newMediaItemResults = await BatchedArray.from(newMediaItems, { batchSize: 50 }).batchedMapInterval( + const newMediaItemResults = await BatchedArray.from(newMediaItems, { batchSize: 50 }).batchedMapPatientInterval( + { magnitude: 100, unit: TimeUnit.Milliseconds }, async (batch: NewMediaItem[]) => { const parameters = { method: 'POST', @@ -72,8 +73,7 @@ export namespace GooglePhotosUploadUtils { } }); })).newMediaItemResults; - }, - { magnitude: 100, unit: TimeUnit.Milliseconds } + } ); return { newMediaItemResults }; }; diff --git a/src/server/index.ts b/src/server/index.ts index 3da726cfa..9a778b88c 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -47,8 +47,8 @@ const mongoose = require('mongoose'); const probe = require("probe-image-size"); import * as qs from 'query-string'; import { Opt } from '../new_fields/Doc'; -import BatchedArray, { TimeUnit } from "array-batcher"; import { DashUploadUtils } from './DashUploadUtils'; +import { BatchedArray, TimeUnit } from 'array-batcher'; const download = (url: string, dest: fs.PathLike) => request.get(url).pipe(fs.createWriteStream(dest)); let youtubeApiKey: string; @@ -848,7 +848,8 @@ app.post(RouteStore.googlePhotosMediaUpload, async (req, res) => { let failed = 0; - const newMediaItems = await BatchedArray.from(media, { batchSize: 25 }).batchedMapInterval( + const newMediaItems = await BatchedArray.from(media, { batchSize: 25 }).batchedMapPatientInterval( + { magnitude: 100, unit: TimeUnit.Milliseconds }, async (batch: GooglePhotosUploadUtils.MediaInput[]) => { const newMediaItems: NewMediaItem[] = []; for (let element of batch) { @@ -863,8 +864,7 @@ app.post(RouteStore.googlePhotosMediaUpload, async (req, res) => { } } return newMediaItems; - }, - { magnitude: 100, unit: TimeUnit.Milliseconds } + } ); if (failed) { -- cgit v1.2.3-70-g09d2 From 0ae9a7f6acbdf6ecade8d349981e8d6badef7ff9 Mon Sep 17 00:00:00 2001 From: Bob Zeleznik Date: Sun, 29 Sep 2019 22:27:03 -0400 Subject: added pdf text searching --- package.json | 2 + src/client/util/SearchUtil.ts | 27 +++++++++++-- src/client/views/collections/CollectionSubView.tsx | 8 +++- src/client/views/search/SearchBox.tsx | 15 ++++--- src/client/views/search/SearchItem.tsx | 3 +- src/server/index.ts | 47 ++++++++++++++++++++-- 6 files changed, 86 insertions(+), 16 deletions(-) (limited to 'package.json') diff --git a/package.json b/package.json index b373ee4a5..307283214 100644 --- a/package.json +++ b/package.json @@ -133,6 +133,7 @@ "express-session": "^1.15.6", "express-validator": "^5.3.1", "expressjs": "^1.0.1", + "find-in-files": "^0.5.0", "flexlayout-react": "^0.3.3", "font-awesome": "^4.7.0", "formidable": "^1.2.1", @@ -165,6 +166,7 @@ "p-limit": "^2.2.0", "passport": "^0.4.0", "passport-local": "^1.0.0", + "pdf-parse": "^1.1.1", "pdfjs-dist": "^2.0.943", "probe-image-size": "^4.0.0", "prosemirror-commands": "^1.0.8", diff --git a/src/client/util/SearchUtil.ts b/src/client/util/SearchUtil.ts index ee5a83710..d8b9dbec6 100644 --- a/src/client/util/SearchUtil.ts +++ b/src/client/util/SearchUtil.ts @@ -3,18 +3,22 @@ import { DocServer } from '../DocServer'; import { Doc } from '../../new_fields/Doc'; import { Id } from '../../new_fields/FieldSymbols'; import { Utils } from '../../Utils'; +import { ResultParameters } from '../northstar/model/idea/idea'; +import { DocumentType } from '../documents/DocumentTypes'; export namespace SearchUtil { export type HighlightingResult = { [id: string]: { [key: string]: string[] } }; export interface IdSearchResult { ids: string[]; + lines: string[][]; numFound: number; highlighting: HighlightingResult | undefined; } export interface DocSearchResult { docs: Doc[]; + lines: string[][]; numFound: number; highlighting: HighlightingResult | undefined; } @@ -30,16 +34,31 @@ export namespace SearchUtil { export function Search(query: string, returnDocs: false, options?: SearchParams): Promise; export async function Search(query: string, returnDocs: boolean, options: SearchParams = {}) { query = query || "*"; //If we just have a filter query, search for * as the query - const result: IdSearchResult = JSON.parse(await rp.get(Utils.prepend("/search"), { + let result: IdSearchResult = JSON.parse(await rp.get(Utils.prepend("/search"), { qs: { ...options, q: query }, })); if (!returnDocs) { return result; } - const { ids, numFound, highlighting } = result; + + let { ids, numFound, highlighting } = result; + let lines: string[][] = ids.map(i => []); + + let txtresult = query !== "*" && JSON.parse(await rp.get(Utils.prepend("/textsearch"), { + qs: { ...options, q: query }, + })); + let fileids = txtresult ? txtresult.ids : []; + await Promise.all(fileids.map(async (tr: string, i: number) => { + let docQuery = "fileUpload_t:" + tr.substr(0, 7); //If we just have a filter query, search for * as the query + let docResult = JSON.parse(await rp.get(Utils.prepend("/search"), { qs: { ...options, q: docQuery } })); + ids.push(...docResult.ids); + lines.push(...docResult.ids.map((dr: any) => txtresult.lines[i])); + numFound += docResult.numFound; + })); + const docMap = await DocServer.GetRefFields(ids); - const docs = ids.map((id: string) => docMap[id]).filter((doc: any) => doc instanceof Doc); - return { docs, numFound, highlighting }; + const docs = ids.map((id: string) => docMap[id]).filter((doc: any) => doc instanceof Doc && doc.type !== DocumentType.KVP); + return { docs, numFound, highlighting, lines }; } export async function GetAliasesOfDocument(doc: Doc): Promise; diff --git a/src/client/views/collections/CollectionSubView.tsx b/src/client/views/collections/CollectionSubView.tsx index 954a27cbd..cdeba6e16 100644 --- a/src/client/views/collections/CollectionSubView.tsx +++ b/src/client/views/collections/CollectionSubView.tsx @@ -22,6 +22,7 @@ import { CollectionPDFView } from "./CollectionPDFView"; import { CollectionVideoView } from "./CollectionVideoView"; import { CollectionView } from "./CollectionView"; import React = require("react"); +var path = require('path'); export interface CollectionViewProps extends FieldViewProps { addDocument: (document: Doc, allowDuplicates?: boolean) => boolean; @@ -261,8 +262,11 @@ export function CollectionSubView(schemaCtor: (doc: Doc) => T) { }).then(async (res: Response) => { (await res.json()).map(action((file: any) => { let full = { ...options, nativeWidth: type.indexOf("video") !== -1 ? 600 : 300, width: 300, title: dropFileName }; - let path = Utils.prepend(file); - Docs.Get.DocumentFromType(type, path, full).then(doc => doc && this.props.addDocument(doc)); + let pathname = Utils.prepend(file); + Docs.Get.DocumentFromType(type, pathname, full).then(doc => { + doc && (doc.fileUpload = path.basename(pathname).replace("upload_", "").replace(/\.[a-z0-9]*$/, "")); + doc && this.props.addDocument(doc); + }); })); }); promises.push(prom); diff --git a/src/client/views/search/SearchBox.tsx b/src/client/views/search/SearchBox.tsx index 0d50124dd..f53270c64 100644 --- a/src/client/views/search/SearchBox.tsx +++ b/src/client/views/search/SearchBox.tsx @@ -18,6 +18,7 @@ import { FilterBox } from './FilterBox'; import "./FilterBox.scss"; import "./SearchBox.scss"; import { SearchItem } from './SearchItem'; +import { string } from 'prop-types'; library.add(faTimes); @@ -27,7 +28,7 @@ export class SearchBox extends React.Component { @observable private _searchString: string = ""; @observable private _resultsOpen: boolean = false; @observable private _searchbarOpen: boolean = false; - @observable private _results: [Doc, string[]][] = []; + @observable private _results: [Doc, string[], string[]][] = []; private _resultsSet = new Map(); @observable private _openNoResults: boolean = false; @observable private _visibleElements: JSX.Element[] = []; @@ -159,6 +160,8 @@ export class SearchBox extends React.Component { const highlighting = res.highlighting || {}; const highlightList = res.docs.map(doc => highlighting[doc[Id]]); + const lines = new Map(); + res.docs.map((doc, i) => lines.set(doc[Id], res.lines[i])); const docs = await Promise.all(res.docs.map(async doc => (await Cast(doc.extendsDoc, Doc)) || doc)); const highlights: typeof res.highlighting = {}; docs.forEach((doc, index) => highlights[doc[Id]] = highlightList[index]); @@ -168,12 +171,14 @@ export class SearchBox extends React.Component { filteredDocs.forEach(doc => { const index = this._resultsSet.get(doc); const highlight = highlights[doc[Id]]; + const line = lines.get(doc[Id]) || []; const hlights = highlight ? Object.keys(highlight).map(key => key.substring(0, key.length - 2)) : []; if (index === undefined) { this._resultsSet.set(doc, this._results.length); - this._results.push([doc, hlights]); + this._results.push([doc, hlights, line]); } else { this._results[index][1].push(...hlights); + this._results[index][2].push(...line); } }); }); @@ -296,13 +301,13 @@ export class SearchBox extends React.Component { } else { if (this._isSearch[i] !== "search") { - let result: [Doc, string[]] | undefined = undefined; + let result: [Doc, string[], string[]] | undefined = undefined; if (i >= this._results.length) { this.getResults(this._searchString); if (i < this._results.length) result = this._results[i]; if (result) { let highlights = Array.from([...Array.from(new Set(result[1]).values())]).filter(v => v !== "search_string"); - this._visibleElements[i] = ; + this._visibleElements[i] = ; this._isSearch[i] = "search"; } } @@ -310,7 +315,7 @@ export class SearchBox extends React.Component { result = this._results[i]; if (result) { let highlights = Array.from([...Array.from(new Set(result[1]).values())]).filter(v => v !== "search_string"); - this._visibleElements[i] = ; + this._visibleElements[i] = ; this._isSearch[i] = "search"; } } diff --git a/src/client/views/search/SearchItem.tsx b/src/client/views/search/SearchItem.tsx index 96eefacc2..4d021216d 100644 --- a/src/client/views/search/SearchItem.tsx +++ b/src/client/views/search/SearchItem.tsx @@ -30,6 +30,7 @@ export interface SearchItemProps { doc: Doc; query: string; highlighting: string[]; + lines: string[]; } library.add(faCaretUp); @@ -288,7 +289,7 @@ export class SearchItem extends React.Component {
{StrCast(this.props.doc.title)}
-
Matched fields: {this.props.highlighting.join(", ")}
+
{this.props.highlighting.length ? "Matched fields:" + this.props.highlighting.join(", ") : this.props.lines.length ? "Text:" + this.props.lines[0] : ""}
diff --git a/src/server/index.ts b/src/server/index.ts index 50ce2b14e..524407a83 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -42,13 +42,11 @@ var AdmZip = require('adm-zip'); import * as YoutubeApi from "./apis/youtube/youtubeApiSample"; import { Response } from 'express-serve-static-core'; import { GoogleApiServerUtils } from "./apis/google/GoogleApiServerUtils"; -import { GaxiosResponse } from 'gaxios'; -import { Opt } from '../new_fields/Doc'; -import { docs_v1 } from 'googleapis'; -import { Endpoint } from 'googleapis-common'; const MongoStore = require('connect-mongo')(session); const mongoose = require('mongoose'); const probe = require("probe-image-size"); +const pdf = require('pdf-parse'); +var findInFiles = require('find-in-files'); const download = (url: string, dest: fs.PathLike) => request.get(url).pipe(fs.createWriteStream(dest)); let youtubeApiKey: string; @@ -196,6 +194,23 @@ const solrURL = "http://localhost:8983/solr/#/dash"; // GETTERS +app.get("/textsearch", async (req, res) => { + let q = req.query.q; + console.log("TEXTSEARCH " + q); + if (q === undefined) { + res.send([]); + return; + } + let results = await findInFiles.find({ 'term': q, 'flags': 'ig' }, uploadDir + "text", ".txt$"); + let resObj: { ids: string[], numFound: number, lines: string[] } = { ids: [], numFound: 0, lines: [] }; + for (var result in results) { + resObj.ids.push(path.basename(result, ".txt").replace(/upload_/, "")); + resObj.lines.push(results[result].line); + resObj.numFound++; + } + res.send(resObj); +}); + app.get("/search", async (req, res) => { const solrQuery: any = {}; ["q", "fq", "start", "rows", "hl", "hl.fl"].forEach(key => solrQuery[key] = req.query[key]); @@ -597,6 +612,30 @@ app.post( fs.createReadStream(uploadDir + file).pipe(resizer.resizer).pipe(fs.createWriteStream(uploadDir + file.substring(0, file.length - ext.length) + resizer.suffix + ext)); }); } + if (ext.endsWith("pdf")) { + var filePath = uploadDir + file; + + let dataBuffer = fs.readFileSync(filePath); + + pdf(dataBuffer).then(async function (data: any) { + + // number of pages + // console.log(data.numpages); + // // number of rendered pages + // console.log(data.numrender); + // // PDF info + // console.log(data.info); + // // PDF metadata + // console.log(data.metadata); + // // PDF.js version + // // check https://mozilla.github.io/pdf.js/getting_started/ + // console.log(data.version); + // // PDF text + // console.log(data.text); + fs.createWriteStream(uploadDir + "text/" + file.substring(0, file.length - ext.length) + ".txt").write(data.text); + }); + + } names.push(`/files/` + file); } res.send(names); -- cgit v1.2.3-70-g09d2 From faaff9fe8aab3bd5d116eab8bd85198a0756fe30 Mon Sep 17 00:00:00 2001 From: bob Date: Tue, 1 Oct 2019 10:12:50 -0400 Subject: fixed issues with pdf loading and layout. --- package.json | 2 +- .../views/collections/CollectionSchemaView.tsx | 16 +++++------ src/client/views/nodes/PDFBox.scss | 31 ++++++++++++++++++++++ src/client/views/nodes/PDFBox.tsx | 5 ++-- src/client/views/pdf/PDFViewer.tsx | 2 +- src/server/index.ts | 8 +++++- 6 files changed, 49 insertions(+), 15 deletions(-) (limited to 'package.json') diff --git a/package.json b/package.json index fa722a6ae..e516a6979 100644 --- a/package.json +++ b/package.json @@ -167,7 +167,7 @@ "nodemailer": "^5.1.1", "nodemon": "^1.18.10", "normalize.css": "^8.0.1", - "npm": "^6.10.3", + "npm": "^6.11.3", "p-limit": "^2.2.0", "passport": "^0.4.0", "passport-google-oauth20": "^2.0.0", diff --git a/src/client/views/collections/CollectionSchemaView.tsx b/src/client/views/collections/CollectionSchemaView.tsx index 59cce7386..853808335 100644 --- a/src/client/views/collections/CollectionSchemaView.tsx +++ b/src/client/views/collections/CollectionSchemaView.tsx @@ -303,21 +303,17 @@ export class SchemaTable extends React.Component { this.props.Document.textwrappedSchemaRows = new List(textWrappedRows); } - @computed get resized(): { "id": string, "value": number }[] { + @computed get resized(): { id: string, value: number }[] { return this.columns.reduce((resized, shf) => { - if (shf.width > -1) { - resized.push({ "id": shf.heading, "value": shf.width }); - } + (shf.width > -1) && resized.push({ id: shf.heading, value: shf.width }); return resized; - }, [] as { "id": string, "value": number }[]); + }, [] as { id: string, value: number }[]); } - @computed get sorted(): { id: string, desc: boolean }[] { + @computed get sorted(): { id: string, desc?: true }[] { return this.columns.reduce((sorted, shf) => { - if (shf.desc) { - sorted.push({ "id": shf.heading, "desc": shf.desc }); - } + shf.desc && sorted.push({ id: shf.heading, desc: shf.desc }); return sorted; - }, [] as { id: string, desc: boolean }[]); + }, [] as { id: string, desc?: true }[]); } @computed get borderWidth() { return Number(COLLECTION_BORDER_WIDTH); } diff --git a/src/client/views/nodes/PDFBox.scss b/src/client/views/nodes/PDFBox.scss index 2917c81cb..3f5f47e05 100644 --- a/src/client/views/nodes/PDFBox.scss +++ b/src/client/views/nodes/PDFBox.scss @@ -9,6 +9,37 @@ z-index: -1; } +.pdfBox-title-outer { + + position: absolute; + width: 100%; + height: 100%; + background: lightslategray; + .pdfBox-title-cont { + width: 150%; + height: 100%; + position: relative; + display: grid; + + .pdfBox-title { + color:lightgray; + margin-top: auto; + margin-bottom: auto; + transform-origin: 42% -18%; + width: 100%; + transform: rotate(55deg); + font-size: 144; + padding: 5%; + overflow: hidden; + display: inline-block; + white-space: pre; + text-overflow: ellipsis; + text-align: center; + } + } +} + + .pdfBox-cont { pointer-events: none; .collectionFreeFormView-none { diff --git a/src/client/views/nodes/PDFBox.tsx b/src/client/views/nodes/PDFBox.tsx index fe71e76fd..3014af944 100644 --- a/src/client/views/nodes/PDFBox.tsx +++ b/src/client/views/nodes/PDFBox.tsx @@ -180,8 +180,9 @@ export class PDFBox extends DocComponent(PdfDocumen render() { const pdfUrl = Cast(this.dataDoc[this.props.fieldKey], PdfField); let classname = "pdfBox-cont" + (InkingControl.Instance.selectedTool || !this.active ? "" : "-interactive"); - return (!(pdfUrl instanceof PdfField) || !this._pdf ? -
{`pdf, ${this.dataDoc[this.props.fieldKey]}, not found`}
: + return (!(pdfUrl instanceof PdfField) || !this._pdf || this.props.ScreenToLocalTransform().Scale > 6 ? +
+ {` ${this.props.Document.title}`}
:
{ let hit = document.elementFromPoint(e.clientX, e.clientY); if (hit && hit.localName === "span" && this.props.isSelected()) { // drag selecting text stops propagation diff --git a/src/client/views/pdf/PDFViewer.tsx b/src/client/views/pdf/PDFViewer.tsx index 13fd8ea98..3bf53340b 100644 --- a/src/client/views/pdf/PDFViewer.tsx +++ b/src/client/views/pdf/PDFViewer.tsx @@ -628,7 +628,7 @@ export class PDFViewer extends React.Component { } let nativeWidth = NumCast(this.props.Document.nativeWidth); let nativeHeight = NumCast(this.props.Document.nativeHeight); - return this._showWaiting = false)} + return this._coverPath.path = "http://www.cs.brown.edu/~bcz/face.gif")} onLoad={action(() => this._showWaiting = false)} style={{ position: "absolute", display: "inline-block", top: 0, left: 0, width: `${nativeWidth}px`, height: `${nativeHeight}px` }} />; } diff --git a/src/server/index.ts b/src/server/index.ts index a265853ff..690836fff 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -613,7 +613,13 @@ app.post( const result: ParsedPDF = await pdf(dataBuffer); await new Promise(resolve => { const path = pdfDirectory + "/" + filename.substring(0, filename.length - ".pdf".length) + ".txt"; - fs.createWriteStream(path).write(result.text, error => error === null ? resolve() : reject(error)); + fs.createWriteStream(path).write(result.text, error => { + if (!error) { + resolve(); + } else { + reject(error); + } + }); }); } else { await DashUploadUtils.UploadImage(uploadDirectory + filename, filename).catch(() => console.log(`Unable to process ${filename}`)); -- cgit v1.2.3-70-g09d2 From 0538db262ff611c5273c363470886d2aa42cf760 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Tue, 1 Oct 2019 20:31:27 -0400 Subject: initial commit --- package.json | 2 ++ src/client/documents/Documents.ts | 2 +- .../util/Import & Export/DirectoryImportBox.tsx | 12 +++++-- src/client/util/Import & Export/ImageUtils.ts | 22 +++++++++++++ src/client/views/collections/CollectionSubView.tsx | 2 ++ src/server/DashUploadUtils.ts | 37 +++++++++++++++++++--- src/server/RouteStore.ts | 1 + src/server/index.ts | 21 +++++++++--- 8 files changed, 87 insertions(+), 12 deletions(-) create mode 100644 src/client/util/Import & Export/ImageUtils.ts (limited to 'package.json') diff --git a/package.json b/package.json index e516a6979..4ac081374 100644 --- a/package.json +++ b/package.json @@ -65,6 +65,7 @@ "@types/cookie-session": "^2.0.36", "@types/d3-format": "^1.3.1", "@types/dotenv": "^6.1.1", + "@types/exif": "^0.6.0", "@types/express": "^4.16.1", "@types/express-flash": "0.0.0", "@types/express-session": "^1.15.12", @@ -132,6 +133,7 @@ "crypto-browserify": "^3.11.0", "d3-format": "^1.3.2", "dotenv": "^8.0.0", + "exif": "^0.6.0", "express": "^4.16.4", "express-flash": "0.0.2", "express-session": "^1.15.6", diff --git a/src/client/documents/Documents.ts b/src/client/documents/Documents.ts index 0d04d044e..eed8d61c2 100644 --- a/src/client/documents/Documents.ts +++ b/src/client/documents/Documents.ts @@ -254,7 +254,7 @@ export namespace Docs { let title = prototypeId.toUpperCase().replace(upper, `_${upper}`); // synthesize the default options, the type and title from computed values and // whatever options pertain to this specific prototype - let options = { title: title, type: type, baseProto: true, ...defaultOptions, ...(template.options || {}) }; + let options = { title, type, baseProto: true, ...defaultOptions, ...(template.options || {}) }; let primary = layout.view.LayoutString(layout.ext); let collectionView = layout.collectionView; if (collectionView) { diff --git a/src/client/util/Import & Export/DirectoryImportBox.tsx b/src/client/util/Import & Export/DirectoryImportBox.tsx index d3f81b992..d74b51993 100644 --- a/src/client/util/Import & Export/DirectoryImportBox.tsx +++ b/src/client/util/Import & Export/DirectoryImportBox.tsx @@ -22,13 +22,15 @@ import { SchemaHeaderField } from "../../../new_fields/SchemaHeaderField"; import "./DirectoryImportBox.scss"; import { Identified } from "../../Network"; import { BatchedArray } from "array-batcher"; +import { ExifData } from "exif"; const unsupported = ["text/html", "text/plain"]; -interface FileResponse { +interface ImageUploadResponse { name: string; path: string; type: string; + exif: any; } @observer @@ -117,7 +119,7 @@ export default class DirectoryImportBox extends React.Component const responses = await Identified.PostFormDataToServer(RouteStore.upload, formData); runInAction(() => this.completed += batch.length); - return responses as FileResponse[]; + return responses as ImageUploadResponse[]; }); await Promise.all(uploads.map(async upload => { @@ -129,7 +131,11 @@ export default class DirectoryImportBox extends React.Component title: upload.name }; const document = await Docs.Get.DocumentFromType(type, path, options); - document && docs.push(document); + const { data, error } = upload.exif; + if (document) { + Doc.GetProto(document).exif = error || Docs.Get.DocumentHierarchyFromJson(data); + docs.push(document); + } })); for (let i = 0; i < docs.length; i++) { diff --git a/src/client/util/Import & Export/ImageUtils.ts b/src/client/util/Import & Export/ImageUtils.ts new file mode 100644 index 000000000..33ca55aa9 --- /dev/null +++ b/src/client/util/Import & Export/ImageUtils.ts @@ -0,0 +1,22 @@ +import { Doc } from "../../../new_fields/Doc"; +import { ImageField } from "../../../new_fields/URLField"; +import { Cast } from "../../../new_fields/Types"; +import { RouteStore } from "../../../server/RouteStore"; +import { Docs } from "../../documents/Documents"; +import { Identified } from "../../Network"; + +export namespace ImageUtils { + + export const ExtractExif = async (document: Doc): Promise => { + const field = Cast(document.data, ImageField); + if (!field) { + return false; + } + const source = field.url.href; + const response = await Identified.PostToServer(RouteStore.inspectImage, { source }); + const { error, data } = response.exifData; + document.exif = error || Docs.Get.DocumentHierarchyFromJson(data); + return data !== undefined; + }; + +} \ No newline at end of file diff --git a/src/client/views/collections/CollectionSubView.tsx b/src/client/views/collections/CollectionSubView.tsx index 155f2718b..7ba9fb5ed 100644 --- a/src/client/views/collections/CollectionSubView.tsx +++ b/src/client/views/collections/CollectionSubView.tsx @@ -24,6 +24,7 @@ import { CollectionView } from "./CollectionView"; import React = require("react"); var path = require('path'); import { GooglePhotos } from "../../apis/google_docs/GooglePhotosClientUtils"; +import { ImageUtils } from "../../util/Import & Export/ImageUtils"; export interface CollectionViewProps extends FieldViewProps { addDocument: (document: Doc, allowDuplicates?: boolean) => boolean; @@ -193,6 +194,7 @@ export function CollectionSubView(schemaCtor: (doc: Doc) => T) { if (img) { let split = img.split("src=\"")[1].split("\"")[0]; let doc = Docs.Create.ImageDocument(split, { ...options, width: 300 }); + ImageUtils.ExtractExif(doc); this.props.addDocument(doc, false); return; } else { diff --git a/src/server/DashUploadUtils.ts b/src/server/DashUploadUtils.ts index 4230e9b17..619cba3c6 100644 --- a/src/server/DashUploadUtils.ts +++ b/src/server/DashUploadUtils.ts @@ -3,6 +3,8 @@ import { Utils } from '../Utils'; import * as path from 'path'; import * as sharp from 'sharp'; import request = require('request-promise'); +import { ExifData, ExifImage } from 'exif'; +import { Opt } from '../new_fields/Doc'; const uploadDirectory = path.join(__dirname, './public/files/'); @@ -31,12 +33,19 @@ export namespace DashUploadUtils { export interface UploadInformation { mediaPaths: string[]; fileNames: { [key: string]: string }; + exifData: EnrichedExifData; contentSize?: number; contentType?: string; } - const generate = (prefix: string, url: string) => `${prefix}upload_${Utils.GenerateGuid()}${path.extname(url).toLowerCase()}`; + const generate = (prefix: string, url: string) => `${prefix}upload_${Utils.GenerateGuid()}${sanitizeExtension(url)}`; const sanitize = (filename: string) => filename.replace(/\s+/g, "_"); + const sanitizeExtension = (source: string) => { + let extension = path.extname(source); + extension = extension.toLowerCase(); + extension = extension.split("?")[0]; + return extension; + }; /** * Uploads an image specified by the @param source to Dash's /public/files/ @@ -64,10 +73,16 @@ export namespace DashUploadUtils { isLocal: boolean; stream: any; normalizedUrl: string; + exifData: EnrichedExifData; contentSize?: number; contentType?: string; } + export interface EnrichedExifData { + data: ExifData; + error?: string; + } + /** * Based on the url's classification as local or remote, gleans * as much information as possible about the specified image @@ -76,7 +91,9 @@ export namespace DashUploadUtils { */ export const InspectImage = async (source: string): Promise => { const { isLocal, stream, normalized: normalizedUrl } = classify(source); + const exifData = await parseExifData(source); const results = { + exifData, isLocal, stream, normalizedUrl @@ -101,13 +118,13 @@ export namespace DashUploadUtils { }; export const UploadInspectedImage = async (metadata: InspectionResults, filename?: string, prefix = ""): Promise => { - const { isLocal, stream, normalizedUrl, contentSize, contentType } = metadata; + const { isLocal, stream, normalizedUrl, contentSize, contentType, exifData } = metadata; const resolved = filename ? sanitize(filename) : generate(prefix, normalizedUrl); - let extension = path.extname(normalizedUrl) || path.extname(resolved); - extension && (extension = extension.toLowerCase()); + const extension = sanitizeExtension(normalizedUrl || resolved); let information: UploadInformation = { mediaPaths: [], fileNames: { clean: resolved }, + exifData, contentSize, contentType, }; @@ -159,6 +176,18 @@ export namespace DashUploadUtils { }; }; + const parseExifData = async (source: string): Promise => { + return new Promise(resolve => { + new ExifImage(source, (error, data) => { + let reason: Opt = undefined; + if (error) { + reason = (error as any).code; + } + resolve({ data, error: reason }); + }); + }); + }; + export const createIfNotExists = async (path: string) => { if (await new Promise(resolve => fs.exists(path, resolve))) { return true; diff --git a/src/server/RouteStore.ts b/src/server/RouteStore.ts index ee9cd8a0e..1e1dd6300 100644 --- a/src/server/RouteStore.ts +++ b/src/server/RouteStore.ts @@ -13,6 +13,7 @@ export enum RouteStore { upload = "/upload", dataUriToImage = "/uploadURI", images = "/images", + inspectImage = "/inspectImage", // USER AND WORKSPACES getCurrUser = "/getCurrentUser", diff --git a/src/server/index.ts b/src/server/index.ts index 690836fff..1ca5d6e11 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -53,6 +53,7 @@ import { DashUploadUtils } from './DashUploadUtils'; import { BatchedArray, TimeUnit } from 'array-batcher'; import { ParsedPDF } from "./PdfTypes"; import { reject } from 'bluebird'; +import { ExifData } from 'exif'; const download = (url: string, dest: fs.PathLike) => request.get(url).pipe(fs.createWriteStream(dest)); let youtubeApiKey: string; @@ -590,10 +591,11 @@ const uploadDirectory = __dirname + "/public/files/"; const pdfDirectory = uploadDirectory + "text"; DashUploadUtils.createIfNotExists(pdfDirectory); -interface FileResponse { +interface ImageFileResponse { name: string; path: string; type: string; + exif: Opt; } // SETTERS @@ -604,10 +606,11 @@ app.post( form.uploadDir = uploadDirectory; form.keepExtensions = true; form.parse(req, async (_err, _fields, files) => { - let results: FileResponse[] = []; + let results: ImageFileResponse[] = []; for (const key in files) { const { type, path: location, name } = files[key]; const filename = path.basename(location); + let uploadInformation: Opt; if (filename.endsWith(".pdf")) { let dataBuffer = fs.readFileSync(uploadDirectory + filename); const result: ParsedPDF = await pdf(dataBuffer); @@ -622,9 +625,10 @@ app.post( }); }); } else { - await DashUploadUtils.UploadImage(uploadDirectory + filename, filename).catch(() => console.log(`Unable to process ${filename}`)); + uploadInformation = await DashUploadUtils.UploadImage(uploadDirectory + filename, filename); } - results.push({ name, type, path: `/files/${filename}` }); + const exif = uploadInformation ? uploadInformation.exifData : undefined; + results.push({ name, type, path: `/files/${filename}`, exif }); } _success(res, results); @@ -632,6 +636,15 @@ app.post( } ); +app.post(RouteStore.inspectImage, async (req, res) => { + const { source } = req.body; + if (typeof source === "string") { + const uploadInformation = await DashUploadUtils.UploadImage(source); + return res.send(await DashUploadUtils.InspectImage(uploadInformation.mediaPaths[0])); + } + res.send({}); +}); + addSecureRoute( Method.POST, (user, res, req) => { -- cgit v1.2.3-70-g09d2 From 128b8f61a8e3c7557bf7c3424a76f9c61ad6a56b Mon Sep 17 00:00:00 2001 From: Bob Zeleznik Date: Tue, 1 Oct 2019 21:13:02 -0400 Subject: fixed issues with isMinimized and with iconBoxs being too big. --- package.json | 2 +- src/client/views/collections/CollectionSchemaView.tsx | 4 ++-- .../collectionFreeForm/CollectionFreeFormView.tsx | 2 +- src/client/views/nodes/DocumentView.tsx | 10 +++++++--- src/client/views/nodes/PDFBox.scss | 2 +- src/client/views/nodes/PDFBox.tsx | 12 ++++++++---- src/new_fields/Doc.ts | 10 ++-------- 7 files changed, 22 insertions(+), 20 deletions(-) (limited to 'package.json') diff --git a/package.json b/package.json index e516a6979..9d12f51b0 100644 --- a/package.json +++ b/package.json @@ -205,7 +205,7 @@ "react-mosaic": "0.0.20", "react-simple-dropdown": "^3.2.3", "react-split-pane": "^0.1.85", - "react-table": "^6.9.2", + "react-table": "^6.10.3", "readline": "^1.3.0", "request": "^2.88.0", "request-promise": "^4.2.4", diff --git a/src/client/views/collections/CollectionSchemaView.tsx b/src/client/views/collections/CollectionSchemaView.tsx index 24f585a07..52a41fc84 100644 --- a/src/client/views/collections/CollectionSchemaView.tsx +++ b/src/client/views/collections/CollectionSchemaView.tsx @@ -309,11 +309,11 @@ export class SchemaTable extends React.Component { return resized; }, [] as { id: string, value: number }[]); } - @computed get sorted(): { id: string, desc?: true }[] { + @computed get sorted(): { id: string, desc: boolean }[] { return this.columns.reduce((sorted, shf) => { shf.desc && sorted.push({ id: shf.heading, desc: shf.desc }); return sorted; - }, [] as { id: string, desc?: true }[]); + }, [] as { id: string, desc: boolean }[]); } @computed get borderWidth() { return Number(COLLECTION_BORDER_WIDTH); } diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx index 27b9a06f5..98730fe13 100644 --- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx +++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx @@ -105,7 +105,7 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) { SelectionManager.DeselectAll(); docs.map(doc => DocumentManager.Instance.getDocumentView(doc)).map(dv => dv && SelectionManager.SelectDoc(dv, true)); } - public isCurrent(doc: Doc) { return !this.props.Document.isMinimized && (Math.abs(NumCast(doc.page, -1) - NumCast(this.Document.curPage, -1)) < 1.5 || NumCast(doc.page, -1) === -1); } + public isCurrent(doc: Doc) { return !doc.isMinimized && (Math.abs(NumCast(doc.page, -1) - NumCast(this.Document.curPage, -1)) < 1.5 || NumCast(doc.page, -1) === -1); } public getActiveDocuments = () => { return this.childLayoutPairs.filter(pair => this.isCurrent(pair.layout)).map(pair => pair.layout); diff --git a/src/client/views/nodes/DocumentView.tsx b/src/client/views/nodes/DocumentView.tsx index ded50890d..a903f8fe2 100644 --- a/src/client/views/nodes/DocumentView.tsx +++ b/src/client/views/nodes/DocumentView.tsx @@ -635,6 +635,7 @@ export class DocumentView extends DocComponent(Docu DataDoc={this.props.DataDoc} />); } render() { + let animDims = this.props.Document.animateToDimensions ? Array.from(Cast(this.props.Document.animateToDimensions, listSpec("number"))!) : undefined; const ruleColor = this.props.ruleProvider ? StrCast(this.props.ruleProvider["ruleColor_" + this.Document.heading]) : undefined; const ruleRounding = this.props.ruleProvider ? StrCast(this.props.ruleProvider["ruleRounding_" + this.Document.heading]) : undefined; const colorSet = this.setsLayoutProp("backgroundColor"); @@ -644,7 +645,7 @@ export class DocumentView extends DocComponent(Docu ruleColor && !colorSet ? ruleColor : StrCast(this.layoutDoc.backgroundColor) || this.props.backgroundColor(this.Document); const nativeWidth = this.props.Document.fitWidth ? this.props.PanelWidth() : this.nativeWidth > 0 && !this.Document.ignoreAspect ? `${this.nativeWidth}px` : "100%"; - const nativeHeight = this.props.Document.fitWidth ? this.props.PanelHeight() : this.Document.ignoreAspect || this.props.Document.fitWidth ? this.props.PanelHeight() / this.props.ContentScaling() : this.nativeHeight > 0 ? `${this.nativeHeight}px` : nativeWidth !== "100%" ? nativeWidth : "100%"; + const nativeHeight = this.props.Document.fitWidth ? this.props.PanelHeight() : this.Document.ignoreAspect ? this.props.PanelHeight() / this.props.ContentScaling() : this.nativeHeight > 0 ? `${this.nativeHeight}px` : "100%"; const showOverlays = this.props.showOverlays ? this.props.showOverlays(this.Document) : undefined; const showTitle = showOverlays && "title" in showOverlays ? showOverlays.title : this.getLayoutPropStr("showTitle"); const showCaption = showOverlays && "caption" in showOverlays ? showOverlays.caption : this.getLayoutPropStr("showCaption"); @@ -678,6 +679,9 @@ export class DocumentView extends DocComponent(Docu SetValue={(value: string) => (Doc.GetProto(this.Document)[showTitle] = value) ? true : true} />
); + let animheight = animDims ? animDims[1] : nativeHeight; + let animwidth = animDims ? animDims[0] : nativeWidth; + return (
(Docu outlineWidth: fullDegree && !borderRounding ? `${localScale}px` : "0px", border: fullDegree && borderRounding ? `${["none", "dashed", "solid", "solid"][fullDegree]} ${["transparent", "maroon", "maroon", "yellow"][fullDegree]} ${localScale}px` : undefined, background: backgroundColor, - width: nativeWidth, - height: nativeHeight, + width: animwidth, + height: animheight, transform: `scale(${this.props.Document.fitWidth ? 1 : this.props.ContentScaling()})`, opacity: this.Document.opacity }} diff --git a/src/client/views/nodes/PDFBox.scss b/src/client/views/nodes/PDFBox.scss index 3f5f47e05..6393f21f7 100644 --- a/src/client/views/nodes/PDFBox.scss +++ b/src/client/views/nodes/PDFBox.scss @@ -15,7 +15,7 @@ width: 100%; height: 100%; background: lightslategray; - .pdfBox-title-cont { + .pdfBox-title-cont, .pdfBox-cont-interactive{ width: 150%; height: 100%; position: relative; diff --git a/src/client/views/nodes/PDFBox.tsx b/src/client/views/nodes/PDFBox.tsx index 3ad0b4010..a3b375be7 100644 --- a/src/client/views/nodes/PDFBox.tsx +++ b/src/client/views/nodes/PDFBox.tsx @@ -183,11 +183,15 @@ export class PDFBox extends DocComponent(PdfDocumen let noPdf = !(pdfUrl instanceof PdfField) || !this._pdf; if (!noPdf && (this.props.isSelected() || this.props.ScreenToLocalTransform().Scale < 2.5)) this._everActive = true; return (!this._everActive ? -
- {` ${this.props.Document.title}`}
: +
+
+ {` ${this.props.Document.title}`} +
+
:
{ let hit = document.elementFromPoint(e.clientX, e.clientY); diff --git a/src/new_fields/Doc.ts b/src/new_fields/Doc.ts index a68692732..aeffc81c4 100644 --- a/src/new_fields/Doc.ts +++ b/src/new_fields/Doc.ts @@ -144,14 +144,8 @@ export class Doc extends RefField { private [Self] = this; private [SelfProxy]: any; - public [WidthSym] = () => { - let animDims = this[SelfProxy].animateToDimensions ? Array.from(Cast(this[SelfProxy].animateToDimensions, listSpec("number"))!) : undefined; - return animDims ? animDims[0] : NumCast(this[SelfProxy].width); - } - public [HeightSym] = () => { - let animDims = this[SelfProxy].animateToDimensions ? Array.from(Cast(this[SelfProxy].animateToDimensions, listSpec("number"))!) : undefined; - return animDims ? animDims[1] : NumCast(this[SelfProxy].height); - } + public [WidthSym] = () => NumCast(this[SelfProxy].width); + public [HeightSym] = () => NumCast(this[SelfProxy].height); [ToScriptString]() { return "invalid"; -- cgit v1.2.3-70-g09d2 From 2cec74403daf057d6e2e830a0544c1254722dcde Mon Sep 17 00:00:00 2001 From: bob Date: Thu, 10 Oct 2019 10:04:46 -0400 Subject: fixed scrolltoannotation bug in pdfviewer. removed CollectionPdfView --- package.json | 2 +- src/client/util/DocumentManager.ts | 5 ++- src/client/util/SharingManager.tsx | 3 +- .../views/collections/CollectionPDFView.scss | 11 ------- src/client/views/collections/CollectionPDFView.tsx | 37 ---------------------- .../views/collections/CollectionSchemaCells.tsx | 5 ++- .../views/collections/CollectionSchemaView.tsx | 7 ++-- src/client/views/collections/CollectionSubView.tsx | 3 +- src/client/views/nodes/DocumentContentsView.tsx | 3 +- src/client/views/nodes/DocumentView.tsx | 3 +- src/client/views/nodes/FieldView.tsx | 3 +- src/client/views/pdf/PDFViewer.tsx | 11 ++++--- 12 files changed, 19 insertions(+), 74 deletions(-) delete mode 100644 src/client/views/collections/CollectionPDFView.scss delete mode 100644 src/client/views/collections/CollectionPDFView.tsx (limited to 'package.json') diff --git a/package.json b/package.json index 6e96f94d2..8cbbb84af 100644 --- a/package.json +++ b/package.json @@ -169,7 +169,7 @@ "nodemailer": "^5.1.1", "nodemon": "^1.18.10", "normalize.css": "^8.0.1", - "npm": "^6.11.3", + "npm": "^6.12.0", "p-limit": "^2.2.0", "passport": "^0.4.0", "passport-google-oauth20": "^2.0.0", diff --git a/src/client/util/DocumentManager.ts b/src/client/util/DocumentManager.ts index c45d3a75f..c95d923cb 100644 --- a/src/client/util/DocumentManager.ts +++ b/src/client/util/DocumentManager.ts @@ -3,7 +3,6 @@ import { Doc, DocListCastAsync } from '../../new_fields/Doc'; import { Id } from '../../new_fields/FieldSymbols'; import { Cast, NumCast, StrCast } from '../../new_fields/Types'; import { CollectionDockingView } from '../views/collections/CollectionDockingView'; -import { CollectionPDFView } from '../views/collections/CollectionPDFView'; import { CollectionVideoView } from '../views/collections/CollectionVideoView'; import { CollectionView } from '../views/collections/CollectionView'; import { DocumentView } from '../views/nodes/DocumentView'; @@ -57,7 +56,7 @@ export class DocumentManager { return this.getDocumentViewsById(doc[Id]); } - public getDocumentViewById(id: string, preferredCollection?: CollectionView | CollectionPDFView | CollectionVideoView): DocumentView | undefined { + public getDocumentViewById(id: string, preferredCollection?: CollectionView | CollectionVideoView): DocumentView | undefined { let toReturn: DocumentView | undefined; let passes = preferredCollection ? [preferredCollection, undefined] : [undefined]; @@ -82,7 +81,7 @@ export class DocumentManager { return toReturn; } - public getDocumentView(toFind: Doc, preferredCollection?: CollectionView | CollectionPDFView | CollectionVideoView): DocumentView | undefined { + public getDocumentView(toFind: Doc, preferredCollection?: CollectionView | CollectionVideoView): DocumentView | undefined { return this.getDocumentViewById(toFind[Id], preferredCollection); } diff --git a/src/client/util/SharingManager.tsx b/src/client/util/SharingManager.tsx index 1541cd6b2..c989b6c17 100644 --- a/src/client/util/SharingManager.tsx +++ b/src/client/util/SharingManager.tsx @@ -18,7 +18,6 @@ import { DocumentView } from "../views/nodes/DocumentView"; import { SelectionManager } from "./SelectionManager"; import { DocumentManager } from "./DocumentManager"; import { CollectionVideoView } from "../views/collections/CollectionVideoView"; -import { CollectionPDFView } from "../views/collections/CollectionPDFView"; import { CollectionView } from "../views/collections/CollectionView"; library.add(fa.faCopy); @@ -186,7 +185,7 @@ export default class SharingManager extends React.Component<{}> { className={"focus-span"} title={title} onClick={() => { - let context: Opt; + let context: Opt; if (this.targetDoc && this.targetDocView && (context = this.targetDocView.props.ContainingCollectionView)) { DocumentManager.Instance.jumpToDocument(this.targetDoc, true, undefined, context.props.Document); } diff --git a/src/client/views/collections/CollectionPDFView.scss b/src/client/views/collections/CollectionPDFView.scss deleted file mode 100644 index 62ec8a5be..000000000 --- a/src/client/views/collections/CollectionPDFView.scss +++ /dev/null @@ -1,11 +0,0 @@ - - -.collectionPdfView-cont { - width: 100%; - height: 100%; - position: absolute; - top: 0; - left: 0; - z-index: -1; - overflow: hidden !important; -} diff --git a/src/client/views/collections/CollectionPDFView.tsx b/src/client/views/collections/CollectionPDFView.tsx deleted file mode 100644 index cc8142ec0..000000000 --- a/src/client/views/collections/CollectionPDFView.tsx +++ /dev/null @@ -1,37 +0,0 @@ -import { trace } from "mobx"; -import { observer } from "mobx-react"; -import { Id } from "../../../new_fields/FieldSymbols"; -import { emptyFunction } from "../../../Utils"; -import { ContextMenu } from "../ContextMenu"; -import { FieldView, FieldViewProps } from "../nodes/FieldView"; -import { CollectionBaseView, CollectionRenderProps, CollectionViewType } from "./CollectionBaseView"; -import { CollectionFreeFormView } from "./collectionFreeForm/CollectionFreeFormView"; -import "./CollectionPDFView.scss"; -import React = require("react"); - - -@observer -export class CollectionPDFView extends React.Component { - public static LayoutString(fieldKey: string = "data", fieldExt: string = "annotations") { - return FieldView.LayoutString(CollectionPDFView, fieldKey, fieldExt); - } - - onContextMenu = (e: React.MouseEvent): void => { - if (!e.isPropagationStopped() && this.props.Document[Id] !== "mainDoc") { // need to test this because GoldenLayout causes a parallel hierarchy in the React DOM for its children and the main document view7 - ContextMenu.Instance.addItem({ description: "PDFOptions", event: emptyFunction, icon: "file-pdf" }); - } - } - - subView = (_type: CollectionViewType, renderProps: CollectionRenderProps) => { - return (); - } - - render() { - trace(); - return ( - - {this.subView} - - ); - } -} \ No newline at end of file diff --git a/src/client/views/collections/CollectionSchemaCells.tsx b/src/client/views/collections/CollectionSchemaCells.tsx index 179e44266..fd1362848 100644 --- a/src/client/views/collections/CollectionSchemaCells.tsx +++ b/src/client/views/collections/CollectionSchemaCells.tsx @@ -13,7 +13,6 @@ import { COLLECTION_BORDER_WIDTH, MAX_ROW_HEIGHT } from '../globalCssVariables.s import '../DocumentDecorations.scss'; import { EditableView } from "../EditableView"; import { FieldView, FieldViewProps } from "../nodes/FieldView"; -import { CollectionPDFView } from "./CollectionPDFView"; import "./CollectionSchemaView.scss"; import { CollectionVideoView } from "./CollectionVideoView"; import { CollectionView } from "./CollectionView"; @@ -34,8 +33,8 @@ export interface CellProps { row: number; col: number; rowProps: CellInfo; - CollectionView: Opt; - ContainingCollection: Opt; + CollectionView: Opt; + ContainingCollection: Opt; Document: Doc; fieldKey: string; renderDepth: number; diff --git a/src/client/views/collections/CollectionSchemaView.tsx b/src/client/views/collections/CollectionSchemaView.tsx index 1ba35a52b..670c6bd43 100644 --- a/src/client/views/collections/CollectionSchemaView.tsx +++ b/src/client/views/collections/CollectionSchemaView.tsx @@ -21,7 +21,6 @@ import { COLLECTION_BORDER_WIDTH } from '../../views/globalCssVariables.scss'; import { ContextMenu } from "../ContextMenu"; import '../DocumentDecorations.scss'; import { DocumentView } from "../nodes/DocumentView"; -import { CollectionPDFView } from "./CollectionPDFView"; import "./CollectionSchemaView.scss"; import { CollectionSubView } from "./CollectionSubView"; import { CollectionVideoView } from "./CollectionVideoView"; @@ -247,8 +246,8 @@ export interface SchemaTableProps { PanelHeight: () => number; PanelWidth: () => number; childDocs?: Doc[]; - CollectionView: Opt; - ContainingCollectionView: Opt; + CollectionView: Opt; + ContainingCollectionView: Opt; ContainingCollectionDoc: Opt; fieldKey: string; renderDepth: number; @@ -905,7 +904,7 @@ interface CollectionSchemaPreviewProps { ruleProvider: Doc | undefined; focus?: (doc: Doc) => void; showOverlays?: (doc: Doc) => { title?: string, caption?: string }; - CollectionView?: CollectionView | CollectionPDFView | CollectionVideoView; + CollectionView?: CollectionView | CollectionVideoView; CollectionDoc?: Doc; onClick?: ScriptField; getTransform: () => Transform; diff --git a/src/client/views/collections/CollectionSubView.tsx b/src/client/views/collections/CollectionSubView.tsx index 10937fba2..5e2b79278 100644 --- a/src/client/views/collections/CollectionSubView.tsx +++ b/src/client/views/collections/CollectionSubView.tsx @@ -18,7 +18,6 @@ import { undoBatch, UndoManager } from "../../util/UndoManager"; import { DocComponent } from "../DocComponent"; import { FieldViewProps } from "../nodes/FieldView"; import { FormattedTextBox, GoogleRef } from "../nodes/FormattedTextBox"; -import { CollectionPDFView } from "./CollectionPDFView"; import { CollectionVideoView } from "./CollectionVideoView"; import { CollectionView } from "./CollectionView"; import React = require("react"); @@ -38,7 +37,7 @@ export interface CollectionViewProps extends FieldViewProps { } export interface SubCollectionViewProps extends CollectionViewProps { - CollectionView: Opt; + CollectionView: Opt; ruleProvider: Doc | undefined; } diff --git a/src/client/views/nodes/DocumentContentsView.tsx b/src/client/views/nodes/DocumentContentsView.tsx index 75dd27f46..a824ae9cc 100644 --- a/src/client/views/nodes/DocumentContentsView.tsx +++ b/src/client/views/nodes/DocumentContentsView.tsx @@ -8,7 +8,6 @@ import { HistogramBox } from "../../northstar/dash-nodes/HistogramBox"; import DirectoryImportBox from "../../util/Import & Export/DirectoryImportBox"; import { CollectionDockingView } from "../collections/CollectionDockingView"; import { CollectionFreeFormView } from "../collections/collectionFreeForm/CollectionFreeFormView"; -import { CollectionPDFView } from "../collections/CollectionPDFView"; import { CollectionSchemaView } from "../collections/CollectionSchemaView"; import { CollectionVideoView } from "../collections/CollectionVideoView"; import { CollectionView } from "../collections/CollectionView"; @@ -101,7 +100,7 @@ export class DocumentContentsView extends React.Component; + ContainingCollectionView: Opt; ContainingCollectionDoc: Opt; Document: Doc; DataDoc?: Doc; diff --git a/src/client/views/nodes/FieldView.tsx b/src/client/views/nodes/FieldView.tsx index b93c78cfd..c17730f48 100644 --- a/src/client/views/nodes/FieldView.tsx +++ b/src/client/views/nodes/FieldView.tsx @@ -8,7 +8,6 @@ import { List } from "../../../new_fields/List"; import { RichTextField } from "../../../new_fields/RichTextField"; import { AudioField, ImageField, VideoField } from "../../../new_fields/URLField"; import { Transform } from "../../util/Transform"; -import { CollectionPDFView } from "../collections/CollectionPDFView"; import { CollectionVideoView } from "../collections/CollectionVideoView"; import { CollectionView } from "../collections/CollectionView"; import { AudioBox } from "./AudioBox"; @@ -29,7 +28,7 @@ export interface FieldViewProps { fieldExt: string; leaveNativeSize?: boolean; fitToBox?: boolean; - ContainingCollectionView: Opt; + ContainingCollectionView: Opt; ContainingCollectionDoc: Opt; ruleProvider: Doc | undefined; Document: Doc; diff --git a/src/client/views/pdf/PDFViewer.tsx b/src/client/views/pdf/PDFViewer.tsx index b010d16c8..65ca830d9 100644 --- a/src/client/views/pdf/PDFViewer.tsx +++ b/src/client/views/pdf/PDFViewer.tsx @@ -18,7 +18,6 @@ import PDFMenu from "./PDFMenu"; import "./PDFViewer.scss"; import React = require("react"); import * as rp from "request-promise"; -import { CollectionPDFView } from "../collections/CollectionPDFView"; import { CollectionVideoView } from "../collections/CollectionVideoView"; import { CollectionView } from "../collections/CollectionView"; import Annotation from "./Annotation"; @@ -54,7 +53,7 @@ interface IViewerProps { addDocument?: (doc: Doc, allowDuplicates?: boolean) => boolean; setPdfViewer: (view: PDFViewer) => void; ScreenToLocalTransform: () => Transform; - ContainingCollectionView: Opt; + ContainingCollectionView: Opt; whenActiveChanged: (isActive: boolean) => void; } @@ -315,9 +314,11 @@ export class PDFViewer extends React.Component { @action scrollToAnnotation = (scrollToAnnotation: Doc) => { - let offset = this.visibleHeight() / 2 * 96 / 72; - this._mainCont.current && smoothScroll(500, this._mainCont.current, NumCast(scrollToAnnotation.y) - offset); - Doc.linkFollowHighlight(scrollToAnnotation); + if (scrollToAnnotation) { + let offset = this.visibleHeight() / 2 * 96 / 72; + this._mainCont.current && smoothScroll(500, this._mainCont.current, NumCast(scrollToAnnotation.y) - offset); + Doc.linkFollowHighlight(scrollToAnnotation); + } } -- cgit v1.2.3-70-g09d2 From eb95a7e5f5e50970d87ae0fd1f76ee080e7fa169 Mon Sep 17 00:00:00 2001 From: Bob Zeleznik Date: Mon, 21 Oct 2019 03:20:58 -0400 Subject: switched to a lightbox package that has rotation --- package.json | 2 +- src/client/views/collections/CollectionView.tsx | 27 ++++++++++++++++++--- src/client/views/nodes/ImageBox.tsx | 31 ++++--------------------- src/client/views/nodes/KeyValueBox.tsx | 1 - src/client/views/nodes/KeyValuePair.tsx | 1 - src/client/views/nodes/PDFBox.tsx | 1 - 6 files changed, 29 insertions(+), 34 deletions(-) (limited to 'package.json') diff --git a/package.json b/package.json index 8cbbb84af..8ee080933 100644 --- a/package.json +++ b/package.json @@ -201,7 +201,7 @@ "react-dimensions": "^1.3.1", "react-dom": "^16.8.4", "react-golden-layout": "^1.0.6", - "react-image-lightbox": "^5.1.0", + "react-image-lightbox-with-rotate": "^5.1.1", "react-jsx-parser": "^1.15.0", "react-measure": "^2.2.4", "react-mosaic": "0.0.20", diff --git a/src/client/views/collections/CollectionView.tsx b/src/client/views/collections/CollectionView.tsx index 3d5b4e562..f8011b1f8 100644 --- a/src/client/views/collections/CollectionView.tsx +++ b/src/client/views/collections/CollectionView.tsx @@ -5,7 +5,7 @@ import { action, IReactionDisposer, observable, reaction, runInAction } from 'mo import { observer } from "mobx-react"; import * as React from 'react'; import { Id } from '../../../new_fields/FieldSymbols'; -import { StrCast } from '../../../new_fields/Types'; +import { StrCast, Cast } from '../../../new_fields/Types'; import { CurrentUserUtils } from '../../../server/authentication/models/current_user_utils'; import { ContextMenu } from "../ContextMenu"; import { ContextMenuProps } from '../ContextMenuItem'; @@ -20,6 +20,11 @@ import { CollectionTreeView } from "./CollectionTreeView"; import { CollectionViewBaseChrome } from './CollectionViewChromes'; import { ImageUtils } from '../../util/Import & Export/ImageUtils'; import { CollectionLinearView } from '../CollectionLinearView'; +import { DocumentType } from '../../documents/DocumentTypes'; +import { ImageField } from '../../../new_fields/URLField'; +import { DocListCast } from '../../../new_fields/Doc'; +import Lightbox from 'react-image-lightbox-with-rotate'; +import 'react-image-lightbox-with-rotate/style.css'; // This only needs to be imported once in your app export const COLLECTION_BORDER_WIDTH = 2; library.add(faTh, faTree, faSquare, faProjectDiagram, faSignature, faThList, faFingerprint, faColumns, faEllipsisV, faImage, faEye as any, faCopy); @@ -120,6 +125,7 @@ export class CollectionView extends React.Component { break; } } + subItems.push({ description: "lightbox", event: action(() => this._isOpen = true), icon: "eye" }); !existingVm && ContextMenu.Instance.addItem({ description: "View Modes...", subitems: subItems, icon: "eye" }); let existing = ContextMenu.Instance.findByDescription("Layout..."); @@ -130,11 +136,26 @@ export class CollectionView extends React.Component { } } + @observable _isOpen = false; + @observable _curPage = 0; + lightbox = (images: string[]) => { + return !this._isOpen ? (null) : ( this._isOpen = false)} + onMovePrevRequest={action(() => this._curPage = (this._curPage + images.length - 1) % images.length)} + onMoveNextRequest={action(() => this._curPage = (this._curPage + 1) % images.length)} />); + } + render() { - return ( - + return (<> + {this.SubView} + + {this.lightbox(DocListCast(this.props.Document[this.props.fieldKey]).filter(d => d.type === DocumentType.IMG).map(d => Cast(d.data, ImageField)!.url!.href))} + ); } } \ No newline at end of file diff --git a/src/client/views/nodes/ImageBox.tsx b/src/client/views/nodes/ImageBox.tsx index fbf05d148..a6d6a16a0 100644 --- a/src/client/views/nodes/ImageBox.tsx +++ b/src/client/views/nodes/ImageBox.tsx @@ -4,8 +4,6 @@ import { faAsterisk, faFileAudio, faImage, faPaintBrush } from '@fortawesome/fre import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; import { action, computed, observable, runInAction, trace } from 'mobx'; import { observer } from "mobx-react"; -import Lightbox from 'react-image-lightbox'; -import 'react-image-lightbox/style.css'; // This only needs to be imported once in your app import { Doc, DocListCast, HeightSym, WidthSym } from '../../../new_fields/Doc'; import { List } from '../../../new_fields/List'; import { createSchema, listSpec, makeInterface } from '../../../new_fields/Schema'; @@ -84,27 +82,6 @@ export class ImageBox extends DocAnnotatableComponent { - if (this._isOpen) { - return ( - this._isOpen = false - )} - onMovePrevRequest={action(() => - this.Document.curPage = ((this.Document.curPage || 0) + images.length - 1) % images.length - )} - onMoveNextRequest={action(() => - this.Document.curPage = ((this.Document.curPage || 0) + 1) % images.length - )} - />); - } - } - recordAudioAnnotation = () => { let gumStream: any; let recorder: any; @@ -294,13 +271,13 @@ export class ImageBox extends DocAnnotatableComponent 20) { let alts = DocListCast(this.extensionDoc.Alternates); - let altpaths: string[] = alts.filter(doc => doc.data instanceof ImageField).map(doc => this.choosePath((doc.data as ImageField).url)); + let altpaths = alts.filter(doc => doc.data instanceof ImageField).map(doc => this.choosePath((doc.data as ImageField).url)); let field = this.dataDoc[this.props.fieldKey]; // if (w < 100 && this._smallRetryCount < 10) this._curSuffix = "_s"; // else if (w < 600 && this._mediumRetryCount < 10) this._curSuffix = "_m"; diff --git a/src/client/views/nodes/KeyValueBox.tsx b/src/client/views/nodes/KeyValueBox.tsx index 41d43b8b8..2b46e9dff 100644 --- a/src/client/views/nodes/KeyValueBox.tsx +++ b/src/client/views/nodes/KeyValueBox.tsx @@ -1,7 +1,6 @@ import { action, computed, observable } from "mobx"; import { observer } from "mobx-react"; -import 'react-image-lightbox/style.css'; // This only needs to be imported once in your app import { Doc, Field, FieldResult } from "../../../new_fields/Doc"; import { List } from "../../../new_fields/List"; import { RichTextField } from "../../../new_fields/RichTextField"; diff --git a/src/client/views/nodes/KeyValuePair.tsx b/src/client/views/nodes/KeyValuePair.tsx index 4e1390200..67d33845d 100644 --- a/src/client/views/nodes/KeyValuePair.tsx +++ b/src/client/views/nodes/KeyValuePair.tsx @@ -1,6 +1,5 @@ import { action, observable } from 'mobx'; import { observer } from "mobx-react"; -import 'react-image-lightbox/style.css'; // This only needs to be imported once in your app import { Doc, Field, Opt } from '../../../new_fields/Doc'; import { emptyFunction, returnFalse, returnOne, returnZero } from '../../../Utils'; import { Docs } from '../../documents/Documents'; diff --git a/src/client/views/nodes/PDFBox.tsx b/src/client/views/nodes/PDFBox.tsx index 4513117b2..5795d6f12 100644 --- a/src/client/views/nodes/PDFBox.tsx +++ b/src/client/views/nodes/PDFBox.tsx @@ -3,7 +3,6 @@ import { action, observable, runInAction, reaction, IReactionDisposer } from 'mo import { observer } from "mobx-react"; import * as Pdfjs from "pdfjs-dist"; import "pdfjs-dist/web/pdf_viewer.css"; -import 'react-image-lightbox/style.css'; import { Opt, WidthSym, Doc } from "../../../new_fields/Doc"; import { makeInterface } from "../../../new_fields/Schema"; import { ScriptField } from '../../../new_fields/ScriptField'; -- cgit v1.2.3-70-g09d2 From f83205e4ea3486ebeeaa52918650bf6521c36cb9 Mon Sep 17 00:00:00 2001 From: bob Date: Wed, 6 Nov 2019 12:35:51 -0500 Subject: schema view cleanups --- package.json | 2 +- src/client/views/MainView.tsx | 2 +- .../CollectionSchemaMovableTableHOC.tsx | 2 +- .../views/collections/CollectionSchemaView.scss | 6 +- .../views/collections/CollectionSchemaView.tsx | 243 +++++++-------------- src/client/views/nodes/DocumentView.tsx | 2 +- src/server/index.ts | 64 +++--- 7 files changed, 116 insertions(+), 205 deletions(-) (limited to 'package.json') diff --git a/package.json b/package.json index 8ee080933..fb978be14 100644 --- a/package.json +++ b/package.json @@ -115,7 +115,7 @@ "@types/youtube": "0.0.38", "adm-zip": "^0.4.13", "archiver": "^3.0.3", - "array-batcher": "^1.1.3", + "array-batcher": "^1.2.3", "async": "^2.6.2", "babel-runtime": "^6.26.0", "bcrypt-nodejs": "0.0.3", diff --git a/src/client/views/MainView.tsx b/src/client/views/MainView.tsx index 383300b22..773da05df 100644 --- a/src/client/views/MainView.tsx +++ b/src/client/views/MainView.tsx @@ -137,7 +137,7 @@ export class MainView extends React.Component { globalPointerDown = action((e: PointerEvent) => { this.isPointerDown = true; - AudioBox.AudioEnabled = true; + AudioBox.Enabled = true; const targets = document.elementsFromPoint(e.x, e.y); if (targets && targets.length && targets[0].className.toString().indexOf("contextMenu") === -1) { ContextMenu.Instance.closeMenu(); diff --git a/src/client/views/collections/CollectionSchemaMovableTableHOC.tsx b/src/client/views/collections/CollectionSchemaMovableTableHOC.tsx index 39abc41ec..274c8b6d1 100644 --- a/src/client/views/collections/CollectionSchemaMovableTableHOC.tsx +++ b/src/client/views/collections/CollectionSchemaMovableTableHOC.tsx @@ -229,7 +229,7 @@ export class MovableRow extends React.Component {
-
this.props.removeDoc(this.props.rowInfo.original)}>
+
this.props.removeDoc(this.props.rowInfo.original))}>
{children} diff --git a/src/client/views/collections/CollectionSchemaView.scss b/src/client/views/collections/CollectionSchemaView.scss index 6a9392253..36c6c7b0e 100644 --- a/src/client/views/collections/CollectionSchemaView.scss +++ b/src/client/views/collections/CollectionSchemaView.scss @@ -10,6 +10,7 @@ top: 0; width: 100%; height: 100%; + margin-top: 0; transition: top 0.5s; display: flex; justify-content: space-between; @@ -43,9 +44,8 @@ height: auto !important; .collectionSchemaView-previewDoc { - height: 100%; - width: 100%; position: absolute; + display: inline; } .collectionSchemaView-input { @@ -469,7 +469,7 @@ button.add-column { overflow: visible; } -.sub { +.reactTable-sub { padding: 10px 30px; background-color: rgb(252, 252, 252); width: calc(100% - 50px); diff --git a/src/client/views/collections/CollectionSchemaView.tsx b/src/client/views/collections/CollectionSchemaView.tsx index 34f642f80..b840dc5f8 100644 --- a/src/client/views/collections/CollectionSchemaView.tsx +++ b/src/client/views/collections/CollectionSchemaView.tsx @@ -4,7 +4,7 @@ import { faCog, faPlus, faTable, faSortUp, faSortDown } from '@fortawesome/free- import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; import { action, computed, observable, trace, untracked } from "mobx"; import { observer } from "mobx-react"; -import ReactTable, { CellInfo, ComponentPropsGetterR, Column, RowInfo, ResizedChangeFunction, Resize } from "react-table"; +import ReactTable, { CellInfo, ComponentPropsGetterR, Column, RowInfo, ResizedChangeFunction, Resize, SortingRule } from "react-table"; import "react-table/react-table.css"; import { emptyFunction, returnOne, returnEmptyString } from "../../../Utils"; import { Doc, DocListCast, Field, Opt } from "../../../new_fields/Doc"; @@ -32,7 +32,6 @@ import { ComputedField, ScriptField } from "../../../new_fields/ScriptField"; import { SchemaHeaderField } from "../../../new_fields/SchemaHeaderField"; import { DocumentType } from "../../documents/DocumentTypes"; - library.add(faCog, faPlus, faSortUp, faSortDown); library.add(faTable); // bcz: need to add drag and drop of rows and columns. This seems like it might work for rows: https://codesandbox.io/s/l94mn1q657 @@ -73,20 +72,14 @@ export class CollectionSchemaView extends CollectionSubView(doc => doc) { super.CreateDropTarget(ele); } - isFocused = (doc: Doc): boolean => { - if (!this.props.isSelected()) return false; - return doc === this._focusedTable; - } + isFocused = (doc: Doc): boolean => !this.props.isSelected() ? false : doc === this._focusedTable; - @action - setFocused = (doc: Doc): void => { - this._focusedTable = doc; - } + @action setFocused = (doc: Doc) => this._focusedTable = doc; - @action - setPreviewDoc = (doc: Doc): void => { - this.previewDoc = doc; - } + @action setPreviewDoc = (doc: Doc) => this.previewDoc = doc; + + @undoBatch + @action setPreviewScript = (script: string) => this.previewScript = script; //toggles preview side-panel of schema @action @@ -128,12 +121,6 @@ export class CollectionSchemaView extends CollectionSubView(doc => doc) { } } - onWheel = (e: React.WheelEvent): void => { - if (this.props.active()) { - e.stopPropagation(); - } - } - @computed get previewDocument(): Doc | undefined { let selected = this.previewDoc; @@ -180,62 +167,51 @@ export class CollectionSchemaView extends CollectionSubView(doc => doc) {
; } - @undoBatch - @action - setPreviewScript = (script: string) => { - this.previewScript = script; - } - @computed get schemaTable() { - return ( - - ); + return ; } @computed public get schemaToolbar() { - return ( -
-
-
Show Preview
-
+ return
+
+
Show Preview
- ); +
; } render() { - return ( -
-
this.onDrop(e, {})} ref={this.createTarget}> - {this.schemaTable} -
- {this.dividerDragger} - {!this.previewWidth() ? (null) : this.previewPanel} + return
+
this.props.active() && e.stopPropagation()} onDrop={e => this.onDrop(e, {})} ref={this.createTarget}> + {this.schemaTable}
- ); + {this.dividerDragger} + {!this.previewWidth() ? (null) : this.previewPanel} +
; } } @@ -251,6 +227,7 @@ export interface SchemaTableProps { fieldKey: string; renderDepth: number; deleteDocument: (document: Doc) => boolean; + addDocument: (document: Doc) => boolean; moveDocument: (document: Doc, targetCollection: Doc, addDocument: (document: Doc) => boolean) => boolean; ScreenToLocalTransform: () => Transform; active: () => boolean; @@ -307,11 +284,11 @@ export class SchemaTable extends React.Component { return resized; }, [] as { id: string, value: number }[]); } - @computed get sorted(): { id: string, desc: boolean }[] { + @computed get sorted(): SortingRule[] { return this.columns.reduce((sorted, shf) => { shf.desc && sorted.push({ id: shf.heading, desc: shf.desc }); return sorted; - }, [] as { id: string, desc: boolean }[]); + }, [] as SortingRule[]); } @computed get borderWidth() { return Number(COLLECTION_BORDER_WIDTH); } @@ -433,26 +410,11 @@ export class SchemaTable extends React.Component { return Doc.AddDocToList(this.props.Document, this.props.fieldKey, doc, relativeTo, before); } - tableRemoveDoc = (document: Doc): boolean => { - - let children = this.childDocs; - if (children.indexOf(document) !== -1) { - children.splice(children.indexOf(document), 1); - this.childDocs = children; - return true; - } - return false; - } - private getTrProps: ComponentPropsGetterR = (state, rowInfo) => { - const that = this; - if (!rowInfo) { - return {}; - } - return { + return !rowInfo ? {} : { ScreenToLocalTransform: this.props.ScreenToLocalTransform, addDoc: this.tableAddDoc, - removeDoc: this.tableRemoveDoc, + removeDoc: this.props.deleteDocument, rowInfo, rowFocused: !this._headerIsEditing && rowInfo.index === this._focusedCell.row && this.props.isFocused(this.props.Document), textWrapRow: this.toggleTextWrapRow, @@ -477,10 +439,7 @@ export class SchemaTable extends React.Component { }; } - @action - onExpandCollection = (collection: Doc): void => { - this._openCollections.push(collection[Id]); - } + @action onExpandCollection = (collection: Doc) => this._openCollections.push(collection[Id]); @action onCloseCollection = (collection: Doc): void => { @@ -488,15 +447,8 @@ export class SchemaTable extends React.Component { if (index > -1) this._openCollections.splice(index, 1); } - @action - setCellIsEditing = (isEditing: boolean): void => { - this._cellIsEditing = isEditing; - } - - @action - setHeaderIsEditing = (isEditing: boolean): void => { - this._headerIsEditing = isEditing; - } + @action setCellIsEditing = (isEditing: boolean) => this._cellIsEditing = isEditing; + @action setHeaderIsEditing = (isEditing: boolean) => this._headerIsEditing = isEditing; onPointerDown = (e: React.PointerEvent): void => { this.props.setFocused(this.props.Document); @@ -505,67 +457,40 @@ export class SchemaTable extends React.Component { } } - onWheel = (e: React.WheelEvent): void => { - if (this.props.active()) { - e.stopPropagation(); - } - } - + @action onKeyDown = (e: KeyboardEvent): void => { if (!this._cellIsEditing && !this._headerIsEditing && this.props.isFocused(this.props.Document)) {// && this.props.isSelected()) { let direction = e.key === "Tab" ? "tab" : e.which === 39 ? "right" : e.which === 37 ? "left" : e.which === 38 ? "up" : e.which === 40 ? "down" : ""; - this.changeFocusedCellByDirection(direction); + this._focusedCell = this.changeFocusedCellByDirection(direction, this._focusedCell.row, this._focusedCell.col); - let children = this.childDocs; - const pdoc = FieldValue(children[this._focusedCell.row]); + const pdoc = FieldValue(this.childDocs[this._focusedCell.row]); pdoc && this.props.setPreviewDoc(pdoc); } } - @action - changeFocusedCellByDirection = (direction: string): void => { - let children = this.childDocs; + changeFocusedCellByDirection = (direction: string, curRow: number, curCol: number) => { switch (direction) { - case "tab": - if (this._focusedCell.col + 1 === this.columns.length && this._focusedCell.row + 1 === children.length) { - this._focusedCell = { row: 0, col: 0 }; - } else if (this._focusedCell.col + 1 === this.columns.length) { - this._focusedCell = { row: this._focusedCell.row + 1, col: 0 }; - } else { - this._focusedCell = { row: this._focusedCell.row, col: this._focusedCell.col + 1 }; - } - break; - case "right": - this._focusedCell = { row: this._focusedCell.row, col: this._focusedCell.col + 1 === this.columns.length ? this._focusedCell.col : this._focusedCell.col + 1 }; - break; - case "left": - this._focusedCell = { row: this._focusedCell.row, col: this._focusedCell.col === 0 ? this._focusedCell.col : this._focusedCell.col - 1 }; - break; - case "up": - this._focusedCell = { row: this._focusedCell.row === 0 ? this._focusedCell.row : this._focusedCell.row - 1, col: this._focusedCell.col }; - break; - case "down": - this._focusedCell = { row: this._focusedCell.row + 1 === children.length ? this._focusedCell.row : this._focusedCell.row + 1, col: this._focusedCell.col }; - break; + case "tab": return { row: (curRow + 1 === this.childDocs.length ? 0 : curRow + 1), col: curCol + 1 === this.columns.length ? 0 : curCol + 1 }; + case "right": return { row: curRow, col: curCol + 1 === this.columns.length ? curCol : curCol + 1 }; + case "left": return { row: curRow, col: curCol === 0 ? curCol : curCol - 1 }; + case "up": return { row: curRow === 0 ? curRow : curRow - 1, col: curCol }; + case "down": return { row: curRow + 1 === this.childDocs.length ? curRow : curRow + 1, col: curCol }; } + return this._focusedCell; } @action changeFocusedCellByIndex = (row: number, col: number): void => { - this._focusedCell = { row: row, col: col }; + if (this._focusedCell.row !== row || this._focusedCell.col !== col) { + this._focusedCell = { row: row, col: col }; + } this.props.setFocused(this.props.Document); } @undoBatch createRow = () => { - let children = this.childDocs; - - let newDoc = Docs.Create.TextDocument({ width: 100, height: 30 }); - let proto = Doc.GetProto(newDoc); - proto.title = ""; - children.push(newDoc); - - this.childDocs = children; + let newDoc = Docs.Create.TextDocument({ title: "", width: 100, height: 30 }); + this.props.addDocument(newDoc); } @undoBatch @@ -677,9 +602,7 @@ export class SchemaTable extends React.Component { } @action - setColumns = (columns: SchemaHeaderField[]) => { - this.columns = columns; - } + setColumns = (columns: SchemaHeaderField[]) => this.columns = columns; @undoBatch reorderColumns = (toMove: SchemaHeaderField, relativeTo: SchemaHeaderField, before: boolean, columnsValues: SchemaHeaderField[]) => { @@ -762,7 +685,7 @@ export class SchemaTable extends React.Component { SubComponent={hasCollectionChild ? row => { if (row.original.type === "collection") { - return
; + return
; } } : undefined} @@ -881,13 +804,11 @@ export class SchemaTable extends React.Component { } render() { - return ( -
this.props.onDrop(e, {})} onContextMenu={this.onContextMenu} > - {this.reactTable} -
this.createRow()}>+ new
-
- ); + return
this.props.active() && e.stopPropagation()} + onDrop={e => this.props.onDrop(e, {})} onContextMenu={this.onContextMenu} > + {this.reactTable} +
this.createRow()}>+ new
+
; } } @@ -922,7 +843,6 @@ interface CollectionSchemaPreviewProps { @observer export class CollectionSchemaPreview extends React.Component{ private dropDisposer?: DragManager.DragDropDisposer; - _mainCont?: HTMLDivElement; private get layoutDoc() { return this.props.Document && Doc.Layout(this.props.Document); } private get nativeWidth() { return NumCast(this.layoutDoc!.nativeWidth, this.props.PanelWidth()); } private get nativeHeight() { return NumCast(this.layoutDoc!.nativeHeight, this.props.PanelHeight()); } @@ -933,10 +853,7 @@ export class CollectionSchemaPreview extends React.Component { - } private createTarget = (ele: HTMLDivElement) => { - this._mainCont = ele; this.dropDisposer && this.dropDisposer(); if (ele) { this.dropDisposer = DragManager.MakeDropTarget(ele, { handlers: { drop: this.drop.bind(this) } }); @@ -960,25 +877,12 @@ export class CollectionSchemaPreview extends React.Component this.nativeHeight && (!this.props.Document || !this.props.Document.fitWidth) ? this.nativeHeight * this.contentScaling() : this.props.PanelHeight(); private getTransform = () => this.props.getTransform().translate(-this.centeringOffset, 0).scale(1 / this.contentScaling()); get centeringOffset() { return this.nativeWidth && (!this.props.Document || !this.props.Document.fitWidth) ? (this.props.PanelWidth() - this.nativeWidth * this.contentScaling()) / 2 : 0; } - @action - onPreviewScriptChange = (e: React.ChangeEvent) => { - this.props.setPreviewScript(e.currentTarget.value); - } - @computed get borderRounding() { - let br = StrCast(this.props.Document!.borderRounding); - if (br.endsWith("%")) { - let percent = Number(br.substr(0, br.length - 1)) / 100; - let nativeDim = Math.min(NumCast(this.layoutDoc!.nativeWidth), NumCast(this.layoutDoc!.nativeHeight)); - let minDim = percent * (nativeDim ? nativeDim : Math.min(this.PanelWidth(), this.PanelHeight())); - return minDim; - } - return undefined; - } + @computed get borderRounding() { return StrCast(this.props.Document!.borderRounding); } render() { let input = this.props.previewScript === undefined ? (null) : -
this.props.setPreviewScript(e.currentTarget.value)} style={{ left: `calc(50% - ${Math.min(75, (this.props.Document ? this.PanelWidth() / 2 : 75))}px)` }} />
; return (
@@ -987,7 +891,6 @@ export class CollectionSchemaPreview extends React.Component diff --git a/src/client/views/nodes/DocumentView.tsx b/src/client/views/nodes/DocumentView.tsx index c091f1260..54a687c34 100644 --- a/src/client/views/nodes/DocumentView.tsx +++ b/src/client/views/nodes/DocumentView.tsx @@ -668,7 +668,7 @@ export class DocumentView extends DocComponent(Docu {searchHighlight}
} -
+
; } } diff --git a/src/server/index.ts b/src/server/index.ts index 1595781dc..c6753a253 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -1049,34 +1049,42 @@ addSecureRoute({ let failed: number[] = []; - const batched = BatchedArray.from(media, { batchSize: 25 }); - const newMediaItems = await batched.batchedMapPatientInterval( - { magnitude: 100, unit: TimeUnit.Milliseconds }, - async (batch, collector) => { - for (let index = 0; index < batch.length; index++) { - const { url, description } = batch[index]; - const uploadToken = await GooglePhotosUploadUtils.DispatchGooglePhotosUpload(url); - if (!uploadToken) { - failed.push(index); - } else { - collector.push({ - description, - simpleMediaItem: { uploadToken } - }); - } - } - } - ); - - const failedCount = failed.length; - if (failedCount) { - console.error(`Unable to upload ${failedCount} image${failedCount === 1 ? "" : "s"} to Google's servers`); - } - - GooglePhotosUploadUtils.CreateMediaItems(newMediaItems, req.body.album).then( - results => _success(res, { results, failed }), - error => _error(res, mediaError, error) - ); + // bcz: this doesn't compile: + // Using ts-node version 7.0.1, typescript version 3.5.3 + // [ERROR] 09:54:48 ⨯ Unable to compile TypeScript: + // src/server/index.ts(1055,13): error TS2345: Argument of type '(batch: MediaInput[], collector: BatchContext) => Promise' is not assignable to parameter of type 'BatchFunction>'. + // Type 'Promise' is not assignable to type 'NewMediaItem[] | Promise'. + // Type 'Promise' is not assignable to type 'Promise'. + // Type 'void' is not assignable to type 'NewMediaItem[]'. + // src/server/index.ts(1062,35): error TS2339: Property 'push' does not exist on type 'BatchContext'. + // const batched = BatchedArray.from(media, { batchSize: 25 }); + // const newMediaItems = await batched.batchedMapPatientInterval( + // { magnitude: 100, unit: TimeUnit.Milliseconds }, + // async (batch, collector) => { + // for (let index = 0; index < batch.length; index++) { + // const { url, description } = batch[index]; + // const uploadToken = await GooglePhotosUploadUtils.DispatchGooglePhotosUpload(url); + // if (!uploadToken) { + // failed.push(index); + // } else { + // collector.push({ + // description, + // simpleMediaItem: { uploadToken } + // }); + // } + // } + // } + // ); + + // const failedCount = failed.length; + // if (failedCount) { + // console.error(`Unable to upload ${failedCount} image${failedCount === 1 ? "" : "s"} to Google's servers`); + // } + + // GooglePhotosUploadUtils.CreateMediaItems(newMediaItems, req.body.album).then( + // results => _success(res, { results, failed }), + // error => _error(res, mediaError, error) + // ); } }); -- cgit v1.2.3-70-g09d2 From 2891b8f3547cb0a2d039099228e30a712457832f Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Wed, 6 Nov 2019 22:32:50 -0500 Subject: updated typescript --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'package.json') diff --git a/package.json b/package.json index fb978be14..40bf40c17 100644 --- a/package.json +++ b/package.json @@ -33,7 +33,7 @@ "ts-node-dev": "^1.0.0-pre.32", "tslint": "^5.15.0", "tslint-loader": "^3.5.4", - "typescript": "^3.5.3", + "typescript": "^3.7.2", "webpack": "^4.29.6", "webpack-cli": "^3.2.3", "webpack-dev-middleware": "^3.6.1", -- cgit v1.2.3-70-g09d2