From 91e4ac65e0b8d1ff5c17ea0e80666038281ec5a6 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Tue, 15 Oct 2019 12:54:57 -0400 Subject: initial commit --- src/server/ActionUtilities.ts | 28 + src/server/Initialization.ts | 110 ++ src/server/RouteManager.ts | 131 ++ src/server/RouteStore.ts | 1 + src/server/apis/google/GoogleApiServerUtils.ts | 21 +- src/server/database.ts | 7 +- src/server/index.ts | 2089 +++++++++++------------- 7 files changed, 1240 insertions(+), 1147 deletions(-) create mode 100644 src/server/ActionUtilities.ts create mode 100644 src/server/Initialization.ts create mode 100644 src/server/RouteManager.ts (limited to 'src') diff --git a/src/server/ActionUtilities.ts b/src/server/ActionUtilities.ts new file mode 100644 index 000000000..9a009791b --- /dev/null +++ b/src/server/ActionUtilities.ts @@ -0,0 +1,28 @@ +import * as fs from 'fs'; +import { ExecOptions } from 'shelljs'; +import { exec } from 'child_process'; +import * as path from 'path'; + +export const command_line = (command: string, fromDirectory?: string) => { + return new Promise((resolve, reject) => { + let options: ExecOptions = {}; + if (fromDirectory) { + options.cwd = path.join(__dirname, fromDirectory); + } + exec(command, options, (err, stdout) => err ? reject(err) : resolve(stdout)); + }); +}; + +export const read_text_file = (relativePath: string) => { + let target = path.join(__dirname, relativePath); + return new Promise((resolve, reject) => { + fs.readFile(target, (err, data) => err ? reject(err) : resolve(data.toString())); + }); +}; + +export const write_text_file = (relativePath: string, contents: any) => { + let target = path.join(__dirname, relativePath); + return new Promise((resolve, reject) => { + fs.writeFile(target, contents, (err) => err ? reject(err) : resolve()); + }); +}; \ No newline at end of file diff --git a/src/server/Initialization.ts b/src/server/Initialization.ts new file mode 100644 index 000000000..e371a3edb --- /dev/null +++ b/src/server/Initialization.ts @@ -0,0 +1,110 @@ +import * as express from 'express'; +import * as expressValidator from 'express-validator'; +import * as session from 'express-session'; +import * as passport from 'passport'; +import * as bodyParser from 'body-parser'; +import * as cookieParser from 'cookie-parser'; +import expressFlash = require('express-flash'); +import flash = require('connect-flash'); +import { Database } from './database'; +const MongoStore = require('connect-mongo')(session); +import mongoose, { ConnectionStates } from 'mongoose'; +import { RouteStore } from './RouteStore'; +import RouteManager from './RouteManager'; +import * as webpack from 'webpack'; +const config = require('../../webpack.config'); +const compiler = webpack(config); +import * as wdm from 'webpack-dev-middleware'; +import * as whm from 'webpack-hot-middleware'; +import * as fs from 'fs'; + +export interface InitializationOptions { + listenAtPort: number; + routeSetter: (server: RouteManager) => void; +} + +export default async function InitializeServer(options: InitializationOptions) { + const { listenAtPort, routeSetter } = options; + const server = injectMiddleware(express()); + const { url } = Database; + try { + await connectToDatabase(url); + } catch (e) { + console.error(`Mongoose FAILED to establish default connection at ${url}`); + console.error(e); + console.log('Since a valid database connection is required to use Dash, killing the server process.\nPlease try again later.'); + process.exit(1); + } + + // static file serving + server.use(express.static(__dirname + RouteStore.public)); + server.use(RouteStore.images, express.static(__dirname + RouteStore.public)); + + routeSetter(new RouteManager(server, determineEnvironment())); + + server.use(wdm(compiler, { publicPath: config.output.publicPath })); + server.use(whm(compiler)); + server.listen(listenAtPort, () => console.log(`server started at http://localhost:${listenAtPort}`)); + + return server; +} + +function determineEnvironment() { + const isRelease = process.env.RELEASE === "true"; + + console.log(`running server in ${isRelease ? 'release' : 'debug'} mode`); + console.log(process.env.PWD); + + let clientUtils = fs.readFileSync("./src/client/util/ClientUtils.ts.temp", "utf8"); + clientUtils = `//AUTO-GENERATED FILE: DO NOT EDIT\n${clientUtils.replace('"mode"', String(isRelease))}`; + fs.writeFileSync("./src/client/util/ClientUtils.ts", clientUtils, "utf8"); + + return isRelease; +} + +const week = 7 * 24 * 60 * 60 * 1000; +const secret = "64d6866242d3b5a5503c675b32c9605e4e90478e9b77bcf2bc"; + +function injectMiddleware(server: express.Express) { + [ + cookieParser(), + session({ + secret, + resave: true, + cookie: { maxAge: week }, + saveUninitialized: true, + store: new MongoStore({ url: Database.url }) + }), + flash(), + expressFlash(), + bodyParser.json({ limit: "10mb" }), + bodyParser.urlencoded({ extended: true }), + expressValidator(), + passport.initialize(), + passport.session(), + (req: express.Request, res: express.Response, next: express.NextFunction) => { + res.locals.user = req.user; + next(); + } + ].forEach(next => server.use(next)); + return server; +} + +async function connectToDatabase(url: string) { + const { connection } = mongoose; + process.on('SIGINT', () => { + connection.close(() => { + console.log('Mongoose default connection disconnected through app termination'); + process.exit(0); + }); + }); + if (connection.readyState === ConnectionStates.disconnected) { + return new Promise((resolve, reject) => { + connection.on('error', reject); + connection.on('connected', () => { + console.log(`Mongoose established default connection at ${url}`); + resolve(); + }); + }); + } +} \ No newline at end of file diff --git a/src/server/RouteManager.ts b/src/server/RouteManager.ts new file mode 100644 index 000000000..cf15e45c9 --- /dev/null +++ b/src/server/RouteManager.ts @@ -0,0 +1,131 @@ +import RouteSubscriber from "./RouteSubscriber"; +import { RouteStore } from "./RouteStore"; +import { DashUserModel } from "./authentication/models/user_model"; +import * as express from 'express'; +import * as qs from 'query-string'; + +export default class RouteManager { + private server: express.Express; + private _isRelease: boolean; + + public get release() { + return this._isRelease; + } + + constructor(server: express.Express, isRelease: boolean) { + this.server = server; + this._isRelease = isRelease; + } + + /** + * Please invoke this function when adding a new route to Dash's server. + * It ensures that any requests leading to or containing user-sensitive information + * does not execute unless Passport authentication detects a user logged in. + * @param method whether or not the request is a GET or a POST + * @param handler the action to invoke, recieving a DashUserModel and, as expected, the Express.Request and Express.Response + * @param onRejection an optional callback invoked on return if no user is found to be logged in + * @param subscribers the forward slash prepended path names (reference and add to RouteStore.ts) that will all invoke the given @param handler + */ + addSupervisedRoute(initializer: RouteInitializer) { + const { method, subscription, onValidation, onRejection, onError } = initializer; + const release = this._isRelease; + let abstracted = async (req: express.Request, res: express.Response) => { + const { user, originalUrl: target } = req; + if (user || isSharedDocAccess(target)) { + try { + await onValidation(user, req, res, release); + } catch (e) { + if (onError) { + onError(req, res, e, release); + } else { + _error(res, `The server encountered an internal error handling ${target}.`, e); + } + } + } else { + req.session!.target = target; + try { + await (onRejection || LoginRedirect)(req, res, release); + } catch (e) { + if (onError) { + onError(req, res, e, this._isRelease); + } else { + _error(res, `The server encountered an internal error when rejecting ${target}.`, e); + } + } + } + }; + const subscribe = (subscriber: RouteSubscriber | string) => { + let route: string; + if (typeof subscriber === "string") { + route = subscriber; + } else { + route = subscriber.build; + } + switch (method) { + case Method.GET: + this.server.get(route, abstracted); + break; + case Method.POST: + this.server.post(route, abstracted); + break; + } + }; + if (Array.isArray(subscription)) { + subscription.forEach(subscribe); + } else { + subscribe(subscription); + } + } + +} + +export enum Method { + GET, + POST +} + +export type ValidationHandler = (user: DashUserModel, req: express.Request, res: express.Response, isRelease: boolean) => any | Promise; +export type RejectionHandler = (req: express.Request, res: express.Response, isRelease: boolean) => any | Promise; +export type ErrorHandler = (req: express.Request, res: express.Response, error: any, isRelease: boolean) => any | Promise; + +const LoginRedirect: RejectionHandler = (_req, res) => res.redirect(RouteStore.login); + +export interface RouteInitializer { + method: Method; + subscription: string | RouteSubscriber | (string | RouteSubscriber)[]; + onValidation: ValidationHandler; + onRejection?: RejectionHandler; + onError?: ErrorHandler; +} + +const isSharedDocAccess = (target: string) => { + const shared = qs.parse(qs.extract(target), { sort: false }).sharing === "true"; + const docAccess = target.startsWith("/doc/"); + return shared && docAccess; +}; + +export const STATUS = { + OK: 200, + BAD_REQUEST: 400, + EXECUTION_ERROR: 500, + PERMISSION_DENIED: 403 +}; + +export function _error(res: express.Response, message: string, error?: any) { + res.statusMessage = message; + res.status(STATUS.EXECUTION_ERROR).send(error); +} + +export function _success(res: express.Response, body: any) { + res.status(STATUS.OK).send(body); +} + +export function _invalid(res: express.Response, message: string) { + res.statusMessage = message; + res.status(STATUS.BAD_REQUEST).send(); +} + +export function _permission_denied(res: express.Response, message: string) { + res.statusMessage = message; + res.status(STATUS.BAD_REQUEST).send("Permission Denied!"); +} diff --git a/src/server/RouteStore.ts b/src/server/RouteStore.ts index 7426ffb39..de2553b2f 100644 --- a/src/server/RouteStore.ts +++ b/src/server/RouteStore.ts @@ -7,6 +7,7 @@ export enum RouteStore { corsProxy = "/corsProxy", delete = "/delete", deleteAll = "/deleteAll", + pull = "/pull", // UPLOAD AND STATIC FILE SERVING public = "/public", diff --git a/src/server/apis/google/GoogleApiServerUtils.ts b/src/server/apis/google/GoogleApiServerUtils.ts index 5714c9928..6093197f1 100644 --- a/src/server/apis/google/GoogleApiServerUtils.ts +++ b/src/server/apis/google/GoogleApiServerUtils.ts @@ -9,6 +9,8 @@ import request = require('request-promise'); import * as qs from 'query-string'; import Photos = require('googlephotos'); import { Database } from "../../database"; +const path = require("path"); + /** * Server side authentication for Google Api queries. */ @@ -76,22 +78,24 @@ export namespace GoogleApiServerUtils { }); }; - const RetrieveOAuthClient = async (information: CredentialInformation) => { - return new Promise((resolve, reject) => { - readFile(information.credentialsPath, async (err, credentials) => { + let AuthorizationManager: OAuth2Client; + + export const LoadOAuthClient = async () => { + return new Promise((resolve, reject) => { + readFile(path.join(__dirname, "../../credentials/google_docs_credentials.json"), async (err, credentials) => { if (err) { reject(err); return console.log('Error loading client secret file:', err); } const { client_secret, client_id, redirect_uris } = parseBuffer(credentials).installed; - resolve(new google.auth.OAuth2(client_id, client_secret, redirect_uris[0])); + AuthorizationManager = new google.auth.OAuth2(client_id, client_secret, redirect_uris[0]); + resolve(); }); }); }; export const GenerateAuthenticationUrl = async (information: CredentialInformation) => { - const client = await RetrieveOAuthClient(information); - return client.generateAuthUrl({ + return AuthorizationManager.generateAuthUrl({ access_type: 'offline', scope: SCOPES.map(relative => prefix + relative), }); @@ -103,14 +107,13 @@ export namespace GoogleApiServerUtils { name: string; } export const ProcessClientSideCode = async (information: CredentialInformation, authenticationCode: string): Promise => { - const oAuth2Client = await RetrieveOAuthClient(information); return new Promise((resolve, reject) => { - oAuth2Client.getToken(authenticationCode, async (err, token) => { + AuthorizationManager.getToken(authenticationCode, async (err, token) => { if (err || !token) { reject(err); return console.error('Error retrieving access token', err); } - oAuth2Client.setCredentials(token); + AuthorizationManager.setCredentials(token); const enriched = injectUserInfo(token); await Database.Auxiliary.GoogleAuthenticationToken.Write(information.userId, enriched); const { given_name, picture } = enriched.userInfo; diff --git a/src/server/database.ts b/src/server/database.ts index db86b472d..25e1e67e0 100644 --- a/src/server/database.ts +++ b/src/server/database.ts @@ -8,16 +8,19 @@ import { GoogleApiServerUtils } from './apis/google/GoogleApiServerUtils'; export namespace Database { + const schema = 'Dash'; + const port = 27017; + export const url = `mongodb://localhost:${port}/${schema}`; + class Database { public static DocumentsCollection = 'documents'; private MongoClient = mongodb.MongoClient; - private url = 'mongodb://localhost:27017/Dash'; private currentWrites: { [id: string]: Promise } = {}; private db?: mongodb.Db; private onConnect: (() => void)[] = []; constructor() { - this.MongoClient.connect(this.url, (err, client) => { + this.MongoClient.connect(url, (_err, client) => { this.db = client.db(); this.onConnect.forEach(fn => fn()); }); diff --git a/src/server/index.ts b/src/server/index.ts index 2203ae2e1..ad18857b6 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -1,1255 +1,1072 @@ require('dotenv').config(); -import * as bodyParser from 'body-parser'; -import { exec, ExecOptions } from 'child_process'; -import * as cookieParser from 'cookie-parser'; -import * as express from 'express'; -import * as session from 'express-session'; -import * as expressValidator from 'express-validator'; +import { exec } from 'child_process'; import * as formidable from 'formidable'; import * as fs from 'fs'; import * as sharp from 'sharp'; import * as Pdfjs from 'pdfjs-dist'; const imageDataUri = require('image-data-uri'); import * as mobileDetect from 'mobile-detect'; -import * as passport from 'passport'; import * as path from 'path'; import * as request from 'request'; -import * as io from 'socket.io'; +import io from 'socket.io'; import { Socket } from 'socket.io'; -import * as webpack from 'webpack'; -import * as wdm from 'webpack-dev-middleware'; -import * as whm from 'webpack-hot-middleware'; import { Utils } from '../Utils'; import { getForgot, getLogin, getLogout, getReset, getSignup, postForgot, postLogin, postReset, postSignup } from './authentication/controllers/user_controller'; -import { DashUserModel } from './authentication/models/user_model'; import { Client } from './Client'; import { Database } from './database'; import { MessageStore, Transferable, Types, Diff, YoutubeQueryTypes as YoutubeQueryType, YoutubeQueryInput } from "./Message"; import { RouteStore } from './RouteStore'; import v4 = require('uuid/v4'); -const app = express(); -const config = require('../../webpack.config'); import { createCanvas } from "canvas"; -const compiler = webpack(config); -const port = 1050; // default port to listen const serverPort = 4321; -import expressFlash = require('express-flash'); -import flash = require('connect-flash'); import { Search } from './Search'; import * as Archiver from 'archiver'; var AdmZip = require('adm-zip'); import * as YoutubeApi from "./apis/youtube/youtubeApiSample"; import { Response } from 'express-serve-static-core'; import { GoogleApiServerUtils } from "./apis/google/GoogleApiServerUtils"; -const MongoStore = require('connect-mongo')(session); -const mongoose = require('mongoose'); const probe = require("probe-image-size"); const pdf = require('pdf-parse'); var findInFiles = require('find-in-files'); import { GooglePhotosUploadUtils } from './apis/google/GooglePhotosUploadUtils'; -import * as qs from 'query-string'; import { Opt } from '../new_fields/Doc'; import { DashUploadUtils } from './DashUploadUtils'; import { BatchedArray, TimeUnit } from 'array-batcher'; import { ParsedPDF } from "./PdfTypes"; import { reject } from 'bluebird'; -import { ExifData } from 'exif'; -import { Result } from '../client/northstar/model/idea/idea'; import RouteSubscriber from './RouteSubscriber'; +import InitializeServer from './Initialization'; +import { Method, _success, _permission_denied, _error, _invalid } from './RouteManager'; +import { command_line, read_text_file } from './ActionUtilities'; -const download = (url: string, dest: fs.PathLike) => request.get(url).pipe(fs.createWriteStream(dest)); let youtubeApiKey: string; -YoutubeApi.readApiKey((apiKey: string) => youtubeApiKey = apiKey); -const release = process.env.RELEASE === "true"; -if (process.env.RELEASE === "true") { - console.log("Running server in release mode"); -} else { - console.log("Running server in debug mode"); -} -console.log(process.env.PWD); -let clientUtils = fs.readFileSync("./src/client/util/ClientUtils.ts.temp", "utf8"); -clientUtils = `//AUTO-GENERATED FILE: DO NOT EDIT\n${clientUtils.replace('"mode"', String(release))}`; -fs.writeFileSync("./src/client/util/ClientUtils.ts", clientUtils, "utf8"); - -const mongoUrl = 'mongodb://localhost:27017/Dash'; -mongoose.connection.readyState === 0 && mongoose.connect(mongoUrl); -mongoose.connection.on('connected', () => console.log("connected")); - -// SESSION MANAGEMENT AND AUTHENTICATION MIDDLEWARE -// ORDER OF IMPORTS MATTERS - -app.use(cookieParser()); -app.use(session({ - secret: "64d6866242d3b5a5503c675b32c9605e4e90478e9b77bcf2bc", - resave: true, - cookie: { maxAge: 7 * 24 * 60 * 60 * 1000 }, - saveUninitialized: true, - store: new MongoStore({ url: 'mongodb://localhost:27017/Dash' }) -})); - -app.use(flash()); -app.use(expressFlash()); -app.use(bodyParser.json({ limit: "10mb" })); -app.use(bodyParser.urlencoded({ extended: true })); -app.use(expressValidator()); -app.use(passport.initialize()); -app.use(passport.session()); -app.use((req, res, next) => { - res.locals.user = req.user; - next(); -}); - -app.get("/hello", (req, res) => res.send("

Hello

")); - -enum Method { - GET, - POST +export type Hierarchy = { [id: string]: string | Hierarchy }; +export type ZipMutator = (file: Archiver.Archiver) => void | Promise; + +export interface NewMediaItem { + description: string; + simpleMediaItem: { + uploadToken: string; + }; } -export type ValidationHandler = (user: DashUserModel, req: express.Request, res: express.Response) => any | Promise; -export type RejectionHandler = (req: express.Request, res: express.Response) => any | Promise; -export type ErrorHandler = (req: express.Request, res: express.Response, error: any) => any | Promise; +(async () => { + YoutubeApi.readApiKey((apiKey: string) => youtubeApiKey = apiKey); + await GoogleApiServerUtils.LoadOAuthClient(); + + await InitializeServer({ + listenAtPort: 1050, + routeSetter: router => { + + router.addSupervisedRoute({ + method: Method.GET, + subscription: "/pull", + onValidation: (_user, _req, res) => { + exec('"C:\\Program Files\\Git\\git-bash.exe" -c "git pull"', err => { + if (err) { + res.send(err.message); + return; + } + res.redirect("/"); + }); + } + }); -const LoginRedirect: RejectionHandler = (_req, res) => res.redirect(RouteStore.login); + router.addSupervisedRoute({ + method: Method.GET, + subscription: "/buxton", + onValidation: (_user, _req, res) => { + let cwd = '../scraping/buxton'; -export interface RouteInitializer { - method: Method; - subscribers: string | RouteSubscriber | (string | RouteSubscriber)[]; - onValidation: ValidationHandler; - onRejection?: RejectionHandler; - onError?: ErrorHandler; -} + let onResolved = (stdout: string) => { console.log(stdout); res.redirect("/"); }; + let onRejected = (err: any) => { console.error(err.message); res.send(err); }; + let tryPython3 = () => command_line('python3 scraper.py', cwd).then(onResolved, onRejected); -const isSharedDocAccess = (target: string) => { - const shared = qs.parse(qs.extract(target), { sort: false }).sharing === "true"; - const docAccess = target.startsWith("/doc/"); - return shared && docAccess; -}; - -/** - * Please invoke this function when adding a new route to Dash's server. - * It ensures that any requests leading to or containing user-sensitive information - * does not execute unless Passport authentication detects a user logged in. - * @param method whether or not the request is a GET or a POST - * @param handler the action to invoke, recieving a DashUserModel and, as expected, the Express.Request and Express.Response - * @param onRejection an optional callback invoked on return if no user is found to be logged in - * @param subscribers the forward slash prepended path names (reference and add to RouteStore.ts) that will all invoke the given @param handler - */ -function addSecureRoute(initializer: RouteInitializer) { - const { method, subscribers, onValidation, onRejection, onError } = initializer; - let abstracted = async (req: express.Request, res: express.Response) => { - const { user, originalUrl: target } = req; - if (user || isSharedDocAccess(target)) { - try { - await onValidation(user, req, res); - } catch (e) { - if (onError) { - onError(req, res, e); - } else { - _error(res, `The server encountered an internal error handling ${target}.`, e); - } - } - } else { - req.session!.target = target; - try { - await (onRejection || LoginRedirect)(req, res); - } catch (e) { - if (onError) { - onError(req, res, e); - } else { - _error(res, `The server encountered an internal error when rejecting ${target}.`, e); + command_line('python scraper.py', cwd).then(onResolved, tryPython3); + }, + }); + + router.addSupervisedRoute({ + method: Method.GET, + subscription: "/version", + onValidation: (_user, _req, res) => { + exec('"C:\\Program Files\\Git\\bin\\git.exe" rev-parse HEAD', (err, stdout) => { + if (err) { + res.send(err.message); + return; + } + res.send(stdout); + }); } - } - } - }; - const subscribe = (subscriber: RouteSubscriber | string) => { - let route: string; - if (typeof subscriber === "string") { - route = subscriber; - } else { - route = subscriber.build; - } - switch (method) { - case Method.GET: - app.get(route, abstracted); - break; - case Method.POST: - app.post(route, abstracted); - break; - } - }; - if (Array.isArray(subscribers)) { - subscribers.forEach(subscribe); - } else { - subscribe(subscribers); - } -} + }); -// STATIC FILE SERVING -app.use(express.static(__dirname + RouteStore.public)); -app.use(RouteStore.images, express.static(__dirname + RouteStore.public)); + // SEARCH + const solrURL = "http://localhost:8983/solr/#/dash"; -app.get("/pull", (req, res) => - exec('"C:\\Program Files\\Git\\git-bash.exe" -c "git pull"', (err, stdout, stderr) => { - if (err) { - res.send(err.message); - return; - } - res.redirect("/"); - })); - -app.get("/buxton", (req, res) => { - let cwd = '../scraping/buxton'; - - let onResolved = (stdout: string) => { console.log(stdout); res.redirect("/"); }; - let onRejected = (err: any) => { console.error(err.message); res.send(err); }; - let tryPython3 = () => command_line('python3 scraper.py', cwd).then(onResolved, onRejected); - - command_line('python scraper.py', cwd).then(onResolved, tryPython3); -}); - -const STATUS = { - OK: 200, - BAD_REQUEST: 400, - EXECUTION_ERROR: 500, - PERMISSION_DENIED: 403 -}; - -const command_line = (command: string, fromDirectory?: string) => { - return new Promise((resolve, reject) => { - let options: ExecOptions = {}; - if (fromDirectory) { - options.cwd = path.join(__dirname, fromDirectory); - } - exec(command, options, (err, stdout) => err ? reject(err) : resolve(stdout)); - }); -}; + // GETTERS -const read_text_file = (relativePath: string) => { - let target = path.join(__dirname, relativePath); - return new Promise((resolve, reject) => { - fs.readFile(target, (err, data) => err ? reject(err) : resolve(data.toString())); - }); -}; + DashServer.get("/textsearch", async (req, res) => { + let q = req.query.q; + if (q === undefined) { + res.send([]); + return; + } + let results = await findInFiles.find({ 'term': q, 'flags': 'ig' }, uploadDirectory + "text", ".txt$"); + let resObj: { ids: string[], numFound: number, lines: string[] } = { ids: [], numFound: 0, lines: [] }; + for (var result in results) { + resObj.ids.push(path.basename(result, ".txt").replace(/upload_/, "")); + resObj.lines.push(results[result].line); + resObj.numFound++; + } + res.send(resObj); + }); -const write_text_file = (relativePath: string, contents: any) => { - let target = path.join(__dirname, relativePath); - return new Promise((resolve, reject) => { - fs.writeFile(target, contents, (err) => err ? reject(err) : resolve()); - }); -}; + DashServer.get("/search", async (req, res) => { + const solrQuery: any = {}; + ["q", "fq", "start", "rows", "hl", "hl.fl"].forEach(key => solrQuery[key] = req.query[key]); + if (solrQuery.q === undefined) { + res.send([]); + return; + } + let results = await Search.Instance.search(solrQuery); + res.send(results); + }); -app.get("/version", (req, res) => { - exec('"C:\\Program Files\\Git\\bin\\git.exe" rev-parse HEAD', (err, stdout, stderr) => { - if (err) { - res.send(err.message); - return; - } - res.send(stdout); - }); -}); - -// SEARCH -const solrURL = "http://localhost:8983/solr/#/dash"; - -// GETTERS - -app.get("/textsearch", async (req, res) => { - let q = req.query.q; - if (q === undefined) { - res.send([]); - return; - } - let results = await findInFiles.find({ 'term': q, 'flags': 'ig' }, uploadDirectory + "text", ".txt$"); - let resObj: { ids: string[], numFound: number, lines: string[] } = { ids: [], numFound: 0, lines: [] }; - for (var result in results) { - resObj.ids.push(path.basename(result, ".txt").replace(/upload_/, "")); - resObj.lines.push(results[result].line); - resObj.numFound++; - } - res.send(resObj); -}); - -app.get("/search", async (req, res) => { - const solrQuery: any = {}; - ["q", "fq", "start", "rows", "hl", "hl.fl"].forEach(key => solrQuery[key] = req.query[key]); - if (solrQuery.q === undefined) { - res.send([]); - return; - } - let results = await Search.Instance.search(solrQuery); - res.send(results); -}); - -function msToTime(duration: number) { - let milliseconds = Math.floor((duration % 1000) / 100), - seconds = Math.floor((duration / 1000) % 60), - minutes = Math.floor((duration / (1000 * 60)) % 60), - hours = Math.floor((duration / (1000 * 60 * 60)) % 24); - - let hoursS = (hours < 10) ? "0" + hours : hours; - let minutesS = (minutes < 10) ? "0" + minutes : minutes; - let secondsS = (seconds < 10) ? "0" + seconds : seconds; - - return hoursS + ":" + minutesS + ":" + secondsS + "." + milliseconds; -} + function msToTime(duration: number) { + let milliseconds = Math.floor((duration % 1000) / 100), + seconds = Math.floor((duration / 1000) % 60), + minutes = Math.floor((duration / (1000 * 60)) % 60), + hours = Math.floor((duration / (1000 * 60 * 60)) % 24); -async function getDocs(id: string) { - const files = new Set(); - const docs: { [id: string]: any } = {}; - const fn = (doc: any): string[] => { - const id = doc.id; - if (typeof id === "string" && id.endsWith("Proto")) { - //Skip protos - return []; - } - const ids: string[] = []; - for (const key in doc.fields) { - if (!doc.fields.hasOwnProperty(key)) { - continue; + let hoursS = (hours < 10) ? "0" + hours : hours; + let minutesS = (minutes < 10) ? "0" + minutes : minutes; + let secondsS = (seconds < 10) ? "0" + seconds : seconds; + + return hoursS + ":" + minutesS + ":" + secondsS + "." + milliseconds; } - const field = doc.fields[key]; - if (field === undefined || field === null) { - continue; + + async function getDocs(id: string) { + const files = new Set(); + const docs: { [id: string]: any } = {}; + const fn = (doc: any): string[] => { + const id = doc.id; + if (typeof id === "string" && id.endsWith("Proto")) { + //Skip protos + return []; + } + const ids: string[] = []; + for (const key in doc.fields) { + if (!doc.fields.hasOwnProperty(key)) { + continue; + } + const field = doc.fields[key]; + if (field === undefined || field === null) { + continue; + } + + if (field.__type === "proxy" || field.__type === "prefetch_proxy") { + ids.push(field.fieldId); + } else if (field.__type === "script" || field.__type === "computed") { + if (field.captures) { + ids.push(field.captures.fieldId); + } + } else if (field.__type === "list") { + ids.push(...fn(field)); + } else if (typeof field === "string") { + const re = /"(?:dataD|d)ocumentId"\s*:\s*"([\w\-]*)"/g; + let match: string[] | null; + while ((match = re.exec(field)) !== null) { + ids.push(match[1]); + } + } else if (field.__type === "RichTextField") { + const re = /"href"\s*:\s*"(.*?)"/g; + let match: string[] | null; + while ((match = re.exec(field.Data)) !== null) { + const urlString = match[1]; + const split = new URL(urlString).pathname.split("doc/"); + if (split.length > 1) { + ids.push(split[split.length - 1]); + } + } + const re2 = /"src"\s*:\s*"(.*?)"/g; + while ((match = re2.exec(field.Data)) !== null) { + const urlString = match[1]; + const pathname = new URL(urlString).pathname; + files.add(pathname); + } + } else if (["audio", "image", "video", "pdf", "web"].includes(field.__type)) { + const url = new URL(field.url); + const pathname = url.pathname; + files.add(pathname); + } + } + + if (doc.id) { + docs[doc.id] = doc; + } + return ids; + }; + await Database.Instance.visit([id], fn); + return { id, docs, files }; } + DashServer.get("/serializeDoc/:docId", async (req, res) => { + const { docs, files } = await getDocs(req.params.docId); + res.send({ docs, files: Array.from(files) }); + }); - if (field.__type === "proxy" || field.__type === "prefetch_proxy") { - ids.push(field.fieldId); - } else if (field.__type === "script" || field.__type === "computed") { - if (field.captures) { - ids.push(field.captures.fieldId); + router.addSupervisedRoute({ + method: Method.GET, + subscription: new RouteSubscriber(RouteStore.imageHierarchyExport).add('docId'), + onValidation: async (_user, req, res) => { + const id = req.params.docId; + const hierarchy: Hierarchy = {}; + await targetedVisitorRecursive(id, hierarchy); + BuildAndDispatchZip(res, async zip => { + await hierarchyTraverserRecursive(zip, hierarchy); + }); } - } else if (field.__type === "list") { - ids.push(...fn(field)); - } else if (typeof field === "string") { - const re = /"(?:dataD|d)ocumentId"\s*:\s*"([\w\-]*)"/g; - let match: string[] | null; - while ((match = re.exec(field)) !== null) { - ids.push(match[1]); + }); + + const BuildAndDispatchZip = async (res: Response, mutator: ZipMutator): Promise => { + const zip = Archiver('zip'); + zip.pipe(res); + await mutator(zip); + return zip.finalize(); + }; + + const targetedVisitorRecursive = async (seedId: string, hierarchy: Hierarchy): Promise => { + const local: Hierarchy = {}; + const { title, data } = await getData(seedId); + const label = `${title} (${seedId})`; + if (Array.isArray(data)) { + hierarchy[label] = local; + await Promise.all(data.map(proxy => targetedVisitorRecursive(proxy.fieldId, local))); + } else { + hierarchy[label + path.extname(data)] = data; } - } else if (field.__type === "RichTextField") { - const re = /"href"\s*:\s*"(.*?)"/g; - let match: string[] | null; - while ((match = re.exec(field.Data)) !== null) { - const urlString = match[1]; - const split = new URL(urlString).pathname.split("doc/"); - if (split.length > 1) { - ids.push(split[split.length - 1]); + }; + + const getData = async (seedId: string): Promise<{ data: string | any[], title: string }> => { + return new Promise<{ data: string | any[], title: string }>((resolve, reject) => { + Database.Instance.getDocument(seedId, async (result: any) => { + const { data, proto, title } = result.fields; + if (data) { + if (data.url) { + resolve({ data: data.url, title }); + } else if (data.fields) { + resolve({ data: data.fields, title }); + } else { + reject(); + } + } + if (proto) { + getData(proto.fieldId).then(resolve, reject); + } + }); + }); + }; + + const hierarchyTraverserRecursive = async (file: Archiver.Archiver, hierarchy: Hierarchy, prefix = "Dash Export"): Promise => { + for (const key of Object.keys(hierarchy)) { + const result = hierarchy[key]; + if (typeof result === "string") { + let path: string; + let matches: RegExpExecArray | null; + if ((matches = /\:1050\/files\/(upload\_[\da-z]{32}.*)/g.exec(result)) !== null) { + path = `${__dirname}/public/files/${matches[1]}`; + } else { + const information = await DashUploadUtils.UploadImage(result); + path = information.mediaPaths[0]; + } + file.file(path, { name: key, prefix }); + } else { + await hierarchyTraverserRecursive(file, result, `${prefix}/${key}`); } } - const re2 = /"src"\s*:\s*"(.*?)"/g; - while ((match = re2.exec(field.Data)) !== null) { - const urlString = match[1]; - const pathname = new URL(urlString).pathname; - files.add(pathname); + }; + + DashServer.get("/downloadId/:docId", async (req, res) => { + res.set('Content-disposition', `attachment;`); + res.set('Content-Type', "application/zip"); + const { id, docs, files } = await getDocs(req.params.docId); + const docString = JSON.stringify({ id, docs }); + const zip = Archiver('zip'); + zip.pipe(res); + zip.append(docString, { name: "doc.json" }); + files.forEach(val => { + zip.file(__dirname + RouteStore.public + val, { name: val.substring(1) }); + }); + zip.finalize(); + }); + + DashServer.post("/uploadDoc", (req, res) => { + let form = new formidable.IncomingForm(); + form.keepExtensions = true; + // let path = req.body.path; + const ids: { [id: string]: string } = {}; + let remap = true; + const getId = (id: string): string => { + if (!remap) return id; + if (id.endsWith("Proto")) return id; + if (id in ids) { + return ids[id]; + } else { + return ids[id] = v4(); + } + }; + const mapFn = (doc: any) => { + if (doc.id) { + doc.id = getId(doc.id); + } + for (const key in doc.fields) { + if (!doc.fields.hasOwnProperty(key)) { + continue; + } + const field = doc.fields[key]; + if (field === undefined || field === null) { + continue; + } + + if (field.__type === "proxy" || field.__type === "prefetch_proxy") { + field.fieldId = getId(field.fieldId); + } else if (field.__type === "script" || field.__type === "computed") { + if (field.captures) { + field.captures.fieldId = getId(field.captures.fieldId); + } + } else if (field.__type === "list") { + mapFn(field); + } else if (typeof field === "string") { + const re = /("(?:dataD|d)ocumentId"\s*:\s*")([\w\-]*)"/g; + doc.fields[key] = (field as any).replace(re, (match: any, p1: string, p2: string) => { + return `${p1}${getId(p2)}"`; + }); + } else if (field.__type === "RichTextField") { + const re = /("href"\s*:\s*")(.*?)"/g; + field.Data = field.Data.replace(re, (match: any, p1: string, p2: string) => { + return `${p1}${getId(p2)}"`; + }); + } + } + }; + form.parse(req, async (err, fields, files) => { + remap = fields.remap !== "false"; + let id: string = ""; + try { + for (const name in files) { + const path_2 = files[name].path; + const zip = new AdmZip(path_2); + zip.getEntries().forEach((entry: any) => { + if (!entry.entryName.startsWith("files/")) return; + let dirname = path.dirname(entry.entryName) + "/"; + let extname = path.extname(entry.entryName); + let basename = path.basename(entry.entryName).split(".")[0]; + // zip.extractEntryTo(dirname + basename + "_o" + extname, __dirname + RouteStore.public, true, false); + // zip.extractEntryTo(dirname + basename + "_s" + extname, __dirname + RouteStore.public, true, false); + // zip.extractEntryTo(dirname + basename + "_m" + extname, __dirname + RouteStore.public, true, false); + // zip.extractEntryTo(dirname + basename + "_l" + extname, __dirname + RouteStore.public, true, false); + try { + zip.extractEntryTo(entry.entryName, __dirname + RouteStore.public, true, false); + dirname = "/" + dirname; + + fs.createReadStream(__dirname + RouteStore.public + dirname + basename + extname).pipe(fs.createWriteStream(__dirname + RouteStore.public + dirname + basename + "_o" + extname)); + fs.createReadStream(__dirname + RouteStore.public + dirname + basename + extname).pipe(fs.createWriteStream(__dirname + RouteStore.public + dirname + basename + "_s" + extname)); + fs.createReadStream(__dirname + RouteStore.public + dirname + basename + extname).pipe(fs.createWriteStream(__dirname + RouteStore.public + dirname + basename + "_m" + extname)); + fs.createReadStream(__dirname + RouteStore.public + dirname + basename + extname).pipe(fs.createWriteStream(__dirname + RouteStore.public + dirname + basename + "_l" + extname)); + } catch (e) { + console.log(e); + } + }); + const json = zip.getEntry("doc.json"); + let docs: any; + try { + let data = JSON.parse(json.getData().toString("utf8")); + docs = data.docs; + id = data.id; + docs = Object.keys(docs).map(key => docs[key]); + docs.forEach(mapFn); + await Promise.all(docs.map((doc: any) => new Promise(res => Database.Instance.replace(doc.id, doc, (err, r) => { + err && console.log(err); + res(); + }, true, "newDocuments")))); + } catch (e) { console.log(e); } + fs.unlink(path_2, () => { }); + } + if (id) { + res.send(JSON.stringify(getId(id))); + } else { + res.send(JSON.stringify("error")); + } + } catch (e) { console.log(e); } + }); + }); + + DashServer.get("/whosOnline", (req, res) => { + let users: any = { active: {}, inactive: {} }; + const now = Date.now(); + + for (const user in timeMap) { + const time = timeMap[user]; + const key = ((now - time) / 1000) < (60 * 5) ? "active" : "inactive"; + users[key][user] = `Last active ${msToTime(now - time)} ago`; } - } else if (["audio", "image", "video", "pdf", "web"].includes(field.__type)) { - const url = new URL(field.url); - const pathname = url.pathname; - files.add(pathname); + + res.send(users); + }); + DashServer.get("/thumbnail/:filename", (req, res) => { + let filename = req.params.filename; + let noExt = filename.substring(0, filename.length - ".png".length); + let pagenumber = parseInt(noExt.split('-')[1]); + fs.exists(uploadDirectory + filename, (exists: boolean) => { + console.log(`${uploadDirectory + filename} ${exists ? "exists" : "does not exist"}`); + if (exists) { + let input = fs.createReadStream(uploadDirectory + filename); + probe(input, (err: any, result: any) => { + if (err) { + console.log(err); + console.log(`error on ${filename}`); + return; + } + res.send({ path: "/files/" + filename, width: result.width, height: result.height }); + }); + } + else { + LoadPage(uploadDirectory + filename.substring(0, filename.length - noExt.split('-')[1].length - ".PNG".length - 1) + ".pdf", pagenumber, res); + } + }); + }); + + function LoadPage(file: string, pageNumber: number, res: Response) { + console.log(file); + Pdfjs.getDocument(file).promise + .then((pdf: Pdfjs.PDFDocumentProxy) => { + let factory = new NodeCanvasFactory(); + console.log(pageNumber); + pdf.getPage(pageNumber).then((page: Pdfjs.PDFPageProxy) => { + console.log("reading " + page); + let viewport = page.getViewport(1 as any); + let canvasAndContext = factory.create(viewport.width, viewport.height); + let renderContext = { + canvasContext: canvasAndContext.context, + viewport: viewport, + canvasFactory: factory + }; + console.log("read " + pageNumber); + + page.render(renderContext).promise + .then(() => { + console.log("saving " + pageNumber); + let stream = canvasAndContext.canvas.createPNGStream(); + let pngFile = `${file.substring(0, file.length - ".pdf".length)}-${pageNumber}.PNG`; + let out = fs.createWriteStream(pngFile); + stream.pipe(out); + out.on("finish", () => { + console.log(`Success! Saved to ${pngFile}`); + let name = path.basename(pngFile); + res.send({ path: "/files/" + name, width: viewport.width, height: viewport.height }); + }); + }, (reason: string) => { + console.error(reason + ` ${pageNumber}`); + }); + }); + }); } - } - if (doc.id) { - docs[doc.id] = doc; - } - return ids; - }; - await Database.Instance.visit([id], fn); - return { id, docs, files }; -} -app.get("/serializeDoc/:docId", async (req, res) => { - const { docs, files } = await getDocs(req.params.docId); - res.send({ docs, files: Array.from(files) }); -}); + /** + * Anyone attempting to navigate to localhost at this port will + * first have to log in. + */ + router.addSupervisedRoute({ + method: Method.GET, + subscription: RouteStore.root, + onValidation: (_user, _req, res) => res.redirect(RouteStore.home) + }); -export type Hierarchy = { [id: string]: string | Hierarchy }; -export type ZipMutator = (file: Archiver.Archiver) => void | Promise; + router.addSupervisedRoute({ + method: Method.GET, + subscription: RouteStore.getUsers, + onValidation: async (_user, _req, res) => { + const cursor = await Database.Instance.query({}, { email: 1, userDocumentId: 1 }, "users"); + const results = await cursor.toArray(); + res.send(results.map(user => ({ email: user.email, userDocumentId: user.userDocumentId }))); + }, + }); -addSecureRoute({ - method: Method.GET, - subscribers: new RouteSubscriber(RouteStore.imageHierarchyExport).add('docId'), - onValidation: async (_user, req, res) => { - const id = req.params.docId; - const hierarchy: Hierarchy = {}; - await targetedVisitorRecursive(id, hierarchy); - BuildAndDispatchZip(res, async zip => { - await hierarchyTraverserRecursive(zip, hierarchy); - }); - } -}); - -const BuildAndDispatchZip = async (res: Response, mutator: ZipMutator): Promise => { - const zip = Archiver('zip'); - zip.pipe(res); - await mutator(zip); - return zip.finalize(); -}; - -const targetedVisitorRecursive = async (seedId: string, hierarchy: Hierarchy): Promise => { - const local: Hierarchy = {}; - const { title, data } = await getData(seedId); - const label = `${title} (${seedId})`; - if (Array.isArray(data)) { - hierarchy[label] = local; - await Promise.all(data.map(proxy => targetedVisitorRecursive(proxy.fieldId, local))); - } else { - hierarchy[label + path.extname(data)] = data; - } -}; - -const getData = async (seedId: string): Promise<{ data: string | any[], title: string }> => { - return new Promise<{ data: string | any[], title: string }>((resolve, reject) => { - Database.Instance.getDocument(seedId, async (result: any) => { - const { data, proto, title } = result.fields; - if (data) { - if (data.url) { - resolve({ data: data.url, title }); - } else if (data.fields) { - resolve({ data: data.fields, title }); - } else { - reject(); + router.addSupervisedRoute({ + method: Method.GET, + subscription: [RouteStore.home, RouteStore.openDocumentWithId], + onValidation: (_user, req, res) => { + let detector = new mobileDetect(req.headers['user-agent'] || ""); + let filename = detector.mobile() !== null ? 'mobile/image.html' : 'index.html'; + res.sendFile(path.join(__dirname, '../../deploy/' + filename)); + }, + }); + + router.addSupervisedRoute({ + method: Method.GET, + subscription: RouteStore.getUserDocumentId, + onValidation: (user, _req, res) => res.send(user.userDocumentId), + onRejection: (_req, res) => res.send(undefined) + }); + + router.addSupervisedRoute({ + method: Method.GET, + subscription: RouteStore.getCurrUser, + onValidation: (user, _req, res) => { res.send(JSON.stringify(user)); }, + onRejection: (_req, res) => res.send(JSON.stringify({ id: "__guest__", email: "" })) + }); + + const ServicesApiKeyMap = new Map([ + ["face", process.env.FACE], + ["vision", process.env.VISION], + ["handwriting", process.env.HANDWRITING] + ]); + + router.addSupervisedRoute({ + method: Method.GET, + subscription: new RouteSubscriber(RouteStore.cognitiveServices).add('requestedservice'), + onValidation: (_user, req, res) => { + let service = req.params.requestedservice; + res.send(ServicesApiKeyMap.get(service)); + } + }); + + class NodeCanvasFactory { + create = (width: number, height: number) => { + var canvas = createCanvas(width, height); + var context = canvas.getContext('2d'); + return { + canvas: canvas, + context: context, + }; + } + + reset = (canvasAndContext: any, width: number, height: number) => { + canvasAndContext.canvas.width = width; + canvasAndContext.canvas.height = height; + } + + destroy = (canvasAndContext: any) => { + canvasAndContext.canvas.width = 0; + canvasAndContext.canvas.height = 0; + canvasAndContext.canvas = null; + canvasAndContext.context = null; } } - if (proto) { - getData(proto.fieldId).then(resolve, reject); - } - }); - }); -}; - -const hierarchyTraverserRecursive = async (file: Archiver.Archiver, hierarchy: Hierarchy, prefix = "Dash Export"): Promise => { - for (const key of Object.keys(hierarchy)) { - const result = hierarchy[key]; - if (typeof result === "string") { - let path: string; - let matches: RegExpExecArray | null; - if ((matches = /\:1050\/files\/(upload\_[\da-z]{32}.*)/g.exec(result)) !== null) { - path = `${__dirname}/public/files/${matches[1]}`; - } else { - const information = await DashUploadUtils.UploadImage(result); - path = information.mediaPaths[0]; - } - file.file(path, { name: key, prefix }); - } else { - await hierarchyTraverserRecursive(file, result, `${prefix}/${key}`); - } - } -}; - -app.get("/downloadId/:docId", async (req, res) => { - res.set('Content-disposition', `attachment;`); - res.set('Content-Type', "application/zip"); - const { id, docs, files } = await getDocs(req.params.docId); - const docString = JSON.stringify({ id, docs }); - const zip = Archiver('zip'); - zip.pipe(res); - zip.append(docString, { name: "doc.json" }); - files.forEach(val => { - zip.file(__dirname + RouteStore.public + val, { name: val.substring(1) }); - }); - zip.finalize(); -}); - -app.post("/uploadDoc", (req, res) => { - let form = new formidable.IncomingForm(); - form.keepExtensions = true; - // let path = req.body.path; - const ids: { [id: string]: string } = {}; - let remap = true; - const getId = (id: string): string => { - if (!remap) return id; - if (id.endsWith("Proto")) return id; - if (id in ids) { - return ids[id]; - } else { - return ids[id] = v4(); - } - }; - const mapFn = (doc: any) => { - if (doc.id) { - doc.id = getId(doc.id); - } - for (const key in doc.fields) { - if (!doc.fields.hasOwnProperty(key)) { - continue; - } - const field = doc.fields[key]; - if (field === undefined || field === null) { - continue; + + const pngTypes = [".png", ".PNG"]; + const jpgTypes = [".jpg", ".JPG", ".jpeg", ".JPEG"]; + const uploadDirectory = __dirname + "/public/files/"; + const pdfDirectory = uploadDirectory + "text"; + DashUploadUtils.createIfNotExists(pdfDirectory); + + interface ImageFileResponse { + name: string; + path: string; + type: string; + exif: Opt; } - if (field.__type === "proxy" || field.__type === "prefetch_proxy") { - field.fieldId = getId(field.fieldId); - } else if (field.__type === "script" || field.__type === "computed") { - if (field.captures) { - field.captures.fieldId = getId(field.captures.fieldId); + router.addSupervisedRoute({ + method: Method.POST, + subscription: RouteStore.upload, + onValidation: (_user, req, res) => { + let form = new formidable.IncomingForm(); + form.uploadDir = uploadDirectory; + form.keepExtensions = true; + form.parse(req, async (_err, _fields, files) => { + let results: ImageFileResponse[] = []; + for (const key in files) { + const { type, path: location, name } = files[key]; + const filename = path.basename(location); + let uploadInformation: Opt; + if (filename.endsWith(".pdf")) { + let dataBuffer = fs.readFileSync(uploadDirectory + filename); + const result: ParsedPDF = await pdf(dataBuffer); + await new Promise(resolve => { + const path = pdfDirectory + "/" + filename.substring(0, filename.length - ".pdf".length) + ".txt"; + fs.createWriteStream(path).write(result.text, error => { + if (!error) { + resolve(); + } else { + reject(error); + } + }); + }); + } else { + uploadInformation = await DashUploadUtils.UploadImage(uploadDirectory + filename, filename); + } + const exif = uploadInformation ? uploadInformation.exifData : undefined; + results.push({ name, type, path: `/files/${filename}`, exif }); + + } + _success(res, results); + }); } - } else if (field.__type === "list") { - mapFn(field); - } else if (typeof field === "string") { - const re = /("(?:dataD|d)ocumentId"\s*:\s*")([\w\-]*)"/g; - doc.fields[key] = (field as any).replace(re, (match: any, p1: string, p2: string) => { - return `${p1}${getId(p2)}"`; - }); - } else if (field.__type === "RichTextField") { - const re = /("href"\s*:\s*")(.*?)"/g; - field.Data = field.Data.replace(re, (match: any, p1: string, p2: string) => { - return `${p1}${getId(p2)}"`; - }); - } - } - }; - form.parse(req, async (err, fields, files) => { - remap = fields.remap !== "false"; - let id: string = ""; - try { - for (const name in files) { - const path_2 = files[name].path; - const zip = new AdmZip(path_2); - zip.getEntries().forEach((entry: any) => { - if (!entry.entryName.startsWith("files/")) return; - let dirname = path.dirname(entry.entryName) + "/"; - let extname = path.extname(entry.entryName); - let basename = path.basename(entry.entryName).split(".")[0]; - // zip.extractEntryTo(dirname + basename + "_o" + extname, __dirname + RouteStore.public, true, false); - // zip.extractEntryTo(dirname + basename + "_s" + extname, __dirname + RouteStore.public, true, false); - // zip.extractEntryTo(dirname + basename + "_m" + extname, __dirname + RouteStore.public, true, false); - // zip.extractEntryTo(dirname + basename + "_l" + extname, __dirname + RouteStore.public, true, false); - try { - zip.extractEntryTo(entry.entryName, __dirname + RouteStore.public, true, false); - dirname = "/" + dirname; - - fs.createReadStream(__dirname + RouteStore.public + dirname + basename + extname).pipe(fs.createWriteStream(__dirname + RouteStore.public + dirname + basename + "_o" + extname)); - fs.createReadStream(__dirname + RouteStore.public + dirname + basename + extname).pipe(fs.createWriteStream(__dirname + RouteStore.public + dirname + basename + "_s" + extname)); - fs.createReadStream(__dirname + RouteStore.public + dirname + basename + extname).pipe(fs.createWriteStream(__dirname + RouteStore.public + dirname + basename + "_m" + extname)); - fs.createReadStream(__dirname + RouteStore.public + dirname + basename + extname).pipe(fs.createWriteStream(__dirname + RouteStore.public + dirname + basename + "_l" + extname)); - } catch (e) { - console.log(e); + }); + + router.addSupervisedRoute({ + method: Method.POST, + subscription: RouteStore.inspectImage, + onValidation: async (_user, req, res) => { + const { source } = req.body; + if (typeof source === "string") { + const uploadInformation = await DashUploadUtils.UploadImage(source); + return res.send(await DashUploadUtils.InspectImage(uploadInformation.mediaPaths[0])); } - }); - const json = zip.getEntry("doc.json"); - let docs: any; - try { - let data = JSON.parse(json.getData().toString("utf8")); - docs = data.docs; - id = data.id; - docs = Object.keys(docs).map(key => docs[key]); - docs.forEach(mapFn); - await Promise.all(docs.map((doc: any) => new Promise(res => Database.Instance.replace(doc.id, doc, (err, r) => { - err && console.log(err); - res(); - }, true, "newDocuments")))); - } catch (e) { console.log(e); } - fs.unlink(path_2, () => { }); - } - if (id) { - res.send(JSON.stringify(getId(id))); - } else { - res.send(JSON.stringify("error")); - } - } catch (e) { console.log(e); } - }); -}); - -app.get("/whosOnline", (req, res) => { - let users: any = { active: {}, inactive: {} }; - const now = Date.now(); - - for (const user in timeMap) { - const time = timeMap[user]; - const key = ((now - time) / 1000) < (60 * 5) ? "active" : "inactive"; - users[key][user] = `Last active ${msToTime(now - time)} ago`; - } - - res.send(users); -}); -app.get("/thumbnail/:filename", (req, res) => { - let filename = req.params.filename; - let noExt = filename.substring(0, filename.length - ".png".length); - let pagenumber = parseInt(noExt.split('-')[1]); - fs.exists(uploadDirectory + filename, (exists: boolean) => { - console.log(`${uploadDirectory + filename} ${exists ? "exists" : "does not exist"}`); - if (exists) { - let input = fs.createReadStream(uploadDirectory + filename); - probe(input, (err: any, result: any) => { - if (err) { - console.log(err); - console.log(`error on ${filename}`); - return; + res.send({}); } - res.send({ path: "/files/" + filename, width: result.width, height: result.height }); }); - } - else { - LoadPage(uploadDirectory + filename.substring(0, filename.length - noExt.split('-')[1].length - ".PNG".length - 1) + ".pdf", pagenumber, res); - } - }); -}); - -function LoadPage(file: string, pageNumber: number, res: Response) { - console.log(file); - Pdfjs.getDocument(file).promise - .then((pdf: Pdfjs.PDFDocumentProxy) => { - let factory = new NodeCanvasFactory(); - console.log(pageNumber); - pdf.getPage(pageNumber).then((page: Pdfjs.PDFPageProxy) => { - console.log("reading " + page); - let viewport = page.getViewport(1 as any); - let canvasAndContext = factory.create(viewport.width, viewport.height); - let renderContext = { - canvasContext: canvasAndContext.context, - viewport: viewport, - canvasFactory: factory - }; - console.log("read " + pageNumber); - - page.render(renderContext).promise - .then(() => { - console.log("saving " + pageNumber); - let stream = canvasAndContext.canvas.createPNGStream(); - let pngFile = `${file.substring(0, file.length - ".pdf".length)}-${pageNumber}.PNG`; - let out = fs.createWriteStream(pngFile); - stream.pipe(out); - out.on("finish", () => { - console.log(`Success! Saved to ${pngFile}`); - let name = path.basename(pngFile); - res.send({ path: "/files/" + name, width: viewport.width, height: viewport.height }); - }); - }, (reason: string) => { - console.error(reason + ` ${pageNumber}`); + + router.addSupervisedRoute({ + method: Method.POST, + subscription: RouteStore.dataUriToImage, + onValidation: (_user, req, res) => { + const uri = req.body.uri; + const filename = req.body.name; + if (!uri || !filename) { + res.status(401).send("incorrect parameters specified"); + return; + } + imageDataUri.outputFile(uri, uploadDirectory + filename).then((savedName: string) => { + const ext = path.extname(savedName); + let resizers = [ + { resizer: sharp().resize(100, undefined, { withoutEnlargement: true }), suffix: "_s" }, + { resizer: sharp().resize(400, undefined, { withoutEnlargement: true }), suffix: "_m" }, + { resizer: sharp().resize(900, undefined, { withoutEnlargement: true }), suffix: "_l" }, + ]; + let isImage = false; + if (pngTypes.includes(ext)) { + resizers.forEach(element => { + element.resizer = element.resizer.png(); + }); + isImage = true; + } else if (jpgTypes.includes(ext)) { + resizers.forEach(element => { + element.resizer = element.resizer.jpeg(); + }); + isImage = true; + } + if (isImage) { + resizers.forEach(resizer => { + fs.createReadStream(savedName).pipe(resizer.resizer).pipe(fs.createWriteStream(uploadDirectory + filename + resizer.suffix + ext)); + }); + } + res.send("/files/" + filename + ext); }); + } }); - }); -} -/** - * Anyone attempting to navigate to localhost at this port will - * first have to log in. - */ -addSecureRoute({ - method: Method.GET, - subscribers: RouteStore.root, - onValidation: (_user, _req, res) => res.redirect(RouteStore.home) -}); - -addSecureRoute({ - method: Method.GET, - subscribers: RouteStore.getUsers, - onValidation: async (_user, _req, res) => { - const cursor = await Database.Instance.query({}, { email: 1, userDocumentId: 1 }, "users"); - const results = await cursor.toArray(); - res.send(results.map(user => ({ email: user.email, userDocumentId: user.userDocumentId }))); - }, -}); - -addSecureRoute({ - method: Method.GET, - subscribers: [RouteStore.home, RouteStore.openDocumentWithId], - onValidation: (_user, req, res) => { - let detector = new mobileDetect(req.headers['user-agent'] || ""); - let filename = detector.mobile() !== null ? 'mobile/image.html' : 'index.html'; - res.sendFile(path.join(__dirname, '../../deploy/' + filename)); - }, -}); - -addSecureRoute({ - method: Method.GET, - subscribers: RouteStore.getUserDocumentId, - onValidation: (user, _req, res) => res.send(user.userDocumentId), - onRejection: (_req, res) => res.send(undefined) -}); - -addSecureRoute({ - method: Method.GET, - subscribers: RouteStore.getCurrUser, - onValidation: (user, _req, res) => { res.send(JSON.stringify(user)); }, - onRejection: (_req, res) => res.send(JSON.stringify({ id: "__guest__", email: "" })) -}); - -const ServicesApiKeyMap = new Map([ - ["face", process.env.FACE], - ["vision", process.env.VISION], - ["handwriting", process.env.HANDWRITING] -]); - -addSecureRoute({ - method: Method.GET, - subscribers: new RouteSubscriber(RouteStore.cognitiveServices).add('requestedservice'), - onValidation: (_user, req, res) => { - let service = req.params.requestedservice; - res.send(ServicesApiKeyMap.get(service)); - } -}); - -class NodeCanvasFactory { - create = (width: number, height: number) => { - var canvas = createCanvas(width, height); - var context = canvas.getContext('2d'); - return { - canvas: canvas, - context: context, - }; - } - - reset = (canvasAndContext: any, width: number, height: number) => { - canvasAndContext.canvas.width = width; - canvasAndContext.canvas.height = height; - } - - destroy = (canvasAndContext: any) => { - canvasAndContext.canvas.width = 0; - canvasAndContext.canvas.height = 0; - canvasAndContext.canvas = null; - canvasAndContext.context = null; - } -} - -const pngTypes = [".png", ".PNG"]; -const jpgTypes = [".jpg", ".JPG", ".jpeg", ".JPEG"]; -const uploadDirectory = __dirname + "/public/files/"; -const pdfDirectory = uploadDirectory + "text"; -DashUploadUtils.createIfNotExists(pdfDirectory); - -interface ImageFileResponse { - name: string; - path: string; - type: string; - exif: Opt; -} - -addSecureRoute({ - method: Method.POST, - subscribers: RouteStore.upload, - onValidation: (_user, req, res) => { - let form = new formidable.IncomingForm(); - form.uploadDir = uploadDirectory; - form.keepExtensions = true; - form.parse(req, async (_err, _fields, files) => { - let results: ImageFileResponse[] = []; - for (const key in files) { - const { type, path: location, name } = files[key]; - const filename = path.basename(location); - let uploadInformation: Opt; - if (filename.endsWith(".pdf")) { - let dataBuffer = fs.readFileSync(uploadDirectory + filename); - const result: ParsedPDF = await pdf(dataBuffer); - await new Promise(resolve => { - const path = pdfDirectory + "/" + filename.substring(0, filename.length - ".pdf".length) + ".txt"; - fs.createWriteStream(path).write(result.text, error => { - if (!error) { - resolve(); - } else { - reject(error); + // AUTHENTICATION + + // Sign Up + DashServer.get(RouteStore.signup, getSignup); + DashServer.post(RouteStore.signup, postSignup); + + // Log In + DashServer.get(RouteStore.login, getLogin); + DashServer.post(RouteStore.login, postLogin); + + // Log Out + DashServer.get(RouteStore.logout, getLogout); + + // FORGOT PASSWORD EMAIL HANDLING + DashServer.get(RouteStore.forgot, getForgot); + DashServer.post(RouteStore.forgot, postForgot); + + // RESET PASSWORD EMAIL HANDLING + DashServer.get(RouteStore.reset, getReset); + DashServer.post(RouteStore.reset, postReset); + + const headerCharRegex = /[^\t\x20-\x7e\x80-\xff]/; + DashServer.use(RouteStore.corsProxy, (req, res) => { + req.pipe(request(decodeURIComponent(req.url.substring(1)))).on("response", res => { + const headers = Object.keys(res.headers); + headers.forEach(headerName => { + const header = res.headers[headerName]; + if (Array.isArray(header)) { + res.headers[headerName] = header.filter(h => !headerCharRegex.test(h)); + } else if (header) { + if (headerCharRegex.test(header as any)) { + delete res.headers[headerName]; } - }); + } }); - } else { - uploadInformation = await DashUploadUtils.UploadImage(uploadDirectory + filename, filename); + }).pipe(res); + }); + + router.addSupervisedRoute({ + method: Method.GET, + subscription: RouteStore.delete, + onValidation: (_user, _req, res, isRelease) => { + if (isRelease) { + return _permission_denied(res, deletionPermissionError); + } + deleteFields().then(() => res.redirect(RouteStore.home)); } - const exif = uploadInformation ? uploadInformation.exifData : undefined; - results.push({ name, type, path: `/files/${filename}`, exif }); + }); + router.addSupervisedRoute({ + method: Method.GET, + subscription: RouteStore.deleteAll, + onValidation: (_user, _req, res, isRelease) => { + if (isRelease) { + return _permission_denied(res, deletionPermissionError); + } + deleteAll().then(() => res.redirect(RouteStore.home)); + } + }); + + const server = io(); + interface Map { + [key: string]: Client; } - _success(res, results); - }); - } -}); - -addSecureRoute({ - method: Method.POST, - subscribers: RouteStore.inspectImage, - onValidation: async (_user, req, res) => { - const { source } = req.body; - if (typeof source === "string") { - const uploadInformation = await DashUploadUtils.UploadImage(source); - return res.send(await DashUploadUtils.InspectImage(uploadInformation.mediaPaths[0])); - } - res.send({}); - } -}); - -addSecureRoute({ - method: Method.POST, - subscribers: RouteStore.dataUriToImage, - onValidation: (_user, req, res) => { - const uri = req.body.uri; - const filename = req.body.name; - if (!uri || !filename) { - res.status(401).send("incorrect parameters specified"); - return; - } - imageDataUri.outputFile(uri, uploadDirectory + filename).then((savedName: string) => { - const ext = path.extname(savedName); - let resizers = [ - { resizer: sharp().resize(100, undefined, { withoutEnlargement: true }), suffix: "_s" }, - { resizer: sharp().resize(400, undefined, { withoutEnlargement: true }), suffix: "_m" }, - { resizer: sharp().resize(900, undefined, { withoutEnlargement: true }), suffix: "_l" }, - ]; - let isImage = false; - if (pngTypes.includes(ext)) { - resizers.forEach(element => { - element.resizer = element.resizer.png(); - }); - isImage = true; - } else if (jpgTypes.includes(ext)) { - resizers.forEach(element => { - element.resizer = element.resizer.jpeg(); + let clients: Map = {}; + + let socketMap = new Map(); + let timeMap: { [id: string]: number } = {}; + + server.on("connection", function (socket: Socket) { + socket.use((packet, next) => { + let id = socketMap.get(socket); + if (id) { + timeMap[id] = Date.now(); + } + next(); }); - isImage = true; + + Utils.Emit(socket, MessageStore.Foo, "handshooken"); + + Utils.AddServerHandler(socket, MessageStore.Bar, guid => barReceived(socket, guid)); + Utils.AddServerHandler(socket, MessageStore.SetField, (args) => setField(socket, args)); + Utils.AddServerHandlerCallback(socket, MessageStore.GetField, getField); + Utils.AddServerHandlerCallback(socket, MessageStore.GetFields, getFields); + if (!router.release) { + Utils.AddServerHandler(socket, MessageStore.DeleteAll, deleteFields); + } + + Utils.AddServerHandler(socket, MessageStore.CreateField, CreateField); + Utils.AddServerHandlerCallback(socket, MessageStore.YoutubeApiQuery, HandleYoutubeQuery); + Utils.AddServerHandler(socket, MessageStore.UpdateField, diff => UpdateField(socket, diff)); + Utils.AddServerHandler(socket, MessageStore.DeleteField, id => DeleteField(socket, id)); + Utils.AddServerHandler(socket, MessageStore.DeleteFields, ids => DeleteFields(socket, ids)); + Utils.AddServerHandlerCallback(socket, MessageStore.GetRefField, GetRefField); + Utils.AddServerHandlerCallback(socket, MessageStore.GetRefFields, GetRefFields); + }); + + async function deleteFields() { + await Database.Instance.deleteAll(); + await Search.Instance.clear(); + await Database.Instance.deleteAll('newDocuments'); } - if (isImage) { - resizers.forEach(resizer => { - fs.createReadStream(savedName).pipe(resizer.resizer).pipe(fs.createWriteStream(uploadDirectory + filename + resizer.suffix + ext)); - }); + + async function deleteAll() { + await Database.Instance.deleteAll(); + await Database.Instance.deleteAll('newDocuments'); + await Database.Instance.deleteAll('sessions'); + await Database.Instance.deleteAll('users'); + await Search.Instance.clear(); } - res.send("/files/" + filename + ext); - }); - } -}); - -// AUTHENTICATION - -// Sign Up -app.get(RouteStore.signup, getSignup); -app.post(RouteStore.signup, postSignup); - -// Log In -app.get(RouteStore.login, getLogin); -app.post(RouteStore.login, postLogin); - -// Log Out -app.get(RouteStore.logout, getLogout); - -// FORGOT PASSWORD EMAIL HANDLING -app.get(RouteStore.forgot, getForgot); -app.post(RouteStore.forgot, postForgot); - -// RESET PASSWORD EMAIL HANDLING -app.get(RouteStore.reset, getReset); -app.post(RouteStore.reset, postReset); - -const headerCharRegex = /[^\t\x20-\x7e\x80-\xff]/; -app.use(RouteStore.corsProxy, (req, res) => { - req.pipe(request(decodeURIComponent(req.url.substring(1)))).on("response", res => { - const headers = Object.keys(res.headers); - headers.forEach(headerName => { - const header = res.headers[headerName]; - if (Array.isArray(header)) { - res.headers[headerName] = header.filter(h => !headerCharRegex.test(h)); - } else if (header) { - if (headerCharRegex.test(header as any)) { - delete res.headers[headerName]; - } + + function barReceived(socket: SocketIO.Socket, guid: string) { + clients[guid] = new Client(guid.toString()); + console.log(`User ${guid} has connected`); + socketMap.set(socket, guid); } - }); - }).pipe(res); -}); - -addSecureRoute({ - method: Method.GET, - subscribers: RouteStore.delete, - onValidation: (_user, _req, res) => { - if (release) { - return _permission_denied(res, deletionPermissionError); - } - deleteFields().then(() => res.redirect(RouteStore.home)); - } -}); - -addSecureRoute({ - method: Method.GET, - subscribers: RouteStore.deleteAll, - onValidation: (_user, _req, res) => { - if (release) { - return _permission_denied(res, deletionPermissionError); - } - deleteAll().then(() => res.redirect(RouteStore.home)); - } -}); -app.use(wdm(compiler, { publicPath: config.output.publicPath })); + function getField([id, callback]: [string, (result?: Transferable) => void]) { + Database.Instance.getDocument(id, (result?: Transferable) => + callback(result ? result : undefined)); + } -app.use(whm(compiler)); + function getFields([ids, callback]: [string[], (result: Transferable[]) => void]) { + Database.Instance.getDocuments(ids, callback); + } -// start the Express server -app.listen(port, () => - console.log(`server started at http://localhost:${port}`)); + function setField(socket: Socket, newValue: Transferable) { + Database.Instance.update(newValue.id, newValue, () => + socket.broadcast.emit(MessageStore.SetField.Message, newValue)); + if (newValue.type === Types.Text) { + Search.Instance.updateDocument({ id: newValue.id, data: (newValue as any).data }); + console.log("set field"); + console.log("checking in"); + } + } -const server = io(); -interface Map { - [key: string]: Client; -} -let clients: Map = {}; + function GetRefField([id, callback]: [string, (result?: Transferable) => void]) { + Database.Instance.getDocument(id, callback, "newDocuments"); + } -let socketMap = new Map(); -let timeMap: { [id: string]: number } = {}; + function GetRefFields([ids, callback]: [string[], (result?: Transferable[]) => void]) { + Database.Instance.getDocuments(ids, callback, "newDocuments"); + } -server.on("connection", function (socket: Socket) { - socket.use((packet, next) => { - let id = socketMap.get(socket); - if (id) { - timeMap[id] = Date.now(); - } - next(); - }); + function HandleYoutubeQuery([query, callback]: [YoutubeQueryInput, (result?: any[]) => void]) { + switch (query.type) { + case YoutubeQueryType.Channels: + YoutubeApi.authorizedGetChannel(youtubeApiKey); + break; + case YoutubeQueryType.SearchVideo: + YoutubeApi.authorizedGetVideos(youtubeApiKey, query.userInput, callback); + case YoutubeQueryType.VideoDetails: + YoutubeApi.authorizedGetVideoDetails(youtubeApiKey, query.videoIds, callback); + } + } - Utils.Emit(socket, MessageStore.Foo, "handshooken"); - - Utils.AddServerHandler(socket, MessageStore.Bar, guid => barReceived(socket, guid)); - Utils.AddServerHandler(socket, MessageStore.SetField, (args) => setField(socket, args)); - Utils.AddServerHandlerCallback(socket, MessageStore.GetField, getField); - Utils.AddServerHandlerCallback(socket, MessageStore.GetFields, getFields); - if (!release) { - Utils.AddServerHandler(socket, MessageStore.DeleteAll, deleteFields); - } - - Utils.AddServerHandler(socket, MessageStore.CreateField, CreateField); - Utils.AddServerHandlerCallback(socket, MessageStore.YoutubeApiQuery, HandleYoutubeQuery); - Utils.AddServerHandler(socket, MessageStore.UpdateField, diff => UpdateField(socket, diff)); - Utils.AddServerHandler(socket, MessageStore.DeleteField, id => DeleteField(socket, id)); - Utils.AddServerHandler(socket, MessageStore.DeleteFields, ids => DeleteFields(socket, ids)); - Utils.AddServerHandlerCallback(socket, MessageStore.GetRefField, GetRefField); - Utils.AddServerHandlerCallback(socket, MessageStore.GetRefFields, GetRefFields); -}); - -async function deleteFields() { - await Database.Instance.deleteAll(); - await Search.Instance.clear(); - await Database.Instance.deleteAll('newDocuments'); -} + const credentialsPath = path.join(__dirname, "./credentials/google_docs_credentials.json"); + + const EndpointHandlerMap = new Map([ + ["create", (api, params) => api.create(params)], + ["retrieve", (api, params) => api.get(params)], + ["update", (api, params) => api.batchUpdate(params)], + ]); + + DashServer.post(RouteStore.googleDocs + "/:sector/:action", (req, res) => { + let sector: GoogleApiServerUtils.Service = req.params.sector as GoogleApiServerUtils.Service; + let action: GoogleApiServerUtils.Action = req.params.action as GoogleApiServerUtils.Action; + GoogleApiServerUtils.GetEndpoint(GoogleApiServerUtils.Service[sector], { credentialsPath, userId: req.headers.userId as string }).then(endpoint => { + let handler = EndpointHandlerMap.get(action); + if (endpoint && handler) { + let execute = handler(endpoint, req.body).then( + response => res.send(response.data), + rejection => res.send(rejection) + ); + execute.catch(exception => res.send(exception)); + return; + } + res.send(undefined); + }); + }); -async function deleteAll() { - await Database.Instance.deleteAll(); - await Database.Instance.deleteAll('newDocuments'); - await Database.Instance.deleteAll('sessions'); - await Database.Instance.deleteAll('users'); - await Search.Instance.clear(); -} + router.addSupervisedRoute({ + method: Method.GET, + subscription: RouteStore.readGoogleAccessToken, + onValidation: async (user, _req, res) => { + const userId = user.id; + const token = await Database.Auxiliary.GoogleAuthenticationToken.Fetch(userId); + const information = { credentialsPath, userId }; + if (!token) { + return res.send(await GoogleApiServerUtils.GenerateAuthenticationUrl(information)); + } + GoogleApiServerUtils.RetrieveAccessToken(information).then(token => res.send(token)); + } + }); -function barReceived(socket: SocketIO.Socket, guid: string) { - clients[guid] = new Client(guid.toString()); - console.log(`User ${guid} has connected`); - socketMap.set(socket, guid); -} + router.addSupervisedRoute({ + method: Method.POST, + subscription: RouteStore.writeGoogleAccessToken, + onValidation: async (user, req, res) => { + const userId = user.id; + const information = { credentialsPath, userId }; + res.send(await GoogleApiServerUtils.ProcessClientSideCode(information, req.body.authenticationCode)); + } + }); -function getField([id, callback]: [string, (result?: Transferable) => void]) { - Database.Instance.getDocument(id, (result?: Transferable) => - callback(result ? result : undefined)); -} + const tokenError = "Unable to successfully upload bytes for all images!"; + const mediaError = "Unable to convert all uploaded bytes to media items!"; + const userIdError = "Unable to parse the identification of the user!"; + + router.addSupervisedRoute({ + method: Method.POST, + subscription: RouteStore.googlePhotosMediaUpload, + onValidation: async (user, req, res) => { + const { media } = req.body; + const userId = user.id; + if (!userId) { + return _error(res, userIdError); + } -function getFields([ids, callback]: [string[], (result: Transferable[]) => void]) { - Database.Instance.getDocuments(ids, callback); -} + await GooglePhotosUploadUtils.initialize({ credentialsPath, userId }); + + let failed: number[] = []; + + const newMediaItems = await BatchedArray.from(media, { batchSize: 25 }).batchedMapPatientInterval( + { magnitude: 100, unit: TimeUnit.Milliseconds }, + async (batch: GooglePhotosUploadUtils.MediaInput[]) => { + const newMediaItems: NewMediaItem[] = []; + for (let index = 0; index < batch.length; index++) { + const element = batch[index]; + const uploadToken = await GooglePhotosUploadUtils.DispatchGooglePhotosUpload(element.url); + if (!uploadToken) { + failed.push(index); + } else { + newMediaItems.push({ + description: element.description, + simpleMediaItem: { uploadToken } + }); + } + } + return newMediaItems; + } + ); -function setField(socket: Socket, newValue: Transferable) { - Database.Instance.update(newValue.id, newValue, () => - socket.broadcast.emit(MessageStore.SetField.Message, newValue)); - if (newValue.type === Types.Text) { - Search.Instance.updateDocument({ id: newValue.id, data: (newValue as any).data }); - console.log("set field"); - console.log("checking in"); - } -} + const failedCount = failed.length; + if (failedCount) { + console.error(`Unable to upload ${failedCount} image${failedCount === 1 ? "" : "s"} to Google's servers`); + } -function GetRefField([id, callback]: [string, (result?: Transferable) => void]) { - Database.Instance.getDocument(id, callback, "newDocuments"); -} + GooglePhotosUploadUtils.CreateMediaItems(newMediaItems, req.body.album).then( + result => _success(res, { results: result.newMediaItemResults, failed }), + error => _error(res, mediaError, error) + ); + } + }); -function GetRefFields([ids, callback]: [string[], (result?: Transferable[]) => void]) { - Database.Instance.getDocuments(ids, callback, "newDocuments"); -} + interface MediaItem { + baseUrl: string; + filename: string; + } + const prefix = "google_photos_"; -function HandleYoutubeQuery([query, callback]: [YoutubeQueryInput, (result?: any[]) => void]) { - switch (query.type) { - case YoutubeQueryType.Channels: - YoutubeApi.authorizedGetChannel(youtubeApiKey); - break; - case YoutubeQueryType.SearchVideo: - YoutubeApi.authorizedGetVideos(youtubeApiKey, query.userInput, callback); - case YoutubeQueryType.VideoDetails: - YoutubeApi.authorizedGetVideoDetails(youtubeApiKey, query.videoIds, callback); - } -} + const downloadError = "Encountered an error while executing downloads."; + const requestError = "Unable to execute download: the body's media items were malformed."; + const deletionPermissionError = "Cannot perform specialized delete outside of the development environment!"; -const credentialsPath = path.join(__dirname, "./credentials/google_docs_credentials.json"); - -const EndpointHandlerMap = new Map([ - ["create", (api, params) => api.create(params)], - ["retrieve", (api, params) => api.get(params)], - ["update", (api, params) => api.batchUpdate(params)], -]); - -app.post(RouteStore.googleDocs + "/:sector/:action", (req, res) => { - let sector: GoogleApiServerUtils.Service = req.params.sector as GoogleApiServerUtils.Service; - let action: GoogleApiServerUtils.Action = req.params.action as GoogleApiServerUtils.Action; - GoogleApiServerUtils.GetEndpoint(GoogleApiServerUtils.Service[sector], { credentialsPath, userId: req.headers.userId as string }).then(endpoint => { - let handler = EndpointHandlerMap.get(action); - if (endpoint && handler) { - let execute = handler(endpoint, req.body).then( - response => res.send(response.data), - rejection => res.send(rejection) - ); - execute.catch(exception => res.send(exception)); - return; - } - res.send(undefined); - }); -}); - -addSecureRoute({ - method: Method.GET, - subscribers: RouteStore.readGoogleAccessToken, - onValidation: async (user, _req, res) => { - const userId = user.id; - const token = await Database.Auxiliary.GoogleAuthenticationToken.Fetch(userId); - const information = { credentialsPath, userId }; - if (!token) { - return res.send(await GoogleApiServerUtils.GenerateAuthenticationUrl(information)); - } - GoogleApiServerUtils.RetrieveAccessToken(information).then(token => res.send(token)); - } -}); - -addSecureRoute({ - method: Method.POST, - subscribers: RouteStore.writeGoogleAccessToken, - onValidation: async (user, req, res) => { - const userId = user.id; - const information = { credentialsPath, userId }; - res.send(await GoogleApiServerUtils.ProcessClientSideCode(information, req.body.authenticationCode)); - } -}); - -const tokenError = "Unable to successfully upload bytes for all images!"; -const mediaError = "Unable to convert all uploaded bytes to media items!"; -const userIdError = "Unable to parse the identification of the user!"; + DashServer.get("/deleteWithAux", async (_req, res) => { + if (release) { + return _permission_denied(res, deletionPermissionError); + } + await Database.Auxiliary.DeleteAll(); + res.redirect(RouteStore.delete); + }); -export interface NewMediaItem { - description: string; - simpleMediaItem: { - uploadToken: string; - }; -} + DashServer.get("/deleteWithGoogleCredentials", async (req, res) => { + if (release) { + return _permission_denied(res, deletionPermissionError); + } + await Database.Auxiliary.GoogleAuthenticationToken.DeleteAll(); + res.redirect(RouteStore.delete); + }); -addSecureRoute({ - method: Method.POST, - subscribers: RouteStore.googlePhotosMediaUpload, - onValidation: async (user, req, res) => { - const { media } = req.body; - const userId = user.id; - if (!userId) { - return _error(res, userIdError); - } + const UploadError = (count: number) => `Unable to upload ${count} images to Dash's server`; + DashServer.post(RouteStore.googlePhotosMediaDownload, async (req, res) => { + const contents: { mediaItems: MediaItem[] } = req.body; + let failed = 0; + if (contents) { + const completed: Opt[] = []; + for (let item of contents.mediaItems) { + const { contentSize, ...attributes } = await DashUploadUtils.InspectImage(item.baseUrl); + const found: Opt = await Database.Auxiliary.QueryUploadHistory(contentSize!); + if (!found) { + const upload = await DashUploadUtils.UploadInspectedImage({ contentSize, ...attributes }, item.filename, prefix).catch(error => _error(res, downloadError, error)); + if (upload) { + completed.push(upload); + await Database.Auxiliary.LogUpload(upload); + } else { + failed++; + } + } else { + completed.push(found); + } + } + if (failed) { + return _error(res, UploadError(failed)); + } + return _success(res, completed); + } + _invalid(res, requestError); + }); - await GooglePhotosUploadUtils.initialize({ credentialsPath, userId }); + const suffixMap: { [type: string]: (string | [string, string | ((json: any) => any)]) } = { + "number": "_n", + "string": "_t", + "boolean": "_b", + "image": ["_t", "url"], + "video": ["_t", "url"], + "pdf": ["_t", "url"], + "audio": ["_t", "url"], + "web": ["_t", "url"], + "date": ["_d", value => new Date(value.date).toISOString()], + "proxy": ["_i", "fieldId"], + "list": ["_l", list => { + const results = []; + for (const value of list.fields) { + const term = ToSearchTerm(value); + if (term) { + results.push(term.value); + } + } + return results.length ? results : null; + }] + }; - let failed: number[] = []; + function ToSearchTerm(val: any): { suffix: string, value: any } | undefined { + if (val === null || val === undefined) { + return; + } + const type = val.__type || typeof val; + let suffix = suffixMap[type]; + if (!suffix) { + return; + } - const newMediaItems = await BatchedArray.from(media, { batchSize: 25 }).batchedMapPatientInterval( - { magnitude: 100, unit: TimeUnit.Milliseconds }, - async (batch: GooglePhotosUploadUtils.MediaInput[]) => { - const newMediaItems: NewMediaItem[] = []; - for (let index = 0; index < batch.length; index++) { - const element = batch[index]; - const uploadToken = await GooglePhotosUploadUtils.DispatchGooglePhotosUpload(element.url); - if (!uploadToken) { - failed.push(index); + if (Array.isArray(suffix)) { + const accessor = suffix[1]; + if (typeof accessor === "function") { + val = accessor(val); } else { - newMediaItems.push({ - description: element.description, - simpleMediaItem: { uploadToken } - }); + val = val[accessor]; } + suffix = suffix[0]; } - return newMediaItems; - } - ); - const failedCount = failed.length; - if (failedCount) { - console.error(`Unable to upload ${failedCount} image${failedCount === 1 ? "" : "s"} to Google's servers`); - } + return { suffix, value: val }; + } - GooglePhotosUploadUtils.CreateMediaItems(newMediaItems, req.body.album).then( - result => _success(res, { results: result.newMediaItemResults, failed }), - error => _error(res, mediaError, error) - ); - } -}); + function getSuffix(value: string | [string, any]): string { + return typeof value === "string" ? value : value[0]; + } -interface MediaItem { - baseUrl: string; - filename: string; -} -const prefix = "google_photos_"; - -const downloadError = "Encountered an error while executing downloads."; -const requestError = "Unable to execute download: the body's media items were malformed."; -const deletionPermissionError = "Cannot perform specialized delete outside of the development environment!"; - -app.get("/deleteWithAux", async (_req, res) => { - if (release) { - return _permission_denied(res, deletionPermissionError); - } - await Database.Auxiliary.DeleteAll(); - res.redirect(RouteStore.delete); -}); - -app.get("/deleteWithGoogleCredentials", async (req, res) => { - if (release) { - return _permission_denied(res, deletionPermissionError); - } - await Database.Auxiliary.GoogleAuthenticationToken.DeleteAll(); - res.redirect(RouteStore.delete); -}); - -const UploadError = (count: number) => `Unable to upload ${count} images to Dash's server`; -app.post(RouteStore.googlePhotosMediaDownload, async (req, res) => { - const contents: { mediaItems: MediaItem[] } = req.body; - let failed = 0; - if (contents) { - const completed: Opt[] = []; - for (let item of contents.mediaItems) { - const { contentSize, ...attributes } = await DashUploadUtils.InspectImage(item.baseUrl); - const found: Opt = await Database.Auxiliary.QueryUploadHistory(contentSize!); - if (!found) { - const upload = await DashUploadUtils.UploadInspectedImage({ contentSize, ...attributes }, item.filename, prefix).catch(error => _error(res, downloadError, error)); - if (upload) { - completed.push(upload); - await Database.Auxiliary.LogUpload(upload); - } else { - failed++; + function UpdateField(socket: Socket, diff: Diff) { + Database.Instance.update(diff.id, diff.diff, + () => socket.broadcast.emit(MessageStore.UpdateField.Message, diff), false, "newDocuments"); + const docfield = diff.diff.$set; + if (!docfield) { + return; + } + const update: any = { id: diff.id }; + let dynfield = false; + for (let key in docfield) { + if (!key.startsWith("fields.")) continue; + dynfield = true; + let val = docfield[key]; + key = key.substring(7); + Object.values(suffixMap).forEach(suf => update[key + getSuffix(suf)] = { set: null }); + let term = ToSearchTerm(val); + if (term !== undefined) { + let { suffix, value } = term; + update[key + suffix] = { set: value }; + } + } + if (dynfield) { + Search.Instance.updateDocument(update); } - } else { - completed.push(found); - } - } - if (failed) { - return _error(res, UploadError(failed)); - } - return _success(res, completed); - } - _invalid(res, requestError); -}); - -const _error = (res: Response, message: string, error?: any) => { - res.statusMessage = message; - res.status(STATUS.EXECUTION_ERROR).send(error); -}; - -const _success = (res: Response, body: any) => { - res.status(STATUS.OK).send(body); -}; - -const _invalid = (res: Response, message: string) => { - res.statusMessage = message; - res.status(STATUS.BAD_REQUEST).send(); -}; - -const _permission_denied = (res: Response, message: string) => { - res.statusMessage = message; - res.status(STATUS.BAD_REQUEST).send("Permission Denied!"); -}; - -const suffixMap: { [type: string]: (string | [string, string | ((json: any) => any)]) } = { - "number": "_n", - "string": "_t", - "boolean": "_b", - "image": ["_t", "url"], - "video": ["_t", "url"], - "pdf": ["_t", "url"], - "audio": ["_t", "url"], - "web": ["_t", "url"], - "date": ["_d", value => new Date(value.date).toISOString()], - "proxy": ["_i", "fieldId"], - "list": ["_l", list => { - const results = []; - for (const value of list.fields) { - const term = ToSearchTerm(value); - if (term) { - results.push(term.value); } - } - return results.length ? results : null; - }] -}; - -function ToSearchTerm(val: any): { suffix: string, value: any } | undefined { - if (val === null || val === undefined) { - return; - } - const type = val.__type || typeof val; - let suffix = suffixMap[type]; - if (!suffix) { - return; - } - - if (Array.isArray(suffix)) { - const accessor = suffix[1]; - if (typeof accessor === "function") { - val = accessor(val); - } else { - val = val[accessor]; - } - suffix = suffix[0]; - } - - return { suffix, value: val }; -} -function getSuffix(value: string | [string, any]): string { - return typeof value === "string" ? value : value[0]; -} - -function UpdateField(socket: Socket, diff: Diff) { - Database.Instance.update(diff.id, diff.diff, - () => socket.broadcast.emit(MessageStore.UpdateField.Message, diff), false, "newDocuments"); - const docfield = diff.diff.$set; - if (!docfield) { - return; - } - const update: any = { id: diff.id }; - let dynfield = false; - for (let key in docfield) { - if (!key.startsWith("fields.")) continue; - dynfield = true; - let val = docfield[key]; - key = key.substring(7); - Object.values(suffixMap).forEach(suf => update[key + getSuffix(suf)] = { set: null }); - let term = ToSearchTerm(val); - if (term !== undefined) { - let { suffix, value } = term; - update[key + suffix] = { set: value }; - } - } - if (dynfield) { - Search.Instance.updateDocument(update); - } -} - -function DeleteField(socket: Socket, id: string) { - Database.Instance.delete({ _id: id }, "newDocuments").then(() => { - socket.broadcast.emit(MessageStore.DeleteField.Message, id); - }); + function DeleteField(socket: Socket, id: string) { + Database.Instance.delete({ _id: id }, "newDocuments").then(() => { + socket.broadcast.emit(MessageStore.DeleteField.Message, id); + }); - Search.Instance.deleteDocuments([id]); -} + Search.Instance.deleteDocuments([id]); + } -function DeleteFields(socket: Socket, ids: string[]) { - Database.Instance.delete({ _id: { $in: ids } }, "newDocuments").then(() => { - socket.broadcast.emit(MessageStore.DeleteFields.Message, ids); - }); + function DeleteFields(socket: Socket, ids: string[]) { + Database.Instance.delete({ _id: { $in: ids } }, "newDocuments").then(() => { + socket.broadcast.emit(MessageStore.DeleteFields.Message, ids); + }); - Search.Instance.deleteDocuments(ids); + Search.Instance.deleteDocuments(ids); -} + } -function CreateField(newValue: any) { - Database.Instance.insert(newValue, "newDocuments"); -} + function CreateField(newValue: any) { + Database.Instance.insert(newValue, "newDocuments"); + } -server.listen(serverPort); -console.log(`listening on port ${serverPort}`); + server.listen(serverPort); + console.log(`listening on port ${serverPort}`); + } + }); +})(); \ No newline at end of file -- cgit v1.2.3-70-g09d2 From 7b43e349d31c911ab43763a4ff7179b3778a2d96 Mon Sep 17 00:00:00 2001 From: Sam Wilkins <35748010+samwilkins333@users.noreply.github.com> Date: Wed, 16 Oct 2019 18:38:31 -0400 Subject: database separation and preliminary functions --- src/server/Initialization.ts | 81 +++++++++++++---------------- src/server/database.ts | 27 ++++++++++ src/server/index.ts | 118 ++++++++++++++++++++----------------------- 3 files changed, 117 insertions(+), 109 deletions(-) (limited to 'src') diff --git a/src/server/Initialization.ts b/src/server/Initialization.ts index e371a3edb..2c343ae90 100644 --- a/src/server/Initialization.ts +++ b/src/server/Initialization.ts @@ -7,8 +7,8 @@ import * as cookieParser from 'cookie-parser'; import expressFlash = require('express-flash'); import flash = require('connect-flash'); import { Database } from './database'; +import { getForgot, getLogin, getLogout, getReset, getSignup, postForgot, postLogin, postReset, postSignup } from './authentication/controllers/user_controller'; const MongoStore = require('connect-mongo')(session); -import mongoose, { ConnectionStates } from 'mongoose'; import { RouteStore } from './RouteStore'; import RouteManager from './RouteManager'; import * as webpack from 'webpack'; @@ -25,47 +25,26 @@ export interface InitializationOptions { export default async function InitializeServer(options: InitializationOptions) { const { listenAtPort, routeSetter } = options; - const server = injectMiddleware(express()); - const { url } = Database; - try { - await connectToDatabase(url); - } catch (e) { - console.error(`Mongoose FAILED to establish default connection at ${url}`); - console.error(e); - console.log('Since a valid database connection is required to use Dash, killing the server process.\nPlease try again later.'); - process.exit(1); - } - - // static file serving - server.use(express.static(__dirname + RouteStore.public)); - server.use(RouteStore.images, express.static(__dirname + RouteStore.public)); + const server = buildWithMiddleware(express()); routeSetter(new RouteManager(server, determineEnvironment())); + server.use(express.static(__dirname + RouteStore.public)); + server.use(RouteStore.images, express.static(__dirname + RouteStore.public)); + server.use(wdm(compiler, { publicPath: config.output.publicPath })); server.use(whm(compiler)); server.listen(listenAtPort, () => console.log(`server started at http://localhost:${listenAtPort}`)); - return server; -} - -function determineEnvironment() { - const isRelease = process.env.RELEASE === "true"; - - console.log(`running server in ${isRelease ? 'release' : 'debug'} mode`); - console.log(process.env.PWD); + registerAuthenticationRoutes(server); - let clientUtils = fs.readFileSync("./src/client/util/ClientUtils.ts.temp", "utf8"); - clientUtils = `//AUTO-GENERATED FILE: DO NOT EDIT\n${clientUtils.replace('"mode"', String(isRelease))}`; - fs.writeFileSync("./src/client/util/ClientUtils.ts", clientUtils, "utf8"); - - return isRelease; + return server; } const week = 7 * 24 * 60 * 60 * 1000; const secret = "64d6866242d3b5a5503c675b32c9605e4e90478e9b77bcf2bc"; -function injectMiddleware(server: express.Express) { +function buildWithMiddleware(server: express.Express) { [ cookieParser(), session({ @@ -90,21 +69,31 @@ function injectMiddleware(server: express.Express) { return server; } -async function connectToDatabase(url: string) { - const { connection } = mongoose; - process.on('SIGINT', () => { - connection.close(() => { - console.log('Mongoose default connection disconnected through app termination'); - process.exit(0); - }); - }); - if (connection.readyState === ConnectionStates.disconnected) { - return new Promise((resolve, reject) => { - connection.on('error', reject); - connection.on('connected', () => { - console.log(`Mongoose established default connection at ${url}`); - resolve(); - }); - }); - } +function determineEnvironment() { + const isRelease = process.env.RELEASE === "true"; + + console.log(`running server in ${isRelease ? 'release' : 'debug'} mode`); + console.log(process.env.PWD); + + let clientUtils = fs.readFileSync("./src/client/util/ClientUtils.ts.temp", "utf8"); + clientUtils = `//AUTO-GENERATED FILE: DO NOT EDIT\n${clientUtils.replace('"mode"', String(isRelease))}`; + fs.writeFileSync("./src/client/util/ClientUtils.ts", clientUtils, "utf8"); + + return isRelease; +} + +function registerAuthenticationRoutes(server: express.Express) { + server.get(RouteStore.signup, getSignup); + server.post(RouteStore.signup, postSignup); + + server.get(RouteStore.login, getLogin); + server.post(RouteStore.login, postLogin); + + server.get(RouteStore.logout, getLogout); + + server.get(RouteStore.forgot, getForgot); + server.post(RouteStore.forgot, postForgot); + + server.get(RouteStore.reset, getReset); + server.post(RouteStore.reset, postReset); } \ No newline at end of file diff --git a/src/server/database.ts b/src/server/database.ts index 25e1e67e0..4f93d1ee6 100644 --- a/src/server/database.ts +++ b/src/server/database.ts @@ -5,6 +5,7 @@ import { Utils, emptyFunction } from '../Utils'; import { DashUploadUtils } from './DashUploadUtils'; import { Credentials } from 'google-auth-library'; import { GoogleApiServerUtils } from './apis/google/GoogleApiServerUtils'; +import mongoose, { ConnectionStates } from 'mongoose'; export namespace Database { @@ -12,6 +13,32 @@ export namespace Database { const port = 27017; export const url = `mongodb://localhost:${port}/${schema}`; + export async function tryInitializeConnection() { + try { + const { connection } = mongoose; + process.on('SIGINT', () => { + connection.close(() => { + console.log('Mongoose default connection disconnected through app termination'); + process.exit(0); + }); + }); + if (connection.readyState === ConnectionStates.disconnected) { + await new Promise((resolve, reject) => { + connection.on('error', reject); + connection.on('connected', () => { + console.log(`Mongoose established default connection at ${url}`); + resolve(); + }); + }); + } + } catch (e) { + console.error(`Mongoose FAILED to establish default connection at ${url} with the following error:`); + console.error(e); + console.log('Since a valid database connection is required to use Dash, the server process will now exit.\nPlease try again later.'); + process.exit(1); + } + } + class Database { public static DocumentsCollection = 'documents'; private MongoClient = mongodb.MongoClient; diff --git a/src/server/index.ts b/src/server/index.ts index ad18857b6..ef618472b 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -11,7 +11,6 @@ import * as request from 'request'; import io from 'socket.io'; import { Socket } from 'socket.io'; import { Utils } from '../Utils'; -import { getForgot, getLogin, getLogout, getReset, getSignup, postForgot, postLogin, postReset, postSignup } from './authentication/controllers/user_controller'; import { Client } from './Client'; import { Database } from './database'; import { MessageStore, Transferable, Types, Diff, YoutubeQueryTypes as YoutubeQueryType, YoutubeQueryInput } from "./Message"; @@ -27,7 +26,6 @@ import { Response } from 'express-serve-static-core'; import { GoogleApiServerUtils } from "./apis/google/GoogleApiServerUtils"; const probe = require("probe-image-size"); const pdf = require('pdf-parse'); -var findInFiles = require('find-in-files'); import { GooglePhotosUploadUtils } from './apis/google/GooglePhotosUploadUtils'; import { Opt } from '../new_fields/Doc'; import { DashUploadUtils } from './DashUploadUtils'; @@ -38,6 +36,7 @@ import RouteSubscriber from './RouteSubscriber'; import InitializeServer from './Initialization'; import { Method, _success, _permission_denied, _error, _invalid } from './RouteManager'; import { command_line, read_text_file } from './ActionUtilities'; +var findInFiles = require('find-in-files'); let youtubeApiKey: string; @@ -51,10 +50,22 @@ export interface NewMediaItem { }; } -(async () => { - YoutubeApi.readApiKey((apiKey: string) => youtubeApiKey = apiKey); +const pngTypes = [".png", ".PNG"]; +const jpgTypes = [".jpg", ".JPG", ".jpeg", ".JPEG"]; +const uploadDirectory = __dirname + "/public/files/"; +const pdfDirectory = uploadDirectory + "text"; +const solrURL = "http://localhost:8983/solr/#/dash"; + +YoutubeApi.readApiKey((apiKey: string) => youtubeApiKey = apiKey); + +async function PreliminaryFunctions() { await GoogleApiServerUtils.LoadOAuthClient(); + await DashUploadUtils.createIfNotExists(pdfDirectory); + await Database.tryInitializeConnection(); +} +(async () => { + await PreliminaryFunctions(); await InitializeServer({ listenAtPort: 1050, routeSetter: router => { @@ -73,6 +84,26 @@ export interface NewMediaItem { } }); + router.addSupervisedRoute({ + method: Method.GET, + subscription: "/textsearch", + onValidation: async (_user, req, res) => { + let q = req.query.q; + if (q === undefined) { + res.send([]); + return; + } + let results = await findInFiles.find({ 'term': q, 'flags': 'ig' }, uploadDirectory + "text", ".txt$"); + let resObj: { ids: string[], numFound: number, lines: string[] } = { ids: [], numFound: 0, lines: [] }; + for (var result in results) { + resObj.ids.push(path.basename(result, ".txt").replace(/upload_/, "")); + resObj.lines.push(results[result].line); + resObj.numFound++; + } + res.send(resObj); + } + }); + router.addSupervisedRoute({ method: Method.GET, subscription: "/buxton", @@ -101,36 +132,19 @@ export interface NewMediaItem { } }); - // SEARCH - const solrURL = "http://localhost:8983/solr/#/dash"; - - // GETTERS - - DashServer.get("/textsearch", async (req, res) => { - let q = req.query.q; - if (q === undefined) { - res.send([]); - return; - } - let results = await findInFiles.find({ 'term': q, 'flags': 'ig' }, uploadDirectory + "text", ".txt$"); - let resObj: { ids: string[], numFound: number, lines: string[] } = { ids: [], numFound: 0, lines: [] }; - for (var result in results) { - resObj.ids.push(path.basename(result, ".txt").replace(/upload_/, "")); - resObj.lines.push(results[result].line); - resObj.numFound++; - } - res.send(resObj); - }); - - DashServer.get("/search", async (req, res) => { - const solrQuery: any = {}; - ["q", "fq", "start", "rows", "hl", "hl.fl"].forEach(key => solrQuery[key] = req.query[key]); - if (solrQuery.q === undefined) { - res.send([]); - return; + router.addSupervisedRoute({ + method: Method.GET, + subscription: "/search", + onValidation: async (_user, req, res) => { + const solrQuery: any = {}; + ["q", "fq", "start", "rows", "hl", "hl.fl"].forEach(key => solrQuery[key] = req.query[key]); + if (solrQuery.q === undefined) { + res.send([]); + return; + } + let results = await Search.Instance.search(solrQuery); + res.send(results); } - let results = await Search.Instance.search(solrQuery); - res.send(results); }); function msToTime(duration: number) { @@ -210,9 +224,14 @@ export interface NewMediaItem { await Database.Instance.visit([id], fn); return { id, docs, files }; } - DashServer.get("/serializeDoc/:docId", async (req, res) => { - const { docs, files } = await getDocs(req.params.docId); - res.send({ docs, files: Array.from(files) }); + + router.addSupervisedRoute({ + method: Method.GET, + subscription: new RouteSubscriber("/serializeDoc").add("docId"), + onValidation: async (_user, req, res) => { + const { docs, files } = await getDocs(req.params.docId); + res.send({ docs, files: Array.from(files) }); + } }); router.addSupervisedRoute({ @@ -554,12 +573,6 @@ export interface NewMediaItem { } } - const pngTypes = [".png", ".PNG"]; - const jpgTypes = [".jpg", ".JPG", ".jpeg", ".JPEG"]; - const uploadDirectory = __dirname + "/public/files/"; - const pdfDirectory = uploadDirectory + "text"; - DashUploadUtils.createIfNotExists(pdfDirectory); - interface ImageFileResponse { name: string; path: string; @@ -657,27 +670,6 @@ export interface NewMediaItem { } }); - // AUTHENTICATION - - // Sign Up - DashServer.get(RouteStore.signup, getSignup); - DashServer.post(RouteStore.signup, postSignup); - - // Log In - DashServer.get(RouteStore.login, getLogin); - DashServer.post(RouteStore.login, postLogin); - - // Log Out - DashServer.get(RouteStore.logout, getLogout); - - // FORGOT PASSWORD EMAIL HANDLING - DashServer.get(RouteStore.forgot, getForgot); - DashServer.post(RouteStore.forgot, postForgot); - - // RESET PASSWORD EMAIL HANDLING - DashServer.get(RouteStore.reset, getReset); - DashServer.post(RouteStore.reset, postReset); - const headerCharRegex = /[^\t\x20-\x7e\x80-\xff]/; DashServer.use(RouteStore.corsProxy, (req, res) => { req.pipe(request(decodeURIComponent(req.url.substring(1)))).on("response", res => { -- cgit v1.2.3-70-g09d2 From 91868727ea6e6443a916cf720d477b1136601b2f Mon Sep 17 00:00:00 2001 From: Sam Wilkins <35748010+samwilkins333@users.noreply.github.com> Date: Thu, 17 Oct 2019 02:53:34 -0400 Subject: refactored handlers --- src/client/util/SelectionManager.ts | 5 +- src/server/Initialization.ts | 28 +- src/server/RouteManager.ts | 60 ++- src/server/authentication/models/user_model.ts | 14 +- src/server/database.ts | 14 +- src/server/index.ts | 495 +++++++++++++------------ 6 files changed, 343 insertions(+), 273 deletions(-) (limited to 'src') diff --git a/src/client/util/SelectionManager.ts b/src/client/util/SelectionManager.ts index df1b46b33..398c90ddb 100644 --- a/src/client/util/SelectionManager.ts +++ b/src/client/util/SelectionManager.ts @@ -54,7 +54,10 @@ export namespace SelectionManager { let stored = StrCast(targetDoc.backgroundColor); stored.length > 0 && (targetColor = stored); } - InkingControl.Instance.updateSelectedColor(targetColor); + const { Instance } = InkingControl; + if (Instance) { + Instance.updateSelectedColor(targetColor); + } }, { fireImmediately: true }); export function DeselectDoc(docView: DocumentView): void { diff --git a/src/server/Initialization.ts b/src/server/Initialization.ts index 2c343ae90..9646dc195 100644 --- a/src/server/Initialization.ts +++ b/src/server/Initialization.ts @@ -17,6 +17,7 @@ const compiler = webpack(config); import * as wdm from 'webpack-dev-middleware'; import * as whm from 'webpack-hot-middleware'; import * as fs from 'fs'; +import * as request from 'request'; export interface InitializationOptions { listenAtPort: number; @@ -27,18 +28,18 @@ export default async function InitializeServer(options: InitializationOptions) { const { listenAtPort, routeSetter } = options; const server = buildWithMiddleware(express()); - routeSetter(new RouteManager(server, determineEnvironment())); - server.use(express.static(__dirname + RouteStore.public)); server.use(RouteStore.images, express.static(__dirname + RouteStore.public)); server.use(wdm(compiler, { publicPath: config.output.publicPath })); server.use(whm(compiler)); - server.listen(listenAtPort, () => console.log(`server started at http://localhost:${listenAtPort}`)); registerAuthenticationRoutes(server); + registerCorsProxy(server); - return server; + routeSetter(new RouteManager(server, determineEnvironment())); + + server.listen(listenAtPort, () => console.log(`server started at http://localhost:${listenAtPort}`)); } const week = 7 * 24 * 60 * 60 * 1000; @@ -96,4 +97,23 @@ function registerAuthenticationRoutes(server: express.Express) { server.get(RouteStore.reset, getReset); server.post(RouteStore.reset, postReset); +} + +function registerCorsProxy(server: express.Express) { + const headerCharRegex = /[^\t\x20-\x7e\x80-\xff]/; + server.use(RouteStore.corsProxy, (req, res) => { + req.pipe(request(decodeURIComponent(req.url.substring(1)))).on("response", res => { + const headers = Object.keys(res.headers); + headers.forEach(headerName => { + const header = res.headers[headerName]; + if (Array.isArray(header)) { + res.headers[headerName] = header.filter(h => !headerCharRegex.test(h)); + } else if (header) { + if (headerCharRegex.test(header as any)) { + delete res.headers[headerName]; + } + } + }); + }).pipe(res); + }); } \ No newline at end of file diff --git a/src/server/RouteManager.ts b/src/server/RouteManager.ts index cf15e45c9..626014d1a 100644 --- a/src/server/RouteManager.ts +++ b/src/server/RouteManager.ts @@ -3,6 +3,7 @@ import { RouteStore } from "./RouteStore"; import { DashUserModel } from "./authentication/models/user_model"; import * as express from 'express'; import * as qs from 'query-string'; +import { Opt } from "../new_fields/Doc"; export default class RouteManager { private server: express.Express; @@ -27,29 +28,34 @@ export default class RouteManager { * @param subscribers the forward slash prepended path names (reference and add to RouteStore.ts) that will all invoke the given @param handler */ addSupervisedRoute(initializer: RouteInitializer) { - const { method, subscription, onValidation, onRejection, onError } = initializer; - const release = this._isRelease; + const { method, subscription, onValidation, onRejection, onError, onGuestAccess } = initializer; + const isRelease = this._isRelease; let abstracted = async (req: express.Request, res: express.Response) => { const { user, originalUrl: target } = req; - if (user || isSharedDocAccess(target)) { + const core = { req, res, isRelease: isRelease }; + if (user) { try { - await onValidation(user, req, res, release); + await onValidation({ ...core, user: user as any }); } catch (e) { if (onError) { - onError(req, res, e, release); + onError({ ...core, error: e }); } else { _error(res, `The server encountered an internal error handling ${target}.`, e); } } } else { - req.session!.target = target; - try { - await (onRejection || LoginRedirect)(req, res, release); - } catch (e) { - if (onError) { - onError(req, res, e, this._isRelease); - } else { - _error(res, `The server encountered an internal error when rejecting ${target}.`, e); + if (isGuestAccess(req) && onGuestAccess) { + await onGuestAccess(core); + } else { + req.session!.target = target; + try { + await (onRejection || LoginRedirect)(core); + } catch (e) { + if (onError) { + onError({ ...core, error: e }); + } else { + _error(res, `The server encountered an internal error when rejecting ${target}.`, e); + } } } } @@ -84,18 +90,25 @@ export enum Method { POST } -export type ValidationHandler = (user: DashUserModel, req: express.Request, res: express.Response, isRelease: boolean) => any | Promise; -export type RejectionHandler = (req: express.Request, res: express.Response, isRelease: boolean) => any | Promise; -export type ErrorHandler = (req: express.Request, res: express.Response, error: any, isRelease: boolean) => any | Promise; +export interface CoreArguments { + req: express.Request, + res: express.Response, + isRelease: boolean; +} -const LoginRedirect: RejectionHandler = (_req, res) => res.redirect(RouteStore.login); +export type OnValidation = (core: CoreArguments & { user: DashUserModel }) => any | Promise; +export type OnUnauthenticated = (core: CoreArguments) => any | Promise; +export type OnError = (core: CoreArguments & { error: any }) => any | Promise; + +const LoginRedirect: OnUnauthenticated = ({ res }) => res.redirect(RouteStore.login); export interface RouteInitializer { method: Method; subscription: string | RouteSubscriber | (string | RouteSubscriber)[]; - onValidation: ValidationHandler; - onRejection?: RejectionHandler; - onError?: ErrorHandler; + onValidation: OnValidation; + onRejection?: OnUnauthenticated; + onGuestAccess?: OnUnauthenticated; + onError?: OnError; } const isSharedDocAccess = (target: string) => { @@ -104,6 +117,13 @@ const isSharedDocAccess = (target: string) => { return shared && docAccess; }; +const isGuestAccess = (req: express.Request) => { + if (isSharedDocAccess(req.originalUrl)) { + return true; + } + return false; +} + export const STATUS = { OK: 200, BAD_REQUEST: 400, diff --git a/src/server/authentication/models/user_model.ts b/src/server/authentication/models/user_model.ts index 45fbf23b1..cc670a03a 100644 --- a/src/server/authentication/models/user_model.ts +++ b/src/server/authentication/models/user_model.ts @@ -1,20 +1,8 @@ //@ts-ignore import * as bcrypt from "bcrypt-nodejs"; //@ts-ignore -import * as mongoose from "mongoose"; -var url = 'mongodb://localhost:27017/Dash'; +import * as mongoose from 'mongoose'; -mongoose.connect(url, { useNewUrlParser: true }); - -mongoose.connection.on('connected', function () { - console.log('Stablished connection on ' + url); -}); -mongoose.connection.on('error', function (error) { - console.log('Something wrong happened: ' + error); -}); -mongoose.connection.on('disconnected', function () { - console.log('connection closed'); -}); export type DashUserModel = mongoose.Document & { email: String, password: string, diff --git a/src/server/database.ts b/src/server/database.ts index 4f93d1ee6..44c49d03e 100644 --- a/src/server/database.ts +++ b/src/server/database.ts @@ -5,7 +5,7 @@ import { Utils, emptyFunction } from '../Utils'; import { DashUploadUtils } from './DashUploadUtils'; import { Credentials } from 'google-auth-library'; import { GoogleApiServerUtils } from './apis/google/GoogleApiServerUtils'; -import mongoose, { ConnectionStates } from 'mongoose'; +import * as mongoose from 'mongoose'; export namespace Database { @@ -13,6 +13,14 @@ export namespace Database { const port = 27017; export const url = `mongodb://localhost:${port}/${schema}`; + enum ConnectionStates { + disconnected = 0, + connected = 1, + connecting = 2, + disconnecting = 3, + uninitialized = 99, + } + export async function tryInitializeConnection() { try { const { connection } = mongoose; @@ -25,10 +33,14 @@ export namespace Database { if (connection.readyState === ConnectionStates.disconnected) { await new Promise((resolve, reject) => { connection.on('error', reject); + connection.on('disconnected', () => { + console.log(`Mongoose connection at ${url} now closed`); + }); connection.on('connected', () => { console.log(`Mongoose established default connection at ${url}`); resolve(); }); + mongoose.connect(url, { useNewUrlParser: true }); }); } } catch (e) { diff --git a/src/server/index.ts b/src/server/index.ts index ef618472b..bba8fc292 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -7,8 +7,7 @@ import * as Pdfjs from 'pdfjs-dist'; const imageDataUri = require('image-data-uri'); import * as mobileDetect from 'mobile-detect'; import * as path from 'path'; -import * as request from 'request'; -import io from 'socket.io'; +import * as io from 'socket.io'; import { Socket } from 'socket.io'; import { Utils } from '../Utils'; import { Client } from './Client'; @@ -34,8 +33,8 @@ import { ParsedPDF } from "./PdfTypes"; import { reject } from 'bluebird'; import RouteSubscriber from './RouteSubscriber'; import InitializeServer from './Initialization'; -import { Method, _success, _permission_denied, _error, _invalid } from './RouteManager'; -import { command_line, read_text_file } from './ActionUtilities'; +import { Method, _success, _permission_denied, _error, _invalid, OnUnauthenticated } from './RouteManager'; +import { command_line } from './ActionUtilities'; var findInFiles = require('find-in-files'); let youtubeApiKey: string; @@ -73,7 +72,7 @@ async function PreliminaryFunctions() { router.addSupervisedRoute({ method: Method.GET, subscription: "/pull", - onValidation: (_user, _req, res) => { + onValidation: ({ res }) => { exec('"C:\\Program Files\\Git\\git-bash.exe" -c "git pull"', err => { if (err) { res.send(err.message); @@ -87,7 +86,7 @@ async function PreliminaryFunctions() { router.addSupervisedRoute({ method: Method.GET, subscription: "/textsearch", - onValidation: async (_user, req, res) => { + onValidation: async ({ req, res }) => { let q = req.query.q; if (q === undefined) { res.send([]); @@ -107,7 +106,7 @@ async function PreliminaryFunctions() { router.addSupervisedRoute({ method: Method.GET, subscription: "/buxton", - onValidation: (_user, _req, res) => { + onValidation: ({ res }) => { let cwd = '../scraping/buxton'; let onResolved = (stdout: string) => { console.log(stdout); res.redirect("/"); }; @@ -121,7 +120,7 @@ async function PreliminaryFunctions() { router.addSupervisedRoute({ method: Method.GET, subscription: "/version", - onValidation: (_user, _req, res) => { + onValidation: ({ res }) => { exec('"C:\\Program Files\\Git\\bin\\git.exe" rev-parse HEAD', (err, stdout) => { if (err) { res.send(err.message); @@ -135,7 +134,7 @@ async function PreliminaryFunctions() { router.addSupervisedRoute({ method: Method.GET, subscription: "/search", - onValidation: async (_user, req, res) => { + onValidation: async ({ req, res }) => { const solrQuery: any = {}; ["q", "fq", "start", "rows", "hl", "hl.fl"].forEach(key => solrQuery[key] = req.query[key]); if (solrQuery.q === undefined) { @@ -228,7 +227,7 @@ async function PreliminaryFunctions() { router.addSupervisedRoute({ method: Method.GET, subscription: new RouteSubscriber("/serializeDoc").add("docId"), - onValidation: async (_user, req, res) => { + onValidation: async ({ req, res }) => { const { docs, files } = await getDocs(req.params.docId); res.send({ docs, files: Array.from(files) }); } @@ -237,7 +236,7 @@ async function PreliminaryFunctions() { router.addSupervisedRoute({ method: Method.GET, subscription: new RouteSubscriber(RouteStore.imageHierarchyExport).add('docId'), - onValidation: async (_user, req, res) => { + onValidation: async ({ req, res }) => { const id = req.params.docId; const hierarchy: Hierarchy = {}; await targetedVisitorRecursive(id, hierarchy); @@ -305,154 +304,171 @@ async function PreliminaryFunctions() { } }; - DashServer.get("/downloadId/:docId", async (req, res) => { - res.set('Content-disposition', `attachment;`); - res.set('Content-Type', "application/zip"); - const { id, docs, files } = await getDocs(req.params.docId); - const docString = JSON.stringify({ id, docs }); - const zip = Archiver('zip'); - zip.pipe(res); - zip.append(docString, { name: "doc.json" }); - files.forEach(val => { - zip.file(__dirname + RouteStore.public + val, { name: val.substring(1) }); - }); - zip.finalize(); - }); + router.addSupervisedRoute({ + method: Method.GET, + subscription: new RouteSubscriber("/downloadId").add("docId"), + onValidation: async ({ req, res }) => { + res.set('Content-disposition', `attachment;`); + res.set('Content-Type', "application/zip"); + const { id, docs, files } = await getDocs(req.params.docId); + const docString = JSON.stringify({ id, docs }); + const zip = Archiver('zip'); + zip.pipe(res); + zip.append(docString, { name: "doc.json" }); + files.forEach(val => { + zip.file(__dirname + RouteStore.public + val, { name: val.substring(1) }); + }); + zip.finalize(); + } + }) - DashServer.post("/uploadDoc", (req, res) => { - let form = new formidable.IncomingForm(); - form.keepExtensions = true; - // let path = req.body.path; - const ids: { [id: string]: string } = {}; - let remap = true; - const getId = (id: string): string => { - if (!remap) return id; - if (id.endsWith("Proto")) return id; - if (id in ids) { - return ids[id]; - } else { - return ids[id] = v4(); - } - }; - const mapFn = (doc: any) => { - if (doc.id) { - doc.id = getId(doc.id); - } - for (const key in doc.fields) { - if (!doc.fields.hasOwnProperty(key)) { - continue; + router.addSupervisedRoute({ + method: Method.POST, + subscription: "/uploadDoc", + onValidation: ({ req, res }) => { + let form = new formidable.IncomingForm(); + form.keepExtensions = true; + // let path = req.body.path; + const ids: { [id: string]: string } = {}; + let remap = true; + const getId = (id: string): string => { + if (!remap) return id; + if (id.endsWith("Proto")) return id; + if (id in ids) { + return ids[id]; + } else { + return ids[id] = v4(); } - const field = doc.fields[key]; - if (field === undefined || field === null) { - continue; + }; + const mapFn = (doc: any) => { + if (doc.id) { + doc.id = getId(doc.id); } + for (const key in doc.fields) { + if (!doc.fields.hasOwnProperty(key)) { + continue; + } + const field = doc.fields[key]; + if (field === undefined || field === null) { + continue; + } - if (field.__type === "proxy" || field.__type === "prefetch_proxy") { - field.fieldId = getId(field.fieldId); - } else if (field.__type === "script" || field.__type === "computed") { - if (field.captures) { - field.captures.fieldId = getId(field.captures.fieldId); + if (field.__type === "proxy" || field.__type === "prefetch_proxy") { + field.fieldId = getId(field.fieldId); + } else if (field.__type === "script" || field.__type === "computed") { + if (field.captures) { + field.captures.fieldId = getId(field.captures.fieldId); + } + } else if (field.__type === "list") { + mapFn(field); + } else if (typeof field === "string") { + const re = /("(?:dataD|d)ocumentId"\s*:\s*")([\w\-]*)"/g; + doc.fields[key] = (field as any).replace(re, (match: any, p1: string, p2: string) => { + return `${p1}${getId(p2)}"`; + }); + } else if (field.__type === "RichTextField") { + const re = /("href"\s*:\s*")(.*?)"/g; + field.Data = field.Data.replace(re, (match: any, p1: string, p2: string) => { + return `${p1}${getId(p2)}"`; + }); } - } else if (field.__type === "list") { - mapFn(field); - } else if (typeof field === "string") { - const re = /("(?:dataD|d)ocumentId"\s*:\s*")([\w\-]*)"/g; - doc.fields[key] = (field as any).replace(re, (match: any, p1: string, p2: string) => { - return `${p1}${getId(p2)}"`; - }); - } else if (field.__type === "RichTextField") { - const re = /("href"\s*:\s*")(.*?)"/g; - field.Data = field.Data.replace(re, (match: any, p1: string, p2: string) => { - return `${p1}${getId(p2)}"`; - }); } - } - }; - form.parse(req, async (err, fields, files) => { - remap = fields.remap !== "false"; - let id: string = ""; - try { - for (const name in files) { - const path_2 = files[name].path; - const zip = new AdmZip(path_2); - zip.getEntries().forEach((entry: any) => { - if (!entry.entryName.startsWith("files/")) return; - let dirname = path.dirname(entry.entryName) + "/"; - let extname = path.extname(entry.entryName); - let basename = path.basename(entry.entryName).split(".")[0]; - // zip.extractEntryTo(dirname + basename + "_o" + extname, __dirname + RouteStore.public, true, false); - // zip.extractEntryTo(dirname + basename + "_s" + extname, __dirname + RouteStore.public, true, false); - // zip.extractEntryTo(dirname + basename + "_m" + extname, __dirname + RouteStore.public, true, false); - // zip.extractEntryTo(dirname + basename + "_l" + extname, __dirname + RouteStore.public, true, false); + }; + form.parse(req, async (err, fields, files) => { + remap = fields.remap !== "false"; + let id: string = ""; + try { + for (const name in files) { + const path_2 = files[name].path; + const zip = new AdmZip(path_2); + zip.getEntries().forEach((entry: any) => { + if (!entry.entryName.startsWith("files/")) return; + let dirname = path.dirname(entry.entryName) + "/"; + let extname = path.extname(entry.entryName); + let basename = path.basename(entry.entryName).split(".")[0]; + // zip.extractEntryTo(dirname + basename + "_o" + extname, __dirname + RouteStore.public, true, false); + // zip.extractEntryTo(dirname + basename + "_s" + extname, __dirname + RouteStore.public, true, false); + // zip.extractEntryTo(dirname + basename + "_m" + extname, __dirname + RouteStore.public, true, false); + // zip.extractEntryTo(dirname + basename + "_l" + extname, __dirname + RouteStore.public, true, false); + try { + zip.extractEntryTo(entry.entryName, __dirname + RouteStore.public, true, false); + dirname = "/" + dirname; + + fs.createReadStream(__dirname + RouteStore.public + dirname + basename + extname).pipe(fs.createWriteStream(__dirname + RouteStore.public + dirname + basename + "_o" + extname)); + fs.createReadStream(__dirname + RouteStore.public + dirname + basename + extname).pipe(fs.createWriteStream(__dirname + RouteStore.public + dirname + basename + "_s" + extname)); + fs.createReadStream(__dirname + RouteStore.public + dirname + basename + extname).pipe(fs.createWriteStream(__dirname + RouteStore.public + dirname + basename + "_m" + extname)); + fs.createReadStream(__dirname + RouteStore.public + dirname + basename + extname).pipe(fs.createWriteStream(__dirname + RouteStore.public + dirname + basename + "_l" + extname)); + } catch (e) { + console.log(e); + } + }); + const json = zip.getEntry("doc.json"); + let docs: any; try { - zip.extractEntryTo(entry.entryName, __dirname + RouteStore.public, true, false); - dirname = "/" + dirname; - - fs.createReadStream(__dirname + RouteStore.public + dirname + basename + extname).pipe(fs.createWriteStream(__dirname + RouteStore.public + dirname + basename + "_o" + extname)); - fs.createReadStream(__dirname + RouteStore.public + dirname + basename + extname).pipe(fs.createWriteStream(__dirname + RouteStore.public + dirname + basename + "_s" + extname)); - fs.createReadStream(__dirname + RouteStore.public + dirname + basename + extname).pipe(fs.createWriteStream(__dirname + RouteStore.public + dirname + basename + "_m" + extname)); - fs.createReadStream(__dirname + RouteStore.public + dirname + basename + extname).pipe(fs.createWriteStream(__dirname + RouteStore.public + dirname + basename + "_l" + extname)); - } catch (e) { - console.log(e); + let data = JSON.parse(json.getData().toString("utf8")); + docs = data.docs; + id = data.id; + docs = Object.keys(docs).map(key => docs[key]); + docs.forEach(mapFn); + await Promise.all(docs.map((doc: any) => new Promise(res => Database.Instance.replace(doc.id, doc, (err, r) => { + err && console.log(err); + res(); + }, true, "newDocuments")))); + } catch (e) { console.log(e); } + fs.unlink(path_2, () => { }); + } + if (id) { + res.send(JSON.stringify(getId(id))); + } else { + res.send(JSON.stringify("error")); + } + } catch (e) { console.log(e); } + }); + } + }) + + router.addSupervisedRoute({ + method: Method.GET, + subscription: "/whosOnline", + onValidation: ({ res }) => { + let users: any = { active: {}, inactive: {} }; + const now = Date.now(); + + for (const user in timeMap) { + const time = timeMap[user]; + const key = ((now - time) / 1000) < (60 * 5) ? "active" : "inactive"; + users[key][user] = `Last active ${msToTime(now - time)} ago`; + } + + res.send(users); + } + }); + + router.addSupervisedRoute({ + method: Method.GET, + subscription: new RouteSubscriber("/thumbnail").add("filename"), + onValidation: ({ req, res }) => { + let filename = req.params.filename; + let noExt = filename.substring(0, filename.length - ".png".length); + let pagenumber = parseInt(noExt.split('-')[1]); + fs.exists(uploadDirectory + filename, (exists: boolean) => { + console.log(`${uploadDirectory + filename} ${exists ? "exists" : "does not exist"}`); + if (exists) { + let input = fs.createReadStream(uploadDirectory + filename); + probe(input, (err: any, result: any) => { + if (err) { + console.log(err); + console.log(`error on ${filename}`); + return; } + res.send({ path: "/files/" + filename, width: result.width, height: result.height }); }); - const json = zip.getEntry("doc.json"); - let docs: any; - try { - let data = JSON.parse(json.getData().toString("utf8")); - docs = data.docs; - id = data.id; - docs = Object.keys(docs).map(key => docs[key]); - docs.forEach(mapFn); - await Promise.all(docs.map((doc: any) => new Promise(res => Database.Instance.replace(doc.id, doc, (err, r) => { - err && console.log(err); - res(); - }, true, "newDocuments")))); - } catch (e) { console.log(e); } - fs.unlink(path_2, () => { }); } - if (id) { - res.send(JSON.stringify(getId(id))); - } else { - res.send(JSON.stringify("error")); + else { + LoadPage(uploadDirectory + filename.substring(0, filename.length - noExt.split('-')[1].length - ".PNG".length - 1) + ".pdf", pagenumber, res); } - } catch (e) { console.log(e); } - }); - }); - - DashServer.get("/whosOnline", (req, res) => { - let users: any = { active: {}, inactive: {} }; - const now = Date.now(); - - for (const user in timeMap) { - const time = timeMap[user]; - const key = ((now - time) / 1000) < (60 * 5) ? "active" : "inactive"; - users[key][user] = `Last active ${msToTime(now - time)} ago`; + }); } - - res.send(users); - }); - DashServer.get("/thumbnail/:filename", (req, res) => { - let filename = req.params.filename; - let noExt = filename.substring(0, filename.length - ".png".length); - let pagenumber = parseInt(noExt.split('-')[1]); - fs.exists(uploadDirectory + filename, (exists: boolean) => { - console.log(`${uploadDirectory + filename} ${exists ? "exists" : "does not exist"}`); - if (exists) { - let input = fs.createReadStream(uploadDirectory + filename); - probe(input, (err: any, result: any) => { - if (err) { - console.log(err); - console.log(`error on ${filename}`); - return; - } - res.send({ path: "/files/" + filename, width: result.width, height: result.height }); - }); - } - else { - LoadPage(uploadDirectory + filename.substring(0, filename.length - noExt.split('-')[1].length - ".PNG".length - 1) + ".pdf", pagenumber, res); - } - }); }); function LoadPage(file: string, pageNumber: number, res: Response) { @@ -498,41 +514,44 @@ async function PreliminaryFunctions() { router.addSupervisedRoute({ method: Method.GET, subscription: RouteStore.root, - onValidation: (_user, _req, res) => res.redirect(RouteStore.home) + onValidation: ({ res }) => res.redirect(RouteStore.home) }); router.addSupervisedRoute({ method: Method.GET, subscription: RouteStore.getUsers, - onValidation: async (_user, _req, res) => { + onValidation: async ({ res }) => { const cursor = await Database.Instance.query({}, { email: 1, userDocumentId: 1 }, "users"); const results = await cursor.toArray(); res.send(results.map(user => ({ email: user.email, userDocumentId: user.userDocumentId }))); - }, + } }); + const serve: OnUnauthenticated = ({ req, res }) => { + let detector = new mobileDetect(req.headers['user-agent'] || ""); + let filename = detector.mobile() !== null ? 'mobile/image.html' : 'index.html'; + res.sendFile(path.join(__dirname, '../../deploy/' + filename)); + } + router.addSupervisedRoute({ method: Method.GET, - subscription: [RouteStore.home, RouteStore.openDocumentWithId], - onValidation: (_user, req, res) => { - let detector = new mobileDetect(req.headers['user-agent'] || ""); - let filename = detector.mobile() !== null ? 'mobile/image.html' : 'index.html'; - res.sendFile(path.join(__dirname, '../../deploy/' + filename)); - }, + subscription: [RouteStore.home, new RouteSubscriber("/doc").add("docId")], + onValidation: serve, + onGuestAccess: serve }); router.addSupervisedRoute({ method: Method.GET, subscription: RouteStore.getUserDocumentId, - onValidation: (user, _req, res) => res.send(user.userDocumentId), - onRejection: (_req, res) => res.send(undefined) + onValidation: ({ res, user }) => res.send(user.userDocumentId), + onRejection: ({ res }) => res.send(undefined) }); router.addSupervisedRoute({ method: Method.GET, subscription: RouteStore.getCurrUser, - onValidation: (user, _req, res) => { res.send(JSON.stringify(user)); }, - onRejection: (_req, res) => res.send(JSON.stringify({ id: "__guest__", email: "" })) + onValidation: ({ res, user }) => { res.send(JSON.stringify(user)); }, + onRejection: ({ res }) => res.send(JSON.stringify({ id: "__guest__", email: "" })) }); const ServicesApiKeyMap = new Map([ @@ -544,7 +563,7 @@ async function PreliminaryFunctions() { router.addSupervisedRoute({ method: Method.GET, subscription: new RouteSubscriber(RouteStore.cognitiveServices).add('requestedservice'), - onValidation: (_user, req, res) => { + onValidation: ({ req, res }) => { let service = req.params.requestedservice; res.send(ServicesApiKeyMap.get(service)); } @@ -583,7 +602,7 @@ async function PreliminaryFunctions() { router.addSupervisedRoute({ method: Method.POST, subscription: RouteStore.upload, - onValidation: (_user, req, res) => { + onValidation: ({ req, res }) => { let form = new formidable.IncomingForm(); form.uploadDir = uploadDirectory; form.keepExtensions = true; @@ -621,7 +640,7 @@ async function PreliminaryFunctions() { router.addSupervisedRoute({ method: Method.POST, subscription: RouteStore.inspectImage, - onValidation: async (_user, req, res) => { + onValidation: async ({ req, res }) => { const { source } = req.body; if (typeof source === "string") { const uploadInformation = await DashUploadUtils.UploadImage(source); @@ -634,7 +653,7 @@ async function PreliminaryFunctions() { router.addSupervisedRoute({ method: Method.POST, subscription: RouteStore.dataUriToImage, - onValidation: (_user, req, res) => { + onValidation: ({ req, res }) => { const uri = req.body.uri; const filename = req.body.name; if (!uri || !filename) { @@ -670,27 +689,10 @@ async function PreliminaryFunctions() { } }); - const headerCharRegex = /[^\t\x20-\x7e\x80-\xff]/; - DashServer.use(RouteStore.corsProxy, (req, res) => { - req.pipe(request(decodeURIComponent(req.url.substring(1)))).on("response", res => { - const headers = Object.keys(res.headers); - headers.forEach(headerName => { - const header = res.headers[headerName]; - if (Array.isArray(header)) { - res.headers[headerName] = header.filter(h => !headerCharRegex.test(h)); - } else if (header) { - if (headerCharRegex.test(header as any)) { - delete res.headers[headerName]; - } - } - }); - }).pipe(res); - }); - router.addSupervisedRoute({ method: Method.GET, subscription: RouteStore.delete, - onValidation: (_user, _req, res, isRelease) => { + onValidation: ({ res, isRelease }) => { if (isRelease) { return _permission_denied(res, deletionPermissionError); } @@ -701,7 +703,7 @@ async function PreliminaryFunctions() { router.addSupervisedRoute({ method: Method.GET, subscription: RouteStore.deleteAll, - onValidation: (_user, _req, res, isRelease) => { + onValidation: ({ res, isRelease }) => { if (isRelease) { return _permission_denied(res, deletionPermissionError); } @@ -813,27 +815,34 @@ async function PreliminaryFunctions() { ["update", (api, params) => api.batchUpdate(params)], ]); - DashServer.post(RouteStore.googleDocs + "/:sector/:action", (req, res) => { - let sector: GoogleApiServerUtils.Service = req.params.sector as GoogleApiServerUtils.Service; - let action: GoogleApiServerUtils.Action = req.params.action as GoogleApiServerUtils.Action; - GoogleApiServerUtils.GetEndpoint(GoogleApiServerUtils.Service[sector], { credentialsPath, userId: req.headers.userId as string }).then(endpoint => { - let handler = EndpointHandlerMap.get(action); - if (endpoint && handler) { - let execute = handler(endpoint, req.body).then( - response => res.send(response.data), - rejection => res.send(rejection) - ); - execute.catch(exception => res.send(exception)); - return; - } - res.send(undefined); - }); + router.addSupervisedRoute({ + method: Method.POST, + subscription: new RouteSubscriber(RouteStore.googleDocs).add("sector", "action"), + onValidation: ({ req, res }) => { + let sector: GoogleApiServerUtils.Service = req.params.sector as GoogleApiServerUtils.Service; + let action: GoogleApiServerUtils.Action = req.params.action as GoogleApiServerUtils.Action; + GoogleApiServerUtils.GetEndpoint(GoogleApiServerUtils.Service[sector], { credentialsPath, userId: req.headers.userId as string }).then(endpoint => { + let handler = EndpointHandlerMap.get(action); + if (endpoint && handler) { + let execute = handler(endpoint, req.body).then( + response => res.send(response.data), + rejection => res.send(rejection) + ); + execute.catch(exception => res.send(exception)); + return; + } + res.send(undefined); + }); + } }); router.addSupervisedRoute({ method: Method.GET, subscription: RouteStore.readGoogleAccessToken, - onValidation: async (user, _req, res) => { + onValidation: async ({ user, res }) => { + if (!user) { + return res.send(undefined); + } const userId = user.id; const token = await Database.Auxiliary.GoogleAuthenticationToken.Fetch(userId); const information = { credentialsPath, userId }; @@ -847,7 +856,10 @@ async function PreliminaryFunctions() { router.addSupervisedRoute({ method: Method.POST, subscription: RouteStore.writeGoogleAccessToken, - onValidation: async (user, req, res) => { + onValidation: async ({ user, req, res }) => { + if (!user) { + return res.send(undefined); + } const userId = user.id; const information = { credentialsPath, userId }; res.send(await GoogleApiServerUtils.ProcessClientSideCode(information, req.body.authenticationCode)); @@ -861,8 +873,11 @@ async function PreliminaryFunctions() { router.addSupervisedRoute({ method: Method.POST, subscription: RouteStore.googlePhotosMediaUpload, - onValidation: async (user, req, res) => { + onValidation: async ({ user, req, res }) => { const { media } = req.body; + if (!user) { + return res.send(undefined); + } const userId = user.id; if (!userId) { return _error(res, userIdError); @@ -914,50 +929,62 @@ async function PreliminaryFunctions() { const requestError = "Unable to execute download: the body's media items were malformed."; const deletionPermissionError = "Cannot perform specialized delete outside of the development environment!"; - DashServer.get("/deleteWithAux", async (_req, res) => { - if (release) { - return _permission_denied(res, deletionPermissionError); + router.addSupervisedRoute({ + method: Method.GET, + subscription: "/deleteWithAux", + onValidation: async ({ res, isRelease }) => { + if (isRelease) { + return _permission_denied(res, deletionPermissionError); + } + await Database.Auxiliary.DeleteAll(); + res.redirect(RouteStore.delete); } - await Database.Auxiliary.DeleteAll(); - res.redirect(RouteStore.delete); - }); + }) - DashServer.get("/deleteWithGoogleCredentials", async (req, res) => { - if (release) { - return _permission_denied(res, deletionPermissionError); + router.addSupervisedRoute({ + method: Method.GET, + subscription: "/deleteWithGoogleCredentials", + onValidation: async ({ res, isRelease }) => { + if (isRelease) { + return _permission_denied(res, deletionPermissionError); + } + await Database.Auxiliary.GoogleAuthenticationToken.DeleteAll(); + res.redirect(RouteStore.delete); } - await Database.Auxiliary.GoogleAuthenticationToken.DeleteAll(); - res.redirect(RouteStore.delete); }); const UploadError = (count: number) => `Unable to upload ${count} images to Dash's server`; - DashServer.post(RouteStore.googlePhotosMediaDownload, async (req, res) => { - const contents: { mediaItems: MediaItem[] } = req.body; - let failed = 0; - if (contents) { - const completed: Opt[] = []; - for (let item of contents.mediaItems) { - const { contentSize, ...attributes } = await DashUploadUtils.InspectImage(item.baseUrl); - const found: Opt = await Database.Auxiliary.QueryUploadHistory(contentSize!); - if (!found) { - const upload = await DashUploadUtils.UploadInspectedImage({ contentSize, ...attributes }, item.filename, prefix).catch(error => _error(res, downloadError, error)); - if (upload) { - completed.push(upload); - await Database.Auxiliary.LogUpload(upload); + router.addSupervisedRoute({ + method: Method.POST, + subscription: RouteStore.googlePhotosMediaDownload, + onValidation: async ({ req, res }) => { + const contents: { mediaItems: MediaItem[] } = req.body; + let failed = 0; + if (contents) { + const completed: Opt[] = []; + for (let item of contents.mediaItems) { + const { contentSize, ...attributes } = await DashUploadUtils.InspectImage(item.baseUrl); + const found: Opt = await Database.Auxiliary.QueryUploadHistory(contentSize!); + if (!found) { + const upload = await DashUploadUtils.UploadInspectedImage({ contentSize, ...attributes }, item.filename, prefix).catch(error => _error(res, downloadError, error)); + if (upload) { + completed.push(upload); + await Database.Auxiliary.LogUpload(upload); + } else { + failed++; + } } else { - failed++; + completed.push(found); } - } else { - completed.push(found); } + if (failed) { + return _error(res, UploadError(failed)); + } + return _success(res, completed); } - if (failed) { - return _error(res, UploadError(failed)); - } - return _success(res, completed); + _invalid(res, requestError); } - _invalid(res, requestError); - }); + }) const suffixMap: { [type: string]: (string | [string, string | ((json: any) => any)]) } = { "number": "_n", -- cgit v1.2.3-70-g09d2 From e20756093c7f3e15795af6b71a4fae3092926452 Mon Sep 17 00:00:00 2001 From: Sam Wilkins <35748010+samwilkins333@users.noreply.github.com> Date: Thu, 17 Oct 2019 03:04:09 -0400 Subject: try execute --- src/server/RouteManager.ts | 28 +++++++++++----------------- 1 file changed, 11 insertions(+), 17 deletions(-) (limited to 'src') diff --git a/src/server/RouteManager.ts b/src/server/RouteManager.ts index 626014d1a..1e6717348 100644 --- a/src/server/RouteManager.ts +++ b/src/server/RouteManager.ts @@ -3,7 +3,6 @@ import { RouteStore } from "./RouteStore"; import { DashUserModel } from "./authentication/models/user_model"; import * as express from 'express'; import * as qs from 'query-string'; -import { Opt } from "../new_fields/Doc"; export default class RouteManager { private server: express.Express; @@ -30,33 +29,28 @@ export default class RouteManager { addSupervisedRoute(initializer: RouteInitializer) { const { method, subscription, onValidation, onRejection, onError, onGuestAccess } = initializer; const isRelease = this._isRelease; - let abstracted = async (req: express.Request, res: express.Response) => { + let supervised = async (req: express.Request, res: express.Response) => { const { user, originalUrl: target } = req; const core = { req, res, isRelease: isRelease }; - if (user) { + const tryExecute = async (target: any, args: any) => { try { - await onValidation({ ...core, user: user as any }); + await target(args); } catch (e) { if (onError) { onError({ ...core, error: e }); } else { - _error(res, `The server encountered an internal error handling ${target}.`, e); + _error(res, `The server encountered an internal error when serving ${target}.`, e); } } + } + if (user) { + await tryExecute(onValidation, { ...core, user: user as any }); } else { if (isGuestAccess(req) && onGuestAccess) { - await onGuestAccess(core); + await tryExecute(onGuestAccess, core); } else { req.session!.target = target; - try { - await (onRejection || LoginRedirect)(core); - } catch (e) { - if (onError) { - onError({ ...core, error: e }); - } else { - _error(res, `The server encountered an internal error when rejecting ${target}.`, e); - } - } + await tryExecute(onRejection || LoginRedirect, core); } } }; @@ -69,10 +63,10 @@ export default class RouteManager { } switch (method) { case Method.GET: - this.server.get(route, abstracted); + this.server.get(route, supervised); break; case Method.POST: - this.server.post(route, abstracted); + this.server.post(route, supervised); break; } }; -- cgit v1.2.3-70-g09d2 From a432dd429540f5e2b5e1efe7cb766ee96d0f857d Mon Sep 17 00:00:00 2001 From: Sam Wilkins <35748010+samwilkins333@users.noreply.github.com> Date: Thu, 17 Oct 2019 03:05:41 -0400 Subject: logical short ciruit --- src/server/RouteManager.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/server/RouteManager.ts b/src/server/RouteManager.ts index 1e6717348..879b115ac 100644 --- a/src/server/RouteManager.ts +++ b/src/server/RouteManager.ts @@ -46,7 +46,7 @@ export default class RouteManager { if (user) { await tryExecute(onValidation, { ...core, user: user as any }); } else { - if (isGuestAccess(req) && onGuestAccess) { + if (onGuestAccess && isGuestAccess(req)) { await tryExecute(onGuestAccess, core); } else { req.session!.target = target; -- cgit v1.2.3-70-g09d2 From 19ebd515630155e95318dc3a8801727d54f2db6e Mon Sep 17 00:00:00 2001 From: Sam Wilkins <35748010+samwilkins333@users.noreply.github.com> Date: Thu, 17 Oct 2019 03:09:16 -0400 Subject: reorder --- src/server/RouteManager.ts | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) (limited to 'src') diff --git a/src/server/RouteManager.ts b/src/server/RouteManager.ts index 879b115ac..5755c1f7e 100644 --- a/src/server/RouteManager.ts +++ b/src/server/RouteManager.ts @@ -4,6 +4,21 @@ import { DashUserModel } from "./authentication/models/user_model"; import * as express from 'express'; import * as qs from 'query-string'; +export enum Method { + GET, + POST +} + +export interface CoreArguments { + req: express.Request, + res: express.Response, + isRelease: boolean; +} + +export type OnValidation = (core: CoreArguments & { user: DashUserModel }) => any | Promise; +export type OnUnauthenticated = (core: CoreArguments) => any | Promise; +export type OnError = (core: CoreArguments & { error: any }) => any | Promise; + export default class RouteManager { private server: express.Express; private _isRelease: boolean; @@ -31,7 +46,7 @@ export default class RouteManager { const isRelease = this._isRelease; let supervised = async (req: express.Request, res: express.Response) => { const { user, originalUrl: target } = req; - const core = { req, res, isRelease: isRelease }; + const core = { req, res, isRelease }; const tryExecute = async (target: any, args: any) => { try { await target(args); @@ -79,21 +94,6 @@ export default class RouteManager { } -export enum Method { - GET, - POST -} - -export interface CoreArguments { - req: express.Request, - res: express.Response, - isRelease: boolean; -} - -export type OnValidation = (core: CoreArguments & { user: DashUserModel }) => any | Promise; -export type OnUnauthenticated = (core: CoreArguments) => any | Promise; -export type OnError = (core: CoreArguments & { error: any }) => any | Promise; - const LoginRedirect: OnUnauthenticated = ({ res }) => res.redirect(RouteStore.login); export interface RouteInitializer { -- cgit v1.2.3-70-g09d2 From e385c9b0285a1d015917eca49dfc190d9810c8d9 Mon Sep 17 00:00:00 2001 From: Sam Wilkins <35748010+samwilkins333@users.noreply.github.com> Date: Thu, 17 Oct 2019 03:19:36 -0400 Subject: sig int log --- src/server/database.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'src') diff --git a/src/server/database.ts b/src/server/database.ts index 44c49d03e..12626e594 100644 --- a/src/server/database.ts +++ b/src/server/database.ts @@ -26,7 +26,7 @@ export namespace Database { const { connection } = mongoose; process.on('SIGINT', () => { connection.close(() => { - console.log('Mongoose default connection disconnected through app termination'); + console.log(`SIGINT closed mongoose connection at ${url}`); process.exit(0); }); }); @@ -34,10 +34,10 @@ export namespace Database { await new Promise((resolve, reject) => { connection.on('error', reject); connection.on('disconnected', () => { - console.log(`Mongoose connection at ${url} now closed`); + console.log(`disconnecting mongoose connection at ${url}`); }); connection.on('connected', () => { - console.log(`Mongoose established default connection at ${url}`); + console.log(`mongoose established default connection at ${url}`); resolve(); }); mongoose.connect(url, { useNewUrlParser: true }); -- cgit v1.2.3-70-g09d2 From df7ed1e41472909e802116adaa285281ec7588ee Mon Sep 17 00:00:00 2001 From: Sam Wilkins <35748010+samwilkins333@users.noreply.github.com> Date: Thu, 17 Oct 2019 04:01:15 -0400 Subject: streamlined --- src/server/RouteManager.ts | 59 ++++++++++++++++++++-------------------------- src/server/index.ts | 18 +++++++++++--- 2 files changed, 40 insertions(+), 37 deletions(-) (limited to 'src') diff --git a/src/server/RouteManager.ts b/src/server/RouteManager.ts index 5755c1f7e..54f9cc460 100644 --- a/src/server/RouteManager.ts +++ b/src/server/RouteManager.ts @@ -2,7 +2,7 @@ import RouteSubscriber from "./RouteSubscriber"; import { RouteStore } from "./RouteStore"; import { DashUserModel } from "./authentication/models/user_model"; import * as express from 'express'; -import * as qs from 'query-string'; +import { Opt } from "../new_fields/Doc"; export enum Method { GET, @@ -19,6 +19,14 @@ export type OnValidation = (core: CoreArguments & { user: DashUserModel }) => an export type OnUnauthenticated = (core: CoreArguments) => any | Promise; export type OnError = (core: CoreArguments & { error: any }) => any | Promise; +export interface RouteInitializer { + method: Method; + subscription: string | RouteSubscriber | (string | RouteSubscriber)[]; + onValidation: OnValidation; + onUnauthenticated?: OnUnauthenticated; + onError?: OnError; +} + export default class RouteManager { private server: express.Express; private _isRelease: boolean; @@ -42,14 +50,15 @@ export default class RouteManager { * @param subscribers the forward slash prepended path names (reference and add to RouteStore.ts) that will all invoke the given @param handler */ addSupervisedRoute(initializer: RouteInitializer) { - const { method, subscription, onValidation, onRejection, onError, onGuestAccess } = initializer; + const { method, subscription, onValidation, onUnauthenticated, onError } = initializer; const isRelease = this._isRelease; let supervised = async (req: express.Request, res: express.Response) => { const { user, originalUrl: target } = req; const core = { req, res, isRelease }; - const tryExecute = async (target: any, args: any) => { + const tryExecute = async (target: (args: any) => T | Promise, args: any) => { try { - await target(args); + const result = await target(args); + return result; } catch (e) { if (onError) { onError({ ...core, error: e }); @@ -61,13 +70,17 @@ export default class RouteManager { if (user) { await tryExecute(onValidation, { ...core, user: user as any }); } else { - if (onGuestAccess && isGuestAccess(req)) { - await tryExecute(onGuestAccess, core); + req.session!.target = target; + if (!onUnauthenticated) { + res.redirect(RouteStore.login); } else { - req.session!.target = target; - await tryExecute(onRejection || LoginRedirect, core); + await tryExecute(onUnauthenticated, core); } } + const warning = `request to ${target} fell through - this is a fallback response`; + if (!res.headersSent) { + res.send({ warning }); + } }; const subscribe = (subscriber: RouteSubscriber | string) => { let route: string; @@ -94,30 +107,6 @@ export default class RouteManager { } -const LoginRedirect: OnUnauthenticated = ({ res }) => res.redirect(RouteStore.login); - -export interface RouteInitializer { - method: Method; - subscription: string | RouteSubscriber | (string | RouteSubscriber)[]; - onValidation: OnValidation; - onRejection?: OnUnauthenticated; - onGuestAccess?: OnUnauthenticated; - onError?: OnError; -} - -const isSharedDocAccess = (target: string) => { - const shared = qs.parse(qs.extract(target), { sort: false }).sharing === "true"; - const docAccess = target.startsWith("/doc/"); - return shared && docAccess; -}; - -const isGuestAccess = (req: express.Request) => { - if (isSharedDocAccess(req.originalUrl)) { - return true; - } - return false; -} - export const STATUS = { OK: 200, BAD_REQUEST: 400, @@ -139,7 +128,9 @@ export function _invalid(res: express.Response, message: string) { res.status(STATUS.BAD_REQUEST).send(); } -export function _permission_denied(res: express.Response, message: string) { - res.statusMessage = message; +export function _permission_denied(res: express.Response, message?: string) { + if (message) { + res.statusMessage = message; + } res.status(STATUS.BAD_REQUEST).send("Permission Denied!"); } diff --git a/src/server/index.ts b/src/server/index.ts index bba8fc292..81e236894 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -36,6 +36,8 @@ import InitializeServer from './Initialization'; import { Method, _success, _permission_denied, _error, _invalid, OnUnauthenticated } from './RouteManager'; import { command_line } from './ActionUtilities'; var findInFiles = require('find-in-files'); +import * as qs from 'query-string'; + let youtubeApiKey: string; @@ -537,21 +539,31 @@ async function PreliminaryFunctions() { method: Method.GET, subscription: [RouteStore.home, new RouteSubscriber("/doc").add("docId")], onValidation: serve, - onGuestAccess: serve + onUnauthenticated: ({ req, ...remaining }) => { + const { originalUrl: target } = req; + const sharing = qs.parse(qs.extract(req.originalUrl), { sort: false }).sharing === "true"; + const docAccess = target.startsWith("/doc/"); + if (sharing && docAccess) { + serve({ req, ...remaining }); + } + } }); router.addSupervisedRoute({ method: Method.GET, subscription: RouteStore.getUserDocumentId, onValidation: ({ res, user }) => res.send(user.userDocumentId), - onRejection: ({ res }) => res.send(undefined) + onUnauthenticated: ({ res }) => _permission_denied(res) }); router.addSupervisedRoute({ method: Method.GET, subscription: RouteStore.getCurrUser, onValidation: ({ res, user }) => { res.send(JSON.stringify(user)); }, - onRejection: ({ res }) => res.send(JSON.stringify({ id: "__guest__", email: "" })) + onUnauthenticated: ({ res }) => { + res.send(JSON.stringify({ id: "__guest__", email: "" })) + return true; + } }); const ServicesApiKeyMap = new Map([ -- cgit v1.2.3-70-g09d2 From 8884e5cf68c3ad34e23a539201fddda169d70262 Mon Sep 17 00:00:00 2001 From: Sam Wilkins <35748010+samwilkins333@users.noreply.github.com> Date: Thu, 17 Oct 2019 04:19:02 -0400 Subject: tweak timeout for fallback response --- src/server/RouteManager.ts | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) (limited to 'src') diff --git a/src/server/RouteManager.ts b/src/server/RouteManager.ts index 54f9cc460..37eaded0d 100644 --- a/src/server/RouteManager.ts +++ b/src/server/RouteManager.ts @@ -55,10 +55,9 @@ export default class RouteManager { let supervised = async (req: express.Request, res: express.Response) => { const { user, originalUrl: target } = req; const core = { req, res, isRelease }; - const tryExecute = async (target: (args: any) => T | Promise, args: any) => { + const tryExecute = async (target: (args: any) => any | Promise, args: any) => { try { - const result = await target(args); - return result; + await target(args); } catch (e) { if (onError) { onError({ ...core, error: e }); @@ -71,16 +70,18 @@ export default class RouteManager { await tryExecute(onValidation, { ...core, user: user as any }); } else { req.session!.target = target; - if (!onUnauthenticated) { - res.redirect(RouteStore.login); - } else { + if (onUnauthenticated) { await tryExecute(onUnauthenticated, core); + } else { + res.redirect(RouteStore.login); } } - const warning = `request to ${target} fell through - this is a fallback response`; - if (!res.headersSent) { - res.send({ warning }); - } + setTimeout(() => { + if (!res.headersSent) { + const warning = `request to ${target} fell through - this is a fallback response`; + res.send({ warning }); + } + }, 1000); }; const subscribe = (subscriber: RouteSubscriber | string) => { let route: string; -- cgit v1.2.3-70-g09d2 From 37fa239403f58de77a5c860ff53909dc624beae0 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Thu, 17 Oct 2019 14:11:15 -0400 Subject: linter errors, small fixes --- src/server/RouteManager.ts | 8 ++++---- src/server/index.ts | 17 +++++++---------- 2 files changed, 11 insertions(+), 14 deletions(-) (limited to 'src') diff --git a/src/server/RouteManager.ts b/src/server/RouteManager.ts index 37eaded0d..a3841249b 100644 --- a/src/server/RouteManager.ts +++ b/src/server/RouteManager.ts @@ -10,8 +10,8 @@ export enum Method { } export interface CoreArguments { - req: express.Request, - res: express.Response, + req: express.Request; + res: express.Response; isRelease: boolean; } @@ -65,9 +65,9 @@ export default class RouteManager { _error(res, `The server encountered an internal error when serving ${target}.`, e); } } - } + }; if (user) { - await tryExecute(onValidation, { ...core, user: user as any }); + await tryExecute(onValidation, { ...core, user }); } else { req.session!.target = target; if (onUnauthenticated) { diff --git a/src/server/index.ts b/src/server/index.ts index 81e236894..70add4ab2 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -322,7 +322,7 @@ async function PreliminaryFunctions() { }); zip.finalize(); } - }) + }); router.addSupervisedRoute({ method: Method.POST, @@ -427,7 +427,7 @@ async function PreliminaryFunctions() { } catch (e) { console.log(e); } }); } - }) + }); router.addSupervisedRoute({ method: Method.GET, @@ -533,7 +533,7 @@ async function PreliminaryFunctions() { let detector = new mobileDetect(req.headers['user-agent'] || ""); let filename = detector.mobile() !== null ? 'mobile/image.html' : 'index.html'; res.sendFile(path.join(__dirname, '../../deploy/' + filename)); - } + }; router.addSupervisedRoute({ method: Method.GET, @@ -559,11 +559,8 @@ async function PreliminaryFunctions() { router.addSupervisedRoute({ method: Method.GET, subscription: RouteStore.getCurrUser, - onValidation: ({ res, user }) => { res.send(JSON.stringify(user)); }, - onUnauthenticated: ({ res }) => { - res.send(JSON.stringify({ id: "__guest__", email: "" })) - return true; - } + onValidation: ({ res, user }) => res.send(JSON.stringify(user)), + onUnauthenticated: ({ res }) => res.send(JSON.stringify({ id: "__guest__", email: "" })) }); const ServicesApiKeyMap = new Map([ @@ -951,7 +948,7 @@ async function PreliminaryFunctions() { await Database.Auxiliary.DeleteAll(); res.redirect(RouteStore.delete); } - }) + }); router.addSupervisedRoute({ method: Method.GET, @@ -996,7 +993,7 @@ async function PreliminaryFunctions() { } _invalid(res, requestError); } - }) + }); const suffixMap: { [type: string]: (string | [string, string | ((json: any) => any)]) } = { "number": "_n", -- cgit v1.2.3-70-g09d2 From 369f5d5e5619b5ecddba4c44a6d134f3444ca544 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Thu, 17 Oct 2019 14:29:02 -0400 Subject: cleanup --- src/server/index.ts | 10 ---------- 1 file changed, 10 deletions(-) (limited to 'src') diff --git a/src/server/index.ts b/src/server/index.ts index 70add4ab2..c7eece703 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -553,7 +553,6 @@ async function PreliminaryFunctions() { method: Method.GET, subscription: RouteStore.getUserDocumentId, onValidation: ({ res, user }) => res.send(user.userDocumentId), - onUnauthenticated: ({ res }) => _permission_denied(res) }); router.addSupervisedRoute({ @@ -849,9 +848,6 @@ async function PreliminaryFunctions() { method: Method.GET, subscription: RouteStore.readGoogleAccessToken, onValidation: async ({ user, res }) => { - if (!user) { - return res.send(undefined); - } const userId = user.id; const token = await Database.Auxiliary.GoogleAuthenticationToken.Fetch(userId); const information = { credentialsPath, userId }; @@ -866,9 +862,6 @@ async function PreliminaryFunctions() { method: Method.POST, subscription: RouteStore.writeGoogleAccessToken, onValidation: async ({ user, req, res }) => { - if (!user) { - return res.send(undefined); - } const userId = user.id; const information = { credentialsPath, userId }; res.send(await GoogleApiServerUtils.ProcessClientSideCode(information, req.body.authenticationCode)); @@ -884,9 +877,6 @@ async function PreliminaryFunctions() { subscription: RouteStore.googlePhotosMediaUpload, onValidation: async ({ user, req, res }) => { const { media } = req.body; - if (!user) { - return res.send(undefined); - } const userId = user.id; if (!userId) { return _error(res, userIdError); -- cgit v1.2.3-70-g09d2 From f42620a33f76f8fdcf7417498f3fb2c1588c064d Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Thu, 17 Oct 2019 14:29:29 -0400 Subject: saved --- src/server/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/server/index.ts b/src/server/index.ts index c7eece703..2f4e65b46 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -552,7 +552,7 @@ async function PreliminaryFunctions() { router.addSupervisedRoute({ method: Method.GET, subscription: RouteStore.getUserDocumentId, - onValidation: ({ res, user }) => res.send(user.userDocumentId), + onValidation: ({ res, user }) => res.send(user.userDocumentId) }); router.addSupervisedRoute({ -- cgit v1.2.3-70-g09d2 From cee55b4a1b13909d55708eee6c364206ae7c0d4f Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Tue, 22 Oct 2019 19:39:19 -0400 Subject: api managers and web socket initial refactoring --- src/server/ApiManagers/ApiManager.ts | 7 + src/server/ApiManagers/SearchManager.ts | 49 +++++ src/server/ApiManagers/UserManager.ts | 40 ++++ src/server/ApiManagers/UtilManager.ts | 54 ++++++ src/server/Initialization.ts | 4 +- src/server/Websocket/Websocket.ts | 213 +++++++++++++++++++++ src/server/index.ts | 327 ++------------------------------ 7 files changed, 378 insertions(+), 316 deletions(-) create mode 100644 src/server/ApiManagers/ApiManager.ts create mode 100644 src/server/ApiManagers/SearchManager.ts create mode 100644 src/server/ApiManagers/UserManager.ts create mode 100644 src/server/ApiManagers/UtilManager.ts create mode 100644 src/server/Websocket/Websocket.ts (limited to 'src') diff --git a/src/server/ApiManagers/ApiManager.ts b/src/server/ApiManagers/ApiManager.ts new file mode 100644 index 000000000..264c78a17 --- /dev/null +++ b/src/server/ApiManagers/ApiManager.ts @@ -0,0 +1,7 @@ +import RouteManager from "../RouteManager"; + +export default abstract class ApiManager { + + public abstract register(router: RouteManager): void; + +} \ No newline at end of file diff --git a/src/server/ApiManagers/SearchManager.ts b/src/server/ApiManagers/SearchManager.ts new file mode 100644 index 000000000..15b87204c --- /dev/null +++ b/src/server/ApiManagers/SearchManager.ts @@ -0,0 +1,49 @@ +import ApiManager from "./ApiManager"; +import RouteManager, { Method } from "../RouteManager"; +import { Search } from "../Search"; +var findInFiles = require('find-in-files'); +import * as path from 'path'; +import { uploadDirectory } from ".."; + +export default class SearchManager extends ApiManager { + + public register(router: RouteManager): void { + + router.addSupervisedRoute({ + method: Method.GET, + subscription: "/textsearch", + onValidation: async ({ req, res }) => { + let q = req.query.q; + if (q === undefined) { + res.send([]); + return; + } + let results = await findInFiles.find({ 'term': q, 'flags': 'ig' }, uploadDirectory + "text", ".txt$"); + let resObj: { ids: string[], numFound: number, lines: string[] } = { ids: [], numFound: 0, lines: [] }; + for (var result in results) { + resObj.ids.push(path.basename(result, ".txt").replace(/upload_/, "")); + resObj.lines.push(results[result].line); + resObj.numFound++; + } + res.send(resObj); + } + }); + + router.addSupervisedRoute({ + method: Method.GET, + subscription: "/search", + onValidation: async ({ req, res }) => { + const solrQuery: any = {}; + ["q", "fq", "start", "rows", "hl", "hl.fl"].forEach(key => solrQuery[key] = req.query[key]); + if (solrQuery.q === undefined) { + res.send([]); + return; + } + let results = await Search.Instance.search(solrQuery); + res.send(results); + } + }); + + } + +} \ No newline at end of file diff --git a/src/server/ApiManagers/UserManager.ts b/src/server/ApiManagers/UserManager.ts new file mode 100644 index 000000000..bb8837dc6 --- /dev/null +++ b/src/server/ApiManagers/UserManager.ts @@ -0,0 +1,40 @@ +import ApiManager from "./ApiManager"; +import RouteManager, { Method } from "../RouteManager"; +import { WebSocket } from "../Websocket/Websocket"; + +export default class UserManager extends ApiManager { + + public register(router: RouteManager): void { + router.addSupervisedRoute({ + method: Method.GET, + subscription: "/whosOnline", + onValidation: ({ res }) => { + let users: any = { active: {}, inactive: {} }; + const now = Date.now(); + + const { timeMap } = WebSocket; + for (const user in timeMap) { + const time = timeMap[user]; + const key = ((now - time) / 1000) < (60 * 5) ? "active" : "inactive"; + users[key][user] = `Last active ${this.msToTime(now - time)} ago`; + } + + res.send(users); + } + }); + } + + private msToTime(duration: number) { + let milliseconds = Math.floor((duration % 1000) / 100), + seconds = Math.floor((duration / 1000) % 60), + minutes = Math.floor((duration / (1000 * 60)) % 60), + hours = Math.floor((duration / (1000 * 60 * 60)) % 24); + + let hoursS = (hours < 10) ? "0" + hours : hours; + let minutesS = (minutes < 10) ? "0" + minutes : minutes; + let secondsS = (seconds < 10) ? "0" + seconds : seconds; + + return hoursS + ":" + minutesS + ":" + secondsS + "." + milliseconds; + } + +} \ No newline at end of file diff --git a/src/server/ApiManagers/UtilManager.ts b/src/server/ApiManagers/UtilManager.ts new file mode 100644 index 000000000..79b904e8a --- /dev/null +++ b/src/server/ApiManagers/UtilManager.ts @@ -0,0 +1,54 @@ +import ApiManager from "./ApiManager"; +import RouteManager, { Method } from "../RouteManager"; +import { exec } from 'child_process'; +import { command_line } from "../ActionUtilities"; + +export default class UtilManager extends ApiManager { + + public register(router: RouteManager): void { + + router.addSupervisedRoute({ + method: Method.GET, + subscription: "/pull", + onValidation: ({ res }) => { + exec('"C:\\Program Files\\Git\\git-bash.exe" -c "git pull"', err => { + if (err) { + res.send(err.message); + return; + } + res.redirect("/"); + }); + } + }); + + router.addSupervisedRoute({ + method: Method.GET, + subscription: "/buxton", + onValidation: ({ res }) => { + let cwd = '../scraping/buxton'; + + let onResolved = (stdout: string) => { console.log(stdout); res.redirect("/"); }; + let onRejected = (err: any) => { console.error(err.message); res.send(err); }; + let tryPython3 = () => command_line('python3 scraper.py', cwd).then(onResolved, onRejected); + + command_line('python scraper.py', cwd).then(onResolved, tryPython3); + }, + }); + + router.addSupervisedRoute({ + method: Method.GET, + subscription: "/version", + onValidation: ({ res }) => { + exec('"C:\\Program Files\\Git\\bin\\git.exe" rev-parse HEAD', (err, stdout) => { + if (err) { + res.send(err.message); + return; + } + res.send(stdout); + }); + } + }); + + } + +} \ No newline at end of file diff --git a/src/server/Initialization.ts b/src/server/Initialization.ts index 9646dc195..e4c97cc48 100644 --- a/src/server/Initialization.ts +++ b/src/server/Initialization.ts @@ -37,9 +37,11 @@ export default async function InitializeServer(options: InitializationOptions) { registerAuthenticationRoutes(server); registerCorsProxy(server); - routeSetter(new RouteManager(server, determineEnvironment())); + const isRelease = determineEnvironment(); + routeSetter(new RouteManager(server, isRelease)); server.listen(listenAtPort, () => console.log(`server started at http://localhost:${listenAtPort}`)); + return isRelease; } const week = 7 * 24 * 60 * 60 * 1000; diff --git a/src/server/Websocket/Websocket.ts b/src/server/Websocket/Websocket.ts new file mode 100644 index 000000000..2461dd8d5 --- /dev/null +++ b/src/server/Websocket/Websocket.ts @@ -0,0 +1,213 @@ +import { Utils } from "../../Utils"; +import { MessageStore, Transferable, Types, Diff, YoutubeQueryInput, YoutubeQueryTypes } from "../Message"; +import { Client } from "../Client"; +import { Socket } from "socket.io"; +import { Database } from "../database"; +import { Search } from "../Search"; +import io from 'socket.io'; +import YoutubeApi from "../apis/youtube/youtubeApiSample"; +import { youtubeApiKey } from ".."; + +export namespace WebSocket { + + interface Map { + [key: string]: Client; + } + let clients: Map = {}; + + export const socketMap = new Map(); + export const timeMap: { [id: string]: number } = {}; + + export function initialize(serverPort: number, isRelease: boolean) { + const endpoint = io(); + endpoint.listen(serverPort); + console.log(`listening on port ${serverPort}`); + + endpoint.on("connection", function (socket: Socket) { + socket.use((_packet, next) => { + let id = socketMap.get(socket); + if (id) { + timeMap[id] = Date.now(); + } + next(); + }); + + Utils.Emit(socket, MessageStore.Foo, "handshooken"); + + Utils.AddServerHandler(socket, MessageStore.Bar, guid => barReceived(socket, guid)); + Utils.AddServerHandler(socket, MessageStore.SetField, (args) => setField(socket, args)); + Utils.AddServerHandlerCallback(socket, MessageStore.GetField, getField); + Utils.AddServerHandlerCallback(socket, MessageStore.GetFields, getFields); + if (isRelease) { + Utils.AddServerHandler(socket, MessageStore.DeleteAll, deleteFields); + } + + Utils.AddServerHandler(socket, MessageStore.CreateField, CreateField); + Utils.AddServerHandlerCallback(socket, MessageStore.YoutubeApiQuery, HandleYoutubeQuery); + Utils.AddServerHandler(socket, MessageStore.UpdateField, diff => UpdateField(socket, diff)); + Utils.AddServerHandler(socket, MessageStore.DeleteField, id => DeleteField(socket, id)); + Utils.AddServerHandler(socket, MessageStore.DeleteFields, ids => DeleteFields(socket, ids)); + Utils.AddServerHandlerCallback(socket, MessageStore.GetRefField, GetRefField); + Utils.AddServerHandlerCallback(socket, MessageStore.GetRefFields, GetRefFields); + }); + + } + + function HandleYoutubeQuery([query, callback]: [YoutubeQueryInput, (result?: any[]) => void]) { + switch (query.type) { + case YoutubeQueryTypes.Channels: + YoutubeApi.authorizedGetChannel(youtubeApiKey); + break; + case YoutubeQueryTypes.SearchVideo: + YoutubeApi.authorizedGetVideos(youtubeApiKey, query.userInput, callback); + case YoutubeQueryTypes.VideoDetails: + YoutubeApi.authorizedGetVideoDetails(youtubeApiKey, query.videoIds, callback); + } + } + + export async function deleteFields() { + await Database.Instance.deleteAll(); + await Search.Instance.clear(); + await Database.Instance.deleteAll('newDocuments'); + } + + export async function deleteAll() { + await Database.Instance.deleteAll(); + await Database.Instance.deleteAll('newDocuments'); + await Database.Instance.deleteAll('sessions'); + await Database.Instance.deleteAll('users'); + await Search.Instance.clear(); + } + + function barReceived(socket: SocketIO.Socket, guid: string) { + clients[guid] = new Client(guid.toString()); + console.log(`User ${guid} has connected`); + socketMap.set(socket, guid); + } + + function getField([id, callback]: [string, (result?: Transferable) => void]) { + Database.Instance.getDocument(id, (result?: Transferable) => + callback(result ? result : undefined)); + } + + function getFields([ids, callback]: [string[], (result: Transferable[]) => void]) { + Database.Instance.getDocuments(ids, callback); + } + + function setField(socket: Socket, newValue: Transferable) { + Database.Instance.update(newValue.id, newValue, () => + socket.broadcast.emit(MessageStore.SetField.Message, newValue)); + if (newValue.type === Types.Text) { + Search.Instance.updateDocument({ id: newValue.id, data: (newValue as any).data }); + console.log("set field"); + console.log("checking in"); + } + } + + function GetRefField([id, callback]: [string, (result?: Transferable) => void]) { + Database.Instance.getDocument(id, callback, "newDocuments"); + } + + function GetRefFields([ids, callback]: [string[], (result?: Transferable[]) => void]) { + Database.Instance.getDocuments(ids, callback, "newDocuments"); + } + + const suffixMap: { [type: string]: (string | [string, string | ((json: any) => any)]) } = { + "number": "_n", + "string": "_t", + "boolean": "_b", + "image": ["_t", "url"], + "video": ["_t", "url"], + "pdf": ["_t", "url"], + "audio": ["_t", "url"], + "web": ["_t", "url"], + "date": ["_d", value => new Date(value.date).toISOString()], + "proxy": ["_i", "fieldId"], + "list": ["_l", list => { + const results = []; + for (const value of list.fields) { + const term = ToSearchTerm(value); + if (term) { + results.push(term.value); + } + } + return results.length ? results : null; + }] + }; + + function ToSearchTerm(val: any): { suffix: string, value: any } | undefined { + if (val === null || val === undefined) { + return; + } + const type = val.__type || typeof val; + let suffix = suffixMap[type]; + if (!suffix) { + return; + } + + if (Array.isArray(suffix)) { + const accessor = suffix[1]; + if (typeof accessor === "function") { + val = accessor(val); + } else { + val = val[accessor]; + } + suffix = suffix[0]; + } + + return { suffix, value: val }; + } + + function getSuffix(value: string | [string, any]): string { + return typeof value === "string" ? value : value[0]; + } + + function UpdateField(socket: Socket, diff: Diff) { + Database.Instance.update(diff.id, diff.diff, + () => socket.broadcast.emit(MessageStore.UpdateField.Message, diff), false, "newDocuments"); + const docfield = diff.diff.$set; + if (!docfield) { + return; + } + const update: any = { id: diff.id }; + let dynfield = false; + for (let key in docfield) { + if (!key.startsWith("fields.")) continue; + dynfield = true; + let val = docfield[key]; + key = key.substring(7); + Object.values(suffixMap).forEach(suf => update[key + getSuffix(suf)] = { set: null }); + let term = ToSearchTerm(val); + if (term !== undefined) { + let { suffix, value } = term; + update[key + suffix] = { set: value }; + } + } + if (dynfield) { + Search.Instance.updateDocument(update); + } + } + + function DeleteField(socket: Socket, id: string) { + Database.Instance.delete({ _id: id }, "newDocuments").then(() => { + socket.broadcast.emit(MessageStore.DeleteField.Message, id); + }); + + Search.Instance.deleteDocuments([id]); + } + + function DeleteFields(socket: Socket, ids: string[]) { + Database.Instance.delete({ _id: { $in: ids } }, "newDocuments").then(() => { + socket.broadcast.emit(MessageStore.DeleteFields.Message, ids); + }); + + Search.Instance.deleteDocuments(ids); + + } + + function CreateField(newValue: any) { + Database.Instance.insert(newValue, "newDocuments"); + } + +} + diff --git a/src/server/index.ts b/src/server/index.ts index 2f4e65b46..ae0e79458 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -1,5 +1,4 @@ require('dotenv').config(); -import { exec } from 'child_process'; import * as formidable from 'formidable'; import * as fs from 'fs'; import * as sharp from 'sharp'; @@ -7,10 +6,6 @@ import * as Pdfjs from 'pdfjs-dist'; const imageDataUri = require('image-data-uri'); import * as mobileDetect from 'mobile-detect'; import * as path from 'path'; -import * as io from 'socket.io'; -import { Socket } from 'socket.io'; -import { Utils } from '../Utils'; -import { Client } from './Client'; import { Database } from './database'; import { MessageStore, Transferable, Types, Diff, YoutubeQueryTypes as YoutubeQueryType, YoutubeQueryInput } from "./Message"; import { RouteStore } from './RouteStore'; @@ -34,12 +29,13 @@ import { reject } from 'bluebird'; import RouteSubscriber from './RouteSubscriber'; import InitializeServer from './Initialization'; import { Method, _success, _permission_denied, _error, _invalid, OnUnauthenticated } from './RouteManager'; -import { command_line } from './ActionUtilities'; -var findInFiles = require('find-in-files'); import * as qs from 'query-string'; +import UtilManager from './ApiManagers/UtilManager'; +import SearchManager from './ApiManagers/SearchManager'; +import UserManager from './ApiManagers/UserManager'; +import { WebSocket } from './Websocket/Websocket'; - -let youtubeApiKey: string; +export let youtubeApiKey: string; export type Hierarchy = { [id: string]: string | Hierarchy }; export type ZipMutator = (file: Archiver.Archiver) => void | Promise; @@ -53,7 +49,7 @@ export interface NewMediaItem { const pngTypes = [".png", ".PNG"]; const jpgTypes = [".jpg", ".JPG", ".jpeg", ".JPEG"]; -const uploadDirectory = __dirname + "/public/files/"; +export const uploadDirectory = __dirname + "/public/files/"; const pdfDirectory = uploadDirectory + "text"; const solrURL = "http://localhost:8983/solr/#/dash"; @@ -70,96 +66,11 @@ async function PreliminaryFunctions() { await InitializeServer({ listenAtPort: 1050, routeSetter: router => { + new UtilManager().register(router); + new SearchManager().register(router); + new UserManager().register(router); - router.addSupervisedRoute({ - method: Method.GET, - subscription: "/pull", - onValidation: ({ res }) => { - exec('"C:\\Program Files\\Git\\git-bash.exe" -c "git pull"', err => { - if (err) { - res.send(err.message); - return; - } - res.redirect("/"); - }); - } - }); - - router.addSupervisedRoute({ - method: Method.GET, - subscription: "/textsearch", - onValidation: async ({ req, res }) => { - let q = req.query.q; - if (q === undefined) { - res.send([]); - return; - } - let results = await findInFiles.find({ 'term': q, 'flags': 'ig' }, uploadDirectory + "text", ".txt$"); - let resObj: { ids: string[], numFound: number, lines: string[] } = { ids: [], numFound: 0, lines: [] }; - for (var result in results) { - resObj.ids.push(path.basename(result, ".txt").replace(/upload_/, "")); - resObj.lines.push(results[result].line); - resObj.numFound++; - } - res.send(resObj); - } - }); - - router.addSupervisedRoute({ - method: Method.GET, - subscription: "/buxton", - onValidation: ({ res }) => { - let cwd = '../scraping/buxton'; - - let onResolved = (stdout: string) => { console.log(stdout); res.redirect("/"); }; - let onRejected = (err: any) => { console.error(err.message); res.send(err); }; - let tryPython3 = () => command_line('python3 scraper.py', cwd).then(onResolved, onRejected); - - command_line('python scraper.py', cwd).then(onResolved, tryPython3); - }, - }); - - router.addSupervisedRoute({ - method: Method.GET, - subscription: "/version", - onValidation: ({ res }) => { - exec('"C:\\Program Files\\Git\\bin\\git.exe" rev-parse HEAD', (err, stdout) => { - if (err) { - res.send(err.message); - return; - } - res.send(stdout); - }); - } - }); - - router.addSupervisedRoute({ - method: Method.GET, - subscription: "/search", - onValidation: async ({ req, res }) => { - const solrQuery: any = {}; - ["q", "fq", "start", "rows", "hl", "hl.fl"].forEach(key => solrQuery[key] = req.query[key]); - if (solrQuery.q === undefined) { - res.send([]); - return; - } - let results = await Search.Instance.search(solrQuery); - res.send(results); - } - }); - - function msToTime(duration: number) { - let milliseconds = Math.floor((duration % 1000) / 100), - seconds = Math.floor((duration / 1000) % 60), - minutes = Math.floor((duration / (1000 * 60)) % 60), - hours = Math.floor((duration / (1000 * 60 * 60)) % 24); - - let hoursS = (hours < 10) ? "0" + hours : hours; - let minutesS = (minutes < 10) ? "0" + minutes : minutes; - let secondsS = (seconds < 10) ? "0" + seconds : seconds; - - return hoursS + ":" + minutesS + ":" + secondsS + "." + milliseconds; - } + WebSocket.initialize(serverPort, router.isRelease); async function getDocs(id: string) { const files = new Set(); @@ -429,23 +340,6 @@ async function PreliminaryFunctions() { } }); - router.addSupervisedRoute({ - method: Method.GET, - subscription: "/whosOnline", - onValidation: ({ res }) => { - let users: any = { active: {}, inactive: {} }; - const now = Date.now(); - - for (const user in timeMap) { - const time = timeMap[user]; - const key = ((now - time) / 1000) < (60 * 5) ? "active" : "inactive"; - users[key][user] = `Last active ${msToTime(now - time)} ago`; - } - - res.send(users); - } - }); - router.addSupervisedRoute({ method: Method.GET, subscription: new RouteSubscriber("/thumbnail").add("filename"), @@ -704,7 +598,7 @@ async function PreliminaryFunctions() { if (isRelease) { return _permission_denied(res, deletionPermissionError); } - deleteFields().then(() => res.redirect(RouteStore.home)); + WebSocket.deleteFields().then(() => res.redirect(RouteStore.home)); } }); @@ -715,106 +609,10 @@ async function PreliminaryFunctions() { if (isRelease) { return _permission_denied(res, deletionPermissionError); } - deleteAll().then(() => res.redirect(RouteStore.home)); - } - }); - - const server = io(); - interface Map { - [key: string]: Client; - } - let clients: Map = {}; - - let socketMap = new Map(); - let timeMap: { [id: string]: number } = {}; - - server.on("connection", function (socket: Socket) { - socket.use((packet, next) => { - let id = socketMap.get(socket); - if (id) { - timeMap[id] = Date.now(); - } - next(); - }); - - Utils.Emit(socket, MessageStore.Foo, "handshooken"); - - Utils.AddServerHandler(socket, MessageStore.Bar, guid => barReceived(socket, guid)); - Utils.AddServerHandler(socket, MessageStore.SetField, (args) => setField(socket, args)); - Utils.AddServerHandlerCallback(socket, MessageStore.GetField, getField); - Utils.AddServerHandlerCallback(socket, MessageStore.GetFields, getFields); - if (!router.release) { - Utils.AddServerHandler(socket, MessageStore.DeleteAll, deleteFields); + WebSocket.deleteAll().then(() => res.redirect(RouteStore.home)); } - - Utils.AddServerHandler(socket, MessageStore.CreateField, CreateField); - Utils.AddServerHandlerCallback(socket, MessageStore.YoutubeApiQuery, HandleYoutubeQuery); - Utils.AddServerHandler(socket, MessageStore.UpdateField, diff => UpdateField(socket, diff)); - Utils.AddServerHandler(socket, MessageStore.DeleteField, id => DeleteField(socket, id)); - Utils.AddServerHandler(socket, MessageStore.DeleteFields, ids => DeleteFields(socket, ids)); - Utils.AddServerHandlerCallback(socket, MessageStore.GetRefField, GetRefField); - Utils.AddServerHandlerCallback(socket, MessageStore.GetRefFields, GetRefFields); }); - async function deleteFields() { - await Database.Instance.deleteAll(); - await Search.Instance.clear(); - await Database.Instance.deleteAll('newDocuments'); - } - - async function deleteAll() { - await Database.Instance.deleteAll(); - await Database.Instance.deleteAll('newDocuments'); - await Database.Instance.deleteAll('sessions'); - await Database.Instance.deleteAll('users'); - await Search.Instance.clear(); - } - - function barReceived(socket: SocketIO.Socket, guid: string) { - clients[guid] = new Client(guid.toString()); - console.log(`User ${guid} has connected`); - socketMap.set(socket, guid); - } - - function getField([id, callback]: [string, (result?: Transferable) => void]) { - Database.Instance.getDocument(id, (result?: Transferable) => - callback(result ? result : undefined)); - } - - function getFields([ids, callback]: [string[], (result: Transferable[]) => void]) { - Database.Instance.getDocuments(ids, callback); - } - - function setField(socket: Socket, newValue: Transferable) { - Database.Instance.update(newValue.id, newValue, () => - socket.broadcast.emit(MessageStore.SetField.Message, newValue)); - if (newValue.type === Types.Text) { - Search.Instance.updateDocument({ id: newValue.id, data: (newValue as any).data }); - console.log("set field"); - console.log("checking in"); - } - } - - function GetRefField([id, callback]: [string, (result?: Transferable) => void]) { - Database.Instance.getDocument(id, callback, "newDocuments"); - } - - function GetRefFields([ids, callback]: [string[], (result?: Transferable[]) => void]) { - Database.Instance.getDocuments(ids, callback, "newDocuments"); - } - - function HandleYoutubeQuery([query, callback]: [YoutubeQueryInput, (result?: any[]) => void]) { - switch (query.type) { - case YoutubeQueryType.Channels: - YoutubeApi.authorizedGetChannel(youtubeApiKey); - break; - case YoutubeQueryType.SearchVideo: - YoutubeApi.authorizedGetVideos(youtubeApiKey, query.userInput, callback); - case YoutubeQueryType.VideoDetails: - YoutubeApi.authorizedGetVideoDetails(youtubeApiKey, query.videoIds, callback); - } - } - const credentialsPath = path.join(__dirname, "./credentials/google_docs_credentials.json"); const EndpointHandlerMap = new Map([ @@ -984,107 +782,6 @@ async function PreliminaryFunctions() { _invalid(res, requestError); } }); - - const suffixMap: { [type: string]: (string | [string, string | ((json: any) => any)]) } = { - "number": "_n", - "string": "_t", - "boolean": "_b", - "image": ["_t", "url"], - "video": ["_t", "url"], - "pdf": ["_t", "url"], - "audio": ["_t", "url"], - "web": ["_t", "url"], - "date": ["_d", value => new Date(value.date).toISOString()], - "proxy": ["_i", "fieldId"], - "list": ["_l", list => { - const results = []; - for (const value of list.fields) { - const term = ToSearchTerm(value); - if (term) { - results.push(term.value); - } - } - return results.length ? results : null; - }] - }; - - function ToSearchTerm(val: any): { suffix: string, value: any } | undefined { - if (val === null || val === undefined) { - return; - } - const type = val.__type || typeof val; - let suffix = suffixMap[type]; - if (!suffix) { - return; - } - - if (Array.isArray(suffix)) { - const accessor = suffix[1]; - if (typeof accessor === "function") { - val = accessor(val); - } else { - val = val[accessor]; - } - suffix = suffix[0]; - } - - return { suffix, value: val }; - } - - function getSuffix(value: string | [string, any]): string { - return typeof value === "string" ? value : value[0]; - } - - function UpdateField(socket: Socket, diff: Diff) { - Database.Instance.update(diff.id, diff.diff, - () => socket.broadcast.emit(MessageStore.UpdateField.Message, diff), false, "newDocuments"); - const docfield = diff.diff.$set; - if (!docfield) { - return; - } - const update: any = { id: diff.id }; - let dynfield = false; - for (let key in docfield) { - if (!key.startsWith("fields.")) continue; - dynfield = true; - let val = docfield[key]; - key = key.substring(7); - Object.values(suffixMap).forEach(suf => update[key + getSuffix(suf)] = { set: null }); - let term = ToSearchTerm(val); - if (term !== undefined) { - let { suffix, value } = term; - update[key + suffix] = { set: value }; - } - } - if (dynfield) { - Search.Instance.updateDocument(update); - } - } - - function DeleteField(socket: Socket, id: string) { - Database.Instance.delete({ _id: id }, "newDocuments").then(() => { - socket.broadcast.emit(MessageStore.DeleteField.Message, id); - }); - - Search.Instance.deleteDocuments([id]); - } - - function DeleteFields(socket: Socket, ids: string[]) { - Database.Instance.delete({ _id: { $in: ids } }, "newDocuments").then(() => { - socket.broadcast.emit(MessageStore.DeleteFields.Message, ids); - }); - - Search.Instance.deleteDocuments(ids); - - } - - function CreateField(newValue: any) { - Database.Instance.insert(newValue, "newDocuments"); - } - - server.listen(serverPort); - console.log(`listening on port ${serverPort}`); } }); - })(); \ No newline at end of file -- cgit v1.2.3-70-g09d2 From fcf67616b9fd6f98d631f6c8eab31a19a2a2e86d Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Tue, 22 Oct 2019 19:45:33 -0400 Subject: start --- src/server/Initialization.ts | 3 +- src/server/RouteManager.ts | 2 +- src/server/index.ts | 1312 +++++++++++++++++++++--------------------- 3 files changed, 661 insertions(+), 656 deletions(-) (limited to 'src') diff --git a/src/server/Initialization.ts b/src/server/Initialization.ts index e4c97cc48..fbb5ae7a6 100644 --- a/src/server/Initialization.ts +++ b/src/server/Initialization.ts @@ -19,9 +19,10 @@ import * as whm from 'webpack-hot-middleware'; import * as fs from 'fs'; import * as request from 'request'; +export type RouteSetter = (server: RouteManager) => void; export interface InitializationOptions { listenAtPort: number; - routeSetter: (server: RouteManager) => void; + routeSetter: RouteSetter; } export default async function InitializeServer(options: InitializationOptions) { diff --git a/src/server/RouteManager.ts b/src/server/RouteManager.ts index a3841249b..b3864e89c 100644 --- a/src/server/RouteManager.ts +++ b/src/server/RouteManager.ts @@ -31,7 +31,7 @@ export default class RouteManager { private server: express.Express; private _isRelease: boolean; - public get release() { + public get isRelease() { return this._isRelease; } diff --git a/src/server/index.ts b/src/server/index.ts index ae0e79458..93f4238bc 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -7,7 +7,6 @@ const imageDataUri = require('image-data-uri'); import * as mobileDetect from 'mobile-detect'; import * as path from 'path'; import { Database } from './database'; -import { MessageStore, Transferable, Types, Diff, YoutubeQueryTypes as YoutubeQueryType, YoutubeQueryInput } from "./Message"; import { RouteStore } from './RouteStore'; import v4 = require('uuid/v4'); import { createCanvas } from "canvas"; @@ -28,7 +27,7 @@ import { ParsedPDF } from "./PdfTypes"; import { reject } from 'bluebird'; import RouteSubscriber from './RouteSubscriber'; import InitializeServer from './Initialization'; -import { Method, _success, _permission_denied, _error, _invalid, OnUnauthenticated } from './RouteManager'; +import RouteManager, { Method, _success, _permission_denied, _error, _invalid, OnUnauthenticated } from './RouteManager'; import * as qs from 'query-string'; import UtilManager from './ApiManagers/UtilManager'; import SearchManager from './ApiManagers/SearchManager'; @@ -53,735 +52,740 @@ export const uploadDirectory = __dirname + "/public/files/"; const pdfDirectory = uploadDirectory + "text"; const solrURL = "http://localhost:8983/solr/#/dash"; -YoutubeApi.readApiKey((apiKey: string) => youtubeApiKey = apiKey); +start(); + +async function start() { + await PreliminaryFunctions(); + await InitializeServer({ listenAtPort: 1050, routeSetter }); +} async function PreliminaryFunctions() { + await new Promise(resolve => { + YoutubeApi.readApiKey((apiKey: string) => { + youtubeApiKey = apiKey; + resolve(); + }); + }); await GoogleApiServerUtils.LoadOAuthClient(); await DashUploadUtils.createIfNotExists(pdfDirectory); await Database.tryInitializeConnection(); } -(async () => { - await PreliminaryFunctions(); - await InitializeServer({ - listenAtPort: 1050, - routeSetter: router => { - new UtilManager().register(router); - new SearchManager().register(router); - new UserManager().register(router); - - WebSocket.initialize(serverPort, router.isRelease); - - async function getDocs(id: string) { - const files = new Set(); - const docs: { [id: string]: any } = {}; - const fn = (doc: any): string[] => { - const id = doc.id; - if (typeof id === "string" && id.endsWith("Proto")) { - //Skip protos - return []; - } - const ids: string[] = []; - for (const key in doc.fields) { - if (!doc.fields.hasOwnProperty(key)) { - continue; - } - const field = doc.fields[key]; - if (field === undefined || field === null) { - continue; - } +function routeSetter(router: RouteManager) { + new UtilManager().register(router); + new SearchManager().register(router); + new UserManager().register(router); + + WebSocket.initialize(serverPort, router.isRelease); + + async function getDocs(id: string) { + const files = new Set(); + const docs: { [id: string]: any } = {}; + const fn = (doc: any): string[] => { + const id = doc.id; + if (typeof id === "string" && id.endsWith("Proto")) { + //Skip protos + return []; + } + const ids: string[] = []; + for (const key in doc.fields) { + if (!doc.fields.hasOwnProperty(key)) { + continue; + } + const field = doc.fields[key]; + if (field === undefined || field === null) { + continue; + } - if (field.__type === "proxy" || field.__type === "prefetch_proxy") { - ids.push(field.fieldId); - } else if (field.__type === "script" || field.__type === "computed") { - if (field.captures) { - ids.push(field.captures.fieldId); - } - } else if (field.__type === "list") { - ids.push(...fn(field)); - } else if (typeof field === "string") { - const re = /"(?:dataD|d)ocumentId"\s*:\s*"([\w\-]*)"/g; - let match: string[] | null; - while ((match = re.exec(field)) !== null) { - ids.push(match[1]); - } - } else if (field.__type === "RichTextField") { - const re = /"href"\s*:\s*"(.*?)"/g; - let match: string[] | null; - while ((match = re.exec(field.Data)) !== null) { - const urlString = match[1]; - const split = new URL(urlString).pathname.split("doc/"); - if (split.length > 1) { - ids.push(split[split.length - 1]); - } - } - const re2 = /"src"\s*:\s*"(.*?)"/g; - while ((match = re2.exec(field.Data)) !== null) { - const urlString = match[1]; - const pathname = new URL(urlString).pathname; - files.add(pathname); - } - } else if (["audio", "image", "video", "pdf", "web"].includes(field.__type)) { - const url = new URL(field.url); - const pathname = url.pathname; - files.add(pathname); + if (field.__type === "proxy" || field.__type === "prefetch_proxy") { + ids.push(field.fieldId); + } else if (field.__type === "script" || field.__type === "computed") { + if (field.captures) { + ids.push(field.captures.fieldId); + } + } else if (field.__type === "list") { + ids.push(...fn(field)); + } else if (typeof field === "string") { + const re = /"(?:dataD|d)ocumentId"\s*:\s*"([\w\-]*)"/g; + let match: string[] | null; + while ((match = re.exec(field)) !== null) { + ids.push(match[1]); + } + } else if (field.__type === "RichTextField") { + const re = /"href"\s*:\s*"(.*?)"/g; + let match: string[] | null; + while ((match = re.exec(field.Data)) !== null) { + const urlString = match[1]; + const split = new URL(urlString).pathname.split("doc/"); + if (split.length > 1) { + ids.push(split[split.length - 1]); } } - - if (doc.id) { - docs[doc.id] = doc; + const re2 = /"src"\s*:\s*"(.*?)"/g; + while ((match = re2.exec(field.Data)) !== null) { + const urlString = match[1]; + const pathname = new URL(urlString).pathname; + files.add(pathname); } - return ids; - }; - await Database.Instance.visit([id], fn); - return { id, docs, files }; + } else if (["audio", "image", "video", "pdf", "web"].includes(field.__type)) { + const url = new URL(field.url); + const pathname = url.pathname; + files.add(pathname); + } } - router.addSupervisedRoute({ - method: Method.GET, - subscription: new RouteSubscriber("/serializeDoc").add("docId"), - onValidation: async ({ req, res }) => { - const { docs, files } = await getDocs(req.params.docId); - res.send({ docs, files: Array.from(files) }); - } + if (doc.id) { + docs[doc.id] = doc; + } + return ids; + }; + await Database.Instance.visit([id], fn); + return { id, docs, files }; + } + + router.addSupervisedRoute({ + method: Method.GET, + subscription: new RouteSubscriber("/serializeDoc").add("docId"), + onValidation: async ({ req, res }) => { + const { docs, files } = await getDocs(req.params.docId); + res.send({ docs, files: Array.from(files) }); + } + }); + + router.addSupervisedRoute({ + method: Method.GET, + subscription: new RouteSubscriber(RouteStore.imageHierarchyExport).add('docId'), + onValidation: async ({ req, res }) => { + const id = req.params.docId; + const hierarchy: Hierarchy = {}; + await targetedVisitorRecursive(id, hierarchy); + BuildAndDispatchZip(res, async zip => { + await hierarchyTraverserRecursive(zip, hierarchy); }); + } + }); - router.addSupervisedRoute({ - method: Method.GET, - subscription: new RouteSubscriber(RouteStore.imageHierarchyExport).add('docId'), - onValidation: async ({ req, res }) => { - const id = req.params.docId; - const hierarchy: Hierarchy = {}; - await targetedVisitorRecursive(id, hierarchy); - BuildAndDispatchZip(res, async zip => { - await hierarchyTraverserRecursive(zip, hierarchy); - }); + const BuildAndDispatchZip = async (res: Response, mutator: ZipMutator): Promise => { + const zip = Archiver('zip'); + zip.pipe(res); + await mutator(zip); + return zip.finalize(); + }; + + const targetedVisitorRecursive = async (seedId: string, hierarchy: Hierarchy): Promise => { + const local: Hierarchy = {}; + const { title, data } = await getData(seedId); + const label = `${title} (${seedId})`; + if (Array.isArray(data)) { + hierarchy[label] = local; + await Promise.all(data.map(proxy => targetedVisitorRecursive(proxy.fieldId, local))); + } else { + hierarchy[label + path.extname(data)] = data; + } + }; + + const getData = async (seedId: string): Promise<{ data: string | any[], title: string }> => { + return new Promise<{ data: string | any[], title: string }>((resolve, reject) => { + Database.Instance.getDocument(seedId, async (result: any) => { + const { data, proto, title } = result.fields; + if (data) { + if (data.url) { + resolve({ data: data.url, title }); + } else if (data.fields) { + resolve({ data: data.fields, title }); + } else { + reject(); + } + } + if (proto) { + getData(proto.fieldId).then(resolve, reject); } }); + }); + }; - const BuildAndDispatchZip = async (res: Response, mutator: ZipMutator): Promise => { - const zip = Archiver('zip'); - zip.pipe(res); - await mutator(zip); - return zip.finalize(); - }; - - const targetedVisitorRecursive = async (seedId: string, hierarchy: Hierarchy): Promise => { - const local: Hierarchy = {}; - const { title, data } = await getData(seedId); - const label = `${title} (${seedId})`; - if (Array.isArray(data)) { - hierarchy[label] = local; - await Promise.all(data.map(proxy => targetedVisitorRecursive(proxy.fieldId, local))); + const hierarchyTraverserRecursive = async (file: Archiver.Archiver, hierarchy: Hierarchy, prefix = "Dash Export"): Promise => { + for (const key of Object.keys(hierarchy)) { + const result = hierarchy[key]; + if (typeof result === "string") { + let path: string; + let matches: RegExpExecArray | null; + if ((matches = /\:1050\/files\/(upload\_[\da-z]{32}.*)/g.exec(result)) !== null) { + path = `${__dirname}/public/files/${matches[1]}`; } else { - hierarchy[label + path.extname(data)] = data; + const information = await DashUploadUtils.UploadImage(result); + path = information.mediaPaths[0]; } - }; + file.file(path, { name: key, prefix }); + } else { + await hierarchyTraverserRecursive(file, result, `${prefix}/${key}`); + } + } + }; - const getData = async (seedId: string): Promise<{ data: string | any[], title: string }> => { - return new Promise<{ data: string | any[], title: string }>((resolve, reject) => { - Database.Instance.getDocument(seedId, async (result: any) => { - const { data, proto, title } = result.fields; - if (data) { - if (data.url) { - resolve({ data: data.url, title }); - } else if (data.fields) { - resolve({ data: data.fields, title }); - } else { - reject(); - } - } - if (proto) { - getData(proto.fieldId).then(resolve, reject); - } - }); - }); - }; + router.addSupervisedRoute({ + method: Method.GET, + subscription: new RouteSubscriber("/downloadId").add("docId"), + onValidation: async ({ req, res }) => { + res.set('Content-disposition', `attachment;`); + res.set('Content-Type', "application/zip"); + const { id, docs, files } = await getDocs(req.params.docId); + const docString = JSON.stringify({ id, docs }); + const zip = Archiver('zip'); + zip.pipe(res); + zip.append(docString, { name: "doc.json" }); + files.forEach(val => { + zip.file(__dirname + RouteStore.public + val, { name: val.substring(1) }); + }); + zip.finalize(); + } + }); - const hierarchyTraverserRecursive = async (file: Archiver.Archiver, hierarchy: Hierarchy, prefix = "Dash Export"): Promise => { - for (const key of Object.keys(hierarchy)) { - const result = hierarchy[key]; - if (typeof result === "string") { - let path: string; - let matches: RegExpExecArray | null; - if ((matches = /\:1050\/files\/(upload\_[\da-z]{32}.*)/g.exec(result)) !== null) { - path = `${__dirname}/public/files/${matches[1]}`; - } else { - const information = await DashUploadUtils.UploadImage(result); - path = information.mediaPaths[0]; - } - file.file(path, { name: key, prefix }); - } else { - await hierarchyTraverserRecursive(file, result, `${prefix}/${key}`); - } + router.addSupervisedRoute({ + method: Method.POST, + subscription: "/uploadDoc", + onValidation: ({ req, res }) => { + let form = new formidable.IncomingForm(); + form.keepExtensions = true; + // let path = req.body.path; + const ids: { [id: string]: string } = {}; + let remap = true; + const getId = (id: string): string => { + if (!remap) return id; + if (id.endsWith("Proto")) return id; + if (id in ids) { + return ids[id]; + } else { + return ids[id] = v4(); } }; - - router.addSupervisedRoute({ - method: Method.GET, - subscription: new RouteSubscriber("/downloadId").add("docId"), - onValidation: async ({ req, res }) => { - res.set('Content-disposition', `attachment;`); - res.set('Content-Type', "application/zip"); - const { id, docs, files } = await getDocs(req.params.docId); - const docString = JSON.stringify({ id, docs }); - const zip = Archiver('zip'); - zip.pipe(res); - zip.append(docString, { name: "doc.json" }); - files.forEach(val => { - zip.file(__dirname + RouteStore.public + val, { name: val.substring(1) }); - }); - zip.finalize(); + const mapFn = (doc: any) => { + if (doc.id) { + doc.id = getId(doc.id); } - }); + for (const key in doc.fields) { + if (!doc.fields.hasOwnProperty(key)) { + continue; + } + const field = doc.fields[key]; + if (field === undefined || field === null) { + continue; + } - router.addSupervisedRoute({ - method: Method.POST, - subscription: "/uploadDoc", - onValidation: ({ req, res }) => { - let form = new formidable.IncomingForm(); - form.keepExtensions = true; - // let path = req.body.path; - const ids: { [id: string]: string } = {}; - let remap = true; - const getId = (id: string): string => { - if (!remap) return id; - if (id.endsWith("Proto")) return id; - if (id in ids) { - return ids[id]; - } else { - return ids[id] = v4(); + if (field.__type === "proxy" || field.__type === "prefetch_proxy") { + field.fieldId = getId(field.fieldId); + } else if (field.__type === "script" || field.__type === "computed") { + if (field.captures) { + field.captures.fieldId = getId(field.captures.fieldId); } - }; - const mapFn = (doc: any) => { - if (doc.id) { - doc.id = getId(doc.id); - } - for (const key in doc.fields) { - if (!doc.fields.hasOwnProperty(key)) { - continue; - } - const field = doc.fields[key]; - if (field === undefined || field === null) { - continue; - } - - if (field.__type === "proxy" || field.__type === "prefetch_proxy") { - field.fieldId = getId(field.fieldId); - } else if (field.__type === "script" || field.__type === "computed") { - if (field.captures) { - field.captures.fieldId = getId(field.captures.fieldId); - } - } else if (field.__type === "list") { - mapFn(field); - } else if (typeof field === "string") { - const re = /("(?:dataD|d)ocumentId"\s*:\s*")([\w\-]*)"/g; - doc.fields[key] = (field as any).replace(re, (match: any, p1: string, p2: string) => { - return `${p1}${getId(p2)}"`; - }); - } else if (field.__type === "RichTextField") { - const re = /("href"\s*:\s*")(.*?)"/g; - field.Data = field.Data.replace(re, (match: any, p1: string, p2: string) => { - return `${p1}${getId(p2)}"`; - }); + } else if (field.__type === "list") { + mapFn(field); + } else if (typeof field === "string") { + const re = /("(?:dataD|d)ocumentId"\s*:\s*")([\w\-]*)"/g; + doc.fields[key] = (field as any).replace(re, (match: any, p1: string, p2: string) => { + return `${p1}${getId(p2)}"`; + }); + } else if (field.__type === "RichTextField") { + const re = /("href"\s*:\s*")(.*?)"/g; + field.Data = field.Data.replace(re, (match: any, p1: string, p2: string) => { + return `${p1}${getId(p2)}"`; + }); + } + } + }; + form.parse(req, async (err, fields, files) => { + remap = fields.remap !== "false"; + let id: string = ""; + try { + for (const name in files) { + const path_2 = files[name].path; + const zip = new AdmZip(path_2); + zip.getEntries().forEach((entry: any) => { + if (!entry.entryName.startsWith("files/")) return; + let dirname = path.dirname(entry.entryName) + "/"; + let extname = path.extname(entry.entryName); + let basename = path.basename(entry.entryName).split(".")[0]; + // zip.extractEntryTo(dirname + basename + "_o" + extname, __dirname + RouteStore.public, true, false); + // zip.extractEntryTo(dirname + basename + "_s" + extname, __dirname + RouteStore.public, true, false); + // zip.extractEntryTo(dirname + basename + "_m" + extname, __dirname + RouteStore.public, true, false); + // zip.extractEntryTo(dirname + basename + "_l" + extname, __dirname + RouteStore.public, true, false); + try { + zip.extractEntryTo(entry.entryName, __dirname + RouteStore.public, true, false); + dirname = "/" + dirname; + + fs.createReadStream(__dirname + RouteStore.public + dirname + basename + extname).pipe(fs.createWriteStream(__dirname + RouteStore.public + dirname + basename + "_o" + extname)); + fs.createReadStream(__dirname + RouteStore.public + dirname + basename + extname).pipe(fs.createWriteStream(__dirname + RouteStore.public + dirname + basename + "_s" + extname)); + fs.createReadStream(__dirname + RouteStore.public + dirname + basename + extname).pipe(fs.createWriteStream(__dirname + RouteStore.public + dirname + basename + "_m" + extname)); + fs.createReadStream(__dirname + RouteStore.public + dirname + basename + extname).pipe(fs.createWriteStream(__dirname + RouteStore.public + dirname + basename + "_l" + extname)); + } catch (e) { + console.log(e); } - } - }; - form.parse(req, async (err, fields, files) => { - remap = fields.remap !== "false"; - let id: string = ""; + }); + const json = zip.getEntry("doc.json"); + let docs: any; try { - for (const name in files) { - const path_2 = files[name].path; - const zip = new AdmZip(path_2); - zip.getEntries().forEach((entry: any) => { - if (!entry.entryName.startsWith("files/")) return; - let dirname = path.dirname(entry.entryName) + "/"; - let extname = path.extname(entry.entryName); - let basename = path.basename(entry.entryName).split(".")[0]; - // zip.extractEntryTo(dirname + basename + "_o" + extname, __dirname + RouteStore.public, true, false); - // zip.extractEntryTo(dirname + basename + "_s" + extname, __dirname + RouteStore.public, true, false); - // zip.extractEntryTo(dirname + basename + "_m" + extname, __dirname + RouteStore.public, true, false); - // zip.extractEntryTo(dirname + basename + "_l" + extname, __dirname + RouteStore.public, true, false); - try { - zip.extractEntryTo(entry.entryName, __dirname + RouteStore.public, true, false); - dirname = "/" + dirname; - - fs.createReadStream(__dirname + RouteStore.public + dirname + basename + extname).pipe(fs.createWriteStream(__dirname + RouteStore.public + dirname + basename + "_o" + extname)); - fs.createReadStream(__dirname + RouteStore.public + dirname + basename + extname).pipe(fs.createWriteStream(__dirname + RouteStore.public + dirname + basename + "_s" + extname)); - fs.createReadStream(__dirname + RouteStore.public + dirname + basename + extname).pipe(fs.createWriteStream(__dirname + RouteStore.public + dirname + basename + "_m" + extname)); - fs.createReadStream(__dirname + RouteStore.public + dirname + basename + extname).pipe(fs.createWriteStream(__dirname + RouteStore.public + dirname + basename + "_l" + extname)); - } catch (e) { - console.log(e); - } - }); - const json = zip.getEntry("doc.json"); - let docs: any; - try { - let data = JSON.parse(json.getData().toString("utf8")); - docs = data.docs; - id = data.id; - docs = Object.keys(docs).map(key => docs[key]); - docs.forEach(mapFn); - await Promise.all(docs.map((doc: any) => new Promise(res => Database.Instance.replace(doc.id, doc, (err, r) => { - err && console.log(err); - res(); - }, true, "newDocuments")))); - } catch (e) { console.log(e); } - fs.unlink(path_2, () => { }); - } - if (id) { - res.send(JSON.stringify(getId(id))); - } else { - res.send(JSON.stringify("error")); - } + let data = JSON.parse(json.getData().toString("utf8")); + docs = data.docs; + id = data.id; + docs = Object.keys(docs).map(key => docs[key]); + docs.forEach(mapFn); + await Promise.all(docs.map((doc: any) => new Promise(res => Database.Instance.replace(doc.id, doc, (err, r) => { + err && console.log(err); + res(); + }, true, "newDocuments")))); } catch (e) { console.log(e); } - }); - } + fs.unlink(path_2, () => { }); + } + if (id) { + res.send(JSON.stringify(getId(id))); + } else { + res.send(JSON.stringify("error")); + } + } catch (e) { console.log(e); } }); + } + }); - router.addSupervisedRoute({ - method: Method.GET, - subscription: new RouteSubscriber("/thumbnail").add("filename"), - onValidation: ({ req, res }) => { - let filename = req.params.filename; - let noExt = filename.substring(0, filename.length - ".png".length); - let pagenumber = parseInt(noExt.split('-')[1]); - fs.exists(uploadDirectory + filename, (exists: boolean) => { - console.log(`${uploadDirectory + filename} ${exists ? "exists" : "does not exist"}`); - if (exists) { - let input = fs.createReadStream(uploadDirectory + filename); - probe(input, (err: any, result: any) => { - if (err) { - console.log(err); - console.log(`error on ${filename}`); - return; - } - res.send({ path: "/files/" + filename, width: result.width, height: result.height }); - }); - } - else { - LoadPage(uploadDirectory + filename.substring(0, filename.length - noExt.split('-')[1].length - ".PNG".length - 1) + ".pdf", pagenumber, res); + router.addSupervisedRoute({ + method: Method.GET, + subscription: new RouteSubscriber("/thumbnail").add("filename"), + onValidation: ({ req, res }) => { + let filename = req.params.filename; + let noExt = filename.substring(0, filename.length - ".png".length); + let pagenumber = parseInt(noExt.split('-')[1]); + fs.exists(uploadDirectory + filename, (exists: boolean) => { + console.log(`${uploadDirectory + filename} ${exists ? "exists" : "does not exist"}`); + if (exists) { + let input = fs.createReadStream(uploadDirectory + filename); + probe(input, (err: any, result: any) => { + if (err) { + console.log(err); + console.log(`error on ${filename}`); + return; } + res.send({ path: "/files/" + filename, width: result.width, height: result.height }); }); } + else { + LoadPage(uploadDirectory + filename.substring(0, filename.length - noExt.split('-')[1].length - ".PNG".length - 1) + ".pdf", pagenumber, res); + } }); + } + }); - function LoadPage(file: string, pageNumber: number, res: Response) { - console.log(file); - Pdfjs.getDocument(file).promise - .then((pdf: Pdfjs.PDFDocumentProxy) => { - let factory = new NodeCanvasFactory(); - console.log(pageNumber); - pdf.getPage(pageNumber).then((page: Pdfjs.PDFPageProxy) => { - console.log("reading " + page); - let viewport = page.getViewport(1 as any); - let canvasAndContext = factory.create(viewport.width, viewport.height); - let renderContext = { - canvasContext: canvasAndContext.context, - viewport: viewport, - canvasFactory: factory - }; - console.log("read " + pageNumber); - - page.render(renderContext).promise - .then(() => { - console.log("saving " + pageNumber); - let stream = canvasAndContext.canvas.createPNGStream(); - let pngFile = `${file.substring(0, file.length - ".pdf".length)}-${pageNumber}.PNG`; - let out = fs.createWriteStream(pngFile); - stream.pipe(out); - out.on("finish", () => { - console.log(`Success! Saved to ${pngFile}`); - let name = path.basename(pngFile); - res.send({ path: "/files/" + name, width: viewport.width, height: viewport.height }); - }); - }, (reason: string) => { - console.error(reason + ` ${pageNumber}`); - }); + function LoadPage(file: string, pageNumber: number, res: Response) { + console.log(file); + Pdfjs.getDocument(file).promise + .then((pdf: Pdfjs.PDFDocumentProxy) => { + let factory = new NodeCanvasFactory(); + console.log(pageNumber); + pdf.getPage(pageNumber).then((page: Pdfjs.PDFPageProxy) => { + console.log("reading " + page); + let viewport = page.getViewport(1 as any); + let canvasAndContext = factory.create(viewport.width, viewport.height); + let renderContext = { + canvasContext: canvasAndContext.context, + viewport: viewport, + canvasFactory: factory + }; + console.log("read " + pageNumber); + + page.render(renderContext).promise + .then(() => { + console.log("saving " + pageNumber); + let stream = canvasAndContext.canvas.createPNGStream(); + let pngFile = `${file.substring(0, file.length - ".pdf".length)}-${pageNumber}.PNG`; + let out = fs.createWriteStream(pngFile); + stream.pipe(out); + out.on("finish", () => { + console.log(`Success! Saved to ${pngFile}`); + let name = path.basename(pngFile); + res.send({ path: "/files/" + name, width: viewport.width, height: viewport.height }); + }); + }, (reason: string) => { + console.error(reason + ` ${pageNumber}`); }); - }); - } - - /** - * Anyone attempting to navigate to localhost at this port will - * first have to log in. - */ - router.addSupervisedRoute({ - method: Method.GET, - subscription: RouteStore.root, - onValidation: ({ res }) => res.redirect(RouteStore.home) + }); }); + } + + /** + * Anyone attempting to navigate to localhost at this port will + * first have to log in. + */ + router.addSupervisedRoute({ + method: Method.GET, + subscription: RouteStore.root, + onValidation: ({ res }) => res.redirect(RouteStore.home) + }); - router.addSupervisedRoute({ - method: Method.GET, - subscription: RouteStore.getUsers, - onValidation: async ({ res }) => { - const cursor = await Database.Instance.query({}, { email: 1, userDocumentId: 1 }, "users"); - const results = await cursor.toArray(); - res.send(results.map(user => ({ email: user.email, userDocumentId: user.userDocumentId }))); - } - }); + router.addSupervisedRoute({ + method: Method.GET, + subscription: RouteStore.getUsers, + onValidation: async ({ res }) => { + const cursor = await Database.Instance.query({}, { email: 1, userDocumentId: 1 }, "users"); + const results = await cursor.toArray(); + res.send(results.map(user => ({ email: user.email, userDocumentId: user.userDocumentId }))); + } + }); - const serve: OnUnauthenticated = ({ req, res }) => { - let detector = new mobileDetect(req.headers['user-agent'] || ""); - let filename = detector.mobile() !== null ? 'mobile/image.html' : 'index.html'; - res.sendFile(path.join(__dirname, '../../deploy/' + filename)); - }; + const serve: OnUnauthenticated = ({ req, res }) => { + let detector = new mobileDetect(req.headers['user-agent'] || ""); + let filename = detector.mobile() !== null ? 'mobile/image.html' : 'index.html'; + res.sendFile(path.join(__dirname, '../../deploy/' + filename)); + }; - router.addSupervisedRoute({ - method: Method.GET, - subscription: [RouteStore.home, new RouteSubscriber("/doc").add("docId")], - onValidation: serve, - onUnauthenticated: ({ req, ...remaining }) => { - const { originalUrl: target } = req; - const sharing = qs.parse(qs.extract(req.originalUrl), { sort: false }).sharing === "true"; - const docAccess = target.startsWith("/doc/"); - if (sharing && docAccess) { - serve({ req, ...remaining }); - } - } - }); + router.addSupervisedRoute({ + method: Method.GET, + subscription: [RouteStore.home, new RouteSubscriber("/doc").add("docId")], + onValidation: serve, + onUnauthenticated: ({ req, ...remaining }) => { + const { originalUrl: target } = req; + const sharing = qs.parse(qs.extract(req.originalUrl), { sort: false }).sharing === "true"; + const docAccess = target.startsWith("/doc/"); + if (sharing && docAccess) { + serve({ req, ...remaining }); + } + } + }); - router.addSupervisedRoute({ - method: Method.GET, - subscription: RouteStore.getUserDocumentId, - onValidation: ({ res, user }) => res.send(user.userDocumentId) - }); + router.addSupervisedRoute({ + method: Method.GET, + subscription: RouteStore.getUserDocumentId, + onValidation: ({ res, user }) => res.send(user.userDocumentId) + }); - router.addSupervisedRoute({ - method: Method.GET, - subscription: RouteStore.getCurrUser, - onValidation: ({ res, user }) => res.send(JSON.stringify(user)), - onUnauthenticated: ({ res }) => res.send(JSON.stringify({ id: "__guest__", email: "" })) - }); + router.addSupervisedRoute({ + method: Method.GET, + subscription: RouteStore.getCurrUser, + onValidation: ({ res, user }) => res.send(JSON.stringify(user)), + onUnauthenticated: ({ res }) => res.send(JSON.stringify({ id: "__guest__", email: "" })) + }); - const ServicesApiKeyMap = new Map([ - ["face", process.env.FACE], - ["vision", process.env.VISION], - ["handwriting", process.env.HANDWRITING] - ]); - - router.addSupervisedRoute({ - method: Method.GET, - subscription: new RouteSubscriber(RouteStore.cognitiveServices).add('requestedservice'), - onValidation: ({ req, res }) => { - let service = req.params.requestedservice; - res.send(ServicesApiKeyMap.get(service)); - } - }); + const ServicesApiKeyMap = new Map([ + ["face", process.env.FACE], + ["vision", process.env.VISION], + ["handwriting", process.env.HANDWRITING] + ]); + + router.addSupervisedRoute({ + method: Method.GET, + subscription: new RouteSubscriber(RouteStore.cognitiveServices).add('requestedservice'), + onValidation: ({ req, res }) => { + let service = req.params.requestedservice; + res.send(ServicesApiKeyMap.get(service)); + } + }); - class NodeCanvasFactory { - create = (width: number, height: number) => { - var canvas = createCanvas(width, height); - var context = canvas.getContext('2d'); - return { - canvas: canvas, - context: context, - }; - } + class NodeCanvasFactory { + create = (width: number, height: number) => { + var canvas = createCanvas(width, height); + var context = canvas.getContext('2d'); + return { + canvas: canvas, + context: context, + }; + } - reset = (canvasAndContext: any, width: number, height: number) => { - canvasAndContext.canvas.width = width; - canvasAndContext.canvas.height = height; - } + reset = (canvasAndContext: any, width: number, height: number) => { + canvasAndContext.canvas.width = width; + canvasAndContext.canvas.height = height; + } + + destroy = (canvasAndContext: any) => { + canvasAndContext.canvas.width = 0; + canvasAndContext.canvas.height = 0; + canvasAndContext.canvas = null; + canvasAndContext.context = null; + } + } + + interface ImageFileResponse { + name: string; + path: string; + type: string; + exif: Opt; + } + + router.addSupervisedRoute({ + method: Method.POST, + subscription: RouteStore.upload, + onValidation: ({ req, res }) => { + let form = new formidable.IncomingForm(); + form.uploadDir = uploadDirectory; + form.keepExtensions = true; + form.parse(req, async (_err, _fields, files) => { + let results: ImageFileResponse[] = []; + for (const key in files) { + const { type, path: location, name } = files[key]; + const filename = path.basename(location); + let uploadInformation: Opt; + if (filename.endsWith(".pdf")) { + let dataBuffer = fs.readFileSync(uploadDirectory + filename); + const result: ParsedPDF = await pdf(dataBuffer); + await new Promise(resolve => { + const path = pdfDirectory + "/" + filename.substring(0, filename.length - ".pdf".length) + ".txt"; + fs.createWriteStream(path).write(result.text, error => { + if (!error) { + resolve(); + } else { + reject(error); + } + }); + }); + } else { + uploadInformation = await DashUploadUtils.UploadImage(uploadDirectory + filename, filename); + } + const exif = uploadInformation ? uploadInformation.exifData : undefined; + results.push({ name, type, path: `/files/${filename}`, exif }); - destroy = (canvasAndContext: any) => { - canvasAndContext.canvas.width = 0; - canvasAndContext.canvas.height = 0; - canvasAndContext.canvas = null; - canvasAndContext.context = null; } - } + _success(res, results); + }); + } + }); - interface ImageFileResponse { - name: string; - path: string; - type: string; - exif: Opt; + router.addSupervisedRoute({ + method: Method.POST, + subscription: RouteStore.inspectImage, + onValidation: async ({ req, res }) => { + const { source } = req.body; + if (typeof source === "string") { + const uploadInformation = await DashUploadUtils.UploadImage(source); + return res.send(await DashUploadUtils.InspectImage(uploadInformation.mediaPaths[0])); } + res.send({}); + } + }); - router.addSupervisedRoute({ - method: Method.POST, - subscription: RouteStore.upload, - onValidation: ({ req, res }) => { - let form = new formidable.IncomingForm(); - form.uploadDir = uploadDirectory; - form.keepExtensions = true; - form.parse(req, async (_err, _fields, files) => { - let results: ImageFileResponse[] = []; - for (const key in files) { - const { type, path: location, name } = files[key]; - const filename = path.basename(location); - let uploadInformation: Opt; - if (filename.endsWith(".pdf")) { - let dataBuffer = fs.readFileSync(uploadDirectory + filename); - const result: ParsedPDF = await pdf(dataBuffer); - await new Promise(resolve => { - const path = pdfDirectory + "/" + filename.substring(0, filename.length - ".pdf".length) + ".txt"; - fs.createWriteStream(path).write(result.text, error => { - if (!error) { - resolve(); - } else { - reject(error); - } - }); - }); - } else { - uploadInformation = await DashUploadUtils.UploadImage(uploadDirectory + filename, filename); - } - const exif = uploadInformation ? uploadInformation.exifData : undefined; - results.push({ name, type, path: `/files/${filename}`, exif }); - - } - _success(res, results); + router.addSupervisedRoute({ + method: Method.POST, + subscription: RouteStore.dataUriToImage, + onValidation: ({ req, res }) => { + const uri = req.body.uri; + const filename = req.body.name; + if (!uri || !filename) { + res.status(401).send("incorrect parameters specified"); + return; + } + imageDataUri.outputFile(uri, uploadDirectory + filename).then((savedName: string) => { + const ext = path.extname(savedName); + let resizers = [ + { resizer: sharp().resize(100, undefined, { withoutEnlargement: true }), suffix: "_s" }, + { resizer: sharp().resize(400, undefined, { withoutEnlargement: true }), suffix: "_m" }, + { resizer: sharp().resize(900, undefined, { withoutEnlargement: true }), suffix: "_l" }, + ]; + let isImage = false; + if (pngTypes.includes(ext)) { + resizers.forEach(element => { + element.resizer = element.resizer.png(); }); + isImage = true; + } else if (jpgTypes.includes(ext)) { + resizers.forEach(element => { + element.resizer = element.resizer.jpeg(); + }); + isImage = true; } - }); - - router.addSupervisedRoute({ - method: Method.POST, - subscription: RouteStore.inspectImage, - onValidation: async ({ req, res }) => { - const { source } = req.body; - if (typeof source === "string") { - const uploadInformation = await DashUploadUtils.UploadImage(source); - return res.send(await DashUploadUtils.InspectImage(uploadInformation.mediaPaths[0])); - } - res.send({}); - } - }); - - router.addSupervisedRoute({ - method: Method.POST, - subscription: RouteStore.dataUriToImage, - onValidation: ({ req, res }) => { - const uri = req.body.uri; - const filename = req.body.name; - if (!uri || !filename) { - res.status(401).send("incorrect parameters specified"); - return; - } - imageDataUri.outputFile(uri, uploadDirectory + filename).then((savedName: string) => { - const ext = path.extname(savedName); - let resizers = [ - { resizer: sharp().resize(100, undefined, { withoutEnlargement: true }), suffix: "_s" }, - { resizer: sharp().resize(400, undefined, { withoutEnlargement: true }), suffix: "_m" }, - { resizer: sharp().resize(900, undefined, { withoutEnlargement: true }), suffix: "_l" }, - ]; - let isImage = false; - if (pngTypes.includes(ext)) { - resizers.forEach(element => { - element.resizer = element.resizer.png(); - }); - isImage = true; - } else if (jpgTypes.includes(ext)) { - resizers.forEach(element => { - element.resizer = element.resizer.jpeg(); - }); - isImage = true; - } - if (isImage) { - resizers.forEach(resizer => { - fs.createReadStream(savedName).pipe(resizer.resizer).pipe(fs.createWriteStream(uploadDirectory + filename + resizer.suffix + ext)); - }); - } - res.send("/files/" + filename + ext); + if (isImage) { + resizers.forEach(resizer => { + fs.createReadStream(savedName).pipe(resizer.resizer).pipe(fs.createWriteStream(uploadDirectory + filename + resizer.suffix + ext)); }); } + res.send("/files/" + filename + ext); }); + } + }); - router.addSupervisedRoute({ - method: Method.GET, - subscription: RouteStore.delete, - onValidation: ({ res, isRelease }) => { - if (isRelease) { - return _permission_denied(res, deletionPermissionError); - } - WebSocket.deleteFields().then(() => res.redirect(RouteStore.home)); - } - }); + router.addSupervisedRoute({ + method: Method.GET, + subscription: RouteStore.delete, + onValidation: ({ res, isRelease }) => { + if (isRelease) { + return _permission_denied(res, deletionPermissionError); + } + WebSocket.deleteFields().then(() => res.redirect(RouteStore.home)); + } + }); - router.addSupervisedRoute({ - method: Method.GET, - subscription: RouteStore.deleteAll, - onValidation: ({ res, isRelease }) => { - if (isRelease) { - return _permission_denied(res, deletionPermissionError); - } - WebSocket.deleteAll().then(() => res.redirect(RouteStore.home)); - } - }); + router.addSupervisedRoute({ + method: Method.GET, + subscription: RouteStore.deleteAll, + onValidation: ({ res, isRelease }) => { + if (isRelease) { + return _permission_denied(res, deletionPermissionError); + } + WebSocket.deleteAll().then(() => res.redirect(RouteStore.home)); + } + }); - const credentialsPath = path.join(__dirname, "./credentials/google_docs_credentials.json"); - - const EndpointHandlerMap = new Map([ - ["create", (api, params) => api.create(params)], - ["retrieve", (api, params) => api.get(params)], - ["update", (api, params) => api.batchUpdate(params)], - ]); - - router.addSupervisedRoute({ - method: Method.POST, - subscription: new RouteSubscriber(RouteStore.googleDocs).add("sector", "action"), - onValidation: ({ req, res }) => { - let sector: GoogleApiServerUtils.Service = req.params.sector as GoogleApiServerUtils.Service; - let action: GoogleApiServerUtils.Action = req.params.action as GoogleApiServerUtils.Action; - GoogleApiServerUtils.GetEndpoint(GoogleApiServerUtils.Service[sector], { credentialsPath, userId: req.headers.userId as string }).then(endpoint => { - let handler = EndpointHandlerMap.get(action); - if (endpoint && handler) { - let execute = handler(endpoint, req.body).then( - response => res.send(response.data), - rejection => res.send(rejection) - ); - execute.catch(exception => res.send(exception)); - return; - } - res.send(undefined); - }); + const credentialsPath = path.join(__dirname, "./credentials/google_docs_credentials.json"); + + const EndpointHandlerMap = new Map([ + ["create", (api, params) => api.create(params)], + ["retrieve", (api, params) => api.get(params)], + ["update", (api, params) => api.batchUpdate(params)], + ]); + + router.addSupervisedRoute({ + method: Method.POST, + subscription: new RouteSubscriber(RouteStore.googleDocs).add("sector", "action"), + onValidation: ({ req, res }) => { + let sector: GoogleApiServerUtils.Service = req.params.sector as GoogleApiServerUtils.Service; + let action: GoogleApiServerUtils.Action = req.params.action as GoogleApiServerUtils.Action; + GoogleApiServerUtils.GetEndpoint(GoogleApiServerUtils.Service[sector], { credentialsPath, userId: req.headers.userId as string }).then(endpoint => { + let handler = EndpointHandlerMap.get(action); + if (endpoint && handler) { + let execute = handler(endpoint, req.body).then( + response => res.send(response.data), + rejection => res.send(rejection) + ); + execute.catch(exception => res.send(exception)); + return; } + res.send(undefined); }); + } + }); - router.addSupervisedRoute({ - method: Method.GET, - subscription: RouteStore.readGoogleAccessToken, - onValidation: async ({ user, res }) => { - const userId = user.id; - const token = await Database.Auxiliary.GoogleAuthenticationToken.Fetch(userId); - const information = { credentialsPath, userId }; - if (!token) { - return res.send(await GoogleApiServerUtils.GenerateAuthenticationUrl(information)); - } - GoogleApiServerUtils.RetrieveAccessToken(information).then(token => res.send(token)); - } - }); + router.addSupervisedRoute({ + method: Method.GET, + subscription: RouteStore.readGoogleAccessToken, + onValidation: async ({ user, res }) => { + const userId = user.id; + const token = await Database.Auxiliary.GoogleAuthenticationToken.Fetch(userId); + const information = { credentialsPath, userId }; + if (!token) { + return res.send(await GoogleApiServerUtils.GenerateAuthenticationUrl(information)); + } + GoogleApiServerUtils.RetrieveAccessToken(information).then(token => res.send(token)); + } + }); - router.addSupervisedRoute({ - method: Method.POST, - subscription: RouteStore.writeGoogleAccessToken, - onValidation: async ({ user, req, res }) => { - const userId = user.id; - const information = { credentialsPath, userId }; - res.send(await GoogleApiServerUtils.ProcessClientSideCode(information, req.body.authenticationCode)); - } - }); + router.addSupervisedRoute({ + method: Method.POST, + subscription: RouteStore.writeGoogleAccessToken, + onValidation: async ({ user, req, res }) => { + const userId = user.id; + const information = { credentialsPath, userId }; + res.send(await GoogleApiServerUtils.ProcessClientSideCode(information, req.body.authenticationCode)); + } + }); - const tokenError = "Unable to successfully upload bytes for all images!"; - const mediaError = "Unable to convert all uploaded bytes to media items!"; - const userIdError = "Unable to parse the identification of the user!"; - - router.addSupervisedRoute({ - method: Method.POST, - subscription: RouteStore.googlePhotosMediaUpload, - onValidation: async ({ user, req, res }) => { - const { media } = req.body; - const userId = user.id; - if (!userId) { - return _error(res, userIdError); - } + const tokenError = "Unable to successfully upload bytes for all images!"; + const mediaError = "Unable to convert all uploaded bytes to media items!"; + const userIdError = "Unable to parse the identification of the user!"; + + router.addSupervisedRoute({ + method: Method.POST, + subscription: RouteStore.googlePhotosMediaUpload, + onValidation: async ({ user, req, res }) => { + const { media } = req.body; + const userId = user.id; + if (!userId) { + return _error(res, userIdError); + } - await GooglePhotosUploadUtils.initialize({ credentialsPath, userId }); + await GooglePhotosUploadUtils.initialize({ credentialsPath, userId }); - let failed: number[] = []; + let failed: number[] = []; - const newMediaItems = await BatchedArray.from(media, { batchSize: 25 }).batchedMapPatientInterval( - { magnitude: 100, unit: TimeUnit.Milliseconds }, - async (batch: GooglePhotosUploadUtils.MediaInput[]) => { - const newMediaItems: NewMediaItem[] = []; - for (let index = 0; index < batch.length; index++) { - const element = batch[index]; - const uploadToken = await GooglePhotosUploadUtils.DispatchGooglePhotosUpload(element.url); - if (!uploadToken) { - failed.push(index); - } else { - newMediaItems.push({ - description: element.description, - simpleMediaItem: { uploadToken } - }); - } - } - return newMediaItems; + const newMediaItems = await BatchedArray.from(media, { batchSize: 25 }).batchedMapPatientInterval( + { magnitude: 100, unit: TimeUnit.Milliseconds }, + async (batch: GooglePhotosUploadUtils.MediaInput[]) => { + const newMediaItems: NewMediaItem[] = []; + for (let index = 0; index < batch.length; index++) { + const element = batch[index]; + const uploadToken = await GooglePhotosUploadUtils.DispatchGooglePhotosUpload(element.url); + if (!uploadToken) { + failed.push(index); + } else { + newMediaItems.push({ + description: element.description, + simpleMediaItem: { uploadToken } + }); } - ); - - const failedCount = failed.length; - if (failedCount) { - console.error(`Unable to upload ${failedCount} image${failedCount === 1 ? "" : "s"} to Google's servers`); } - - GooglePhotosUploadUtils.CreateMediaItems(newMediaItems, req.body.album).then( - result => _success(res, { results: result.newMediaItemResults, failed }), - error => _error(res, mediaError, error) - ); + return newMediaItems; } - }); + ); - interface MediaItem { - baseUrl: string; - filename: string; + const failedCount = failed.length; + if (failedCount) { + console.error(`Unable to upload ${failedCount} image${failedCount === 1 ? "" : "s"} to Google's servers`); } - const prefix = "google_photos_"; - - const downloadError = "Encountered an error while executing downloads."; - const requestError = "Unable to execute download: the body's media items were malformed."; - const deletionPermissionError = "Cannot perform specialized delete outside of the development environment!"; - - router.addSupervisedRoute({ - method: Method.GET, - subscription: "/deleteWithAux", - onValidation: async ({ res, isRelease }) => { - if (isRelease) { - return _permission_denied(res, deletionPermissionError); - } - await Database.Auxiliary.DeleteAll(); - res.redirect(RouteStore.delete); - } - }); - router.addSupervisedRoute({ - method: Method.GET, - subscription: "/deleteWithGoogleCredentials", - onValidation: async ({ res, isRelease }) => { - if (isRelease) { - return _permission_denied(res, deletionPermissionError); - } - await Database.Auxiliary.GoogleAuthenticationToken.DeleteAll(); - res.redirect(RouteStore.delete); - } - }); + GooglePhotosUploadUtils.CreateMediaItems(newMediaItems, req.body.album).then( + result => _success(res, { results: result.newMediaItemResults, failed }), + error => _error(res, mediaError, error) + ); + } + }); - const UploadError = (count: number) => `Unable to upload ${count} images to Dash's server`; - router.addSupervisedRoute({ - method: Method.POST, - subscription: RouteStore.googlePhotosMediaDownload, - onValidation: async ({ req, res }) => { - const contents: { mediaItems: MediaItem[] } = req.body; - let failed = 0; - if (contents) { - const completed: Opt[] = []; - for (let item of contents.mediaItems) { - const { contentSize, ...attributes } = await DashUploadUtils.InspectImage(item.baseUrl); - const found: Opt = await Database.Auxiliary.QueryUploadHistory(contentSize!); - if (!found) { - const upload = await DashUploadUtils.UploadInspectedImage({ contentSize, ...attributes }, item.filename, prefix).catch(error => _error(res, downloadError, error)); - if (upload) { - completed.push(upload); - await Database.Auxiliary.LogUpload(upload); - } else { - failed++; - } - } else { - completed.push(found); - } - } - if (failed) { - return _error(res, UploadError(failed)); + interface MediaItem { + baseUrl: string; + filename: string; + } + const prefix = "google_photos_"; + + const downloadError = "Encountered an error while executing downloads."; + const requestError = "Unable to execute download: the body's media items were malformed."; + const deletionPermissionError = "Cannot perform specialized delete outside of the development environment!"; + + router.addSupervisedRoute({ + method: Method.GET, + subscription: "/deleteWithAux", + onValidation: async ({ res, isRelease }) => { + if (isRelease) { + return _permission_denied(res, deletionPermissionError); + } + await Database.Auxiliary.DeleteAll(); + res.redirect(RouteStore.delete); + } + }); + + router.addSupervisedRoute({ + method: Method.GET, + subscription: "/deleteWithGoogleCredentials", + onValidation: async ({ res, isRelease }) => { + if (isRelease) { + return _permission_denied(res, deletionPermissionError); + } + await Database.Auxiliary.GoogleAuthenticationToken.DeleteAll(); + res.redirect(RouteStore.delete); + } + }); + + const UploadError = (count: number) => `Unable to upload ${count} images to Dash's server`; + router.addSupervisedRoute({ + method: Method.POST, + subscription: RouteStore.googlePhotosMediaDownload, + onValidation: async ({ req, res }) => { + const contents: { mediaItems: MediaItem[] } = req.body; + let failed = 0; + if (contents) { + const completed: Opt[] = []; + for (let item of contents.mediaItems) { + const { contentSize, ...attributes } = await DashUploadUtils.InspectImage(item.baseUrl); + const found: Opt = await Database.Auxiliary.QueryUploadHistory(contentSize!); + if (!found) { + const upload = await DashUploadUtils.UploadInspectedImage({ contentSize, ...attributes }, item.filename, prefix).catch(error => _error(res, downloadError, error)); + if (upload) { + completed.push(upload); + await Database.Auxiliary.LogUpload(upload); + } else { + failed++; } - return _success(res, completed); + } else { + completed.push(found); } - _invalid(res, requestError); } - }); + if (failed) { + return _error(res, UploadError(failed)); + } + return _success(res, completed); + } + _invalid(res, requestError); } }); -})(); \ No newline at end of file +} \ No newline at end of file -- cgit v1.2.3-70-g09d2 From e6bd33867cc7f7185575666255369f55cacb9856 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Sat, 26 Oct 2019 18:28:38 -0400 Subject: restructured route registration and added preliminary comments for exporter --- src/server/ApiManagers/ApiManager.ts | 10 ++- src/server/ApiManagers/ExportManager.ts | 133 ++++++++++++++++++++++++++++++++ src/server/ApiManagers/SearchManager.ts | 10 +-- src/server/ApiManagers/UserManager.ts | 10 ++- src/server/ApiManagers/UtilManager.ts | 12 +-- src/server/RouteManager.ts | 15 ++-- src/server/Websocket/Websocket.ts | 2 +- src/server/index.ts | 96 ++++------------------- 8 files changed, 179 insertions(+), 109 deletions(-) create mode 100644 src/server/ApiManagers/ExportManager.ts (limited to 'src') diff --git a/src/server/ApiManagers/ApiManager.ts b/src/server/ApiManagers/ApiManager.ts index 264c78a17..9fd726060 100644 --- a/src/server/ApiManagers/ApiManager.ts +++ b/src/server/ApiManagers/ApiManager.ts @@ -1,7 +1,11 @@ -import RouteManager from "../RouteManager"; +import RouteManager, { RouteInitializer } from "../RouteManager"; -export default abstract class ApiManager { +export type Registration = (initializer: RouteInitializer) => void; - public abstract register(router: RouteManager): void; +export default abstract class ApiManager { + protected abstract initialize(register: Registration): void; + public register(router: RouteManager) { + this.initialize(router.addSupervisedRoute); + } } \ No newline at end of file diff --git a/src/server/ApiManagers/ExportManager.ts b/src/server/ApiManagers/ExportManager.ts new file mode 100644 index 000000000..261acbbe0 --- /dev/null +++ b/src/server/ApiManagers/ExportManager.ts @@ -0,0 +1,133 @@ +import ApiManager, { Registration } from "./ApiManager"; +import RouteManager, { Method } from "../RouteManager"; +import RouteSubscriber from "../RouteSubscriber"; +import { RouteStore } from "../RouteStore"; +import * as Archiver from 'archiver'; +import * as express from 'express'; +import { Database } from "../database"; +import * as path from "path"; +import { DashUploadUtils } from "../DashUploadUtils"; + +export type Hierarchy = { [id: string]: string | Hierarchy }; +export type ZipMutator = (file: Archiver.Archiver) => void | Promise; +export interface DocumentElements { + data: string | any[]; + title: string; +} + +export default class ExportManager extends ApiManager { + + protected initialize(register: Registration): void { + + register({ + method: Method.GET, + subscription: new RouteSubscriber(RouteStore.imageHierarchyExport).add('docId'), + onValidation: async ({ req, res }) => { + const id = req.params.docId; + const hierarchy: Hierarchy = {}; + await buildHierarchyRecursive(id, hierarchy); + BuildAndDispatchZip(res, zip => writeHierarchyRecursive(zip, hierarchy)); + } + }); + } + +} + +/** + * This utility function factors out the process + * of creating a zip file and sending it back to the client + * by piping it into a response. + * + * Learn more about piping and readable / writable streams here! + * https://www.freecodecamp.org/news/node-js-streams-everything-you-need-to-know-c9141306be93/ + * + * @param res the writable stream response object that will transfer the generated zip file + * @param mutator the callback function used to actually modify and insert information into the zip instance + */ +export async function BuildAndDispatchZip(res: express.Response, mutator: ZipMutator): Promise { + const zip = Archiver('zip'); + zip.pipe(res); + await mutator(zip); + zip.finalize(); +} + +/** + * This function starts with a single document id as a seed, + * typically that of a collection, and then descends the entire tree + * of image or collection documents that are reachable from that seed. + * @param seedId the id of the root of the subtree we're trying to capture, interesting only if it's a collection + * @param hierarchy the data structure we're going to use to record the nesting of the collections and images as we descend + */ + +/* +Below is an example of the JSON hierarchy built from two images contained inside a collection titled 'a nested collection', +following the general recursive structure shown immediately below +{ + "parent folder name":{ + "first child's fild name":"first child's url" + ... + "nth child's fild name":"nth child's url" + } +} +{ + "a nested collection (865c4734-c036-4d67-a588-c71bb43d1440)":{ + "an image of a cat (ace99ffd-8ed8-4026-a5d5-a353fff57bdd).jpg":"https://upload.wikimedia.org/wikipedia/commons/thumb/3/3a/Cat03.jpg/1200px-Cat03.jpg", + "1*SGJw31T5Q9Zfsk24l2yirg.gif (9321cc9b-9b3e-4cb6-b99c-b7e667340f05).gif":"https://cdn-media-1.freecodecamp.org/images/1*SGJw31T5Q9Zfsk24l2yirg.gif" + } +} +*/ + +async function buildHierarchyRecursive(seedId: string, hierarchy: Hierarchy): Promise { + const { title, data } = await getData(seedId); + const label = `${title} (${seedId})`; + // is the document a collection? + if (Array.isArray(data)) { + // recurse over all documents in the collection. + const local: Hierarchy = {}; // create a child hierarchy for this level, which will get passed in as the parent of the recursive call + hierarchy[label] = local; // store it at the index in the parent, so we'll end up with a map of maps of maps + await Promise.all(data.map(proxy => buildHierarchyRecursive(proxy.fieldId, local))); + } else { + // now, data can only be a string, namely the url of the image + const filename = label + path.extname(data); // this is the file name under which the output image will be stored + hierarchy[filename] = data; + } +} + +async function getData(seedId: string): Promise { + return new Promise((resolve, reject) => { + Database.Instance.getDocument(seedId, async (result: any) => { + const { data, proto, title } = result.fields; + if (data) { + if (data.url) { + resolve({ data: data.url, title }); + } else if (data.fields) { + resolve({ data: data.fields, title }); + } else { + reject(); + } + } + if (proto) { + getData(proto.fieldId).then(resolve, reject); + } + }); + }); +} + +async function writeHierarchyRecursive(file: Archiver.Archiver, hierarchy: Hierarchy, prefix = "Dash Export"): Promise { + for (const key of Object.keys(hierarchy)) { + const result = hierarchy[key]; + if (typeof result === "string") { + let path: string; + let matches: RegExpExecArray | null; + if ((matches = /\:1050\/files\/(upload\_[\da-z]{32}.*)/g.exec(result)) !== null) { + path = `${__dirname}/public/files/${matches[1]}`; + } else { + const information = await DashUploadUtils.UploadImage(result); + path = information.mediaPaths[0]; + } + file.file(path, { name: key, prefix }); + } else { + await writeHierarchyRecursive(file, result, `${prefix}/${key}`); + } + } +} \ No newline at end of file diff --git a/src/server/ApiManagers/SearchManager.ts b/src/server/ApiManagers/SearchManager.ts index 15b87204c..1c4b805e5 100644 --- a/src/server/ApiManagers/SearchManager.ts +++ b/src/server/ApiManagers/SearchManager.ts @@ -1,5 +1,5 @@ -import ApiManager from "./ApiManager"; -import RouteManager, { Method } from "../RouteManager"; +import ApiManager, { Registration } from "./ApiManager"; +import { Method } from "../RouteManager"; import { Search } from "../Search"; var findInFiles = require('find-in-files'); import * as path from 'path'; @@ -7,9 +7,9 @@ import { uploadDirectory } from ".."; export default class SearchManager extends ApiManager { - public register(router: RouteManager): void { + protected initialize(register: Registration): void { - router.addSupervisedRoute({ + register({ method: Method.GET, subscription: "/textsearch", onValidation: async ({ req, res }) => { @@ -29,7 +29,7 @@ export default class SearchManager extends ApiManager { } }); - router.addSupervisedRoute({ + register({ method: Method.GET, subscription: "/search", onValidation: async ({ req, res }) => { diff --git a/src/server/ApiManagers/UserManager.ts b/src/server/ApiManagers/UserManager.ts index bb8837dc6..dd1e50133 100644 --- a/src/server/ApiManagers/UserManager.ts +++ b/src/server/ApiManagers/UserManager.ts @@ -1,11 +1,12 @@ -import ApiManager from "./ApiManager"; -import RouteManager, { Method } from "../RouteManager"; +import ApiManager, { Registration } from "./ApiManager"; +import { Method } from "../RouteManager"; import { WebSocket } from "../Websocket/Websocket"; export default class UserManager extends ApiManager { - public register(router: RouteManager): void { - router.addSupervisedRoute({ + protected initialize(register: Registration): void { + + register({ method: Method.GET, subscription: "/whosOnline", onValidation: ({ res }) => { @@ -22,6 +23,7 @@ export default class UserManager extends ApiManager { res.send(users); } }); + } private msToTime(duration: number) { diff --git a/src/server/ApiManagers/UtilManager.ts b/src/server/ApiManagers/UtilManager.ts index 79b904e8a..a3f802b20 100644 --- a/src/server/ApiManagers/UtilManager.ts +++ b/src/server/ApiManagers/UtilManager.ts @@ -1,13 +1,13 @@ -import ApiManager from "./ApiManager"; -import RouteManager, { Method } from "../RouteManager"; +import ApiManager, { Registration } from "./ApiManager"; +import { Method } from "../RouteManager"; import { exec } from 'child_process'; import { command_line } from "../ActionUtilities"; export default class UtilManager extends ApiManager { - public register(router: RouteManager): void { + protected initialize(register: Registration): void { - router.addSupervisedRoute({ + register({ method: Method.GET, subscription: "/pull", onValidation: ({ res }) => { @@ -21,7 +21,7 @@ export default class UtilManager extends ApiManager { } }); - router.addSupervisedRoute({ + register({ method: Method.GET, subscription: "/buxton", onValidation: ({ res }) => { @@ -35,7 +35,7 @@ export default class UtilManager extends ApiManager { }, }); - router.addSupervisedRoute({ + register({ method: Method.GET, subscription: "/version", onValidation: ({ res }) => { diff --git a/src/server/RouteManager.ts b/src/server/RouteManager.ts index b3864e89c..ef083a88a 100644 --- a/src/server/RouteManager.ts +++ b/src/server/RouteManager.ts @@ -2,7 +2,6 @@ import RouteSubscriber from "./RouteSubscriber"; import { RouteStore } from "./RouteStore"; import { DashUserModel } from "./authentication/models/user_model"; import * as express from 'express'; -import { Opt } from "../new_fields/Doc"; export enum Method { GET, @@ -41,15 +40,10 @@ export default class RouteManager { } /** - * Please invoke this function when adding a new route to Dash's server. - * It ensures that any requests leading to or containing user-sensitive information - * does not execute unless Passport authentication detects a user logged in. - * @param method whether or not the request is a GET or a POST - * @param handler the action to invoke, recieving a DashUserModel and, as expected, the Express.Request and Express.Response - * @param onRejection an optional callback invoked on return if no user is found to be logged in - * @param subscribers the forward slash prepended path names (reference and add to RouteStore.ts) that will all invoke the given @param handler + * + * @param initializer */ - addSupervisedRoute(initializer: RouteInitializer) { + addSupervisedRoute = (initializer: RouteInitializer): void => { const { method, subscription, onValidation, onUnauthenticated, onError } = initializer; const isRelease = this._isRelease; let supervised = async (req: express.Request, res: express.Response) => { @@ -72,6 +66,9 @@ export default class RouteManager { req.session!.target = target; if (onUnauthenticated) { await tryExecute(onUnauthenticated, core); + if (!res.headersSent) { + res.redirect(RouteStore.login); + } } else { res.redirect(RouteStore.login); } diff --git a/src/server/Websocket/Websocket.ts b/src/server/Websocket/Websocket.ts index 2461dd8d5..cd2813d99 100644 --- a/src/server/Websocket/Websocket.ts +++ b/src/server/Websocket/Websocket.ts @@ -4,7 +4,7 @@ import { Client } from "../Client"; import { Socket } from "socket.io"; import { Database } from "../database"; import { Search } from "../Search"; -import io from 'socket.io'; +import * as io from 'socket.io'; import YoutubeApi from "../apis/youtube/youtubeApiSample"; import { youtubeApiKey } from ".."; diff --git a/src/server/index.ts b/src/server/index.ts index 93f4238bc..384800f23 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -33,12 +33,11 @@ import UtilManager from './ApiManagers/UtilManager'; import SearchManager from './ApiManagers/SearchManager'; import UserManager from './ApiManagers/UserManager'; import { WebSocket } from './Websocket/Websocket'; +import ExportManager from './ApiManagers/ExportManager'; +import ApiManager from './ApiManagers/ApiManager'; export let youtubeApiKey: string; -export type Hierarchy = { [id: string]: string | Hierarchy }; -export type ZipMutator = (file: Archiver.Archiver) => void | Promise; - export interface NewMediaItem { description: string; simpleMediaItem: { @@ -72,9 +71,13 @@ async function PreliminaryFunctions() { } function routeSetter(router: RouteManager) { - new UtilManager().register(router); - new SearchManager().register(router); - new UserManager().register(router); + const managers: ApiManager[] = [ + new UtilManager(), + new SearchManager(), + new UserManager(), + new ExportManager() + ]; + managers.forEach(manager => manager.register(router)); WebSocket.initialize(serverPort, router.isRelease); @@ -152,77 +155,6 @@ function routeSetter(router: RouteManager) { } }); - router.addSupervisedRoute({ - method: Method.GET, - subscription: new RouteSubscriber(RouteStore.imageHierarchyExport).add('docId'), - onValidation: async ({ req, res }) => { - const id = req.params.docId; - const hierarchy: Hierarchy = {}; - await targetedVisitorRecursive(id, hierarchy); - BuildAndDispatchZip(res, async zip => { - await hierarchyTraverserRecursive(zip, hierarchy); - }); - } - }); - - const BuildAndDispatchZip = async (res: Response, mutator: ZipMutator): Promise => { - const zip = Archiver('zip'); - zip.pipe(res); - await mutator(zip); - return zip.finalize(); - }; - - const targetedVisitorRecursive = async (seedId: string, hierarchy: Hierarchy): Promise => { - const local: Hierarchy = {}; - const { title, data } = await getData(seedId); - const label = `${title} (${seedId})`; - if (Array.isArray(data)) { - hierarchy[label] = local; - await Promise.all(data.map(proxy => targetedVisitorRecursive(proxy.fieldId, local))); - } else { - hierarchy[label + path.extname(data)] = data; - } - }; - - const getData = async (seedId: string): Promise<{ data: string | any[], title: string }> => { - return new Promise<{ data: string | any[], title: string }>((resolve, reject) => { - Database.Instance.getDocument(seedId, async (result: any) => { - const { data, proto, title } = result.fields; - if (data) { - if (data.url) { - resolve({ data: data.url, title }); - } else if (data.fields) { - resolve({ data: data.fields, title }); - } else { - reject(); - } - } - if (proto) { - getData(proto.fieldId).then(resolve, reject); - } - }); - }); - }; - - const hierarchyTraverserRecursive = async (file: Archiver.Archiver, hierarchy: Hierarchy, prefix = "Dash Export"): Promise => { - for (const key of Object.keys(hierarchy)) { - const result = hierarchy[key]; - if (typeof result === "string") { - let path: string; - let matches: RegExpExecArray | null; - if ((matches = /\:1050\/files\/(upload\_[\da-z]{32}.*)/g.exec(result)) !== null) { - path = `${__dirname}/public/files/${matches[1]}`; - } else { - const information = await DashUploadUtils.UploadImage(result); - path = information.mediaPaths[0]; - } - file.file(path, { name: key, prefix }); - } else { - await hierarchyTraverserRecursive(file, result, `${prefix}/${key}`); - } - } - }; - router.addSupervisedRoute({ method: Method.GET, subscription: new RouteSubscriber("/downloadId").add("docId"), @@ -600,22 +532,24 @@ function routeSetter(router: RouteManager) { router.addSupervisedRoute({ method: Method.GET, subscription: RouteStore.delete, - onValidation: ({ res, isRelease }) => { + onValidation: async ({ res, isRelease }) => { if (isRelease) { return _permission_denied(res, deletionPermissionError); } - WebSocket.deleteFields().then(() => res.redirect(RouteStore.home)); + await WebSocket.deleteFields(); + res.redirect(RouteStore.home); } }); router.addSupervisedRoute({ method: Method.GET, subscription: RouteStore.deleteAll, - onValidation: ({ res, isRelease }) => { + onValidation: async ({ res, isRelease }) => { if (isRelease) { return _permission_denied(res, deletionPermissionError); } - WebSocket.deleteAll().then(() => res.redirect(RouteStore.home)); + await WebSocket.deleteAll(); + res.redirect(RouteStore.home); } }); -- cgit v1.2.3-70-g09d2 From feec691275ec83e4ddd8fd8ea803f004a371cf11 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Sun, 27 Oct 2019 18:46:57 -0400 Subject: refactoring oauth2 client use in google api serverside --- src/server/apis/google/GoogleApiServerUtils.ts | 87 +++++++++++--------------- src/server/index.ts | 10 ++- 2 files changed, 40 insertions(+), 57 deletions(-) (limited to 'src') diff --git a/src/server/apis/google/GoogleApiServerUtils.ts b/src/server/apis/google/GoogleApiServerUtils.ts index 6093197f1..c0824cfb7 100644 --- a/src/server/apis/google/GoogleApiServerUtils.ts +++ b/src/server/apis/google/GoogleApiServerUtils.ts @@ -1,7 +1,6 @@ import { google } from "googleapis"; -import { createInterface } from "readline"; -import { readFile, writeFile } from "fs"; -import { OAuth2Client, Credentials } from "google-auth-library"; +import { readFile } from "fs"; +import { OAuth2Client, Credentials, OAuth2ClientOptions } from "google-auth-library"; import { Opt } from "../../../new_fields/Doc"; import { GlobalOptions } from "googleapis-common"; import { GaxiosResponse } from "gaxios"; @@ -31,6 +30,8 @@ export namespace GoogleApiServerUtils { 'userinfo.profile' ]; + const ClientMapping = new Map(); + export const parseBuffer = (data: Buffer) => JSON.parse(data.toString()); export enum Service { @@ -51,11 +52,11 @@ export namespace GoogleApiServerUtils { export type Endpoint = { get: ApiHandler, create: ApiHandler, batchUpdate: ApiHandler }; export type EndpointParameters = GlobalOptions & { version: "v1" }; - export const GetEndpoint = (sector: string, paths: CredentialInformation) => { + export const GetEndpoint = (sector: string, userId: string) => { return new Promise>(resolve => { - RetrieveCredentials(paths).then(authentication => { + authorize(userId).then(({ client: auth }) => { let routed: Opt; - let parameters: EndpointParameters = { auth: authentication.client, version: "v1" }; + let parameters: EndpointParameters = { auth, version: "v1" }; switch (sector) { case Service.Documents: routed = google.docs(parameters).documents; @@ -69,16 +70,17 @@ export namespace GoogleApiServerUtils { }); }; - export const RetrieveAccessToken = (information: CredentialInformation) => { + export const RetrieveAccessToken = (userId: string): Promise => { return new Promise((resolve, reject) => { - RetrieveCredentials(information).then( - credentials => resolve(credentials.token.access_token!), + authorize(userId).then( + ({ token: { access_token } }) => resolve(access_token!), error => reject(`Error: unable to authenticate Google Photos API request.\n${error}`) ); }); }; - let AuthorizationManager: OAuth2Client; + let installed: OAuth2ClientOptions; + let worker: OAuth2Client; export const LoadOAuthClient = async () => { return new Promise((resolve, reject) => { @@ -87,15 +89,17 @@ export namespace GoogleApiServerUtils { reject(err); return console.log('Error loading client secret file:', err); } - const { client_secret, client_id, redirect_uris } = parseBuffer(credentials).installed; - AuthorizationManager = new google.auth.OAuth2(client_id, client_secret, redirect_uris[0]); + installed = parseBuffer(credentials).installed; + worker = new google.auth.OAuth2(installed); resolve(); }); }); }; + const generateClient = () => new google.auth.OAuth2(installed); + export const GenerateAuthenticationUrl = async (information: CredentialInformation) => { - return AuthorizationManager.generateAuthUrl({ + return worker.generateAuthUrl({ access_type: 'offline', scope: SCOPES.map(relative => prefix + relative), }); @@ -106,16 +110,15 @@ export namespace GoogleApiServerUtils { avatar: string; name: string; } - export const ProcessClientSideCode = async (information: CredentialInformation, authenticationCode: string): Promise => { + export const ProcessClientSideCode = async (userId: string, authenticationCode: string): Promise => { return new Promise((resolve, reject) => { - AuthorizationManager.getToken(authenticationCode, async (err, token) => { + worker.getToken(authenticationCode, async (err, token) => { if (err || !token) { reject(err); return console.error('Error retrieving access token', err); } - AuthorizationManager.setCredentials(token); const enriched = injectUserInfo(token); - await Database.Auxiliary.GoogleAuthenticationToken.Write(information.userId, enriched); + await Database.Auxiliary.GoogleAuthenticationToken.Write(userId, enriched); const { given_name, picture } = enriched.userInfo; resolve({ access_token: enriched.access_token!, @@ -155,57 +158,39 @@ export namespace GoogleApiServerUtils { sub: string; } - export const RetrieveCredentials = (information: CredentialInformation) => { - return new Promise((resolve, reject) => { - readFile(information.credentialsPath, async (err, credentials) => { - if (err) { - reject(err); - return console.log('Error loading client secret file:', err); + export const authorize = async (userId: string): Promise => { + return Database.Auxiliary.GoogleAuthenticationToken.Fetch(userId).then(token => { + return new Promise((resolve, reject) => { + const client = generateClient(); + if (token!.expiry_date! < new Date().getTime()) { + // Token has expired, so submitting a request for a refreshed access token + return refreshToken(token!, client, userId).then(resolve, reject); } - authorize(parseBuffer(credentials), information.userId).then(resolve, reject); + // Authentication successful! + client.setCredentials(token!); + resolve({ token: token!, client }); }); }); }; - export const RetrievePhotosEndpoint = (paths: CredentialInformation) => { + export const RetrievePhotosEndpoint = (userId: string) => { return new Promise((resolve, reject) => { - RetrieveAccessToken(paths).then( + RetrieveAccessToken(userId).then( token => resolve(new Photos(token)), reject ); }); }; - type TokenResult = { token: Credentials, client: OAuth2Client }; - /** - * Create an OAuth2 client with the given credentials, and returns the promise resolving to the authenticated client - * @param {Object} credentials The authorization client credentials. - */ - export function authorize(credentials: any, userId: string): Promise { - const { client_secret, client_id, redirect_uris } = credentials.installed; - const oAuth2Client = new google.auth.OAuth2(client_id, client_secret, redirect_uris[0]); - return new Promise((resolve, reject) => { - // Attempting to authorize user (${userId}) - Database.Auxiliary.GoogleAuthenticationToken.Fetch(userId).then(token => { - if (token!.expiry_date! < new Date().getTime()) { - // Token has expired, so submitting a request for a refreshed access token - return refreshToken(token!, client_id, client_secret, oAuth2Client, userId).then(resolve, reject); - } - // Authentication successful! - oAuth2Client.setCredentials(token!); - resolve({ token: token!, client: oAuth2Client }); - }); - }); - } + type AuthenticationResult = { token: Credentials, client: OAuth2Client }; const refreshEndpoint = "https://oauth2.googleapis.com/token"; - const refreshToken = (credentials: Credentials, client_id: string, client_secret: string, oAuth2Client: OAuth2Client, userId: string) => { - return new Promise(resolve => { + const refreshToken = (credentials: Credentials, oAuth2Client: OAuth2Client, userId: string) => { + return new Promise(resolve => { let headerParameters = { headers: { 'Content-Type': 'application/x-www-form-urlencoded' } }; let queryParameters = { refreshToken: credentials.refresh_token, - client_id, - client_secret, + ...installed, grant_type: "refresh_token" }; let url = `${refreshEndpoint}?${qs.stringify(queryParameters)}`; diff --git a/src/server/index.ts b/src/server/index.ts index 384800f23..3220a9533 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -564,10 +564,10 @@ function routeSetter(router: RouteManager) { router.addSupervisedRoute({ method: Method.POST, subscription: new RouteSubscriber(RouteStore.googleDocs).add("sector", "action"), - onValidation: ({ req, res }) => { + onValidation: ({ req, res, user }) => { let sector: GoogleApiServerUtils.Service = req.params.sector as GoogleApiServerUtils.Service; let action: GoogleApiServerUtils.Action = req.params.action as GoogleApiServerUtils.Action; - GoogleApiServerUtils.GetEndpoint(GoogleApiServerUtils.Service[sector], { credentialsPath, userId: req.headers.userId as string }).then(endpoint => { + GoogleApiServerUtils.GetEndpoint(GoogleApiServerUtils.Service[sector], user.id).then(endpoint => { let handler = EndpointHandlerMap.get(action); if (endpoint && handler) { let execute = handler(endpoint, req.body).then( @@ -592,7 +592,7 @@ function routeSetter(router: RouteManager) { if (!token) { return res.send(await GoogleApiServerUtils.GenerateAuthenticationUrl(information)); } - GoogleApiServerUtils.RetrieveAccessToken(information).then(token => res.send(token)); + GoogleApiServerUtils.RetrieveAccessToken(userId).then(token => res.send(token)); } }); @@ -600,9 +600,7 @@ function routeSetter(router: RouteManager) { method: Method.POST, subscription: RouteStore.writeGoogleAccessToken, onValidation: async ({ user, req, res }) => { - const userId = user.id; - const information = { credentialsPath, userId }; - res.send(await GoogleApiServerUtils.ProcessClientSideCode(information, req.body.authenticationCode)); + res.send(await GoogleApiServerUtils.ProcessClientSideCode(user.id, req.body.authenticationCode)); } }); -- cgit v1.2.3-70-g09d2 From f0f3dddbe1d3ac54d3754bb913b8ecd9eb6fcc63 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Sun, 27 Oct 2019 21:05:01 -0400 Subject: further cleanup for oauth, separated access token function from client function --- src/server/apis/google/GoogleApiServerUtils.ts | 65 ++++++++++++----------- src/server/apis/google/GooglePhotosUploadUtils.ts | 19 +++---- src/server/database.ts | 2 +- src/server/index.ts | 24 +++------ 4 files changed, 47 insertions(+), 63 deletions(-) (limited to 'src') diff --git a/src/server/apis/google/GoogleApiServerUtils.ts b/src/server/apis/google/GoogleApiServerUtils.ts index c0824cfb7..88f0f3377 100644 --- a/src/server/apis/google/GoogleApiServerUtils.ts +++ b/src/server/apis/google/GoogleApiServerUtils.ts @@ -15,7 +15,6 @@ const path = require("path"); */ export namespace GoogleApiServerUtils { - // If modifying these scopes, delete token.json. const prefix = 'https://www.googleapis.com/auth/'; const SCOPES = [ 'documents.readonly', @@ -54,7 +53,7 @@ export namespace GoogleApiServerUtils { export const GetEndpoint = (sector: string, userId: string) => { return new Promise>(resolve => { - authorize(userId).then(({ client: auth }) => { + retrieveOAuthClient(userId).then(auth => { let routed: Opt; let parameters: EndpointParameters = { auth, version: "v1" }; switch (sector) { @@ -70,10 +69,23 @@ export namespace GoogleApiServerUtils { }); }; - export const RetrieveAccessToken = (userId: string): Promise => { + export const retrieveAccessToken = (userId: string): Promise => { return new Promise((resolve, reject) => { - authorize(userId).then( - ({ token: { access_token } }) => resolve(access_token!), + retrieveCredentials(userId).then( + ({ access_token }) => resolve(access_token!), + error => reject(`Error: unable to authenticate Google Photos API request.\n${error}`) + ); + }); + }; + + export const retrieveOAuthClient = (userId: string): Promise => { + return new Promise((resolve, reject) => { + retrieveCredentials(userId).then( + credentials => { + const client = generateClient(); + client.setCredentials(credentials); + resolve(client); + }, error => reject(`Error: unable to authenticate Google Photos API request.\n${error}`) ); }); @@ -82,7 +94,7 @@ export namespace GoogleApiServerUtils { let installed: OAuth2ClientOptions; let worker: OAuth2Client; - export const LoadOAuthClient = async () => { + export const loadClientSecret = async () => { return new Promise((resolve, reject) => { readFile(path.join(__dirname, "../../credentials/google_docs_credentials.json"), async (err, credentials) => { if (err) { @@ -90,7 +102,7 @@ export namespace GoogleApiServerUtils { return console.log('Error loading client secret file:', err); } installed = parseBuffer(credentials).installed; - worker = new google.auth.OAuth2(installed); + worker = generateClient(); resolve(); }); }); @@ -98,7 +110,7 @@ export namespace GoogleApiServerUtils { const generateClient = () => new google.auth.OAuth2(installed); - export const GenerateAuthenticationUrl = async (information: CredentialInformation) => { + export const generateAuthenticationUrl = async () => { return worker.generateAuthUrl({ access_type: 'offline', scope: SCOPES.map(relative => prefix + relative), @@ -110,7 +122,7 @@ export namespace GoogleApiServerUtils { avatar: string; name: string; } - export const ProcessClientSideCode = async (userId: string, authenticationCode: string): Promise => { + export const processNewUser = async (userId: string, authenticationCode: string): Promise => { return new Promise((resolve, reject) => { worker.getToken(authenticationCode, async (err, token) => { if (err || !token) { @@ -158,35 +170,25 @@ export namespace GoogleApiServerUtils { sub: string; } - export const authorize = async (userId: string): Promise => { - return Database.Auxiliary.GoogleAuthenticationToken.Fetch(userId).then(token => { - return new Promise((resolve, reject) => { - const client = generateClient(); - if (token!.expiry_date! < new Date().getTime()) { + const retrieveCredentials = async (userId: string): Promise => { + return new Promise((resolve, reject) => { + Database.Auxiliary.GoogleAuthenticationToken.Fetch(userId).then(credentials => { + if (!credentials) { + return reject(); + } + if (credentials!.expiry_date! < new Date().getTime()) { // Token has expired, so submitting a request for a refreshed access token - return refreshToken(token!, client, userId).then(resolve, reject); + return refreshAccessToken(credentials!, userId).then(resolve, reject); } // Authentication successful! - client.setCredentials(token!); - resolve({ token: token!, client }); + resolve(credentials); }); }); }; - export const RetrievePhotosEndpoint = (userId: string) => { - return new Promise((resolve, reject) => { - RetrieveAccessToken(userId).then( - token => resolve(new Photos(token)), - reject - ); - }); - }; - - type AuthenticationResult = { token: Credentials, client: OAuth2Client }; - const refreshEndpoint = "https://oauth2.googleapis.com/token"; - const refreshToken = (credentials: Credentials, oAuth2Client: OAuth2Client, userId: string) => { - return new Promise(resolve => { + const refreshAccessToken = (credentials: Credentials, userId: string) => { + return new Promise(resolve => { let headerParameters = { headers: { 'Content-Type': 'application/x-www-form-urlencoded' } }; let queryParameters = { refreshToken: credentials.refresh_token, @@ -200,8 +202,7 @@ export namespace GoogleApiServerUtils { await Database.Auxiliary.GoogleAuthenticationToken.Update(userId, access_token, expiry_date); credentials.access_token = access_token; credentials.expiry_date = expiry_date; - oAuth2Client.setCredentials(credentials); - resolve({ token: credentials, client: oAuth2Client }); + resolve(credentials); }); }); }; diff --git a/src/server/apis/google/GooglePhotosUploadUtils.ts b/src/server/apis/google/GooglePhotosUploadUtils.ts index 4a67e57cc..d704faa71 100644 --- a/src/server/apis/google/GooglePhotosUploadUtils.ts +++ b/src/server/apis/google/GooglePhotosUploadUtils.ts @@ -20,19 +20,12 @@ export namespace GooglePhotosUploadUtils { } const prepend = (extension: string) => `https://photoslibrary.googleapis.com/v1/${extension}`; - const headers = (type: string) => ({ + const headers = (type: string, token: string) => ({ 'Content-Type': `application/${type}`, - 'Authorization': Bearer, + 'Authorization': token, }); - let Bearer: string; - - export const initialize = async (information: GoogleApiServerUtils.CredentialInformation) => { - const token = await GoogleApiServerUtils.RetrieveAccessToken(information); - Bearer = `Bearer ${token}`; - }; - - export const DispatchGooglePhotosUpload = async (url: string) => { + export const DispatchGooglePhotosUpload = async (bearerToken: string, url: string) => { if (!DashUploadUtils.imageFormats.includes(path.extname(url))) { return undefined; } @@ -40,7 +33,7 @@ export namespace GooglePhotosUploadUtils { const parameters = { method: 'POST', headers: { - ...headers('octet-stream'), + ...headers('octet-stream', bearerToken), 'X-Goog-Upload-File-Name': path.basename(url), 'X-Goog-Upload-Protocol': 'raw' }, @@ -56,13 +49,13 @@ export namespace GooglePhotosUploadUtils { })); }; - export const CreateMediaItems = async (newMediaItems: NewMediaItem[], album?: { id: string }): Promise => { + export const CreateMediaItems = async (bearerToken: string, newMediaItems: NewMediaItem[], album?: { id: string }): Promise => { const newMediaItemResults = await BatchedArray.from(newMediaItems, { batchSize: 50 }).batchedMapPatientInterval( { magnitude: 100, unit: TimeUnit.Milliseconds }, async (batch: NewMediaItem[]) => { const parameters = { method: 'POST', - headers: headers('json'), + headers: headers('json', bearerToken), uri: prepend('mediaItems:batchCreate'), body: { newMediaItems: batch } as any, json: true diff --git a/src/server/database.ts b/src/server/database.ts index 12626e594..79dd26b7d 100644 --- a/src/server/database.ts +++ b/src/server/database.ts @@ -298,7 +298,7 @@ export namespace Database { export type StoredCredentials = Credentials & { _id: string }; - export const Fetch = async (userId: string, removeId = true) => { + export const Fetch = async (userId: string, removeId = true): Promise> => { return SanitizedSingletonQuery({ userId }, GoogleAuthentication, removeId); }; diff --git a/src/server/index.ts b/src/server/index.ts index 3220a9533..24866a5e5 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -65,7 +65,7 @@ async function PreliminaryFunctions() { resolve(); }); }); - await GoogleApiServerUtils.LoadOAuthClient(); + await GoogleApiServerUtils.loadClientSecret(); await DashUploadUtils.createIfNotExists(pdfDirectory); await Database.tryInitializeConnection(); } @@ -553,8 +553,6 @@ function routeSetter(router: RouteManager) { } }); - const credentialsPath = path.join(__dirname, "./credentials/google_docs_credentials.json"); - const EndpointHandlerMap = new Map([ ["create", (api, params) => api.create(params)], ["retrieve", (api, params) => api.get(params)], @@ -588,11 +586,10 @@ function routeSetter(router: RouteManager) { onValidation: async ({ user, res }) => { const userId = user.id; const token = await Database.Auxiliary.GoogleAuthenticationToken.Fetch(userId); - const information = { credentialsPath, userId }; if (!token) { - return res.send(await GoogleApiServerUtils.GenerateAuthenticationUrl(information)); + return res.send(await GoogleApiServerUtils.generateAuthenticationUrl()); } - GoogleApiServerUtils.RetrieveAccessToken(userId).then(token => res.send(token)); + GoogleApiServerUtils.retrieveAccessToken(userId).then(token => res.send(token)); } }); @@ -600,35 +597,28 @@ function routeSetter(router: RouteManager) { method: Method.POST, subscription: RouteStore.writeGoogleAccessToken, onValidation: async ({ user, req, res }) => { - res.send(await GoogleApiServerUtils.ProcessClientSideCode(user.id, req.body.authenticationCode)); + res.send(await GoogleApiServerUtils.processNewUser(user.id, req.body.authenticationCode)); } }); const tokenError = "Unable to successfully upload bytes for all images!"; const mediaError = "Unable to convert all uploaded bytes to media items!"; - const userIdError = "Unable to parse the identification of the user!"; router.addSupervisedRoute({ method: Method.POST, subscription: RouteStore.googlePhotosMediaUpload, onValidation: async ({ user, req, res }) => { const { media } = req.body; - const userId = user.id; - if (!userId) { - return _error(res, userIdError); - } - - await GooglePhotosUploadUtils.initialize({ credentialsPath, userId }); let failed: number[] = []; - + const token = await GoogleApiServerUtils.retrieveAccessToken(user.id); const newMediaItems = await BatchedArray.from(media, { batchSize: 25 }).batchedMapPatientInterval( { magnitude: 100, unit: TimeUnit.Milliseconds }, async (batch: GooglePhotosUploadUtils.MediaInput[]) => { const newMediaItems: NewMediaItem[] = []; for (let index = 0; index < batch.length; index++) { const element = batch[index]; - const uploadToken = await GooglePhotosUploadUtils.DispatchGooglePhotosUpload(element.url); + const uploadToken = await GooglePhotosUploadUtils.DispatchGooglePhotosUpload(token, element.url); if (!uploadToken) { failed.push(index); } else { @@ -647,7 +637,7 @@ function routeSetter(router: RouteManager) { console.error(`Unable to upload ${failedCount} image${failedCount === 1 ? "" : "s"} to Google's servers`); } - GooglePhotosUploadUtils.CreateMediaItems(newMediaItems, req.body.album).then( + GooglePhotosUploadUtils.CreateMediaItems(token, newMediaItems, req.body.album).then( result => _success(res, { results: result.newMediaItemResults, failed }), error => _error(res, mediaError, error) ); -- cgit v1.2.3-70-g09d2 From c4e832aa5c384c9d5f018ed1148cc003e988a45e Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Sun, 27 Oct 2019 21:09:53 -0400 Subject: cleanup --- src/server/apis/google/GoogleApiServerUtils.ts | 115 +++++++++++++------------ 1 file changed, 58 insertions(+), 57 deletions(-) (limited to 'src') diff --git a/src/server/apis/google/GoogleApiServerUtils.ts b/src/server/apis/google/GoogleApiServerUtils.ts index 88f0f3377..5a6aa7abe 100644 --- a/src/server/apis/google/GoogleApiServerUtils.ts +++ b/src/server/apis/google/GoogleApiServerUtils.ts @@ -6,28 +6,68 @@ import { GlobalOptions } from "googleapis-common"; import { GaxiosResponse } from "gaxios"; import request = require('request-promise'); import * as qs from 'query-string'; -import Photos = require('googlephotos'); import { Database } from "../../database"; const path = require("path"); +const prefix = 'https://www.googleapis.com/auth/'; +const refreshEndpoint = "https://oauth2.googleapis.com/token"; +const SCOPES = [ + 'documents.readonly', + 'documents', + 'presentations', + 'presentations.readonly', + 'drive', + 'drive.file', + 'photoslibrary', + 'photoslibrary.appendonly', + 'photoslibrary.sharing', + 'userinfo.profile' +]; + /** * Server side authentication for Google Api queries. */ export namespace GoogleApiServerUtils { - const prefix = 'https://www.googleapis.com/auth/'; - const SCOPES = [ - 'documents.readonly', - 'documents', - 'presentations', - 'presentations.readonly', - 'drive', - 'drive.file', - 'photoslibrary', - 'photoslibrary.appendonly', - 'photoslibrary.sharing', - 'userinfo.profile' - ]; + export type EnrichedCredentials = Credentials & { userInfo: UserInfo }; + + export interface GoogleAuthenticationResult { + access_token: string; + avatar: string; + name: string; + } + + export interface UserInfo { + at_hash: string; + aud: string; + azp: string; + exp: number; + family_name: string; + given_name: string; + iat: number; + iss: string; + locale: string; + name: string; + picture: string; + sub: string; + } + + let installed: OAuth2ClientOptions; + let worker: OAuth2Client; + + export const loadClientSecret = async () => { + return new Promise((resolve, reject) => { + readFile(path.join(__dirname, "../../credentials/google_docs_credentials.json"), async (err, credentials) => { + if (err) { + reject(err); + return console.log('Error loading client secret file:', err); + } + installed = parseBuffer(credentials).installed; + worker = generateClient(); + resolve(); + }); + }); + }; const ClientMapping = new Map(); @@ -71,7 +111,7 @@ export namespace GoogleApiServerUtils { export const retrieveAccessToken = (userId: string): Promise => { return new Promise((resolve, reject) => { - retrieveCredentials(userId).then( + retrieveCurrentCredentials(userId).then( ({ access_token }) => resolve(access_token!), error => reject(`Error: unable to authenticate Google Photos API request.\n${error}`) ); @@ -80,34 +120,17 @@ export namespace GoogleApiServerUtils { export const retrieveOAuthClient = (userId: string): Promise => { return new Promise((resolve, reject) => { - retrieveCredentials(userId).then( + retrieveCurrentCredentials(userId).then( credentials => { const client = generateClient(); client.setCredentials(credentials); resolve(client); }, - error => reject(`Error: unable to authenticate Google Photos API request.\n${error}`) + error => reject(`Error: unable to instantiate and certify a new OAuth2 client.\n${error}`) ); }); }; - let installed: OAuth2ClientOptions; - let worker: OAuth2Client; - - export const loadClientSecret = async () => { - return new Promise((resolve, reject) => { - readFile(path.join(__dirname, "../../credentials/google_docs_credentials.json"), async (err, credentials) => { - if (err) { - reject(err); - return console.log('Error loading client secret file:', err); - } - installed = parseBuffer(credentials).installed; - worker = generateClient(); - resolve(); - }); - }); - }; - const generateClient = () => new google.auth.OAuth2(installed); export const generateAuthenticationUrl = async () => { @@ -117,11 +140,6 @@ export namespace GoogleApiServerUtils { }); }; - export interface GoogleAuthenticationResult { - access_token: string; - avatar: string; - name: string; - } export const processNewUser = async (userId: string, authenticationCode: string): Promise => { return new Promise((resolve, reject) => { worker.getToken(authenticationCode, async (err, token) => { @@ -154,23 +172,7 @@ export namespace GoogleApiServerUtils { return { ...credentials, userInfo }; }; - export type EnrichedCredentials = Credentials & { userInfo: UserInfo }; - export interface UserInfo { - at_hash: string; - aud: string; - azp: string; - exp: number; - family_name: string; - given_name: string; - iat: number; - iss: string; - locale: string; - name: string; - picture: string; - sub: string; - } - - const retrieveCredentials = async (userId: string): Promise => { + const retrieveCurrentCredentials = async (userId: string): Promise => { return new Promise((resolve, reject) => { Database.Auxiliary.GoogleAuthenticationToken.Fetch(userId).then(credentials => { if (!credentials) { @@ -186,7 +188,6 @@ export namespace GoogleApiServerUtils { }); }; - const refreshEndpoint = "https://oauth2.googleapis.com/token"; const refreshAccessToken = (credentials: Credentials, userId: string) => { return new Promise(resolve => { let headerParameters = { headers: { 'Content-Type': 'application/x-www-form-urlencoded' } }; -- cgit v1.2.3-70-g09d2 From b217bd842356deace1e6620625b8f1841a9bce7b Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Mon, 28 Oct 2019 02:10:58 -0400 Subject: using client mapping --- src/server/apis/google/GoogleApiServerUtils.ts | 43 ++++++++++++++++++-------- 1 file changed, 30 insertions(+), 13 deletions(-) (limited to 'src') diff --git a/src/server/apis/google/GoogleApiServerUtils.ts b/src/server/apis/google/GoogleApiServerUtils.ts index 5a6aa7abe..ad7540e5d 100644 --- a/src/server/apis/google/GoogleApiServerUtils.ts +++ b/src/server/apis/google/GoogleApiServerUtils.ts @@ -37,6 +37,11 @@ export namespace GoogleApiServerUtils { name: string; } + export interface CredentialsResult { + credentials: Credentials; + refreshed: boolean; + } + export interface UserInfo { at_hash: string; aud: string; @@ -69,7 +74,7 @@ export namespace GoogleApiServerUtils { }); }; - const ClientMapping = new Map(); + const authenticationClients = new Map(); export const parseBuffer = (data: Buffer) => JSON.parse(data.toString()); @@ -111,8 +116,8 @@ export namespace GoogleApiServerUtils { export const retrieveAccessToken = (userId: string): Promise => { return new Promise((resolve, reject) => { - retrieveCurrentCredentials(userId).then( - ({ access_token }) => resolve(access_token!), + retrieveCredentials(userId).then( + ({ credentials }) => resolve(credentials.access_token!), error => reject(`Error: unable to authenticate Google Photos API request.\n${error}`) ); }); @@ -120,10 +125,14 @@ export namespace GoogleApiServerUtils { export const retrieveOAuthClient = (userId: string): Promise => { return new Promise((resolve, reject) => { - retrieveCurrentCredentials(userId).then( - credentials => { - const client = generateClient(); - client.setCredentials(credentials); + retrieveCredentials(userId).then( + ({ credentials, refreshed }) => { + let client = authenticationClients.get(userId); + if (!client) { + authenticationClients.set(userId, client = generateClientWith(credentials)); + } else if (refreshed) { + client.setCredentials(credentials); + } resolve(client); }, error => reject(`Error: unable to instantiate and certify a new OAuth2 client.\n${error}`) @@ -131,7 +140,15 @@ export namespace GoogleApiServerUtils { }); }; - const generateClient = () => new google.auth.OAuth2(installed); + function generateClient() { + return new google.auth.OAuth2(installed); + } + + function generateClientWith(credentials: Credentials) { + const client = new google.auth.OAuth2(installed); + client.setCredentials(credentials); + return client; + } export const generateAuthenticationUrl = async () => { return worker.generateAuthUrl({ @@ -172,8 +189,8 @@ export namespace GoogleApiServerUtils { return { ...credentials, userInfo }; }; - const retrieveCurrentCredentials = async (userId: string): Promise => { - return new Promise((resolve, reject) => { + const retrieveCredentials = async (userId: string): Promise => { + return new Promise((resolve, reject) => { Database.Auxiliary.GoogleAuthenticationToken.Fetch(userId).then(credentials => { if (!credentials) { return reject(); @@ -183,13 +200,13 @@ export namespace GoogleApiServerUtils { return refreshAccessToken(credentials!, userId).then(resolve, reject); } // Authentication successful! - resolve(credentials); + resolve({ credentials, refreshed: false }); }); }); }; const refreshAccessToken = (credentials: Credentials, userId: string) => { - return new Promise(resolve => { + return new Promise(resolve => { let headerParameters = { headers: { 'Content-Type': 'application/x-www-form-urlencoded' } }; let queryParameters = { refreshToken: credentials.refresh_token, @@ -203,7 +220,7 @@ export namespace GoogleApiServerUtils { await Database.Auxiliary.GoogleAuthenticationToken.Update(userId, access_token, expiry_date); credentials.access_token = access_token; credentials.expiry_date = expiry_date; - resolve(credentials); + resolve({ credentials, refreshed: true }); }); }); }; -- cgit v1.2.3-70-g09d2 From 1f6e1d7e063f9ce1c08486f8c0c11b6c2c4198dc Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Mon, 28 Oct 2019 04:11:53 -0400 Subject: repaired google photos routine, no route handlers can have dangling promises --- src/server/ApiManagers/ExportManager.ts | 4 +- src/server/ApiManagers/UtilManager.ts | 36 ++-- src/server/RouteManager.ts | 1 + src/server/apis/google/GoogleApiServerUtils.ts | 9 +- src/server/apis/google/GooglePhotosUploadUtils.ts | 2 +- src/server/index.ts | 202 +++++++++++----------- 6 files changed, 137 insertions(+), 117 deletions(-) (limited to 'src') diff --git a/src/server/ApiManagers/ExportManager.ts b/src/server/ApiManagers/ExportManager.ts index 261acbbe0..14ac7dd5b 100644 --- a/src/server/ApiManagers/ExportManager.ts +++ b/src/server/ApiManagers/ExportManager.ts @@ -26,7 +26,7 @@ export default class ExportManager extends ApiManager { const id = req.params.docId; const hierarchy: Hierarchy = {}; await buildHierarchyRecursive(id, hierarchy); - BuildAndDispatchZip(res, zip => writeHierarchyRecursive(zip, hierarchy)); + return BuildAndDispatchZip(res, zip => writeHierarchyRecursive(zip, hierarchy)); } }); } @@ -48,7 +48,7 @@ export async function BuildAndDispatchZip(res: express.Response, mutator: ZipMut const zip = Archiver('zip'); zip.pipe(res); await mutator(zip); - zip.finalize(); + return zip.finalize(); } /** diff --git a/src/server/ApiManagers/UtilManager.ts b/src/server/ApiManagers/UtilManager.ts index a3f802b20..61cda2e9b 100644 --- a/src/server/ApiManagers/UtilManager.ts +++ b/src/server/ApiManagers/UtilManager.ts @@ -10,13 +10,16 @@ export default class UtilManager extends ApiManager { register({ method: Method.GET, subscription: "/pull", - onValidation: ({ res }) => { - exec('"C:\\Program Files\\Git\\git-bash.exe" -c "git pull"', err => { - if (err) { - res.send(err.message); - return; - } - res.redirect("/"); + onValidation: async ({ res }) => { + return new Promise(resolve => { + exec('"C:\\Program Files\\Git\\git-bash.exe" -c "git pull"', err => { + if (err) { + res.send(err.message); + return; + } + res.redirect("/"); + resolve(); + }); }); } }); @@ -24,14 +27,14 @@ export default class UtilManager extends ApiManager { register({ method: Method.GET, subscription: "/buxton", - onValidation: ({ res }) => { + onValidation: async ({ res }) => { let cwd = '../scraping/buxton'; let onResolved = (stdout: string) => { console.log(stdout); res.redirect("/"); }; let onRejected = (err: any) => { console.error(err.message); res.send(err); }; let tryPython3 = () => command_line('python3 scraper.py', cwd).then(onResolved, onRejected); - command_line('python scraper.py', cwd).then(onResolved, tryPython3); + return command_line('python scraper.py', cwd).then(onResolved, tryPython3); }, }); @@ -39,12 +42,15 @@ export default class UtilManager extends ApiManager { method: Method.GET, subscription: "/version", onValidation: ({ res }) => { - exec('"C:\\Program Files\\Git\\bin\\git.exe" rev-parse HEAD', (err, stdout) => { - if (err) { - res.send(err.message); - return; - } - res.send(stdout); + return new Promise(resolve => { + exec('"C:\\Program Files\\Git\\bin\\git.exe" rev-parse HEAD', (err, stdout) => { + if (err) { + res.send(err.message); + return; + } + res.send(stdout); + }); + resolve(); }); } }); diff --git a/src/server/RouteManager.ts b/src/server/RouteManager.ts index ef083a88a..21ce9c9e4 100644 --- a/src/server/RouteManager.ts +++ b/src/server/RouteManager.ts @@ -75,6 +75,7 @@ export default class RouteManager { } setTimeout(() => { if (!res.headersSent) { + console.log("Initiating fallback for ", target); const warning = `request to ${target} fell through - this is a fallback response`; res.send({ warning }); } diff --git a/src/server/apis/google/GoogleApiServerUtils.ts b/src/server/apis/google/GoogleApiServerUtils.ts index ad7540e5d..1cca07036 100644 --- a/src/server/apis/google/GoogleApiServerUtils.ts +++ b/src/server/apis/google/GoogleApiServerUtils.ts @@ -62,12 +62,17 @@ export namespace GoogleApiServerUtils { export const loadClientSecret = async () => { return new Promise((resolve, reject) => { - readFile(path.join(__dirname, "../../credentials/google_docs_credentials.json"), async (err, credentials) => { + readFile(path.join(__dirname, "../../credentials/google_docs_credentials.json"), async (err, projectCredentials) => { if (err) { reject(err); return console.log('Error loading client secret file:', err); } - installed = parseBuffer(credentials).installed; + const { client_secret, client_id, redirect_uris } = parseBuffer(projectCredentials).installed; + installed = { + clientId: client_id, + clientSecret: client_secret, + redirectUri: redirect_uris[0] + }; worker = generateClient(); resolve(); }); diff --git a/src/server/apis/google/GooglePhotosUploadUtils.ts b/src/server/apis/google/GooglePhotosUploadUtils.ts index d704faa71..172fa8d46 100644 --- a/src/server/apis/google/GooglePhotosUploadUtils.ts +++ b/src/server/apis/google/GooglePhotosUploadUtils.ts @@ -22,7 +22,7 @@ export namespace GooglePhotosUploadUtils { const prepend = (extension: string) => `https://photoslibrary.googleapis.com/v1/${extension}`; const headers = (type: string, token: string) => ({ 'Content-Type': `application/${type}`, - 'Authorization': token, + 'Authorization': `Bearer ${token}`, }); export const DispatchGooglePhotosUpload = async (bearerToken: string, url: string) => { diff --git a/src/server/index.ts b/src/server/index.ts index 24866a5e5..eb19c71a9 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -225,55 +225,58 @@ function routeSetter(router: RouteManager) { } } }; - form.parse(req, async (err, fields, files) => { - remap = fields.remap !== "false"; - let id: string = ""; - try { - for (const name in files) { - const path_2 = files[name].path; - const zip = new AdmZip(path_2); - zip.getEntries().forEach((entry: any) => { - if (!entry.entryName.startsWith("files/")) return; - let dirname = path.dirname(entry.entryName) + "/"; - let extname = path.extname(entry.entryName); - let basename = path.basename(entry.entryName).split(".")[0]; - // zip.extractEntryTo(dirname + basename + "_o" + extname, __dirname + RouteStore.public, true, false); - // zip.extractEntryTo(dirname + basename + "_s" + extname, __dirname + RouteStore.public, true, false); - // zip.extractEntryTo(dirname + basename + "_m" + extname, __dirname + RouteStore.public, true, false); - // zip.extractEntryTo(dirname + basename + "_l" + extname, __dirname + RouteStore.public, true, false); + return new Promise(resolve => { + form.parse(req, async (_err, fields, files) => { + remap = fields.remap !== "false"; + let id: string = ""; + try { + for (const name in files) { + const path_2 = files[name].path; + const zip = new AdmZip(path_2); + zip.getEntries().forEach((entry: any) => { + if (!entry.entryName.startsWith("files/")) return; + let dirname = path.dirname(entry.entryName) + "/"; + let extname = path.extname(entry.entryName); + let basename = path.basename(entry.entryName).split(".")[0]; + // zip.extractEntryTo(dirname + basename + "_o" + extname, __dirname + RouteStore.public, true, false); + // zip.extractEntryTo(dirname + basename + "_s" + extname, __dirname + RouteStore.public, true, false); + // zip.extractEntryTo(dirname + basename + "_m" + extname, __dirname + RouteStore.public, true, false); + // zip.extractEntryTo(dirname + basename + "_l" + extname, __dirname + RouteStore.public, true, false); + try { + zip.extractEntryTo(entry.entryName, __dirname + RouteStore.public, true, false); + dirname = "/" + dirname; + + fs.createReadStream(__dirname + RouteStore.public + dirname + basename + extname).pipe(fs.createWriteStream(__dirname + RouteStore.public + dirname + basename + "_o" + extname)); + fs.createReadStream(__dirname + RouteStore.public + dirname + basename + extname).pipe(fs.createWriteStream(__dirname + RouteStore.public + dirname + basename + "_s" + extname)); + fs.createReadStream(__dirname + RouteStore.public + dirname + basename + extname).pipe(fs.createWriteStream(__dirname + RouteStore.public + dirname + basename + "_m" + extname)); + fs.createReadStream(__dirname + RouteStore.public + dirname + basename + extname).pipe(fs.createWriteStream(__dirname + RouteStore.public + dirname + basename + "_l" + extname)); + } catch (e) { + console.log(e); + } + }); + const json = zip.getEntry("doc.json"); + let docs: any; try { - zip.extractEntryTo(entry.entryName, __dirname + RouteStore.public, true, false); - dirname = "/" + dirname; - - fs.createReadStream(__dirname + RouteStore.public + dirname + basename + extname).pipe(fs.createWriteStream(__dirname + RouteStore.public + dirname + basename + "_o" + extname)); - fs.createReadStream(__dirname + RouteStore.public + dirname + basename + extname).pipe(fs.createWriteStream(__dirname + RouteStore.public + dirname + basename + "_s" + extname)); - fs.createReadStream(__dirname + RouteStore.public + dirname + basename + extname).pipe(fs.createWriteStream(__dirname + RouteStore.public + dirname + basename + "_m" + extname)); - fs.createReadStream(__dirname + RouteStore.public + dirname + basename + extname).pipe(fs.createWriteStream(__dirname + RouteStore.public + dirname + basename + "_l" + extname)); - } catch (e) { - console.log(e); - } - }); - const json = zip.getEntry("doc.json"); - let docs: any; - try { - let data = JSON.parse(json.getData().toString("utf8")); - docs = data.docs; - id = data.id; - docs = Object.keys(docs).map(key => docs[key]); - docs.forEach(mapFn); - await Promise.all(docs.map((doc: any) => new Promise(res => Database.Instance.replace(doc.id, doc, (err, r) => { - err && console.log(err); - res(); - }, true, "newDocuments")))); - } catch (e) { console.log(e); } - fs.unlink(path_2, () => { }); - } - if (id) { - res.send(JSON.stringify(getId(id))); - } else { - res.send(JSON.stringify("error")); - } - } catch (e) { console.log(e); } + let data = JSON.parse(json.getData().toString("utf8")); + docs = data.docs; + id = data.id; + docs = Object.keys(docs).map(key => docs[key]); + docs.forEach(mapFn); + await Promise.all(docs.map((doc: any) => new Promise(res => Database.Instance.replace(doc.id, doc, (err, r) => { + err && console.log(err); + res(); + }, true, "newDocuments")))); + } catch (e) { console.log(e); } + fs.unlink(path_2, () => { }); + } + if (id) { + res.send(JSON.stringify(getId(id))); + } else { + res.send(JSON.stringify("error")); + } + } catch (e) { console.log(e); } + resolve(); + }); }); } }); @@ -285,22 +288,25 @@ function routeSetter(router: RouteManager) { let filename = req.params.filename; let noExt = filename.substring(0, filename.length - ".png".length); let pagenumber = parseInt(noExt.split('-')[1]); - fs.exists(uploadDirectory + filename, (exists: boolean) => { - console.log(`${uploadDirectory + filename} ${exists ? "exists" : "does not exist"}`); - if (exists) { - let input = fs.createReadStream(uploadDirectory + filename); - probe(input, (err: any, result: any) => { - if (err) { - console.log(err); - console.log(`error on ${filename}`); - return; - } - res.send({ path: "/files/" + filename, width: result.width, height: result.height }); - }); - } - else { - LoadPage(uploadDirectory + filename.substring(0, filename.length - noExt.split('-')[1].length - ".PNG".length - 1) + ".pdf", pagenumber, res); - } + return new Promise(resolve => { + fs.exists(uploadDirectory + filename, (exists: boolean) => { + console.log(`${uploadDirectory + filename} ${exists ? "exists" : "does not exist"}`); + if (exists) { + let input = fs.createReadStream(uploadDirectory + filename); + probe(input, (err: any, result: any) => { + if (err) { + console.log(err); + console.log(`error on ${filename}`); + return; + } + res.send({ path: "/files/" + filename, width: result.width, height: result.height }); + }); + } + else { + LoadPage(uploadDirectory + filename.substring(0, filename.length - noExt.split('-')[1].length - ".PNG".length - 1) + ".pdf", pagenumber, res); + } + resolve(); + }); }); } }); @@ -414,8 +420,8 @@ function routeSetter(router: RouteManager) { var canvas = createCanvas(width, height); var context = canvas.getContext('2d'); return { - canvas: canvas, - context: context, + canvas, + context }; } @@ -442,37 +448,39 @@ function routeSetter(router: RouteManager) { router.addSupervisedRoute({ method: Method.POST, subscription: RouteStore.upload, - onValidation: ({ req, res }) => { + onValidation: async ({ req, res }) => { let form = new formidable.IncomingForm(); form.uploadDir = uploadDirectory; form.keepExtensions = true; - form.parse(req, async (_err, _fields, files) => { - let results: ImageFileResponse[] = []; - for (const key in files) { - const { type, path: location, name } = files[key]; - const filename = path.basename(location); - let uploadInformation: Opt; - if (filename.endsWith(".pdf")) { - let dataBuffer = fs.readFileSync(uploadDirectory + filename); - const result: ParsedPDF = await pdf(dataBuffer); - await new Promise(resolve => { - const path = pdfDirectory + "/" + filename.substring(0, filename.length - ".pdf".length) + ".txt"; - fs.createWriteStream(path).write(result.text, error => { - if (!error) { - resolve(); - } else { - reject(error); - } + return new Promise(resolve => { + form.parse(req, async (_err, _fields, files) => { + let results: ImageFileResponse[] = []; + for (const key in files) { + const { type, path: location, name } = files[key]; + const filename = path.basename(location); + let uploadInformation: Opt; + if (filename.endsWith(".pdf")) { + let dataBuffer = fs.readFileSync(uploadDirectory + filename); + const result: ParsedPDF = await pdf(dataBuffer); + await new Promise(resolve => { + const path = pdfDirectory + "/" + filename.substring(0, filename.length - ".pdf".length) + ".txt"; + fs.createWriteStream(path).write(result.text, error => { + if (!error) { + resolve(); + } else { + reject(error); + } + }); }); - }); - } else { - uploadInformation = await DashUploadUtils.UploadImage(uploadDirectory + filename, filename); + } else { + uploadInformation = await DashUploadUtils.UploadImage(uploadDirectory + filename, filename); + } + const exif = uploadInformation ? uploadInformation.exifData : undefined; + results.push({ name, type, path: `/files/${filename}`, exif }); } - const exif = uploadInformation ? uploadInformation.exifData : undefined; - results.push({ name, type, path: `/files/${filename}`, exif }); - - } - _success(res, results); + _success(res, results); + resolve(); + }); }); } }); @@ -500,7 +508,7 @@ function routeSetter(router: RouteManager) { res.status(401).send("incorrect parameters specified"); return; } - imageDataUri.outputFile(uri, uploadDirectory + filename).then((savedName: string) => { + return imageDataUri.outputFile(uri, uploadDirectory + filename).then((savedName: string) => { const ext = path.extname(savedName); let resizers = [ { resizer: sharp().resize(100, undefined, { withoutEnlargement: true }), suffix: "_s" }, @@ -562,10 +570,10 @@ function routeSetter(router: RouteManager) { router.addSupervisedRoute({ method: Method.POST, subscription: new RouteSubscriber(RouteStore.googleDocs).add("sector", "action"), - onValidation: ({ req, res, user }) => { + onValidation: async ({ req, res, user }) => { let sector: GoogleApiServerUtils.Service = req.params.sector as GoogleApiServerUtils.Service; let action: GoogleApiServerUtils.Action = req.params.action as GoogleApiServerUtils.Action; - GoogleApiServerUtils.GetEndpoint(GoogleApiServerUtils.Service[sector], user.id).then(endpoint => { + return GoogleApiServerUtils.GetEndpoint(GoogleApiServerUtils.Service[sector], user.id).then(endpoint => { let handler = EndpointHandlerMap.get(action); if (endpoint && handler) { let execute = handler(endpoint, req.body).then( @@ -589,7 +597,7 @@ function routeSetter(router: RouteManager) { if (!token) { return res.send(await GoogleApiServerUtils.generateAuthenticationUrl()); } - GoogleApiServerUtils.retrieveAccessToken(userId).then(token => res.send(token)); + return GoogleApiServerUtils.retrieveAccessToken(userId).then(token => res.send(token)); } }); @@ -637,7 +645,7 @@ function routeSetter(router: RouteManager) { console.error(`Unable to upload ${failedCount} image${failedCount === 1 ? "" : "s"} to Google's servers`); } - GooglePhotosUploadUtils.CreateMediaItems(token, newMediaItems, req.body.album).then( + return GooglePhotosUploadUtils.CreateMediaItems(token, newMediaItems, req.body.album).then( result => _success(res, { results: result.newMediaItemResults, failed }), error => _error(res, mediaError, error) ); -- cgit v1.2.3-70-g09d2 From acea9d7aa984fe8b1eeac0546833d3dca3c844e3 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Mon, 28 Oct 2019 04:14:37 -0400 Subject: removed one-line functions --- src/server/apis/google/GoogleApiServerUtils.ts | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) (limited to 'src') diff --git a/src/server/apis/google/GoogleApiServerUtils.ts b/src/server/apis/google/GoogleApiServerUtils.ts index 1cca07036..92bb8d072 100644 --- a/src/server/apis/google/GoogleApiServerUtils.ts +++ b/src/server/apis/google/GoogleApiServerUtils.ts @@ -67,13 +67,12 @@ export namespace GoogleApiServerUtils { reject(err); return console.log('Error loading client secret file:', err); } - const { client_secret, client_id, redirect_uris } = parseBuffer(projectCredentials).installed; - installed = { + const { client_secret, client_id, redirect_uris } = JSON.parse(projectCredentials.toString()).installed; + worker = new google.auth.OAuth2({ clientId: client_id, clientSecret: client_secret, redirectUri: redirect_uris[0] - }; - worker = generateClient(); + }); resolve(); }); }); @@ -81,8 +80,6 @@ export namespace GoogleApiServerUtils { const authenticationClients = new Map(); - export const parseBuffer = (data: Buffer) => JSON.parse(data.toString()); - export enum Service { Documents = "Documents", Slides = "Slides" @@ -145,10 +142,6 @@ export namespace GoogleApiServerUtils { }); }; - function generateClient() { - return new google.auth.OAuth2(installed); - } - function generateClientWith(credentials: Credentials) { const client = new google.auth.OAuth2(installed); client.setCredentials(credentials); -- cgit v1.2.3-70-g09d2 From b259472385b03099380f22c7c19ae135b2adf30c Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Mon, 28 Oct 2019 04:17:06 -0400 Subject: rename --- src/server/apis/google/GoogleApiServerUtils.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'src') diff --git a/src/server/apis/google/GoogleApiServerUtils.ts b/src/server/apis/google/GoogleApiServerUtils.ts index 92bb8d072..4e5175a2b 100644 --- a/src/server/apis/google/GoogleApiServerUtils.ts +++ b/src/server/apis/google/GoogleApiServerUtils.ts @@ -157,12 +157,12 @@ export namespace GoogleApiServerUtils { export const processNewUser = async (userId: string, authenticationCode: string): Promise => { return new Promise((resolve, reject) => { - worker.getToken(authenticationCode, async (err, token) => { - if (err || !token) { + worker.getToken(authenticationCode, async (err, credentials) => { + if (err || !credentials) { reject(err); return console.error('Error retrieving access token', err); } - const enriched = injectUserInfo(token); + const enriched = injectUserInfo(credentials); await Database.Auxiliary.GoogleAuthenticationToken.Write(userId, enriched); const { given_name, picture } = enriched.userInfo; resolve({ -- cgit v1.2.3-70-g09d2 From c56b602e892707dbc7e22be2edba75f49a465ec7 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Mon, 28 Oct 2019 15:48:54 -0400 Subject: server utils beginning commenting --- src/server/apis/google/GoogleApiServerUtils.ts | 168 ++++++++++++++++++++----- 1 file changed, 139 insertions(+), 29 deletions(-) (limited to 'src') diff --git a/src/server/apis/google/GoogleApiServerUtils.ts b/src/server/apis/google/GoogleApiServerUtils.ts index 4e5175a2b..9071b0485 100644 --- a/src/server/apis/google/GoogleApiServerUtils.ts +++ b/src/server/apis/google/GoogleApiServerUtils.ts @@ -7,10 +7,21 @@ import { GaxiosResponse } from "gaxios"; import request = require('request-promise'); import * as qs from 'query-string'; import { Database } from "../../database"; -const path = require("path"); +import path from "path"; +/** + * + */ const prefix = 'https://www.googleapis.com/auth/'; + +/** + * + */ const refreshEndpoint = "https://oauth2.googleapis.com/token"; + +/** + * + */ const SCOPES = [ 'documents.readonly', 'documents', @@ -25,23 +36,31 @@ const SCOPES = [ ]; /** - * Server side authentication for Google Api queries. + * This namespace manages server side authentication for Google API queries, either + * from the standard v1 APIs or the Google Photos REST API. */ export namespace GoogleApiServerUtils { - export type EnrichedCredentials = Credentials & { userInfo: UserInfo }; - + /** + * + */ export interface GoogleAuthenticationResult { access_token: string; avatar: string; name: string; } + /** + * + */ export interface CredentialsResult { credentials: Credentials; refreshed: boolean; } + /** + * + */ export interface UserInfo { at_hash: string; aud: string; @@ -57,9 +76,74 @@ export namespace GoogleApiServerUtils { sub: string; } + /** + * + */ + export enum Service { + Documents = "Documents", + Slides = "Slides" + } + + /** + * + */ + export interface CredentialInformation { + credentialsPath: string; + userId: string; + } + + /** + * + */ let installed: OAuth2ClientOptions; + + /** + * This is a global authorization client that is never + * passed around, and whose credentials are never set. + * Its job is purely to generate new authentication urls + * (users will follow to get to Google's permissions GUI) + * and to use the codes returned from that process to generate the + * initial credentials. + */ let worker: OAuth2Client; + /** + * + */ + export type ApiResponse = Promise; + + /** + * + */ + export type ApiRouter = (endpoint: Endpoint, parameters: any) => ApiResponse; + + /** + * + */ + export type ApiHandler = (parameters: any, methodOptions?: any) => ApiResponse; + + /** + * + */ + export type Action = "create" | "retrieve" | "update"; + + /** + * + */ + export interface Endpoint { + get: ApiHandler; + create: ApiHandler; + batchUpdate: ApiHandler; + } + + /** + * + */ + export type EndpointParameters = GlobalOptions & { version: "v1" }; + + /** + * + */ export const loadClientSecret = async () => { return new Promise((resolve, reject) => { readFile(path.join(__dirname, "../../credentials/google_docs_credentials.json"), async (err, projectCredentials) => { @@ -68,36 +152,28 @@ export namespace GoogleApiServerUtils { return console.log('Error loading client secret file:', err); } const { client_secret, client_id, redirect_uris } = JSON.parse(projectCredentials.toString()).installed; - worker = new google.auth.OAuth2({ + // initialize the global authorization client + installed = { clientId: client_id, clientSecret: client_secret, redirectUri: redirect_uris[0] - }); + }; + worker = generateClient(); resolve(); }); }); }; + /** + * + */ const authenticationClients = new Map(); - export enum Service { - Documents = "Documents", - Slides = "Slides" - } - - export interface CredentialInformation { - credentialsPath: string; - userId: string; - } - - export type ApiResponse = Promise; - export type ApiRouter = (endpoint: Endpoint, parameters: any) => ApiResponse; - export type ApiHandler = (parameters: any, methodOptions?: any) => ApiResponse; - export type Action = "create" | "retrieve" | "update"; - - export type Endpoint = { get: ApiHandler, create: ApiHandler, batchUpdate: ApiHandler }; - export type EndpointParameters = GlobalOptions & { version: "v1" }; - + /** + * + * @param sector + * @param userId + */ export const GetEndpoint = (sector: string, userId: string) => { return new Promise>(resolve => { retrieveOAuthClient(userId).then(auth => { @@ -116,6 +192,10 @@ export namespace GoogleApiServerUtils { }); }; + /** + * + * @param userId + */ export const retrieveAccessToken = (userId: string): Promise => { return new Promise((resolve, reject) => { retrieveCredentials(userId).then( @@ -125,13 +205,17 @@ export namespace GoogleApiServerUtils { }); }; + /** + * + * @param userId + */ export const retrieveOAuthClient = (userId: string): Promise => { return new Promise((resolve, reject) => { retrieveCredentials(userId).then( ({ credentials, refreshed }) => { let client = authenticationClients.get(userId); if (!client) { - authenticationClients.set(userId, client = generateClientWith(credentials)); + authenticationClients.set(userId, client = generateClient(credentials)); } else if (refreshed) { client.setCredentials(credentials); } @@ -142,12 +226,19 @@ export namespace GoogleApiServerUtils { }); }; - function generateClientWith(credentials: Credentials) { + /** + * + * @param credentials + */ + function generateClient(credentials?: Credentials) { const client = new google.auth.OAuth2(installed); - client.setCredentials(credentials); + credentials && client.setCredentials(credentials); return client; } + /** + * + */ export const generateAuthenticationUrl = async () => { return worker.generateAuthUrl({ access_type: 'offline', @@ -155,6 +246,11 @@ export namespace GoogleApiServerUtils { }); }; + /** + * + * @param userId + * @param authenticationCode + */ export const processNewUser = async (userId: string, authenticationCode: string): Promise => { return new Promise((resolve, reject) => { worker.getToken(authenticationCode, async (err, credentials) => { @@ -174,6 +270,11 @@ export namespace GoogleApiServerUtils { }); }; + /** + * + */ + export type EnrichedCredentials = Credentials & { userInfo: UserInfo }; + /** * It's pretty cool: the credentials id_token is split into thirds by periods. * The middle third contains a base64-encoded JSON string with all the @@ -187,6 +288,10 @@ export namespace GoogleApiServerUtils { return { ...credentials, userInfo }; }; + /** + * + * @param userId + */ const retrieveCredentials = async (userId: string): Promise => { return new Promise((resolve, reject) => { Database.Auxiliary.GoogleAuthenticationToken.Fetch(userId).then(credentials => { @@ -203,13 +308,18 @@ export namespace GoogleApiServerUtils { }); }; + /** + * + * @param credentials + * @param userId + */ const refreshAccessToken = (credentials: Credentials, userId: string) => { return new Promise(resolve => { let headerParameters = { headers: { 'Content-Type': 'application/x-www-form-urlencoded' } }; let queryParameters = { refreshToken: credentials.refresh_token, - ...installed, - grant_type: "refresh_token" + grant_type: "refresh_token", + ...installed }; let url = `${refreshEndpoint}?${qs.stringify(queryParameters)}`; request.post(url, headerParameters).then(async response => { -- cgit v1.2.3-70-g09d2 From c1c919d4d44a40d59f2ec714c143cd8f03ad3481 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Mon, 28 Oct 2019 20:04:51 -0400 Subject: clean up --- src/server/apis/google/GoogleApiServerUtils.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/server/apis/google/GoogleApiServerUtils.ts b/src/server/apis/google/GoogleApiServerUtils.ts index 9071b0485..884487509 100644 --- a/src/server/apis/google/GoogleApiServerUtils.ts +++ b/src/server/apis/google/GoogleApiServerUtils.ts @@ -298,9 +298,9 @@ export namespace GoogleApiServerUtils { if (!credentials) { return reject(); } - if (credentials!.expiry_date! < new Date().getTime()) { + if (credentials.expiry_date! < new Date().getTime()) { // Token has expired, so submitting a request for a refreshed access token - return refreshAccessToken(credentials!, userId).then(resolve, reject); + return refreshAccessToken(credentials, userId).then(resolve, reject); } // Authentication successful! resolve({ credentials, refreshed: false }); -- cgit v1.2.3-70-g09d2 From ba7568e4fe2e9323a66a91876305f829487bffb9 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Tue, 29 Oct 2019 16:00:21 -0400 Subject: beginning commenting --- src/server/apis/google/GoogleApiServerUtils.ts | 57 ++++++++++++++++++-------- 1 file changed, 40 insertions(+), 17 deletions(-) (limited to 'src') diff --git a/src/server/apis/google/GoogleApiServerUtils.ts b/src/server/apis/google/GoogleApiServerUtils.ts index 884487509..b9984649e 100644 --- a/src/server/apis/google/GoogleApiServerUtils.ts +++ b/src/server/apis/google/GoogleApiServerUtils.ts @@ -41,15 +41,6 @@ const SCOPES = [ */ export namespace GoogleApiServerUtils { - /** - * - */ - export interface GoogleAuthenticationResult { - access_token: string; - avatar: string; - name: string; - } - /** * */ @@ -247,9 +238,30 @@ export namespace GoogleApiServerUtils { }; /** - * - * @param userId - * @param authenticationCode + * This is what we return to the server in processNewUser(), after the + * worker OAuth2Client has used the user-pasted authentication code + * to retrieve an access token and an info token. The avatar is the + * URL to the Google-hosted mono-color, single white letter profile 'image'. + */ + export interface GoogleAuthenticationResult { + access_token: string; + avatar: string; + name: string; + } + + /** + * This method receives the authentication code that the + * user pasted into the overlay in the client side and uses the worker + * and the authentication code to fetch the full set of credentials that + * we'll store in the database for each user. This is called once per + * new account integration. + * @param userId The Dash user id of the user requesting account integration, used to associate the new credentials + * with a Dash user in the googleAuthentication table of the database. + * @param authenticationCode the Google-provided authentication code that the user copied + * from Google's permissions UI and pasted into the overlay. + * @returns the information necessary to authenticate a client side google photos request + * and display basic user information in the overlay on successful authentication. + * This can be expanded as needed by adding properties to the interface GoogleAuthenticationResult. */ export const processNewUser = async (userId: string, authenticationCode: string): Promise => { return new Promise((resolve, reject) => { @@ -271,7 +283,9 @@ export namespace GoogleApiServerUtils { }; /** - * + * This type represents the union of the full set of OAuth2 credentials + * and all of a Google user's publically available information. This is the strucure + * of the JSON object we ultimately store in the googleAuthentication table of the database. */ export type EnrichedCredentials = Credentials & { userInfo: UserInfo }; @@ -282,6 +296,8 @@ export namespace GoogleApiServerUtils { * base64 decode with atob and parse the JSON. * @param credentials the client credentials returned from OAuth after the user * has executed the authentication routine + * @returns the full set of credentials in the structure in which they'll be stored + * in the database. */ const injectUserInfo = (credentials: Credentials): EnrichedCredentials => { const userInfo = JSON.parse(atob(credentials.id_token!.split(".")[1])); @@ -289,8 +305,12 @@ export namespace GoogleApiServerUtils { }; /** - * - * @param userId + * Looks in the database for any credentials object with the given user id, + * and returns them. If the credentials are found but expired, the function will + * automatically refresh the credentials and then resolve with the updated values. + * @param userId the id of the Dash user requesting his/her credentials. Eventually + * might have multiple. + * @returns the credentials and whether or not they were updated in the process */ const retrieveCredentials = async (userId: string): Promise => { return new Promise((resolve, reject) => { @@ -309,8 +329,11 @@ export namespace GoogleApiServerUtils { }; /** - * - * @param credentials + * This function submits a request to OAuth with the local refresh token + * to revalidate the credentials for a given Google user associated with + * the Dash user id passed in. In addition to returning the credentials, it + * writes the diff to the database. + * @param credentials the credentials * @param userId */ const refreshAccessToken = (credentials: Credentials, userId: string) => { -- cgit v1.2.3-70-g09d2 From 3db5ea503754b74681f44ebffeb251dfad8ee65e Mon Sep 17 00:00:00 2001 From: Mohammad Amoush Date: Tue, 29 Oct 2019 17:05:01 -0400 Subject: a few comments --- src/server/Initialization.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/server/Initialization.ts b/src/server/Initialization.ts index fbb5ae7a6..3ea8f2085 100644 --- a/src/server/Initialization.ts +++ b/src/server/Initialization.ts @@ -19,6 +19,8 @@ import * as whm from 'webpack-hot-middleware'; import * as fs from 'fs'; import * as request from 'request'; +/* RouteSetter is a wrapper around the server that prevents the server + from being exposed. */ export type RouteSetter = (server: RouteManager) => void; export interface InitializationOptions { listenAtPort: number; @@ -38,7 +40,7 @@ export default async function InitializeServer(options: InitializationOptions) { registerAuthenticationRoutes(server); registerCorsProxy(server); - const isRelease = determineEnvironment(); + const isRelease = determineEnvironment(); //vs. dev mode routeSetter(new RouteManager(server, isRelease)); server.listen(listenAtPort, () => console.log(`server started at http://localhost:${listenAtPort}`)); @@ -73,6 +75,7 @@ function buildWithMiddleware(server: express.Express) { return server; } +/* Determine if the enviroment is dev mode or release mode. */ function determineEnvironment() { const isRelease = process.env.RELEASE === "true"; -- cgit v1.2.3-70-g09d2 From af25eaf2a848278a58f0993cba2e68c05da0760c Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Tue, 29 Oct 2019 20:00:54 -0400 Subject: comments and fixes for google photos server sid --- src/client/Network.ts | 12 +- src/client/apis/GoogleAuthenticationManager.tsx | 39 ++--- .../apis/google_docs/GoogleApiClientUtils.ts | 8 +- .../apis/google_docs/GooglePhotosClientUtils.ts | 10 +- .../util/Import & Export/DirectoryImportBox.tsx | 5 +- src/client/util/Import & Export/ImageUtils.ts | 4 +- src/new_fields/RichTextUtils.ts | 4 +- src/server/RouteManager.ts | 4 +- src/server/RouteStore.ts | 3 +- src/server/apis/google/GoogleApiServerUtils.ts | 171 +++++++++++---------- src/server/index.ts | 14 +- 11 files changed, 147 insertions(+), 127 deletions(-) (limited to 'src') diff --git a/src/client/Network.ts b/src/client/Network.ts index 75ccb5e99..f9ef27267 100644 --- a/src/client/Network.ts +++ b/src/client/Network.ts @@ -1,18 +1,16 @@ import { Utils } from "../Utils"; -import { CurrentUserUtils } from "../server/authentication/models/current_user_utils"; import requestPromise = require('request-promise'); -export namespace Identified { +export namespace Networking { export async function FetchFromServer(relativeRoute: string) { - return (await fetch(relativeRoute, { headers: { userId: CurrentUserUtils.id } })).text(); + return (await fetch(relativeRoute)).text(); } export async function PostToServer(relativeRoute: string, body?: any) { let options = { uri: Utils.prepend(relativeRoute), method: "POST", - headers: { userId: CurrentUserUtils.id }, body, json: true }; @@ -22,12 +20,10 @@ export namespace Identified { export async function PostFormDataToServer(relativeRoute: string, formData: FormData) { const parameters = { method: 'POST', - headers: { userId: CurrentUserUtils.id }, - body: formData, + body: formData }; const response = await fetch(relativeRoute, parameters); - const text = await response.json(); - return text; + return response.json(); } } \ No newline at end of file diff --git a/src/client/apis/GoogleAuthenticationManager.tsx b/src/client/apis/GoogleAuthenticationManager.tsx index 01dac3996..1ec9d8412 100644 --- a/src/client/apis/GoogleAuthenticationManager.tsx +++ b/src/client/apis/GoogleAuthenticationManager.tsx @@ -3,7 +3,7 @@ import { observer } from "mobx-react"; import * as React from "react"; import MainViewModal from "../views/MainViewModal"; import { Opt } from "../../new_fields/Doc"; -import { Identified } from "../Network"; +import { Networking } from "../Network"; import { RouteStore } from "../../server/RouteStore"; import "./GoogleAuthenticationManager.scss"; @@ -31,7 +31,7 @@ export default class GoogleAuthenticationManager extends React.Component<{}> { } public fetchOrGenerateAccessToken = async () => { - let response = await Identified.FetchFromServer(RouteStore.readGoogleAccessToken); + let response = await Networking.FetchFromServer(RouteStore.readGoogleAccessToken); // if this is an authentication url, activate the UI to register the new access token if (new RegExp(AuthenticationUrl).test(response)) { this.isOpen = true; @@ -39,24 +39,25 @@ export default class GoogleAuthenticationManager extends React.Component<{}> { return new Promise(async resolve => { const disposer = reaction( () => this.authenticationCode, - authenticationCode => { - if (authenticationCode) { - Identified.PostToServer(RouteStore.writeGoogleAccessToken, { authenticationCode }).then( - ({ access_token, avatar, name }) => { - runInAction(() => { - this.avatar = avatar; - this.username = name; - }); - this.beginFadeout(); - disposer(); - resolve(access_token); - }, - action(() => { - this.hasBeenClicked = false; - this.success = false; - }) - ); + async authenticationCode => { + if (!authenticationCode) { + return; } + const { access_token, avatar, name } = await Networking.PostToServer( + RouteStore.writeGoogleAccessToken, + { authenticationCode } + ); + runInAction(() => { + this.avatar = avatar; + this.username = name; + }); + this.beginFadeout(); + disposer(); + resolve(access_token); + action(() => { + this.hasBeenClicked = false; + this.success = false; + }); } ); }); diff --git a/src/client/apis/google_docs/GoogleApiClientUtils.ts b/src/client/apis/google_docs/GoogleApiClientUtils.ts index 1cf01fc3d..183679317 100644 --- a/src/client/apis/google_docs/GoogleApiClientUtils.ts +++ b/src/client/apis/google_docs/GoogleApiClientUtils.ts @@ -3,7 +3,7 @@ import { RouteStore } from "../../../server/RouteStore"; import { Opt } from "../../../new_fields/Doc"; import { isArray } from "util"; import { EditorState } from "prosemirror-state"; -import { Identified } from "../../Network"; +import { Networking } from "../../Network"; export const Pulls = "googleDocsPullCount"; export const Pushes = "googleDocsPushCount"; @@ -84,7 +84,7 @@ export namespace GoogleApiClientUtils { } }; try { - const schema: docs_v1.Schema$Document = await Identified.PostToServer(path, parameters); + const schema: docs_v1.Schema$Document = await Networking.PostToServer(path, parameters); return schema.documentId; } catch { return undefined; @@ -157,7 +157,7 @@ export namespace GoogleApiClientUtils { const path = `${RouteStore.googleDocs}/Documents/${Actions.Retrieve}`; try { const parameters = { documentId: options.documentId }; - const schema: RetrievalResult = await Identified.PostToServer(path, parameters); + const schema: RetrievalResult = await Networking.PostToServer(path, parameters); return schema; } catch { return undefined; @@ -173,7 +173,7 @@ export namespace GoogleApiClientUtils { } }; try { - const replies: UpdateResult = await Identified.PostToServer(path, parameters); + const replies: UpdateResult = await Networking.PostToServer(path, parameters); return replies; } catch { return undefined; diff --git a/src/client/apis/google_docs/GooglePhotosClientUtils.ts b/src/client/apis/google_docs/GooglePhotosClientUtils.ts index e93fa6eb4..402fc64b5 100644 --- a/src/client/apis/google_docs/GooglePhotosClientUtils.ts +++ b/src/client/apis/google_docs/GooglePhotosClientUtils.ts @@ -13,7 +13,7 @@ import { Docs, DocumentOptions } from "../../documents/Documents"; import { NewMediaItemResult, MediaItem } from "../../../server/apis/google/SharedTypes"; import { AssertionError } from "assert"; import { DocumentView } from "../../views/nodes/DocumentView"; -import { Identified } from "../../Network"; +import { Networking } from "../../Network"; import GoogleAuthenticationManager from "../GoogleAuthenticationManager"; export namespace GooglePhotos { @@ -78,6 +78,7 @@ export namespace GooglePhotos { } export const CollectionToAlbum = async (options: AlbumCreationOptions): Promise> => { + await GoogleAuthenticationManager.Instance.fetchOrGenerateAccessToken(); const { collection, title, descriptionKey, tag } = options; const dataDocument = Doc.GetProto(collection); const images = ((await DocListCastAsync(dataDocument.data)) || []).filter(doc => Cast(doc.data, ImageField)); @@ -127,6 +128,7 @@ export namespace GooglePhotos { export type CollectionConstructor = (data: Array, options: DocumentOptions, ...args: any) => Doc; export const CollectionFromSearch = async (constructor: CollectionConstructor, requested: Opt>): Promise => { + await GoogleAuthenticationManager.Instance.fetchOrGenerateAccessToken(); let response = await Query.ContentSearch(requested); let uploads = await Transactions.WriteMediaItemsToServer(response); const children = uploads.map((upload: Transactions.UploadInformation) => { @@ -147,6 +149,7 @@ export namespace GooglePhotos { const comparator = (a: string, b: string) => (a < b) ? -1 : (a > b ? 1 : 0); export const TagChildImages = async (collection: Doc) => { + await GoogleAuthenticationManager.Instance.fetchOrGenerateAccessToken(); const idMapping = await Cast(collection.googlePhotosIdMapping, Doc); if (!idMapping) { throw new Error("Appending image metadata requires that the targeted collection have already been mapped to an album!"); @@ -304,7 +307,7 @@ export namespace GooglePhotos { }; export const WriteMediaItemsToServer = async (body: { mediaItems: any[] }): Promise => { - const uploads = await Identified.PostToServer(RouteStore.googlePhotosMediaDownload, body); + const uploads = await Networking.PostToServer(RouteStore.googlePhotosMediaDownload, body); return uploads; }; @@ -325,6 +328,7 @@ export namespace GooglePhotos { } export const UploadImages = async (sources: Doc[], album?: AlbumReference, descriptionKey = "caption"): Promise> => { + await GoogleAuthenticationManager.Instance.fetchOrGenerateAccessToken(); if (album && "title" in album) { album = await Create.Album(album.title); } @@ -341,7 +345,7 @@ export namespace GooglePhotos { media.push({ url, description }); } if (media.length) { - const results = await Identified.PostToServer(RouteStore.googlePhotosMediaUpload, { media, album }); + const results = await Networking.PostToServer(RouteStore.googlePhotosMediaUpload, { media, album }); return results; } }; diff --git a/src/client/util/Import & Export/DirectoryImportBox.tsx b/src/client/util/Import & Export/DirectoryImportBox.tsx index d74b51993..2d1b6fe20 100644 --- a/src/client/util/Import & Export/DirectoryImportBox.tsx +++ b/src/client/util/Import & Export/DirectoryImportBox.tsx @@ -20,9 +20,8 @@ import { listSpec } from "../../../new_fields/Schema"; import { GooglePhotos } from "../../apis/google_docs/GooglePhotosClientUtils"; import { SchemaHeaderField } from "../../../new_fields/SchemaHeaderField"; import "./DirectoryImportBox.scss"; -import { Identified } from "../../Network"; +import { Networking } from "../../Network"; import { BatchedArray } from "array-batcher"; -import { ExifData } from "exif"; const unsupported = ["text/html", "text/plain"]; @@ -117,7 +116,7 @@ export default class DirectoryImportBox extends React.Component formData.append(Utils.GenerateGuid(), file); }); - const responses = await Identified.PostFormDataToServer(RouteStore.upload, formData); + const responses = await Networking.PostFormDataToServer(RouteStore.upload, formData); runInAction(() => this.completed += batch.length); return responses as ImageUploadResponse[]; }); diff --git a/src/client/util/Import & Export/ImageUtils.ts b/src/client/util/Import & Export/ImageUtils.ts index c9abf38fa..914f4870a 100644 --- a/src/client/util/Import & Export/ImageUtils.ts +++ b/src/client/util/Import & Export/ImageUtils.ts @@ -3,7 +3,7 @@ import { ImageField } from "../../../new_fields/URLField"; import { Cast, StrCast } from "../../../new_fields/Types"; import { RouteStore } from "../../../server/RouteStore"; import { Docs } from "../../documents/Documents"; -import { Identified } from "../../Network"; +import { Networking } from "../../Network"; import { Id } from "../../../new_fields/FieldSymbols"; import { Utils } from "../../../Utils"; @@ -15,7 +15,7 @@ export namespace ImageUtils { return false; } const source = field.url.href; - const response = await Identified.PostToServer(RouteStore.inspectImage, { source }); + const response = await Networking.PostToServer(RouteStore.inspectImage, { source }); const { error, data } = response.exifData; document.exif = error || Docs.Get.DocumentHierarchyFromJson(data); return data !== undefined; diff --git a/src/new_fields/RichTextUtils.ts b/src/new_fields/RichTextUtils.ts index 601939ed2..63d718ce8 100644 --- a/src/new_fields/RichTextUtils.ts +++ b/src/new_fields/RichTextUtils.ts @@ -17,7 +17,7 @@ import { Cast, StrCast } from "./Types"; import { Id } from "./FieldSymbols"; import { DocumentView } from "../client/views/nodes/DocumentView"; import { AssertionError } from "assert"; -import { Identified } from "../client/Network"; +import { Networking } from "../client/Network"; export namespace RichTextUtils { @@ -129,7 +129,7 @@ export namespace RichTextUtils { return { baseUrl, filename }; }); - const uploads = await Identified.PostToServer(RouteStore.googlePhotosMediaDownload, { mediaItems }); + const uploads = await Networking.PostToServer(RouteStore.googlePhotosMediaDownload, { mediaItems }); if (uploads.length !== mediaItems.length) { throw new AssertionError({ expected: mediaItems.length, actual: uploads.length, message: "Error with internally uploading inlineObjects!" }); diff --git a/src/server/RouteManager.ts b/src/server/RouteManager.ts index 21ce9c9e4..eda2a49d2 100644 --- a/src/server/RouteManager.ts +++ b/src/server/RouteManager.ts @@ -49,9 +49,9 @@ export default class RouteManager { let supervised = async (req: express.Request, res: express.Response) => { const { user, originalUrl: target } = req; const core = { req, res, isRelease }; - const tryExecute = async (target: (args: any) => any | Promise, args: any) => { + const tryExecute = async (toExecute: (args: any) => any | Promise, args: any) => { try { - await target(args); + await toExecute(args); } catch (e) { if (onError) { onError({ ...core, error: e }); diff --git a/src/server/RouteStore.ts b/src/server/RouteStore.ts index de2553b2f..a310d0c95 100644 --- a/src/server/RouteStore.ts +++ b/src/server/RouteStore.ts @@ -39,6 +39,7 @@ export enum RouteStore { writeGoogleAccessToken = "/writeGoogleAccessToken", googlePhotosMediaUpload = "/googlePhotosMediaUpload", googlePhotosMediaDownload = "/googlePhotosMediaDownload", - googleDocsGet = "/googleDocsGet" + googleDocsGet = "/googleDocsGet", + checkGoogle = "/checkGoogleAuthentication" } \ No newline at end of file diff --git a/src/server/apis/google/GoogleApiServerUtils.ts b/src/server/apis/google/GoogleApiServerUtils.ts index b9984649e..ff5dc7081 100644 --- a/src/server/apis/google/GoogleApiServerUtils.ts +++ b/src/server/apis/google/GoogleApiServerUtils.ts @@ -7,7 +7,7 @@ import { GaxiosResponse } from "gaxios"; import request = require('request-promise'); import * as qs from 'query-string'; import { Database } from "../../database"; -import path from "path"; +import * as path from "path"; /** * @@ -45,7 +45,7 @@ export namespace GoogleApiServerUtils { * */ export interface CredentialsResult { - credentials: Credentials; + credentials: Opt; refreshed: boolean; } @@ -135,8 +135,8 @@ export namespace GoogleApiServerUtils { /** * */ - export const loadClientSecret = async () => { - return new Promise((resolve, reject) => { + export async function loadClientSecret(): Promise { + return new Promise((resolve, reject) => { readFile(path.join(__dirname, "../../credentials/google_docs_credentials.json"), async (err, projectCredentials) => { if (err) { reject(err); @@ -153,7 +153,7 @@ export namespace GoogleApiServerUtils { resolve(); }); }); - }; + } /** * @@ -165,9 +165,12 @@ export namespace GoogleApiServerUtils { * @param sector * @param userId */ - export const GetEndpoint = (sector: string, userId: string) => { - return new Promise>(resolve => { + export async function GetEndpoint(sector: string, userId: string): Promise> { + return new Promise(resolve => { retrieveOAuthClient(userId).then(auth => { + if (!auth) { + return resolve(); + } let routed: Opt; let parameters: EndpointParameters = { auth, version: "v1" }; switch (sector) { @@ -181,29 +184,38 @@ export namespace GoogleApiServerUtils { resolve(routed); }); }); - }; + } /** * * @param userId */ - export const retrieveAccessToken = (userId: string): Promise => { - return new Promise((resolve, reject) => { + export async function retrieveAccessToken(userId: string): Promise { + return new Promise(resolve => { retrieveCredentials(userId).then( - ({ credentials }) => resolve(credentials.access_token!), - error => reject(`Error: unable to authenticate Google Photos API request.\n${error}`) + ({ credentials }) => { + if (credentials) { + return resolve(credentials.access_token!); + } + resolve(); + } ); }); - }; + } /** - * - * @param userId + * Returns an initialized OAuth2 client instance, likely to be passed into Google's + * npm-installed API wrappers that use authenticated client instances rather than access codes for + * security. + * @param userId the Dash user id of the user requesting account integration */ - export const retrieveOAuthClient = (userId: string): Promise => { - return new Promise((resolve, reject) => { + export async function retrieveOAuthClient(userId: string): Promise { + return new Promise((resolve, reject) => { retrieveCredentials(userId).then( ({ credentials, refreshed }) => { + if (!credentials) { + return resolve(); + } let client = authenticationClients.get(userId); if (!client) { authenticationClients.set(userId, client = generateClient(credentials)); @@ -211,31 +223,34 @@ export namespace GoogleApiServerUtils { client.setCredentials(credentials); } resolve(client); - }, - error => reject(`Error: unable to instantiate and certify a new OAuth2 client.\n${error}`) + } ); }); - }; + } /** - * - * @param credentials + * Creates a new OAuth2Client instance, and if provided, sets + * the specific credentials on the client + * @param credentials if you have access to the credentials that you'll eventually set on + * the client, just pass them in at initialization */ - function generateClient(credentials?: Credentials) { + function generateClient(credentials?: Credentials): OAuth2Client { const client = new google.auth.OAuth2(installed); credentials && client.setCredentials(credentials); return client; } /** - * + * Calls on the worker (which does not have and does not need + * any credentials) to produce a url to which the user can + * navigate to give Dash the necessary Google permissions. */ - export const generateAuthenticationUrl = async () => { + export function generateAuthenticationUrl(): string { return worker.generateAuthUrl({ access_type: 'offline', scope: SCOPES.map(relative => prefix + relative), }); - }; + } /** * This is what we return to the server in processNewUser(), after the @@ -255,7 +270,7 @@ export namespace GoogleApiServerUtils { * and the authentication code to fetch the full set of credentials that * we'll store in the database for each user. This is called once per * new account integration. - * @param userId The Dash user id of the user requesting account integration, used to associate the new credentials + * @param userId the Dash user id of the user requesting account integration, used to associate the new credentials * with a Dash user in the googleAuthentication table of the database. * @param authenticationCode the Google-provided authentication code that the user copied * from Google's permissions UI and pasted into the overlay. @@ -263,24 +278,25 @@ export namespace GoogleApiServerUtils { * and display basic user information in the overlay on successful authentication. * This can be expanded as needed by adding properties to the interface GoogleAuthenticationResult. */ - export const processNewUser = async (userId: string, authenticationCode: string): Promise => { - return new Promise((resolve, reject) => { + export async function processNewUser(userId: string, authenticationCode: string): Promise { + const credentials = await new Promise((resolve, reject) => { worker.getToken(authenticationCode, async (err, credentials) => { if (err || !credentials) { reject(err); - return console.error('Error retrieving access token', err); + return; } - const enriched = injectUserInfo(credentials); - await Database.Auxiliary.GoogleAuthenticationToken.Write(userId, enriched); - const { given_name, picture } = enriched.userInfo; - resolve({ - access_token: enriched.access_token!, - avatar: picture, - name: given_name - }); + resolve(credentials); }); }); - }; + const enriched = injectUserInfo(credentials); + await Database.Auxiliary.GoogleAuthenticationToken.Write(userId, enriched); + const { given_name, picture } = enriched.userInfo; + return { + access_token: enriched.access_token!, + avatar: picture, + name: given_name + }; + } /** * This type represents the union of the full set of OAuth2 credentials @@ -299,34 +315,31 @@ export namespace GoogleApiServerUtils { * @returns the full set of credentials in the structure in which they'll be stored * in the database. */ - const injectUserInfo = (credentials: Credentials): EnrichedCredentials => { + function injectUserInfo(credentials: Credentials): EnrichedCredentials { const userInfo = JSON.parse(atob(credentials.id_token!.split(".")[1])); return { ...credentials, userInfo }; - }; + } /** * Looks in the database for any credentials object with the given user id, * and returns them. If the credentials are found but expired, the function will * automatically refresh the credentials and then resolve with the updated values. - * @param userId the id of the Dash user requesting his/her credentials. Eventually - * might have multiple. - * @returns the credentials and whether or not they were updated in the process + * @param userId the id of the Dash user requesting his/her credentials. Eventually, each user might + * be associated with multiple different sets of Google credentials. + * @returns the credentials and a flag indicating whether or not they were refreshed during retrieval */ - const retrieveCredentials = async (userId: string): Promise => { - return new Promise((resolve, reject) => { - Database.Auxiliary.GoogleAuthenticationToken.Fetch(userId).then(credentials => { - if (!credentials) { - return reject(); - } - if (credentials.expiry_date! < new Date().getTime()) { - // Token has expired, so submitting a request for a refreshed access token - return refreshAccessToken(credentials, userId).then(resolve, reject); - } - // Authentication successful! - resolve({ credentials, refreshed: false }); - }); - }); - }; + async function retrieveCredentials(userId: string): Promise { + let credentials: Opt = await Database.Auxiliary.GoogleAuthenticationToken.Fetch(userId); + let refreshed = false; + if (!credentials) { + return { credentials: undefined, refreshed }; + } + // if the token has expired, submit a request for a refreshed access token + if (credentials.expiry_date! <= new Date().getTime()) { + credentials = await refreshAccessToken(credentials, userId); + } + return { credentials, refreshed }; + } /** * This function submits a request to OAuth with the local refresh token @@ -334,26 +347,28 @@ export namespace GoogleApiServerUtils { * the Dash user id passed in. In addition to returning the credentials, it * writes the diff to the database. * @param credentials the credentials - * @param userId + * @param userId the id of the Dash user implicitly requesting that + * his/her credentials be refreshed + * @returns the updated credentials */ - const refreshAccessToken = (credentials: Credentials, userId: string) => { - return new Promise(resolve => { - let headerParameters = { headers: { 'Content-Type': 'application/x-www-form-urlencoded' } }; - let queryParameters = { - refreshToken: credentials.refresh_token, - grant_type: "refresh_token", - ...installed - }; - let url = `${refreshEndpoint}?${qs.stringify(queryParameters)}`; - request.post(url, headerParameters).then(async response => { - let { access_token, expires_in } = JSON.parse(response); - const expiry_date = new Date().getTime() + (expires_in * 1000); - await Database.Auxiliary.GoogleAuthenticationToken.Update(userId, access_token, expiry_date); - credentials.access_token = access_token; - credentials.expiry_date = expiry_date; - resolve({ credentials, refreshed: true }); - }); + async function refreshAccessToken(credentials: Credentials, userId: string): Promise { + let headerParameters = { headers: { 'Content-Type': 'application/x-www-form-urlencoded' } }; + let url = `${refreshEndpoint}?${qs.stringify({ + refreshToken: credentials.refresh_token, + grant_type: "refresh_token", + ...installed + })}`; + const { access_token, expires_in } = await new Promise(async resolve => { + const response = await request.post(url, headerParameters); + resolve(JSON.parse(response)); }); - }; + // expires_in is in seconds, but we're building the new expiry date in milliseconds + const expiry_date = new Date().getTime() + (expires_in * 1000); + await Database.Auxiliary.GoogleAuthenticationToken.Update(userId, access_token, expiry_date); + // update the relevant properties + credentials.access_token = access_token; + credentials.expiry_date = expiry_date; + return credentials; + } } \ No newline at end of file diff --git a/src/server/index.ts b/src/server/index.ts index eb19c71a9..860cde3b5 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -593,11 +593,11 @@ function routeSetter(router: RouteManager) { subscription: RouteStore.readGoogleAccessToken, onValidation: async ({ user, res }) => { const userId = user.id; - const token = await Database.Auxiliary.GoogleAuthenticationToken.Fetch(userId); + const token = await GoogleApiServerUtils.retrieveAccessToken(userId); if (!token) { - return res.send(await GoogleApiServerUtils.generateAuthenticationUrl()); + return res.send(GoogleApiServerUtils.generateAuthenticationUrl()); } - return GoogleApiServerUtils.retrieveAccessToken(userId).then(token => res.send(token)); + return res.send(token); } }); @@ -609,7 +609,7 @@ function routeSetter(router: RouteManager) { } }); - const tokenError = "Unable to successfully upload bytes for all images!"; + const authenticationError = "Unable to authenticate Google credentials before uploading to Google Photos!"; const mediaError = "Unable to convert all uploaded bytes to media items!"; router.addSupervisedRoute({ @@ -618,8 +618,12 @@ function routeSetter(router: RouteManager) { onValidation: async ({ user, req, res }) => { const { media } = req.body; - let failed: number[] = []; const token = await GoogleApiServerUtils.retrieveAccessToken(user.id); + if (!token) { + return _error(res, authenticationError); + } + + let failed: number[] = []; const newMediaItems = await BatchedArray.from(media, { batchSize: 25 }).batchedMapPatientInterval( { magnitude: 100, unit: TimeUnit.Milliseconds }, async (batch: GooglePhotosUploadUtils.MediaInput[]) => { -- cgit v1.2.3-70-g09d2 From d4d8c2835c8e1e943f77a14e2b87df05f5848dbd Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Tue, 29 Oct 2019 23:26:20 -0400 Subject: finished cleaning and commenting GoogleApiServerUtils --- src/server/apis/google/GoogleApiServerUtils.ts | 233 ++++++++++++------------- 1 file changed, 114 insertions(+), 119 deletions(-) (limited to 'src') diff --git a/src/server/apis/google/GoogleApiServerUtils.ts b/src/server/apis/google/GoogleApiServerUtils.ts index ff5dc7081..ec7c2cfe1 100644 --- a/src/server/apis/google/GoogleApiServerUtils.ts +++ b/src/server/apis/google/GoogleApiServerUtils.ts @@ -2,7 +2,6 @@ import { google } from "googleapis"; import { readFile } from "fs"; import { OAuth2Client, Credentials, OAuth2ClientOptions } from "google-auth-library"; import { Opt } from "../../../new_fields/Doc"; -import { GlobalOptions } from "googleapis-common"; import { GaxiosResponse } from "gaxios"; import request = require('request-promise'); import * as qs from 'query-string'; @@ -10,19 +9,12 @@ import { Database } from "../../database"; import * as path from "path"; /** - * + * Scopes give Google users fine granularity of control + * over the information they make accessible via the API. + * This is the somewhat overkill list of what Dash requests + * from the user. */ -const prefix = 'https://www.googleapis.com/auth/'; - -/** - * - */ -const refreshEndpoint = "https://oauth2.googleapis.com/token"; - -/** - * - */ -const SCOPES = [ +const scope = [ 'documents.readonly', 'documents', 'presentations', @@ -33,7 +25,7 @@ const SCOPES = [ 'photoslibrary.appendonly', 'photoslibrary.sharing', 'userinfo.profile' -]; +].map(relative => `https://www.googleapis.com/auth/${relative}`); /** * This namespace manages server side authentication for Google API queries, either @@ -42,33 +34,9 @@ const SCOPES = [ export namespace GoogleApiServerUtils { /** - * - */ - export interface CredentialsResult { - credentials: Opt; - refreshed: boolean; - } - - /** - * - */ - export interface UserInfo { - at_hash: string; - aud: string; - azp: string; - exp: number; - family_name: string; - given_name: string; - iat: number; - iss: string; - locale: string; - name: string; - picture: string; - sub: string; - } - - /** - * + * As we expand out to more Google APIs that are accessible from + * the 'googleapis' module imported above, this enum will record + * the list and provide a unified string representation of each API. */ export enum Service { Documents = "Documents", @@ -76,15 +44,10 @@ export namespace GoogleApiServerUtils { } /** - * - */ - export interface CredentialInformation { - credentialsPath: string; - userId: string; - } - - /** - * + * Global credentials read once from a JSON file + * before the server is started that + * allow us to build OAuth2 clients with Dash's + * application specific credentials. */ let installed: OAuth2ClientOptions; @@ -99,27 +62,33 @@ export namespace GoogleApiServerUtils { let worker: OAuth2Client; /** - * + * A briefer format for the response from a 'googleapis' API request */ export type ApiResponse = Promise; /** - * + * A generic form for a handler that executes some request on the endpoint */ export type ApiRouter = (endpoint: Endpoint, parameters: any) => ApiResponse; /** - * + * A generic form for the asynchronous function that actually submits the + * request to the API and returns the corresporing response. Helpful when + * making an extensible endpoint definition. */ export type ApiHandler = (parameters: any, methodOptions?: any) => ApiResponse; /** - * + * A literal union type indicating the valid actions for these 'googleapis' + * requestions */ export type Action = "create" | "retrieve" | "update"; /** - * + * An interface defining any entity on which one can invoke + * anuy of the following handlers. All 'googleapis' wrappers + * such as google.docs().documents and google.slides().presentations + * satisfy this interface. */ export interface Endpoint { get: ApiHandler; @@ -128,12 +97,10 @@ export namespace GoogleApiServerUtils { } /** - * - */ - export type EndpointParameters = GlobalOptions & { version: "v1" }; - - /** - * + * This function is called once before the server is started, + * reading in Dash's project-specific credentials (client secret + * and client id) for later repeated access. It also sets up the + * global, intentionally unauthenticated worker OAuth2 client instance. */ export async function loadClientSecret(): Promise { return new Promise((resolve, reject) => { @@ -156,75 +123,83 @@ export namespace GoogleApiServerUtils { } /** - * + * Maps the Dash user id of a given user to their single + * associated OAuth2 client, mitigating the creation + * of needless duplicate clients that would arise from + * making one new client instance per request. */ const authenticationClients = new Map(); /** - * - * @param sector - * @param userId + * This function receives the target sector ("which G-Suite app's API am I interested in?") + * and the id of the Dash user making the request to the API. With this information, it generates + * an authenticated OAuth2 client and passes it into the relevant 'googleapis' wrapper. + * @param sector the particular desired G-Suite 'googleapis' API (docs, slides, etc.) + * @param userId the id of the Dash user making the request to the API + * @returns the relevant 'googleapis' wrapper, if any */ export async function GetEndpoint(sector: string, userId: string): Promise> { - return new Promise(resolve => { - retrieveOAuthClient(userId).then(auth => { - if (!auth) { - return resolve(); - } - let routed: Opt; - let parameters: EndpointParameters = { auth, version: "v1" }; - switch (sector) { - case Service.Documents: - routed = google.docs(parameters).documents; - break; - case Service.Slides: - routed = google.slides(parameters).presentations; - break; - } - resolve(routed); - }); + return new Promise(async resolve => { + const auth = await retrieveOAuthClient(userId); + if (!auth) { + return resolve(); + } + let routed: Opt; + let parameters: any = { auth, version: "v1" }; + switch (sector) { + case Service.Documents: + routed = google.docs(parameters).documents; + break; + case Service.Slides: + routed = google.slides(parameters).presentations; + break; + } + resolve(routed); }); } /** - * - * @param userId + * Returns the lengthy string or access token that can be passed into + * the headers of an API request or into the constructor of the Photos + * client API wrapper. + * @param userId the Dash user id of the user requesting his/her associated + * access_token + * @returns the current access_token associated with the requesting + * Dash user. The access_token is valid for only an hour, and + * is then refreshed. */ export async function retrieveAccessToken(userId: string): Promise { - return new Promise(resolve => { - retrieveCredentials(userId).then( - ({ credentials }) => { - if (credentials) { - return resolve(credentials.access_token!); - } - resolve(); - } - ); + return new Promise(async resolve => { + const { credentials } = await retrieveCredentials(userId); + if (!credentials) { + return resolve(); + } + resolve(credentials.access_token!); }); } /** - * Returns an initialized OAuth2 client instance, likely to be passed into Google's + * Manipulates a mapping such that, in the limit, each Dash user has + * an associated authenticated OAuth2 client at their disposal. This + * function ensures that the client's credentials always remain up to date + * @param userId the Dash user id of the user requesting account integration + * @returns returns an initialized OAuth2 client instance, likely to be passed into Google's * npm-installed API wrappers that use authenticated client instances rather than access codes for * security. - * @param userId the Dash user id of the user requesting account integration */ export async function retrieveOAuthClient(userId: string): Promise { - return new Promise((resolve, reject) => { - retrieveCredentials(userId).then( - ({ credentials, refreshed }) => { - if (!credentials) { - return resolve(); - } - let client = authenticationClients.get(userId); - if (!client) { - authenticationClients.set(userId, client = generateClient(credentials)); - } else if (refreshed) { - client.setCredentials(credentials); - } - resolve(client); - } - ); + return new Promise(async resolve => { + const { credentials, refreshed } = await retrieveCredentials(userId); + if (!credentials) { + return resolve(); + } + let client = authenticationClients.get(userId); + if (!client) { + authenticationClients.set(userId, client = generateClient(credentials)); + } else if (refreshed) { + client.setCredentials(credentials); + } + resolve(client); }); } @@ -233,6 +208,7 @@ export namespace GoogleApiServerUtils { * the specific credentials on the client * @param credentials if you have access to the credentials that you'll eventually set on * the client, just pass them in at initialization + * @returns the newly created, potentially certified, OAuth2 client instance */ function generateClient(credentials?: Credentials): OAuth2Client { const client = new google.auth.OAuth2(installed); @@ -244,12 +220,10 @@ export namespace GoogleApiServerUtils { * Calls on the worker (which does not have and does not need * any credentials) to produce a url to which the user can * navigate to give Dash the necessary Google permissions. + * @returns the newly generated url to the authentication landing page */ export function generateAuthenticationUrl(): string { - return worker.generateAuthUrl({ - access_type: 'offline', - scope: SCOPES.map(relative => prefix + relative), - }); + return worker.generateAuthUrl({ scope, access_type: 'offline' }); } /** @@ -305,6 +279,26 @@ export namespace GoogleApiServerUtils { */ export type EnrichedCredentials = Credentials & { userInfo: UserInfo }; + /** + * This interface defines all of the information we + * receive from parsing the base64 encoded info-token + * for a Google user. + */ + export interface UserInfo { + at_hash: string; + aud: string; + azp: string; + exp: number; + family_name: string; + given_name: string; + iat: number; + iss: string; + locale: string; + name: string; + picture: string; + sub: string; + } + /** * It's pretty cool: the credentials id_token is split into thirds by periods. * The middle third contains a base64-encoded JSON string with all the @@ -316,7 +310,7 @@ export namespace GoogleApiServerUtils { * in the database. */ function injectUserInfo(credentials: Credentials): EnrichedCredentials { - const userInfo = JSON.parse(atob(credentials.id_token!.split(".")[1])); + const userInfo: UserInfo = JSON.parse(atob(credentials.id_token!.split(".")[1])); return { ...credentials, userInfo }; } @@ -326,15 +320,16 @@ export namespace GoogleApiServerUtils { * automatically refresh the credentials and then resolve with the updated values. * @param userId the id of the Dash user requesting his/her credentials. Eventually, each user might * be associated with multiple different sets of Google credentials. - * @returns the credentials and a flag indicating whether or not they were refreshed during retrieval + * @returns the credentials, or undefined if the user has no stored associated credentials, + * and a flag indicating whether or not they were refreshed during retrieval */ - async function retrieveCredentials(userId: string): Promise { + async function retrieveCredentials(userId: string): Promise<{ credentials: Opt, refreshed: boolean }> { let credentials: Opt = await Database.Auxiliary.GoogleAuthenticationToken.Fetch(userId); let refreshed = false; if (!credentials) { return { credentials: undefined, refreshed }; } - // if the token has expired, submit a request for a refreshed access token + // check for token expiry if (credentials.expiry_date! <= new Date().getTime()) { credentials = await refreshAccessToken(credentials, userId); } @@ -353,7 +348,7 @@ export namespace GoogleApiServerUtils { */ async function refreshAccessToken(credentials: Credentials, userId: string): Promise { let headerParameters = { headers: { 'Content-Type': 'application/x-www-form-urlencoded' } }; - let url = `${refreshEndpoint}?${qs.stringify({ + let url = `https://oauth2.googleapis.com/token?${qs.stringify({ refreshToken: credentials.refresh_token, grant_type: "refresh_token", ...installed -- cgit v1.2.3-70-g09d2 From 109be54065038392b19d9dbafbccc9205f198766 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Tue, 29 Oct 2019 23:37:42 -0400 Subject: db error handling and example code --- src/server/apis/google/GoogleApiServerUtils.ts | 3 +++ src/server/database.ts | 4 ++++ 2 files changed, 7 insertions(+) (limited to 'src') diff --git a/src/server/apis/google/GoogleApiServerUtils.ts b/src/server/apis/google/GoogleApiServerUtils.ts index ec7c2cfe1..35a2541a9 100644 --- a/src/server/apis/google/GoogleApiServerUtils.ts +++ b/src/server/apis/google/GoogleApiServerUtils.ts @@ -248,6 +248,9 @@ export namespace GoogleApiServerUtils { * with a Dash user in the googleAuthentication table of the database. * @param authenticationCode the Google-provided authentication code that the user copied * from Google's permissions UI and pasted into the overlay. + * + * EXAMPLE CODE: 4/sgF2A5uGg4xASHf7VQDnLtdqo3mUlfQqLSce_HYz5qf1nFtHj9YTeGs + * * @returns the information necessary to authenticate a client side google photos request * and display basic user information in the overlay on successful authentication. * This can be expanded as needed by adding properties to the interface GoogleAuthenticationResult. diff --git a/src/server/database.ts b/src/server/database.ts index 79dd26b7d..b81fc03a4 100644 --- a/src/server/database.ts +++ b/src/server/database.ts @@ -60,6 +60,10 @@ export namespace Database { constructor() { this.MongoClient.connect(url, (_err, client) => { + if (!client) { + console.error("\nPlease start MongoDB by running 'mongod' in a terminal before continuing...\n"); + process.exit(0); + } this.db = client.db(); this.onConnect.forEach(fn => fn()); }); -- cgit v1.2.3-70-g09d2 From 9c7e619fb9d3116649ec3779bd528b947235d5a4 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Wed, 30 Oct 2019 15:51:29 -0400 Subject: updated array batcher --- package.json | 2 +- .../util/Import & Export/DirectoryImportBox.tsx | 5 +-- src/server/RouteManager.ts | 1 + src/server/apis/google/GooglePhotosUploadUtils.ts | 11 ++--- src/server/index.ts | 48 ++++++++++++---------- 5 files changed, 36 insertions(+), 31 deletions(-) (limited to 'src') diff --git a/package.json b/package.json index 8cbbb84af..4572a3f73 100644 --- a/package.json +++ b/package.json @@ -115,7 +115,7 @@ "@types/youtube": "0.0.38", "adm-zip": "^0.4.13", "archiver": "^3.0.3", - "array-batcher": "^1.1.3", + "array-batcher": "^1.2.3", "async": "^2.6.2", "babel-runtime": "^6.26.0", "bcrypt-nodejs": "0.0.3", diff --git a/src/client/util/Import & Export/DirectoryImportBox.tsx b/src/client/util/Import & Export/DirectoryImportBox.tsx index 2d1b6fe20..bdd59cb16 100644 --- a/src/client/util/Import & Export/DirectoryImportBox.tsx +++ b/src/client/util/Import & Export/DirectoryImportBox.tsx @@ -107,7 +107,7 @@ export default class DirectoryImportBox extends React.Component runInAction(() => this.phase = `Internal: uploading ${this.quota - this.completed} files to Dash...`); - const uploads = await BatchedArray.from(validated, { batchSize: 15 }).batchedMapAsync(async batch => { + const uploads = await BatchedArray.from(validated, { batchSize: 15 }).batchedMapAsync(async (batch, collector) => { const formData = new FormData(); batch.forEach(file => { @@ -116,9 +116,8 @@ export default class DirectoryImportBox extends React.Component formData.append(Utils.GenerateGuid(), file); }); - const responses = await Networking.PostFormDataToServer(RouteStore.upload, formData); + collector.push(...(await Networking.PostFormDataToServer(RouteStore.upload, formData))); runInAction(() => this.completed += batch.length); - return responses as ImageUploadResponse[]; }); await Promise.all(uploads.map(async upload => { diff --git a/src/server/RouteManager.ts b/src/server/RouteManager.ts index eda2a49d2..c1d38327f 100644 --- a/src/server/RouteManager.ts +++ b/src/server/RouteManager.ts @@ -114,6 +114,7 @@ export const STATUS = { }; export function _error(res: express.Response, message: string, error?: any) { + console.error(message); res.statusMessage = message; res.status(STATUS.EXECUTION_ERROR).send(error); } diff --git a/src/server/apis/google/GooglePhotosUploadUtils.ts b/src/server/apis/google/GooglePhotosUploadUtils.ts index 172fa8d46..d3442338b 100644 --- a/src/server/apis/google/GooglePhotosUploadUtils.ts +++ b/src/server/apis/google/GooglePhotosUploadUtils.ts @@ -1,7 +1,7 @@ import request = require('request-promise'); import { GoogleApiServerUtils } from './GoogleApiServerUtils'; import * as path from 'path'; -import { MediaItemCreationResult } from './SharedTypes'; +import { MediaItemCreationResult, NewMediaItemResult } from './SharedTypes'; import { NewMediaItem } from "../../index"; import { BatchedArray, TimeUnit } from 'array-batcher'; import { DashUploadUtils } from '../../DashUploadUtils'; @@ -50,9 +50,9 @@ export namespace GooglePhotosUploadUtils { }; export const CreateMediaItems = async (bearerToken: string, newMediaItems: NewMediaItem[], album?: { id: string }): Promise => { - const newMediaItemResults = await BatchedArray.from(newMediaItems, { batchSize: 50 }).batchedMapPatientInterval( + const newMediaItemResults = await BatchedArray.from(newMediaItems, { batchSize: 50 }).batchedMapPatientInterval( { magnitude: 100, unit: TimeUnit.Milliseconds }, - async (batch: NewMediaItem[]) => { + async (batch: NewMediaItem[], collector) => { const parameters = { method: 'POST', headers: headers('json', bearerToken), @@ -61,7 +61,7 @@ export namespace GooglePhotosUploadUtils { json: true }; album && (parameters.body.albumId = album.id); - return (await new Promise((resolve, reject) => { + const { newMediaItemResults } = await new Promise((resolve, reject) => { request(parameters, (error, _response, body) => { if (error) { reject(error); @@ -69,7 +69,8 @@ export namespace GooglePhotosUploadUtils { resolve(body); } }); - })).newMediaItemResults; + }); + collector.push(...newMediaItemResults); } ); return { newMediaItemResults }; diff --git a/src/server/index.ts b/src/server/index.ts index 860cde3b5..05c866eae 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -573,18 +573,15 @@ function routeSetter(router: RouteManager) { onValidation: async ({ req, res, user }) => { let sector: GoogleApiServerUtils.Service = req.params.sector as GoogleApiServerUtils.Service; let action: GoogleApiServerUtils.Action = req.params.action as GoogleApiServerUtils.Action; - return GoogleApiServerUtils.GetEndpoint(GoogleApiServerUtils.Service[sector], user.id).then(endpoint => { - let handler = EndpointHandlerMap.get(action); - if (endpoint && handler) { - let execute = handler(endpoint, req.body).then( - response => res.send(response.data), - rejection => res.send(rejection) - ); - execute.catch(exception => res.send(exception)); - return; - } - res.send(undefined); - }); + const endpoint = await GoogleApiServerUtils.GetEndpoint(GoogleApiServerUtils.Service[sector], user.id); + let handler = EndpointHandlerMap.get(action); + if (endpoint && handler) { + handler(endpoint, req.body) + .then(response => res.send(response.data)) + .catch(exception => res.send(exception)); + return; + } + res.send(undefined); } }); @@ -611,6 +608,12 @@ function routeSetter(router: RouteManager) { const authenticationError = "Unable to authenticate Google credentials before uploading to Google Photos!"; const mediaError = "Unable to convert all uploaded bytes to media items!"; + interface GooglePhotosUploadFailure { + batch: number; + index: number; + url: string; + reason: string; + } router.addSupervisedRoute({ method: Method.POST, @@ -623,30 +626,31 @@ function routeSetter(router: RouteManager) { return _error(res, authenticationError); } - let failed: number[] = []; - const newMediaItems = await BatchedArray.from(media, { batchSize: 25 }).batchedMapPatientInterval( + let failed: GooglePhotosUploadFailure[] = []; + const batched = BatchedArray.from(media, { batchSize: 25 }); + const newMediaItems = await batched.batchedMapPatientInterval( { magnitude: 100, unit: TimeUnit.Milliseconds }, - async (batch: GooglePhotosUploadUtils.MediaInput[]) => { - const newMediaItems: NewMediaItem[] = []; + async (batch, collector, { completedBatches }) => { for (let index = 0; index < batch.length; index++) { - const element = batch[index]; - const uploadToken = await GooglePhotosUploadUtils.DispatchGooglePhotosUpload(token, element.url); + const { url, description } = batch[index]; + const fail = (reason: string) => failed.push({ reason, batch: completedBatches + 1, index, url }); + const uploadToken = await GooglePhotosUploadUtils.DispatchGooglePhotosUpload(token, url).catch(fail); if (!uploadToken) { - failed.push(index); + fail(`${path.extname(url)} is not an accepted extension`); } else { - newMediaItems.push({ - description: element.description, + collector.push({ + description, simpleMediaItem: { uploadToken } }); } } - return newMediaItems; } ); const failedCount = failed.length; if (failedCount) { console.error(`Unable to upload ${failedCount} image${failedCount === 1 ? "" : "s"} to Google's servers`); + console.log(failed.map(({ reason, batch, index, url }) => `@${batch}.${index}: ${url} failed: ${reason}`).join('\n')); } return GooglePhotosUploadUtils.CreateMediaItems(token, newMediaItems, req.body.album).then( -- cgit v1.2.3-70-g09d2 From f48b2729b294d08da0c99a242f9ebb4d7aab4407 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Thu, 31 Oct 2019 01:58:42 -0400 Subject: commented and cleaned google photos upload utils --- src/server/DashUploadUtils.ts | 6 +- src/server/apis/google/GooglePhotosUploadUtils.ts | 100 +++++++++++++++++----- src/server/index.ts | 2 +- 3 files changed, 84 insertions(+), 24 deletions(-) (limited to 'src') diff --git a/src/server/DashUploadUtils.ts b/src/server/DashUploadUtils.ts index 46d897339..9fddb466c 100644 --- a/src/server/DashUploadUtils.ts +++ b/src/server/DashUploadUtils.ts @@ -24,9 +24,13 @@ export namespace DashUploadUtils { const gifs = [".gif"]; const pngs = [".png"]; const jpgs = [".jpg", ".jpeg"]; - export const imageFormats = [...pngs, ...jpgs, ...gifs]; + const imageFormats = [...pngs, ...jpgs, ...gifs]; const videoFormats = [".mov", ".mp4"]; + export function validateExtension(url: string) { + return imageFormats.includes(path.extname(url).toLowerCase()); + } + const size = "content-length"; const type = "content-type"; diff --git a/src/server/apis/google/GooglePhotosUploadUtils.ts b/src/server/apis/google/GooglePhotosUploadUtils.ts index d3442338b..a98399621 100644 --- a/src/server/apis/google/GooglePhotosUploadUtils.ts +++ b/src/server/apis/google/GooglePhotosUploadUtils.ts @@ -1,56 +1,111 @@ import request = require('request-promise'); -import { GoogleApiServerUtils } from './GoogleApiServerUtils'; import * as path from 'path'; import { MediaItemCreationResult, NewMediaItemResult } from './SharedTypes'; import { NewMediaItem } from "../../index"; import { BatchedArray, TimeUnit } from 'array-batcher'; import { DashUploadUtils } from '../../DashUploadUtils'; +/** + * This namespace encompasses the logic + * necessary to upload images to Google's server, + * and then initialize / create those images in the Photos + * API given the upload tokens returned from the initial + * uploading process. + * + * https://developers.google.com/photos/library/reference/rest/v1/mediaItems/batchCreate + */ export namespace GooglePhotosUploadUtils { - export interface Paths { - uploadDirectory: string; - credentialsPath: string; - tokenPath: string; - } - - export interface MediaInput { + /** + * Specifies the structure of the object + * necessary to upload bytes to Google's servers. + * The url is streamed to access the image's bytes, + * and the description is what appears in Google Photos' + * description field. + */ + export interface UploadSource { url: string; description: string; } - const prepend = (extension: string) => `https://photoslibrary.googleapis.com/v1/${extension}`; - const headers = (type: string, token: string) => ({ - 'Content-Type': `application/${type}`, - 'Authorization': `Bearer ${token}`, - }); + /** + * A utility function to streamline making + * calls to the API's url - accentuates + * the relative path in the caller. + * @param extension the desired + * subset of the API + */ + function prepend(extension: string): string { + return `https://photoslibrary.googleapis.com/v1/${extension}`; + } + + /** + * Factors out the creation of the API request's + * authentication elements stored in the header. + * @param type the contents of the request + * @param token the user-specific Google access token + */ + function headers(type: string, token: string) { + return { + 'Content-Type': `application/${type}`, + 'Authorization': `Bearer ${token}`, + }; + } - export const DispatchGooglePhotosUpload = async (bearerToken: string, url: string) => { - if (!DashUploadUtils.imageFormats.includes(path.extname(url))) { + /** + * This is the first step in the remote image creation process. + * Here we upload the raw bytes of the image to Google's servers by + * setting authentication and other required header properties and including + * the raw bytes to the image, to be uploaded, in the body of the request. + * @param bearerToken the user-specific Google access token, specifies the account associated + * with the eventual image creation + * @param url the url of the image to upload + * @param filename an optional name associated with the uploaded image - if not specified + * defaults to the filename (basename) in the url + */ + export const DispatchGooglePhotosUpload = async (bearerToken: string, url: string, filename?: string): Promise => { + // check if the url points to a non-image or an unsupported format + if (!DashUploadUtils.validateExtension(url)) { return undefined; } - const body = await request(url, { encoding: null }); const parameters = { method: 'POST', + uri: prepend('uploads'), headers: { ...headers('octet-stream', bearerToken), - 'X-Goog-Upload-File-Name': path.basename(url), + 'X-Goog-Upload-File-Name': filename || path.basename(url), 'X-Goog-Upload-Protocol': 'raw' }, - uri: prepend('uploads'), - body + body: await request(url, { encoding: null }) // returns a readable stream with the unencoded binary image data }; - return new Promise((resolve, reject) => request(parameters, (error, _response, body) => { + return new Promise((resolve, reject) => request(parameters, (error, _response, body) => { if (error) { - console.log(error); + // on rejection, the server logs the error and the offending image return reject(error); } resolve(body); })); }; + /** + * This is the second step in the remote image creation process: having uploaded + * the raw bytes of the image and received / stored pointers (upload tokens) to those + * bytes, we can now instruct the API to finalize the creation of those images by + * submitting a batch create request with the list of upload tokens and the description + * to be associated with reach resulting new image. + * @param bearerToken the user-specific Google access token, specifies the account associated + * with the eventual image creation + * @param newMediaItems a list of objects containing a description and, effectively, the + * pointer to the uploaded bytes + * @param album if included, will add all of the newly created remote images to the album + * with the specified id + */ export const CreateMediaItems = async (bearerToken: string, newMediaItems: NewMediaItem[], album?: { id: string }): Promise => { - const newMediaItemResults = await BatchedArray.from(newMediaItems, { batchSize: 50 }).batchedMapPatientInterval( + // it's important to note that the API can't handle more than 50 items in each request and + // seems to need at least some latency between requests (spamming it synchronously has led to the server returning errors)... + const batched = BatchedArray.from(newMediaItems, { batchSize: 50 }); + // ...so we execute them in delayed batches and await the entire execution + const newMediaItemResults = await batched.batchedMapPatientInterval( { magnitude: 100, unit: TimeUnit.Milliseconds }, async (batch: NewMediaItem[], collector) => { const parameters = { @@ -60,6 +115,7 @@ export namespace GooglePhotosUploadUtils { body: { newMediaItems: batch } as any, json: true }; + // register the target album, if provided album && (parameters.body.albumId = album.id); const { newMediaItemResults } = await new Promise((resolve, reject) => { request(parameters, (error, _response, body) => { diff --git a/src/server/index.ts b/src/server/index.ts index 05c866eae..9f3e34761 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -627,7 +627,7 @@ function routeSetter(router: RouteManager) { } let failed: GooglePhotosUploadFailure[] = []; - const batched = BatchedArray.from(media, { batchSize: 25 }); + const batched = BatchedArray.from(media, { batchSize: 25 }); const newMediaItems = await batched.batchedMapPatientInterval( { magnitude: 100, unit: TimeUnit.Milliseconds }, async (batch, collector, { completedBatches }) => { -- cgit v1.2.3-70-g09d2 From c53d599f8ecffe173d8df06777721658f065674a Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Thu, 31 Oct 2019 14:32:39 -0400 Subject: cleanup --- src/server/apis/google/GooglePhotosUploadUtils.ts | 12 +++++------- src/server/index.ts | 4 ++-- 2 files changed, 7 insertions(+), 9 deletions(-) (limited to 'src') diff --git a/src/server/apis/google/GooglePhotosUploadUtils.ts b/src/server/apis/google/GooglePhotosUploadUtils.ts index a98399621..d8cf795b5 100644 --- a/src/server/apis/google/GooglePhotosUploadUtils.ts +++ b/src/server/apis/google/GooglePhotosUploadUtils.ts @@ -100,12 +100,12 @@ export namespace GooglePhotosUploadUtils { * @param album if included, will add all of the newly created remote images to the album * with the specified id */ - export const CreateMediaItems = async (bearerToken: string, newMediaItems: NewMediaItem[], album?: { id: string }): Promise => { + export const CreateMediaItems = async (bearerToken: string, newMediaItems: NewMediaItem[], album?: { id: string }): Promise => { // it's important to note that the API can't handle more than 50 items in each request and // seems to need at least some latency between requests (spamming it synchronously has led to the server returning errors)... const batched = BatchedArray.from(newMediaItems, { batchSize: 50 }); // ...so we execute them in delayed batches and await the entire execution - const newMediaItemResults = await batched.batchedMapPatientInterval( + return batched.batchedMapPatientInterval( { magnitude: 100, unit: TimeUnit.Milliseconds }, async (batch: NewMediaItem[], collector) => { const parameters = { @@ -117,19 +117,17 @@ export namespace GooglePhotosUploadUtils { }; // register the target album, if provided album && (parameters.body.albumId = album.id); - const { newMediaItemResults } = await new Promise((resolve, reject) => { + collector.push(...(await new Promise((resolve, reject) => { request(parameters, (error, _response, body) => { if (error) { reject(error); } else { - resolve(body); + resolve(body.newMediaItemResults); } }); - }); - collector.push(...newMediaItemResults); + }))); } ); - return { newMediaItemResults }; }; } \ No newline at end of file diff --git a/src/server/index.ts b/src/server/index.ts index 9f3e34761..25697e71f 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -650,11 +650,11 @@ function routeSetter(router: RouteManager) { const failedCount = failed.length; if (failedCount) { console.error(`Unable to upload ${failedCount} image${failedCount === 1 ? "" : "s"} to Google's servers`); - console.log(failed.map(({ reason, batch, index, url }) => `@${batch}.${index}: ${url} failed: ${reason}`).join('\n')); + console.log(failed.map(({ reason, batch, index, url }) => `@${batch}.${index}: ${url} failed:\n${reason}`).join('\n\n')); } return GooglePhotosUploadUtils.CreateMediaItems(token, newMediaItems, req.body.album).then( - result => _success(res, { results: result.newMediaItemResults, failed }), + results => _success(res, { results, failed }), error => _error(res, mediaError, error) ); } -- cgit v1.2.3-70-g09d2 From 0b72a27ead9d1e933ae349b8a3e9e9b8702664d1 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Sat, 9 Nov 2019 16:18:23 -0500 Subject: factored out all but google resources into managers --- client_secret.json | 1 - .../util/Import & Export/DirectoryImportBox.tsx | 10 +- src/server/ApiManagers/DeleteManager.ts | 65 +++ src/server/ApiManagers/ExportManager.ts | 157 +++++- src/server/ApiManagers/PDFManager.ts | 107 ++++ src/server/ApiManagers/SearchManager.ts | 4 +- src/server/ApiManagers/UploadManager.ts | 227 ++++++++ src/server/ApiManagers/UserManager.ts | 47 +- src/server/DashUploadUtils.ts | 17 +- src/server/SharedMediaTypes.ts | 9 + src/server/Websocket/Websocket.ts | 19 +- src/server/apis/google/GoogleApiServerUtils.ts | 46 +- src/server/apis/google/GooglePhotosUploadUtils.ts | 19 +- src/server/credentials/CredentialsLoader.ts | 29 ++ .../credentials/google_docs_credentials.json | 11 - .../credentials/google_project_credentials.json | 14 + src/server/credentials/test.json | 14 + src/server/index.ts | 570 ++------------------- 18 files changed, 770 insertions(+), 596 deletions(-) delete mode 100644 client_secret.json create mode 100644 src/server/ApiManagers/DeleteManager.ts create mode 100644 src/server/ApiManagers/PDFManager.ts create mode 100644 src/server/ApiManagers/UploadManager.ts create mode 100644 src/server/SharedMediaTypes.ts create mode 100644 src/server/credentials/CredentialsLoader.ts delete mode 100644 src/server/credentials/google_docs_credentials.json create mode 100644 src/server/credentials/google_project_credentials.json create mode 100644 src/server/credentials/test.json (limited to 'src') diff --git a/client_secret.json b/client_secret.json deleted file mode 100644 index a9c698421..000000000 --- a/client_secret.json +++ /dev/null @@ -1 +0,0 @@ -{"installed":{"client_id":"1005546247619-kqpnvh42mpa803tem8556b87umi4j9r0.apps.googleusercontent.com","project_id":"brown-dash","auth_uri":"https://accounts.google.com/o/oauth2/auth","token_uri":"https://oauth2.googleapis.com/token","auth_provider_x509_cert_url":"https://www.googleapis.com/oauth2/v1/certs","client_secret":"WshLb5TH9SdFVGGbQcnYj7IU","redirect_uris":["urn:ietf:wg:oauth:2.0:oob","http://localhost"]}} \ No newline at end of file diff --git a/src/client/util/Import & Export/DirectoryImportBox.tsx b/src/client/util/Import & Export/DirectoryImportBox.tsx index bdd59cb16..2e0ba25eb 100644 --- a/src/client/util/Import & Export/DirectoryImportBox.tsx +++ b/src/client/util/Import & Export/DirectoryImportBox.tsx @@ -22,6 +22,9 @@ import { SchemaHeaderField } from "../../../new_fields/SchemaHeaderField"; import "./DirectoryImportBox.scss"; import { Networking } from "../../Network"; import { BatchedArray } from "array-batcher"; +import * as path from 'path'; +import { DashUploadUtils } from "../../../server/DashUploadUtils"; +import { SharedMediaTypes } from "../../../server/SharedMediaTypes"; const unsupported = ["text/html", "text/plain"]; @@ -94,7 +97,12 @@ export default class DirectoryImportBox extends React.Component let validated: File[] = []; for (let i = 0; i < files.length; i++) { let file = files.item(i); - file && !unsupported.includes(file.type) && validated.push(file); + if (file && !unsupported.includes(file.type)) { + const ext = path.extname(file.name).toLowerCase(); + if (SharedMediaTypes.imageFormats.includes(ext)) { + validated.push(file); + } + } } runInAction(() => { diff --git a/src/server/ApiManagers/DeleteManager.ts b/src/server/ApiManagers/DeleteManager.ts new file mode 100644 index 000000000..bbf1d0425 --- /dev/null +++ b/src/server/ApiManagers/DeleteManager.ts @@ -0,0 +1,65 @@ +import ApiManager, { Registration } from "./ApiManager"; +import { Method, _permission_denied } from "../RouteManager"; +import { RouteStore } from "../RouteStore"; +import { WebSocket } from "../Websocket/Websocket"; +import { Database } from "../database"; + +export default class DeleteManager extends ApiManager { + + protected initialize(register: Registration): void { + + register({ + method: Method.GET, + subscription: RouteStore.delete, + onValidation: async ({ res, isRelease }) => { + if (isRelease) { + return _permission_denied(res, deletionPermissionError); + } + await WebSocket.deleteFields(); + res.redirect(RouteStore.home); + } + }); + + register({ + method: Method.GET, + subscription: RouteStore.deleteAll, + onValidation: async ({ res, isRelease }) => { + if (isRelease) { + return _permission_denied(res, deletionPermissionError); + } + await WebSocket.deleteAll(); + res.redirect(RouteStore.home); + } + }); + + + register({ + method: Method.GET, + subscription: "/deleteWithAux", + onValidation: async ({ res, isRelease }) => { + if (isRelease) { + return _permission_denied(res, deletionPermissionError); + } + await Database.Auxiliary.DeleteAll(); + res.redirect(RouteStore.delete); + } + }); + + register({ + method: Method.GET, + subscription: "/deleteWithGoogleCredentials", + onValidation: async ({ res, isRelease }) => { + if (isRelease) { + return _permission_denied(res, deletionPermissionError); + } + await Database.Auxiliary.GoogleAuthenticationToken.DeleteAll(); + res.redirect(RouteStore.delete); + } + }); + + + } + +} + +const deletionPermissionError = "Cannot perform a delete operation outside of the development environment!"; diff --git a/src/server/ApiManagers/ExportManager.ts b/src/server/ApiManagers/ExportManager.ts index 14ac7dd5b..d42db1056 100644 --- a/src/server/ApiManagers/ExportManager.ts +++ b/src/server/ApiManagers/ExportManager.ts @@ -1,5 +1,5 @@ import ApiManager, { Registration } from "./ApiManager"; -import RouteManager, { Method } from "../RouteManager"; +import { Method } from "../RouteManager"; import RouteSubscriber from "../RouteSubscriber"; import { RouteStore } from "../RouteStore"; import * as Archiver from 'archiver'; @@ -7,6 +7,7 @@ import * as express from 'express'; import { Database } from "../database"; import * as path from "path"; import { DashUploadUtils } from "../DashUploadUtils"; +import { publicDirectory } from ".."; export type Hierarchy = { [id: string]: string | Hierarchy }; export type ZipMutator = (file: Archiver.Archiver) => void | Promise; @@ -15,10 +16,20 @@ export interface DocumentElements { title: string; } -export default class ExportManager extends ApiManager { +export default class DownloadManager extends ApiManager { protected initialize(register: Registration): void { + /** + * Let's say someone's using Dash to organize images in collections. + * This lets them export the hierarchy they've built to their + * own file system in a useful format. + * + * This handler starts with a single document id (interesting only + * if it's that of a collection). It traverses the database, captures + * the nesting of only nested images or collections, writes + * that to a zip file and returns it to the client for download. + */ register({ method: Method.GET, subscription: new RouteSubscriber(RouteStore.imageHierarchyExport).add('docId'), @@ -29,10 +40,101 @@ export default class ExportManager extends ApiManager { return BuildAndDispatchZip(res, zip => writeHierarchyRecursive(zip, hierarchy)); } }); + + register({ + method: Method.GET, + subscription: new RouteSubscriber("/downloadId").add("docId"), + onValidation: async ({ req, res }) => { + return BuildAndDispatchZip(res, async zip => { + const { id, docs, files } = await getDocs(req.params.docId); + const docString = JSON.stringify({ id, docs }); + zip.append(docString, { name: "doc.json" }); + files.forEach(val => { + zip.file(publicDirectory + val, { name: val.substring(1) }); + }); + }); + } + }); + + register({ + method: Method.GET, + subscription: new RouteSubscriber("/serializeDoc").add("docId"), + onValidation: async ({ req, res }) => { + const { docs, files } = await getDocs(req.params.docId); + res.send({ docs, files: Array.from(files) }); + } + }); + + } } +async function getDocs(id: string) { + const files = new Set(); + const docs: { [id: string]: any } = {}; + const fn = (doc: any): string[] => { + const id = doc.id; + if (typeof id === "string" && id.endsWith("Proto")) { + //Skip protos + return []; + } + const ids: string[] = []; + for (const key in doc.fields) { + if (!doc.fields.hasOwnProperty(key)) { + continue; + } + const field = doc.fields[key]; + if (field === undefined || field === null) { + continue; + } + + if (field.__type === "proxy" || field.__type === "prefetch_proxy") { + ids.push(field.fieldId); + } else if (field.__type === "script" || field.__type === "computed") { + if (field.captures) { + ids.push(field.captures.fieldId); + } + } else if (field.__type === "list") { + ids.push(...fn(field)); + } else if (typeof field === "string") { + const re = /"(?:dataD|d)ocumentId"\s*:\s*"([\w\-]*)"/g; + let match: string[] | null; + while ((match = re.exec(field)) !== null) { + ids.push(match[1]); + } + } else if (field.__type === "RichTextField") { + const re = /"href"\s*:\s*"(.*?)"/g; + let match: string[] | null; + while ((match = re.exec(field.Data)) !== null) { + const urlString = match[1]; + const split = new URL(urlString).pathname.split("doc/"); + if (split.length > 1) { + ids.push(split[split.length - 1]); + } + } + const re2 = /"src"\s*:\s*"(.*?)"/g; + while ((match = re2.exec(field.Data)) !== null) { + const urlString = match[1]; + const pathname = new URL(urlString).pathname; + files.add(pathname); + } + } else if (["audio", "image", "video", "pdf", "web"].includes(field.__type)) { + const url = new URL(field.url); + const pathname = url.pathname; + files.add(pathname); + } + } + + if (doc.id) { + docs[doc.id] = doc; + } + return ids; + }; + await Database.Instance.visit([id], fn); + return { id, docs, files }; +} + /** * This utility function factors out the process * of creating a zip file and sending it back to the client @@ -45,6 +147,8 @@ export default class ExportManager extends ApiManager { * @param mutator the callback function used to actually modify and insert information into the zip instance */ export async function BuildAndDispatchZip(res: express.Response, mutator: ZipMutator): Promise { + res.set('Content-disposition', `attachment;`); + res.set('Content-Type', "application/zip"); const zip = Archiver('zip'); zip.pipe(res); await mutator(zip); @@ -76,7 +180,6 @@ following the general recursive structure shown immediately below } } */ - async function buildHierarchyRecursive(seedId: string, hierarchy: Hierarchy): Promise { const { title, data } = await getData(seedId); const label = `${title} (${seedId})`; @@ -93,9 +196,20 @@ async function buildHierarchyRecursive(seedId: string, hierarchy: Hierarchy): Pr } } -async function getData(seedId: string): Promise { +/** + * This is a very specific utility method to help traverse the database + * to parse data and titles out of images and collections alone. + * + * We don't know if the document id given to is corresponds to a view document or a data + * document. If it's a data document, the response from the database will have + * a data field. If not, call recursively on the proto, and resolve with *its* data + * + * @param targetId the id of the Dash document whose data is being requests + * @returns the data of the document, as well as its title + */ +async function getData(targetId: string): Promise { return new Promise((resolve, reject) => { - Database.Instance.getDocument(seedId, async (result: any) => { + Database.Instance.getDocument(targetId, async (result: any) => { const { data, proto, title } = result.fields; if (data) { if (data.url) { @@ -105,29 +219,50 @@ async function getData(seedId: string): Promise { } else { reject(); } - } - if (proto) { + } else if (proto) { getData(proto.fieldId).then(resolve, reject); + } else { + reject(); } }); }); } +/** + * + * @param file the zip file to which we write the files + * @param hierarchy the data structure from which we read, defining the nesting of the documents in the zip + * @param prefix lets us create nested folders in the zip file by continually appending to the end + * of the prefix with each layer of recursion. + * + * Function Call #1 => "Dash Export" + * Function Call #2 => "Dash Export/a nested collection" + * Function Call #3 => "Dash Export/a nested collection/lowest level collection" + * ... + */ async function writeHierarchyRecursive(file: Archiver.Archiver, hierarchy: Hierarchy, prefix = "Dash Export"): Promise { - for (const key of Object.keys(hierarchy)) { - const result = hierarchy[key]; + for (const documentTitle of Object.keys(hierarchy)) { + const result = hierarchy[documentTitle]; + // base case or leaf node, we've hit a url (image) if (typeof result === "string") { let path: string; let matches: RegExpExecArray | null; if ((matches = /\:1050\/files\/(upload\_[\da-z]{32}.*)/g.exec(result)) !== null) { + // image already exists on our server path = `${__dirname}/public/files/${matches[1]}`; } else { + // the image doesn't already exist on our server (may have been dragged + // and dropped in the browser and thus hosted remotely) so we upload it + // to our server and point the zip file to it, so it can bundle up the bytes const information = await DashUploadUtils.UploadImage(result); path = information.mediaPaths[0]; } - file.file(path, { name: key, prefix }); + // write the file specified by the path to the directory in the + // zip file given by the prefix. + file.file(path, { name: documentTitle, prefix }); } else { - await writeHierarchyRecursive(file, result, `${prefix}/${key}`); + // we've hit a collection, so we have to recurse + await writeHierarchyRecursive(file, result, `${prefix}/${documentTitle}`); } } } \ No newline at end of file diff --git a/src/server/ApiManagers/PDFManager.ts b/src/server/ApiManagers/PDFManager.ts new file mode 100644 index 000000000..f328557b4 --- /dev/null +++ b/src/server/ApiManagers/PDFManager.ts @@ -0,0 +1,107 @@ +import ApiManager, { Registration } from "./ApiManager"; +import { Method } from "../RouteManager"; +import RouteSubscriber from "../RouteSubscriber"; +import { exists, createReadStream, createWriteStream } from "fs"; +import { filesDirectory } from ".."; +import * as Pdfjs from 'pdfjs-dist'; +import { createCanvas } from "canvas"; +const probe = require("probe-image-size"); +import * as express from "express"; +import * as path from "path"; + +export default class PDFManager extends ApiManager { + + protected initialize(register: Registration): void { + + register({ + method: Method.GET, + subscription: new RouteSubscriber("/thumbnail").add("filename"), + onValidation: ({ req, res }) => { + let filename = req.params.filename; + let noExt = filename.substring(0, filename.length - ".png".length); + let pagenumber = parseInt(noExt.split('-')[1]); + return new Promise(resolve => { + exists(filesDirectory + filename, (exists: boolean) => { + console.log(`${filesDirectory + filename} ${exists ? "exists" : "does not exist"}`); + if (exists) { + let input = createReadStream(filesDirectory + filename); + probe(input, (err: any, result: any) => { + if (err) { + console.log(err); + console.log(`error on ${filename}`); + return; + } + res.send({ path: "/files/" + filename, width: result.width, height: result.height }); + }); + } + else { + LoadPage(filesDirectory + filename.substring(0, filename.length - noExt.split('-')[1].length - ".PNG".length - 1) + ".pdf", pagenumber, res); + } + resolve(); + }); + }); + } + }); + + function LoadPage(file: string, pageNumber: number, res: express.Response) { + console.log(file); + Pdfjs.getDocument(file).promise + .then((pdf: Pdfjs.PDFDocumentProxy) => { + let factory = new NodeCanvasFactory(); + console.log(pageNumber); + pdf.getPage(pageNumber).then((page: Pdfjs.PDFPageProxy) => { + console.log("reading " + page); + let viewport = page.getViewport(1 as any); + let canvasAndContext = factory.create(viewport.width, viewport.height); + let renderContext = { + canvasContext: canvasAndContext.context, + viewport: viewport, + canvasFactory: factory + }; + console.log("read " + pageNumber); + + page.render(renderContext).promise + .then(() => { + console.log("saving " + pageNumber); + let stream = canvasAndContext.canvas.createPNGStream(); + let pngFile = `${file.substring(0, file.length - ".pdf".length)}-${pageNumber}.PNG`; + let out = createWriteStream(pngFile); + stream.pipe(out); + out.on("finish", () => { + console.log(`Success! Saved to ${pngFile}`); + let name = path.basename(pngFile); + res.send({ path: "/files/" + name, width: viewport.width, height: viewport.height }); + }); + }, (reason: string) => { + console.error(reason + ` ${pageNumber}`); + }); + }); + }); + } + + } + +} + +class NodeCanvasFactory { + create = (width: number, height: number) => { + var canvas = createCanvas(width, height); + var context = canvas.getContext('2d'); + return { + canvas, + context + }; + } + + reset = (canvasAndContext: any, width: number, height: number) => { + canvasAndContext.canvas.width = width; + canvasAndContext.canvas.height = height; + } + + destroy = (canvasAndContext: any) => { + canvasAndContext.canvas.width = 0; + canvasAndContext.canvas.height = 0; + canvasAndContext.canvas = null; + canvasAndContext.context = null; + } +} \ No newline at end of file diff --git a/src/server/ApiManagers/SearchManager.ts b/src/server/ApiManagers/SearchManager.ts index 1c4b805e5..1c801715a 100644 --- a/src/server/ApiManagers/SearchManager.ts +++ b/src/server/ApiManagers/SearchManager.ts @@ -3,7 +3,7 @@ import { Method } from "../RouteManager"; import { Search } from "../Search"; var findInFiles = require('find-in-files'); import * as path from 'path'; -import { uploadDirectory } from ".."; +import { filesDirectory } from ".."; export default class SearchManager extends ApiManager { @@ -18,7 +18,7 @@ export default class SearchManager extends ApiManager { res.send([]); return; } - let results = await findInFiles.find({ 'term': q, 'flags': 'ig' }, uploadDirectory + "text", ".txt$"); + let results = await findInFiles.find({ 'term': q, 'flags': 'ig' }, filesDirectory + "text", ".txt$"); let resObj: { ids: string[], numFound: number, lines: string[] } = { ids: [], numFound: 0, lines: [] }; for (var result in results) { resObj.ids.push(path.basename(result, ".txt").replace(/upload_/, "")); diff --git a/src/server/ApiManagers/UploadManager.ts b/src/server/ApiManagers/UploadManager.ts new file mode 100644 index 000000000..38635eda5 --- /dev/null +++ b/src/server/ApiManagers/UploadManager.ts @@ -0,0 +1,227 @@ +import ApiManager, { Registration } from "./ApiManager"; +import { Method, _success } from "../RouteManager"; +import * as formidable from 'formidable'; +import v4 = require('uuid/v4'); +var AdmZip = require('adm-zip'); +import * as path from 'path'; +import { createReadStream, createWriteStream, unlink, readFileSync } from "fs"; +import { publicDirectory, filesDirectory, Partitions } from ".."; +import { RouteStore } from "../RouteStore"; +import { Database } from "../database"; +import { DashUploadUtils } from "../DashUploadUtils"; +import { Opt } from "../../new_fields/Doc"; +import { ParsedPDF } from "../PdfTypes"; +const pdf = require('pdf-parse'); +import * as sharp from 'sharp'; +import { SharedMediaTypes } from "../SharedMediaTypes"; +const imageDataUri = require('image-data-uri'); + +export default class UploadManager extends ApiManager { + + protected initialize(register: Registration): void { + + register({ + method: Method.POST, + subscription: "/uploadDoc", + onValidation: ({ req, res }) => { + let form = new formidable.IncomingForm(); + form.keepExtensions = true; + // let path = req.body.path; + const ids: { [id: string]: string } = {}; + let remap = true; + const getId = (id: string): string => { + if (!remap) return id; + if (id.endsWith("Proto")) return id; + if (id in ids) { + return ids[id]; + } else { + return ids[id] = v4(); + } + }; + const mapFn = (doc: any) => { + if (doc.id) { + doc.id = getId(doc.id); + } + for (const key in doc.fields) { + if (!doc.fields.hasOwnProperty(key)) { + continue; + } + const field = doc.fields[key]; + if (field === undefined || field === null) { + continue; + } + + if (field.__type === "proxy" || field.__type === "prefetch_proxy") { + field.fieldId = getId(field.fieldId); + } else if (field.__type === "script" || field.__type === "computed") { + if (field.captures) { + field.captures.fieldId = getId(field.captures.fieldId); + } + } else if (field.__type === "list") { + mapFn(field); + } else if (typeof field === "string") { + const re = /("(?:dataD|d)ocumentId"\s*:\s*")([\w\-]*)"/g; + doc.fields[key] = (field as any).replace(re, (match: any, p1: string, p2: string) => { + return `${p1}${getId(p2)}"`; + }); + } else if (field.__type === "RichTextField") { + const re = /("href"\s*:\s*")(.*?)"/g; + field.Data = field.Data.replace(re, (match: any, p1: string, p2: string) => { + return `${p1}${getId(p2)}"`; + }); + } + } + }; + return new Promise(resolve => { + form.parse(req, async (_err, fields, files) => { + remap = fields.remap !== "false"; + let id: string = ""; + try { + for (const name in files) { + const path_2 = files[name].path; + const zip = new AdmZip(path_2); + zip.getEntries().forEach((entry: any) => { + if (!entry.entryName.startsWith("files/")) return; + let dirname = path.dirname(entry.entryName) + "/"; + let extname = path.extname(entry.entryName); + let basename = path.basename(entry.entryName).split(".")[0]; + // zip.extractEntryTo(dirname + basename + "_o" + extname, __dirname + RouteStore.public, true, false); + // zip.extractEntryTo(dirname + basename + "_s" + extname, __dirname + RouteStore.public, true, false); + // zip.extractEntryTo(dirname + basename + "_m" + extname, __dirname + RouteStore.public, true, false); + // zip.extractEntryTo(dirname + basename + "_l" + extname, __dirname + RouteStore.public, true, false); + try { + zip.extractEntryTo(entry.entryName, __dirname + RouteStore.public, true, false); + dirname = "/" + dirname; + + createReadStream(publicDirectory + dirname + basename + extname).pipe(createWriteStream(publicDirectory + dirname + basename + "_o" + extname)); + createReadStream(publicDirectory + dirname + basename + extname).pipe(createWriteStream(publicDirectory + dirname + basename + "_s" + extname)); + createReadStream(publicDirectory + dirname + basename + extname).pipe(createWriteStream(publicDirectory + dirname + basename + "_m" + extname)); + createReadStream(publicDirectory + dirname + basename + extname).pipe(createWriteStream(publicDirectory + dirname + basename + "_l" + extname)); + } catch (e) { + console.log(e); + } + }); + const json = zip.getEntry("doc.json"); + let docs: any; + try { + let data = JSON.parse(json.getData().toString("utf8")); + docs = data.docs; + id = data.id; + docs = Object.keys(docs).map(key => docs[key]); + docs.forEach(mapFn); + await Promise.all(docs.map((doc: any) => new Promise(res => Database.Instance.replace(doc.id, doc, (err, r) => { + err && console.log(err); + res(); + }, true, "newDocuments")))); + } catch (e) { console.log(e); } + unlink(path_2, () => { }); + } + if (id) { + res.send(JSON.stringify(getId(id))); + } else { + res.send(JSON.stringify("error")); + } + } catch (e) { console.log(e); } + resolve(); + }); + }); + } + }); + + + register({ + method: Method.POST, + subscription: RouteStore.upload, + onValidation: async ({ req, res }) => { + let form = new formidable.IncomingForm(); + form.uploadDir = filesDirectory; + form.keepExtensions = true; + return new Promise(resolve => { + form.parse(req, async (_err, _fields, files) => { + let results: DashUploadUtils.ImageFileResponse[] = []; + for (const key in files) { + const { type, path: location, name } = files[key]; + const filename = path.basename(location); + let uploadInformation: Opt; + if (filename.endsWith(".pdf")) { + let dataBuffer = readFileSync(filesDirectory + filename); + const result: ParsedPDF = await pdf(dataBuffer); + await new Promise((resolve, reject) => { + const path = filesDirectory + Partitions.PdfText + "/" + filename.substring(0, filename.length - ".pdf".length) + ".txt"; + createWriteStream(path).write(result.text, error => { + if (!error) { + resolve(); + } else { + reject(error); + } + }); + }); + } else { + uploadInformation = await DashUploadUtils.UploadImage(filesDirectory + filename, filename); + } + const exif = uploadInformation ? uploadInformation.exifData : undefined; + results.push({ name, type, path: `/files/${filename}`, exif }); + } + _success(res, results); + resolve(); + }); + }); + } + }); + + register({ + method: Method.POST, + subscription: RouteStore.inspectImage, + onValidation: async ({ req, res }) => { + const { source } = req.body; + if (typeof source === "string") { + const uploadInformation = await DashUploadUtils.UploadImage(source); + return res.send(await DashUploadUtils.InspectImage(uploadInformation.mediaPaths[0])); + } + res.send({}); + } + }); + + register({ + method: Method.POST, + subscription: RouteStore.dataUriToImage, + onValidation: ({ req, res }) => { + const uri = req.body.uri; + const filename = req.body.name; + if (!uri || !filename) { + res.status(401).send("incorrect parameters specified"); + return; + } + return imageDataUri.outputFile(uri, filesDirectory + filename).then((savedName: string) => { + const ext = path.extname(savedName).toLowerCase(); + const { pngs, jpgs } = SharedMediaTypes; + let resizers = [ + { resizer: sharp().resize(100, undefined, { withoutEnlargement: true }), suffix: "_s" }, + { resizer: sharp().resize(400, undefined, { withoutEnlargement: true }), suffix: "_m" }, + { resizer: sharp().resize(900, undefined, { withoutEnlargement: true }), suffix: "_l" }, + ]; + let isImage = false; + if (pngs.includes(ext)) { + resizers.forEach(element => { + element.resizer = element.resizer.png(); + }); + isImage = true; + } else if (jpgs.includes(ext)) { + resizers.forEach(element => { + element.resizer = element.resizer.jpeg(); + }); + isImage = true; + } + if (isImage) { + resizers.forEach(resizer => { + createReadStream(savedName).pipe(resizer.resizer).pipe(createWriteStream(filesDirectory + filename + resizer.suffix + ext)); + }); + } + res.send("/files/" + filename + ext); + }); + } + }); + + } + +} \ No newline at end of file diff --git a/src/server/ApiManagers/UserManager.ts b/src/server/ApiManagers/UserManager.ts index dd1e50133..fe1ce7f2b 100644 --- a/src/server/ApiManagers/UserManager.ts +++ b/src/server/ApiManagers/UserManager.ts @@ -1,11 +1,36 @@ import ApiManager, { Registration } from "./ApiManager"; import { Method } from "../RouteManager"; import { WebSocket } from "../Websocket/Websocket"; +import { RouteStore } from "../RouteStore"; +import { Database } from "../database"; export default class UserManager extends ApiManager { protected initialize(register: Registration): void { + register({ + method: Method.GET, + subscription: RouteStore.getUsers, + onValidation: async ({ res }) => { + const cursor = await Database.Instance.query({}, { email: 1, userDocumentId: 1 }, "users"); + const results = await cursor.toArray(); + res.send(results.map(user => ({ email: user.email, userDocumentId: user.userDocumentId }))); + } + }); + + register({ + method: Method.GET, + subscription: RouteStore.getUserDocumentId, + onValidation: ({ res, user }) => res.send(user.userDocumentId) + }); + + register({ + method: Method.GET, + subscription: RouteStore.getCurrUser, + onValidation: ({ res, user }) => res.send(JSON.stringify(user)), + onUnauthenticated: ({ res }) => res.send(JSON.stringify({ id: "__guest__", email: "" })) + }); + register({ method: Method.GET, subscription: "/whosOnline", @@ -17,7 +42,7 @@ export default class UserManager extends ApiManager { for (const user in timeMap) { const time = timeMap[user]; const key = ((now - time) / 1000) < (60 * 5) ? "active" : "inactive"; - users[key][user] = `Last active ${this.msToTime(now - time)} ago`; + users[key][user] = `Last active ${msToTime(now - time)} ago`; } res.send(users); @@ -26,17 +51,17 @@ export default class UserManager extends ApiManager { } - private msToTime(duration: number) { - let milliseconds = Math.floor((duration % 1000) / 100), - seconds = Math.floor((duration / 1000) % 60), - minutes = Math.floor((duration / (1000 * 60)) % 60), - hours = Math.floor((duration / (1000 * 60 * 60)) % 24); +} - let hoursS = (hours < 10) ? "0" + hours : hours; - let minutesS = (minutes < 10) ? "0" + minutes : minutes; - let secondsS = (seconds < 10) ? "0" + seconds : seconds; +function msToTime(duration: number) { + let milliseconds = Math.floor((duration % 1000) / 100), + seconds = Math.floor((duration / 1000) % 60), + minutes = Math.floor((duration / (1000 * 60)) % 60), + hours = Math.floor((duration / (1000 * 60 * 60)) % 24); - return hoursS + ":" + minutesS + ":" + secondsS + "." + milliseconds; - } + let hoursS = (hours < 10) ? "0" + hours : hours; + let minutesS = (minutes < 10) ? "0" + minutes : minutes; + let secondsS = (seconds < 10) ? "0" + seconds : seconds; + return hoursS + ":" + minutesS + ":" + secondsS + "." + milliseconds; } \ No newline at end of file diff --git a/src/server/DashUploadUtils.ts b/src/server/DashUploadUtils.ts index 9fddb466c..8f5b0e1a8 100644 --- a/src/server/DashUploadUtils.ts +++ b/src/server/DashUploadUtils.ts @@ -5,6 +5,7 @@ import * as sharp from 'sharp'; import request = require('request-promise'); import { ExifData, ExifImage } from 'exif'; import { Opt } from '../new_fields/Doc'; +import { SharedMediaTypes } from './SharedMediaTypes'; const uploadDirectory = path.join(__dirname, './public/files/'); @@ -15,20 +16,21 @@ export namespace DashUploadUtils { suffix: string; } + export interface ImageFileResponse { + name: string; + path: string; + type: string; + exif: Opt; + } + export const Sizes: { [size: string]: Size } = { SMALL: { width: 100, suffix: "_s" }, MEDIUM: { width: 400, suffix: "_m" }, LARGE: { width: 900, suffix: "_l" }, }; - const gifs = [".gif"]; - const pngs = [".png"]; - const jpgs = [".jpg", ".jpeg"]; - const imageFormats = [...pngs, ...jpgs, ...gifs]; - const videoFormats = [".mov", ".mp4"]; - export function validateExtension(url: string) { - return imageFormats.includes(path.extname(url).toLowerCase()); + return SharedMediaTypes.imageFormats.includes(path.extname(url).toLowerCase()); } const size = "content-length"; @@ -132,6 +134,7 @@ export namespace DashUploadUtils { contentSize, contentType, }; + const { pngs, imageFormats, jpgs, videoFormats } = SharedMediaTypes; return new Promise(async (resolve, reject) => { const resizers = [ { resizer: sharp().rotate(), suffix: "_o" }, diff --git a/src/server/SharedMediaTypes.ts b/src/server/SharedMediaTypes.ts new file mode 100644 index 000000000..3d3234125 --- /dev/null +++ b/src/server/SharedMediaTypes.ts @@ -0,0 +1,9 @@ +export namespace SharedMediaTypes { + + export const gifs = [".gif"]; + export const pngs = [".png"]; + export const jpgs = [".jpg", ".jpeg"]; + export const imageFormats = [...pngs, ...jpgs, ...gifs]; + export const videoFormats = [".mov", ".mp4"]; + +} \ No newline at end of file diff --git a/src/server/Websocket/Websocket.ts b/src/server/Websocket/Websocket.ts index cd2813d99..f6a6c8718 100644 --- a/src/server/Websocket/Websocket.ts +++ b/src/server/Websocket/Websocket.ts @@ -6,7 +6,9 @@ import { Database } from "../database"; import { Search } from "../Search"; import * as io from 'socket.io'; import YoutubeApi from "../apis/youtube/youtubeApiSample"; -import { youtubeApiKey } from ".."; +import { readFile } from "fs"; +import { Credentials } from "google-auth-library"; +import { GoogleCredentialsLoader } from "../credentials/CredentialsLoader"; export namespace WebSocket { @@ -18,6 +20,14 @@ export namespace WebSocket { export const socketMap = new Map(); export const timeMap: { [id: string]: number } = {}; + export async function start(serverPort: number, isRelease: boolean) { + await preliminaryFunctions(); + initialize(serverPort, isRelease); + } + + async function preliminaryFunctions() { + } + export function initialize(serverPort: number, isRelease: boolean) { const endpoint = io(); endpoint.listen(serverPort); @@ -54,14 +64,15 @@ export namespace WebSocket { } function HandleYoutubeQuery([query, callback]: [YoutubeQueryInput, (result?: any[]) => void]) { + const { ProjectCredentials } = GoogleCredentialsLoader; switch (query.type) { case YoutubeQueryTypes.Channels: - YoutubeApi.authorizedGetChannel(youtubeApiKey); + YoutubeApi.authorizedGetChannel(ProjectCredentials); break; case YoutubeQueryTypes.SearchVideo: - YoutubeApi.authorizedGetVideos(youtubeApiKey, query.userInput, callback); + YoutubeApi.authorizedGetVideos(ProjectCredentials, query.userInput, callback); case YoutubeQueryTypes.VideoDetails: - YoutubeApi.authorizedGetVideoDetails(youtubeApiKey, query.videoIds, callback); + YoutubeApi.authorizedGetVideoDetails(ProjectCredentials, query.videoIds, callback); } } diff --git a/src/server/apis/google/GoogleApiServerUtils.ts b/src/server/apis/google/GoogleApiServerUtils.ts index 35a2541a9..b3657ee43 100644 --- a/src/server/apis/google/GoogleApiServerUtils.ts +++ b/src/server/apis/google/GoogleApiServerUtils.ts @@ -1,12 +1,11 @@ import { google } from "googleapis"; -import { readFile } from "fs"; import { OAuth2Client, Credentials, OAuth2ClientOptions } from "google-auth-library"; import { Opt } from "../../../new_fields/Doc"; import { GaxiosResponse } from "gaxios"; import request = require('request-promise'); import * as qs from 'query-string'; import { Database } from "../../database"; -import * as path from "path"; +import { GoogleCredentialsLoader } from "../../credentials/CredentialsLoader"; /** * Scopes give Google users fine granularity of control @@ -61,6 +60,23 @@ export namespace GoogleApiServerUtils { */ let worker: OAuth2Client; + /** + * This function is called once before the server is started, + * reading in Dash's project-specific credentials (client secret + * and client id) for later repeated access. It also sets up the + * global, intentionally unauthenticated worker OAuth2 client instance. + */ + export function processProjectCredentials(): void { + const { client_secret, client_id, redirect_uris } = GoogleCredentialsLoader.ProjectCredentials; + // initialize the global authorization client + installed = { + clientId: client_id, + clientSecret: client_secret, + redirectUri: redirect_uris[0] + }; + worker = generateClient(); + } + /** * A briefer format for the response from a 'googleapis' API request */ @@ -96,32 +112,6 @@ export namespace GoogleApiServerUtils { batchUpdate: ApiHandler; } - /** - * This function is called once before the server is started, - * reading in Dash's project-specific credentials (client secret - * and client id) for later repeated access. It also sets up the - * global, intentionally unauthenticated worker OAuth2 client instance. - */ - export async function loadClientSecret(): Promise { - return new Promise((resolve, reject) => { - readFile(path.join(__dirname, "../../credentials/google_docs_credentials.json"), async (err, projectCredentials) => { - if (err) { - reject(err); - return console.log('Error loading client secret file:', err); - } - const { client_secret, client_id, redirect_uris } = JSON.parse(projectCredentials.toString()).installed; - // initialize the global authorization client - installed = { - clientId: client_id, - clientSecret: client_secret, - redirectUri: redirect_uris[0] - }; - worker = generateClient(); - resolve(); - }); - }); - } - /** * Maps the Dash user id of a given user to their single * associated OAuth2 client, mitigating the creation diff --git a/src/server/apis/google/GooglePhotosUploadUtils.ts b/src/server/apis/google/GooglePhotosUploadUtils.ts index d8cf795b5..0abed3f1d 100644 --- a/src/server/apis/google/GooglePhotosUploadUtils.ts +++ b/src/server/apis/google/GooglePhotosUploadUtils.ts @@ -1,7 +1,6 @@ import request = require('request-promise'); import * as path from 'path'; -import { MediaItemCreationResult, NewMediaItemResult } from './SharedTypes'; -import { NewMediaItem } from "../../index"; +import { NewMediaItemResult } from './SharedTypes'; import { BatchedArray, TimeUnit } from 'array-batcher'; import { DashUploadUtils } from '../../DashUploadUtils'; @@ -28,6 +27,22 @@ export namespace GooglePhotosUploadUtils { description: string; } + /** + * This is the format needed to pass + * into the BatchCreate API request + * to take a reference to raw uploaded bytes + * and actually create an image in Google Photos. + * + * So, to instantiate this interface you must have already dispatched an upload + * and received an upload token. + */ + export interface NewMediaItem { + description: string; + simpleMediaItem: { + uploadToken: string; + }; + } + /** * A utility function to streamline making * calls to the API's url - accentuates diff --git a/src/server/credentials/CredentialsLoader.ts b/src/server/credentials/CredentialsLoader.ts new file mode 100644 index 000000000..e3f4d167b --- /dev/null +++ b/src/server/credentials/CredentialsLoader.ts @@ -0,0 +1,29 @@ +import { readFile } from "fs"; + +export namespace GoogleCredentialsLoader { + + export interface InstalledCredentials { + client_id: string; + project_id: string; + auth_uri: string; + token_uri: string; + auth_provider_x509_cert_url: string; + client_secret: string; + redirect_uris: string[]; + } + + export let ProjectCredentials: InstalledCredentials; + + export async function loadCredentials() { + ProjectCredentials = await new Promise(resolve => { + readFile(__dirname + '/google_project_credentials.json', function processClientSecrets(err, content) { + if (err) { + console.log('Error loading client secret file: ' + err); + return; + } + resolve(JSON.parse(content.toString()).installed); + }); + }); + } + +} diff --git a/src/server/credentials/google_docs_credentials.json b/src/server/credentials/google_docs_credentials.json deleted file mode 100644 index 955c5a3c1..000000000 --- a/src/server/credentials/google_docs_credentials.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "installed": { - "client_id": "343179513178-ud6tvmh275r2fq93u9eesrnc66t6akh9.apps.googleusercontent.com", - "project_id": "quickstart-1565056383187", - "auth_uri": "https://accounts.google.com/o/oauth2/auth", - "token_uri": "https://oauth2.googleapis.com/token", - "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", - "client_secret": "w8KIFSc0MQpmUYHed4qEzn8b", - "redirect_uris": ["urn:ietf:wg:oauth:2.0:oob", "http://localhost"] - } -} \ No newline at end of file diff --git a/src/server/credentials/google_project_credentials.json b/src/server/credentials/google_project_credentials.json new file mode 100644 index 000000000..5d9c62eb1 --- /dev/null +++ b/src/server/credentials/google_project_credentials.json @@ -0,0 +1,14 @@ +{ + "installed": { + "client_id": "1005546247619-kqpnvh42mpa803tem8556b87umi4j9r0.apps.googleusercontent.com", + "project_id": "brown-dash", + "auth_uri": "https://accounts.google.com/o/oauth2/auth", + "token_uri": "https://oauth2.googleapis.com/token", + "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", + "client_secret": "WshLb5TH9SdFVGGbQcnYj7IU", + "redirect_uris": [ + "urn:ietf:wg:oauth:2.0:oob", + "http://localhost" + ] + } +} \ No newline at end of file diff --git a/src/server/credentials/test.json b/src/server/credentials/test.json new file mode 100644 index 000000000..0a032cc2d --- /dev/null +++ b/src/server/credentials/test.json @@ -0,0 +1,14 @@ +{ + "installed": { + "client_id": "343179513178-ud6tvmh275r2fq93u9eesrnc66t6akh9.apps.googleusercontent.com", + "project_id": "quickstart-1565056383187", + "auth_uri": "https://accounts.google.com/o/oauth2/auth", + "token_uri": "https://oauth2.googleapis.com/token", + "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", + "client_secret": "w8KIFSc0MQpmUYHed4qEzn8b", + "redirect_uris": [ + "urn:ietf:wg:oauth:2.0:oob", + "http://localhost" + ] + } +} \ No newline at end of file diff --git a/src/server/index.ts b/src/server/index.ts index 25697e71f..8eb88cf8b 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -1,352 +1,75 @@ require('dotenv').config(); -import * as formidable from 'formidable'; -import * as fs from 'fs'; -import * as sharp from 'sharp'; -import * as Pdfjs from 'pdfjs-dist'; -const imageDataUri = require('image-data-uri'); +import { GoogleApiServerUtils } from "./apis/google/GoogleApiServerUtils"; import * as mobileDetect from 'mobile-detect'; import * as path from 'path'; import { Database } from './database'; import { RouteStore } from './RouteStore'; -import v4 = require('uuid/v4'); -import { createCanvas } from "canvas"; const serverPort = 4321; -import { Search } from './Search'; -import * as Archiver from 'archiver'; -var AdmZip = require('adm-zip'); -import * as YoutubeApi from "./apis/youtube/youtubeApiSample"; -import { Response } from 'express-serve-static-core'; -import { GoogleApiServerUtils } from "./apis/google/GoogleApiServerUtils"; -const probe = require("probe-image-size"); -const pdf = require('pdf-parse'); import { GooglePhotosUploadUtils } from './apis/google/GooglePhotosUploadUtils'; import { Opt } from '../new_fields/Doc'; import { DashUploadUtils } from './DashUploadUtils'; import { BatchedArray, TimeUnit } from 'array-batcher'; -import { ParsedPDF } from "./PdfTypes"; -import { reject } from 'bluebird'; import RouteSubscriber from './RouteSubscriber'; -import InitializeServer from './Initialization'; +import initializeServer from './Initialization'; import RouteManager, { Method, _success, _permission_denied, _error, _invalid, OnUnauthenticated } from './RouteManager'; import * as qs from 'query-string'; import UtilManager from './ApiManagers/UtilManager'; import SearchManager from './ApiManagers/SearchManager'; import UserManager from './ApiManagers/UserManager'; import { WebSocket } from './Websocket/Websocket'; -import ExportManager from './ApiManagers/ExportManager'; -import ApiManager from './ApiManagers/ApiManager'; - -export let youtubeApiKey: string; - -export interface NewMediaItem { - description: string; - simpleMediaItem: { - uploadToken: string; - }; +import DownloadManager from './ApiManagers/ExportManager'; +import { GoogleCredentialsLoader } from './credentials/CredentialsLoader'; +import DeleteManager from "./ApiManagers/DeleteManager"; +import PDFManager from "./ApiManagers/PDFManager"; +import UploadManager from "./ApiManagers/UploadManager"; + +export const publicDirectory = __dirname + RouteStore.public; +export const filesDirectory = publicDirectory + "/files/"; +export enum Partitions { + PdfText = "pdf_text" } -const pngTypes = [".png", ".PNG"]; -const jpgTypes = [".jpg", ".JPG", ".jpeg", ".JPEG"]; -export const uploadDirectory = __dirname + "/public/files/"; -const pdfDirectory = uploadDirectory + "text"; -const solrURL = "http://localhost:8983/solr/#/dash"; - -start(); - -async function start() { - await PreliminaryFunctions(); - await InitializeServer({ listenAtPort: 1050, routeSetter }); -} - -async function PreliminaryFunctions() { - await new Promise(resolve => { - YoutubeApi.readApiKey((apiKey: string) => { - youtubeApiKey = apiKey; - resolve(); - }); - }); - await GoogleApiServerUtils.loadClientSecret(); - await DashUploadUtils.createIfNotExists(pdfDirectory); +/** + * These are the functions run before the server starts + * listening. Anything that must be complete + * before clients can access the server should be run or awaited here. + */ +async function preliminaryFunctions() { + // make project credentials globally accessible + await GoogleCredentialsLoader.loadCredentials(); + // read the resulting credentials into a different namespace + GoogleApiServerUtils.processProjectCredentials(); + // divide the public directory based on type + await Promise.all(Object.keys(Partitions).map(partition => DashUploadUtils.createIfNotExists(filesDirectory + partition))); + // connect to the database await Database.tryInitializeConnection(); } +/** + * Either clustered together as an API manager + * or individually referenced below, by the completion + * of this function's execution, all routes will + * be registered on the server + * @param router the instance of the route manager + * that will manage the registration of new routes + * with the server + */ function routeSetter(router: RouteManager) { - const managers: ApiManager[] = [ - new UtilManager(), - new SearchManager(), + // initialize API Managers + [ new UserManager(), - new ExportManager() - ]; - managers.forEach(manager => manager.register(router)); + new UploadManager(), + new DownloadManager(), + new SearchManager(), + new PDFManager(), + new DeleteManager(), + new UtilManager() + ].forEach(manager => manager.register(router)); + // initialize the web socket (bidirectional communication: if a user changes + // a field on one client, that change must be broadcast to all other clients) WebSocket.initialize(serverPort, router.isRelease); - async function getDocs(id: string) { - const files = new Set(); - const docs: { [id: string]: any } = {}; - const fn = (doc: any): string[] => { - const id = doc.id; - if (typeof id === "string" && id.endsWith("Proto")) { - //Skip protos - return []; - } - const ids: string[] = []; - for (const key in doc.fields) { - if (!doc.fields.hasOwnProperty(key)) { - continue; - } - const field = doc.fields[key]; - if (field === undefined || field === null) { - continue; - } - - if (field.__type === "proxy" || field.__type === "prefetch_proxy") { - ids.push(field.fieldId); - } else if (field.__type === "script" || field.__type === "computed") { - if (field.captures) { - ids.push(field.captures.fieldId); - } - } else if (field.__type === "list") { - ids.push(...fn(field)); - } else if (typeof field === "string") { - const re = /"(?:dataD|d)ocumentId"\s*:\s*"([\w\-]*)"/g; - let match: string[] | null; - while ((match = re.exec(field)) !== null) { - ids.push(match[1]); - } - } else if (field.__type === "RichTextField") { - const re = /"href"\s*:\s*"(.*?)"/g; - let match: string[] | null; - while ((match = re.exec(field.Data)) !== null) { - const urlString = match[1]; - const split = new URL(urlString).pathname.split("doc/"); - if (split.length > 1) { - ids.push(split[split.length - 1]); - } - } - const re2 = /"src"\s*:\s*"(.*?)"/g; - while ((match = re2.exec(field.Data)) !== null) { - const urlString = match[1]; - const pathname = new URL(urlString).pathname; - files.add(pathname); - } - } else if (["audio", "image", "video", "pdf", "web"].includes(field.__type)) { - const url = new URL(field.url); - const pathname = url.pathname; - files.add(pathname); - } - } - - if (doc.id) { - docs[doc.id] = doc; - } - return ids; - }; - await Database.Instance.visit([id], fn); - return { id, docs, files }; - } - - router.addSupervisedRoute({ - method: Method.GET, - subscription: new RouteSubscriber("/serializeDoc").add("docId"), - onValidation: async ({ req, res }) => { - const { docs, files } = await getDocs(req.params.docId); - res.send({ docs, files: Array.from(files) }); - } - }); - - router.addSupervisedRoute({ - method: Method.GET, - subscription: new RouteSubscriber("/downloadId").add("docId"), - onValidation: async ({ req, res }) => { - res.set('Content-disposition', `attachment;`); - res.set('Content-Type', "application/zip"); - const { id, docs, files } = await getDocs(req.params.docId); - const docString = JSON.stringify({ id, docs }); - const zip = Archiver('zip'); - zip.pipe(res); - zip.append(docString, { name: "doc.json" }); - files.forEach(val => { - zip.file(__dirname + RouteStore.public + val, { name: val.substring(1) }); - }); - zip.finalize(); - } - }); - - router.addSupervisedRoute({ - method: Method.POST, - subscription: "/uploadDoc", - onValidation: ({ req, res }) => { - let form = new formidable.IncomingForm(); - form.keepExtensions = true; - // let path = req.body.path; - const ids: { [id: string]: string } = {}; - let remap = true; - const getId = (id: string): string => { - if (!remap) return id; - if (id.endsWith("Proto")) return id; - if (id in ids) { - return ids[id]; - } else { - return ids[id] = v4(); - } - }; - const mapFn = (doc: any) => { - if (doc.id) { - doc.id = getId(doc.id); - } - for (const key in doc.fields) { - if (!doc.fields.hasOwnProperty(key)) { - continue; - } - const field = doc.fields[key]; - if (field === undefined || field === null) { - continue; - } - - if (field.__type === "proxy" || field.__type === "prefetch_proxy") { - field.fieldId = getId(field.fieldId); - } else if (field.__type === "script" || field.__type === "computed") { - if (field.captures) { - field.captures.fieldId = getId(field.captures.fieldId); - } - } else if (field.__type === "list") { - mapFn(field); - } else if (typeof field === "string") { - const re = /("(?:dataD|d)ocumentId"\s*:\s*")([\w\-]*)"/g; - doc.fields[key] = (field as any).replace(re, (match: any, p1: string, p2: string) => { - return `${p1}${getId(p2)}"`; - }); - } else if (field.__type === "RichTextField") { - const re = /("href"\s*:\s*")(.*?)"/g; - field.Data = field.Data.replace(re, (match: any, p1: string, p2: string) => { - return `${p1}${getId(p2)}"`; - }); - } - } - }; - return new Promise(resolve => { - form.parse(req, async (_err, fields, files) => { - remap = fields.remap !== "false"; - let id: string = ""; - try { - for (const name in files) { - const path_2 = files[name].path; - const zip = new AdmZip(path_2); - zip.getEntries().forEach((entry: any) => { - if (!entry.entryName.startsWith("files/")) return; - let dirname = path.dirname(entry.entryName) + "/"; - let extname = path.extname(entry.entryName); - let basename = path.basename(entry.entryName).split(".")[0]; - // zip.extractEntryTo(dirname + basename + "_o" + extname, __dirname + RouteStore.public, true, false); - // zip.extractEntryTo(dirname + basename + "_s" + extname, __dirname + RouteStore.public, true, false); - // zip.extractEntryTo(dirname + basename + "_m" + extname, __dirname + RouteStore.public, true, false); - // zip.extractEntryTo(dirname + basename + "_l" + extname, __dirname + RouteStore.public, true, false); - try { - zip.extractEntryTo(entry.entryName, __dirname + RouteStore.public, true, false); - dirname = "/" + dirname; - - fs.createReadStream(__dirname + RouteStore.public + dirname + basename + extname).pipe(fs.createWriteStream(__dirname + RouteStore.public + dirname + basename + "_o" + extname)); - fs.createReadStream(__dirname + RouteStore.public + dirname + basename + extname).pipe(fs.createWriteStream(__dirname + RouteStore.public + dirname + basename + "_s" + extname)); - fs.createReadStream(__dirname + RouteStore.public + dirname + basename + extname).pipe(fs.createWriteStream(__dirname + RouteStore.public + dirname + basename + "_m" + extname)); - fs.createReadStream(__dirname + RouteStore.public + dirname + basename + extname).pipe(fs.createWriteStream(__dirname + RouteStore.public + dirname + basename + "_l" + extname)); - } catch (e) { - console.log(e); - } - }); - const json = zip.getEntry("doc.json"); - let docs: any; - try { - let data = JSON.parse(json.getData().toString("utf8")); - docs = data.docs; - id = data.id; - docs = Object.keys(docs).map(key => docs[key]); - docs.forEach(mapFn); - await Promise.all(docs.map((doc: any) => new Promise(res => Database.Instance.replace(doc.id, doc, (err, r) => { - err && console.log(err); - res(); - }, true, "newDocuments")))); - } catch (e) { console.log(e); } - fs.unlink(path_2, () => { }); - } - if (id) { - res.send(JSON.stringify(getId(id))); - } else { - res.send(JSON.stringify("error")); - } - } catch (e) { console.log(e); } - resolve(); - }); - }); - } - }); - - router.addSupervisedRoute({ - method: Method.GET, - subscription: new RouteSubscriber("/thumbnail").add("filename"), - onValidation: ({ req, res }) => { - let filename = req.params.filename; - let noExt = filename.substring(0, filename.length - ".png".length); - let pagenumber = parseInt(noExt.split('-')[1]); - return new Promise(resolve => { - fs.exists(uploadDirectory + filename, (exists: boolean) => { - console.log(`${uploadDirectory + filename} ${exists ? "exists" : "does not exist"}`); - if (exists) { - let input = fs.createReadStream(uploadDirectory + filename); - probe(input, (err: any, result: any) => { - if (err) { - console.log(err); - console.log(`error on ${filename}`); - return; - } - res.send({ path: "/files/" + filename, width: result.width, height: result.height }); - }); - } - else { - LoadPage(uploadDirectory + filename.substring(0, filename.length - noExt.split('-')[1].length - ".PNG".length - 1) + ".pdf", pagenumber, res); - } - resolve(); - }); - }); - } - }); - - function LoadPage(file: string, pageNumber: number, res: Response) { - console.log(file); - Pdfjs.getDocument(file).promise - .then((pdf: Pdfjs.PDFDocumentProxy) => { - let factory = new NodeCanvasFactory(); - console.log(pageNumber); - pdf.getPage(pageNumber).then((page: Pdfjs.PDFPageProxy) => { - console.log("reading " + page); - let viewport = page.getViewport(1 as any); - let canvasAndContext = factory.create(viewport.width, viewport.height); - let renderContext = { - canvasContext: canvasAndContext.context, - viewport: viewport, - canvasFactory: factory - }; - console.log("read " + pageNumber); - - page.render(renderContext).promise - .then(() => { - console.log("saving " + pageNumber); - let stream = canvasAndContext.canvas.createPNGStream(); - let pngFile = `${file.substring(0, file.length - ".pdf".length)}-${pageNumber}.PNG`; - let out = fs.createWriteStream(pngFile); - stream.pipe(out); - out.on("finish", () => { - console.log(`Success! Saved to ${pngFile}`); - let name = path.basename(pngFile); - res.send({ path: "/files/" + name, width: viewport.width, height: viewport.height }); - }); - }, (reason: string) => { - console.error(reason + ` ${pageNumber}`); - }); - }); - }); - } - /** * Anyone attempting to navigate to localhost at this port will * first have to log in. @@ -357,16 +80,6 @@ function routeSetter(router: RouteManager) { onValidation: ({ res }) => res.redirect(RouteStore.home) }); - router.addSupervisedRoute({ - method: Method.GET, - subscription: RouteStore.getUsers, - onValidation: async ({ res }) => { - const cursor = await Database.Instance.query({}, { email: 1, userDocumentId: 1 }, "users"); - const results = await cursor.toArray(); - res.send(results.map(user => ({ email: user.email, userDocumentId: user.userDocumentId }))); - } - }); - const serve: OnUnauthenticated = ({ req, res }) => { let detector = new mobileDetect(req.headers['user-agent'] || ""); let filename = detector.mobile() !== null ? 'mobile/image.html' : 'index.html'; @@ -387,19 +100,6 @@ function routeSetter(router: RouteManager) { } }); - router.addSupervisedRoute({ - method: Method.GET, - subscription: RouteStore.getUserDocumentId, - onValidation: ({ res, user }) => res.send(user.userDocumentId) - }); - - router.addSupervisedRoute({ - method: Method.GET, - subscription: RouteStore.getCurrUser, - onValidation: ({ res, user }) => res.send(JSON.stringify(user)), - onUnauthenticated: ({ res }) => res.send(JSON.stringify({ id: "__guest__", email: "" })) - }); - const ServicesApiKeyMap = new Map([ ["face", process.env.FACE], ["vision", process.env.VISION], @@ -415,152 +115,6 @@ function routeSetter(router: RouteManager) { } }); - class NodeCanvasFactory { - create = (width: number, height: number) => { - var canvas = createCanvas(width, height); - var context = canvas.getContext('2d'); - return { - canvas, - context - }; - } - - reset = (canvasAndContext: any, width: number, height: number) => { - canvasAndContext.canvas.width = width; - canvasAndContext.canvas.height = height; - } - - destroy = (canvasAndContext: any) => { - canvasAndContext.canvas.width = 0; - canvasAndContext.canvas.height = 0; - canvasAndContext.canvas = null; - canvasAndContext.context = null; - } - } - - interface ImageFileResponse { - name: string; - path: string; - type: string; - exif: Opt; - } - - router.addSupervisedRoute({ - method: Method.POST, - subscription: RouteStore.upload, - onValidation: async ({ req, res }) => { - let form = new formidable.IncomingForm(); - form.uploadDir = uploadDirectory; - form.keepExtensions = true; - return new Promise(resolve => { - form.parse(req, async (_err, _fields, files) => { - let results: ImageFileResponse[] = []; - for (const key in files) { - const { type, path: location, name } = files[key]; - const filename = path.basename(location); - let uploadInformation: Opt; - if (filename.endsWith(".pdf")) { - let dataBuffer = fs.readFileSync(uploadDirectory + filename); - const result: ParsedPDF = await pdf(dataBuffer); - await new Promise(resolve => { - const path = pdfDirectory + "/" + filename.substring(0, filename.length - ".pdf".length) + ".txt"; - fs.createWriteStream(path).write(result.text, error => { - if (!error) { - resolve(); - } else { - reject(error); - } - }); - }); - } else { - uploadInformation = await DashUploadUtils.UploadImage(uploadDirectory + filename, filename); - } - const exif = uploadInformation ? uploadInformation.exifData : undefined; - results.push({ name, type, path: `/files/${filename}`, exif }); - } - _success(res, results); - resolve(); - }); - }); - } - }); - - router.addSupervisedRoute({ - method: Method.POST, - subscription: RouteStore.inspectImage, - onValidation: async ({ req, res }) => { - const { source } = req.body; - if (typeof source === "string") { - const uploadInformation = await DashUploadUtils.UploadImage(source); - return res.send(await DashUploadUtils.InspectImage(uploadInformation.mediaPaths[0])); - } - res.send({}); - } - }); - - router.addSupervisedRoute({ - method: Method.POST, - subscription: RouteStore.dataUriToImage, - onValidation: ({ req, res }) => { - const uri = req.body.uri; - const filename = req.body.name; - if (!uri || !filename) { - res.status(401).send("incorrect parameters specified"); - return; - } - return imageDataUri.outputFile(uri, uploadDirectory + filename).then((savedName: string) => { - const ext = path.extname(savedName); - let resizers = [ - { resizer: sharp().resize(100, undefined, { withoutEnlargement: true }), suffix: "_s" }, - { resizer: sharp().resize(400, undefined, { withoutEnlargement: true }), suffix: "_m" }, - { resizer: sharp().resize(900, undefined, { withoutEnlargement: true }), suffix: "_l" }, - ]; - let isImage = false; - if (pngTypes.includes(ext)) { - resizers.forEach(element => { - element.resizer = element.resizer.png(); - }); - isImage = true; - } else if (jpgTypes.includes(ext)) { - resizers.forEach(element => { - element.resizer = element.resizer.jpeg(); - }); - isImage = true; - } - if (isImage) { - resizers.forEach(resizer => { - fs.createReadStream(savedName).pipe(resizer.resizer).pipe(fs.createWriteStream(uploadDirectory + filename + resizer.suffix + ext)); - }); - } - res.send("/files/" + filename + ext); - }); - } - }); - - router.addSupervisedRoute({ - method: Method.GET, - subscription: RouteStore.delete, - onValidation: async ({ res, isRelease }) => { - if (isRelease) { - return _permission_denied(res, deletionPermissionError); - } - await WebSocket.deleteFields(); - res.redirect(RouteStore.home); - } - }); - - router.addSupervisedRoute({ - method: Method.GET, - subscription: RouteStore.deleteAll, - onValidation: async ({ res, isRelease }) => { - if (isRelease) { - return _permission_denied(res, deletionPermissionError); - } - await WebSocket.deleteAll(); - res.redirect(RouteStore.home); - } - }); - const EndpointHandlerMap = new Map([ ["create", (api, params) => api.create(params)], ["retrieve", (api, params) => api.get(params)], @@ -628,7 +182,7 @@ function routeSetter(router: RouteManager) { let failed: GooglePhotosUploadFailure[] = []; const batched = BatchedArray.from(media, { batchSize: 25 }); - const newMediaItems = await batched.batchedMapPatientInterval( + const newMediaItems = await batched.batchedMapPatientInterval( { magnitude: 100, unit: TimeUnit.Milliseconds }, async (batch, collector, { completedBatches }) => { for (let index = 0; index < batch.length; index++) { @@ -668,31 +222,6 @@ function routeSetter(router: RouteManager) { const downloadError = "Encountered an error while executing downloads."; const requestError = "Unable to execute download: the body's media items were malformed."; - const deletionPermissionError = "Cannot perform specialized delete outside of the development environment!"; - - router.addSupervisedRoute({ - method: Method.GET, - subscription: "/deleteWithAux", - onValidation: async ({ res, isRelease }) => { - if (isRelease) { - return _permission_denied(res, deletionPermissionError); - } - await Database.Auxiliary.DeleteAll(); - res.redirect(RouteStore.delete); - } - }); - - router.addSupervisedRoute({ - method: Method.GET, - subscription: "/deleteWithGoogleCredentials", - onValidation: async ({ res, isRelease }) => { - if (isRelease) { - return _permission_denied(res, deletionPermissionError); - } - await Database.Auxiliary.GoogleAuthenticationToken.DeleteAll(); - res.redirect(RouteStore.delete); - } - }); const UploadError = (count: number) => `Unable to upload ${count} images to Dash's server`; router.addSupervisedRoute({ @@ -726,4 +255,9 @@ function routeSetter(router: RouteManager) { _invalid(res, requestError); } }); -} \ No newline at end of file +} + +(async function start() { + await preliminaryFunctions(); + await initializeServer({ listenAtPort: 1050, routeSetter }); +})(); -- cgit v1.2.3-70-g09d2 From ed0657b6b0ca1566af06038e418feda8a680419d Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Sat, 9 Nov 2019 16:23:15 -0500 Subject: added more partitions --- src/server/index.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/server/index.ts b/src/server/index.ts index 8eb88cf8b..aec301a74 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -26,7 +26,9 @@ import UploadManager from "./ApiManagers/UploadManager"; export const publicDirectory = __dirname + RouteStore.public; export const filesDirectory = publicDirectory + "/files/"; export enum Partitions { - PdfText = "pdf_text" + pdf_text, + images, + videos } /** -- cgit v1.2.3-70-g09d2 From ee4910b1a90284c71ebdaa5fbd9243148ae113f6 Mon Sep 17 00:00:00 2001 From: Mohammad Amoush Date: Sat, 9 Nov 2019 16:29:20 -0500 Subject: initial --- src/server/ApiManagers/GeneralGoogleManager.ts | 48 ++++++++++ src/server/ApiManagers/GooglePhotosManager.ts | 108 ++++++++++++++++++++++ src/server/index.ts | 123 ------------------------- 3 files changed, 156 insertions(+), 123 deletions(-) create mode 100644 src/server/ApiManagers/GeneralGoogleManager.ts create mode 100644 src/server/ApiManagers/GooglePhotosManager.ts (limited to 'src') diff --git a/src/server/ApiManagers/GeneralGoogleManager.ts b/src/server/ApiManagers/GeneralGoogleManager.ts new file mode 100644 index 000000000..cb37b0dce --- /dev/null +++ b/src/server/ApiManagers/GeneralGoogleManager.ts @@ -0,0 +1,48 @@ +import ApiManager, { Registration } from "./ApiManager"; +import { Method, _permission_denied } from "../RouteManager"; +import { uploadDirectory } from ".."; +import { path } from "animejs"; +import { RouteStore } from "../RouteStore"; +import { GoogleApiServerUtils } from "../apis/google/GoogleApiServerUtils"; +import { Database } from "../database"; + +const deletionPermissionError = "Cannot perform specialized delete outside of the development environment!"; + +export default class GeneralGoogleManager extends ApiManager { + + protected initialize(register: Registration): void { + + register({ + method: Method.GET, + subscription: RouteStore.readGoogleAccessToken, + onValidation: async ({ user, res }) => { + const userId = user.id; + const token = await GoogleApiServerUtils.retrieveAccessToken(userId); + if (!token) { + return res.send(GoogleApiServerUtils.generateAuthenticationUrl()); + } + return res.send(token); + } + }); + + register({ + method: Method.POST, + subscription: RouteStore.writeGoogleAccessToken, + onValidation: async ({ user, req, res }) => { + res.send(await GoogleApiServerUtils.processNewUser(user.id, req.body.authenticationCode)); + } + }); + + register({ + method: Method.GET, + subscription: "/deleteWithGoogleCredentials", + onValidation: async ({ res, isRelease }) => { + if (isRelease) { + return _permission_denied(res, deletionPermissionError); + } + await Database.Auxiliary.GoogleAuthenticationToken.DeleteAll(); + res.redirect(RouteStore.delete); + } + }); + } +} \ No newline at end of file diff --git a/src/server/ApiManagers/GooglePhotosManager.ts b/src/server/ApiManagers/GooglePhotosManager.ts new file mode 100644 index 000000000..b5e9caa38 --- /dev/null +++ b/src/server/ApiManagers/GooglePhotosManager.ts @@ -0,0 +1,108 @@ +import ApiManager, { Registration } from "./ApiManager"; +import { Method, _error, _success, _invalid } from "../RouteManager"; +import { uploadDirectory, NewMediaItem } from ".."; +import { path } from "animejs"; +import { RouteStore } from "../RouteStore"; +import { GoogleApiServerUtils } from "../apis/google/GoogleApiServerUtils"; +import { BatchedArray, TimeUnit } from "array-batcher"; +import { GooglePhotosUploadUtils } from "../apis/google/GooglePhotosUploadUtils"; +import { MediaItem } from "../apis/google/SharedTypes"; +import { Opt } from "../../new_fields/Doc"; +import { DashUploadUtils } from "../DashUploadUtils"; +import { Database } from "../database"; +import { prefix } from "@fortawesome/free-solid-svg-icons"; + +const authenticationError = "Unable to authenticate Google credentials before uploading to Google Photos!"; +const mediaError = "Unable to convert all uploaded bytes to media items!"; +const UploadError = (count: number) => `Unable to upload ${count} images to Dash's server`; +const requestError = "Unable to execute download: the body's media items were malformed."; +const downloadError = "Encountered an error while executing downloads."; +interface GooglePhotosUploadFailure { + batch: number; + index: number; + url: string; + reason: string; +} + +export default class GooglePhotosManager extends ApiManager { + + protected initialize(register: Registration): void { + + register({ + method: Method.POST, + subscription: RouteStore.googlePhotosMediaUpload, + onValidation: async ({ user, req, res }) => { + const { media } = req.body; + + const token = await GoogleApiServerUtils.retrieveAccessToken(user.id); + if (!token) { + return _error(res, authenticationError); + } + + let failed: GooglePhotosUploadFailure[] = []; + const batched = BatchedArray.from(media, { batchSize: 25 }); + const newMediaItems = await batched.batchedMapPatientInterval( + { magnitude: 100, unit: TimeUnit.Milliseconds }, + async (batch, collector, { completedBatches }) => { + for (let index = 0; index < batch.length; index++) { + const { url, description } = batch[index]; + const fail = (reason: string) => failed.push({ reason, batch: completedBatches + 1, index, url }); + const uploadToken = await GooglePhotosUploadUtils.DispatchGooglePhotosUpload(token, url).catch(fail); + if (!uploadToken) { + fail(`${path.extname(url)} is not an accepted extension`); + } else { + collector.push({ + description, + simpleMediaItem: { uploadToken } + }); + } + } + } + ); + + const failedCount = failed.length; + if (failedCount) { + console.error(`Unable to upload ${failedCount} image${failedCount === 1 ? "" : "s"} to Google's servers`); + console.log(failed.map(({ reason, batch, index, url }) => `@${batch}.${index}: ${url} failed:\n${reason}`).join('\n\n')); + } + + return GooglePhotosUploadUtils.CreateMediaItems(token, newMediaItems, req.body.album).then( + results => _success(res, { results, failed }), + error => _error(res, mediaError, error) + ); + } + }); + + register({ + method: Method.POST, + subscription: RouteStore.googlePhotosMediaDownload, + onValidation: async ({ req, res }) => { + const contents: { mediaItems: MediaItem[] } = req.body; + let failed = 0; + if (contents) { + const completed: Opt[] = []; + for (let item of contents.mediaItems) { + const { contentSize, ...attributes } = await DashUploadUtils.InspectImage(item.baseUrl); + const found: Opt = await Database.Auxiliary.QueryUploadHistory(contentSize!); + if (!found) { + const upload = await DashUploadUtils.UploadInspectedImage({ contentSize, ...attributes }, item.filename, prefix).catch(error => _error(res, downloadError, error)); + if (upload) { + completed.push(upload); + await Database.Auxiliary.LogUpload(upload); + } else { + failed++; + } + } else { + completed.push(found); + } + } + if (failed) { + return _error(res, UploadError(failed)); + } + return _success(res, completed); + } + _invalid(res, requestError); + } + }); + } +} \ No newline at end of file diff --git a/src/server/index.ts b/src/server/index.ts index 25697e71f..9a5099d0d 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -585,89 +585,11 @@ function routeSetter(router: RouteManager) { } }); - router.addSupervisedRoute({ - method: Method.GET, - subscription: RouteStore.readGoogleAccessToken, - onValidation: async ({ user, res }) => { - const userId = user.id; - const token = await GoogleApiServerUtils.retrieveAccessToken(userId); - if (!token) { - return res.send(GoogleApiServerUtils.generateAuthenticationUrl()); - } - return res.send(token); - } - }); - - router.addSupervisedRoute({ - method: Method.POST, - subscription: RouteStore.writeGoogleAccessToken, - onValidation: async ({ user, req, res }) => { - res.send(await GoogleApiServerUtils.processNewUser(user.id, req.body.authenticationCode)); - } - }); - - const authenticationError = "Unable to authenticate Google credentials before uploading to Google Photos!"; - const mediaError = "Unable to convert all uploaded bytes to media items!"; - interface GooglePhotosUploadFailure { - batch: number; - index: number; - url: string; - reason: string; - } - - router.addSupervisedRoute({ - method: Method.POST, - subscription: RouteStore.googlePhotosMediaUpload, - onValidation: async ({ user, req, res }) => { - const { media } = req.body; - - const token = await GoogleApiServerUtils.retrieveAccessToken(user.id); - if (!token) { - return _error(res, authenticationError); - } - - let failed: GooglePhotosUploadFailure[] = []; - const batched = BatchedArray.from(media, { batchSize: 25 }); - const newMediaItems = await batched.batchedMapPatientInterval( - { magnitude: 100, unit: TimeUnit.Milliseconds }, - async (batch, collector, { completedBatches }) => { - for (let index = 0; index < batch.length; index++) { - const { url, description } = batch[index]; - const fail = (reason: string) => failed.push({ reason, batch: completedBatches + 1, index, url }); - const uploadToken = await GooglePhotosUploadUtils.DispatchGooglePhotosUpload(token, url).catch(fail); - if (!uploadToken) { - fail(`${path.extname(url)} is not an accepted extension`); - } else { - collector.push({ - description, - simpleMediaItem: { uploadToken } - }); - } - } - } - ); - - const failedCount = failed.length; - if (failedCount) { - console.error(`Unable to upload ${failedCount} image${failedCount === 1 ? "" : "s"} to Google's servers`); - console.log(failed.map(({ reason, batch, index, url }) => `@${batch}.${index}: ${url} failed:\n${reason}`).join('\n\n')); - } - - return GooglePhotosUploadUtils.CreateMediaItems(token, newMediaItems, req.body.album).then( - results => _success(res, { results, failed }), - error => _error(res, mediaError, error) - ); - } - }); - interface MediaItem { baseUrl: string; filename: string; } const prefix = "google_photos_"; - - const downloadError = "Encountered an error while executing downloads."; - const requestError = "Unable to execute download: the body's media items were malformed."; const deletionPermissionError = "Cannot perform specialized delete outside of the development environment!"; router.addSupervisedRoute({ @@ -681,49 +603,4 @@ function routeSetter(router: RouteManager) { res.redirect(RouteStore.delete); } }); - - router.addSupervisedRoute({ - method: Method.GET, - subscription: "/deleteWithGoogleCredentials", - onValidation: async ({ res, isRelease }) => { - if (isRelease) { - return _permission_denied(res, deletionPermissionError); - } - await Database.Auxiliary.GoogleAuthenticationToken.DeleteAll(); - res.redirect(RouteStore.delete); - } - }); - - const UploadError = (count: number) => `Unable to upload ${count} images to Dash's server`; - router.addSupervisedRoute({ - method: Method.POST, - subscription: RouteStore.googlePhotosMediaDownload, - onValidation: async ({ req, res }) => { - const contents: { mediaItems: MediaItem[] } = req.body; - let failed = 0; - if (contents) { - const completed: Opt[] = []; - for (let item of contents.mediaItems) { - const { contentSize, ...attributes } = await DashUploadUtils.InspectImage(item.baseUrl); - const found: Opt = await Database.Auxiliary.QueryUploadHistory(contentSize!); - if (!found) { - const upload = await DashUploadUtils.UploadInspectedImage({ contentSize, ...attributes }, item.filename, prefix).catch(error => _error(res, downloadError, error)); - if (upload) { - completed.push(upload); - await Database.Auxiliary.LogUpload(upload); - } else { - failed++; - } - } else { - completed.push(found); - } - } - if (failed) { - return _error(res, UploadError(failed)); - } - return _success(res, completed); - } - _invalid(res, requestError); - } - }); } \ No newline at end of file -- cgit v1.2.3-70-g09d2 From 36ad83493d2bd58dc6fe62df6002789ccc1b06a1 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Sun, 10 Nov 2019 14:56:58 -0500 Subject: no more RouteStore --- src/Utils.ts | 3 +- src/client/apis/GoogleAuthenticationManager.tsx | 5 +-- .../apis/google_docs/GoogleApiClientUtils.ts | 7 ++-- .../apis/google_docs/GooglePhotosClientUtils.ts | 5 +-- src/client/cognitive_services/CognitiveServices.ts | 3 +- src/client/util/History.ts | 3 +- .../util/Import & Export/DirectoryImportBox.tsx | 3 +- src/client/util/Import & Export/ImageUtils.ts | 7 ++-- src/client/util/SharingManager.tsx | 3 +- src/client/views/MainView.tsx | 5 +-- src/client/views/collections/CollectionSubView.tsx | 4 +- src/client/views/nodes/ImageBox.tsx | 3 +- src/client/views/nodes/VideoBox.tsx | 3 +- src/client/views/search/SearchBox.tsx | 3 +- src/mobile/ImageUpload.tsx | 3 +- src/new_fields/RichTextUtils.ts | 3 +- src/server/ApiManagers/DeleteManager.ts | 13 +++---- src/server/ApiManagers/ExportManager.ts | 5 +-- src/server/ApiManagers/PDFManager.ts | 2 +- src/server/ApiManagers/UploadManager.ts | 15 +++----- src/server/ApiManagers/UserManager.ts | 7 ++-- src/server/ApiManagers/UtilManager.ts | 7 ++++ src/server/Initialization.ts | 28 +++++++------- src/server/RouteManager.ts | 5 +-- src/server/RouteStore.ts | 45 ---------------------- src/server/RouteSubscriber.ts | 2 +- src/server/authentication/config/passport.ts | 5 +-- .../authentication/controllers/user_controller.ts | 32 +++++++-------- .../authentication/models/current_user_utils.ts | 7 ++-- src/server/index.ts | 26 ++++++------- 30 files changed, 96 insertions(+), 166 deletions(-) delete mode 100644 src/server/RouteStore.ts (limited to 'src') diff --git a/src/Utils.ts b/src/Utils.ts index 9a2f01f80..abff2eaba 100644 --- a/src/Utils.ts +++ b/src/Utils.ts @@ -2,7 +2,6 @@ import v4 = require('uuid/v4'); import v5 = require("uuid/v5"); import { Socket } from 'socket.io'; import { Message } from './server/Message'; -import { RouteStore } from './server/RouteStore'; export namespace Utils { @@ -46,7 +45,7 @@ export namespace Utils { } export function CorsProxy(url: string): string { - return prepend(RouteStore.corsProxy + "/") + encodeURIComponent(url); + return prepend("/corsProxy/") + encodeURIComponent(url); } export function CopyText(text: string) { diff --git a/src/client/apis/GoogleAuthenticationManager.tsx b/src/client/apis/GoogleAuthenticationManager.tsx index 1ec9d8412..ae77c4b7b 100644 --- a/src/client/apis/GoogleAuthenticationManager.tsx +++ b/src/client/apis/GoogleAuthenticationManager.tsx @@ -4,7 +4,6 @@ import * as React from "react"; import MainViewModal from "../views/MainViewModal"; import { Opt } from "../../new_fields/Doc"; import { Networking } from "../Network"; -import { RouteStore } from "../../server/RouteStore"; import "./GoogleAuthenticationManager.scss"; const AuthenticationUrl = "https://accounts.google.com/o/oauth2/v2/auth"; @@ -31,7 +30,7 @@ export default class GoogleAuthenticationManager extends React.Component<{}> { } public fetchOrGenerateAccessToken = async () => { - let response = await Networking.FetchFromServer(RouteStore.readGoogleAccessToken); + let response = await Networking.FetchFromServer("/readGoogleAccessToken"); // if this is an authentication url, activate the UI to register the new access token if (new RegExp(AuthenticationUrl).test(response)) { this.isOpen = true; @@ -44,7 +43,7 @@ export default class GoogleAuthenticationManager extends React.Component<{}> { return; } const { access_token, avatar, name } = await Networking.PostToServer( - RouteStore.writeGoogleAccessToken, + "/writeGoogleAccessToken", { authenticationCode } ); runInAction(() => { diff --git a/src/client/apis/google_docs/GoogleApiClientUtils.ts b/src/client/apis/google_docs/GoogleApiClientUtils.ts index 183679317..26c7f8d2e 100644 --- a/src/client/apis/google_docs/GoogleApiClientUtils.ts +++ b/src/client/apis/google_docs/GoogleApiClientUtils.ts @@ -1,5 +1,4 @@ import { docs_v1, slides_v1 } from "googleapis"; -import { RouteStore } from "../../../server/RouteStore"; import { Opt } from "../../../new_fields/Doc"; import { isArray } from "util"; import { EditorState } from "prosemirror-state"; @@ -77,7 +76,7 @@ export namespace GoogleApiClientUtils { * @returns the documentId of the newly generated document, or undefined if the creation process fails. */ export const create = async (options: CreateOptions): Promise => { - const path = `${RouteStore.googleDocs}/Documents/${Actions.Create}`; + const path = `/googleDocs/Documents/${Actions.Create}`; const parameters = { requestBody: { title: options.title || `Dash Export (${new Date().toDateString()})` @@ -154,7 +153,7 @@ export namespace GoogleApiClientUtils { } export const retrieve = async (options: RetrieveOptions): Promise => { - const path = `${RouteStore.googleDocs}/Documents/${Actions.Retrieve}`; + const path = `/googleDocs/Documents/${Actions.Retrieve}`; try { const parameters = { documentId: options.documentId }; const schema: RetrievalResult = await Networking.PostToServer(path, parameters); @@ -165,7 +164,7 @@ export namespace GoogleApiClientUtils { }; export const update = async (options: UpdateOptions): Promise => { - const path = `${RouteStore.googleDocs}/Documents/${Actions.Update}`; + const path = `/googleDocs/Documents/${Actions.Update}`; const parameters = { documentId: options.documentId, requestBody: { diff --git a/src/client/apis/google_docs/GooglePhotosClientUtils.ts b/src/client/apis/google_docs/GooglePhotosClientUtils.ts index 402fc64b5..bf8897061 100644 --- a/src/client/apis/google_docs/GooglePhotosClientUtils.ts +++ b/src/client/apis/google_docs/GooglePhotosClientUtils.ts @@ -1,5 +1,4 @@ import { Utils } from "../../../Utils"; -import { RouteStore } from "../../../server/RouteStore"; import { ImageField } from "../../../new_fields/URLField"; import { Cast, StrCast } from "../../../new_fields/Types"; import { Doc, Opt, DocListCastAsync } from "../../../new_fields/Doc"; @@ -307,7 +306,7 @@ export namespace GooglePhotos { }; export const WriteMediaItemsToServer = async (body: { mediaItems: any[] }): Promise => { - const uploads = await Networking.PostToServer(RouteStore.googlePhotosMediaDownload, body); + const uploads = await Networking.PostToServer("/googlePhotosMediaDownload", body); return uploads; }; @@ -345,7 +344,7 @@ export namespace GooglePhotos { media.push({ url, description }); } if (media.length) { - const results = await Networking.PostToServer(RouteStore.googlePhotosMediaUpload, { media, album }); + const results = await Networking.PostToServer("/googlePhotosMediaUpload", { media, album }); return results; } }; diff --git a/src/client/cognitive_services/CognitiveServices.ts b/src/client/cognitive_services/CognitiveServices.ts index 08fcb4883..af5fb39fc 100644 --- a/src/client/cognitive_services/CognitiveServices.ts +++ b/src/client/cognitive_services/CognitiveServices.ts @@ -2,7 +2,6 @@ import * as request from "request-promise"; import { Doc, Field, Opt } from "../../new_fields/Doc"; import { Cast } from "../../new_fields/Types"; import { Docs } from "../documents/Documents"; -import { RouteStore } from "../../server/RouteStore"; import { Utils } from "../../Utils"; import { InkData } from "../../new_fields/InkField"; import { UndoManager } from "../util/UndoManager"; @@ -39,7 +38,7 @@ export enum Confidence { export namespace CognitiveServices { const ExecuteQuery = async (service: Service, manager: APIManager, data: D): Promise => { - return fetch(Utils.prepend(`${RouteStore.cognitiveServices}/${service}`)).then(async response => { + return fetch(Utils.prepend(`cognitiveServices/${service}`)).then(async response => { let apiKey = await response.text(); if (!apiKey) { console.log(`No API key found for ${service}: ensure index.ts has access to a .env file in your root directory`); diff --git a/src/client/util/History.ts b/src/client/util/History.ts index 899abbe40..1c51236cb 100644 --- a/src/client/util/History.ts +++ b/src/client/util/History.ts @@ -1,6 +1,5 @@ import { Doc, Opt, Field } from "../../new_fields/Doc"; import { DocServer } from "../DocServer"; -import { RouteStore } from "../../server/RouteStore"; import { MainView } from "../views/MainView"; import * as qs from 'query-string'; import { Utils, OmitKeys } from "../../Utils"; @@ -26,7 +25,7 @@ export namespace HistoryUtil { // const handlers: ((state: ParsedUrl | null) => void)[] = []; function onHistory(e: PopStateEvent) { - if (window.location.pathname !== RouteStore.home) { + if (window.location.pathname !== "/home") { const url = e.state as ParsedUrl || parseUrl(window.location); if (url) { switch (url.type) { diff --git a/src/client/util/Import & Export/DirectoryImportBox.tsx b/src/client/util/Import & Export/DirectoryImportBox.tsx index 2e0ba25eb..437e7766b 100644 --- a/src/client/util/Import & Export/DirectoryImportBox.tsx +++ b/src/client/util/Import & Export/DirectoryImportBox.tsx @@ -1,7 +1,6 @@ import "fs"; import React = require("react"); import { Doc, DocListCast, DocListCastAsync, Opt } from "../../../new_fields/Doc"; -import { RouteStore } from "../../../server/RouteStore"; import { action, observable, autorun, runInAction, computed, reaction, IReactionDisposer } from "mobx"; import { FieldViewProps, FieldView } from "../../views/nodes/FieldView"; import Measure, { ContentRect } from "react-measure"; @@ -124,7 +123,7 @@ export default class DirectoryImportBox extends React.Component formData.append(Utils.GenerateGuid(), file); }); - collector.push(...(await Networking.PostFormDataToServer(RouteStore.upload, formData))); + collector.push(...(await Networking.PostFormDataToServer("/upload", formData))); runInAction(() => this.completed += batch.length); }); diff --git a/src/client/util/Import & Export/ImageUtils.ts b/src/client/util/Import & Export/ImageUtils.ts index 914f4870a..ca80f3bca 100644 --- a/src/client/util/Import & Export/ImageUtils.ts +++ b/src/client/util/Import & Export/ImageUtils.ts @@ -1,7 +1,6 @@ -import { Doc, DocListCast, DocListCastAsync, Opt } from "../../../new_fields/Doc"; +import { Doc } from "../../../new_fields/Doc"; import { ImageField } from "../../../new_fields/URLField"; import { Cast, StrCast } from "../../../new_fields/Types"; -import { RouteStore } from "../../../server/RouteStore"; import { Docs } from "../../documents/Documents"; import { Networking } from "../../Network"; import { Id } from "../../../new_fields/FieldSymbols"; @@ -15,7 +14,7 @@ export namespace ImageUtils { return false; } const source = field.url.href; - const response = await Networking.PostToServer(RouteStore.inspectImage, { source }); + const response = await Networking.PostToServer("/inspectImage", { source }); const { error, data } = response.exifData; document.exif = error || Docs.Get.DocumentHierarchyFromJson(data); return data !== undefined; @@ -23,7 +22,7 @@ export namespace ImageUtils { export const ExportHierarchyToFileSystem = async (collection: Doc): Promise => { const a = document.createElement("a"); - a.href = Utils.prepend(`${RouteStore.imageHierarchyExport}/${collection[Id]}`); + a.href = Utils.prepend(`imageHierarchyExport/${collection[Id]}`); a.download = `Dash Export [${StrCast(collection.title)}].zip`; a.click(); }; diff --git a/src/client/util/SharingManager.tsx b/src/client/util/SharingManager.tsx index 2082d6324..cc1d628b1 100644 --- a/src/client/util/SharingManager.tsx +++ b/src/client/util/SharingManager.tsx @@ -4,7 +4,6 @@ import MainViewModal from "../views/MainViewModal"; import { Doc, Opt, DocCastAsync } from "../../new_fields/Doc"; import { DocServer } from "../DocServer"; import { Cast, StrCast } from "../../new_fields/Types"; -import { RouteStore } from "../../server/RouteStore"; import * as RequestPromise from "request-promise"; import { Utils } from "../../Utils"; import "./SharingManager.scss"; @@ -104,7 +103,7 @@ export default class SharingManager extends React.Component<{}> { } populateUsers = async () => { - let userList = await RequestPromise.get(Utils.prepend(RouteStore.getUsers)); + let userList = await RequestPromise.get(Utils.prepend("/getUsers")); const raw = JSON.parse(userList) as User[]; const evaluating = raw.map(async user => { let isCandidate = user.email !== Doc.CurrentUserEmail; diff --git a/src/client/views/MainView.tsx b/src/client/views/MainView.tsx index 39585113b..0c5a1003b 100644 --- a/src/client/views/MainView.tsx +++ b/src/client/views/MainView.tsx @@ -12,7 +12,6 @@ import { List } from '../../new_fields/List'; import { listSpec } from '../../new_fields/Schema'; import { Cast, FieldValue, StrCast } from '../../new_fields/Types'; import { CurrentUserUtils } from '../../server/authentication/models/current_user_utils'; -import { RouteStore } from '../../server/RouteStore'; import { emptyFunction, returnEmptyString, returnFalse, returnOne, returnTrue, Utils } from '../../Utils'; import GoogleAuthenticationManager from '../apis/GoogleAuthenticationManager'; import { DocServer } from '../DocServer'; @@ -74,7 +73,7 @@ export class MainView extends React.Component { this._urlState = HistoryUtil.parseUrl(window.location) || {} as any; // causes errors to be generated when modifying an observable outside of an action configure({ enforceActions: "observed" }); - if (window.location.pathname !== RouteStore.home) { + if (window.location.pathname !== "/home") { let pathname = window.location.pathname.substr(1).split("/"); if (pathname.length > 1) { let type = pathname[0]; @@ -395,7 +394,7 @@ export class MainView extends React.Component { zoomToScale={emptyFunction} getScale={returnOne}> - ; diff --git a/src/client/views/collections/CollectionSubView.tsx b/src/client/views/collections/CollectionSubView.tsx index 6e8e4fa12..306f8e052 100644 --- a/src/client/views/collections/CollectionSubView.tsx +++ b/src/client/views/collections/CollectionSubView.tsx @@ -8,7 +8,6 @@ import { listSpec } from "../../../new_fields/Schema"; import { ScriptField } from "../../../new_fields/ScriptField"; import { Cast } from "../../../new_fields/Types"; import { CurrentUserUtils } from "../../../server/authentication/models/current_user_utils"; -import { RouteStore } from "../../../server/RouteStore"; import { Utils } from "../../../Utils"; import { DocServer } from "../../DocServer"; import { DocumentType } from "../../documents/DocumentTypes"; @@ -243,7 +242,6 @@ export function CollectionSubView(schemaCtor: (doc: Doc) => T) { let promises: Promise[] = []; // tslint:disable-next-line:prefer-for-of for (let i = 0; i < e.dataTransfer.items.length; i++) { - const upload = window.location.origin + RouteStore.upload; let item = e.dataTransfer.items[i]; if (item.kind === "string" && item.type.indexOf("uri") !== -1) { let str: string; @@ -268,7 +266,7 @@ export function CollectionSubView(schemaCtor: (doc: Doc) => T) { } let dropFileName = file ? file.name : "-empty-"; - let prom = fetch(upload, { + let prom = fetch(Utils.prepend("/upload"), { method: 'POST', body: formData }).then(async (res: Response) => { diff --git a/src/client/views/nodes/ImageBox.tsx b/src/client/views/nodes/ImageBox.tsx index 9f39eccea..07fd832be 100644 --- a/src/client/views/nodes/ImageBox.tsx +++ b/src/client/views/nodes/ImageBox.tsx @@ -12,7 +12,6 @@ import { createSchema, listSpec, makeInterface } from '../../../new_fields/Schem import { ComputedField } from '../../../new_fields/ScriptField'; import { BoolCast, Cast, FieldValue, NumCast, StrCast } from '../../../new_fields/Types'; import { AudioField, ImageField } from '../../../new_fields/URLField'; -import { RouteStore } from '../../../server/RouteStore'; import { Utils, returnOne, emptyFunction } from '../../../Utils'; import { CognitiveServices, Confidence, Service, Tag } from '../../cognitive_services/CognitiveServices'; import { Docs } from '../../documents/Documents'; @@ -152,7 +151,7 @@ export class ImageBox extends DocAnnotatableComponent { if (isRelease) { return _permission_denied(res, deletionPermissionError); } await WebSocket.deleteFields(); - res.redirect(RouteStore.home); + res.redirect("/home"); } }); register({ method: Method.GET, - subscription: RouteStore.deleteAll, + subscription: "/deleteAll", onValidation: async ({ res, isRelease }) => { if (isRelease) { return _permission_denied(res, deletionPermissionError); } await WebSocket.deleteAll(); - res.redirect(RouteStore.home); + res.redirect("/home"); } }); @@ -41,7 +40,7 @@ export default class DeleteManager extends ApiManager { return _permission_denied(res, deletionPermissionError); } await Database.Auxiliary.DeleteAll(); - res.redirect(RouteStore.delete); + res.redirect("/delete"); } }); @@ -53,7 +52,7 @@ export default class DeleteManager extends ApiManager { return _permission_denied(res, deletionPermissionError); } await Database.Auxiliary.GoogleAuthenticationToken.DeleteAll(); - res.redirect(RouteStore.delete); + res.redirect("/delete"); } }); diff --git a/src/server/ApiManagers/ExportManager.ts b/src/server/ApiManagers/ExportManager.ts index d42db1056..fc6ba0d22 100644 --- a/src/server/ApiManagers/ExportManager.ts +++ b/src/server/ApiManagers/ExportManager.ts @@ -1,7 +1,6 @@ import ApiManager, { Registration } from "./ApiManager"; import { Method } from "../RouteManager"; import RouteSubscriber from "../RouteSubscriber"; -import { RouteStore } from "../RouteStore"; import * as Archiver from 'archiver'; import * as express from 'express'; import { Database } from "../database"; @@ -32,7 +31,7 @@ export default class DownloadManager extends ApiManager { */ register({ method: Method.GET, - subscription: new RouteSubscriber(RouteStore.imageHierarchyExport).add('docId'), + subscription: new RouteSubscriber("imageHierarchyExport").add('docId'), onValidation: async ({ req, res }) => { const id = req.params.docId; const hierarchy: Hierarchy = {}; @@ -43,7 +42,7 @@ export default class DownloadManager extends ApiManager { register({ method: Method.GET, - subscription: new RouteSubscriber("/downloadId").add("docId"), + subscription: new RouteSubscriber("downloadId").add("docId"), onValidation: async ({ req, res }) => { return BuildAndDispatchZip(res, async zip => { const { id, docs, files } = await getDocs(req.params.docId); diff --git a/src/server/ApiManagers/PDFManager.ts b/src/server/ApiManagers/PDFManager.ts index f328557b4..632b4965a 100644 --- a/src/server/ApiManagers/PDFManager.ts +++ b/src/server/ApiManagers/PDFManager.ts @@ -15,7 +15,7 @@ export default class PDFManager extends ApiManager { register({ method: Method.GET, - subscription: new RouteSubscriber("/thumbnail").add("filename"), + subscription: new RouteSubscriber("thumbnail").add("filename"), onValidation: ({ req, res }) => { let filename = req.params.filename; let noExt = filename.substring(0, filename.length - ".png".length); diff --git a/src/server/ApiManagers/UploadManager.ts b/src/server/ApiManagers/UploadManager.ts index 38635eda5..01abdab54 100644 --- a/src/server/ApiManagers/UploadManager.ts +++ b/src/server/ApiManagers/UploadManager.ts @@ -6,7 +6,6 @@ var AdmZip = require('adm-zip'); import * as path from 'path'; import { createReadStream, createWriteStream, unlink, readFileSync } from "fs"; import { publicDirectory, filesDirectory, Partitions } from ".."; -import { RouteStore } from "../RouteStore"; import { Database } from "../database"; import { DashUploadUtils } from "../DashUploadUtils"; import { Opt } from "../../new_fields/Doc"; @@ -85,12 +84,8 @@ export default class UploadManager extends ApiManager { let dirname = path.dirname(entry.entryName) + "/"; let extname = path.extname(entry.entryName); let basename = path.basename(entry.entryName).split(".")[0]; - // zip.extractEntryTo(dirname + basename + "_o" + extname, __dirname + RouteStore.public, true, false); - // zip.extractEntryTo(dirname + basename + "_s" + extname, __dirname + RouteStore.public, true, false); - // zip.extractEntryTo(dirname + basename + "_m" + extname, __dirname + RouteStore.public, true, false); - // zip.extractEntryTo(dirname + basename + "_l" + extname, __dirname + RouteStore.public, true, false); try { - zip.extractEntryTo(entry.entryName, __dirname + RouteStore.public, true, false); + zip.extractEntryTo(entry.entryName, publicDirectory, true, false); dirname = "/" + dirname; createReadStream(publicDirectory + dirname + basename + extname).pipe(createWriteStream(publicDirectory + dirname + basename + "_o" + extname)); @@ -131,7 +126,7 @@ export default class UploadManager extends ApiManager { register({ method: Method.POST, - subscription: RouteStore.upload, + subscription: "/upload", onValidation: async ({ req, res }) => { let form = new formidable.IncomingForm(); form.uploadDir = filesDirectory; @@ -147,7 +142,7 @@ export default class UploadManager extends ApiManager { let dataBuffer = readFileSync(filesDirectory + filename); const result: ParsedPDF = await pdf(dataBuffer); await new Promise((resolve, reject) => { - const path = filesDirectory + Partitions.PdfText + "/" + filename.substring(0, filename.length - ".pdf".length) + ".txt"; + const path = filesDirectory + Partitions.pdf_text + "/" + filename.substring(0, filename.length - ".pdf".length) + ".txt"; createWriteStream(path).write(result.text, error => { if (!error) { resolve(); @@ -171,7 +166,7 @@ export default class UploadManager extends ApiManager { register({ method: Method.POST, - subscription: RouteStore.inspectImage, + subscription: "/inspectImage", onValidation: async ({ req, res }) => { const { source } = req.body; if (typeof source === "string") { @@ -184,7 +179,7 @@ export default class UploadManager extends ApiManager { register({ method: Method.POST, - subscription: RouteStore.dataUriToImage, + subscription: "/uploadURI", onValidation: ({ req, res }) => { const uri = req.body.uri; const filename = req.body.name; diff --git a/src/server/ApiManagers/UserManager.ts b/src/server/ApiManagers/UserManager.ts index fe1ce7f2b..51a434fcf 100644 --- a/src/server/ApiManagers/UserManager.ts +++ b/src/server/ApiManagers/UserManager.ts @@ -1,7 +1,6 @@ import ApiManager, { Registration } from "./ApiManager"; import { Method } from "../RouteManager"; import { WebSocket } from "../Websocket/Websocket"; -import { RouteStore } from "../RouteStore"; import { Database } from "../database"; export default class UserManager extends ApiManager { @@ -10,7 +9,7 @@ export default class UserManager extends ApiManager { register({ method: Method.GET, - subscription: RouteStore.getUsers, + subscription: "/getUsers", onValidation: async ({ res }) => { const cursor = await Database.Instance.query({}, { email: 1, userDocumentId: 1 }, "users"); const results = await cursor.toArray(); @@ -20,13 +19,13 @@ export default class UserManager extends ApiManager { register({ method: Method.GET, - subscription: RouteStore.getUserDocumentId, + subscription: "/getUserDocumentId", onValidation: ({ res, user }) => res.send(user.userDocumentId) }); register({ method: Method.GET, - subscription: RouteStore.getCurrUser, + subscription: "/getCurrentUser", onValidation: ({ res, user }) => res.send(JSON.stringify(user)), onUnauthenticated: ({ res }) => res.send(JSON.stringify({ id: "__guest__", email: "" })) }); diff --git a/src/server/ApiManagers/UtilManager.ts b/src/server/ApiManagers/UtilManager.ts index 61cda2e9b..c1234be6c 100644 --- a/src/server/ApiManagers/UtilManager.ts +++ b/src/server/ApiManagers/UtilManager.ts @@ -2,11 +2,18 @@ import ApiManager, { Registration } from "./ApiManager"; import { Method } from "../RouteManager"; import { exec } from 'child_process'; import { command_line } from "../ActionUtilities"; +import RouteSubscriber from "../RouteSubscriber"; export default class UtilManager extends ApiManager { protected initialize(register: Registration): void { + register({ + method: Method.GET, + subscription: new RouteSubscriber("environment").add("key"), + onValidation: ({ req, res }) => res.send(process.env[req.params.key]) + }); + register({ method: Method.GET, subscription: "/pull", diff --git a/src/server/Initialization.ts b/src/server/Initialization.ts index fbb5ae7a6..306058d81 100644 --- a/src/server/Initialization.ts +++ b/src/server/Initialization.ts @@ -9,7 +9,6 @@ import flash = require('connect-flash'); import { Database } from './database'; import { getForgot, getLogin, getLogout, getReset, getSignup, postForgot, postLogin, postReset, postSignup } from './authentication/controllers/user_controller'; const MongoStore = require('connect-mongo')(session); -import { RouteStore } from './RouteStore'; import RouteManager from './RouteManager'; import * as webpack from 'webpack'; const config = require('../../webpack.config'); @@ -18,6 +17,8 @@ import * as wdm from 'webpack-dev-middleware'; import * as whm from 'webpack-hot-middleware'; import * as fs from 'fs'; import * as request from 'request'; +import RouteSubscriber from './RouteSubscriber'; +import { publicDirectory } from '.'; export type RouteSetter = (server: RouteManager) => void; export interface InitializationOptions { @@ -29,8 +30,8 @@ export default async function InitializeServer(options: InitializationOptions) { const { listenAtPort, routeSetter } = options; const server = buildWithMiddleware(express()); - server.use(express.static(__dirname + RouteStore.public)); - server.use(RouteStore.images, express.static(__dirname + RouteStore.public)); + server.use(express.static(publicDirectory)); + server.use("/images", express.static(publicDirectory)); server.use(wdm(compiler, { publicPath: config.output.publicPath })); server.use(whm(compiler)); @@ -87,24 +88,25 @@ function determineEnvironment() { } function registerAuthenticationRoutes(server: express.Express) { - server.get(RouteStore.signup, getSignup); - server.post(RouteStore.signup, postSignup); + server.get("/signup", getSignup); + server.post("/signup", postSignup); - server.get(RouteStore.login, getLogin); - server.post(RouteStore.login, postLogin); + server.get("/login", getLogin); + server.post("/login", postLogin); - server.get(RouteStore.logout, getLogout); + server.get("/logout", getLogout); - server.get(RouteStore.forgot, getForgot); - server.post(RouteStore.forgot, postForgot); + server.get("/forgotPassword", getForgot); + server.post("/forgotPassword", postForgot); - server.get(RouteStore.reset, getReset); - server.post(RouteStore.reset, postReset); + const reset = new RouteSubscriber("resetPassword").add("token").build; + server.get(reset, getReset); + server.post(reset, postReset); } function registerCorsProxy(server: express.Express) { const headerCharRegex = /[^\t\x20-\x7e\x80-\xff]/; - server.use(RouteStore.corsProxy, (req, res) => { + server.use("/corsProxy", (req, res) => { req.pipe(request(decodeURIComponent(req.url.substring(1)))).on("response", res => { const headers = Object.keys(res.headers); headers.forEach(headerName => { diff --git a/src/server/RouteManager.ts b/src/server/RouteManager.ts index c1d38327f..3aae5734a 100644 --- a/src/server/RouteManager.ts +++ b/src/server/RouteManager.ts @@ -1,5 +1,4 @@ import RouteSubscriber from "./RouteSubscriber"; -import { RouteStore } from "./RouteStore"; import { DashUserModel } from "./authentication/models/user_model"; import * as express from 'express'; @@ -67,10 +66,10 @@ export default class RouteManager { if (onUnauthenticated) { await tryExecute(onUnauthenticated, core); if (!res.headersSent) { - res.redirect(RouteStore.login); + res.redirect("/login"); } } else { - res.redirect(RouteStore.login); + res.redirect("/login"); } } setTimeout(() => { diff --git a/src/server/RouteStore.ts b/src/server/RouteStore.ts deleted file mode 100644 index a310d0c95..000000000 --- a/src/server/RouteStore.ts +++ /dev/null @@ -1,45 +0,0 @@ -// PREPEND ALL ROUTES WITH FORWARD SLASHES! - -export enum RouteStore { - // GENERAL - root = "/", - home = "/home", - corsProxy = "/corsProxy", - delete = "/delete", - deleteAll = "/deleteAll", - pull = "/pull", - - // UPLOAD AND STATIC FILE SERVING - public = "/public", - upload = "/upload", - dataUriToImage = "/uploadURI", - images = "/images", - inspectImage = "/inspectImage", - imageHierarchyExport = "/imageHierarchyExport", - - // USER AND WORKSPACES - getCurrUser = "/getCurrentUser", - getUsers = "/getUsers", - getUserDocumentId = "/getUserDocumentId", - updateCursor = "/updateCursor", - - openDocumentWithId = "/doc/:docId", - - // AUTHENTICATION - signup = "/signup", - login = "/login", - logout = "/logout", - forgot = "/forgotpassword", - reset = "/reset/:token", - - // APIS - cognitiveServices = "/cognitiveservices", - googleDocs = "/googleDocs", - readGoogleAccessToken = "/readGoogleAccessToken", - writeGoogleAccessToken = "/writeGoogleAccessToken", - googlePhotosMediaUpload = "/googlePhotosMediaUpload", - googlePhotosMediaDownload = "/googlePhotosMediaDownload", - googleDocsGet = "/googleDocsGet", - checkGoogle = "/checkGoogleAuthentication" - -} \ No newline at end of file diff --git a/src/server/RouteSubscriber.ts b/src/server/RouteSubscriber.ts index e49be8af5..a1cf7c1c4 100644 --- a/src/server/RouteSubscriber.ts +++ b/src/server/RouteSubscriber.ts @@ -3,7 +3,7 @@ export default class RouteSubscriber { private requestParameters: string[] = []; constructor(root: string) { - this._root = root; + this._root = `/${root}`; } add(...parameters: string[]) { diff --git a/src/server/authentication/config/passport.ts b/src/server/authentication/config/passport.ts index 8915a4abf..0b15c3a36 100644 --- a/src/server/authentication/config/passport.ts +++ b/src/server/authentication/config/passport.ts @@ -3,7 +3,6 @@ import * as passportLocal from 'passport-local'; import _ from "lodash"; import { default as User } from '../models/user_model'; import { Request, Response, NextFunction } from "express"; -import { RouteStore } from '../../RouteStore'; const LocalStrategy = passportLocal.Strategy; @@ -35,13 +34,13 @@ export let isAuthenticated = (req: Request, res: Response, next: NextFunction) = if (req.isAuthenticated()) { return next(); } - return res.redirect(RouteStore.login); + return res.redirect("/login"); }; export let isAuthorized = (req: Request, res: Response, next: NextFunction) => { const provider = req.path.split("/").slice(-1)[0]; - if (_.find((req.user as any).tokens, { kind: provider })) { + if (_.find((req.user).tokens, { kind: provider })) { next(); } else { res.redirect(`/auth/${provider}`); diff --git a/src/server/authentication/controllers/user_controller.ts b/src/server/authentication/controllers/user_controller.ts index f5c6e1610..b2b9d33f6 100644 --- a/src/server/authentication/controllers/user_controller.ts +++ b/src/server/authentication/controllers/user_controller.ts @@ -10,10 +10,7 @@ import * as pug from 'pug'; import * as async from 'async'; import * as nodemailer from 'nodemailer'; import c = require("crypto"); -import { RouteStore } from "../../RouteStore"; import { Utils } from "../../../Utils"; -import { Schema } from "mongoose"; -import { Opt } from "../../../new_fields/Doc"; import { MailOptions } from "nodemailer/lib/stream-transport"; /** @@ -23,8 +20,7 @@ import { MailOptions } from "nodemailer/lib/stream-transport"; */ export let getSignup = (req: Request, res: Response) => { if (req.user) { - let user = req.user; - return res.redirect(RouteStore.home); + return res.redirect("/home"); } res.render("signup.pug", { title: "Sign Up", @@ -45,7 +41,7 @@ export let postSignup = (req: Request, res: Response, next: NextFunction) => { const errors = req.validationErrors(); if (errors) { - return res.redirect(RouteStore.signup); + return res.redirect("/signup"); } const email = req.body.email as String; @@ -62,7 +58,7 @@ export let postSignup = (req: Request, res: Response, next: NextFunction) => { User.findOne({ email }, (err, existingUser) => { if (err) { return next(err); } if (existingUser) { - return res.redirect(RouteStore.login); + return res.redirect("/login"); } user.save((err: any) => { if (err) { return next(err); } @@ -81,7 +77,7 @@ let tryRedirectToTarget = (req: Request, res: Response) => { req.session.target = undefined; res.redirect(target); } else { - res.redirect(RouteStore.home); + res.redirect("/home"); } }; @@ -93,7 +89,7 @@ let tryRedirectToTarget = (req: Request, res: Response) => { export let getLogin = (req: Request, res: Response) => { if (req.user) { req.session!.target = undefined; - return res.redirect(RouteStore.home); + return res.redirect("/home"); } res.render("login.pug", { title: "Log In", @@ -115,13 +111,13 @@ export let postLogin = (req: Request, res: Response, next: NextFunction) => { if (errors) { req.flash("errors", "Unable to login at this time. Please try again."); - return res.redirect(RouteStore.signup); + return res.redirect("/signup"); } passport.authenticate("local", (err: Error, user: DashUserModel, info: IVerifyOptions) => { if (err) { next(err); return; } if (!user) { - return res.redirect(RouteStore.signup); + return res.redirect("/signup"); } req.logIn(user, (err) => { if (err) { next(err); return; } @@ -141,7 +137,7 @@ export let getLogout = (req: Request, res: Response) => { if (sess) { sess.destroy((err) => { if (err) { console.log(err); } }); } - res.redirect(RouteStore.login); + res.redirect("/login"); }; export let getForgot = function (req: Request, res: Response) { @@ -168,7 +164,7 @@ export let postForgot = function (req: Request, res: Response, next: NextFunctio User.findOne({ email }, function (err, user: DashUserModel) { if (!user) { // NO ACCOUNT WITH SUBMITTED EMAIL - res.redirect(RouteStore.forgot); + res.redirect("/forgotPassword"); return; } user.passwordResetToken = token; @@ -192,7 +188,7 @@ export let postForgot = function (req: Request, res: Response, next: NextFunctio subject: 'Dash Password Reset', text: 'You are receiving this because you (or someone else) have requested the reset of the password for your account.\n\n' + 'Please click on the following link, or paste this into your browser to complete the process:\n\n' + - 'http://' + req.headers.host + '/reset/' + token + '\n\n' + + 'http://' + req.headers.host + '/resetPassword/' + token + '\n\n' + 'If you did not request this, please ignore this email and your password will remain unchanged.\n' } as MailOptions; smtpTransport.sendMail(mailOptions, function (err: Error | null) { @@ -202,14 +198,14 @@ export let postForgot = function (req: Request, res: Response, next: NextFunctio } ], function (err) { if (err) return next(err); - res.redirect(RouteStore.forgot); + res.redirect("/forgotPassword"); }); }; export let getReset = function (req: Request, res: Response) { User.findOne({ passwordResetToken: req.params.token, passwordResetExpires: { $gt: Date.now() } }, function (err, user: DashUserModel) { if (!user || err) { - return res.redirect(RouteStore.forgot); + return res.redirect("/forgotPassword"); } res.render("reset.pug", { title: "Reset Password", @@ -239,7 +235,7 @@ export let postReset = function (req: Request, res: Response) { user.save(function (err) { if (err) { - res.redirect(RouteStore.login); + res.redirect("/login"); return; } req.logIn(user, function (err) { @@ -271,6 +267,6 @@ export let postReset = function (req: Request, res: Response) { }); } ], function (err) { - res.redirect(RouteStore.login); + res.redirect("/login"); }); }; \ No newline at end of file diff --git a/src/server/authentication/models/current_user_utils.ts b/src/server/authentication/models/current_user_utils.ts index 73cac879e..5a8815983 100644 --- a/src/server/authentication/models/current_user_utils.ts +++ b/src/server/authentication/models/current_user_utils.ts @@ -11,7 +11,6 @@ import { List } from "../../../new_fields/List"; import { listSpec } from "../../../new_fields/Schema"; import { Cast, StrCast, PromiseValue } from "../../../new_fields/Types"; import { Utils } from "../../../Utils"; -import { RouteStore } from "../../RouteStore"; import { ScriptField } from "../../../new_fields/ScriptField"; import { ButtonBox } from "../../../client/views/nodes/ButtonBox"; import { UndoManager } from "../../../client/util/UndoManager"; @@ -198,8 +197,8 @@ export class CurrentUserUtils { return doc; } - public static loadCurrentUser() { - return rp.get(Utils.prepend(RouteStore.getCurrUser)).then(response => { + public static async loadCurrentUser() { + return rp.get(Utils.prepend("/getCurrentUser")).then(response => { if (response) { const result: { id: string, email: string } = JSON.parse(response); return result; @@ -212,7 +211,7 @@ export class CurrentUserUtils { public static async loadUserDocument({ id, email }: { id: string, email: string }) { this.curr_id = id; Doc.CurrentUserEmail = email; - await rp.get(Utils.prepend(RouteStore.getUserDocumentId)).then(id => { + await rp.get(Utils.prepend("/getUserDocumentId")).then(id => { if (id && id !== "guest") { return DocServer.GetRefField(id).then(async field => { if (field instanceof Doc) { diff --git a/src/server/index.ts b/src/server/index.ts index aec301a74..8fc402cc9 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -3,7 +3,6 @@ import { GoogleApiServerUtils } from "./apis/google/GoogleApiServerUtils"; import * as mobileDetect from 'mobile-detect'; import * as path from 'path'; import { Database } from './database'; -import { RouteStore } from './RouteStore'; const serverPort = 4321; import { GooglePhotosUploadUtils } from './apis/google/GooglePhotosUploadUtils'; import { Opt } from '../new_fields/Doc'; @@ -23,7 +22,7 @@ import DeleteManager from "./ApiManagers/DeleteManager"; import PDFManager from "./ApiManagers/PDFManager"; import UploadManager from "./ApiManagers/UploadManager"; -export const publicDirectory = __dirname + RouteStore.public; +export const publicDirectory = __dirname + "/public"; export const filesDirectory = publicDirectory + "/files/"; export enum Partitions { pdf_text, @@ -73,13 +72,12 @@ function routeSetter(router: RouteManager) { WebSocket.initialize(serverPort, router.isRelease); /** - * Anyone attempting to navigate to localhost at this port will - * first have to log in. + * Accessing root index redirects to home */ router.addSupervisedRoute({ method: Method.GET, - subscription: RouteStore.root, - onValidation: ({ res }) => res.redirect(RouteStore.home) + subscription: "/", + onValidation: ({ res }) => res.redirect("/home") }); const serve: OnUnauthenticated = ({ req, res }) => { @@ -90,7 +88,7 @@ function routeSetter(router: RouteManager) { router.addSupervisedRoute({ method: Method.GET, - subscription: [RouteStore.home, new RouteSubscriber("/doc").add("docId")], + subscription: ["/home", new RouteSubscriber("doc").add("docId")], onValidation: serve, onUnauthenticated: ({ req, ...remaining }) => { const { originalUrl: target } = req; @@ -110,9 +108,9 @@ function routeSetter(router: RouteManager) { router.addSupervisedRoute({ method: Method.GET, - subscription: new RouteSubscriber(RouteStore.cognitiveServices).add('requestedservice'), + subscription: new RouteSubscriber("cognitiveServices").add('requestedService'), onValidation: ({ req, res }) => { - let service = req.params.requestedservice; + let service = req.params.requestedService; res.send(ServicesApiKeyMap.get(service)); } }); @@ -125,7 +123,7 @@ function routeSetter(router: RouteManager) { router.addSupervisedRoute({ method: Method.POST, - subscription: new RouteSubscriber(RouteStore.googleDocs).add("sector", "action"), + subscription: new RouteSubscriber("googleDocs").add("sector", "action"), onValidation: async ({ req, res, user }) => { let sector: GoogleApiServerUtils.Service = req.params.sector as GoogleApiServerUtils.Service; let action: GoogleApiServerUtils.Action = req.params.action as GoogleApiServerUtils.Action; @@ -143,7 +141,7 @@ function routeSetter(router: RouteManager) { router.addSupervisedRoute({ method: Method.GET, - subscription: RouteStore.readGoogleAccessToken, + subscription: "/readGoogleAccessToken", onValidation: async ({ user, res }) => { const userId = user.id; const token = await GoogleApiServerUtils.retrieveAccessToken(userId); @@ -156,7 +154,7 @@ function routeSetter(router: RouteManager) { router.addSupervisedRoute({ method: Method.POST, - subscription: RouteStore.writeGoogleAccessToken, + subscription: "/writeGoogleAccessToken", onValidation: async ({ user, req, res }) => { res.send(await GoogleApiServerUtils.processNewUser(user.id, req.body.authenticationCode)); } @@ -173,7 +171,7 @@ function routeSetter(router: RouteManager) { router.addSupervisedRoute({ method: Method.POST, - subscription: RouteStore.googlePhotosMediaUpload, + subscription: "/googlePhotosMediaUpload", onValidation: async ({ user, req, res }) => { const { media } = req.body; @@ -228,7 +226,7 @@ function routeSetter(router: RouteManager) { const UploadError = (count: number) => `Unable to upload ${count} images to Dash's server`; router.addSupervisedRoute({ method: Method.POST, - subscription: RouteStore.googlePhotosMediaDownload, + subscription: "/googlePhotosMediaDownload", onValidation: async ({ req, res }) => { const contents: { mediaItems: MediaItem[] } = req.body; let failed = 0; -- cgit v1.2.3-70-g09d2 From 63ea273820eb5ab8388e2744fc7988f10746f25f Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Sun, 10 Nov 2019 14:59:49 -0500 Subject: now using environment variable route --- src/client/cognitive_services/CognitiveServices.ts | 2 +- src/server/index.ts | 15 --------------- 2 files changed, 1 insertion(+), 16 deletions(-) (limited to 'src') diff --git a/src/client/cognitive_services/CognitiveServices.ts b/src/client/cognitive_services/CognitiveServices.ts index af5fb39fc..e74aef998 100644 --- a/src/client/cognitive_services/CognitiveServices.ts +++ b/src/client/cognitive_services/CognitiveServices.ts @@ -38,7 +38,7 @@ export enum Confidence { export namespace CognitiveServices { const ExecuteQuery = async (service: Service, manager: APIManager, data: D): Promise => { - return fetch(Utils.prepend(`cognitiveServices/${service}`)).then(async response => { + return fetch(Utils.prepend(`environment/${service}`)).then(async response => { let apiKey = await response.text(); if (!apiKey) { console.log(`No API key found for ${service}: ensure index.ts has access to a .env file in your root directory`); diff --git a/src/server/index.ts b/src/server/index.ts index 8fc402cc9..d68e9faa1 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -100,21 +100,6 @@ function routeSetter(router: RouteManager) { } }); - const ServicesApiKeyMap = new Map([ - ["face", process.env.FACE], - ["vision", process.env.VISION], - ["handwriting", process.env.HANDWRITING] - ]); - - router.addSupervisedRoute({ - method: Method.GET, - subscription: new RouteSubscriber("cognitiveServices").add('requestedService'), - onValidation: ({ req, res }) => { - let service = req.params.requestedService; - res.send(ServicesApiKeyMap.get(service)); - } - }); - const EndpointHandlerMap = new Map([ ["create", (api, params) => api.create(params)], ["retrieve", (api, params) => api.get(params)], -- cgit v1.2.3-70-g09d2 From 06a9b3477dfef93af3c2715f5512d0d883191b58 Mon Sep 17 00:00:00 2001 From: Mohammad Amoush Date: Tue, 12 Nov 2019 17:26:58 -0500 Subject: fixed everything except for async --- src/server/ApiManagers/GeneralGoogleManager.ts | 47 ++++++++++++++++++++--- src/server/ApiManagers/GooglePhotosManager.ts | 27 +++++++------ src/server/apis/google/GooglePhotosUploadUtils.ts | 2 +- src/server/index.ts | 9 +++-- 4 files changed, 62 insertions(+), 23 deletions(-) (limited to 'src') diff --git a/src/server/ApiManagers/GeneralGoogleManager.ts b/src/server/ApiManagers/GeneralGoogleManager.ts index cb37b0dce..89efebf78 100644 --- a/src/server/ApiManagers/GeneralGoogleManager.ts +++ b/src/server/ApiManagers/GeneralGoogleManager.ts @@ -1,12 +1,20 @@ import ApiManager, { Registration } from "./ApiManager"; import { Method, _permission_denied } from "../RouteManager"; -import { uploadDirectory } from ".."; -import { path } from "animejs"; -import { RouteStore } from "../RouteStore"; import { GoogleApiServerUtils } from "../apis/google/GoogleApiServerUtils"; import { Database } from "../database"; +import RouteSubscriber from "../RouteSubscriber"; const deletionPermissionError = "Cannot perform specialized delete outside of the development environment!"; +const ServicesApiKeyMap = new Map([ + ["face", process.env.FACE], + ["vision", process.env.VISION], + ["handwriting", process.env.HANDWRITING] +]); +const EndpointHandlerMap = new Map([ + ["create", (api, params) => api.create(params)], + ["retrieve", (api, params) => api.get(params)], + ["update", (api, params) => api.batchUpdate(params)], +]); export default class GeneralGoogleManager extends ApiManager { @@ -14,7 +22,7 @@ export default class GeneralGoogleManager extends ApiManager { register({ method: Method.GET, - subscription: RouteStore.readGoogleAccessToken, + subscription: "/readGoogleAccessToken", onValidation: async ({ user, res }) => { const userId = user.id; const token = await GoogleApiServerUtils.retrieveAccessToken(userId); @@ -27,7 +35,7 @@ export default class GeneralGoogleManager extends ApiManager { register({ method: Method.POST, - subscription: RouteStore.writeGoogleAccessToken, + subscription: "/writeGoogleAccessToken", onValidation: async ({ user, req, res }) => { res.send(await GoogleApiServerUtils.processNewUser(user.id, req.body.authenticationCode)); } @@ -41,7 +49,34 @@ export default class GeneralGoogleManager extends ApiManager { return _permission_denied(res, deletionPermissionError); } await Database.Auxiliary.GoogleAuthenticationToken.DeleteAll(); - res.redirect(RouteStore.delete); + res.redirect("/delete"); + } + }); + + register({ + method: Method.GET, + subscription: new RouteSubscriber("/cognitiveServices").add('requestedservice'), + onValidation: ({ req, res }) => { + let service = req.params.requestedservice; + res.send(ServicesApiKeyMap.get(service)); + } + }); + + register({ + method: Method.POST, + subscription: new RouteSubscriber("/googleDocs").add("sector", "action"), + onValidation: async ({ req, res, user }) => { + let sector: GoogleApiServerUtils.Service = req.params.sector as GoogleApiServerUtils.Service; + let action: GoogleApiServerUtils.Action = req.params.action as GoogleApiServerUtils.Action; + const endpoint = await GoogleApiServerUtils.GetEndpoint(GoogleApiServerUtils.Service[sector], user.id); + let handler = EndpointHandlerMap.get(action); + if (endpoint && handler) { + handler(endpoint, req.body) + .then(response => res.send(response.data)) + .catch(exception => res.send(exception)); + return; + } + res.send(undefined); } }); } diff --git a/src/server/ApiManagers/GooglePhotosManager.ts b/src/server/ApiManagers/GooglePhotosManager.ts index b5e9caa38..1f6051c28 100644 --- a/src/server/ApiManagers/GooglePhotosManager.ts +++ b/src/server/ApiManagers/GooglePhotosManager.ts @@ -1,16 +1,12 @@ import ApiManager, { Registration } from "./ApiManager"; import { Method, _error, _success, _invalid } from "../RouteManager"; -import { uploadDirectory, NewMediaItem } from ".."; -import { path } from "animejs"; -import { RouteStore } from "../RouteStore"; +import * as path from "path"; import { GoogleApiServerUtils } from "../apis/google/GoogleApiServerUtils"; import { BatchedArray, TimeUnit } from "array-batcher"; import { GooglePhotosUploadUtils } from "../apis/google/GooglePhotosUploadUtils"; -import { MediaItem } from "../apis/google/SharedTypes"; import { Opt } from "../../new_fields/Doc"; import { DashUploadUtils } from "../DashUploadUtils"; import { Database } from "../database"; -import { prefix } from "@fortawesome/free-solid-svg-icons"; const authenticationError = "Unable to authenticate Google credentials before uploading to Google Photos!"; const mediaError = "Unable to convert all uploaded bytes to media items!"; @@ -23,6 +19,17 @@ interface GooglePhotosUploadFailure { url: string; reason: string; } +interface MediaItem { + baseUrl: string; + filename: string; +} +interface NewMediaItem { + description: string; + simpleMediaItem: { + uploadToken: string; + }; +} +const prefix = "google_photos_"; export default class GooglePhotosManager extends ApiManager { @@ -30,20 +37,18 @@ export default class GooglePhotosManager extends ApiManager { register({ method: Method.POST, - subscription: RouteStore.googlePhotosMediaUpload, + subscription: "/googlePhotosMediaUpload", onValidation: async ({ user, req, res }) => { const { media } = req.body; - const token = await GoogleApiServerUtils.retrieveAccessToken(user.id); if (!token) { return _error(res, authenticationError); } - let failed: GooglePhotosUploadFailure[] = []; const batched = BatchedArray.from(media, { batchSize: 25 }); const newMediaItems = await batched.batchedMapPatientInterval( { magnitude: 100, unit: TimeUnit.Milliseconds }, - async (batch, collector, { completedBatches }) => { + async (batch: any, collector: any, { completedBatches }: any) => { for (let index = 0; index < batch.length; index++) { const { url, description } = batch[index]; const fail = (reason: string) => failed.push({ reason, batch: completedBatches + 1, index, url }); @@ -59,13 +64,11 @@ export default class GooglePhotosManager extends ApiManager { } } ); - const failedCount = failed.length; if (failedCount) { console.error(`Unable to upload ${failedCount} image${failedCount === 1 ? "" : "s"} to Google's servers`); console.log(failed.map(({ reason, batch, index, url }) => `@${batch}.${index}: ${url} failed:\n${reason}`).join('\n\n')); } - return GooglePhotosUploadUtils.CreateMediaItems(token, newMediaItems, req.body.album).then( results => _success(res, { results, failed }), error => _error(res, mediaError, error) @@ -75,7 +78,7 @@ export default class GooglePhotosManager extends ApiManager { register({ method: Method.POST, - subscription: RouteStore.googlePhotosMediaDownload, + subscription: "/googlePhotosMediaDownload", onValidation: async ({ req, res }) => { const contents: { mediaItems: MediaItem[] } = req.body; let failed = 0; diff --git a/src/server/apis/google/GooglePhotosUploadUtils.ts b/src/server/apis/google/GooglePhotosUploadUtils.ts index 0abed3f1d..27532d7f0 100644 --- a/src/server/apis/google/GooglePhotosUploadUtils.ts +++ b/src/server/apis/google/GooglePhotosUploadUtils.ts @@ -122,7 +122,7 @@ export namespace GooglePhotosUploadUtils { // ...so we execute them in delayed batches and await the entire execution return batched.batchedMapPatientInterval( { magnitude: 100, unit: TimeUnit.Milliseconds }, - async (batch: NewMediaItem[], collector) => { + async (batch: NewMediaItem[], collector: any) => { const parameters = { method: 'POST', headers: headers('json', bearerToken), diff --git a/src/server/index.ts b/src/server/index.ts index 59752d6de..773b84403 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -4,10 +4,7 @@ import * as mobileDetect from 'mobile-detect'; import * as path from 'path'; import { Database } from './database'; const serverPort = 4321; -import { GooglePhotosUploadUtils } from './apis/google/GooglePhotosUploadUtils'; -import { Opt } from '../new_fields/Doc'; import { DashUploadUtils } from './DashUploadUtils'; -import { BatchedArray, TimeUnit } from 'array-batcher'; import RouteSubscriber from './RouteSubscriber'; import initializeServer from './Initialization'; import RouteManager, { Method, _success, _permission_denied, _error, _invalid, OnUnauthenticated } from './RouteManager'; @@ -21,6 +18,8 @@ import { GoogleCredentialsLoader } from './credentials/CredentialsLoader'; import DeleteManager from "./ApiManagers/DeleteManager"; import PDFManager from "./ApiManagers/PDFManager"; import UploadManager from "./ApiManagers/UploadManager"; +import GeneralGoogleManager from "./ApiManagers/GeneralGoogleManager"; +import GooglePhotosManager from "./ApiManagers/GooglePhotosManager"; export const publicDirectory = __dirname + "/public"; export const filesDirectory = publicDirectory + "/files/"; @@ -64,7 +63,9 @@ function routeSetter(router: RouteManager) { new SearchManager(), new PDFManager(), new DeleteManager(), - new UtilManager() + new UtilManager(), + new GeneralGoogleManager(), + new GooglePhotosManager(), ].forEach(manager => manager.register(router)); // initialize the web socket (bidirectional communication: if a user changes -- cgit v1.2.3-70-g09d2 From ab285371f6fb2a4f1e64888bafbc84b602f23416 Mon Sep 17 00:00:00 2001 From: Mohammad Amoush Date: Tue, 12 Nov 2019 18:04:41 -0500 Subject: only one more problem... --- src/server/apis/google/GooglePhotosUploadUtils.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/server/apis/google/GooglePhotosUploadUtils.ts b/src/server/apis/google/GooglePhotosUploadUtils.ts index 27532d7f0..3862fedc8 100644 --- a/src/server/apis/google/GooglePhotosUploadUtils.ts +++ b/src/server/apis/google/GooglePhotosUploadUtils.ts @@ -122,7 +122,7 @@ export namespace GooglePhotosUploadUtils { // ...so we execute them in delayed batches and await the entire execution return batched.batchedMapPatientInterval( { magnitude: 100, unit: TimeUnit.Milliseconds }, - async (batch: NewMediaItem[], collector: any) => { + async (batch: NewMediaItem[], collector: any): Promise => { const parameters = { method: 'POST', headers: headers('json', bearerToken), -- cgit v1.2.3-70-g09d2 From 8441282d3d07d90536464e7b084b278ea9957cf8 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Mon, 18 Nov 2019 16:59:26 -0500 Subject: log --- src/server/ActionUtilities.ts | 8 +++++++- src/server/index.ts | 5 +++-- 2 files changed, 10 insertions(+), 3 deletions(-) (limited to 'src') diff --git a/src/server/ActionUtilities.ts b/src/server/ActionUtilities.ts index 9a009791b..b418772a6 100644 --- a/src/server/ActionUtilities.ts +++ b/src/server/ActionUtilities.ts @@ -25,4 +25,10 @@ export const write_text_file = (relativePath: string, contents: any) => { return new Promise((resolve, reject) => { fs.writeFile(target, contents, (err) => err ? reject(err) : resolve()); }); -}; \ No newline at end of file +}; + +export async function log_execution(startMessage: string, endMessage: string, contents: () => void | Promise) { + console.log('\x1b[36m%s\x1b[0m', `${startMessage}...`); + await contents(); + console.log(endMessage); +} \ No newline at end of file diff --git a/src/server/index.ts b/src/server/index.ts index d68e9faa1..fb031a254 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -21,6 +21,7 @@ import { GoogleCredentialsLoader } from './credentials/CredentialsLoader'; import DeleteManager from "./ApiManagers/DeleteManager"; import PDFManager from "./ApiManagers/PDFManager"; import UploadManager from "./ApiManagers/UploadManager"; +import { log_execution } from "./ActionUtilities"; export const publicDirectory = __dirname + "/public"; export const filesDirectory = publicDirectory + "/files/"; @@ -43,7 +44,7 @@ async function preliminaryFunctions() { // divide the public directory based on type await Promise.all(Object.keys(Partitions).map(partition => DashUploadUtils.createIfNotExists(filesDirectory + partition))); // connect to the database - await Database.tryInitializeConnection(); + await log_execution("attempting to initialize database connection", "connected", Database.tryInitializeConnection); } /** @@ -243,6 +244,6 @@ function routeSetter(router: RouteManager) { } (async function start() { - await preliminaryFunctions(); + await log_execution("starting execution of preliminary functions", "completed preliminary functions", preliminaryFunctions); await initializeServer({ listenAtPort: 1050, routeSetter }); })(); -- cgit v1.2.3-70-g09d2 From 00633c834c725bab78cef5bd7b9c4ff2b1449ccf Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Tue, 19 Nov 2019 11:18:08 -0500 Subject: api key route switched to environment, added client side util --- src/Utils.ts | 7 +++++- src/client/cognitive_services/CognitiveServices.ts | 28 ++++++++++------------ src/server/ApiManagers/GeneralGoogleManager.ts | 16 ++----------- src/server/ApiManagers/GooglePhotosManager.ts | 1 + 4 files changed, 22 insertions(+), 30 deletions(-) (limited to 'src') diff --git a/src/Utils.ts b/src/Utils.ts index abff2eaba..91fa459c6 100644 --- a/src/Utils.ts +++ b/src/Utils.ts @@ -48,6 +48,11 @@ export namespace Utils { return prepend("/corsProxy/") + encodeURIComponent(url); } + export async function getApiKey(target: string): Promise { + const response = await fetch(prepend(`environment/${target.toUpperCase()}`)); + return response.text(); + } + export function CopyText(text: string) { var textArea = document.createElement("textarea"); textArea.value = text; @@ -174,7 +179,7 @@ export namespace Utils { } let idString = (message.id || "").padStart(36, ' '); prefix = prefix.padEnd(16, ' '); - console.log(`${prefix}: ${idString}, ${receiving ? 'receiving' : 'sending'} ${messageName} with data ${JSON.stringify(message)}`); + console.log(`${prefix}: ${idString}, ${receiving ? 'receiving' : 'sending'} ${messageName} with data ${JSON.stringify(message)} `); } function loggingCallback(prefix: string, func: (args: any) => any, messageName: string) { diff --git a/src/client/cognitive_services/CognitiveServices.ts b/src/client/cognitive_services/CognitiveServices.ts index e74aef998..5a7f5e991 100644 --- a/src/client/cognitive_services/CognitiveServices.ts +++ b/src/client/cognitive_services/CognitiveServices.ts @@ -38,21 +38,19 @@ export enum Confidence { export namespace CognitiveServices { const ExecuteQuery = async (service: Service, manager: APIManager, data: D): Promise => { - return fetch(Utils.prepend(`environment/${service}`)).then(async response => { - let apiKey = await response.text(); - if (!apiKey) { - console.log(`No API key found for ${service}: ensure index.ts has access to a .env file in your root directory`); - return undefined; - } - - let results: any; - try { - results = await manager.requester(apiKey, manager.converter(data), service).then(json => JSON.parse(json)); - } catch { - results = undefined; - } - return results; - }); + const apiKey = await Utils.getApiKey(service); + if (!apiKey) { + console.log(`No API key found for ${service}: ensure index.ts has access to a .env file in your root directory.`); + return undefined; + } + + let results: any; + try { + results = await manager.requester(apiKey, manager.converter(data), service).then(json => JSON.parse(json)); + } catch { + results = undefined; + } + return results; }; export namespace Image { diff --git a/src/server/ApiManagers/GeneralGoogleManager.ts b/src/server/ApiManagers/GeneralGoogleManager.ts index 89efebf78..171912185 100644 --- a/src/server/ApiManagers/GeneralGoogleManager.ts +++ b/src/server/ApiManagers/GeneralGoogleManager.ts @@ -5,11 +5,7 @@ import { Database } from "../database"; import RouteSubscriber from "../RouteSubscriber"; const deletionPermissionError = "Cannot perform specialized delete outside of the development environment!"; -const ServicesApiKeyMap = new Map([ - ["face", process.env.FACE], - ["vision", process.env.VISION], - ["handwriting", process.env.HANDWRITING] -]); + const EndpointHandlerMap = new Map([ ["create", (api, params) => api.create(params)], ["retrieve", (api, params) => api.get(params)], @@ -53,15 +49,6 @@ export default class GeneralGoogleManager extends ApiManager { } }); - register({ - method: Method.GET, - subscription: new RouteSubscriber("/cognitiveServices").add('requestedservice'), - onValidation: ({ req, res }) => { - let service = req.params.requestedservice; - res.send(ServicesApiKeyMap.get(service)); - } - }); - register({ method: Method.POST, subscription: new RouteSubscriber("/googleDocs").add("sector", "action"), @@ -79,5 +66,6 @@ export default class GeneralGoogleManager extends ApiManager { res.send(undefined); } }); + } } \ No newline at end of file diff --git a/src/server/ApiManagers/GooglePhotosManager.ts b/src/server/ApiManagers/GooglePhotosManager.ts index 1f6051c28..1138dede1 100644 --- a/src/server/ApiManagers/GooglePhotosManager.ts +++ b/src/server/ApiManagers/GooglePhotosManager.ts @@ -107,5 +107,6 @@ export default class GooglePhotosManager extends ApiManager { _invalid(res, requestError); } }); + } } \ No newline at end of file -- cgit v1.2.3-70-g09d2 From 611bb858265b6667f2b7db858d183cea16f273aa Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Tue, 19 Nov 2019 11:23:06 -0500 Subject: rename --- src/server/ApiManagers/DownloadManager.ts | 267 ++++++++++++++++++++++++++++++ src/server/ApiManagers/ExportManager.ts | 267 ------------------------------ src/server/index.ts | 2 +- 3 files changed, 268 insertions(+), 268 deletions(-) create mode 100644 src/server/ApiManagers/DownloadManager.ts delete mode 100644 src/server/ApiManagers/ExportManager.ts (limited to 'src') diff --git a/src/server/ApiManagers/DownloadManager.ts b/src/server/ApiManagers/DownloadManager.ts new file mode 100644 index 000000000..fc6ba0d22 --- /dev/null +++ b/src/server/ApiManagers/DownloadManager.ts @@ -0,0 +1,267 @@ +import ApiManager, { Registration } from "./ApiManager"; +import { Method } from "../RouteManager"; +import RouteSubscriber from "../RouteSubscriber"; +import * as Archiver from 'archiver'; +import * as express from 'express'; +import { Database } from "../database"; +import * as path from "path"; +import { DashUploadUtils } from "../DashUploadUtils"; +import { publicDirectory } from ".."; + +export type Hierarchy = { [id: string]: string | Hierarchy }; +export type ZipMutator = (file: Archiver.Archiver) => void | Promise; +export interface DocumentElements { + data: string | any[]; + title: string; +} + +export default class DownloadManager extends ApiManager { + + protected initialize(register: Registration): void { + + /** + * Let's say someone's using Dash to organize images in collections. + * This lets them export the hierarchy they've built to their + * own file system in a useful format. + * + * This handler starts with a single document id (interesting only + * if it's that of a collection). It traverses the database, captures + * the nesting of only nested images or collections, writes + * that to a zip file and returns it to the client for download. + */ + register({ + method: Method.GET, + subscription: new RouteSubscriber("imageHierarchyExport").add('docId'), + onValidation: async ({ req, res }) => { + const id = req.params.docId; + const hierarchy: Hierarchy = {}; + await buildHierarchyRecursive(id, hierarchy); + return BuildAndDispatchZip(res, zip => writeHierarchyRecursive(zip, hierarchy)); + } + }); + + register({ + method: Method.GET, + subscription: new RouteSubscriber("downloadId").add("docId"), + onValidation: async ({ req, res }) => { + return BuildAndDispatchZip(res, async zip => { + const { id, docs, files } = await getDocs(req.params.docId); + const docString = JSON.stringify({ id, docs }); + zip.append(docString, { name: "doc.json" }); + files.forEach(val => { + zip.file(publicDirectory + val, { name: val.substring(1) }); + }); + }); + } + }); + + register({ + method: Method.GET, + subscription: new RouteSubscriber("/serializeDoc").add("docId"), + onValidation: async ({ req, res }) => { + const { docs, files } = await getDocs(req.params.docId); + res.send({ docs, files: Array.from(files) }); + } + }); + + + } + +} + +async function getDocs(id: string) { + const files = new Set(); + const docs: { [id: string]: any } = {}; + const fn = (doc: any): string[] => { + const id = doc.id; + if (typeof id === "string" && id.endsWith("Proto")) { + //Skip protos + return []; + } + const ids: string[] = []; + for (const key in doc.fields) { + if (!doc.fields.hasOwnProperty(key)) { + continue; + } + const field = doc.fields[key]; + if (field === undefined || field === null) { + continue; + } + + if (field.__type === "proxy" || field.__type === "prefetch_proxy") { + ids.push(field.fieldId); + } else if (field.__type === "script" || field.__type === "computed") { + if (field.captures) { + ids.push(field.captures.fieldId); + } + } else if (field.__type === "list") { + ids.push(...fn(field)); + } else if (typeof field === "string") { + const re = /"(?:dataD|d)ocumentId"\s*:\s*"([\w\-]*)"/g; + let match: string[] | null; + while ((match = re.exec(field)) !== null) { + ids.push(match[1]); + } + } else if (field.__type === "RichTextField") { + const re = /"href"\s*:\s*"(.*?)"/g; + let match: string[] | null; + while ((match = re.exec(field.Data)) !== null) { + const urlString = match[1]; + const split = new URL(urlString).pathname.split("doc/"); + if (split.length > 1) { + ids.push(split[split.length - 1]); + } + } + const re2 = /"src"\s*:\s*"(.*?)"/g; + while ((match = re2.exec(field.Data)) !== null) { + const urlString = match[1]; + const pathname = new URL(urlString).pathname; + files.add(pathname); + } + } else if (["audio", "image", "video", "pdf", "web"].includes(field.__type)) { + const url = new URL(field.url); + const pathname = url.pathname; + files.add(pathname); + } + } + + if (doc.id) { + docs[doc.id] = doc; + } + return ids; + }; + await Database.Instance.visit([id], fn); + return { id, docs, files }; +} + +/** + * This utility function factors out the process + * of creating a zip file and sending it back to the client + * by piping it into a response. + * + * Learn more about piping and readable / writable streams here! + * https://www.freecodecamp.org/news/node-js-streams-everything-you-need-to-know-c9141306be93/ + * + * @param res the writable stream response object that will transfer the generated zip file + * @param mutator the callback function used to actually modify and insert information into the zip instance + */ +export async function BuildAndDispatchZip(res: express.Response, mutator: ZipMutator): Promise { + res.set('Content-disposition', `attachment;`); + res.set('Content-Type', "application/zip"); + const zip = Archiver('zip'); + zip.pipe(res); + await mutator(zip); + return zip.finalize(); +} + +/** + * This function starts with a single document id as a seed, + * typically that of a collection, and then descends the entire tree + * of image or collection documents that are reachable from that seed. + * @param seedId the id of the root of the subtree we're trying to capture, interesting only if it's a collection + * @param hierarchy the data structure we're going to use to record the nesting of the collections and images as we descend + */ + +/* +Below is an example of the JSON hierarchy built from two images contained inside a collection titled 'a nested collection', +following the general recursive structure shown immediately below +{ + "parent folder name":{ + "first child's fild name":"first child's url" + ... + "nth child's fild name":"nth child's url" + } +} +{ + "a nested collection (865c4734-c036-4d67-a588-c71bb43d1440)":{ + "an image of a cat (ace99ffd-8ed8-4026-a5d5-a353fff57bdd).jpg":"https://upload.wikimedia.org/wikipedia/commons/thumb/3/3a/Cat03.jpg/1200px-Cat03.jpg", + "1*SGJw31T5Q9Zfsk24l2yirg.gif (9321cc9b-9b3e-4cb6-b99c-b7e667340f05).gif":"https://cdn-media-1.freecodecamp.org/images/1*SGJw31T5Q9Zfsk24l2yirg.gif" + } +} +*/ +async function buildHierarchyRecursive(seedId: string, hierarchy: Hierarchy): Promise { + const { title, data } = await getData(seedId); + const label = `${title} (${seedId})`; + // is the document a collection? + if (Array.isArray(data)) { + // recurse over all documents in the collection. + const local: Hierarchy = {}; // create a child hierarchy for this level, which will get passed in as the parent of the recursive call + hierarchy[label] = local; // store it at the index in the parent, so we'll end up with a map of maps of maps + await Promise.all(data.map(proxy => buildHierarchyRecursive(proxy.fieldId, local))); + } else { + // now, data can only be a string, namely the url of the image + const filename = label + path.extname(data); // this is the file name under which the output image will be stored + hierarchy[filename] = data; + } +} + +/** + * This is a very specific utility method to help traverse the database + * to parse data and titles out of images and collections alone. + * + * We don't know if the document id given to is corresponds to a view document or a data + * document. If it's a data document, the response from the database will have + * a data field. If not, call recursively on the proto, and resolve with *its* data + * + * @param targetId the id of the Dash document whose data is being requests + * @returns the data of the document, as well as its title + */ +async function getData(targetId: string): Promise { + return new Promise((resolve, reject) => { + Database.Instance.getDocument(targetId, async (result: any) => { + const { data, proto, title } = result.fields; + if (data) { + if (data.url) { + resolve({ data: data.url, title }); + } else if (data.fields) { + resolve({ data: data.fields, title }); + } else { + reject(); + } + } else if (proto) { + getData(proto.fieldId).then(resolve, reject); + } else { + reject(); + } + }); + }); +} + +/** + * + * @param file the zip file to which we write the files + * @param hierarchy the data structure from which we read, defining the nesting of the documents in the zip + * @param prefix lets us create nested folders in the zip file by continually appending to the end + * of the prefix with each layer of recursion. + * + * Function Call #1 => "Dash Export" + * Function Call #2 => "Dash Export/a nested collection" + * Function Call #3 => "Dash Export/a nested collection/lowest level collection" + * ... + */ +async function writeHierarchyRecursive(file: Archiver.Archiver, hierarchy: Hierarchy, prefix = "Dash Export"): Promise { + for (const documentTitle of Object.keys(hierarchy)) { + const result = hierarchy[documentTitle]; + // base case or leaf node, we've hit a url (image) + if (typeof result === "string") { + let path: string; + let matches: RegExpExecArray | null; + if ((matches = /\:1050\/files\/(upload\_[\da-z]{32}.*)/g.exec(result)) !== null) { + // image already exists on our server + path = `${__dirname}/public/files/${matches[1]}`; + } else { + // the image doesn't already exist on our server (may have been dragged + // and dropped in the browser and thus hosted remotely) so we upload it + // to our server and point the zip file to it, so it can bundle up the bytes + const information = await DashUploadUtils.UploadImage(result); + path = information.mediaPaths[0]; + } + // write the file specified by the path to the directory in the + // zip file given by the prefix. + file.file(path, { name: documentTitle, prefix }); + } else { + // we've hit a collection, so we have to recurse + await writeHierarchyRecursive(file, result, `${prefix}/${documentTitle}`); + } + } +} \ No newline at end of file diff --git a/src/server/ApiManagers/ExportManager.ts b/src/server/ApiManagers/ExportManager.ts deleted file mode 100644 index fc6ba0d22..000000000 --- a/src/server/ApiManagers/ExportManager.ts +++ /dev/null @@ -1,267 +0,0 @@ -import ApiManager, { Registration } from "./ApiManager"; -import { Method } from "../RouteManager"; -import RouteSubscriber from "../RouteSubscriber"; -import * as Archiver from 'archiver'; -import * as express from 'express'; -import { Database } from "../database"; -import * as path from "path"; -import { DashUploadUtils } from "../DashUploadUtils"; -import { publicDirectory } from ".."; - -export type Hierarchy = { [id: string]: string | Hierarchy }; -export type ZipMutator = (file: Archiver.Archiver) => void | Promise; -export interface DocumentElements { - data: string | any[]; - title: string; -} - -export default class DownloadManager extends ApiManager { - - protected initialize(register: Registration): void { - - /** - * Let's say someone's using Dash to organize images in collections. - * This lets them export the hierarchy they've built to their - * own file system in a useful format. - * - * This handler starts with a single document id (interesting only - * if it's that of a collection). It traverses the database, captures - * the nesting of only nested images or collections, writes - * that to a zip file and returns it to the client for download. - */ - register({ - method: Method.GET, - subscription: new RouteSubscriber("imageHierarchyExport").add('docId'), - onValidation: async ({ req, res }) => { - const id = req.params.docId; - const hierarchy: Hierarchy = {}; - await buildHierarchyRecursive(id, hierarchy); - return BuildAndDispatchZip(res, zip => writeHierarchyRecursive(zip, hierarchy)); - } - }); - - register({ - method: Method.GET, - subscription: new RouteSubscriber("downloadId").add("docId"), - onValidation: async ({ req, res }) => { - return BuildAndDispatchZip(res, async zip => { - const { id, docs, files } = await getDocs(req.params.docId); - const docString = JSON.stringify({ id, docs }); - zip.append(docString, { name: "doc.json" }); - files.forEach(val => { - zip.file(publicDirectory + val, { name: val.substring(1) }); - }); - }); - } - }); - - register({ - method: Method.GET, - subscription: new RouteSubscriber("/serializeDoc").add("docId"), - onValidation: async ({ req, res }) => { - const { docs, files } = await getDocs(req.params.docId); - res.send({ docs, files: Array.from(files) }); - } - }); - - - } - -} - -async function getDocs(id: string) { - const files = new Set(); - const docs: { [id: string]: any } = {}; - const fn = (doc: any): string[] => { - const id = doc.id; - if (typeof id === "string" && id.endsWith("Proto")) { - //Skip protos - return []; - } - const ids: string[] = []; - for (const key in doc.fields) { - if (!doc.fields.hasOwnProperty(key)) { - continue; - } - const field = doc.fields[key]; - if (field === undefined || field === null) { - continue; - } - - if (field.__type === "proxy" || field.__type === "prefetch_proxy") { - ids.push(field.fieldId); - } else if (field.__type === "script" || field.__type === "computed") { - if (field.captures) { - ids.push(field.captures.fieldId); - } - } else if (field.__type === "list") { - ids.push(...fn(field)); - } else if (typeof field === "string") { - const re = /"(?:dataD|d)ocumentId"\s*:\s*"([\w\-]*)"/g; - let match: string[] | null; - while ((match = re.exec(field)) !== null) { - ids.push(match[1]); - } - } else if (field.__type === "RichTextField") { - const re = /"href"\s*:\s*"(.*?)"/g; - let match: string[] | null; - while ((match = re.exec(field.Data)) !== null) { - const urlString = match[1]; - const split = new URL(urlString).pathname.split("doc/"); - if (split.length > 1) { - ids.push(split[split.length - 1]); - } - } - const re2 = /"src"\s*:\s*"(.*?)"/g; - while ((match = re2.exec(field.Data)) !== null) { - const urlString = match[1]; - const pathname = new URL(urlString).pathname; - files.add(pathname); - } - } else if (["audio", "image", "video", "pdf", "web"].includes(field.__type)) { - const url = new URL(field.url); - const pathname = url.pathname; - files.add(pathname); - } - } - - if (doc.id) { - docs[doc.id] = doc; - } - return ids; - }; - await Database.Instance.visit([id], fn); - return { id, docs, files }; -} - -/** - * This utility function factors out the process - * of creating a zip file and sending it back to the client - * by piping it into a response. - * - * Learn more about piping and readable / writable streams here! - * https://www.freecodecamp.org/news/node-js-streams-everything-you-need-to-know-c9141306be93/ - * - * @param res the writable stream response object that will transfer the generated zip file - * @param mutator the callback function used to actually modify and insert information into the zip instance - */ -export async function BuildAndDispatchZip(res: express.Response, mutator: ZipMutator): Promise { - res.set('Content-disposition', `attachment;`); - res.set('Content-Type', "application/zip"); - const zip = Archiver('zip'); - zip.pipe(res); - await mutator(zip); - return zip.finalize(); -} - -/** - * This function starts with a single document id as a seed, - * typically that of a collection, and then descends the entire tree - * of image or collection documents that are reachable from that seed. - * @param seedId the id of the root of the subtree we're trying to capture, interesting only if it's a collection - * @param hierarchy the data structure we're going to use to record the nesting of the collections and images as we descend - */ - -/* -Below is an example of the JSON hierarchy built from two images contained inside a collection titled 'a nested collection', -following the general recursive structure shown immediately below -{ - "parent folder name":{ - "first child's fild name":"first child's url" - ... - "nth child's fild name":"nth child's url" - } -} -{ - "a nested collection (865c4734-c036-4d67-a588-c71bb43d1440)":{ - "an image of a cat (ace99ffd-8ed8-4026-a5d5-a353fff57bdd).jpg":"https://upload.wikimedia.org/wikipedia/commons/thumb/3/3a/Cat03.jpg/1200px-Cat03.jpg", - "1*SGJw31T5Q9Zfsk24l2yirg.gif (9321cc9b-9b3e-4cb6-b99c-b7e667340f05).gif":"https://cdn-media-1.freecodecamp.org/images/1*SGJw31T5Q9Zfsk24l2yirg.gif" - } -} -*/ -async function buildHierarchyRecursive(seedId: string, hierarchy: Hierarchy): Promise { - const { title, data } = await getData(seedId); - const label = `${title} (${seedId})`; - // is the document a collection? - if (Array.isArray(data)) { - // recurse over all documents in the collection. - const local: Hierarchy = {}; // create a child hierarchy for this level, which will get passed in as the parent of the recursive call - hierarchy[label] = local; // store it at the index in the parent, so we'll end up with a map of maps of maps - await Promise.all(data.map(proxy => buildHierarchyRecursive(proxy.fieldId, local))); - } else { - // now, data can only be a string, namely the url of the image - const filename = label + path.extname(data); // this is the file name under which the output image will be stored - hierarchy[filename] = data; - } -} - -/** - * This is a very specific utility method to help traverse the database - * to parse data and titles out of images and collections alone. - * - * We don't know if the document id given to is corresponds to a view document or a data - * document. If it's a data document, the response from the database will have - * a data field. If not, call recursively on the proto, and resolve with *its* data - * - * @param targetId the id of the Dash document whose data is being requests - * @returns the data of the document, as well as its title - */ -async function getData(targetId: string): Promise { - return new Promise((resolve, reject) => { - Database.Instance.getDocument(targetId, async (result: any) => { - const { data, proto, title } = result.fields; - if (data) { - if (data.url) { - resolve({ data: data.url, title }); - } else if (data.fields) { - resolve({ data: data.fields, title }); - } else { - reject(); - } - } else if (proto) { - getData(proto.fieldId).then(resolve, reject); - } else { - reject(); - } - }); - }); -} - -/** - * - * @param file the zip file to which we write the files - * @param hierarchy the data structure from which we read, defining the nesting of the documents in the zip - * @param prefix lets us create nested folders in the zip file by continually appending to the end - * of the prefix with each layer of recursion. - * - * Function Call #1 => "Dash Export" - * Function Call #2 => "Dash Export/a nested collection" - * Function Call #3 => "Dash Export/a nested collection/lowest level collection" - * ... - */ -async function writeHierarchyRecursive(file: Archiver.Archiver, hierarchy: Hierarchy, prefix = "Dash Export"): Promise { - for (const documentTitle of Object.keys(hierarchy)) { - const result = hierarchy[documentTitle]; - // base case or leaf node, we've hit a url (image) - if (typeof result === "string") { - let path: string; - let matches: RegExpExecArray | null; - if ((matches = /\:1050\/files\/(upload\_[\da-z]{32}.*)/g.exec(result)) !== null) { - // image already exists on our server - path = `${__dirname}/public/files/${matches[1]}`; - } else { - // the image doesn't already exist on our server (may have been dragged - // and dropped in the browser and thus hosted remotely) so we upload it - // to our server and point the zip file to it, so it can bundle up the bytes - const information = await DashUploadUtils.UploadImage(result); - path = information.mediaPaths[0]; - } - // write the file specified by the path to the directory in the - // zip file given by the prefix. - file.file(path, { name: documentTitle, prefix }); - } else { - // we've hit a collection, so we have to recurse - await writeHierarchyRecursive(file, result, `${prefix}/${documentTitle}`); - } - } -} \ No newline at end of file diff --git a/src/server/index.ts b/src/server/index.ts index 569f2e139..57c66bc28 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -13,7 +13,7 @@ import UtilManager from './ApiManagers/UtilManager'; import SearchManager from './ApiManagers/SearchManager'; import UserManager from './ApiManagers/UserManager'; import { WebSocket } from './Websocket/Websocket'; -import DownloadManager from './ApiManagers/ExportManager'; +import DownloadManager from './ApiManagers/DownloadManager'; import { GoogleCredentialsLoader } from './credentials/CredentialsLoader'; import DeleteManager from "./ApiManagers/DeleteManager"; import PDFManager from "./ApiManagers/PDFManager"; -- cgit v1.2.3-70-g09d2 From 025a25a7fbf88c53d153cec19b8f93097116210e Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Tue, 19 Nov 2019 13:30:27 -0500 Subject: fixed authorization --- src/client/views/nodes/AudioBox.tsx | 3 +-- src/server/apis/google/GoogleApiServerUtils.ts | 12 +++++++----- src/server/credentials/google_project_credentials.json | 11 ++++------- 3 files changed, 12 insertions(+), 14 deletions(-) (limited to 'src') diff --git a/src/client/views/nodes/AudioBox.tsx b/src/client/views/nodes/AudioBox.tsx index 86bd23b67..77b10e395 100644 --- a/src/client/views/nodes/AudioBox.tsx +++ b/src/client/views/nodes/AudioBox.tsx @@ -8,7 +8,6 @@ import { DocExtendableComponent } from "../DocComponent"; import { makeInterface, createSchema } from "../../../new_fields/Schema"; import { documentSchema } from "../../../new_fields/documentSchemas"; import { Utils, returnTrue, emptyFunction, returnOne, returnTransparent } from "../../../Utils"; -import { RouteStore } from "../../../server/RouteStore"; import { runInAction, observable, reaction, IReactionDisposer, computed, action } from "mobx"; import { DateField } from "../../../new_fields/DateField"; import { SelectionManager } from "../../util/SelectionManager"; @@ -140,7 +139,7 @@ export class AudioBox extends DocExtendableComponent { let headerParameters = { headers: { 'Content-Type': 'application/x-www-form-urlencoded' } }; + const { client_id, client_secret } = GoogleCredentialsLoader.ProjectCredentials; let url = `https://oauth2.googleapis.com/token?${qs.stringify({ refreshToken: credentials.refresh_token, - grant_type: "refresh_token", - ...installed + client_id, + client_secret, + grant_type: "refresh_token" })}`; const { access_token, expires_in } = await new Promise(async resolve => { const response = await request.post(url, headerParameters); diff --git a/src/server/credentials/google_project_credentials.json b/src/server/credentials/google_project_credentials.json index 5d9c62eb1..955c5a3c1 100644 --- a/src/server/credentials/google_project_credentials.json +++ b/src/server/credentials/google_project_credentials.json @@ -1,14 +1,11 @@ { "installed": { - "client_id": "1005546247619-kqpnvh42mpa803tem8556b87umi4j9r0.apps.googleusercontent.com", - "project_id": "brown-dash", + "client_id": "343179513178-ud6tvmh275r2fq93u9eesrnc66t6akh9.apps.googleusercontent.com", + "project_id": "quickstart-1565056383187", "auth_uri": "https://accounts.google.com/o/oauth2/auth", "token_uri": "https://oauth2.googleapis.com/token", "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", - "client_secret": "WshLb5TH9SdFVGGbQcnYj7IU", - "redirect_uris": [ - "urn:ietf:wg:oauth:2.0:oob", - "http://localhost" - ] + "client_secret": "w8KIFSc0MQpmUYHed4qEzn8b", + "redirect_uris": ["urn:ietf:wg:oauth:2.0:oob", "http://localhost"] } } \ No newline at end of file -- cgit v1.2.3-70-g09d2 From cabb1c986be9a15cfb650fd365283557c1347737 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Tue, 19 Nov 2019 14:28:27 -0500 Subject: commented out traces --- .../views/collections/collectionFreeForm/CollectionFreeFormView.tsx | 2 +- src/client/views/nodes/CollectionFreeFormDocumentView.tsx | 2 +- src/client/views/nodes/DocumentView.tsx | 6 +++--- src/client/views/nodes/FormattedTextBox.tsx | 2 +- src/client/views/nodes/ImageBox.tsx | 2 +- src/client/views/pdf/PDFViewer.tsx | 4 ++-- 6 files changed, 9 insertions(+), 9 deletions(-) (limited to 'src') diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx index 3b313c34a..59858df8d 100644 --- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx +++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx @@ -858,7 +858,7 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) { return eles; } render() { - trace(); + // trace(); // update the actual dimensions of the collection so that they can inquired (e.g., by a minimap) this.Document.fitX = this.contentBounds && this.contentBounds.x; this.Document.fitY = this.contentBounds && this.contentBounds.y; diff --git a/src/client/views/nodes/CollectionFreeFormDocumentView.tsx b/src/client/views/nodes/CollectionFreeFormDocumentView.tsx index d0e1d1922..bbd9859a8 100644 --- a/src/client/views/nodes/CollectionFreeFormDocumentView.tsx +++ b/src/client/views/nodes/CollectionFreeFormDocumentView.tsx @@ -87,7 +87,7 @@ export class CollectionFreeFormDocumentView extends DocComponent this.dataProvider ? this.dataProvider.height : this.panelHeight(); render() { - trace(); + // trace(); return
(Docu @computed get finalLayoutKey() { return this.props.layoutKey || "layout" } childScaling = () => (this.layoutDoc.fitWidth ? this.props.PanelWidth() / this.nativeWidth : this.props.ContentScaling()); @computed get contents() { - trace(); + // trace(); return ((Docu } @computed get innards() { - trace(); + // trace(); const showOverlays = this.props.showOverlays ? this.props.showOverlays(this.Document) : undefined; const showTitle = showOverlays && "title" in showOverlays ? showOverlays.title : this.getLayoutPropStr("showTitle"); const showCaption = showOverlays && "caption" in showOverlays ? showOverlays.caption : this.getLayoutPropStr("showCaption"); @@ -640,7 +640,7 @@ export class DocumentView extends DocComponent(Docu } render() { if (!this.props.Document) return (null); - trace(); + // trace(); const ruleColor = this.props.ruleProvider ? StrCast(this.props.ruleProvider["ruleColor_" + this.Document.heading]) : undefined; const ruleRounding = this.props.ruleProvider ? StrCast(this.props.ruleProvider["ruleRounding_" + this.Document.heading]) : undefined; const colorSet = this.setsLayoutProp("backgroundColor"); diff --git a/src/client/views/nodes/FormattedTextBox.tsx b/src/client/views/nodes/FormattedTextBox.tsx index 4a79a44d4..ef54a387c 100644 --- a/src/client/views/nodes/FormattedTextBox.tsx +++ b/src/client/views/nodes/FormattedTextBox.tsx @@ -1015,7 +1015,7 @@ export class FormattedTextBox extends DocAnnotatableComponent<(FieldViewProps & @computed get sidebarWidth() { return Number(this.sidebarWidthPercent.substring(0, this.sidebarWidthPercent.length - 1)) / 100 * this.props.PanelWidth(); } @computed get annotationsKey() { return "annotations"; } render() { - trace(); + // trace(); let rounded = StrCast(this.layoutDoc.borderRounding) === "100%" ? "-rounded" : ""; let interactive = InkingControl.Instance.selectedTool || this.layoutDoc.isBackground; if (this.props.isSelected()) { diff --git a/src/client/views/nodes/ImageBox.tsx b/src/client/views/nodes/ImageBox.tsx index 4ed350d2b..7a9ebc00a 100644 --- a/src/client/views/nodes/ImageBox.tsx +++ b/src/client/views/nodes/ImageBox.tsx @@ -267,7 +267,7 @@ export class ImageBox extends DocAnnotatableComponent {this.nonDocAnnotations.sort((a, b) => NumCast(a.y) - NumCast(b.y)).map((anno, index) => )} @@ -671,7 +671,7 @@ export class PDFViewer extends DocAnnotatableComponent this.props.PanelHeight() / this.props.ContentScaling() * 72 / 96; contentZoom = () => this._zoomed; render() { - trace(); + // trace(); return !this.extensionDoc ? (null) :
Date: Tue, 19 Nov 2019 16:34:20 -0500 Subject: passport fix --- src/server/authentication/config/passport.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/server/authentication/config/passport.ts b/src/server/authentication/config/passport.ts index 0b15c3a36..726df7fd7 100644 --- a/src/server/authentication/config/passport.ts +++ b/src/server/authentication/config/passport.ts @@ -40,7 +40,7 @@ export let isAuthenticated = (req: Request, res: Response, next: NextFunction) = export let isAuthorized = (req: Request, res: Response, next: NextFunction) => { const provider = req.path.split("/").slice(-1)[0]; - if (_.find((req.user).tokens, { kind: provider })) { + if (_.find((req.user as any).tokens!, { kind: provider })) { next(); } else { res.redirect(`/auth/${provider}`); -- cgit v1.2.3-70-g09d2 From 8b6f70cd0ddac6d2669c3b0624d59a866737497c Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Tue, 19 Nov 2019 17:23:55 -0500 Subject: small --- src/server/index.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'src') diff --git a/src/server/index.ts b/src/server/index.ts index 57c66bc28..ad9c5886c 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -56,8 +56,7 @@ async function preliminaryFunctions() { * with the server */ function routeSetter(router: RouteManager) { - // initialize API Managers - [ + const managers = [ new UserManager(), new UploadManager(), new DownloadManager(), @@ -67,7 +66,10 @@ function routeSetter(router: RouteManager) { new UtilManager(), new GeneralGoogleManager(), new GooglePhotosManager(), - ].forEach(manager => manager.register(router)); + ]; + + // initialize API Managers + managers.forEach(manager => manager.register(router)); // initialize the web socket (bidirectional communication: if a user changes // a field on one client, that change must be broadcast to all other clients) -- cgit v1.2.3-70-g09d2 From f194abe2a54158bb041e1e0e7cfa6e22c669629f Mon Sep 17 00:00:00 2001 From: eeng5 Date: Tue, 19 Nov 2019 17:47:28 -0500 Subject: one line --- src/server/ApiManagers/GooglePhotosManager.ts | 3 +++ 1 file changed, 3 insertions(+) (limited to 'src') diff --git a/src/server/ApiManagers/GooglePhotosManager.ts b/src/server/ApiManagers/GooglePhotosManager.ts index 1f6051c28..67eb92a18 100644 --- a/src/server/ApiManagers/GooglePhotosManager.ts +++ b/src/server/ApiManagers/GooglePhotosManager.ts @@ -31,6 +31,9 @@ interface NewMediaItem { } const prefix = "google_photos_"; +/** + * This manager handles the creation of routes for google photos functionality. + */ export default class GooglePhotosManager extends ApiManager { protected initialize(register: Registration): void { -- cgit v1.2.3-70-g09d2 From 9a72d65baaa293fa84592de9bc43d555b5cae319 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Tue, 19 Nov 2019 19:52:56 -0500 Subject: logging --- src/Utils.ts | 5 +++++ src/server/ActionUtilities.ts | 31 +++++++++++++++++++++++++++---- src/server/Initialization.ts | 8 +++++++- src/server/Websocket/Websocket.ts | 12 ++++++------ src/server/index.ts | 12 ++++++++++-- 5 files changed, 55 insertions(+), 13 deletions(-) (limited to 'src') diff --git a/src/Utils.ts b/src/Utils.ts index 13d1be767..891a3826b 100644 --- a/src/Utils.ts +++ b/src/Utils.ts @@ -2,6 +2,7 @@ import v4 = require('uuid/v4'); import v5 = require("uuid/v5"); import { Socket } from 'socket.io'; import { Message } from './server/Message'; +import { EventEmitter } from 'events'; export namespace Utils { @@ -238,6 +239,10 @@ export namespace Utils { handler([arg, loggingCallback('S sending', fn, message.Name)]); }); } + + export function InjectLogger() { + + } } export function OmitKeys(obj: any, keys: string[], addKeyFunc?: (dup: any) => void): { omit: any, extract: any } { diff --git a/src/server/ActionUtilities.ts b/src/server/ActionUtilities.ts index b418772a6..5e88ea460 100644 --- a/src/server/ActionUtilities.ts +++ b/src/server/ActionUtilities.ts @@ -27,8 +27,31 @@ export const write_text_file = (relativePath: string, contents: any) => { }); }; -export async function log_execution(startMessage: string, endMessage: string, contents: () => void | Promise) { - console.log('\x1b[36m%s\x1b[0m', `${startMessage}...`); - await contents(); - console.log(endMessage); +export interface LogData { + startMessage: string; + endMessage: string; + action: () => void | Promise; +} + +export async function log_execution({ startMessage, endMessage, action }: LogData) { + const color = `\x1b[${30 + Math.ceil(Math.random() * 6)}m%s\x1b[0m`; + console.log(color, `${startMessage}...`); + await action(); + console.log(color, endMessage); +} + +export enum ConsoleColors { + Black = `\x1b[30m%s\x1b[0m`, + Red = `\x1b[31m%s\x1b[0m`, + Green = `\x1b[32m%s\x1b[0m`, + Yellow = `\x1b[33m%s\x1b[0m`, + Blue = `\x1b[34m%s\x1b[0m`, + Magenta = `\x1b[35m%s\x1b[0m`, + Cyan = `\x1b[36m%s\x1b[0m`, + White = `\x1b[37m%s\x1b[0m` +} + +export function logPort(listener: string, port: number) { + process.stdout.write(`${listener} listening on port `); + console.log(ConsoleColors.Yellow, port); } \ No newline at end of file diff --git a/src/server/Initialization.ts b/src/server/Initialization.ts index 6ac0787c5..357d7837e 100644 --- a/src/server/Initialization.ts +++ b/src/server/Initialization.ts @@ -19,6 +19,7 @@ import * as fs from 'fs'; import * as request from 'request'; import RouteSubscriber from './RouteSubscriber'; import { publicDirectory } from '.'; +import { ConsoleColors, logPort } from './ActionUtilities'; /* RouteSetter is a wrapper around the server that prevents the server from being exposed. */ @@ -35,6 +36,11 @@ export default async function InitializeServer(options: InitializationOptions) { server.use(express.static(publicDirectory)); server.use("/images", express.static(publicDirectory)); + server.use("*", (req, _res, next) => { + console.log(ConsoleColors.Cyan, req.originalUrl, req.user.id); + next(); + }); + server.use(wdm(compiler, { publicPath: config.output.publicPath })); server.use(whm(compiler)); @@ -44,7 +50,7 @@ export default async function InitializeServer(options: InitializationOptions) { const isRelease = determineEnvironment(); //vs. dev mode routeSetter(new RouteManager(server, isRelease)); - server.listen(listenAtPort, () => console.log(`server started at http://localhost:${listenAtPort}`)); + server.listen(listenAtPort, () => logPort("server", listenAtPort)); return isRelease; } diff --git a/src/server/Websocket/Websocket.ts b/src/server/Websocket/Websocket.ts index f6a6c8718..810b3410c 100644 --- a/src/server/Websocket/Websocket.ts +++ b/src/server/Websocket/Websocket.ts @@ -6,9 +6,9 @@ import { Database } from "../database"; import { Search } from "../Search"; import * as io from 'socket.io'; import YoutubeApi from "../apis/youtube/youtubeApiSample"; -import { readFile } from "fs"; -import { Credentials } from "google-auth-library"; import { GoogleCredentialsLoader } from "../credentials/CredentialsLoader"; +import { ConsoleColors, logPort } from "../ActionUtilities"; +import { EventEmitter } from "events"; export namespace WebSocket { @@ -28,10 +28,10 @@ export namespace WebSocket { async function preliminaryFunctions() { } - export function initialize(serverPort: number, isRelease: boolean) { + export function initialize(socketPort: number, isRelease: boolean) { const endpoint = io(); - endpoint.listen(serverPort); - console.log(`listening on port ${serverPort}`); + endpoint.listen(socketPort); + logPort("websocket", socketPort); endpoint.on("connection", function (socket: Socket) { socket.use((_packet, next) => { @@ -92,7 +92,7 @@ export namespace WebSocket { function barReceived(socket: SocketIO.Socket, guid: string) { clients[guid] = new Client(guid.toString()); - console.log(`User ${guid} has connected`); + console.log(ConsoleColors.Green, `User ${guid} has connected`); socketMap.set(socket, guid); } diff --git a/src/server/index.ts b/src/server/index.ts index 01ebf0ac1..618940c1a 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -43,7 +43,11 @@ async function preliminaryFunctions() { // divide the public directory based on type await Promise.all(Object.keys(Partitions).map(partition => DashUploadUtils.createIfNotExists(filesDirectory + partition))); // connect to the database - await Database.tryInitializeConnection(); + await log_execution({ + startMessage: "attempting to initialize mongodb connection", + endMessage: "connection outcome determined", + action: Database.tryInitializeConnection + }); } /** @@ -106,6 +110,10 @@ function routeSetter(router: RouteManager) { } (async function start() { - await log_execution("starting execution of preliminary functions", "completed preliminary functions", preliminaryFunctions); + await log_execution({ + startMessage: "starting execution of preliminary functions", + endMessage: "completed preliminary functions", + action: preliminaryFunctions + }); await initializeServer({ listenAtPort: 1050, routeSetter }); })(); -- cgit v1.2.3-70-g09d2 From cc96b05f8695f84c547438fed7300db0ac6d4266 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Tue, 19 Nov 2019 19:53:39 -0500 Subject: missed --- src/Utils.ts | 2 +- src/server/Websocket/Websocket.ts | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/Utils.ts b/src/Utils.ts index 891a3826b..2c9e50808 100644 --- a/src/Utils.ts +++ b/src/Utils.ts @@ -240,7 +240,7 @@ export namespace Utils { }); } - export function InjectLogger() { + export function InjectLogger(color: string) { } } diff --git a/src/server/Websocket/Websocket.ts b/src/server/Websocket/Websocket.ts index 810b3410c..9f0959e65 100644 --- a/src/server/Websocket/Websocket.ts +++ b/src/server/Websocket/Websocket.ts @@ -59,6 +59,8 @@ export namespace WebSocket { Utils.AddServerHandler(socket, MessageStore.DeleteFields, ids => DeleteFields(socket, ids)); Utils.AddServerHandlerCallback(socket, MessageStore.GetRefField, GetRefField); Utils.AddServerHandlerCallback(socket, MessageStore.GetRefFields, GetRefFields); + + Utils.InjectLogger(ConsoleColors.Magenta); }); } -- cgit v1.2.3-70-g09d2 From 23e72d25506a1cb87b15a34dbf7d85b61bfe063b Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Tue, 19 Nov 2019 20:05:12 -0500 Subject: inject logger, not working --- src/Utils.ts | 10 ++++++++-- src/server/Websocket/Websocket.ts | 2 +- 2 files changed, 9 insertions(+), 3 deletions(-) (limited to 'src') diff --git a/src/Utils.ts b/src/Utils.ts index 2c9e50808..1e69cfbfe 100644 --- a/src/Utils.ts +++ b/src/Utils.ts @@ -3,6 +3,7 @@ import v5 = require("uuid/v5"); import { Socket } from 'socket.io'; import { Message } from './server/Message'; import { EventEmitter } from 'events'; +import { ConsoleColors } from './server/ActionUtilities'; export namespace Utils { @@ -240,8 +241,13 @@ export namespace Utils { }); } - export function InjectLogger(color: string) { - + export function InjectLogger(socket: Socket, color: string) { + const on = socket.on; + socket.on = function (event: string | symbol, listener: (...args: any[]) => void) { + console.log(color, event); + on(event, listener); + return socket; + }; } } diff --git a/src/server/Websocket/Websocket.ts b/src/server/Websocket/Websocket.ts index 9f0959e65..308b32988 100644 --- a/src/server/Websocket/Websocket.ts +++ b/src/server/Websocket/Websocket.ts @@ -60,7 +60,7 @@ export namespace WebSocket { Utils.AddServerHandlerCallback(socket, MessageStore.GetRefField, GetRefField); Utils.AddServerHandlerCallback(socket, MessageStore.GetRefFields, GetRefFields); - Utils.InjectLogger(ConsoleColors.Magenta); + Utils.InjectLogger(socket, ConsoleColors.Yellow); }); } -- cgit v1.2.3-70-g09d2 From 6ddb23d48a8c97cbe12f841791825567c3a48e69 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Tue, 19 Nov 2019 22:17:34 -0500 Subject: removed call --- src/server/Websocket/Websocket.ts | 2 -- 1 file changed, 2 deletions(-) (limited to 'src') diff --git a/src/server/Websocket/Websocket.ts b/src/server/Websocket/Websocket.ts index 308b32988..810b3410c 100644 --- a/src/server/Websocket/Websocket.ts +++ b/src/server/Websocket/Websocket.ts @@ -59,8 +59,6 @@ export namespace WebSocket { Utils.AddServerHandler(socket, MessageStore.DeleteFields, ids => DeleteFields(socket, ids)); Utils.AddServerHandlerCallback(socket, MessageStore.GetRefField, GetRefField); Utils.AddServerHandlerCallback(socket, MessageStore.GetRefFields, GetRefFields); - - Utils.InjectLogger(socket, ConsoleColors.Yellow); }); } -- cgit v1.2.3-70-g09d2 From 1722788bf7d672043b1d51026b553426459bbd22 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Wed, 20 Nov 2019 19:17:48 -0500 Subject: small --- src/Utils.ts | 9 --------- src/server/Initialization.ts | 2 +- src/server/Websocket/Websocket.ts | 9 +++------ 3 files changed, 4 insertions(+), 16 deletions(-) (limited to 'src') diff --git a/src/Utils.ts b/src/Utils.ts index 1e69cfbfe..fc12c1e14 100644 --- a/src/Utils.ts +++ b/src/Utils.ts @@ -240,15 +240,6 @@ export namespace Utils { handler([arg, loggingCallback('S sending', fn, message.Name)]); }); } - - export function InjectLogger(socket: Socket, color: string) { - const on = socket.on; - socket.on = function (event: string | symbol, listener: (...args: any[]) => void) { - console.log(color, event); - on(event, listener); - return socket; - }; - } } export function OmitKeys(obj: any, keys: string[], addKeyFunc?: (dup: any) => void): { omit: any, extract: any } { diff --git a/src/server/Initialization.ts b/src/server/Initialization.ts index 357d7837e..08b476822 100644 --- a/src/server/Initialization.ts +++ b/src/server/Initialization.ts @@ -37,7 +37,7 @@ export default async function InitializeServer(options: InitializationOptions) { server.use("/images", express.static(publicDirectory)); server.use("*", (req, _res, next) => { - console.log(ConsoleColors.Cyan, req.originalUrl, req.user.id); + console.log(ConsoleColors.Cyan, req.originalUrl, req.user.email); next(); }); diff --git a/src/server/Websocket/Websocket.ts b/src/server/Websocket/Websocket.ts index 810b3410c..74a6b4263 100644 --- a/src/server/Websocket/Websocket.ts +++ b/src/server/Websocket/Websocket.ts @@ -8,7 +8,6 @@ import * as io from 'socket.io'; import YoutubeApi from "../apis/youtube/youtubeApiSample"; import { GoogleCredentialsLoader } from "../credentials/CredentialsLoader"; import { ConsoleColors, logPort } from "../ActionUtilities"; -import { EventEmitter } from "events"; export namespace WebSocket { @@ -30,9 +29,6 @@ export namespace WebSocket { export function initialize(socketPort: number, isRelease: boolean) { const endpoint = io(); - endpoint.listen(socketPort); - logPort("websocket", socketPort); - endpoint.on("connection", function (socket: Socket) { socket.use((_packet, next) => { let id = socketMap.get(socket); @@ -60,7 +56,8 @@ export namespace WebSocket { Utils.AddServerHandlerCallback(socket, MessageStore.GetRefField, GetRefField); Utils.AddServerHandlerCallback(socket, MessageStore.GetRefFields, GetRefFields); }); - + endpoint.listen(socketPort); + logPort("websocket", socketPort); } function HandleYoutubeQuery([query, callback]: [YoutubeQueryInput, (result?: any[]) => void]) { @@ -92,7 +89,7 @@ export namespace WebSocket { function barReceived(socket: SocketIO.Socket, guid: string) { clients[guid] = new Client(guid.toString()); - console.log(ConsoleColors.Green, `User ${guid} has connected`); + console.log(ConsoleColors.Green, `user ${guid} has connected to the web socket`); socketMap.set(socket, guid); } -- cgit v1.2.3-70-g09d2 From 39bac937106e77679b2dc76078b812a6b6b11a94 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Thu, 21 Nov 2019 18:20:01 -0500 Subject: last cleanup --- src/server/ActionUtilities.ts | 3 ++- src/server/ApiManagers/ApiManager.ts | 4 ++-- src/server/ApiManagers/UploadManager.ts | 4 ++-- src/server/DashUploadUtils.ts | 12 ++++++++++++ src/server/index.ts | 25 ++++++++----------------- 5 files changed, 26 insertions(+), 22 deletions(-) (limited to 'src') diff --git a/src/server/ActionUtilities.ts b/src/server/ActionUtilities.ts index 5e88ea460..a5f33833d 100644 --- a/src/server/ActionUtilities.ts +++ b/src/server/ActionUtilities.ts @@ -33,8 +33,9 @@ export interface LogData { action: () => void | Promise; } +let current = Math.ceil(Math.random() * 20); export async function log_execution({ startMessage, endMessage, action }: LogData) { - const color = `\x1b[${30 + Math.ceil(Math.random() * 6)}m%s\x1b[0m`; + const color = `\x1b[${31 + current++ % 6}m%s\x1b[0m`; console.log(color, `${startMessage}...`); await action(); console.log(color, endMessage); diff --git a/src/server/ApiManagers/ApiManager.ts b/src/server/ApiManagers/ApiManager.ts index 9fd726060..e2b01d585 100644 --- a/src/server/ApiManagers/ApiManager.ts +++ b/src/server/ApiManagers/ApiManager.ts @@ -5,7 +5,7 @@ export type Registration = (initializer: RouteInitializer) => void; export default abstract class ApiManager { protected abstract initialize(register: Registration): void; - public register(router: RouteManager) { - this.initialize(router.addSupervisedRoute); + public register(register: Registration) { + this.initialize(register); } } \ No newline at end of file diff --git a/src/server/ApiManagers/UploadManager.ts b/src/server/ApiManagers/UploadManager.ts index 01abdab54..aca63a918 100644 --- a/src/server/ApiManagers/UploadManager.ts +++ b/src/server/ApiManagers/UploadManager.ts @@ -5,7 +5,7 @@ import v4 = require('uuid/v4'); var AdmZip = require('adm-zip'); import * as path from 'path'; import { createReadStream, createWriteStream, unlink, readFileSync } from "fs"; -import { publicDirectory, filesDirectory, Partitions } from ".."; +import { publicDirectory, filesDirectory } from ".."; import { Database } from "../database"; import { DashUploadUtils } from "../DashUploadUtils"; import { Opt } from "../../new_fields/Doc"; @@ -142,7 +142,7 @@ export default class UploadManager extends ApiManager { let dataBuffer = readFileSync(filesDirectory + filename); const result: ParsedPDF = await pdf(dataBuffer); await new Promise((resolve, reject) => { - const path = filesDirectory + Partitions.pdf_text + "/" + filename.substring(0, filename.length - ".pdf".length) + ".txt"; + const path = filesDirectory + DashUploadUtils.Partitions.pdf_text + "/" + filename.substring(0, filename.length - ".pdf".length) + ".txt"; createWriteStream(path).write(result.text, error => { if (!error) { resolve(); diff --git a/src/server/DashUploadUtils.ts b/src/server/DashUploadUtils.ts index 8f5b0e1a8..8a429b81b 100644 --- a/src/server/DashUploadUtils.ts +++ b/src/server/DashUploadUtils.ts @@ -6,6 +6,7 @@ import request = require('request-promise'); import { ExifData, ExifImage } from 'exif'; import { Opt } from '../new_fields/Doc'; import { SharedMediaTypes } from './SharedMediaTypes'; +import { filesDirectory } from '.'; const uploadDirectory = path.join(__dirname, './public/files/'); @@ -89,6 +90,17 @@ export namespace DashUploadUtils { error?: string; } + export enum Partitions { + pdf_text, + images, + videos + } + + export async function buildFilePartitions() { + const pending = Object.keys(Partitions).map(sub => createIfNotExists(filesDirectory + sub)); + return Promise.all(pending); + } + /** * Based on the url's classification as local or remote, gleans * as much information as possible about the specified image diff --git a/src/server/index.ts b/src/server/index.ts index 618940c1a..9c48aca45 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -22,13 +22,8 @@ import { log_execution } from "./ActionUtilities"; import GeneralGoogleManager from "./ApiManagers/GeneralGoogleManager"; import GooglePhotosManager from "./ApiManagers/GooglePhotosManager"; -export const publicDirectory = __dirname + "/public"; -export const filesDirectory = publicDirectory + "/files/"; -export enum Partitions { - pdf_text, - images, - videos -} +export const publicDirectory = path.resolve(__dirname, "public"); +export const filesDirectory = path.resolve(publicDirectory, "files") + "/"; /** * These are the functions run before the server starts @@ -36,13 +31,9 @@ export enum Partitions { * before clients can access the server should be run or awaited here. */ async function preliminaryFunctions() { - // make project credentials globally accessible await GoogleCredentialsLoader.loadCredentials(); - // read the resulting credentials into a different namespace GoogleApiServerUtils.processProjectCredentials(); - // divide the public directory based on type - await Promise.all(Object.keys(Partitions).map(partition => DashUploadUtils.createIfNotExists(filesDirectory + partition))); - // connect to the database + await DashUploadUtils.buildFilePartitions(); await log_execution({ startMessage: "attempting to initialize mongodb connection", endMessage: "connection outcome determined", @@ -59,7 +50,7 @@ async function preliminaryFunctions() { * that will manage the registration of new routes * with the server */ -function routeSetter(router: RouteManager) { +function routeSetter({ isRelease, addSupervisedRoute }: RouteManager) { const managers = [ new UserManager(), new UploadManager(), @@ -73,16 +64,16 @@ function routeSetter(router: RouteManager) { ]; // initialize API Managers - managers.forEach(manager => manager.register(router)); + managers.forEach(manager => manager.register(addSupervisedRoute)); // initialize the web socket (bidirectional communication: if a user changes // a field on one client, that change must be broadcast to all other clients) - WebSocket.initialize(serverPort, router.isRelease); + WebSocket.initialize(serverPort, isRelease); /** * Accessing root index redirects to home */ - router.addSupervisedRoute({ + addSupervisedRoute({ method: Method.GET, subscription: "/", onValidation: ({ res }) => res.redirect("/home") @@ -94,7 +85,7 @@ function routeSetter(router: RouteManager) { res.sendFile(path.join(__dirname, '../../deploy/' + filename)); }; - router.addSupervisedRoute({ + addSupervisedRoute({ method: Method.GET, subscription: ["/home", new RouteSubscriber("doc").add("docId")], onValidation: serve, -- cgit v1.2.3-70-g09d2 From b831be86743e329cce441b3d7ae2aa5321e7fb9c Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Sat, 23 Nov 2019 17:09:13 -0500 Subject: improved user activity log --- src/server/ActionUtilities.ts | 13 ++++ src/server/ApiManagers/UserManager.ts | 46 ++++++------ src/server/Initialization.ts | 8 ++- src/server/Websocket/Websocket.ts | 22 +++--- .../authentication/controllers/user_controller.ts | 3 - views/login.pug | 2 - views/stylesheets/authentication.css | 81 ++++++++++++++++++++++ views/user_activity.pug | 19 +++++ 8 files changed, 155 insertions(+), 39 deletions(-) create mode 100644 views/user_activity.pug (limited to 'src') diff --git a/src/server/ActionUtilities.ts b/src/server/ActionUtilities.ts index a5f33833d..7f493dd70 100644 --- a/src/server/ActionUtilities.ts +++ b/src/server/ActionUtilities.ts @@ -55,4 +55,17 @@ export enum ConsoleColors { export function logPort(listener: string, port: number) { process.stdout.write(`${listener} listening on port `); console.log(ConsoleColors.Yellow, port); +} + +export function msToTime(duration: number) { + let milliseconds = Math.floor((duration % 1000) / 100), + seconds = Math.floor((duration / 1000) % 60), + minutes = Math.floor((duration / (1000 * 60)) % 60), + hours = Math.floor((duration / (1000 * 60 * 60)) % 24); + + let hoursS = (hours < 10) ? "0" + hours : hours; + let minutesS = (minutes < 10) ? "0" + minutes : minutes; + let secondsS = (seconds < 10) ? "0" + seconds : seconds; + + return hoursS + ":" + minutesS + ":" + secondsS + "." + milliseconds; } \ No newline at end of file diff --git a/src/server/ApiManagers/UserManager.ts b/src/server/ApiManagers/UserManager.ts index 51a434fcf..4ee5a2b85 100644 --- a/src/server/ApiManagers/UserManager.ts +++ b/src/server/ApiManagers/UserManager.ts @@ -1,7 +1,14 @@ import ApiManager, { Registration } from "./ApiManager"; import { Method } from "../RouteManager"; -import { WebSocket } from "../Websocket/Websocket"; import { Database } from "../database"; +import { msToTime } from "../ActionUtilities"; + +export const timeMap: { [id: string]: number } = {}; +interface ActivityUnit { + user: string; + duration: number; +} + export default class UserManager extends ApiManager { @@ -32,35 +39,34 @@ export default class UserManager extends ApiManager { register({ method: Method.GET, - subscription: "/whosOnline", + subscription: "/activity", onValidation: ({ res }) => { - let users: any = { active: {}, inactive: {} }; const now = Date.now(); - const { timeMap } = WebSocket; + const activeTimes: ActivityUnit[] = []; + const inactiveTimes: ActivityUnit[] = []; + for (const user in timeMap) { const time = timeMap[user]; - const key = ((now - time) / 1000) < (60 * 5) ? "active" : "inactive"; - users[key][user] = `Last active ${msToTime(now - time)} ago`; + const duration = now - time; + const target = (duration / 1000) < (60 * 5) ? activeTimes : inactiveTimes; + target.push({ user, duration }); } - res.send(users); + const process = (target: { user: string, duration: number }[]) => { + const comparator = (first: ActivityUnit, second: ActivityUnit) => first.duration - second.duration; + const sorted = target.sort(comparator); + return sorted.map(({ user, duration }) => `${user} (last active ${msToTime(duration)} ago)`); + }; + + res.render("user_activity.pug", { + title: "User Activity", + active: process(activeTimes), + inactive: process(inactiveTimes) + }); } }); } -} - -function msToTime(duration: number) { - let milliseconds = Math.floor((duration % 1000) / 100), - seconds = Math.floor((duration / 1000) % 60), - minutes = Math.floor((duration / (1000 * 60)) % 60), - hours = Math.floor((duration / (1000 * 60 * 60)) % 24); - - let hoursS = (hours < 10) ? "0" + hours : hours; - let minutesS = (minutes < 10) ? "0" + minutes : minutes; - let secondsS = (seconds < 10) ? "0" + seconds : seconds; - - return hoursS + ":" + minutesS + ":" + secondsS + "." + milliseconds; } \ No newline at end of file diff --git a/src/server/Initialization.ts b/src/server/Initialization.ts index 08b476822..7fad5556d 100644 --- a/src/server/Initialization.ts +++ b/src/server/Initialization.ts @@ -20,6 +20,8 @@ import * as request from 'request'; import RouteSubscriber from './RouteSubscriber'; import { publicDirectory } from '.'; import { ConsoleColors, logPort } from './ActionUtilities'; +import { WebSocket } from './Websocket/Websocket'; +import { timeMap } from './ApiManagers/UserManager'; /* RouteSetter is a wrapper around the server that prevents the server from being exposed. */ @@ -37,7 +39,11 @@ export default async function InitializeServer(options: InitializationOptions) { server.use("/images", express.static(publicDirectory)); server.use("*", (req, _res, next) => { - console.log(ConsoleColors.Cyan, req.originalUrl, req.user.email); + const userEmail = req.user?.email; + console.log(ConsoleColors.Cyan, req.originalUrl, userEmail ?? ""); + if (userEmail) { + timeMap[userEmail] = Date.now(); + } next(); }); diff --git a/src/server/Websocket/Websocket.ts b/src/server/Websocket/Websocket.ts index 74a6b4263..fbf71f707 100644 --- a/src/server/Websocket/Websocket.ts +++ b/src/server/Websocket/Websocket.ts @@ -8,16 +8,12 @@ import * as io from 'socket.io'; import YoutubeApi from "../apis/youtube/youtubeApiSample"; import { GoogleCredentialsLoader } from "../credentials/CredentialsLoader"; import { ConsoleColors, logPort } from "../ActionUtilities"; +import { timeMap } from "../ApiManagers/UserManager"; export namespace WebSocket { - interface Map { - [key: string]: Client; - } - let clients: Map = {}; - + let clients: { [key: string]: Client } = {}; export const socketMap = new Map(); - export const timeMap: { [id: string]: number } = {}; export async function start(serverPort: number, isRelease: boolean) { await preliminaryFunctions(); @@ -31,9 +27,9 @@ export namespace WebSocket { const endpoint = io(); endpoint.on("connection", function (socket: Socket) { socket.use((_packet, next) => { - let id = socketMap.get(socket); - if (id) { - timeMap[id] = Date.now(); + let userEmail = socketMap.get(socket); + if (userEmail) { + timeMap[userEmail] = Date.now(); } next(); }); @@ -87,10 +83,10 @@ export namespace WebSocket { await Search.Instance.clear(); } - function barReceived(socket: SocketIO.Socket, guid: string) { - clients[guid] = new Client(guid.toString()); - console.log(ConsoleColors.Green, `user ${guid} has connected to the web socket`); - socketMap.set(socket, guid); + function barReceived(socket: SocketIO.Socket, userEmail: string) { + clients[userEmail] = new Client(userEmail.toString()); + console.log(ConsoleColors.Green, `user ${userEmail} has connected to the web socket`); + socketMap.set(socket, userEmail); } function getField([id, callback]: [string, (result?: Transferable) => void]) { diff --git a/src/server/authentication/controllers/user_controller.ts b/src/server/authentication/controllers/user_controller.ts index b2b9d33f6..517353479 100644 --- a/src/server/authentication/controllers/user_controller.ts +++ b/src/server/authentication/controllers/user_controller.ts @@ -3,10 +3,7 @@ import { Request, Response, NextFunction } from "express"; import * as passport from "passport"; import { IVerifyOptions } from "passport-local"; import "../config/passport"; -import * as request from "express-validator"; import flash = require("express-flash"); -import * as session from "express-session"; -import * as pug from 'pug'; import * as async from 'async'; import * as nodemailer from 'nodemailer'; import c = require("crypto"); diff --git a/views/login.pug b/views/login.pug index 9bc40a495..26da5e29e 100644 --- a/views/login.pug +++ b/views/login.pug @@ -14,11 +14,9 @@ block content .inner.login h3.auth_header Log In .form-group - //- label.col-sm-3.control-label(for='email', id='email_label') Email .col-sm-7 input.form-control(type='email', name='email', id='email', placeholder='Email', autofocus, required) .form-group - //- label.col-sm-3.control-label(for='password') Password .col-sm-7 input.form-control(type='password', name='password', id='password', placeholder='Password', required) .form-group diff --git a/views/stylesheets/authentication.css b/views/stylesheets/authentication.css index 36bb880af..ff1f4aace 100644 --- a/views/stylesheets/authentication.css +++ b/views/stylesheets/authentication.css @@ -139,4 +139,85 @@ body { padding-right: 10px; font-family: Arial, Helvetica, sans-serif; font-size: 16px; +} + +.outermost, .online-container { + display: flex; + flex-direction: row; + height: 98vh; + justify-content: center; +} + +.online-container { + background: white; + display: flex; + flex-direction: row; + height: 80%; + width: 80%; + align-self: center; + justify-content: center; + border-radius: 8px; + box-shadow: 10px 10px 10px #00000099; +} + +.partition { + width: 50%; + display: flex; + flex-direction: column; + border: 1px solid black; +} + +.inner-activity { + display: flex; + flex-direction: column; + justify-content: center; + height: 100%; + border-top: 2px solid black; + background: white; + padding: 20px; + overflow: scroll; +} + +ol { + align-self: center; +} + +li { + font-family: Arial, Helvetica, sans-serif; + border: 1px solid black; + padding: 10px; + border-radius: 5px; + margin-bottom: 5px; +} + +.duration { + font-style: italic; +} + +span.user-type { + align-self: center; + font-family: Arial, Helvetica, sans-serif; + font-weight: bold; + font-size: 20px; + margin: 50px; +} + +#active-partition { + background: green; + border-top-left-radius: 8px; + border-bottom-left-radius: 8px; +} + +#active-inner { + border-bottom-left-radius: 8px; +} + +#inactive-partition { + background: red; + border-top-right-radius: 8px; + border-bottom-right-radius: 8px; +} + +#inactive-inner { + border-bottom-right-radius: 8px; } \ No newline at end of file diff --git a/views/user_activity.pug b/views/user_activity.pug new file mode 100644 index 000000000..68e42140d --- /dev/null +++ b/views/user_activity.pug @@ -0,0 +1,19 @@ +extends ./layout + +block content + style + include ./stylesheets/authentication.css + .outermost + .online-container + .partition(id="active-partition") + span.user-type Active Users + .inner-activity(id="active-inner") + ol + each val in active + li= val + .partition(id="inactive-partition") + span.user-type Inactive Users + .inner-activity(id="inactive-inner") + ol + each val in inactive + li= val \ No newline at end of file -- cgit v1.2.3-70-g09d2 From e324248724a130a84b459a072dc846f500f8d9b0 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Sat, 23 Nov 2019 18:19:27 -0500 Subject: heartbeats --- src/client/DocServer.ts | 2 ++ src/client/util/ClientDiagnostics.ts | 29 +++++++++++++++++++++++++++++ src/client/views/Main.tsx | 2 ++ src/client/views/MainView.tsx | 6 ++++-- src/server/ApiManagers/DiagnosticManager.ts | 26 ++++++++++++++++++++++++++ src/server/ApiManagers/UserManager.ts | 2 +- src/server/Initialization.ts | 13 +++++++------ src/server/index.ts | 2 ++ 8 files changed, 73 insertions(+), 9 deletions(-) create mode 100644 src/client/util/ClientDiagnostics.ts create mode 100644 src/server/ApiManagers/DiagnosticManager.ts (limited to 'src') diff --git a/src/client/DocServer.ts b/src/client/DocServer.ts index 2cec1046b..14479694c 100644 --- a/src/client/DocServer.ts +++ b/src/client/DocServer.ts @@ -64,6 +64,8 @@ export namespace DocServer { } } + let connection_error = false; + export function init(protocol: string, hostname: string, port: number, identifier: string) { _cache = {}; GUID = identifier; diff --git a/src/client/util/ClientDiagnostics.ts b/src/client/util/ClientDiagnostics.ts new file mode 100644 index 000000000..e454cdecb --- /dev/null +++ b/src/client/util/ClientDiagnostics.ts @@ -0,0 +1,29 @@ +import { observable, runInAction } from "mobx"; +import { MainView } from "../views/MainView"; + +export namespace ClientDiagnostics { + + export function start() { + + let serverPolls = 0; + const serverHandle = setInterval(async () => { + if (++serverPolls === 20) { + alert("Your connection to the server has been terminated."); + clearInterval(serverHandle); + } + await fetch("/serverHeartbeat"); + serverPolls--; + }, 100); + + + const solrHandle = setInterval(async () => { + const response = await fetch("/solrHeartbeat"); + if (!response) { + alert("Looks like SOLR is not running on your machine."); + clearInterval(solrHandle); + } + }, 100); + + } + +} \ No newline at end of file diff --git a/src/client/views/Main.tsx b/src/client/views/Main.tsx index b21eb9c8f..dec4a24e4 100644 --- a/src/client/views/Main.tsx +++ b/src/client/views/Main.tsx @@ -5,10 +5,12 @@ import * as ReactDOM from 'react-dom'; import * as React from 'react'; import { DocServer } from "../DocServer"; import { AssignAllExtensions } from "../../extensions/General/Extensions"; +import { ClientDiagnostics } from "../util/ClientDiagnostics"; AssignAllExtensions(); (async () => { + ClientDiagnostics.start(); const info = await CurrentUserUtils.loadCurrentUser(); DocServer.init(window.location.protocol, window.location.hostname, 4321, info.email); await Docs.Prototypes.initialize(); diff --git a/src/client/views/MainView.tsx b/src/client/views/MainView.tsx index 031926604..d352ad776 100644 --- a/src/client/views/MainView.tsx +++ b/src/client/views/MainView.tsx @@ -4,7 +4,7 @@ import { faMusic, faObjectGroup, faPause, faMousePointer, faPenNib, faFileAudio, faPen, faEraser, faPlay, faPortrait, faRedoAlt, faThumbtack, faTree, faTv, faUndoAlt, faHighlighter, faMicrophone, faCompressArrowsAlt } from '@fortawesome/free-solid-svg-icons'; import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; -import { action, computed, configure, observable, reaction, runInAction } from 'mobx'; +import { action, computed, configure, observable, reaction, runInAction, autorun } from 'mobx'; import { observer } from 'mobx-react'; import "normalize.css"; import * as React from 'react'; @@ -414,7 +414,8 @@ export class MainView extends React.Component { -
; + + ; } @computed get mainContent() { @@ -422,6 +423,7 @@ export class MainView extends React.Component { return !this.userDoc || !(sidebar instanceof Doc) ? (null) : (
+
HEY!
diff --git a/src/server/ApiManagers/DiagnosticManager.ts b/src/server/ApiManagers/DiagnosticManager.ts new file mode 100644 index 000000000..b775167b6 --- /dev/null +++ b/src/server/ApiManagers/DiagnosticManager.ts @@ -0,0 +1,26 @@ +import ApiManager, { Registration } from "./ApiManager"; +import { Method } from "../RouteManager"; +import request = require('request-promise'); + +export default class DiagnosticManager extends ApiManager { + + protected initialize(register: Registration): void { + + register({ + method: Method.GET, + subscription: "/serverHeartbeat", + onValidation: ({ res }) => res.send(true) + }); + + register({ + method: Method.GET, + subscription: "/solrHeartbeat", + onValidation: async ({ res }) => { + const response = await request("http://localhost:8983"); + res.send(response !== undefined); + } + }); + + } + +} \ No newline at end of file diff --git a/src/server/ApiManagers/UserManager.ts b/src/server/ApiManagers/UserManager.ts index 4ee5a2b85..8edeab16d 100644 --- a/src/server/ApiManagers/UserManager.ts +++ b/src/server/ApiManagers/UserManager.ts @@ -56,7 +56,7 @@ export default class UserManager extends ApiManager { const process = (target: { user: string, duration: number }[]) => { const comparator = (first: ActivityUnit, second: ActivityUnit) => first.duration - second.duration; const sorted = target.sort(comparator); - return sorted.map(({ user, duration }) => `${user} (last active ${msToTime(duration)} ago)`); + return sorted.map(({ user, duration }) => `${user} (${msToTime(duration)})`); }; res.render("user_activity.pug", { diff --git a/src/server/Initialization.ts b/src/server/Initialization.ts index 7fad5556d..76acb4363 100644 --- a/src/server/Initialization.ts +++ b/src/server/Initialization.ts @@ -20,7 +20,6 @@ import * as request from 'request'; import RouteSubscriber from './RouteSubscriber'; import { publicDirectory } from '.'; import { ConsoleColors, logPort } from './ActionUtilities'; -import { WebSocket } from './Websocket/Websocket'; import { timeMap } from './ApiManagers/UserManager'; /* RouteSetter is a wrapper around the server that prevents the server @@ -38,11 +37,13 @@ export default async function InitializeServer(options: InitializationOptions) { server.use(express.static(publicDirectory)); server.use("/images", express.static(publicDirectory)); - server.use("*", (req, _res, next) => { - const userEmail = req.user?.email; - console.log(ConsoleColors.Cyan, req.originalUrl, userEmail ?? ""); - if (userEmail) { - timeMap[userEmail] = Date.now(); + server.use("*", ({ user, originalUrl }, _res, next) => { + if (!originalUrl.includes("Heartbeat")) { + const userEmail = user?.email; + console.log(ConsoleColors.Cyan, originalUrl, userEmail ?? ""); + if (userEmail) { + timeMap[userEmail] = Date.now(); + } } next(); }); diff --git a/src/server/index.ts b/src/server/index.ts index 9c48aca45..d02a6005e 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -21,6 +21,7 @@ import UploadManager from "./ApiManagers/UploadManager"; import { log_execution } from "./ActionUtilities"; import GeneralGoogleManager from "./ApiManagers/GeneralGoogleManager"; import GooglePhotosManager from "./ApiManagers/GooglePhotosManager"; +import DiagnosticManager from "./ApiManagers/DiagnosticManager"; export const publicDirectory = path.resolve(__dirname, "public"); export const filesDirectory = path.resolve(publicDirectory, "files") + "/"; @@ -55,6 +56,7 @@ function routeSetter({ isRelease, addSupervisedRoute }: RouteManager) { new UserManager(), new UploadManager(), new DownloadManager(), + new DiagnosticManager(), new SearchManager(), new PDFManager(), new DeleteManager(), -- cgit v1.2.3-70-g09d2 From 96519f4537895edcd1a23f353322459de328a330 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Tue, 26 Nov 2019 19:46:54 -0500 Subject: pivot label fixes --- .../CollectionFreeFormLayoutEngines.tsx | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormLayoutEngines.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormLayoutEngines.tsx index 48d330674..42e987a9a 100644 --- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormLayoutEngines.tsx +++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormLayoutEngines.tsx @@ -1,11 +1,12 @@ import { Doc, Field, FieldResult } from "../../../../new_fields/Doc"; -import { NumCast, StrCast, Cast } from "../../../../new_fields/Types"; +import { NumCast, StrCast, Cast, DateCast } from "../../../../new_fields/Types"; import { ScriptBox } from "../../ScriptBox"; import { CompileScript } from "../../../util/Scripting"; import { ScriptField } from "../../../../new_fields/ScriptField"; import { OverlayView, OverlayElementOptions } from "../../OverlayView"; import { emptyFunction } from "../../../../Utils"; import React = require("react"); +import { DateField } from "../../../../new_fields/DateField"; interface PivotData { type: string; @@ -31,6 +32,16 @@ export interface ViewDefResult { bounds?: ViewDefBounds; } +function toLabel(target: FieldResult) { + if (target instanceof DateField) { + const date = DateCast(target).date; + if (date) { + return `${date.toDateString()} ${date.toTimeString()}`; + } + } + return String(target); +} + export function computePivotLayout(pivotDoc: Doc, childDocs: Doc[], childPairs: { layout: Doc, data?: Doc }[], viewDefsToJSX: (views: any) => ViewDefResult[]) { let layoutPoolData: Map<{ layout: Doc, data?: Doc }, any> = new Map(); const pivotAxisWidth = NumCast(pivotDoc.pivotWidth, 200); @@ -55,7 +66,7 @@ export function computePivotLayout(pivotDoc: Doc, childDocs: Doc[], childPairs: let xCount = 0; groupNames.push({ type: "text", - text: String(key), + text: toLabel(key), x, y: pivotAxisWidth + 50, width: pivotAxisWidth * 1.25 * numCols, -- cgit v1.2.3-70-g09d2 From 0e5445c6eb3cb04b2657d5b5abeb89e0b1538220 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Tue, 26 Nov 2019 20:18:38 -0500 Subject: improved diagnostics routine --- src/client/util/ClientDiagnostics.ts | 6 ++++-- src/server/ApiManagers/DiagnosticManager.ts | 8 ++++++-- 2 files changed, 10 insertions(+), 4 deletions(-) (limited to 'src') diff --git a/src/client/util/ClientDiagnostics.ts b/src/client/util/ClientDiagnostics.ts index e454cdecb..6f82a47db 100644 --- a/src/client/util/ClientDiagnostics.ts +++ b/src/client/util/ClientDiagnostics.ts @@ -16,10 +16,12 @@ export namespace ClientDiagnostics { }, 100); + let executed = false; const solrHandle = setInterval(async () => { const response = await fetch("/solrHeartbeat"); - if (!response) { - alert("Looks like SOLR is not running on your machine."); + if (!(await response.json()).running) { + !executed && alert("Looks like SOLR is not running on your machine."); + executed = true; clearInterval(solrHandle); } }, 100); diff --git a/src/server/ApiManagers/DiagnosticManager.ts b/src/server/ApiManagers/DiagnosticManager.ts index b775167b6..104985481 100644 --- a/src/server/ApiManagers/DiagnosticManager.ts +++ b/src/server/ApiManagers/DiagnosticManager.ts @@ -16,8 +16,12 @@ export default class DiagnosticManager extends ApiManager { method: Method.GET, subscription: "/solrHeartbeat", onValidation: async ({ res }) => { - const response = await request("http://localhost:8983"); - res.send(response !== undefined); + try { + await request("http://localhost:8983"); + res.send({ running: true }); + } catch (e) { + res.send({ running: false }); + } } }); -- cgit v1.2.3-70-g09d2 From 780240515a06d9d71a4b58a2559d8661478a560f Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Tue, 26 Nov 2019 20:34:33 -0500 Subject: cleanup after looking at changed files --- src/client/DocServer.ts | 2 -- src/client/util/ClientDiagnostics.ts | 4 ++-- src/client/util/Import & Export/ImageUtils.ts | 2 +- src/client/views/MainView.tsx | 1 - src/client/views/search/SearchBox.tsx | 2 +- src/server/ApiManagers/UserManager.ts | 1 - src/server/Initialization.ts | 1 - src/server/RouteManager.ts | 2 ++ 8 files changed, 6 insertions(+), 9 deletions(-) (limited to 'src') diff --git a/src/client/DocServer.ts b/src/client/DocServer.ts index 14479694c..2cec1046b 100644 --- a/src/client/DocServer.ts +++ b/src/client/DocServer.ts @@ -64,8 +64,6 @@ export namespace DocServer { } } - let connection_error = false; - export function init(protocol: string, hostname: string, port: number, identifier: string) { _cache = {}; GUID = identifier; diff --git a/src/client/util/ClientDiagnostics.ts b/src/client/util/ClientDiagnostics.ts index 6f82a47db..24f196252 100644 --- a/src/client/util/ClientDiagnostics.ts +++ b/src/client/util/ClientDiagnostics.ts @@ -13,7 +13,7 @@ export namespace ClientDiagnostics { } await fetch("/serverHeartbeat"); serverPolls--; - }, 100); + }, 1000 * 15); let executed = false; @@ -24,7 +24,7 @@ export namespace ClientDiagnostics { executed = true; clearInterval(solrHandle); } - }, 100); + }, 1000 * 15); } diff --git a/src/client/util/Import & Export/ImageUtils.ts b/src/client/util/Import & Export/ImageUtils.ts index ca80f3bca..6a9486f83 100644 --- a/src/client/util/Import & Export/ImageUtils.ts +++ b/src/client/util/Import & Export/ImageUtils.ts @@ -22,7 +22,7 @@ export namespace ImageUtils { export const ExportHierarchyToFileSystem = async (collection: Doc): Promise => { const a = document.createElement("a"); - a.href = Utils.prepend(`imageHierarchyExport/${collection[Id]}`); + a.href = Utils.prepend(`/imageHierarchyExport/${collection[Id]}`); a.download = `Dash Export [${StrCast(collection.title)}].zip`; a.click(); }; diff --git a/src/client/views/MainView.tsx b/src/client/views/MainView.tsx index f59da3cde..5231075a1 100644 --- a/src/client/views/MainView.tsx +++ b/src/client/views/MainView.tsx @@ -423,7 +423,6 @@ export class MainView extends React.Component { return !this.userDoc || !(sidebar instanceof Doc) ? (null) : (
-
HEY!
diff --git a/src/client/views/search/SearchBox.tsx b/src/client/views/search/SearchBox.tsx index 1337d3f95..ff35542ed 100644 --- a/src/client/views/search/SearchBox.tsx +++ b/src/client/views/search/SearchBox.tsx @@ -89,7 +89,7 @@ export class SearchBox extends React.Component { public static async convertDataUri(imageUri: string, returnedFilename: string) { try { - let posting = Utils.prepend("uploadURI"); + let posting = Utils.prepend("/uploadURI"); const returnedUri = await rp.post(posting, { body: { uri: imageUri, diff --git a/src/server/ApiManagers/UserManager.ts b/src/server/ApiManagers/UserManager.ts index 8edeab16d..0f7d14320 100644 --- a/src/server/ApiManagers/UserManager.ts +++ b/src/server/ApiManagers/UserManager.ts @@ -9,7 +9,6 @@ interface ActivityUnit { duration: number; } - export default class UserManager extends ApiManager { protected initialize(register: Registration): void { diff --git a/src/server/Initialization.ts b/src/server/Initialization.ts index 76acb4363..8b633a7cd 100644 --- a/src/server/Initialization.ts +++ b/src/server/Initialization.ts @@ -40,7 +40,6 @@ export default async function InitializeServer(options: InitializationOptions) { server.use("*", ({ user, originalUrl }, _res, next) => { if (!originalUrl.includes("Heartbeat")) { const userEmail = user?.email; - console.log(ConsoleColors.Cyan, originalUrl, userEmail ?? ""); if (userEmail) { timeMap[userEmail] = Date.now(); } diff --git a/src/server/RouteManager.ts b/src/server/RouteManager.ts index 3aae5734a..3a20d5af5 100644 --- a/src/server/RouteManager.ts +++ b/src/server/RouteManager.ts @@ -1,6 +1,7 @@ import RouteSubscriber from "./RouteSubscriber"; import { DashUserModel } from "./authentication/models/user_model"; import * as express from 'express'; +import { ConsoleColors } from "./ActionUtilities"; export enum Method { GET, @@ -52,6 +53,7 @@ export default class RouteManager { try { await toExecute(args); } catch (e) { + console.log(ConsoleColors.Red, target, user?.email ?? ""); if (onError) { onError({ ...core, error: e }); } else { -- cgit v1.2.3-70-g09d2 From 2f4c58306af19954b0c849efb503b9620fab6efe Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Tue, 26 Nov 2019 21:45:09 -0500 Subject: intermediate, not working --- .gitignore | 1 + src/server/ApiManagers/UploadManager.ts | 44 ++-- src/server/DashUploadUtils.ts | 342 +++++++++++++++++--------------- src/server/public/files/.gitignore | 2 - 4 files changed, 195 insertions(+), 194 deletions(-) delete mode 100644 src/server/public/files/.gitignore (limited to 'src') diff --git a/.gitignore b/.gitignore index 5161268ac..e5048cfc4 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ dist/ .env ClientUtils.ts solr-8.1.1/server/ +src/server/public/files/ \ No newline at end of file diff --git a/src/server/ApiManagers/UploadManager.ts b/src/server/ApiManagers/UploadManager.ts index aca63a918..2a9faacd8 100644 --- a/src/server/ApiManagers/UploadManager.ts +++ b/src/server/ApiManagers/UploadManager.ts @@ -3,7 +3,7 @@ import { Method, _success } from "../RouteManager"; import * as formidable from 'formidable'; import v4 = require('uuid/v4'); var AdmZip = require('adm-zip'); -import * as path from 'path'; +import { extname, basename, dirname } from 'path'; import { createReadStream, createWriteStream, unlink, readFileSync } from "fs"; import { publicDirectory, filesDirectory } from ".."; import { Database } from "../database"; @@ -81,17 +81,17 @@ export default class UploadManager extends ApiManager { const zip = new AdmZip(path_2); zip.getEntries().forEach((entry: any) => { if (!entry.entryName.startsWith("files/")) return; - let dirname = path.dirname(entry.entryName) + "/"; - let extname = path.extname(entry.entryName); - let basename = path.basename(entry.entryName).split(".")[0]; + let directory = dirname(entry.entryName) + "/"; + let extension = extname(entry.entryName); + let base = basename(entry.entryName).split(".")[0]; try { zip.extractEntryTo(entry.entryName, publicDirectory, true, false); - dirname = "/" + dirname; + directory = "/" + directory; - createReadStream(publicDirectory + dirname + basename + extname).pipe(createWriteStream(publicDirectory + dirname + basename + "_o" + extname)); - createReadStream(publicDirectory + dirname + basename + extname).pipe(createWriteStream(publicDirectory + dirname + basename + "_s" + extname)); - createReadStream(publicDirectory + dirname + basename + extname).pipe(createWriteStream(publicDirectory + dirname + basename + "_m" + extname)); - createReadStream(publicDirectory + dirname + basename + extname).pipe(createWriteStream(publicDirectory + dirname + basename + "_l" + extname)); + createReadStream(publicDirectory + directory + base + extension).pipe(createWriteStream(publicDirectory + directory + base + "_o" + extension)); + createReadStream(publicDirectory + directory + base + extension).pipe(createWriteStream(publicDirectory + directory + base + "_s" + extension)); + createReadStream(publicDirectory + directory + base + extension).pipe(createWriteStream(publicDirectory + directory + base + "_m" + extension)); + createReadStream(publicDirectory + directory + base + extension).pipe(createWriteStream(publicDirectory + directory + base + "_l" + extension)); } catch (e) { console.log(e); } @@ -133,29 +133,9 @@ export default class UploadManager extends ApiManager { form.keepExtensions = true; return new Promise(resolve => { form.parse(req, async (_err, _fields, files) => { - let results: DashUploadUtils.ImageFileResponse[] = []; + let results: any[] = []; for (const key in files) { - const { type, path: location, name } = files[key]; - const filename = path.basename(location); - let uploadInformation: Opt; - if (filename.endsWith(".pdf")) { - let dataBuffer = readFileSync(filesDirectory + filename); - const result: ParsedPDF = await pdf(dataBuffer); - await new Promise((resolve, reject) => { - const path = filesDirectory + DashUploadUtils.Partitions.pdf_text + "/" + filename.substring(0, filename.length - ".pdf".length) + ".txt"; - createWriteStream(path).write(result.text, error => { - if (!error) { - resolve(); - } else { - reject(error); - } - }); - }); - } else { - uploadInformation = await DashUploadUtils.UploadImage(filesDirectory + filename, filename); - } - const exif = uploadInformation ? uploadInformation.exifData : undefined; - results.push({ name, type, path: `/files/${filename}`, exif }); + results.push(DashUploadUtils.upload(files[key])); } _success(res, results); resolve(); @@ -188,7 +168,7 @@ export default class UploadManager extends ApiManager { return; } return imageDataUri.outputFile(uri, filesDirectory + filename).then((savedName: string) => { - const ext = path.extname(savedName).toLowerCase(); + const ext = extname(savedName).toLowerCase(); const { pngs, jpgs } = SharedMediaTypes; let resizers = [ { resizer: sharp().resize(100, undefined, { withoutEnlargement: true }), suffix: "_s" }, diff --git a/src/server/DashUploadUtils.ts b/src/server/DashUploadUtils.ts index 8a429b81b..839aada4b 100644 --- a/src/server/DashUploadUtils.ts +++ b/src/server/DashUploadUtils.ts @@ -7,6 +7,8 @@ import { ExifData, ExifImage } from 'exif'; import { Opt } from '../new_fields/Doc'; import { SharedMediaTypes } from './SharedMediaTypes'; import { filesDirectory } from '.'; +import { File } from 'formidable'; +import { extname, basename } from "path"; const uploadDirectory = path.join(__dirname, './public/files/'); @@ -45,175 +47,195 @@ export namespace DashUploadUtils { contentType?: string; } - const generate = (prefix: string, url: string) => `${prefix}upload_${Utils.GenerateGuid()}${sanitizeExtension(url)}`; - const sanitize = (filename: string) => filename.replace(/\s+/g, "_"); - const sanitizeExtension = (source: string) => { - let extension = path.extname(source); - extension = extension.toLowerCase(); - extension = extension.split("?")[0]; - return extension; - }; - - /** - * Uploads an image specified by the @param source to Dash's /public/files/ - * directory, and returns information generated during that upload - * - * @param {string} source is either the absolute path of an already uploaded image or - * the url of a remote image - * @param {string} filename dictates what to call the image. If not specified, - * the name {@param prefix}_upload_{GUID} - * @param {string} prefix is a string prepended to the generated image name in the - * event that @param filename is not specified - * - * @returns {UploadInformation} This method returns - * 1) the paths to the uploaded images (plural due to resizing) - * 2) the file name of each of the resized images - * 3) the size of the image, in bytes (4432130) - * 4) the content type of the image, i.e. image/(jpeg | png | ...) - */ - export const UploadImage = async (source: string, filename?: string, prefix: string = ""): Promise => { - const metadata = await InspectImage(source); - return UploadInspectedImage(metadata, filename, prefix); - }; - - export interface InspectionResults { - isLocal: boolean; - stream: any; - normalizedUrl: string; - exifData: EnrichedExifData; - contentSize?: number; - contentType?: string; - } - - export interface EnrichedExifData { - data: ExifData; - error?: string; - } - - export enum Partitions { - pdf_text, - images, - videos - } - - export async function buildFilePartitions() { - const pending = Object.keys(Partitions).map(sub => createIfNotExists(filesDirectory + sub)); - return Promise.all(pending); - } - - /** - * Based on the url's classification as local or remote, gleans - * as much information as possible about the specified image - * - * @param source is the path or url to the image in question - */ - export const InspectImage = async (source: string): Promise => { - const { isLocal, stream, normalized: normalizedUrl } = classify(source); - const exifData = await parseExifData(source); - const results = { - exifData, - isLocal, - stream, - normalizedUrl - }; - // stop here if local, since request.head() can't handle local paths, only urls on the web - if (isLocal) { - return results; + export function upload(file: File): any { + const { type, path, name } = file; + const filename = basename(path); + const extension = extname(path).toLowerCase(); + if (extension === ".pdf") { + + } else if { + let partition: Opt; + if(imageFormats.includes(extension)) { + partition = DashUploadUtils.Partitions.images; + } else if (videoFormats.includes(extension)) { + partition = DashUploadUtils.Partitions.videos; } - const metadata = (await new Promise((resolve, reject) => { - request.head(source, async (error, res) => { - if (error) { - return reject(error); - } - resolve(res); - }); - })).headers; - return { - contentSize: parseInt(metadata[size]), - contentType: metadata[type], - ...results - }; + let uploadInformation: Opt; + if (partition) { + uploadInformation = await DashUploadUtils.UploadImage(`${filesDirectory}/${partition}/${filename}`, filename); + } else { + console.log(`Unable to accommodate, and ignored, the following file upload: ${filename}`); + } + } + const exif = uploadInformation ? uploadInformation.exifData : undefined; + results.push({ name, type, path: `/files/${filename}`, exif }); + +} + +const generate = (prefix: string, url: string) => `${prefix}upload_${Utils.GenerateGuid()}${sanitizeExtension(url)}`; +const sanitize = (filename: string) => filename.replace(/\s+/g, "_"); +const sanitizeExtension = (source: string) => { + let extension = path.extname(source); + extension = extension.toLowerCase(); + extension = extension.split("?")[0]; + return extension; +}; + +/** + * Uploads an image specified by the @param source to Dash's /public/files/ + * directory, and returns information generated during that upload + * + * @param {string} source is either the absolute path of an already uploaded image or + * the url of a remote image + * @param {string} filename dictates what to call the image. If not specified, + * the name {@param prefix}_upload_{GUID} + * @param {string} prefix is a string prepended to the generated image name in the + * event that @param filename is not specified + * + * @returns {UploadInformation} This method returns + * 1) the paths to the uploaded images (plural due to resizing) + * 2) the file name of each of the resized images + * 3) the size of the image, in bytes (4432130) + * 4) the content type of the image, i.e. image/(jpeg | png | ...) + */ +export const UploadImage = async (source: string, filename?: string, prefix: string = ""): Promise => { + const metadata = await InspectImage(source); + return UploadInspectedImage(metadata, filename, prefix); +}; + +export interface InspectionResults { + isLocal: boolean; + stream: any; + normalizedUrl: string; + exifData: EnrichedExifData; + contentSize?: number; + contentType?: string; +} + +export interface EnrichedExifData { + data: ExifData; + error?: string; +} + +export enum Partitions { + pdf_text = "pdf_text", + images = "images", + videos = "videos" +} + +export async function buildFilePartitions() { + const pending = Object.keys(Partitions).map(sub => createIfNotExists(filesDirectory + sub)); + return Promise.all(pending); +} + +/** + * Based on the url's classification as local or remote, gleans + * as much information as possible about the specified image + * + * @param source is the path or url to the image in question + */ +export const InspectImage = async (source: string): Promise => { + const { isLocal, stream, normalized: normalizedUrl } = classify(source); + const exifData = await parseExifData(source); + const results = { + exifData, + isLocal, + stream, + normalizedUrl }; - - export const UploadInspectedImage = async (metadata: InspectionResults, filename?: string, prefix = ""): Promise => { - const { isLocal, stream, normalizedUrl, contentSize, contentType, exifData } = metadata; - const resolved = filename ? sanitize(filename) : generate(prefix, normalizedUrl); - const extension = sanitizeExtension(normalizedUrl || resolved); - let information: UploadInformation = { - mediaPaths: [], - fileNames: { clean: resolved }, - exifData, - contentSize, - contentType, - }; - const { pngs, imageFormats, jpgs, videoFormats } = SharedMediaTypes; - return new Promise(async (resolve, reject) => { - const resizers = [ - { resizer: sharp().rotate(), suffix: "_o" }, - ...Object.values(Sizes).map(size => ({ - resizer: sharp().resize(size.width, undefined, { withoutEnlargement: true }).rotate(), - suffix: size.suffix - })) - ]; - let nonVisual = false; - if (pngs.includes(extension)) { - resizers.forEach(element => element.resizer = element.resizer.png()); - } else if (jpgs.includes(extension)) { - resizers.forEach(element => element.resizer = element.resizer.jpeg()); - } else if (![...imageFormats, ...videoFormats].includes(extension.toLowerCase())) { - nonVisual = true; - } - if (imageFormats.includes(extension)) { - for (let resizer of resizers) { - const suffix = resizer.suffix; - let mediaPath: string; - await new Promise(resolve => { - const filename = resolved.substring(0, resolved.length - extension.length) + suffix + extension; - information.mediaPaths.push(mediaPath = uploadDirectory + filename); - information.fileNames[suffix] = filename; - stream(normalizedUrl).pipe(resizer.resizer).pipe(fs.createWriteStream(mediaPath)) - .on('close', resolve) - .on('error', reject); - }); - } - } - if (!isLocal || nonVisual) { - await new Promise(resolve => { - stream(normalizedUrl).pipe(fs.createWriteStream(uploadDirectory + resolved)).on('close', resolve); - }); + // stop here if local, since request.head() can't handle local paths, only urls on the web + if (isLocal) { + return results; + } + const metadata = (await new Promise((resolve, reject) => { + request.head(source, async (error, res) => { + if (error) { + return reject(error); } - resolve(information); + resolve(res); }); + })).headers; + return { + contentSize: parseInt(metadata[size]), + contentType: metadata[type], + ...results }; - - const classify = (url: string) => { - const isLocal = /Dash-Web(\\|\/)src(\\|\/)server(\\|\/)public(\\|\/)files/g.test(url); - return { - isLocal, - stream: isLocal ? fs.createReadStream : request, - normalized: isLocal ? path.normalize(url) : url - }; +}; + +export const UploadInspectedImage = async (metadata: InspectionResults, filename?: string, prefix = ""): Promise => { + const { isLocal, stream, normalizedUrl, contentSize, contentType, exifData } = metadata; + const resolved = filename ? sanitize(filename) : generate(prefix, normalizedUrl); + const extension = sanitizeExtension(normalizedUrl || resolved); + let information: UploadInformation = { + mediaPaths: [], + fileNames: { clean: resolved }, + exifData, + contentSize, + contentType, }; - - const parseExifData = async (source: string): Promise => { - return new Promise(resolve => { - new ExifImage(source, (error, data) => { - let reason: Opt = undefined; - if (error) { - reason = (error as any).code; - } - resolve({ data, error: reason }); + const { pngs, jpgs } = SharedMediaTypes; + return new Promise(async (resolve, reject) => { + const resizers = [ + { resizer: sharp().rotate(), suffix: "_o" }, + ...Object.values(Sizes).map(size => ({ + resizer: sharp().resize(size.width, undefined, { withoutEnlargement: true }).rotate(), + suffix: size.suffix + })) + ]; + if (pngs.includes(extension)) { + resizers.forEach(element => element.resizer = element.resizer.png()); + } else if (jpgs.includes(extension)) { + resizers.forEach(element => element.resizer = element.resizer.jpeg()); + } + for (let resizer of resizers) { + const suffix = resizer.suffix; + let mediaPath: string; + await new Promise(resolve => { + const filename = resolved.substring(0, resolved.length - extension.length) + suffix + extension; + information.mediaPaths.push(mediaPath = uploadDirectory + filename); + information.fileNames[suffix] = filename; + stream(normalizedUrl).pipe(resizer.resizer).pipe(fs.createWriteStream(mediaPath)) + .on('close', resolve) + .on('error', reject); + }); + } + if (!isLocal) { + await new Promise(resolve => { + stream(normalizedUrl).pipe(fs.createWriteStream(uploadDirectory + resolved)).on('close', resolve); }); - }); - }; - - export const createIfNotExists = async (path: string) => { - if (await new Promise(resolve => fs.exists(path, resolve))) { - return true; } - return new Promise(resolve => fs.mkdir(path, error => resolve(error === null))); + resolve(information); + }); +}; + +const classify = (url: string) => { + const isLocal = /Dash-Web(\\|\/)src(\\|\/)server(\\|\/)public(\\|\/)files/g.test(url); + return { + isLocal, + stream: isLocal ? fs.createReadStream : request, + normalized: isLocal ? path.normalize(url) : url }; +}; + +const parseExifData = async (source: string): Promise => { + return new Promise(resolve => { + new ExifImage(source, (error, data) => { + let reason: Opt = undefined; + if (error) { + reason = (error as any).code; + } + resolve({ data, error: reason }); + }); + }); +}; + +export const createIfNotExists = async (path: string) => { + if (await new Promise(resolve => fs.exists(path, resolve))) { + return true; + } + return new Promise(resolve => fs.mkdir(path, error => resolve(error === null))); +}; - export const Destroy = (mediaPath: string) => new Promise(resolve => fs.unlink(mediaPath, error => resolve(error === null))); +export const Destroy = (mediaPath: string) => new Promise(resolve => fs.unlink(mediaPath, error => resolve(error === null))); } \ No newline at end of file diff --git a/src/server/public/files/.gitignore b/src/server/public/files/.gitignore deleted file mode 100644 index c96a04f00..000000000 --- a/src/server/public/files/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -* -!.gitignore \ No newline at end of file -- cgit v1.2.3-70-g09d2 From df5584ccd40bd83f1362b32db67969e7ffbf2e3f Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Wed, 27 Nov 2019 04:03:30 -0500 Subject: improved file partitioning in server and generified upload method --- package.json | 2 + src/client/documents/Documents.ts | 1 - src/client/util/ClientDiagnostics.ts | 11 +- .../util/Import & Export/DirectoryImportBox.tsx | 19 +- src/client/views/Main.tsx | 2 +- src/client/views/collections/CollectionSubView.tsx | 22 +- src/client/views/pdf/PDFViewer.tsx | 3 +- src/server/ActionUtilities.ts | 17 +- src/server/ApiManagers/DownloadManager.ts | 4 +- src/server/ApiManagers/GooglePhotosManager.ts | 4 +- src/server/ApiManagers/PDFManager.ts | 34 +- src/server/ApiManagers/SearchManager.ts | 4 +- src/server/ApiManagers/UploadManager.ts | 47 ++- src/server/DashUploadUtils.ts | 388 +++++++++++---------- src/server/SharedMediaTypes.ts | 5 +- src/server/database.ts | 4 +- src/server/index.ts | 4 +- 17 files changed, 314 insertions(+), 257 deletions(-) (limited to 'src') diff --git a/package.json b/package.json index 393df8574..3725d76eb 100644 --- a/package.json +++ b/package.json @@ -105,6 +105,7 @@ "@types/react-table": "^6.7.22", "@types/request": "^2.48.1", "@types/request-promise": "^4.1.42", + "@types/rimraf": "^2.0.3", "@types/sharp": "^0.22.2", "@types/shelljs": "^0.8.5", "@types/socket.io": "^2.1.2", @@ -211,6 +212,7 @@ "readline": "^1.3.0", "request": "^2.88.0", "request-promise": "^4.2.4", + "rimraf": "^3.0.0", "serializr": "^1.5.1", "sharp": "^0.22.1", "shelljs": "^0.8.3", diff --git a/src/client/documents/Documents.ts b/src/client/documents/Documents.ts index dea057b93..a2f4d23c7 100644 --- a/src/client/documents/Documents.ts +++ b/src/client/documents/Documents.ts @@ -377,7 +377,6 @@ export namespace Docs { let extension = path.extname(target); target = `${target.substring(0, target.length - extension.length)}_o${extension}`; } - // if (target !== "http://www.cs.brown.edu/") { requestImageSize(target) .then((size: any) => { let aspect = size.height / size.width; diff --git a/src/client/util/ClientDiagnostics.ts b/src/client/util/ClientDiagnostics.ts index 24f196252..7eef935fd 100644 --- a/src/client/util/ClientDiagnostics.ts +++ b/src/client/util/ClientDiagnostics.ts @@ -1,9 +1,6 @@ -import { observable, runInAction } from "mobx"; -import { MainView } from "../views/MainView"; - export namespace ClientDiagnostics { - export function start() { + export async function start() { let serverPolls = 0; const serverHandle = setInterval(async () => { @@ -17,14 +14,16 @@ export namespace ClientDiagnostics { let executed = false; - const solrHandle = setInterval(async () => { + const handle = async () => { const response = await fetch("/solrHeartbeat"); if (!(await response.json()).running) { !executed && alert("Looks like SOLR is not running on your machine."); executed = true; clearInterval(solrHandle); } - }, 1000 * 15); + }; + await handle(); + const solrHandle = setInterval(handle, 1000 * 15); } diff --git a/src/client/util/Import & Export/DirectoryImportBox.tsx b/src/client/util/Import & Export/DirectoryImportBox.tsx index f0880f193..16ae50685 100644 --- a/src/client/util/Import & Export/DirectoryImportBox.tsx +++ b/src/client/util/Import & Export/DirectoryImportBox.tsx @@ -22,18 +22,10 @@ import "./DirectoryImportBox.scss"; import { Networking } from "../../Network"; import { BatchedArray } from "array-batcher"; import * as path from 'path'; -import { DashUploadUtils } from "../../../server/DashUploadUtils"; -import { SharedMediaTypes } from "../../../server/SharedMediaTypes"; +import { AcceptibleMedia } from "../../../server/SharedMediaTypes"; const unsupported = ["text/html", "text/plain"]; -interface ImageUploadResponse { - name: string; - path: string; - type: string; - exif: any; -} - @observer export default class DirectoryImportBox extends React.Component { private selector = React.createRef(); @@ -98,7 +90,7 @@ export default class DirectoryImportBox extends React.Component let file = files.item(i); if (file && !unsupported.includes(file.type)) { const ext = path.extname(file.name).toLowerCase(); - if (SharedMediaTypes.imageFormats.includes(ext)) { + if (AcceptibleMedia.imageFormats.includes(ext)) { validated.push(file); } } @@ -114,7 +106,7 @@ export default class DirectoryImportBox extends React.Component runInAction(() => this.phase = `Internal: uploading ${this.quota - this.completed} files to Dash...`); - const uploads = await BatchedArray.from(validated, { batchSize: 15 }).batchedMapAsync(async (batch, collector) => { + const uploads = await BatchedArray.from(validated, { batchSize: 15 }).batchedMapAsync(async (batch, collector) => { const formData = new FormData(); batch.forEach(file => { @@ -127,16 +119,17 @@ export default class DirectoryImportBox extends React.Component runInAction(() => this.completed += batch.length); }); + const size = "_o"; await Promise.all(uploads.map(async upload => { const type = upload.type; - const path = Utils.prepend(upload.path); + const path = Utils.prepend(upload.clientAccessPath); const options = { nativeWidth: 300, width: 300, title: upload.name }; const document = await Docs.Get.DocumentFromType(type, path, options); - const { data, error } = upload.exif; + const { data, error } = upload.exifData; if (document) { Doc.GetProto(document).exif = error || Docs.Get.DocumentHierarchyFromJson(data); docs.push(document); diff --git a/src/client/views/Main.tsx b/src/client/views/Main.tsx index dec4a24e4..9e699978f 100644 --- a/src/client/views/Main.tsx +++ b/src/client/views/Main.tsx @@ -10,7 +10,7 @@ import { ClientDiagnostics } from "../util/ClientDiagnostics"; AssignAllExtensions(); (async () => { - ClientDiagnostics.start(); + await ClientDiagnostics.start(); const info = await CurrentUserUtils.loadCurrentUser(); DocServer.init(window.location.protocol, window.location.hostname, 4321, info.email); await Docs.Prototypes.initialize(); diff --git a/src/client/views/collections/CollectionSubView.tsx b/src/client/views/collections/CollectionSubView.tsx index 1c3ff37ee..a1bd1527e 100644 --- a/src/client/views/collections/CollectionSubView.tsx +++ b/src/client/views/collections/CollectionSubView.tsx @@ -22,6 +22,7 @@ import React = require("react"); var path = require('path'); import { GooglePhotos } from "../../apis/google_docs/GooglePhotosClientUtils"; import { ImageUtils } from "../../util/Import & Export/ImageUtils"; +import { Networking } from "../../Network"; export interface CollectionViewProps extends FieldViewProps { addDocument: (document: Doc) => boolean; @@ -271,28 +272,25 @@ export function CollectionSubView(schemaCtor: (doc: Doc) => T) { let file = item.getAsFile(); let formData = new FormData(); - if (file) { - formData.append('file', file); + if (!file || !file.type) { + continue; } - let dropFileName = file ? file.name : "-empty-"; - let prom = fetch(Utils.prepend("/upload"), { - method: 'POST', - body: formData - }).then(async (res: Response) => { - (await res.json()).map(action((file: any) => { + formData.append('file', file); + let dropFileName = file ? file.name : "-empty-"; + promises.push(Networking.PostFormDataToServer("/upload", formData).then(results => { + results.map(action((file: any) => { let full = { ...options, nativeWidth: type.indexOf("video") !== -1 ? 600 : 300, width: 300, title: dropFileName }; - let pathname = Utils.prepend(file.path); + let pathname = Utils.prepend(file.clientAccessPath); Docs.Get.DocumentFromType(type, pathname, full).then(doc => { doc && (Doc.GetProto(doc).fileUpload = path.basename(pathname).replace("upload_", "").replace(/\.[a-z0-9]*$/, "")); doc && this.props.addDocument(doc); }); })); - }); - promises.push(prom); + })); } } - if (text) { + if (text && !text.includes("https://")) { this.props.addDocument(Docs.Create.TextDocument({ ...options, documentText: "@@@" + text, width: 400, height: 315 })); return; } diff --git a/src/client/views/pdf/PDFViewer.tsx b/src/client/views/pdf/PDFViewer.tsx index b737ce221..c075a4f99 100644 --- a/src/client/views/pdf/PDFViewer.tsx +++ b/src/client/views/pdf/PDFViewer.tsx @@ -125,7 +125,8 @@ export class PDFViewer extends DocAnnotatableComponent this._showWaiting = this._showCover = true); this.props.startupLive && this.setupPdfJsViewer(); this._searchReactionDisposer = reaction(() => this.Document.search_string, searchString => { diff --git a/src/server/ActionUtilities.ts b/src/server/ActionUtilities.ts index 7f493dd70..c9fc86fea 100644 --- a/src/server/ActionUtilities.ts +++ b/src/server/ActionUtilities.ts @@ -2,6 +2,7 @@ import * as fs from 'fs'; import { ExecOptions } from 'shelljs'; import { exec } from 'child_process'; import * as path from 'path'; +import * as rimraf from "rimraf"; export const command_line = (command: string, fromDirectory?: string) => { return new Promise((resolve, reject) => { @@ -68,4 +69,18 @@ export function msToTime(duration: number) { let secondsS = (seconds < 10) ? "0" + seconds : seconds; return hoursS + ":" + minutesS + ":" + secondsS + "." + milliseconds; -} \ No newline at end of file +} + +export const createIfNotExists = async (path: string) => { + if (await new Promise(resolve => fs.exists(path, resolve))) { + return true; + } + return new Promise(resolve => fs.mkdir(path, error => resolve(error === null))); +}; + +export async function Prune(rootDirectory: string): Promise { + const error = await new Promise(resolve => rimraf(rootDirectory, resolve)); + return error === null; +} + +export const Destroy = (mediaPath: string) => new Promise(resolve => fs.unlink(mediaPath, error => resolve(error === null))); \ No newline at end of file diff --git a/src/server/ApiManagers/DownloadManager.ts b/src/server/ApiManagers/DownloadManager.ts index fc6ba0d22..5bad46eda 100644 --- a/src/server/ApiManagers/DownloadManager.ts +++ b/src/server/ApiManagers/DownloadManager.ts @@ -5,7 +5,7 @@ import * as Archiver from 'archiver'; import * as express from 'express'; import { Database } from "../database"; import * as path from "path"; -import { DashUploadUtils } from "../DashUploadUtils"; +import { DashUploadUtils, SizeSuffix } from "../DashUploadUtils"; import { publicDirectory } from ".."; export type Hierarchy = { [id: string]: string | Hierarchy }; @@ -254,7 +254,7 @@ async function writeHierarchyRecursive(file: Archiver.Archiver, hierarchy: Hiera // and dropped in the browser and thus hosted remotely) so we upload it // to our server and point the zip file to it, so it can bundle up the bytes const information = await DashUploadUtils.UploadImage(result); - path = information.mediaPaths[0]; + path = information.serverAccessPaths[SizeSuffix.Original]; } // write the file specified by the path to the directory in the // zip file given by the prefix. diff --git a/src/server/ApiManagers/GooglePhotosManager.ts b/src/server/ApiManagers/GooglePhotosManager.ts index c7af69375..5a709688b 100644 --- a/src/server/ApiManagers/GooglePhotosManager.ts +++ b/src/server/ApiManagers/GooglePhotosManager.ts @@ -86,10 +86,10 @@ export default class GooglePhotosManager extends ApiManager { const contents: { mediaItems: MediaItem[] } = req.body; let failed = 0; if (contents) { - const completed: Opt[] = []; + const completed: Opt[] = []; for (let item of contents.mediaItems) { const { contentSize, ...attributes } = await DashUploadUtils.InspectImage(item.baseUrl); - const found: Opt = await Database.Auxiliary.QueryUploadHistory(contentSize!); + const found: Opt = await Database.Auxiliary.QueryUploadHistory(contentSize!); if (!found) { const upload = await DashUploadUtils.UploadInspectedImage({ contentSize, ...attributes }, item.filename, prefix).catch(error => _error(res, downloadError, error)); if (upload) { diff --git a/src/server/ApiManagers/PDFManager.ts b/src/server/ApiManagers/PDFManager.ts index 632b4965a..4bd750aaf 100644 --- a/src/server/ApiManagers/PDFManager.ts +++ b/src/server/ApiManagers/PDFManager.ts @@ -2,12 +2,12 @@ import ApiManager, { Registration } from "./ApiManager"; import { Method } from "../RouteManager"; import RouteSubscriber from "../RouteSubscriber"; import { exists, createReadStream, createWriteStream } from "fs"; -import { filesDirectory } from ".."; import * as Pdfjs from 'pdfjs-dist'; import { createCanvas } from "canvas"; const probe = require("probe-image-size"); import * as express from "express"; import * as path from "path"; +import { Directory, serverPathToFile, clientPathToFile } from "./UploadManager"; export default class PDFManager extends ApiManager { @@ -21,21 +21,27 @@ export default class PDFManager extends ApiManager { let noExt = filename.substring(0, filename.length - ".png".length); let pagenumber = parseInt(noExt.split('-')[1]); return new Promise(resolve => { - exists(filesDirectory + filename, (exists: boolean) => { - console.log(`${filesDirectory + filename} ${exists ? "exists" : "does not exist"}`); + const path = serverPathToFile(Directory.pdf_thumbnails, filename); + exists(path, (exists: boolean) => { + console.log(`${path} ${exists ? "exists" : "does not exist"}`); if (exists) { - let input = createReadStream(filesDirectory + filename); - probe(input, (err: any, result: any) => { + let input = createReadStream(path); + probe(input, (err: any, { width, height }: any) => { if (err) { console.log(err); console.log(`error on ${filename}`); return; } - res.send({ path: "/files/" + filename, width: result.width, height: result.height }); + res.send({ + path: clientPathToFile(Directory.pdf_thumbnails, filename), + width, + height + }); }); } else { - LoadPage(filesDirectory + filename.substring(0, filename.length - noExt.split('-')[1].length - ".PNG".length - 1) + ".pdf", pagenumber, res); + const name = filename.substring(0, filename.length - noExt.split('-')[1].length - ".PNG".length - 1) + ".pdf"; + LoadPage(serverPathToFile(Directory.pdfs, name), pagenumber, res); } resolve(); }); @@ -55,8 +61,8 @@ export default class PDFManager extends ApiManager { let canvasAndContext = factory.create(viewport.width, viewport.height); let renderContext = { canvasContext: canvasAndContext.context, - viewport: viewport, - canvasFactory: factory + canvasFactory: factory, + viewport }; console.log("read " + pageNumber); @@ -64,13 +70,17 @@ export default class PDFManager extends ApiManager { .then(() => { console.log("saving " + pageNumber); let stream = canvasAndContext.canvas.createPNGStream(); - let pngFile = `${file.substring(0, file.length - ".pdf".length)}-${pageNumber}.PNG`; + let filenames = path.basename(file).split("."); + const pngFile = serverPathToFile(Directory.pdf_thumbnails, `${filenames[0]}-${pageNumber}.png`); let out = createWriteStream(pngFile); stream.pipe(out); out.on("finish", () => { console.log(`Success! Saved to ${pngFile}`); - let name = path.basename(pngFile); - res.send({ path: "/files/" + name, width: viewport.width, height: viewport.height }); + res.send({ + path: pngFile, + width: viewport.width, + height: viewport.height + }); }); }, (reason: string) => { console.error(reason + ` ${pageNumber}`); diff --git a/src/server/ApiManagers/SearchManager.ts b/src/server/ApiManagers/SearchManager.ts index 1c801715a..d3f8995b0 100644 --- a/src/server/ApiManagers/SearchManager.ts +++ b/src/server/ApiManagers/SearchManager.ts @@ -3,7 +3,7 @@ import { Method } from "../RouteManager"; import { Search } from "../Search"; var findInFiles = require('find-in-files'); import * as path from 'path'; -import { filesDirectory } from ".."; +import { pathToDirectory, Directory } from "./UploadManager"; export default class SearchManager extends ApiManager { @@ -18,7 +18,7 @@ export default class SearchManager extends ApiManager { res.send([]); return; } - let results = await findInFiles.find({ 'term': q, 'flags': 'ig' }, filesDirectory + "text", ".txt$"); + let results = await findInFiles.find({ 'term': q, 'flags': 'ig' }, pathToDirectory(Directory.text), ".txt$"); let resObj: { ids: string[], numFound: number, lines: string[] } = { ids: [], numFound: 0, lines: [] }; for (var result in results) { resObj.ids.push(path.basename(result, ".txt").replace(/upload_/, "")); diff --git a/src/server/ApiManagers/UploadManager.ts b/src/server/ApiManagers/UploadManager.ts index 2a9faacd8..2f76871a6 100644 --- a/src/server/ApiManagers/UploadManager.ts +++ b/src/server/ApiManagers/UploadManager.ts @@ -7,14 +7,33 @@ import { extname, basename, dirname } from 'path'; import { createReadStream, createWriteStream, unlink, readFileSync } from "fs"; import { publicDirectory, filesDirectory } from ".."; import { Database } from "../database"; -import { DashUploadUtils } from "../DashUploadUtils"; -import { Opt } from "../../new_fields/Doc"; -import { ParsedPDF } from "../PdfTypes"; -const pdf = require('pdf-parse'); +import { DashUploadUtils, SizeSuffix } from "../DashUploadUtils"; import * as sharp from 'sharp'; -import { SharedMediaTypes } from "../SharedMediaTypes"; +import { AcceptibleMedia } from "../SharedMediaTypes"; +import { normalize } from "path"; const imageDataUri = require('image-data-uri'); +export enum Directory { + parsed_files = "parsed_files", + images = "images", + videos = "videos", + pdfs = "pdfs", + text = "text", + pdf_thumbnails = "pdf_thumbnails" +} + +export function serverPathToFile(directory: Directory, filename: string) { + return normalize(`${filesDirectory}/${directory}/${filename}`); +} + +export function pathToDirectory(directory: Directory) { + return normalize(`${filesDirectory}/${directory}`); +} + +export function clientPathToFile(directory: Directory, filename: string) { + return `/files/${directory}/${filename}`; +} + export default class UploadManager extends ApiManager { protected initialize(register: Registration): void { @@ -129,13 +148,14 @@ export default class UploadManager extends ApiManager { subscription: "/upload", onValidation: async ({ req, res }) => { let form = new formidable.IncomingForm(); - form.uploadDir = filesDirectory; + form.uploadDir = pathToDirectory(Directory.parsed_files); form.keepExtensions = true; return new Promise(resolve => { form.parse(req, async (_err, _fields, files) => { let results: any[] = []; for (const key in files) { - results.push(DashUploadUtils.upload(files[key])); + const result = await DashUploadUtils.upload(files[key]); + result && results.push(result); } _success(res, results); resolve(); @@ -150,8 +170,8 @@ export default class UploadManager extends ApiManager { onValidation: async ({ req, res }) => { const { source } = req.body; if (typeof source === "string") { - const uploadInformation = await DashUploadUtils.UploadImage(source); - return res.send(await DashUploadUtils.InspectImage(uploadInformation.mediaPaths[0])); + const { serverAccessPaths } = await DashUploadUtils.UploadImage(source); + return res.send(await DashUploadUtils.InspectImage(serverAccessPaths[SizeSuffix.Original])); } res.send({}); } @@ -167,9 +187,9 @@ export default class UploadManager extends ApiManager { res.status(401).send("incorrect parameters specified"); return; } - return imageDataUri.outputFile(uri, filesDirectory + filename).then((savedName: string) => { + return imageDataUri.outputFile(uri, serverPathToFile(Directory.images, filename)).then((savedName: string) => { const ext = extname(savedName).toLowerCase(); - const { pngs, jpgs } = SharedMediaTypes; + const { pngs, jpgs } = AcceptibleMedia; let resizers = [ { resizer: sharp().resize(100, undefined, { withoutEnlargement: true }), suffix: "_s" }, { resizer: sharp().resize(400, undefined, { withoutEnlargement: true }), suffix: "_m" }, @@ -189,10 +209,11 @@ export default class UploadManager extends ApiManager { } if (isImage) { resizers.forEach(resizer => { - createReadStream(savedName).pipe(resizer.resizer).pipe(createWriteStream(filesDirectory + filename + resizer.suffix + ext)); + const path = serverPathToFile(Directory.images, filename + resizer.suffix + ext); + createReadStream(savedName).pipe(resizer.resizer).pipe(createWriteStream(path)); }); } - res.send("/files/" + filename + ext); + res.send(clientPathToFile(Directory.images, filename + ext)); }); } }); diff --git a/src/server/DashUploadUtils.ts b/src/server/DashUploadUtils.ts index 839aada4b..0a670ec01 100644 --- a/src/server/DashUploadUtils.ts +++ b/src/server/DashUploadUtils.ts @@ -5,18 +5,32 @@ import * as sharp from 'sharp'; import request = require('request-promise'); import { ExifData, ExifImage } from 'exif'; import { Opt } from '../new_fields/Doc'; -import { SharedMediaTypes } from './SharedMediaTypes'; +import { AcceptibleMedia } from './SharedMediaTypes'; import { filesDirectory } from '.'; import { File } from 'formidable'; -import { extname, basename } from "path"; +import { basename } from "path"; +import { ConsoleColors, createIfNotExists } from './ActionUtilities'; +import { ParsedPDF } from "../server/PdfTypes"; +const parse = require('pdf-parse'); +import { Directory, serverPathToFile, clientPathToFile } from './ApiManagers/UploadManager'; -const uploadDirectory = path.join(__dirname, './public/files/'); +export enum SizeSuffix { + Small = "_s", + Medium = "_m", + Large = "_l", + Original = "_o" +} export namespace DashUploadUtils { + function InjectSize(filename: string, size: SizeSuffix) { + const extension = path.extname(filename).toLowerCase(); + return filename.substring(0, filename.length - extension.length) + size + extension; + } + export interface Size { width: number; - suffix: string; + suffix: SizeSuffix; } export interface ImageFileResponse { @@ -27,215 +41,221 @@ export namespace DashUploadUtils { } export const Sizes: { [size: string]: Size } = { - SMALL: { width: 100, suffix: "_s" }, - MEDIUM: { width: 400, suffix: "_m" }, - LARGE: { width: 900, suffix: "_l" }, + SMALL: { width: 100, suffix: SizeSuffix.Small }, + MEDIUM: { width: 400, suffix: SizeSuffix.Medium }, + LARGE: { width: 900, suffix: SizeSuffix.Large }, }; export function validateExtension(url: string) { - return SharedMediaTypes.imageFormats.includes(path.extname(url).toLowerCase()); + return AcceptibleMedia.imageFormats.includes(path.extname(url).toLowerCase()); } const size = "content-length"; const type = "content-type"; - export interface UploadInformation { - mediaPaths: string[]; - fileNames: { [key: string]: string }; + export interface ImageUploadInformation { + clientAccessPath: string; + serverAccessPaths: { [key: string]: string }; exifData: EnrichedExifData; contentSize?: number; contentType?: string; } - export function upload(file: File): any { + export async function upload(file: File): Promise { const { type, path, name } = file; - const filename = basename(path); - const extension = extname(path).toLowerCase(); - if (extension === ".pdf") { - - } else if { - let partition: Opt; - if(imageFormats.includes(extension)) { - partition = DashUploadUtils.Partitions.images; - } else if (videoFormats.includes(extension)) { - partition = DashUploadUtils.Partitions.videos; - } - let uploadInformation: Opt; - if (partition) { - uploadInformation = await DashUploadUtils.UploadImage(`${filesDirectory}/${partition}/${filename}`, filename); - } else { - console.log(`Unable to accommodate, and ignored, the following file upload: ${filename}`); + const { imageFormats, videoFormats, applicationFormats } = AcceptibleMedia; + const types = type.split("/"); + + const category = types[0]; + const format = `.${types[1]}`; + + switch (category) { + case "image": + if (imageFormats.includes(format)) { + const { clientAccessPath } = await UploadImage(path, basename(path), format); + return { clientAccessPath, name, type }; + } + case "video": + if (videoFormats.includes(format)) { + return MoveParsedFile(path, Directory.videos); + } + case "application": + if (applicationFormats.includes(format)) { + return UploadPdf(path); + } } + console.log(ConsoleColors.Red, `Ignoring unsupported file ${name} with upload type (${type}).`); + return { clientAccessPath: undefined }; } - const exif = uploadInformation ? uploadInformation.exifData : undefined; - results.push({ name, type, path: `/files/${filename}`, exif }); -} + async function UploadPdf(absolutePath: string) { + let dataBuffer = fs.readFileSync(absolutePath); + const result: ParsedPDF = await parse(dataBuffer); + const parsedName = basename(absolutePath); + await new Promise((resolve, reject) => { + const textFilename = `${parsedName.substring(0, parsedName.length - 4)}.txt`; + const writeStream = fs.createWriteStream(serverPathToFile(Directory.text, textFilename)); + writeStream.write(result.text, error => error ? reject(error) : resolve()); + }); + return MoveParsedFile(absolutePath, Directory.pdfs); + } -const generate = (prefix: string, url: string) => `${prefix}upload_${Utils.GenerateGuid()}${sanitizeExtension(url)}`; -const sanitize = (filename: string) => filename.replace(/\s+/g, "_"); -const sanitizeExtension = (source: string) => { - let extension = path.extname(source); - extension = extension.toLowerCase(); - extension = extension.split("?")[0]; - return extension; -}; - -/** - * Uploads an image specified by the @param source to Dash's /public/files/ - * directory, and returns information generated during that upload - * - * @param {string} source is either the absolute path of an already uploaded image or - * the url of a remote image - * @param {string} filename dictates what to call the image. If not specified, - * the name {@param prefix}_upload_{GUID} - * @param {string} prefix is a string prepended to the generated image name in the - * event that @param filename is not specified - * - * @returns {UploadInformation} This method returns - * 1) the paths to the uploaded images (plural due to resizing) - * 2) the file name of each of the resized images - * 3) the size of the image, in bytes (4432130) - * 4) the content type of the image, i.e. image/(jpeg | png | ...) - */ -export const UploadImage = async (source: string, filename?: string, prefix: string = ""): Promise => { - const metadata = await InspectImage(source); - return UploadInspectedImage(metadata, filename, prefix); -}; - -export interface InspectionResults { - isLocal: boolean; - stream: any; - normalizedUrl: string; - exifData: EnrichedExifData; - contentSize?: number; - contentType?: string; -} + const generate = (prefix: string, url: string) => `${prefix}upload_${Utils.GenerateGuid()}${sanitizeExtension(url)}`; + const sanitizeExtension = (source: string) => { + let extension = path.extname(source); + extension = extension.toLowerCase(); + extension = extension.split("?")[0]; + return extension; + }; -export interface EnrichedExifData { - data: ExifData; - error?: string; -} + /** + * Uploads an image specified by the @param source to Dash's /public/files/ + * directory, and returns information generated during that upload + * + * @param {string} source is either the absolute path of an already uploaded image or + * the url of a remote image + * @param {string} filename dictates what to call the image. If not specified, + * the name {@param prefix}_upload_{GUID} + * @param {string} prefix is a string prepended to the generated image name in the + * event that @param filename is not specified + * + * @returns {ImageUploadInformation} This method returns + * 1) the paths to the uploaded images (plural due to resizing) + * 2) the file name of each of the resized images + * 3) the size of the image, in bytes (4432130) + * 4) the content type of the image, i.e. image/(jpeg | png | ...) + */ + export const UploadImage = async (source: string, filename?: string, format?: string, prefix: string = ""): Promise => { + const metadata = await InspectImage(source); + return UploadInspectedImage(metadata, filename, format, prefix); + }; -export enum Partitions { - pdf_text = "pdf_text", - images = "images", - videos = "videos" -} + export interface InspectionResults { + isLocal: boolean; + stream: any; + normalizedUrl: string; + exifData: EnrichedExifData; + contentSize?: number; + contentType?: string; + } -export async function buildFilePartitions() { - const pending = Object.keys(Partitions).map(sub => createIfNotExists(filesDirectory + sub)); - return Promise.all(pending); -} + export interface EnrichedExifData { + data: ExifData; + error?: string; + } -/** - * Based on the url's classification as local or remote, gleans - * as much information as possible about the specified image - * - * @param source is the path or url to the image in question - */ -export const InspectImage = async (source: string): Promise => { - const { isLocal, stream, normalized: normalizedUrl } = classify(source); - const exifData = await parseExifData(source); - const results = { - exifData, - isLocal, - stream, - normalizedUrl - }; - // stop here if local, since request.head() can't handle local paths, only urls on the web - if (isLocal) { - return results; + export async function buildFileDirectories() { + const pending = Object.keys(Directory).map(sub => createIfNotExists(`${filesDirectory}/${sub}`)); + return Promise.all(pending); } - const metadata = (await new Promise((resolve, reject) => { - request.head(source, async (error, res) => { - if (error) { - return reject(error); - } - resolve(res); - }); - })).headers; - return { - contentSize: parseInt(metadata[size]), - contentType: metadata[type], - ...results - }; -}; - -export const UploadInspectedImage = async (metadata: InspectionResults, filename?: string, prefix = ""): Promise => { - const { isLocal, stream, normalizedUrl, contentSize, contentType, exifData } = metadata; - const resolved = filename ? sanitize(filename) : generate(prefix, normalizedUrl); - const extension = sanitizeExtension(normalizedUrl || resolved); - let information: UploadInformation = { - mediaPaths: [], - fileNames: { clean: resolved }, - exifData, - contentSize, - contentType, - }; - const { pngs, jpgs } = SharedMediaTypes; - return new Promise(async (resolve, reject) => { - const resizers = [ - { resizer: sharp().rotate(), suffix: "_o" }, - ...Object.values(Sizes).map(size => ({ - resizer: sharp().resize(size.width, undefined, { withoutEnlargement: true }).rotate(), - suffix: size.suffix - })) - ]; - if (pngs.includes(extension)) { - resizers.forEach(element => element.resizer = element.resizer.png()); - } else if (jpgs.includes(extension)) { - resizers.forEach(element => element.resizer = element.resizer.jpeg()); - } - for (let resizer of resizers) { - const suffix = resizer.suffix; - let mediaPath: string; - await new Promise(resolve => { - const filename = resolved.substring(0, resolved.length - extension.length) + suffix + extension; - information.mediaPaths.push(mediaPath = uploadDirectory + filename); - information.fileNames[suffix] = filename; - stream(normalizedUrl).pipe(resizer.resizer).pipe(fs.createWriteStream(mediaPath)) - .on('close', resolve) - .on('error', reject); - }); + + /** + * Based on the url's classification as local or remote, gleans + * as much information as possible about the specified image + * + * @param source is the path or url to the image in question + */ + export const InspectImage = async (source: string): Promise => { + const { isLocal, stream, normalized: normalizedUrl } = classify(source); + const exifData = await parseExifData(source); + const results = { + exifData, + isLocal, + stream, + normalizedUrl + }; + // stop here if local, since request.head() can't handle local paths, only urls on the web + if (isLocal) { + return results; } - if (!isLocal) { - await new Promise(resolve => { - stream(normalizedUrl).pipe(fs.createWriteStream(uploadDirectory + resolved)).on('close', resolve); + const metadata = (await new Promise((resolve, reject) => { + request.head(source, async (error, res) => { + if (error) { + return reject(error); + } + resolve(res); }); - } - resolve(information); - }); -}; - -const classify = (url: string) => { - const isLocal = /Dash-Web(\\|\/)src(\\|\/)server(\\|\/)public(\\|\/)files/g.test(url); - return { - isLocal, - stream: isLocal ? fs.createReadStream : request, - normalized: isLocal ? path.normalize(url) : url + })).headers; + return { + contentSize: parseInt(metadata[size]), + contentType: metadata[type], + ...results + }; }; -}; - -const parseExifData = async (source: string): Promise => { - return new Promise(resolve => { - new ExifImage(source, (error, data) => { - let reason: Opt = undefined; - if (error) { - reason = (error as any).code; + + export async function MoveParsedFile(absolutePath: string, destination: Directory): Promise<{ clientAccessPath: Opt }> { + return new Promise<{ clientAccessPath: Opt }>(resolve => { + const filename = basename(absolutePath); + const destinationPath = serverPathToFile(destination, filename); + fs.rename(absolutePath, destinationPath, error => { + resolve({ clientAccessPath: error ? undefined : clientPathToFile(destination, filename) }); + }); + }); + } + + export const UploadInspectedImage = async (metadata: InspectionResults, filename?: string, format?: string, prefix = ""): Promise => { + const { isLocal, stream, normalizedUrl, contentSize, contentType, exifData } = metadata; + const resolved = filename || generate(prefix, normalizedUrl); + const extension = format || sanitizeExtension(normalizedUrl || resolved); + let information: ImageUploadInformation = { + clientAccessPath: clientPathToFile(Directory.images, resolved), + serverAccessPaths: {}, + exifData, + contentSize, + contentType, + }; + const { pngs, jpgs } = AcceptibleMedia; + return new Promise(async (resolve, reject) => { + const resizers = [ + { resizer: sharp().rotate(), suffix: SizeSuffix.Original }, + ...Object.values(Sizes).map(size => ({ + resizer: sharp().resize(size.width, undefined, { withoutEnlargement: true }).rotate(), + suffix: size.suffix + })) + ]; + if (pngs.includes(extension)) { + resizers.forEach(element => element.resizer = element.resizer.png()); + } else if (jpgs.includes(extension)) { + resizers.forEach(element => element.resizer = element.resizer.jpeg()); + } + for (let { resizer, suffix } of resizers) { + let mediaPath: string; + await new Promise(resolve => { + const filename = InjectSize(resolved, suffix); + information.serverAccessPaths[suffix] = serverPathToFile(Directory.images, filename); + stream(normalizedUrl).pipe(resizer).pipe(fs.createWriteStream(serverPathToFile(Directory.images, filename))) + .on('close', resolve) + .on('error', reject); + }); } - resolve({ data, error: reason }); + if (isLocal) { + await new Promise(resolve => { + fs.unlink(normalizedUrl, error => resolve(error === null)); + }); + } + resolve(information); }); - }); -}; + }; -export const createIfNotExists = async (path: string) => { - if (await new Promise(resolve => fs.exists(path, resolve))) { - return true; - } - return new Promise(resolve => fs.mkdir(path, error => resolve(error === null))); -}; + const classify = (url: string) => { + const isLocal = /Dash-Web(\\|\/)src(\\|\/)server(\\|\/)public(\\|\/)files/g.test(url); + return { + isLocal, + stream: isLocal ? fs.createReadStream : request, + normalized: isLocal ? path.normalize(url) : url + }; + }; -export const Destroy = (mediaPath: string) => new Promise(resolve => fs.unlink(mediaPath, error => resolve(error === null))); + const parseExifData = async (source: string): Promise => { + return new Promise(resolve => { + new ExifImage(source, (error, data) => { + let reason: Opt = undefined; + if (error) { + reason = (error as any).code; + } + resolve({ data, error: reason }); + }); + }); + }; } \ No newline at end of file diff --git a/src/server/SharedMediaTypes.ts b/src/server/SharedMediaTypes.ts index 3d3234125..8d0f441f0 100644 --- a/src/server/SharedMediaTypes.ts +++ b/src/server/SharedMediaTypes.ts @@ -1,9 +1,8 @@ -export namespace SharedMediaTypes { - +export namespace AcceptibleMedia { export const gifs = [".gif"]; export const pngs = [".png"]; export const jpgs = [".jpg", ".jpeg"]; export const imageFormats = [...pngs, ...jpgs, ...gifs]; export const videoFormats = [".mov", ".mp4"]; - + export const applicationFormats = [".pdf"]; } \ No newline at end of file diff --git a/src/server/database.ts b/src/server/database.ts index b81fc03a4..db81245c1 100644 --- a/src/server/database.ts +++ b/src/server/database.ts @@ -293,7 +293,7 @@ export namespace Database { }; export const QueryUploadHistory = async (contentSize: number) => { - return SanitizedSingletonQuery({ contentSize }, AuxiliaryCollections.GooglePhotosUploadHistory); + return SanitizedSingletonQuery({ contentSize }, AuxiliaryCollections.GooglePhotosUploadHistory); }; export namespace GoogleAuthenticationToken { @@ -322,7 +322,7 @@ export namespace Database { } - export const LogUpload = async (information: DashUploadUtils.UploadInformation) => { + export const LogUpload = async (information: DashUploadUtils.ImageUploadInformation) => { const bundle = { _id: Utils.GenerateDeterministicGuid(String(information.contentSize!)), ...information diff --git a/src/server/index.ts b/src/server/index.ts index d02a6005e..d77923710 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -24,7 +24,7 @@ import GooglePhotosManager from "./ApiManagers/GooglePhotosManager"; import DiagnosticManager from "./ApiManagers/DiagnosticManager"; export const publicDirectory = path.resolve(__dirname, "public"); -export const filesDirectory = path.resolve(publicDirectory, "files") + "/"; +export const filesDirectory = path.resolve(publicDirectory, "files"); /** * These are the functions run before the server starts @@ -34,7 +34,7 @@ export const filesDirectory = path.resolve(publicDirectory, "files") + "/"; async function preliminaryFunctions() { await GoogleCredentialsLoader.loadCredentials(); GoogleApiServerUtils.processProjectCredentials(); - await DashUploadUtils.buildFilePartitions(); + await DashUploadUtils.buildFileDirectories(); await log_execution({ startMessage: "attempting to initialize mongodb connection", endMessage: "connection outcome determined", -- cgit v1.2.3-70-g09d2 From db6d3f77b1b429c1942019b79c44e378eb8b1ee4 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Wed, 27 Nov 2019 04:25:55 -0500 Subject: fixed google photos upload by appending size suffix --- src/client/util/Import & Export/DirectoryImportBox.tsx | 10 ++++------ src/server/ApiManagers/GooglePhotosManager.ts | 4 ++-- src/server/DashUploadUtils.ts | 14 +++++++------- 3 files changed, 13 insertions(+), 15 deletions(-) (limited to 'src') diff --git a/src/client/util/Import & Export/DirectoryImportBox.tsx b/src/client/util/Import & Export/DirectoryImportBox.tsx index 16ae50685..b5e806a97 100644 --- a/src/client/util/Import & Export/DirectoryImportBox.tsx +++ b/src/client/util/Import & Export/DirectoryImportBox.tsx @@ -119,17 +119,15 @@ export default class DirectoryImportBox extends React.Component runInAction(() => this.completed += batch.length); }); - const size = "_o"; - await Promise.all(uploads.map(async upload => { - const type = upload.type; - const path = Utils.prepend(upload.clientAccessPath); + await Promise.all(uploads.map(async ({ name, type, clientAccessPath, exifData }) => { + const path = Utils.prepend(clientAccessPath); const options = { nativeWidth: 300, width: 300, - title: upload.name + title: name }; const document = await Docs.Get.DocumentFromType(type, path, options); - const { data, error } = upload.exifData; + const { data, error } = exifData; if (document) { Doc.GetProto(document).exif = error || Docs.Get.DocumentHierarchyFromJson(data); docs.push(document); diff --git a/src/server/ApiManagers/GooglePhotosManager.ts b/src/server/ApiManagers/GooglePhotosManager.ts index 5a709688b..4a0c0b936 100644 --- a/src/server/ApiManagers/GooglePhotosManager.ts +++ b/src/server/ApiManagers/GooglePhotosManager.ts @@ -5,7 +5,7 @@ import { GoogleApiServerUtils } from "../apis/google/GoogleApiServerUtils"; import { BatchedArray, TimeUnit } from "array-batcher"; import { GooglePhotosUploadUtils } from "../apis/google/GooglePhotosUploadUtils"; import { Opt } from "../../new_fields/Doc"; -import { DashUploadUtils } from "../DashUploadUtils"; +import { DashUploadUtils, InjectSize, SizeSuffix } from "../DashUploadUtils"; import { Database } from "../database"; const authenticationError = "Unable to authenticate Google credentials before uploading to Google Photos!"; @@ -55,7 +55,7 @@ export default class GooglePhotosManager extends ApiManager { for (let index = 0; index < batch.length; index++) { const { url, description } = batch[index]; const fail = (reason: string) => failed.push({ reason, batch: completedBatches + 1, index, url }); - const uploadToken = await GooglePhotosUploadUtils.DispatchGooglePhotosUpload(token, url).catch(fail); + const uploadToken = await GooglePhotosUploadUtils.DispatchGooglePhotosUpload(token, InjectSize(url, SizeSuffix.Original)).catch(fail); if (!uploadToken) { fail(`${path.extname(url)} is not an accepted extension`); } else { diff --git a/src/server/DashUploadUtils.ts b/src/server/DashUploadUtils.ts index 0a670ec01..81cd2d602 100644 --- a/src/server/DashUploadUtils.ts +++ b/src/server/DashUploadUtils.ts @@ -21,12 +21,12 @@ export enum SizeSuffix { Original = "_o" } -export namespace DashUploadUtils { +export function InjectSize(filename: string, size: SizeSuffix) { + const extension = path.extname(filename).toLowerCase(); + return filename.substring(0, filename.length - extension.length) + size + extension; +} - function InjectSize(filename: string, size: SizeSuffix) { - const extension = path.extname(filename).toLowerCase(); - return filename.substring(0, filename.length - extension.length) + size + extension; - } +export namespace DashUploadUtils { export interface Size { width: number; @@ -72,8 +72,8 @@ export namespace DashUploadUtils { switch (category) { case "image": if (imageFormats.includes(format)) { - const { clientAccessPath } = await UploadImage(path, basename(path), format); - return { clientAccessPath, name, type }; + const results = await UploadImage(path, basename(path), format); + return { ...results, name, type }; } case "video": if (videoFormats.includes(format)) { -- cgit v1.2.3-70-g09d2 From ee1ac7b8c9550cc842e91985c1a92d79ce0e5235 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Wed, 27 Nov 2019 05:34:27 -0500 Subject: constant --- src/server/DashUploadUtils.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/server/DashUploadUtils.ts b/src/server/DashUploadUtils.ts index 81cd2d602..c831eb072 100644 --- a/src/server/DashUploadUtils.ts +++ b/src/server/DashUploadUtils.ts @@ -61,9 +61,10 @@ export namespace DashUploadUtils { contentType?: string; } + const { imageFormats, videoFormats, applicationFormats } = AcceptibleMedia; + export async function upload(file: File): Promise { const { type, path, name } = file; - const { imageFormats, videoFormats, applicationFormats } = AcceptibleMedia; const types = type.split("/"); const category = types[0]; -- cgit v1.2.3-70-g09d2 From 77ee66de66a411f79bbbc036d379d09be38d172f Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Mon, 2 Dec 2019 12:12:58 -0500 Subject: further cleanup --- src/Utils.ts | 2 - src/client/util/ClientDiagnostics.ts | 18 +-- .../util/Import & Export/DirectoryImportBox.tsx | 3 +- src/client/views/MainView.tsx | 2 +- src/client/views/collections/CollectionSubView.tsx | 6 +- src/server/ApiManagers/DeleteManager.ts | 1 - src/server/ApiManagers/GeneralGoogleManager.ts | 15 +-- src/server/ApiManagers/PDFManager.ts | 134 ++++++++++----------- src/server/ApiManagers/UploadManager.ts | 43 ++++--- src/server/DashUploadUtils.ts | 18 ++- src/server/RouteManager.ts | 14 +++ .../authentication/models/current_user_utils.ts | 3 +- src/server/credentials/test.json | 14 --- 13 files changed, 125 insertions(+), 148 deletions(-) delete mode 100644 src/server/credentials/test.json (limited to 'src') diff --git a/src/Utils.ts b/src/Utils.ts index b60e9e023..2543743a4 100644 --- a/src/Utils.ts +++ b/src/Utils.ts @@ -2,8 +2,6 @@ import v4 = require('uuid/v4'); import v5 = require("uuid/v5"); import { Socket } from 'socket.io'; import { Message } from './server/Message'; -import { EventEmitter } from 'events'; -import { ConsoleColors } from './server/ActionUtilities'; export namespace Utils { diff --git a/src/client/util/ClientDiagnostics.ts b/src/client/util/ClientDiagnostics.ts index 7eef935fd..0a213aa1c 100644 --- a/src/client/util/ClientDiagnostics.ts +++ b/src/client/util/ClientDiagnostics.ts @@ -12,18 +12,22 @@ export namespace ClientDiagnostics { serverPolls--; }, 1000 * 15); - let executed = false; - const handle = async () => { + let solrHandle: NodeJS.Timeout | undefined; + const handler = async () => { const response = await fetch("/solrHeartbeat"); if (!(await response.json()).running) { - !executed && alert("Looks like SOLR is not running on your machine."); - executed = true; - clearInterval(solrHandle); + if (!executed) { + alert("Looks like SOLR is not running on your machine."); + executed = true; + solrHandle && clearInterval(solrHandle); + } } }; - await handle(); - const solrHandle = setInterval(handle, 1000 * 15); + await handler(); + if (!executed) { + solrHandle = setInterval(handler, 1000 * 15); + } } diff --git a/src/client/util/Import & Export/DirectoryImportBox.tsx b/src/client/util/Import & Export/DirectoryImportBox.tsx index b5e806a97..104d9e099 100644 --- a/src/client/util/Import & Export/DirectoryImportBox.tsx +++ b/src/client/util/Import & Export/DirectoryImportBox.tsx @@ -106,7 +106,8 @@ export default class DirectoryImportBox extends React.Component runInAction(() => this.phase = `Internal: uploading ${this.quota - this.completed} files to Dash...`); - const uploads = await BatchedArray.from(validated, { batchSize: 15 }).batchedMapAsync(async (batch, collector) => { + const batched = BatchedArray.from(validated, { batchSize: 15 }); + const uploads = await batched.batchedMapAsync(async (batch, collector) => { const formData = new FormData(); batch.forEach(file => { diff --git a/src/client/views/MainView.tsx b/src/client/views/MainView.tsx index 5231075a1..85dfd8be2 100644 --- a/src/client/views/MainView.tsx +++ b/src/client/views/MainView.tsx @@ -4,7 +4,7 @@ import { faMusic, faObjectGroup, faPause, faMousePointer, faPenNib, faFileAudio, faPen, faEraser, faPlay, faPortrait, faRedoAlt, faThumbtack, faTree, faTv, faUndoAlt, faHighlighter, faMicrophone, faCompressArrowsAlt } from '@fortawesome/free-solid-svg-icons'; import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; -import { action, computed, configure, observable, reaction, runInAction, autorun } from 'mobx'; +import { action, computed, configure, observable, reaction, runInAction } from 'mobx'; import { observer } from 'mobx-react'; import "normalize.css"; import * as React from 'react'; diff --git a/src/client/views/collections/CollectionSubView.tsx b/src/client/views/collections/CollectionSubView.tsx index a1bd1527e..a1ae77fef 100644 --- a/src/client/views/collections/CollectionSubView.tsx +++ b/src/client/views/collections/CollectionSubView.tsx @@ -6,7 +6,7 @@ import { Id } from "../../../new_fields/FieldSymbols"; import { List } from "../../../new_fields/List"; import { listSpec } from "../../../new_fields/Schema"; import { ScriptField } from "../../../new_fields/ScriptField"; -import { Cast, StrCast } from "../../../new_fields/Types"; +import { Cast } from "../../../new_fields/Types"; import { CurrentUserUtils } from "../../../server/authentication/models/current_user_utils"; import { Utils } from "../../../Utils"; import { DocServer } from "../../DocServer"; @@ -279,9 +279,9 @@ export function CollectionSubView(schemaCtor: (doc: Doc) => T) { formData.append('file', file); let dropFileName = file ? file.name : "-empty-"; promises.push(Networking.PostFormDataToServer("/upload", formData).then(results => { - results.map(action((file: any) => { + results.map(action(({ clientAccessPath }: any) => { let full = { ...options, nativeWidth: type.indexOf("video") !== -1 ? 600 : 300, width: 300, title: dropFileName }; - let pathname = Utils.prepend(file.clientAccessPath); + let pathname = Utils.prepend(clientAccessPath); Docs.Get.DocumentFromType(type, pathname, full).then(doc => { doc && (Doc.GetProto(doc).fileUpload = path.basename(pathname).replace("upload_", "").replace(/\.[a-z0-9]*$/, "")); doc && this.props.addDocument(doc); diff --git a/src/server/ApiManagers/DeleteManager.ts b/src/server/ApiManagers/DeleteManager.ts index f58a28ce5..71818c673 100644 --- a/src/server/ApiManagers/DeleteManager.ts +++ b/src/server/ApiManagers/DeleteManager.ts @@ -56,7 +56,6 @@ export default class DeleteManager extends ApiManager { } }); - } } diff --git a/src/server/ApiManagers/GeneralGoogleManager.ts b/src/server/ApiManagers/GeneralGoogleManager.ts index 171912185..629684e0c 100644 --- a/src/server/ApiManagers/GeneralGoogleManager.ts +++ b/src/server/ApiManagers/GeneralGoogleManager.ts @@ -20,8 +20,7 @@ export default class GeneralGoogleManager extends ApiManager { method: Method.GET, subscription: "/readGoogleAccessToken", onValidation: async ({ user, res }) => { - const userId = user.id; - const token = await GoogleApiServerUtils.retrieveAccessToken(userId); + const token = await GoogleApiServerUtils.retrieveAccessToken(user.id); if (!token) { return res.send(GoogleApiServerUtils.generateAuthenticationUrl()); } @@ -37,18 +36,6 @@ export default class GeneralGoogleManager extends ApiManager { } }); - register({ - method: Method.GET, - subscription: "/deleteWithGoogleCredentials", - onValidation: async ({ res, isRelease }) => { - if (isRelease) { - return _permission_denied(res, deletionPermissionError); - } - await Database.Auxiliary.GoogleAuthenticationToken.DeleteAll(); - res.redirect("/delete"); - } - }); - register({ method: Method.POST, subscription: new RouteSubscriber("/googleDocs").add("sector", "action"), diff --git a/src/server/ApiManagers/PDFManager.ts b/src/server/ApiManagers/PDFManager.ts index 4bd750aaf..151b48dd9 100644 --- a/src/server/ApiManagers/PDFManager.ts +++ b/src/server/ApiManagers/PDFManager.ts @@ -4,10 +4,11 @@ import RouteSubscriber from "../RouteSubscriber"; import { exists, createReadStream, createWriteStream } from "fs"; import * as Pdfjs from 'pdfjs-dist'; import { createCanvas } from "canvas"; -const probe = require("probe-image-size"); +const imageSize = require("probe-image-size"); import * as express from "express"; import * as path from "path"; import { Directory, serverPathToFile, clientPathToFile } from "./UploadManager"; +import { ConsoleColors } from "../ActionUtilities"; export default class PDFManager extends ApiManager { @@ -16,84 +17,77 @@ export default class PDFManager extends ApiManager { register({ method: Method.GET, subscription: new RouteSubscriber("thumbnail").add("filename"), - onValidation: ({ req, res }) => { - let filename = req.params.filename; - let noExt = filename.substring(0, filename.length - ".png".length); - let pagenumber = parseInt(noExt.split('-')[1]); - return new Promise(resolve => { - const path = serverPathToFile(Directory.pdf_thumbnails, filename); - exists(path, (exists: boolean) => { - console.log(`${path} ${exists ? "exists" : "does not exist"}`); - if (exists) { - let input = createReadStream(path); - probe(input, (err: any, { width, height }: any) => { - if (err) { - console.log(err); - console.log(`error on ${filename}`); - return; - } - res.send({ - path: clientPathToFile(Directory.pdf_thumbnails, filename), - width, - height - }); - }); - } - else { - const name = filename.substring(0, filename.length - noExt.split('-')[1].length - ".PNG".length - 1) + ".pdf"; - LoadPage(serverPathToFile(Directory.pdfs, name), pagenumber, res); - } - resolve(); - }); - }); - } + onValidation: ({ req, res }) => getOrCreateThumbnail(req.params.filename, res) }); - function LoadPage(file: string, pageNumber: number, res: express.Response) { - console.log(file); - Pdfjs.getDocument(file).promise - .then((pdf: Pdfjs.PDFDocumentProxy) => { - let factory = new NodeCanvasFactory(); - console.log(pageNumber); - pdf.getPage(pageNumber).then((page: Pdfjs.PDFPageProxy) => { - console.log("reading " + page); - let viewport = page.getViewport(1 as any); - let canvasAndContext = factory.create(viewport.width, viewport.height); - let renderContext = { - canvasContext: canvasAndContext.context, - canvasFactory: factory, - viewport - }; - console.log("read " + pageNumber); + } - page.render(renderContext).promise - .then(() => { - console.log("saving " + pageNumber); - let stream = canvasAndContext.canvas.createPNGStream(); - let filenames = path.basename(file).split("."); - const pngFile = serverPathToFile(Directory.pdf_thumbnails, `${filenames[0]}-${pageNumber}.png`); - let out = createWriteStream(pngFile); - stream.pipe(out); - out.on("finish", () => { - console.log(`Success! Saved to ${pngFile}`); - res.send({ - path: pngFile, - width: viewport.width, - height: viewport.height - }); - }); - }, (reason: string) => { - console.error(reason + ` ${pageNumber}`); - }); +} + +function getOrCreateThumbnail(thumbnailName: string, res: express.Response) { + const noExtension = thumbnailName.substring(0, thumbnailName.length - ".png".length); + const pageString = noExtension.split('-')[1]; + const pageNumber = parseInt(pageString); + return new Promise(resolve => { + const path = serverPathToFile(Directory.pdf_thumbnails, thumbnailName); + exists(path, (exists: boolean) => { + if (exists) { + let existingThumbnail = createReadStream(path); + imageSize(existingThumbnail, (err: any, { width, height }: any) => { + if (err) { + console.log(ConsoleColors.Red, `In PDF thumbnail response, unable to determine dimensions of ${thumbnailName}:`); + console.log(err); + return; + } + res.send({ + path: clientPathToFile(Directory.pdf_thumbnails, thumbnailName), + width, + height }); }); - } - - } + } else { + const offset = thumbnailName.length - pageString.length - 5; + const name = thumbnailName.substring(0, offset) + ".pdf"; + const path = serverPathToFile(Directory.pdfs, name); + CreateThumbnail(path, pageNumber, res); + } + resolve(); + }); + }); +} +async function CreateThumbnail(file: string, pageNumber: number, res: express.Response) { + const documentProxy = await Pdfjs.getDocument(file).promise; + const factory = new NodeCanvasFactory(); + const page = await documentProxy.getPage(pageNumber); + const viewport = page.getViewport(1 as any); + const { canvas, context } = factory.create(viewport.width, viewport.height); + const renderContext = { + canvasContext: context, + canvasFactory: factory, + viewport + }; + await page.render(renderContext).promise; + const pngStream = canvas.createPNGStream(); + const filenames = path.basename(file).split("."); + const pngFile = serverPathToFile(Directory.pdf_thumbnails, `${filenames[0]}-${pageNumber}.png`); + const out = createWriteStream(pngFile); + pngStream.pipe(out); + out.on("finish", () => { + res.send({ + path: pngFile, + width: viewport.width, + height: viewport.height + }); + }); + out.on("error", error => { + console.log(ConsoleColors.Red, `In PDF thumbnail creation, encountered the following error when piping ${pngFile}:`); + console.log(error); + }); } class NodeCanvasFactory { + create = (width: number, height: number) => { var canvas = createCanvas(width, height); var context = canvas.getContext('2d'); diff --git a/src/server/ApiManagers/UploadManager.ts b/src/server/ApiManagers/UploadManager.ts index 2f76871a6..80ae0ad61 100644 --- a/src/server/ApiManagers/UploadManager.ts +++ b/src/server/ApiManagers/UploadManager.ts @@ -38,6 +38,27 @@ export default class UploadManager extends ApiManager { protected initialize(register: Registration): void { + register({ + method: Method.POST, + subscription: "/upload", + onValidation: async ({ req, res }) => { + let form = new formidable.IncomingForm(); + form.uploadDir = pathToDirectory(Directory.parsed_files); + form.keepExtensions = true; + return new Promise(resolve => { + form.parse(req, async (_err, _fields, files) => { + let results: any[] = []; + for (const key in files) { + const result = await DashUploadUtils.upload(files[key]); + result && results.push(result); + } + _success(res, results); + resolve(); + }); + }); + } + }); + register({ method: Method.POST, subscription: "/uploadDoc", @@ -142,28 +163,6 @@ export default class UploadManager extends ApiManager { } }); - - register({ - method: Method.POST, - subscription: "/upload", - onValidation: async ({ req, res }) => { - let form = new formidable.IncomingForm(); - form.uploadDir = pathToDirectory(Directory.parsed_files); - form.keepExtensions = true; - return new Promise(resolve => { - form.parse(req, async (_err, _fields, files) => { - let results: any[] = []; - for (const key in files) { - const result = await DashUploadUtils.upload(files[key]); - result && results.push(result); - } - _success(res, results); - resolve(); - }); - }); - } - }); - register({ method: Method.POST, subscription: "/inspectImage", diff --git a/src/server/DashUploadUtils.ts b/src/server/DashUploadUtils.ts index c831eb072..9ccc72e35 100644 --- a/src/server/DashUploadUtils.ts +++ b/src/server/DashUploadUtils.ts @@ -85,7 +85,8 @@ export namespace DashUploadUtils { return UploadPdf(path); } } - console.log(ConsoleColors.Red, `Ignoring unsupported file ${name} with upload type (${type}).`); + + console.log(ConsoleColors.Red, `Ignoring unsupported file (${name}) with upload type (${type}).`); return { clientAccessPath: undefined }; } @@ -169,17 +170,12 @@ export namespace DashUploadUtils { if (isLocal) { return results; } - const metadata = (await new Promise((resolve, reject) => { - request.head(source, async (error, res) => { - if (error) { - return reject(error); - } - resolve(res); - }); - })).headers; + const { headers } = (await new Promise((resolve, reject) => { + request.head(source, (error, res) => error ? reject(error) : resolve(res)); + })); return { - contentSize: parseInt(metadata[size]), - contentType: metadata[type], + contentSize: parseInt(headers[size]), + contentType: headers[type], ...results }; }; diff --git a/src/server/RouteManager.ts b/src/server/RouteManager.ts index 3a20d5af5..7c49485f1 100644 --- a/src/server/RouteManager.ts +++ b/src/server/RouteManager.ts @@ -26,6 +26,8 @@ export interface RouteInitializer { onError?: OnError; } +const registered = new Map>(); + export default class RouteManager { private server: express.Express; private _isRelease: boolean; @@ -89,6 +91,18 @@ export default class RouteManager { } else { route = subscriber.build; } + const existing = registered.get(route); + if (existing) { + if (existing.has(method)) { + console.log(ConsoleColors.Red, `\nDuplicate registration error: already registered ${route} with Method[${method}]`); + console.log('Please remove duplicate registrations before continuing...\n'); + process.exit(0); + } + } else { + const specific = new Set(); + specific.add(method); + registered.set(route, specific); + } switch (method) { case Method.GET: this.server.get(route, supervised); diff --git a/src/server/authentication/models/current_user_utils.ts b/src/server/authentication/models/current_user_utils.ts index 052aa54a6..ac4462f78 100644 --- a/src/server/authentication/models/current_user_utils.ts +++ b/src/server/authentication/models/current_user_utils.ts @@ -1,4 +1,4 @@ -import { action, computed, observable, reaction, runInAction } from "mobx"; +import { action, computed, observable, reaction } from "mobx"; import * as rp from 'request-promise'; import { DocServer } from "../../../client/DocServer"; import { Docs } from "../../../client/documents/Documents"; @@ -11,7 +11,6 @@ import { listSpec } from "../../../new_fields/Schema"; import { ScriptField, ComputedField } from "../../../new_fields/ScriptField"; import { Cast, PromiseValue } from "../../../new_fields/Types"; import { Utils } from "../../../Utils"; -import { ButtonBox } from "../../../client/views/nodes/ButtonBox"; import { nullAudio } from "../../../new_fields/URLField"; import { DragManager } from "../../../client/util/DragManager"; import { InkingControl } from "../../../client/views/InkingControl"; diff --git a/src/server/credentials/test.json b/src/server/credentials/test.json deleted file mode 100644 index 0a032cc2d..000000000 --- a/src/server/credentials/test.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "installed": { - "client_id": "343179513178-ud6tvmh275r2fq93u9eesrnc66t6akh9.apps.googleusercontent.com", - "project_id": "quickstart-1565056383187", - "auth_uri": "https://accounts.google.com/o/oauth2/auth", - "token_uri": "https://oauth2.googleapis.com/token", - "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", - "client_secret": "w8KIFSc0MQpmUYHed4qEzn8b", - "redirect_uris": [ - "urn:ietf:wg:oauth:2.0:oob", - "http://localhost" - ] - } -} \ No newline at end of file -- cgit v1.2.3-70-g09d2