From 91e4ac65e0b8d1ff5c17ea0e80666038281ec5a6 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Tue, 15 Oct 2019 12:54:57 -0400 Subject: initial commit --- src/server/database.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) (limited to 'src/server/database.ts') diff --git a/src/server/database.ts b/src/server/database.ts index db86b472d..25e1e67e0 100644 --- a/src/server/database.ts +++ b/src/server/database.ts @@ -8,16 +8,19 @@ import { GoogleApiServerUtils } from './apis/google/GoogleApiServerUtils'; export namespace Database { + const schema = 'Dash'; + const port = 27017; + export const url = `mongodb://localhost:${port}/${schema}`; + class Database { public static DocumentsCollection = 'documents'; private MongoClient = mongodb.MongoClient; - private url = 'mongodb://localhost:27017/Dash'; private currentWrites: { [id: string]: Promise } = {}; private db?: mongodb.Db; private onConnect: (() => void)[] = []; constructor() { - this.MongoClient.connect(this.url, (err, client) => { + this.MongoClient.connect(url, (_err, client) => { this.db = client.db(); this.onConnect.forEach(fn => fn()); }); -- cgit v1.2.3-70-g09d2 From 7b43e349d31c911ab43763a4ff7179b3778a2d96 Mon Sep 17 00:00:00 2001 From: Sam Wilkins <35748010+samwilkins333@users.noreply.github.com> Date: Wed, 16 Oct 2019 18:38:31 -0400 Subject: database separation and preliminary functions --- src/server/Initialization.ts | 81 +++++++++++++---------------- src/server/database.ts | 27 ++++++++++ src/server/index.ts | 118 ++++++++++++++++++++----------------------- 3 files changed, 117 insertions(+), 109 deletions(-) (limited to 'src/server/database.ts') diff --git a/src/server/Initialization.ts b/src/server/Initialization.ts index e371a3edb..2c343ae90 100644 --- a/src/server/Initialization.ts +++ b/src/server/Initialization.ts @@ -7,8 +7,8 @@ import * as cookieParser from 'cookie-parser'; import expressFlash = require('express-flash'); import flash = require('connect-flash'); import { Database } from './database'; +import { getForgot, getLogin, getLogout, getReset, getSignup, postForgot, postLogin, postReset, postSignup } from './authentication/controllers/user_controller'; const MongoStore = require('connect-mongo')(session); -import mongoose, { ConnectionStates } from 'mongoose'; import { RouteStore } from './RouteStore'; import RouteManager from './RouteManager'; import * as webpack from 'webpack'; @@ -25,47 +25,26 @@ export interface InitializationOptions { export default async function InitializeServer(options: InitializationOptions) { const { listenAtPort, routeSetter } = options; - const server = injectMiddleware(express()); - const { url } = Database; - try { - await connectToDatabase(url); - } catch (e) { - console.error(`Mongoose FAILED to establish default connection at ${url}`); - console.error(e); - console.log('Since a valid database connection is required to use Dash, killing the server process.\nPlease try again later.'); - process.exit(1); - } - - // static file serving - server.use(express.static(__dirname + RouteStore.public)); - server.use(RouteStore.images, express.static(__dirname + RouteStore.public)); + const server = buildWithMiddleware(express()); routeSetter(new RouteManager(server, determineEnvironment())); + server.use(express.static(__dirname + RouteStore.public)); + server.use(RouteStore.images, express.static(__dirname + RouteStore.public)); + server.use(wdm(compiler, { publicPath: config.output.publicPath })); server.use(whm(compiler)); server.listen(listenAtPort, () => console.log(`server started at http://localhost:${listenAtPort}`)); - return server; -} - -function determineEnvironment() { - const isRelease = process.env.RELEASE === "true"; - - console.log(`running server in ${isRelease ? 'release' : 'debug'} mode`); - console.log(process.env.PWD); + registerAuthenticationRoutes(server); - let clientUtils = fs.readFileSync("./src/client/util/ClientUtils.ts.temp", "utf8"); - clientUtils = `//AUTO-GENERATED FILE: DO NOT EDIT\n${clientUtils.replace('"mode"', String(isRelease))}`; - fs.writeFileSync("./src/client/util/ClientUtils.ts", clientUtils, "utf8"); - - return isRelease; + return server; } const week = 7 * 24 * 60 * 60 * 1000; const secret = "64d6866242d3b5a5503c675b32c9605e4e90478e9b77bcf2bc"; -function injectMiddleware(server: express.Express) { +function buildWithMiddleware(server: express.Express) { [ cookieParser(), session({ @@ -90,21 +69,31 @@ function injectMiddleware(server: express.Express) { return server; } -async function connectToDatabase(url: string) { - const { connection } = mongoose; - process.on('SIGINT', () => { - connection.close(() => { - console.log('Mongoose default connection disconnected through app termination'); - process.exit(0); - }); - }); - if (connection.readyState === ConnectionStates.disconnected) { - return new Promise((resolve, reject) => { - connection.on('error', reject); - connection.on('connected', () => { - console.log(`Mongoose established default connection at ${url}`); - resolve(); - }); - }); - } +function determineEnvironment() { + const isRelease = process.env.RELEASE === "true"; + + console.log(`running server in ${isRelease ? 'release' : 'debug'} mode`); + console.log(process.env.PWD); + + let clientUtils = fs.readFileSync("./src/client/util/ClientUtils.ts.temp", "utf8"); + clientUtils = `//AUTO-GENERATED FILE: DO NOT EDIT\n${clientUtils.replace('"mode"', String(isRelease))}`; + fs.writeFileSync("./src/client/util/ClientUtils.ts", clientUtils, "utf8"); + + return isRelease; +} + +function registerAuthenticationRoutes(server: express.Express) { + server.get(RouteStore.signup, getSignup); + server.post(RouteStore.signup, postSignup); + + server.get(RouteStore.login, getLogin); + server.post(RouteStore.login, postLogin); + + server.get(RouteStore.logout, getLogout); + + server.get(RouteStore.forgot, getForgot); + server.post(RouteStore.forgot, postForgot); + + server.get(RouteStore.reset, getReset); + server.post(RouteStore.reset, postReset); } \ No newline at end of file diff --git a/src/server/database.ts b/src/server/database.ts index 25e1e67e0..4f93d1ee6 100644 --- a/src/server/database.ts +++ b/src/server/database.ts @@ -5,6 +5,7 @@ import { Utils, emptyFunction } from '../Utils'; import { DashUploadUtils } from './DashUploadUtils'; import { Credentials } from 'google-auth-library'; import { GoogleApiServerUtils } from './apis/google/GoogleApiServerUtils'; +import mongoose, { ConnectionStates } from 'mongoose'; export namespace Database { @@ -12,6 +13,32 @@ export namespace Database { const port = 27017; export const url = `mongodb://localhost:${port}/${schema}`; + export async function tryInitializeConnection() { + try { + const { connection } = mongoose; + process.on('SIGINT', () => { + connection.close(() => { + console.log('Mongoose default connection disconnected through app termination'); + process.exit(0); + }); + }); + if (connection.readyState === ConnectionStates.disconnected) { + await new Promise((resolve, reject) => { + connection.on('error', reject); + connection.on('connected', () => { + console.log(`Mongoose established default connection at ${url}`); + resolve(); + }); + }); + } + } catch (e) { + console.error(`Mongoose FAILED to establish default connection at ${url} with the following error:`); + console.error(e); + console.log('Since a valid database connection is required to use Dash, the server process will now exit.\nPlease try again later.'); + process.exit(1); + } + } + class Database { public static DocumentsCollection = 'documents'; private MongoClient = mongodb.MongoClient; diff --git a/src/server/index.ts b/src/server/index.ts index ad18857b6..ef618472b 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -11,7 +11,6 @@ import * as request from 'request'; import io from 'socket.io'; import { Socket } from 'socket.io'; import { Utils } from '../Utils'; -import { getForgot, getLogin, getLogout, getReset, getSignup, postForgot, postLogin, postReset, postSignup } from './authentication/controllers/user_controller'; import { Client } from './Client'; import { Database } from './database'; import { MessageStore, Transferable, Types, Diff, YoutubeQueryTypes as YoutubeQueryType, YoutubeQueryInput } from "./Message"; @@ -27,7 +26,6 @@ import { Response } from 'express-serve-static-core'; import { GoogleApiServerUtils } from "./apis/google/GoogleApiServerUtils"; const probe = require("probe-image-size"); const pdf = require('pdf-parse'); -var findInFiles = require('find-in-files'); import { GooglePhotosUploadUtils } from './apis/google/GooglePhotosUploadUtils'; import { Opt } from '../new_fields/Doc'; import { DashUploadUtils } from './DashUploadUtils'; @@ -38,6 +36,7 @@ import RouteSubscriber from './RouteSubscriber'; import InitializeServer from './Initialization'; import { Method, _success, _permission_denied, _error, _invalid } from './RouteManager'; import { command_line, read_text_file } from './ActionUtilities'; +var findInFiles = require('find-in-files'); let youtubeApiKey: string; @@ -51,10 +50,22 @@ export interface NewMediaItem { }; } -(async () => { - YoutubeApi.readApiKey((apiKey: string) => youtubeApiKey = apiKey); +const pngTypes = [".png", ".PNG"]; +const jpgTypes = [".jpg", ".JPG", ".jpeg", ".JPEG"]; +const uploadDirectory = __dirname + "/public/files/"; +const pdfDirectory = uploadDirectory + "text"; +const solrURL = "http://localhost:8983/solr/#/dash"; + +YoutubeApi.readApiKey((apiKey: string) => youtubeApiKey = apiKey); + +async function PreliminaryFunctions() { await GoogleApiServerUtils.LoadOAuthClient(); + await DashUploadUtils.createIfNotExists(pdfDirectory); + await Database.tryInitializeConnection(); +} +(async () => { + await PreliminaryFunctions(); await InitializeServer({ listenAtPort: 1050, routeSetter: router => { @@ -73,6 +84,26 @@ export interface NewMediaItem { } }); + router.addSupervisedRoute({ + method: Method.GET, + subscription: "/textsearch", + onValidation: async (_user, req, res) => { + let q = req.query.q; + if (q === undefined) { + res.send([]); + return; + } + let results = await findInFiles.find({ 'term': q, 'flags': 'ig' }, uploadDirectory + "text", ".txt$"); + let resObj: { ids: string[], numFound: number, lines: string[] } = { ids: [], numFound: 0, lines: [] }; + for (var result in results) { + resObj.ids.push(path.basename(result, ".txt").replace(/upload_/, "")); + resObj.lines.push(results[result].line); + resObj.numFound++; + } + res.send(resObj); + } + }); + router.addSupervisedRoute({ method: Method.GET, subscription: "/buxton", @@ -101,36 +132,19 @@ export interface NewMediaItem { } }); - // SEARCH - const solrURL = "http://localhost:8983/solr/#/dash"; - - // GETTERS - - DashServer.get("/textsearch", async (req, res) => { - let q = req.query.q; - if (q === undefined) { - res.send([]); - return; - } - let results = await findInFiles.find({ 'term': q, 'flags': 'ig' }, uploadDirectory + "text", ".txt$"); - let resObj: { ids: string[], numFound: number, lines: string[] } = { ids: [], numFound: 0, lines: [] }; - for (var result in results) { - resObj.ids.push(path.basename(result, ".txt").replace(/upload_/, "")); - resObj.lines.push(results[result].line); - resObj.numFound++; - } - res.send(resObj); - }); - - DashServer.get("/search", async (req, res) => { - const solrQuery: any = {}; - ["q", "fq", "start", "rows", "hl", "hl.fl"].forEach(key => solrQuery[key] = req.query[key]); - if (solrQuery.q === undefined) { - res.send([]); - return; + router.addSupervisedRoute({ + method: Method.GET, + subscription: "/search", + onValidation: async (_user, req, res) => { + const solrQuery: any = {}; + ["q", "fq", "start", "rows", "hl", "hl.fl"].forEach(key => solrQuery[key] = req.query[key]); + if (solrQuery.q === undefined) { + res.send([]); + return; + } + let results = await Search.Instance.search(solrQuery); + res.send(results); } - let results = await Search.Instance.search(solrQuery); - res.send(results); }); function msToTime(duration: number) { @@ -210,9 +224,14 @@ export interface NewMediaItem { await Database.Instance.visit([id], fn); return { id, docs, files }; } - DashServer.get("/serializeDoc/:docId", async (req, res) => { - const { docs, files } = await getDocs(req.params.docId); - res.send({ docs, files: Array.from(files) }); + + router.addSupervisedRoute({ + method: Method.GET, + subscription: new RouteSubscriber("/serializeDoc").add("docId"), + onValidation: async (_user, req, res) => { + const { docs, files } = await getDocs(req.params.docId); + res.send({ docs, files: Array.from(files) }); + } }); router.addSupervisedRoute({ @@ -554,12 +573,6 @@ export interface NewMediaItem { } } - const pngTypes = [".png", ".PNG"]; - const jpgTypes = [".jpg", ".JPG", ".jpeg", ".JPEG"]; - const uploadDirectory = __dirname + "/public/files/"; - const pdfDirectory = uploadDirectory + "text"; - DashUploadUtils.createIfNotExists(pdfDirectory); - interface ImageFileResponse { name: string; path: string; @@ -657,27 +670,6 @@ export interface NewMediaItem { } }); - // AUTHENTICATION - - // Sign Up - DashServer.get(RouteStore.signup, getSignup); - DashServer.post(RouteStore.signup, postSignup); - - // Log In - DashServer.get(RouteStore.login, getLogin); - DashServer.post(RouteStore.login, postLogin); - - // Log Out - DashServer.get(RouteStore.logout, getLogout); - - // FORGOT PASSWORD EMAIL HANDLING - DashServer.get(RouteStore.forgot, getForgot); - DashServer.post(RouteStore.forgot, postForgot); - - // RESET PASSWORD EMAIL HANDLING - DashServer.get(RouteStore.reset, getReset); - DashServer.post(RouteStore.reset, postReset); - const headerCharRegex = /[^\t\x20-\x7e\x80-\xff]/; DashServer.use(RouteStore.corsProxy, (req, res) => { req.pipe(request(decodeURIComponent(req.url.substring(1)))).on("response", res => { -- cgit v1.2.3-70-g09d2 From 91868727ea6e6443a916cf720d477b1136601b2f Mon Sep 17 00:00:00 2001 From: Sam Wilkins <35748010+samwilkins333@users.noreply.github.com> Date: Thu, 17 Oct 2019 02:53:34 -0400 Subject: refactored handlers --- src/client/util/SelectionManager.ts | 5 +- src/server/Initialization.ts | 28 +- src/server/RouteManager.ts | 60 ++- src/server/authentication/models/user_model.ts | 14 +- src/server/database.ts | 14 +- src/server/index.ts | 495 +++++++++++++------------ 6 files changed, 343 insertions(+), 273 deletions(-) (limited to 'src/server/database.ts') diff --git a/src/client/util/SelectionManager.ts b/src/client/util/SelectionManager.ts index df1b46b33..398c90ddb 100644 --- a/src/client/util/SelectionManager.ts +++ b/src/client/util/SelectionManager.ts @@ -54,7 +54,10 @@ export namespace SelectionManager { let stored = StrCast(targetDoc.backgroundColor); stored.length > 0 && (targetColor = stored); } - InkingControl.Instance.updateSelectedColor(targetColor); + const { Instance } = InkingControl; + if (Instance) { + Instance.updateSelectedColor(targetColor); + } }, { fireImmediately: true }); export function DeselectDoc(docView: DocumentView): void { diff --git a/src/server/Initialization.ts b/src/server/Initialization.ts index 2c343ae90..9646dc195 100644 --- a/src/server/Initialization.ts +++ b/src/server/Initialization.ts @@ -17,6 +17,7 @@ const compiler = webpack(config); import * as wdm from 'webpack-dev-middleware'; import * as whm from 'webpack-hot-middleware'; import * as fs from 'fs'; +import * as request from 'request'; export interface InitializationOptions { listenAtPort: number; @@ -27,18 +28,18 @@ export default async function InitializeServer(options: InitializationOptions) { const { listenAtPort, routeSetter } = options; const server = buildWithMiddleware(express()); - routeSetter(new RouteManager(server, determineEnvironment())); - server.use(express.static(__dirname + RouteStore.public)); server.use(RouteStore.images, express.static(__dirname + RouteStore.public)); server.use(wdm(compiler, { publicPath: config.output.publicPath })); server.use(whm(compiler)); - server.listen(listenAtPort, () => console.log(`server started at http://localhost:${listenAtPort}`)); registerAuthenticationRoutes(server); + registerCorsProxy(server); - return server; + routeSetter(new RouteManager(server, determineEnvironment())); + + server.listen(listenAtPort, () => console.log(`server started at http://localhost:${listenAtPort}`)); } const week = 7 * 24 * 60 * 60 * 1000; @@ -96,4 +97,23 @@ function registerAuthenticationRoutes(server: express.Express) { server.get(RouteStore.reset, getReset); server.post(RouteStore.reset, postReset); +} + +function registerCorsProxy(server: express.Express) { + const headerCharRegex = /[^\t\x20-\x7e\x80-\xff]/; + server.use(RouteStore.corsProxy, (req, res) => { + req.pipe(request(decodeURIComponent(req.url.substring(1)))).on("response", res => { + const headers = Object.keys(res.headers); + headers.forEach(headerName => { + const header = res.headers[headerName]; + if (Array.isArray(header)) { + res.headers[headerName] = header.filter(h => !headerCharRegex.test(h)); + } else if (header) { + if (headerCharRegex.test(header as any)) { + delete res.headers[headerName]; + } + } + }); + }).pipe(res); + }); } \ No newline at end of file diff --git a/src/server/RouteManager.ts b/src/server/RouteManager.ts index cf15e45c9..626014d1a 100644 --- a/src/server/RouteManager.ts +++ b/src/server/RouteManager.ts @@ -3,6 +3,7 @@ import { RouteStore } from "./RouteStore"; import { DashUserModel } from "./authentication/models/user_model"; import * as express from 'express'; import * as qs from 'query-string'; +import { Opt } from "../new_fields/Doc"; export default class RouteManager { private server: express.Express; @@ -27,29 +28,34 @@ export default class RouteManager { * @param subscribers the forward slash prepended path names (reference and add to RouteStore.ts) that will all invoke the given @param handler */ addSupervisedRoute(initializer: RouteInitializer) { - const { method, subscription, onValidation, onRejection, onError } = initializer; - const release = this._isRelease; + const { method, subscription, onValidation, onRejection, onError, onGuestAccess } = initializer; + const isRelease = this._isRelease; let abstracted = async (req: express.Request, res: express.Response) => { const { user, originalUrl: target } = req; - if (user || isSharedDocAccess(target)) { + const core = { req, res, isRelease: isRelease }; + if (user) { try { - await onValidation(user, req, res, release); + await onValidation({ ...core, user: user as any }); } catch (e) { if (onError) { - onError(req, res, e, release); + onError({ ...core, error: e }); } else { _error(res, `The server encountered an internal error handling ${target}.`, e); } } } else { - req.session!.target = target; - try { - await (onRejection || LoginRedirect)(req, res, release); - } catch (e) { - if (onError) { - onError(req, res, e, this._isRelease); - } else { - _error(res, `The server encountered an internal error when rejecting ${target}.`, e); + if (isGuestAccess(req) && onGuestAccess) { + await onGuestAccess(core); + } else { + req.session!.target = target; + try { + await (onRejection || LoginRedirect)(core); + } catch (e) { + if (onError) { + onError({ ...core, error: e }); + } else { + _error(res, `The server encountered an internal error when rejecting ${target}.`, e); + } } } } @@ -84,18 +90,25 @@ export enum Method { POST } -export type ValidationHandler = (user: DashUserModel, req: express.Request, res: express.Response, isRelease: boolean) => any | Promise; -export type RejectionHandler = (req: express.Request, res: express.Response, isRelease: boolean) => any | Promise; -export type ErrorHandler = (req: express.Request, res: express.Response, error: any, isRelease: boolean) => any | Promise; +export interface CoreArguments { + req: express.Request, + res: express.Response, + isRelease: boolean; +} -const LoginRedirect: RejectionHandler = (_req, res) => res.redirect(RouteStore.login); +export type OnValidation = (core: CoreArguments & { user: DashUserModel }) => any | Promise; +export type OnUnauthenticated = (core: CoreArguments) => any | Promise; +export type OnError = (core: CoreArguments & { error: any }) => any | Promise; + +const LoginRedirect: OnUnauthenticated = ({ res }) => res.redirect(RouteStore.login); export interface RouteInitializer { method: Method; subscription: string | RouteSubscriber | (string | RouteSubscriber)[]; - onValidation: ValidationHandler; - onRejection?: RejectionHandler; - onError?: ErrorHandler; + onValidation: OnValidation; + onRejection?: OnUnauthenticated; + onGuestAccess?: OnUnauthenticated; + onError?: OnError; } const isSharedDocAccess = (target: string) => { @@ -104,6 +117,13 @@ const isSharedDocAccess = (target: string) => { return shared && docAccess; }; +const isGuestAccess = (req: express.Request) => { + if (isSharedDocAccess(req.originalUrl)) { + return true; + } + return false; +} + export const STATUS = { OK: 200, BAD_REQUEST: 400, diff --git a/src/server/authentication/models/user_model.ts b/src/server/authentication/models/user_model.ts index 45fbf23b1..cc670a03a 100644 --- a/src/server/authentication/models/user_model.ts +++ b/src/server/authentication/models/user_model.ts @@ -1,20 +1,8 @@ //@ts-ignore import * as bcrypt from "bcrypt-nodejs"; //@ts-ignore -import * as mongoose from "mongoose"; -var url = 'mongodb://localhost:27017/Dash'; +import * as mongoose from 'mongoose'; -mongoose.connect(url, { useNewUrlParser: true }); - -mongoose.connection.on('connected', function () { - console.log('Stablished connection on ' + url); -}); -mongoose.connection.on('error', function (error) { - console.log('Something wrong happened: ' + error); -}); -mongoose.connection.on('disconnected', function () { - console.log('connection closed'); -}); export type DashUserModel = mongoose.Document & { email: String, password: string, diff --git a/src/server/database.ts b/src/server/database.ts index 4f93d1ee6..44c49d03e 100644 --- a/src/server/database.ts +++ b/src/server/database.ts @@ -5,7 +5,7 @@ import { Utils, emptyFunction } from '../Utils'; import { DashUploadUtils } from './DashUploadUtils'; import { Credentials } from 'google-auth-library'; import { GoogleApiServerUtils } from './apis/google/GoogleApiServerUtils'; -import mongoose, { ConnectionStates } from 'mongoose'; +import * as mongoose from 'mongoose'; export namespace Database { @@ -13,6 +13,14 @@ export namespace Database { const port = 27017; export const url = `mongodb://localhost:${port}/${schema}`; + enum ConnectionStates { + disconnected = 0, + connected = 1, + connecting = 2, + disconnecting = 3, + uninitialized = 99, + } + export async function tryInitializeConnection() { try { const { connection } = mongoose; @@ -25,10 +33,14 @@ export namespace Database { if (connection.readyState === ConnectionStates.disconnected) { await new Promise((resolve, reject) => { connection.on('error', reject); + connection.on('disconnected', () => { + console.log(`Mongoose connection at ${url} now closed`); + }); connection.on('connected', () => { console.log(`Mongoose established default connection at ${url}`); resolve(); }); + mongoose.connect(url, { useNewUrlParser: true }); }); } } catch (e) { diff --git a/src/server/index.ts b/src/server/index.ts index ef618472b..bba8fc292 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -7,8 +7,7 @@ import * as Pdfjs from 'pdfjs-dist'; const imageDataUri = require('image-data-uri'); import * as mobileDetect from 'mobile-detect'; import * as path from 'path'; -import * as request from 'request'; -import io from 'socket.io'; +import * as io from 'socket.io'; import { Socket } from 'socket.io'; import { Utils } from '../Utils'; import { Client } from './Client'; @@ -34,8 +33,8 @@ import { ParsedPDF } from "./PdfTypes"; import { reject } from 'bluebird'; import RouteSubscriber from './RouteSubscriber'; import InitializeServer from './Initialization'; -import { Method, _success, _permission_denied, _error, _invalid } from './RouteManager'; -import { command_line, read_text_file } from './ActionUtilities'; +import { Method, _success, _permission_denied, _error, _invalid, OnUnauthenticated } from './RouteManager'; +import { command_line } from './ActionUtilities'; var findInFiles = require('find-in-files'); let youtubeApiKey: string; @@ -73,7 +72,7 @@ async function PreliminaryFunctions() { router.addSupervisedRoute({ method: Method.GET, subscription: "/pull", - onValidation: (_user, _req, res) => { + onValidation: ({ res }) => { exec('"C:\\Program Files\\Git\\git-bash.exe" -c "git pull"', err => { if (err) { res.send(err.message); @@ -87,7 +86,7 @@ async function PreliminaryFunctions() { router.addSupervisedRoute({ method: Method.GET, subscription: "/textsearch", - onValidation: async (_user, req, res) => { + onValidation: async ({ req, res }) => { let q = req.query.q; if (q === undefined) { res.send([]); @@ -107,7 +106,7 @@ async function PreliminaryFunctions() { router.addSupervisedRoute({ method: Method.GET, subscription: "/buxton", - onValidation: (_user, _req, res) => { + onValidation: ({ res }) => { let cwd = '../scraping/buxton'; let onResolved = (stdout: string) => { console.log(stdout); res.redirect("/"); }; @@ -121,7 +120,7 @@ async function PreliminaryFunctions() { router.addSupervisedRoute({ method: Method.GET, subscription: "/version", - onValidation: (_user, _req, res) => { + onValidation: ({ res }) => { exec('"C:\\Program Files\\Git\\bin\\git.exe" rev-parse HEAD', (err, stdout) => { if (err) { res.send(err.message); @@ -135,7 +134,7 @@ async function PreliminaryFunctions() { router.addSupervisedRoute({ method: Method.GET, subscription: "/search", - onValidation: async (_user, req, res) => { + onValidation: async ({ req, res }) => { const solrQuery: any = {}; ["q", "fq", "start", "rows", "hl", "hl.fl"].forEach(key => solrQuery[key] = req.query[key]); if (solrQuery.q === undefined) { @@ -228,7 +227,7 @@ async function PreliminaryFunctions() { router.addSupervisedRoute({ method: Method.GET, subscription: new RouteSubscriber("/serializeDoc").add("docId"), - onValidation: async (_user, req, res) => { + onValidation: async ({ req, res }) => { const { docs, files } = await getDocs(req.params.docId); res.send({ docs, files: Array.from(files) }); } @@ -237,7 +236,7 @@ async function PreliminaryFunctions() { router.addSupervisedRoute({ method: Method.GET, subscription: new RouteSubscriber(RouteStore.imageHierarchyExport).add('docId'), - onValidation: async (_user, req, res) => { + onValidation: async ({ req, res }) => { const id = req.params.docId; const hierarchy: Hierarchy = {}; await targetedVisitorRecursive(id, hierarchy); @@ -305,154 +304,171 @@ async function PreliminaryFunctions() { } }; - DashServer.get("/downloadId/:docId", async (req, res) => { - res.set('Content-disposition', `attachment;`); - res.set('Content-Type', "application/zip"); - const { id, docs, files } = await getDocs(req.params.docId); - const docString = JSON.stringify({ id, docs }); - const zip = Archiver('zip'); - zip.pipe(res); - zip.append(docString, { name: "doc.json" }); - files.forEach(val => { - zip.file(__dirname + RouteStore.public + val, { name: val.substring(1) }); - }); - zip.finalize(); - }); + router.addSupervisedRoute({ + method: Method.GET, + subscription: new RouteSubscriber("/downloadId").add("docId"), + onValidation: async ({ req, res }) => { + res.set('Content-disposition', `attachment;`); + res.set('Content-Type', "application/zip"); + const { id, docs, files } = await getDocs(req.params.docId); + const docString = JSON.stringify({ id, docs }); + const zip = Archiver('zip'); + zip.pipe(res); + zip.append(docString, { name: "doc.json" }); + files.forEach(val => { + zip.file(__dirname + RouteStore.public + val, { name: val.substring(1) }); + }); + zip.finalize(); + } + }) - DashServer.post("/uploadDoc", (req, res) => { - let form = new formidable.IncomingForm(); - form.keepExtensions = true; - // let path = req.body.path; - const ids: { [id: string]: string } = {}; - let remap = true; - const getId = (id: string): string => { - if (!remap) return id; - if (id.endsWith("Proto")) return id; - if (id in ids) { - return ids[id]; - } else { - return ids[id] = v4(); - } - }; - const mapFn = (doc: any) => { - if (doc.id) { - doc.id = getId(doc.id); - } - for (const key in doc.fields) { - if (!doc.fields.hasOwnProperty(key)) { - continue; + router.addSupervisedRoute({ + method: Method.POST, + subscription: "/uploadDoc", + onValidation: ({ req, res }) => { + let form = new formidable.IncomingForm(); + form.keepExtensions = true; + // let path = req.body.path; + const ids: { [id: string]: string } = {}; + let remap = true; + const getId = (id: string): string => { + if (!remap) return id; + if (id.endsWith("Proto")) return id; + if (id in ids) { + return ids[id]; + } else { + return ids[id] = v4(); } - const field = doc.fields[key]; - if (field === undefined || field === null) { - continue; + }; + const mapFn = (doc: any) => { + if (doc.id) { + doc.id = getId(doc.id); } + for (const key in doc.fields) { + if (!doc.fields.hasOwnProperty(key)) { + continue; + } + const field = doc.fields[key]; + if (field === undefined || field === null) { + continue; + } - if (field.__type === "proxy" || field.__type === "prefetch_proxy") { - field.fieldId = getId(field.fieldId); - } else if (field.__type === "script" || field.__type === "computed") { - if (field.captures) { - field.captures.fieldId = getId(field.captures.fieldId); + if (field.__type === "proxy" || field.__type === "prefetch_proxy") { + field.fieldId = getId(field.fieldId); + } else if (field.__type === "script" || field.__type === "computed") { + if (field.captures) { + field.captures.fieldId = getId(field.captures.fieldId); + } + } else if (field.__type === "list") { + mapFn(field); + } else if (typeof field === "string") { + const re = /("(?:dataD|d)ocumentId"\s*:\s*")([\w\-]*)"/g; + doc.fields[key] = (field as any).replace(re, (match: any, p1: string, p2: string) => { + return `${p1}${getId(p2)}"`; + }); + } else if (field.__type === "RichTextField") { + const re = /("href"\s*:\s*")(.*?)"/g; + field.Data = field.Data.replace(re, (match: any, p1: string, p2: string) => { + return `${p1}${getId(p2)}"`; + }); } - } else if (field.__type === "list") { - mapFn(field); - } else if (typeof field === "string") { - const re = /("(?:dataD|d)ocumentId"\s*:\s*")([\w\-]*)"/g; - doc.fields[key] = (field as any).replace(re, (match: any, p1: string, p2: string) => { - return `${p1}${getId(p2)}"`; - }); - } else if (field.__type === "RichTextField") { - const re = /("href"\s*:\s*")(.*?)"/g; - field.Data = field.Data.replace(re, (match: any, p1: string, p2: string) => { - return `${p1}${getId(p2)}"`; - }); } - } - }; - form.parse(req, async (err, fields, files) => { - remap = fields.remap !== "false"; - let id: string = ""; - try { - for (const name in files) { - const path_2 = files[name].path; - const zip = new AdmZip(path_2); - zip.getEntries().forEach((entry: any) => { - if (!entry.entryName.startsWith("files/")) return; - let dirname = path.dirname(entry.entryName) + "/"; - let extname = path.extname(entry.entryName); - let basename = path.basename(entry.entryName).split(".")[0]; - // zip.extractEntryTo(dirname + basename + "_o" + extname, __dirname + RouteStore.public, true, false); - // zip.extractEntryTo(dirname + basename + "_s" + extname, __dirname + RouteStore.public, true, false); - // zip.extractEntryTo(dirname + basename + "_m" + extname, __dirname + RouteStore.public, true, false); - // zip.extractEntryTo(dirname + basename + "_l" + extname, __dirname + RouteStore.public, true, false); + }; + form.parse(req, async (err, fields, files) => { + remap = fields.remap !== "false"; + let id: string = ""; + try { + for (const name in files) { + const path_2 = files[name].path; + const zip = new AdmZip(path_2); + zip.getEntries().forEach((entry: any) => { + if (!entry.entryName.startsWith("files/")) return; + let dirname = path.dirname(entry.entryName) + "/"; + let extname = path.extname(entry.entryName); + let basename = path.basename(entry.entryName).split(".")[0]; + // zip.extractEntryTo(dirname + basename + "_o" + extname, __dirname + RouteStore.public, true, false); + // zip.extractEntryTo(dirname + basename + "_s" + extname, __dirname + RouteStore.public, true, false); + // zip.extractEntryTo(dirname + basename + "_m" + extname, __dirname + RouteStore.public, true, false); + // zip.extractEntryTo(dirname + basename + "_l" + extname, __dirname + RouteStore.public, true, false); + try { + zip.extractEntryTo(entry.entryName, __dirname + RouteStore.public, true, false); + dirname = "/" + dirname; + + fs.createReadStream(__dirname + RouteStore.public + dirname + basename + extname).pipe(fs.createWriteStream(__dirname + RouteStore.public + dirname + basename + "_o" + extname)); + fs.createReadStream(__dirname + RouteStore.public + dirname + basename + extname).pipe(fs.createWriteStream(__dirname + RouteStore.public + dirname + basename + "_s" + extname)); + fs.createReadStream(__dirname + RouteStore.public + dirname + basename + extname).pipe(fs.createWriteStream(__dirname + RouteStore.public + dirname + basename + "_m" + extname)); + fs.createReadStream(__dirname + RouteStore.public + dirname + basename + extname).pipe(fs.createWriteStream(__dirname + RouteStore.public + dirname + basename + "_l" + extname)); + } catch (e) { + console.log(e); + } + }); + const json = zip.getEntry("doc.json"); + let docs: any; try { - zip.extractEntryTo(entry.entryName, __dirname + RouteStore.public, true, false); - dirname = "/" + dirname; - - fs.createReadStream(__dirname + RouteStore.public + dirname + basename + extname).pipe(fs.createWriteStream(__dirname + RouteStore.public + dirname + basename + "_o" + extname)); - fs.createReadStream(__dirname + RouteStore.public + dirname + basename + extname).pipe(fs.createWriteStream(__dirname + RouteStore.public + dirname + basename + "_s" + extname)); - fs.createReadStream(__dirname + RouteStore.public + dirname + basename + extname).pipe(fs.createWriteStream(__dirname + RouteStore.public + dirname + basename + "_m" + extname)); - fs.createReadStream(__dirname + RouteStore.public + dirname + basename + extname).pipe(fs.createWriteStream(__dirname + RouteStore.public + dirname + basename + "_l" + extname)); - } catch (e) { - console.log(e); + let data = JSON.parse(json.getData().toString("utf8")); + docs = data.docs; + id = data.id; + docs = Object.keys(docs).map(key => docs[key]); + docs.forEach(mapFn); + await Promise.all(docs.map((doc: any) => new Promise(res => Database.Instance.replace(doc.id, doc, (err, r) => { + err && console.log(err); + res(); + }, true, "newDocuments")))); + } catch (e) { console.log(e); } + fs.unlink(path_2, () => { }); + } + if (id) { + res.send(JSON.stringify(getId(id))); + } else { + res.send(JSON.stringify("error")); + } + } catch (e) { console.log(e); } + }); + } + }) + + router.addSupervisedRoute({ + method: Method.GET, + subscription: "/whosOnline", + onValidation: ({ res }) => { + let users: any = { active: {}, inactive: {} }; + const now = Date.now(); + + for (const user in timeMap) { + const time = timeMap[user]; + const key = ((now - time) / 1000) < (60 * 5) ? "active" : "inactive"; + users[key][user] = `Last active ${msToTime(now - time)} ago`; + } + + res.send(users); + } + }); + + router.addSupervisedRoute({ + method: Method.GET, + subscription: new RouteSubscriber("/thumbnail").add("filename"), + onValidation: ({ req, res }) => { + let filename = req.params.filename; + let noExt = filename.substring(0, filename.length - ".png".length); + let pagenumber = parseInt(noExt.split('-')[1]); + fs.exists(uploadDirectory + filename, (exists: boolean) => { + console.log(`${uploadDirectory + filename} ${exists ? "exists" : "does not exist"}`); + if (exists) { + let input = fs.createReadStream(uploadDirectory + filename); + probe(input, (err: any, result: any) => { + if (err) { + console.log(err); + console.log(`error on ${filename}`); + return; } + res.send({ path: "/files/" + filename, width: result.width, height: result.height }); }); - const json = zip.getEntry("doc.json"); - let docs: any; - try { - let data = JSON.parse(json.getData().toString("utf8")); - docs = data.docs; - id = data.id; - docs = Object.keys(docs).map(key => docs[key]); - docs.forEach(mapFn); - await Promise.all(docs.map((doc: any) => new Promise(res => Database.Instance.replace(doc.id, doc, (err, r) => { - err && console.log(err); - res(); - }, true, "newDocuments")))); - } catch (e) { console.log(e); } - fs.unlink(path_2, () => { }); } - if (id) { - res.send(JSON.stringify(getId(id))); - } else { - res.send(JSON.stringify("error")); + else { + LoadPage(uploadDirectory + filename.substring(0, filename.length - noExt.split('-')[1].length - ".PNG".length - 1) + ".pdf", pagenumber, res); } - } catch (e) { console.log(e); } - }); - }); - - DashServer.get("/whosOnline", (req, res) => { - let users: any = { active: {}, inactive: {} }; - const now = Date.now(); - - for (const user in timeMap) { - const time = timeMap[user]; - const key = ((now - time) / 1000) < (60 * 5) ? "active" : "inactive"; - users[key][user] = `Last active ${msToTime(now - time)} ago`; + }); } - - res.send(users); - }); - DashServer.get("/thumbnail/:filename", (req, res) => { - let filename = req.params.filename; - let noExt = filename.substring(0, filename.length - ".png".length); - let pagenumber = parseInt(noExt.split('-')[1]); - fs.exists(uploadDirectory + filename, (exists: boolean) => { - console.log(`${uploadDirectory + filename} ${exists ? "exists" : "does not exist"}`); - if (exists) { - let input = fs.createReadStream(uploadDirectory + filename); - probe(input, (err: any, result: any) => { - if (err) { - console.log(err); - console.log(`error on ${filename}`); - return; - } - res.send({ path: "/files/" + filename, width: result.width, height: result.height }); - }); - } - else { - LoadPage(uploadDirectory + filename.substring(0, filename.length - noExt.split('-')[1].length - ".PNG".length - 1) + ".pdf", pagenumber, res); - } - }); }); function LoadPage(file: string, pageNumber: number, res: Response) { @@ -498,41 +514,44 @@ async function PreliminaryFunctions() { router.addSupervisedRoute({ method: Method.GET, subscription: RouteStore.root, - onValidation: (_user, _req, res) => res.redirect(RouteStore.home) + onValidation: ({ res }) => res.redirect(RouteStore.home) }); router.addSupervisedRoute({ method: Method.GET, subscription: RouteStore.getUsers, - onValidation: async (_user, _req, res) => { + onValidation: async ({ res }) => { const cursor = await Database.Instance.query({}, { email: 1, userDocumentId: 1 }, "users"); const results = await cursor.toArray(); res.send(results.map(user => ({ email: user.email, userDocumentId: user.userDocumentId }))); - }, + } }); + const serve: OnUnauthenticated = ({ req, res }) => { + let detector = new mobileDetect(req.headers['user-agent'] || ""); + let filename = detector.mobile() !== null ? 'mobile/image.html' : 'index.html'; + res.sendFile(path.join(__dirname, '../../deploy/' + filename)); + } + router.addSupervisedRoute({ method: Method.GET, - subscription: [RouteStore.home, RouteStore.openDocumentWithId], - onValidation: (_user, req, res) => { - let detector = new mobileDetect(req.headers['user-agent'] || ""); - let filename = detector.mobile() !== null ? 'mobile/image.html' : 'index.html'; - res.sendFile(path.join(__dirname, '../../deploy/' + filename)); - }, + subscription: [RouteStore.home, new RouteSubscriber("/doc").add("docId")], + onValidation: serve, + onGuestAccess: serve }); router.addSupervisedRoute({ method: Method.GET, subscription: RouteStore.getUserDocumentId, - onValidation: (user, _req, res) => res.send(user.userDocumentId), - onRejection: (_req, res) => res.send(undefined) + onValidation: ({ res, user }) => res.send(user.userDocumentId), + onRejection: ({ res }) => res.send(undefined) }); router.addSupervisedRoute({ method: Method.GET, subscription: RouteStore.getCurrUser, - onValidation: (user, _req, res) => { res.send(JSON.stringify(user)); }, - onRejection: (_req, res) => res.send(JSON.stringify({ id: "__guest__", email: "" })) + onValidation: ({ res, user }) => { res.send(JSON.stringify(user)); }, + onRejection: ({ res }) => res.send(JSON.stringify({ id: "__guest__", email: "" })) }); const ServicesApiKeyMap = new Map([ @@ -544,7 +563,7 @@ async function PreliminaryFunctions() { router.addSupervisedRoute({ method: Method.GET, subscription: new RouteSubscriber(RouteStore.cognitiveServices).add('requestedservice'), - onValidation: (_user, req, res) => { + onValidation: ({ req, res }) => { let service = req.params.requestedservice; res.send(ServicesApiKeyMap.get(service)); } @@ -583,7 +602,7 @@ async function PreliminaryFunctions() { router.addSupervisedRoute({ method: Method.POST, subscription: RouteStore.upload, - onValidation: (_user, req, res) => { + onValidation: ({ req, res }) => { let form = new formidable.IncomingForm(); form.uploadDir = uploadDirectory; form.keepExtensions = true; @@ -621,7 +640,7 @@ async function PreliminaryFunctions() { router.addSupervisedRoute({ method: Method.POST, subscription: RouteStore.inspectImage, - onValidation: async (_user, req, res) => { + onValidation: async ({ req, res }) => { const { source } = req.body; if (typeof source === "string") { const uploadInformation = await DashUploadUtils.UploadImage(source); @@ -634,7 +653,7 @@ async function PreliminaryFunctions() { router.addSupervisedRoute({ method: Method.POST, subscription: RouteStore.dataUriToImage, - onValidation: (_user, req, res) => { + onValidation: ({ req, res }) => { const uri = req.body.uri; const filename = req.body.name; if (!uri || !filename) { @@ -670,27 +689,10 @@ async function PreliminaryFunctions() { } }); - const headerCharRegex = /[^\t\x20-\x7e\x80-\xff]/; - DashServer.use(RouteStore.corsProxy, (req, res) => { - req.pipe(request(decodeURIComponent(req.url.substring(1)))).on("response", res => { - const headers = Object.keys(res.headers); - headers.forEach(headerName => { - const header = res.headers[headerName]; - if (Array.isArray(header)) { - res.headers[headerName] = header.filter(h => !headerCharRegex.test(h)); - } else if (header) { - if (headerCharRegex.test(header as any)) { - delete res.headers[headerName]; - } - } - }); - }).pipe(res); - }); - router.addSupervisedRoute({ method: Method.GET, subscription: RouteStore.delete, - onValidation: (_user, _req, res, isRelease) => { + onValidation: ({ res, isRelease }) => { if (isRelease) { return _permission_denied(res, deletionPermissionError); } @@ -701,7 +703,7 @@ async function PreliminaryFunctions() { router.addSupervisedRoute({ method: Method.GET, subscription: RouteStore.deleteAll, - onValidation: (_user, _req, res, isRelease) => { + onValidation: ({ res, isRelease }) => { if (isRelease) { return _permission_denied(res, deletionPermissionError); } @@ -813,27 +815,34 @@ async function PreliminaryFunctions() { ["update", (api, params) => api.batchUpdate(params)], ]); - DashServer.post(RouteStore.googleDocs + "/:sector/:action", (req, res) => { - let sector: GoogleApiServerUtils.Service = req.params.sector as GoogleApiServerUtils.Service; - let action: GoogleApiServerUtils.Action = req.params.action as GoogleApiServerUtils.Action; - GoogleApiServerUtils.GetEndpoint(GoogleApiServerUtils.Service[sector], { credentialsPath, userId: req.headers.userId as string }).then(endpoint => { - let handler = EndpointHandlerMap.get(action); - if (endpoint && handler) { - let execute = handler(endpoint, req.body).then( - response => res.send(response.data), - rejection => res.send(rejection) - ); - execute.catch(exception => res.send(exception)); - return; - } - res.send(undefined); - }); + router.addSupervisedRoute({ + method: Method.POST, + subscription: new RouteSubscriber(RouteStore.googleDocs).add("sector", "action"), + onValidation: ({ req, res }) => { + let sector: GoogleApiServerUtils.Service = req.params.sector as GoogleApiServerUtils.Service; + let action: GoogleApiServerUtils.Action = req.params.action as GoogleApiServerUtils.Action; + GoogleApiServerUtils.GetEndpoint(GoogleApiServerUtils.Service[sector], { credentialsPath, userId: req.headers.userId as string }).then(endpoint => { + let handler = EndpointHandlerMap.get(action); + if (endpoint && handler) { + let execute = handler(endpoint, req.body).then( + response => res.send(response.data), + rejection => res.send(rejection) + ); + execute.catch(exception => res.send(exception)); + return; + } + res.send(undefined); + }); + } }); router.addSupervisedRoute({ method: Method.GET, subscription: RouteStore.readGoogleAccessToken, - onValidation: async (user, _req, res) => { + onValidation: async ({ user, res }) => { + if (!user) { + return res.send(undefined); + } const userId = user.id; const token = await Database.Auxiliary.GoogleAuthenticationToken.Fetch(userId); const information = { credentialsPath, userId }; @@ -847,7 +856,10 @@ async function PreliminaryFunctions() { router.addSupervisedRoute({ method: Method.POST, subscription: RouteStore.writeGoogleAccessToken, - onValidation: async (user, req, res) => { + onValidation: async ({ user, req, res }) => { + if (!user) { + return res.send(undefined); + } const userId = user.id; const information = { credentialsPath, userId }; res.send(await GoogleApiServerUtils.ProcessClientSideCode(information, req.body.authenticationCode)); @@ -861,8 +873,11 @@ async function PreliminaryFunctions() { router.addSupervisedRoute({ method: Method.POST, subscription: RouteStore.googlePhotosMediaUpload, - onValidation: async (user, req, res) => { + onValidation: async ({ user, req, res }) => { const { media } = req.body; + if (!user) { + return res.send(undefined); + } const userId = user.id; if (!userId) { return _error(res, userIdError); @@ -914,50 +929,62 @@ async function PreliminaryFunctions() { const requestError = "Unable to execute download: the body's media items were malformed."; const deletionPermissionError = "Cannot perform specialized delete outside of the development environment!"; - DashServer.get("/deleteWithAux", async (_req, res) => { - if (release) { - return _permission_denied(res, deletionPermissionError); + router.addSupervisedRoute({ + method: Method.GET, + subscription: "/deleteWithAux", + onValidation: async ({ res, isRelease }) => { + if (isRelease) { + return _permission_denied(res, deletionPermissionError); + } + await Database.Auxiliary.DeleteAll(); + res.redirect(RouteStore.delete); } - await Database.Auxiliary.DeleteAll(); - res.redirect(RouteStore.delete); - }); + }) - DashServer.get("/deleteWithGoogleCredentials", async (req, res) => { - if (release) { - return _permission_denied(res, deletionPermissionError); + router.addSupervisedRoute({ + method: Method.GET, + subscription: "/deleteWithGoogleCredentials", + onValidation: async ({ res, isRelease }) => { + if (isRelease) { + return _permission_denied(res, deletionPermissionError); + } + await Database.Auxiliary.GoogleAuthenticationToken.DeleteAll(); + res.redirect(RouteStore.delete); } - await Database.Auxiliary.GoogleAuthenticationToken.DeleteAll(); - res.redirect(RouteStore.delete); }); const UploadError = (count: number) => `Unable to upload ${count} images to Dash's server`; - DashServer.post(RouteStore.googlePhotosMediaDownload, async (req, res) => { - const contents: { mediaItems: MediaItem[] } = req.body; - let failed = 0; - if (contents) { - const completed: Opt[] = []; - for (let item of contents.mediaItems) { - const { contentSize, ...attributes } = await DashUploadUtils.InspectImage(item.baseUrl); - const found: Opt = await Database.Auxiliary.QueryUploadHistory(contentSize!); - if (!found) { - const upload = await DashUploadUtils.UploadInspectedImage({ contentSize, ...attributes }, item.filename, prefix).catch(error => _error(res, downloadError, error)); - if (upload) { - completed.push(upload); - await Database.Auxiliary.LogUpload(upload); + router.addSupervisedRoute({ + method: Method.POST, + subscription: RouteStore.googlePhotosMediaDownload, + onValidation: async ({ req, res }) => { + const contents: { mediaItems: MediaItem[] } = req.body; + let failed = 0; + if (contents) { + const completed: Opt[] = []; + for (let item of contents.mediaItems) { + const { contentSize, ...attributes } = await DashUploadUtils.InspectImage(item.baseUrl); + const found: Opt = await Database.Auxiliary.QueryUploadHistory(contentSize!); + if (!found) { + const upload = await DashUploadUtils.UploadInspectedImage({ contentSize, ...attributes }, item.filename, prefix).catch(error => _error(res, downloadError, error)); + if (upload) { + completed.push(upload); + await Database.Auxiliary.LogUpload(upload); + } else { + failed++; + } } else { - failed++; + completed.push(found); } - } else { - completed.push(found); } + if (failed) { + return _error(res, UploadError(failed)); + } + return _success(res, completed); } - if (failed) { - return _error(res, UploadError(failed)); - } - return _success(res, completed); + _invalid(res, requestError); } - _invalid(res, requestError); - }); + }) const suffixMap: { [type: string]: (string | [string, string | ((json: any) => any)]) } = { "number": "_n", -- cgit v1.2.3-70-g09d2 From e385c9b0285a1d015917eca49dfc190d9810c8d9 Mon Sep 17 00:00:00 2001 From: Sam Wilkins <35748010+samwilkins333@users.noreply.github.com> Date: Thu, 17 Oct 2019 03:19:36 -0400 Subject: sig int log --- src/server/database.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'src/server/database.ts') diff --git a/src/server/database.ts b/src/server/database.ts index 44c49d03e..12626e594 100644 --- a/src/server/database.ts +++ b/src/server/database.ts @@ -26,7 +26,7 @@ export namespace Database { const { connection } = mongoose; process.on('SIGINT', () => { connection.close(() => { - console.log('Mongoose default connection disconnected through app termination'); + console.log(`SIGINT closed mongoose connection at ${url}`); process.exit(0); }); }); @@ -34,10 +34,10 @@ export namespace Database { await new Promise((resolve, reject) => { connection.on('error', reject); connection.on('disconnected', () => { - console.log(`Mongoose connection at ${url} now closed`); + console.log(`disconnecting mongoose connection at ${url}`); }); connection.on('connected', () => { - console.log(`Mongoose established default connection at ${url}`); + console.log(`mongoose established default connection at ${url}`); resolve(); }); mongoose.connect(url, { useNewUrlParser: true }); -- cgit v1.2.3-70-g09d2 From f0f3dddbe1d3ac54d3754bb913b8ecd9eb6fcc63 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Sun, 27 Oct 2019 21:05:01 -0400 Subject: further cleanup for oauth, separated access token function from client function --- src/server/apis/google/GoogleApiServerUtils.ts | 65 ++++++++++++----------- src/server/apis/google/GooglePhotosUploadUtils.ts | 19 +++---- src/server/database.ts | 2 +- src/server/index.ts | 24 +++------ 4 files changed, 47 insertions(+), 63 deletions(-) (limited to 'src/server/database.ts') diff --git a/src/server/apis/google/GoogleApiServerUtils.ts b/src/server/apis/google/GoogleApiServerUtils.ts index c0824cfb7..88f0f3377 100644 --- a/src/server/apis/google/GoogleApiServerUtils.ts +++ b/src/server/apis/google/GoogleApiServerUtils.ts @@ -15,7 +15,6 @@ const path = require("path"); */ export namespace GoogleApiServerUtils { - // If modifying these scopes, delete token.json. const prefix = 'https://www.googleapis.com/auth/'; const SCOPES = [ 'documents.readonly', @@ -54,7 +53,7 @@ export namespace GoogleApiServerUtils { export const GetEndpoint = (sector: string, userId: string) => { return new Promise>(resolve => { - authorize(userId).then(({ client: auth }) => { + retrieveOAuthClient(userId).then(auth => { let routed: Opt; let parameters: EndpointParameters = { auth, version: "v1" }; switch (sector) { @@ -70,10 +69,23 @@ export namespace GoogleApiServerUtils { }); }; - export const RetrieveAccessToken = (userId: string): Promise => { + export const retrieveAccessToken = (userId: string): Promise => { return new Promise((resolve, reject) => { - authorize(userId).then( - ({ token: { access_token } }) => resolve(access_token!), + retrieveCredentials(userId).then( + ({ access_token }) => resolve(access_token!), + error => reject(`Error: unable to authenticate Google Photos API request.\n${error}`) + ); + }); + }; + + export const retrieveOAuthClient = (userId: string): Promise => { + return new Promise((resolve, reject) => { + retrieveCredentials(userId).then( + credentials => { + const client = generateClient(); + client.setCredentials(credentials); + resolve(client); + }, error => reject(`Error: unable to authenticate Google Photos API request.\n${error}`) ); }); @@ -82,7 +94,7 @@ export namespace GoogleApiServerUtils { let installed: OAuth2ClientOptions; let worker: OAuth2Client; - export const LoadOAuthClient = async () => { + export const loadClientSecret = async () => { return new Promise((resolve, reject) => { readFile(path.join(__dirname, "../../credentials/google_docs_credentials.json"), async (err, credentials) => { if (err) { @@ -90,7 +102,7 @@ export namespace GoogleApiServerUtils { return console.log('Error loading client secret file:', err); } installed = parseBuffer(credentials).installed; - worker = new google.auth.OAuth2(installed); + worker = generateClient(); resolve(); }); }); @@ -98,7 +110,7 @@ export namespace GoogleApiServerUtils { const generateClient = () => new google.auth.OAuth2(installed); - export const GenerateAuthenticationUrl = async (information: CredentialInformation) => { + export const generateAuthenticationUrl = async () => { return worker.generateAuthUrl({ access_type: 'offline', scope: SCOPES.map(relative => prefix + relative), @@ -110,7 +122,7 @@ export namespace GoogleApiServerUtils { avatar: string; name: string; } - export const ProcessClientSideCode = async (userId: string, authenticationCode: string): Promise => { + export const processNewUser = async (userId: string, authenticationCode: string): Promise => { return new Promise((resolve, reject) => { worker.getToken(authenticationCode, async (err, token) => { if (err || !token) { @@ -158,35 +170,25 @@ export namespace GoogleApiServerUtils { sub: string; } - export const authorize = async (userId: string): Promise => { - return Database.Auxiliary.GoogleAuthenticationToken.Fetch(userId).then(token => { - return new Promise((resolve, reject) => { - const client = generateClient(); - if (token!.expiry_date! < new Date().getTime()) { + const retrieveCredentials = async (userId: string): Promise => { + return new Promise((resolve, reject) => { + Database.Auxiliary.GoogleAuthenticationToken.Fetch(userId).then(credentials => { + if (!credentials) { + return reject(); + } + if (credentials!.expiry_date! < new Date().getTime()) { // Token has expired, so submitting a request for a refreshed access token - return refreshToken(token!, client, userId).then(resolve, reject); + return refreshAccessToken(credentials!, userId).then(resolve, reject); } // Authentication successful! - client.setCredentials(token!); - resolve({ token: token!, client }); + resolve(credentials); }); }); }; - export const RetrievePhotosEndpoint = (userId: string) => { - return new Promise((resolve, reject) => { - RetrieveAccessToken(userId).then( - token => resolve(new Photos(token)), - reject - ); - }); - }; - - type AuthenticationResult = { token: Credentials, client: OAuth2Client }; - const refreshEndpoint = "https://oauth2.googleapis.com/token"; - const refreshToken = (credentials: Credentials, oAuth2Client: OAuth2Client, userId: string) => { - return new Promise(resolve => { + const refreshAccessToken = (credentials: Credentials, userId: string) => { + return new Promise(resolve => { let headerParameters = { headers: { 'Content-Type': 'application/x-www-form-urlencoded' } }; let queryParameters = { refreshToken: credentials.refresh_token, @@ -200,8 +202,7 @@ export namespace GoogleApiServerUtils { await Database.Auxiliary.GoogleAuthenticationToken.Update(userId, access_token, expiry_date); credentials.access_token = access_token; credentials.expiry_date = expiry_date; - oAuth2Client.setCredentials(credentials); - resolve({ token: credentials, client: oAuth2Client }); + resolve(credentials); }); }); }; diff --git a/src/server/apis/google/GooglePhotosUploadUtils.ts b/src/server/apis/google/GooglePhotosUploadUtils.ts index 4a67e57cc..d704faa71 100644 --- a/src/server/apis/google/GooglePhotosUploadUtils.ts +++ b/src/server/apis/google/GooglePhotosUploadUtils.ts @@ -20,19 +20,12 @@ export namespace GooglePhotosUploadUtils { } const prepend = (extension: string) => `https://photoslibrary.googleapis.com/v1/${extension}`; - const headers = (type: string) => ({ + const headers = (type: string, token: string) => ({ 'Content-Type': `application/${type}`, - 'Authorization': Bearer, + 'Authorization': token, }); - let Bearer: string; - - export const initialize = async (information: GoogleApiServerUtils.CredentialInformation) => { - const token = await GoogleApiServerUtils.RetrieveAccessToken(information); - Bearer = `Bearer ${token}`; - }; - - export const DispatchGooglePhotosUpload = async (url: string) => { + export const DispatchGooglePhotosUpload = async (bearerToken: string, url: string) => { if (!DashUploadUtils.imageFormats.includes(path.extname(url))) { return undefined; } @@ -40,7 +33,7 @@ export namespace GooglePhotosUploadUtils { const parameters = { method: 'POST', headers: { - ...headers('octet-stream'), + ...headers('octet-stream', bearerToken), 'X-Goog-Upload-File-Name': path.basename(url), 'X-Goog-Upload-Protocol': 'raw' }, @@ -56,13 +49,13 @@ export namespace GooglePhotosUploadUtils { })); }; - export const CreateMediaItems = async (newMediaItems: NewMediaItem[], album?: { id: string }): Promise => { + export const CreateMediaItems = async (bearerToken: string, newMediaItems: NewMediaItem[], album?: { id: string }): Promise => { const newMediaItemResults = await BatchedArray.from(newMediaItems, { batchSize: 50 }).batchedMapPatientInterval( { magnitude: 100, unit: TimeUnit.Milliseconds }, async (batch: NewMediaItem[]) => { const parameters = { method: 'POST', - headers: headers('json'), + headers: headers('json', bearerToken), uri: prepend('mediaItems:batchCreate'), body: { newMediaItems: batch } as any, json: true diff --git a/src/server/database.ts b/src/server/database.ts index 12626e594..79dd26b7d 100644 --- a/src/server/database.ts +++ b/src/server/database.ts @@ -298,7 +298,7 @@ export namespace Database { export type StoredCredentials = Credentials & { _id: string }; - export const Fetch = async (userId: string, removeId = true) => { + export const Fetch = async (userId: string, removeId = true): Promise> => { return SanitizedSingletonQuery({ userId }, GoogleAuthentication, removeId); }; diff --git a/src/server/index.ts b/src/server/index.ts index 3220a9533..24866a5e5 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -65,7 +65,7 @@ async function PreliminaryFunctions() { resolve(); }); }); - await GoogleApiServerUtils.LoadOAuthClient(); + await GoogleApiServerUtils.loadClientSecret(); await DashUploadUtils.createIfNotExists(pdfDirectory); await Database.tryInitializeConnection(); } @@ -553,8 +553,6 @@ function routeSetter(router: RouteManager) { } }); - const credentialsPath = path.join(__dirname, "./credentials/google_docs_credentials.json"); - const EndpointHandlerMap = new Map([ ["create", (api, params) => api.create(params)], ["retrieve", (api, params) => api.get(params)], @@ -588,11 +586,10 @@ function routeSetter(router: RouteManager) { onValidation: async ({ user, res }) => { const userId = user.id; const token = await Database.Auxiliary.GoogleAuthenticationToken.Fetch(userId); - const information = { credentialsPath, userId }; if (!token) { - return res.send(await GoogleApiServerUtils.GenerateAuthenticationUrl(information)); + return res.send(await GoogleApiServerUtils.generateAuthenticationUrl()); } - GoogleApiServerUtils.RetrieveAccessToken(userId).then(token => res.send(token)); + GoogleApiServerUtils.retrieveAccessToken(userId).then(token => res.send(token)); } }); @@ -600,35 +597,28 @@ function routeSetter(router: RouteManager) { method: Method.POST, subscription: RouteStore.writeGoogleAccessToken, onValidation: async ({ user, req, res }) => { - res.send(await GoogleApiServerUtils.ProcessClientSideCode(user.id, req.body.authenticationCode)); + res.send(await GoogleApiServerUtils.processNewUser(user.id, req.body.authenticationCode)); } }); const tokenError = "Unable to successfully upload bytes for all images!"; const mediaError = "Unable to convert all uploaded bytes to media items!"; - const userIdError = "Unable to parse the identification of the user!"; router.addSupervisedRoute({ method: Method.POST, subscription: RouteStore.googlePhotosMediaUpload, onValidation: async ({ user, req, res }) => { const { media } = req.body; - const userId = user.id; - if (!userId) { - return _error(res, userIdError); - } - - await GooglePhotosUploadUtils.initialize({ credentialsPath, userId }); let failed: number[] = []; - + const token = await GoogleApiServerUtils.retrieveAccessToken(user.id); const newMediaItems = await BatchedArray.from(media, { batchSize: 25 }).batchedMapPatientInterval( { magnitude: 100, unit: TimeUnit.Milliseconds }, async (batch: GooglePhotosUploadUtils.MediaInput[]) => { const newMediaItems: NewMediaItem[] = []; for (let index = 0; index < batch.length; index++) { const element = batch[index]; - const uploadToken = await GooglePhotosUploadUtils.DispatchGooglePhotosUpload(element.url); + const uploadToken = await GooglePhotosUploadUtils.DispatchGooglePhotosUpload(token, element.url); if (!uploadToken) { failed.push(index); } else { @@ -647,7 +637,7 @@ function routeSetter(router: RouteManager) { console.error(`Unable to upload ${failedCount} image${failedCount === 1 ? "" : "s"} to Google's servers`); } - GooglePhotosUploadUtils.CreateMediaItems(newMediaItems, req.body.album).then( + GooglePhotosUploadUtils.CreateMediaItems(token, newMediaItems, req.body.album).then( result => _success(res, { results: result.newMediaItemResults, failed }), error => _error(res, mediaError, error) ); -- cgit v1.2.3-70-g09d2 From 109be54065038392b19d9dbafbccc9205f198766 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Tue, 29 Oct 2019 23:37:42 -0400 Subject: db error handling and example code --- src/server/apis/google/GoogleApiServerUtils.ts | 3 +++ src/server/database.ts | 4 ++++ 2 files changed, 7 insertions(+) (limited to 'src/server/database.ts') diff --git a/src/server/apis/google/GoogleApiServerUtils.ts b/src/server/apis/google/GoogleApiServerUtils.ts index ec7c2cfe1..35a2541a9 100644 --- a/src/server/apis/google/GoogleApiServerUtils.ts +++ b/src/server/apis/google/GoogleApiServerUtils.ts @@ -248,6 +248,9 @@ export namespace GoogleApiServerUtils { * with a Dash user in the googleAuthentication table of the database. * @param authenticationCode the Google-provided authentication code that the user copied * from Google's permissions UI and pasted into the overlay. + * + * EXAMPLE CODE: 4/sgF2A5uGg4xASHf7VQDnLtdqo3mUlfQqLSce_HYz5qf1nFtHj9YTeGs + * * @returns the information necessary to authenticate a client side google photos request * and display basic user information in the overlay on successful authentication. * This can be expanded as needed by adding properties to the interface GoogleAuthenticationResult. diff --git a/src/server/database.ts b/src/server/database.ts index 79dd26b7d..b81fc03a4 100644 --- a/src/server/database.ts +++ b/src/server/database.ts @@ -60,6 +60,10 @@ export namespace Database { constructor() { this.MongoClient.connect(url, (_err, client) => { + if (!client) { + console.error("\nPlease start MongoDB by running 'mongod' in a terminal before continuing...\n"); + process.exit(0); + } this.db = client.db(); this.onConnect.forEach(fn => fn()); }); -- cgit v1.2.3-70-g09d2 From df5584ccd40bd83f1362b32db67969e7ffbf2e3f Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Wed, 27 Nov 2019 04:03:30 -0500 Subject: improved file partitioning in server and generified upload method --- package.json | 2 + src/client/documents/Documents.ts | 1 - src/client/util/ClientDiagnostics.ts | 11 +- .../util/Import & Export/DirectoryImportBox.tsx | 19 +- src/client/views/Main.tsx | 2 +- src/client/views/collections/CollectionSubView.tsx | 22 +- src/client/views/pdf/PDFViewer.tsx | 3 +- src/server/ActionUtilities.ts | 17 +- src/server/ApiManagers/DownloadManager.ts | 4 +- src/server/ApiManagers/GooglePhotosManager.ts | 4 +- src/server/ApiManagers/PDFManager.ts | 34 +- src/server/ApiManagers/SearchManager.ts | 4 +- src/server/ApiManagers/UploadManager.ts | 47 ++- src/server/DashUploadUtils.ts | 388 +++++++++++---------- src/server/SharedMediaTypes.ts | 5 +- src/server/database.ts | 4 +- src/server/index.ts | 4 +- 17 files changed, 314 insertions(+), 257 deletions(-) (limited to 'src/server/database.ts') diff --git a/package.json b/package.json index 393df8574..3725d76eb 100644 --- a/package.json +++ b/package.json @@ -105,6 +105,7 @@ "@types/react-table": "^6.7.22", "@types/request": "^2.48.1", "@types/request-promise": "^4.1.42", + "@types/rimraf": "^2.0.3", "@types/sharp": "^0.22.2", "@types/shelljs": "^0.8.5", "@types/socket.io": "^2.1.2", @@ -211,6 +212,7 @@ "readline": "^1.3.0", "request": "^2.88.0", "request-promise": "^4.2.4", + "rimraf": "^3.0.0", "serializr": "^1.5.1", "sharp": "^0.22.1", "shelljs": "^0.8.3", diff --git a/src/client/documents/Documents.ts b/src/client/documents/Documents.ts index dea057b93..a2f4d23c7 100644 --- a/src/client/documents/Documents.ts +++ b/src/client/documents/Documents.ts @@ -377,7 +377,6 @@ export namespace Docs { let extension = path.extname(target); target = `${target.substring(0, target.length - extension.length)}_o${extension}`; } - // if (target !== "http://www.cs.brown.edu/") { requestImageSize(target) .then((size: any) => { let aspect = size.height / size.width; diff --git a/src/client/util/ClientDiagnostics.ts b/src/client/util/ClientDiagnostics.ts index 24f196252..7eef935fd 100644 --- a/src/client/util/ClientDiagnostics.ts +++ b/src/client/util/ClientDiagnostics.ts @@ -1,9 +1,6 @@ -import { observable, runInAction } from "mobx"; -import { MainView } from "../views/MainView"; - export namespace ClientDiagnostics { - export function start() { + export async function start() { let serverPolls = 0; const serverHandle = setInterval(async () => { @@ -17,14 +14,16 @@ export namespace ClientDiagnostics { let executed = false; - const solrHandle = setInterval(async () => { + const handle = async () => { const response = await fetch("/solrHeartbeat"); if (!(await response.json()).running) { !executed && alert("Looks like SOLR is not running on your machine."); executed = true; clearInterval(solrHandle); } - }, 1000 * 15); + }; + await handle(); + const solrHandle = setInterval(handle, 1000 * 15); } diff --git a/src/client/util/Import & Export/DirectoryImportBox.tsx b/src/client/util/Import & Export/DirectoryImportBox.tsx index f0880f193..16ae50685 100644 --- a/src/client/util/Import & Export/DirectoryImportBox.tsx +++ b/src/client/util/Import & Export/DirectoryImportBox.tsx @@ -22,18 +22,10 @@ import "./DirectoryImportBox.scss"; import { Networking } from "../../Network"; import { BatchedArray } from "array-batcher"; import * as path from 'path'; -import { DashUploadUtils } from "../../../server/DashUploadUtils"; -import { SharedMediaTypes } from "../../../server/SharedMediaTypes"; +import { AcceptibleMedia } from "../../../server/SharedMediaTypes"; const unsupported = ["text/html", "text/plain"]; -interface ImageUploadResponse { - name: string; - path: string; - type: string; - exif: any; -} - @observer export default class DirectoryImportBox extends React.Component { private selector = React.createRef(); @@ -98,7 +90,7 @@ export default class DirectoryImportBox extends React.Component let file = files.item(i); if (file && !unsupported.includes(file.type)) { const ext = path.extname(file.name).toLowerCase(); - if (SharedMediaTypes.imageFormats.includes(ext)) { + if (AcceptibleMedia.imageFormats.includes(ext)) { validated.push(file); } } @@ -114,7 +106,7 @@ export default class DirectoryImportBox extends React.Component runInAction(() => this.phase = `Internal: uploading ${this.quota - this.completed} files to Dash...`); - const uploads = await BatchedArray.from(validated, { batchSize: 15 }).batchedMapAsync(async (batch, collector) => { + const uploads = await BatchedArray.from(validated, { batchSize: 15 }).batchedMapAsync(async (batch, collector) => { const formData = new FormData(); batch.forEach(file => { @@ -127,16 +119,17 @@ export default class DirectoryImportBox extends React.Component runInAction(() => this.completed += batch.length); }); + const size = "_o"; await Promise.all(uploads.map(async upload => { const type = upload.type; - const path = Utils.prepend(upload.path); + const path = Utils.prepend(upload.clientAccessPath); const options = { nativeWidth: 300, width: 300, title: upload.name }; const document = await Docs.Get.DocumentFromType(type, path, options); - const { data, error } = upload.exif; + const { data, error } = upload.exifData; if (document) { Doc.GetProto(document).exif = error || Docs.Get.DocumentHierarchyFromJson(data); docs.push(document); diff --git a/src/client/views/Main.tsx b/src/client/views/Main.tsx index dec4a24e4..9e699978f 100644 --- a/src/client/views/Main.tsx +++ b/src/client/views/Main.tsx @@ -10,7 +10,7 @@ import { ClientDiagnostics } from "../util/ClientDiagnostics"; AssignAllExtensions(); (async () => { - ClientDiagnostics.start(); + await ClientDiagnostics.start(); const info = await CurrentUserUtils.loadCurrentUser(); DocServer.init(window.location.protocol, window.location.hostname, 4321, info.email); await Docs.Prototypes.initialize(); diff --git a/src/client/views/collections/CollectionSubView.tsx b/src/client/views/collections/CollectionSubView.tsx index 1c3ff37ee..a1bd1527e 100644 --- a/src/client/views/collections/CollectionSubView.tsx +++ b/src/client/views/collections/CollectionSubView.tsx @@ -22,6 +22,7 @@ import React = require("react"); var path = require('path'); import { GooglePhotos } from "../../apis/google_docs/GooglePhotosClientUtils"; import { ImageUtils } from "../../util/Import & Export/ImageUtils"; +import { Networking } from "../../Network"; export interface CollectionViewProps extends FieldViewProps { addDocument: (document: Doc) => boolean; @@ -271,28 +272,25 @@ export function CollectionSubView(schemaCtor: (doc: Doc) => T) { let file = item.getAsFile(); let formData = new FormData(); - if (file) { - formData.append('file', file); + if (!file || !file.type) { + continue; } - let dropFileName = file ? file.name : "-empty-"; - let prom = fetch(Utils.prepend("/upload"), { - method: 'POST', - body: formData - }).then(async (res: Response) => { - (await res.json()).map(action((file: any) => { + formData.append('file', file); + let dropFileName = file ? file.name : "-empty-"; + promises.push(Networking.PostFormDataToServer("/upload", formData).then(results => { + results.map(action((file: any) => { let full = { ...options, nativeWidth: type.indexOf("video") !== -1 ? 600 : 300, width: 300, title: dropFileName }; - let pathname = Utils.prepend(file.path); + let pathname = Utils.prepend(file.clientAccessPath); Docs.Get.DocumentFromType(type, pathname, full).then(doc => { doc && (Doc.GetProto(doc).fileUpload = path.basename(pathname).replace("upload_", "").replace(/\.[a-z0-9]*$/, "")); doc && this.props.addDocument(doc); }); })); - }); - promises.push(prom); + })); } } - if (text) { + if (text && !text.includes("https://")) { this.props.addDocument(Docs.Create.TextDocument({ ...options, documentText: "@@@" + text, width: 400, height: 315 })); return; } diff --git a/src/client/views/pdf/PDFViewer.tsx b/src/client/views/pdf/PDFViewer.tsx index b737ce221..c075a4f99 100644 --- a/src/client/views/pdf/PDFViewer.tsx +++ b/src/client/views/pdf/PDFViewer.tsx @@ -125,7 +125,8 @@ export class PDFViewer extends DocAnnotatableComponent this._showWaiting = this._showCover = true); this.props.startupLive && this.setupPdfJsViewer(); this._searchReactionDisposer = reaction(() => this.Document.search_string, searchString => { diff --git a/src/server/ActionUtilities.ts b/src/server/ActionUtilities.ts index 7f493dd70..c9fc86fea 100644 --- a/src/server/ActionUtilities.ts +++ b/src/server/ActionUtilities.ts @@ -2,6 +2,7 @@ import * as fs from 'fs'; import { ExecOptions } from 'shelljs'; import { exec } from 'child_process'; import * as path from 'path'; +import * as rimraf from "rimraf"; export const command_line = (command: string, fromDirectory?: string) => { return new Promise((resolve, reject) => { @@ -68,4 +69,18 @@ export function msToTime(duration: number) { let secondsS = (seconds < 10) ? "0" + seconds : seconds; return hoursS + ":" + minutesS + ":" + secondsS + "." + milliseconds; -} \ No newline at end of file +} + +export const createIfNotExists = async (path: string) => { + if (await new Promise(resolve => fs.exists(path, resolve))) { + return true; + } + return new Promise(resolve => fs.mkdir(path, error => resolve(error === null))); +}; + +export async function Prune(rootDirectory: string): Promise { + const error = await new Promise(resolve => rimraf(rootDirectory, resolve)); + return error === null; +} + +export const Destroy = (mediaPath: string) => new Promise(resolve => fs.unlink(mediaPath, error => resolve(error === null))); \ No newline at end of file diff --git a/src/server/ApiManagers/DownloadManager.ts b/src/server/ApiManagers/DownloadManager.ts index fc6ba0d22..5bad46eda 100644 --- a/src/server/ApiManagers/DownloadManager.ts +++ b/src/server/ApiManagers/DownloadManager.ts @@ -5,7 +5,7 @@ import * as Archiver from 'archiver'; import * as express from 'express'; import { Database } from "../database"; import * as path from "path"; -import { DashUploadUtils } from "../DashUploadUtils"; +import { DashUploadUtils, SizeSuffix } from "../DashUploadUtils"; import { publicDirectory } from ".."; export type Hierarchy = { [id: string]: string | Hierarchy }; @@ -254,7 +254,7 @@ async function writeHierarchyRecursive(file: Archiver.Archiver, hierarchy: Hiera // and dropped in the browser and thus hosted remotely) so we upload it // to our server and point the zip file to it, so it can bundle up the bytes const information = await DashUploadUtils.UploadImage(result); - path = information.mediaPaths[0]; + path = information.serverAccessPaths[SizeSuffix.Original]; } // write the file specified by the path to the directory in the // zip file given by the prefix. diff --git a/src/server/ApiManagers/GooglePhotosManager.ts b/src/server/ApiManagers/GooglePhotosManager.ts index c7af69375..5a709688b 100644 --- a/src/server/ApiManagers/GooglePhotosManager.ts +++ b/src/server/ApiManagers/GooglePhotosManager.ts @@ -86,10 +86,10 @@ export default class GooglePhotosManager extends ApiManager { const contents: { mediaItems: MediaItem[] } = req.body; let failed = 0; if (contents) { - const completed: Opt[] = []; + const completed: Opt[] = []; for (let item of contents.mediaItems) { const { contentSize, ...attributes } = await DashUploadUtils.InspectImage(item.baseUrl); - const found: Opt = await Database.Auxiliary.QueryUploadHistory(contentSize!); + const found: Opt = await Database.Auxiliary.QueryUploadHistory(contentSize!); if (!found) { const upload = await DashUploadUtils.UploadInspectedImage({ contentSize, ...attributes }, item.filename, prefix).catch(error => _error(res, downloadError, error)); if (upload) { diff --git a/src/server/ApiManagers/PDFManager.ts b/src/server/ApiManagers/PDFManager.ts index 632b4965a..4bd750aaf 100644 --- a/src/server/ApiManagers/PDFManager.ts +++ b/src/server/ApiManagers/PDFManager.ts @@ -2,12 +2,12 @@ import ApiManager, { Registration } from "./ApiManager"; import { Method } from "../RouteManager"; import RouteSubscriber from "../RouteSubscriber"; import { exists, createReadStream, createWriteStream } from "fs"; -import { filesDirectory } from ".."; import * as Pdfjs from 'pdfjs-dist'; import { createCanvas } from "canvas"; const probe = require("probe-image-size"); import * as express from "express"; import * as path from "path"; +import { Directory, serverPathToFile, clientPathToFile } from "./UploadManager"; export default class PDFManager extends ApiManager { @@ -21,21 +21,27 @@ export default class PDFManager extends ApiManager { let noExt = filename.substring(0, filename.length - ".png".length); let pagenumber = parseInt(noExt.split('-')[1]); return new Promise(resolve => { - exists(filesDirectory + filename, (exists: boolean) => { - console.log(`${filesDirectory + filename} ${exists ? "exists" : "does not exist"}`); + const path = serverPathToFile(Directory.pdf_thumbnails, filename); + exists(path, (exists: boolean) => { + console.log(`${path} ${exists ? "exists" : "does not exist"}`); if (exists) { - let input = createReadStream(filesDirectory + filename); - probe(input, (err: any, result: any) => { + let input = createReadStream(path); + probe(input, (err: any, { width, height }: any) => { if (err) { console.log(err); console.log(`error on ${filename}`); return; } - res.send({ path: "/files/" + filename, width: result.width, height: result.height }); + res.send({ + path: clientPathToFile(Directory.pdf_thumbnails, filename), + width, + height + }); }); } else { - LoadPage(filesDirectory + filename.substring(0, filename.length - noExt.split('-')[1].length - ".PNG".length - 1) + ".pdf", pagenumber, res); + const name = filename.substring(0, filename.length - noExt.split('-')[1].length - ".PNG".length - 1) + ".pdf"; + LoadPage(serverPathToFile(Directory.pdfs, name), pagenumber, res); } resolve(); }); @@ -55,8 +61,8 @@ export default class PDFManager extends ApiManager { let canvasAndContext = factory.create(viewport.width, viewport.height); let renderContext = { canvasContext: canvasAndContext.context, - viewport: viewport, - canvasFactory: factory + canvasFactory: factory, + viewport }; console.log("read " + pageNumber); @@ -64,13 +70,17 @@ export default class PDFManager extends ApiManager { .then(() => { console.log("saving " + pageNumber); let stream = canvasAndContext.canvas.createPNGStream(); - let pngFile = `${file.substring(0, file.length - ".pdf".length)}-${pageNumber}.PNG`; + let filenames = path.basename(file).split("."); + const pngFile = serverPathToFile(Directory.pdf_thumbnails, `${filenames[0]}-${pageNumber}.png`); let out = createWriteStream(pngFile); stream.pipe(out); out.on("finish", () => { console.log(`Success! Saved to ${pngFile}`); - let name = path.basename(pngFile); - res.send({ path: "/files/" + name, width: viewport.width, height: viewport.height }); + res.send({ + path: pngFile, + width: viewport.width, + height: viewport.height + }); }); }, (reason: string) => { console.error(reason + ` ${pageNumber}`); diff --git a/src/server/ApiManagers/SearchManager.ts b/src/server/ApiManagers/SearchManager.ts index 1c801715a..d3f8995b0 100644 --- a/src/server/ApiManagers/SearchManager.ts +++ b/src/server/ApiManagers/SearchManager.ts @@ -3,7 +3,7 @@ import { Method } from "../RouteManager"; import { Search } from "../Search"; var findInFiles = require('find-in-files'); import * as path from 'path'; -import { filesDirectory } from ".."; +import { pathToDirectory, Directory } from "./UploadManager"; export default class SearchManager extends ApiManager { @@ -18,7 +18,7 @@ export default class SearchManager extends ApiManager { res.send([]); return; } - let results = await findInFiles.find({ 'term': q, 'flags': 'ig' }, filesDirectory + "text", ".txt$"); + let results = await findInFiles.find({ 'term': q, 'flags': 'ig' }, pathToDirectory(Directory.text), ".txt$"); let resObj: { ids: string[], numFound: number, lines: string[] } = { ids: [], numFound: 0, lines: [] }; for (var result in results) { resObj.ids.push(path.basename(result, ".txt").replace(/upload_/, "")); diff --git a/src/server/ApiManagers/UploadManager.ts b/src/server/ApiManagers/UploadManager.ts index 2a9faacd8..2f76871a6 100644 --- a/src/server/ApiManagers/UploadManager.ts +++ b/src/server/ApiManagers/UploadManager.ts @@ -7,14 +7,33 @@ import { extname, basename, dirname } from 'path'; import { createReadStream, createWriteStream, unlink, readFileSync } from "fs"; import { publicDirectory, filesDirectory } from ".."; import { Database } from "../database"; -import { DashUploadUtils } from "../DashUploadUtils"; -import { Opt } from "../../new_fields/Doc"; -import { ParsedPDF } from "../PdfTypes"; -const pdf = require('pdf-parse'); +import { DashUploadUtils, SizeSuffix } from "../DashUploadUtils"; import * as sharp from 'sharp'; -import { SharedMediaTypes } from "../SharedMediaTypes"; +import { AcceptibleMedia } from "../SharedMediaTypes"; +import { normalize } from "path"; const imageDataUri = require('image-data-uri'); +export enum Directory { + parsed_files = "parsed_files", + images = "images", + videos = "videos", + pdfs = "pdfs", + text = "text", + pdf_thumbnails = "pdf_thumbnails" +} + +export function serverPathToFile(directory: Directory, filename: string) { + return normalize(`${filesDirectory}/${directory}/${filename}`); +} + +export function pathToDirectory(directory: Directory) { + return normalize(`${filesDirectory}/${directory}`); +} + +export function clientPathToFile(directory: Directory, filename: string) { + return `/files/${directory}/${filename}`; +} + export default class UploadManager extends ApiManager { protected initialize(register: Registration): void { @@ -129,13 +148,14 @@ export default class UploadManager extends ApiManager { subscription: "/upload", onValidation: async ({ req, res }) => { let form = new formidable.IncomingForm(); - form.uploadDir = filesDirectory; + form.uploadDir = pathToDirectory(Directory.parsed_files); form.keepExtensions = true; return new Promise(resolve => { form.parse(req, async (_err, _fields, files) => { let results: any[] = []; for (const key in files) { - results.push(DashUploadUtils.upload(files[key])); + const result = await DashUploadUtils.upload(files[key]); + result && results.push(result); } _success(res, results); resolve(); @@ -150,8 +170,8 @@ export default class UploadManager extends ApiManager { onValidation: async ({ req, res }) => { const { source } = req.body; if (typeof source === "string") { - const uploadInformation = await DashUploadUtils.UploadImage(source); - return res.send(await DashUploadUtils.InspectImage(uploadInformation.mediaPaths[0])); + const { serverAccessPaths } = await DashUploadUtils.UploadImage(source); + return res.send(await DashUploadUtils.InspectImage(serverAccessPaths[SizeSuffix.Original])); } res.send({}); } @@ -167,9 +187,9 @@ export default class UploadManager extends ApiManager { res.status(401).send("incorrect parameters specified"); return; } - return imageDataUri.outputFile(uri, filesDirectory + filename).then((savedName: string) => { + return imageDataUri.outputFile(uri, serverPathToFile(Directory.images, filename)).then((savedName: string) => { const ext = extname(savedName).toLowerCase(); - const { pngs, jpgs } = SharedMediaTypes; + const { pngs, jpgs } = AcceptibleMedia; let resizers = [ { resizer: sharp().resize(100, undefined, { withoutEnlargement: true }), suffix: "_s" }, { resizer: sharp().resize(400, undefined, { withoutEnlargement: true }), suffix: "_m" }, @@ -189,10 +209,11 @@ export default class UploadManager extends ApiManager { } if (isImage) { resizers.forEach(resizer => { - createReadStream(savedName).pipe(resizer.resizer).pipe(createWriteStream(filesDirectory + filename + resizer.suffix + ext)); + const path = serverPathToFile(Directory.images, filename + resizer.suffix + ext); + createReadStream(savedName).pipe(resizer.resizer).pipe(createWriteStream(path)); }); } - res.send("/files/" + filename + ext); + res.send(clientPathToFile(Directory.images, filename + ext)); }); } }); diff --git a/src/server/DashUploadUtils.ts b/src/server/DashUploadUtils.ts index 839aada4b..0a670ec01 100644 --- a/src/server/DashUploadUtils.ts +++ b/src/server/DashUploadUtils.ts @@ -5,18 +5,32 @@ import * as sharp from 'sharp'; import request = require('request-promise'); import { ExifData, ExifImage } from 'exif'; import { Opt } from '../new_fields/Doc'; -import { SharedMediaTypes } from './SharedMediaTypes'; +import { AcceptibleMedia } from './SharedMediaTypes'; import { filesDirectory } from '.'; import { File } from 'formidable'; -import { extname, basename } from "path"; +import { basename } from "path"; +import { ConsoleColors, createIfNotExists } from './ActionUtilities'; +import { ParsedPDF } from "../server/PdfTypes"; +const parse = require('pdf-parse'); +import { Directory, serverPathToFile, clientPathToFile } from './ApiManagers/UploadManager'; -const uploadDirectory = path.join(__dirname, './public/files/'); +export enum SizeSuffix { + Small = "_s", + Medium = "_m", + Large = "_l", + Original = "_o" +} export namespace DashUploadUtils { + function InjectSize(filename: string, size: SizeSuffix) { + const extension = path.extname(filename).toLowerCase(); + return filename.substring(0, filename.length - extension.length) + size + extension; + } + export interface Size { width: number; - suffix: string; + suffix: SizeSuffix; } export interface ImageFileResponse { @@ -27,215 +41,221 @@ export namespace DashUploadUtils { } export const Sizes: { [size: string]: Size } = { - SMALL: { width: 100, suffix: "_s" }, - MEDIUM: { width: 400, suffix: "_m" }, - LARGE: { width: 900, suffix: "_l" }, + SMALL: { width: 100, suffix: SizeSuffix.Small }, + MEDIUM: { width: 400, suffix: SizeSuffix.Medium }, + LARGE: { width: 900, suffix: SizeSuffix.Large }, }; export function validateExtension(url: string) { - return SharedMediaTypes.imageFormats.includes(path.extname(url).toLowerCase()); + return AcceptibleMedia.imageFormats.includes(path.extname(url).toLowerCase()); } const size = "content-length"; const type = "content-type"; - export interface UploadInformation { - mediaPaths: string[]; - fileNames: { [key: string]: string }; + export interface ImageUploadInformation { + clientAccessPath: string; + serverAccessPaths: { [key: string]: string }; exifData: EnrichedExifData; contentSize?: number; contentType?: string; } - export function upload(file: File): any { + export async function upload(file: File): Promise { const { type, path, name } = file; - const filename = basename(path); - const extension = extname(path).toLowerCase(); - if (extension === ".pdf") { - - } else if { - let partition: Opt; - if(imageFormats.includes(extension)) { - partition = DashUploadUtils.Partitions.images; - } else if (videoFormats.includes(extension)) { - partition = DashUploadUtils.Partitions.videos; - } - let uploadInformation: Opt; - if (partition) { - uploadInformation = await DashUploadUtils.UploadImage(`${filesDirectory}/${partition}/${filename}`, filename); - } else { - console.log(`Unable to accommodate, and ignored, the following file upload: ${filename}`); + const { imageFormats, videoFormats, applicationFormats } = AcceptibleMedia; + const types = type.split("/"); + + const category = types[0]; + const format = `.${types[1]}`; + + switch (category) { + case "image": + if (imageFormats.includes(format)) { + const { clientAccessPath } = await UploadImage(path, basename(path), format); + return { clientAccessPath, name, type }; + } + case "video": + if (videoFormats.includes(format)) { + return MoveParsedFile(path, Directory.videos); + } + case "application": + if (applicationFormats.includes(format)) { + return UploadPdf(path); + } } + console.log(ConsoleColors.Red, `Ignoring unsupported file ${name} with upload type (${type}).`); + return { clientAccessPath: undefined }; } - const exif = uploadInformation ? uploadInformation.exifData : undefined; - results.push({ name, type, path: `/files/${filename}`, exif }); -} + async function UploadPdf(absolutePath: string) { + let dataBuffer = fs.readFileSync(absolutePath); + const result: ParsedPDF = await parse(dataBuffer); + const parsedName = basename(absolutePath); + await new Promise((resolve, reject) => { + const textFilename = `${parsedName.substring(0, parsedName.length - 4)}.txt`; + const writeStream = fs.createWriteStream(serverPathToFile(Directory.text, textFilename)); + writeStream.write(result.text, error => error ? reject(error) : resolve()); + }); + return MoveParsedFile(absolutePath, Directory.pdfs); + } -const generate = (prefix: string, url: string) => `${prefix}upload_${Utils.GenerateGuid()}${sanitizeExtension(url)}`; -const sanitize = (filename: string) => filename.replace(/\s+/g, "_"); -const sanitizeExtension = (source: string) => { - let extension = path.extname(source); - extension = extension.toLowerCase(); - extension = extension.split("?")[0]; - return extension; -}; - -/** - * Uploads an image specified by the @param source to Dash's /public/files/ - * directory, and returns information generated during that upload - * - * @param {string} source is either the absolute path of an already uploaded image or - * the url of a remote image - * @param {string} filename dictates what to call the image. If not specified, - * the name {@param prefix}_upload_{GUID} - * @param {string} prefix is a string prepended to the generated image name in the - * event that @param filename is not specified - * - * @returns {UploadInformation} This method returns - * 1) the paths to the uploaded images (plural due to resizing) - * 2) the file name of each of the resized images - * 3) the size of the image, in bytes (4432130) - * 4) the content type of the image, i.e. image/(jpeg | png | ...) - */ -export const UploadImage = async (source: string, filename?: string, prefix: string = ""): Promise => { - const metadata = await InspectImage(source); - return UploadInspectedImage(metadata, filename, prefix); -}; - -export interface InspectionResults { - isLocal: boolean; - stream: any; - normalizedUrl: string; - exifData: EnrichedExifData; - contentSize?: number; - contentType?: string; -} + const generate = (prefix: string, url: string) => `${prefix}upload_${Utils.GenerateGuid()}${sanitizeExtension(url)}`; + const sanitizeExtension = (source: string) => { + let extension = path.extname(source); + extension = extension.toLowerCase(); + extension = extension.split("?")[0]; + return extension; + }; -export interface EnrichedExifData { - data: ExifData; - error?: string; -} + /** + * Uploads an image specified by the @param source to Dash's /public/files/ + * directory, and returns information generated during that upload + * + * @param {string} source is either the absolute path of an already uploaded image or + * the url of a remote image + * @param {string} filename dictates what to call the image. If not specified, + * the name {@param prefix}_upload_{GUID} + * @param {string} prefix is a string prepended to the generated image name in the + * event that @param filename is not specified + * + * @returns {ImageUploadInformation} This method returns + * 1) the paths to the uploaded images (plural due to resizing) + * 2) the file name of each of the resized images + * 3) the size of the image, in bytes (4432130) + * 4) the content type of the image, i.e. image/(jpeg | png | ...) + */ + export const UploadImage = async (source: string, filename?: string, format?: string, prefix: string = ""): Promise => { + const metadata = await InspectImage(source); + return UploadInspectedImage(metadata, filename, format, prefix); + }; -export enum Partitions { - pdf_text = "pdf_text", - images = "images", - videos = "videos" -} + export interface InspectionResults { + isLocal: boolean; + stream: any; + normalizedUrl: string; + exifData: EnrichedExifData; + contentSize?: number; + contentType?: string; + } -export async function buildFilePartitions() { - const pending = Object.keys(Partitions).map(sub => createIfNotExists(filesDirectory + sub)); - return Promise.all(pending); -} + export interface EnrichedExifData { + data: ExifData; + error?: string; + } -/** - * Based on the url's classification as local or remote, gleans - * as much information as possible about the specified image - * - * @param source is the path or url to the image in question - */ -export const InspectImage = async (source: string): Promise => { - const { isLocal, stream, normalized: normalizedUrl } = classify(source); - const exifData = await parseExifData(source); - const results = { - exifData, - isLocal, - stream, - normalizedUrl - }; - // stop here if local, since request.head() can't handle local paths, only urls on the web - if (isLocal) { - return results; + export async function buildFileDirectories() { + const pending = Object.keys(Directory).map(sub => createIfNotExists(`${filesDirectory}/${sub}`)); + return Promise.all(pending); } - const metadata = (await new Promise((resolve, reject) => { - request.head(source, async (error, res) => { - if (error) { - return reject(error); - } - resolve(res); - }); - })).headers; - return { - contentSize: parseInt(metadata[size]), - contentType: metadata[type], - ...results - }; -}; - -export const UploadInspectedImage = async (metadata: InspectionResults, filename?: string, prefix = ""): Promise => { - const { isLocal, stream, normalizedUrl, contentSize, contentType, exifData } = metadata; - const resolved = filename ? sanitize(filename) : generate(prefix, normalizedUrl); - const extension = sanitizeExtension(normalizedUrl || resolved); - let information: UploadInformation = { - mediaPaths: [], - fileNames: { clean: resolved }, - exifData, - contentSize, - contentType, - }; - const { pngs, jpgs } = SharedMediaTypes; - return new Promise(async (resolve, reject) => { - const resizers = [ - { resizer: sharp().rotate(), suffix: "_o" }, - ...Object.values(Sizes).map(size => ({ - resizer: sharp().resize(size.width, undefined, { withoutEnlargement: true }).rotate(), - suffix: size.suffix - })) - ]; - if (pngs.includes(extension)) { - resizers.forEach(element => element.resizer = element.resizer.png()); - } else if (jpgs.includes(extension)) { - resizers.forEach(element => element.resizer = element.resizer.jpeg()); - } - for (let resizer of resizers) { - const suffix = resizer.suffix; - let mediaPath: string; - await new Promise(resolve => { - const filename = resolved.substring(0, resolved.length - extension.length) + suffix + extension; - information.mediaPaths.push(mediaPath = uploadDirectory + filename); - information.fileNames[suffix] = filename; - stream(normalizedUrl).pipe(resizer.resizer).pipe(fs.createWriteStream(mediaPath)) - .on('close', resolve) - .on('error', reject); - }); + + /** + * Based on the url's classification as local or remote, gleans + * as much information as possible about the specified image + * + * @param source is the path or url to the image in question + */ + export const InspectImage = async (source: string): Promise => { + const { isLocal, stream, normalized: normalizedUrl } = classify(source); + const exifData = await parseExifData(source); + const results = { + exifData, + isLocal, + stream, + normalizedUrl + }; + // stop here if local, since request.head() can't handle local paths, only urls on the web + if (isLocal) { + return results; } - if (!isLocal) { - await new Promise(resolve => { - stream(normalizedUrl).pipe(fs.createWriteStream(uploadDirectory + resolved)).on('close', resolve); + const metadata = (await new Promise((resolve, reject) => { + request.head(source, async (error, res) => { + if (error) { + return reject(error); + } + resolve(res); }); - } - resolve(information); - }); -}; - -const classify = (url: string) => { - const isLocal = /Dash-Web(\\|\/)src(\\|\/)server(\\|\/)public(\\|\/)files/g.test(url); - return { - isLocal, - stream: isLocal ? fs.createReadStream : request, - normalized: isLocal ? path.normalize(url) : url + })).headers; + return { + contentSize: parseInt(metadata[size]), + contentType: metadata[type], + ...results + }; }; -}; - -const parseExifData = async (source: string): Promise => { - return new Promise(resolve => { - new ExifImage(source, (error, data) => { - let reason: Opt = undefined; - if (error) { - reason = (error as any).code; + + export async function MoveParsedFile(absolutePath: string, destination: Directory): Promise<{ clientAccessPath: Opt }> { + return new Promise<{ clientAccessPath: Opt }>(resolve => { + const filename = basename(absolutePath); + const destinationPath = serverPathToFile(destination, filename); + fs.rename(absolutePath, destinationPath, error => { + resolve({ clientAccessPath: error ? undefined : clientPathToFile(destination, filename) }); + }); + }); + } + + export const UploadInspectedImage = async (metadata: InspectionResults, filename?: string, format?: string, prefix = ""): Promise => { + const { isLocal, stream, normalizedUrl, contentSize, contentType, exifData } = metadata; + const resolved = filename || generate(prefix, normalizedUrl); + const extension = format || sanitizeExtension(normalizedUrl || resolved); + let information: ImageUploadInformation = { + clientAccessPath: clientPathToFile(Directory.images, resolved), + serverAccessPaths: {}, + exifData, + contentSize, + contentType, + }; + const { pngs, jpgs } = AcceptibleMedia; + return new Promise(async (resolve, reject) => { + const resizers = [ + { resizer: sharp().rotate(), suffix: SizeSuffix.Original }, + ...Object.values(Sizes).map(size => ({ + resizer: sharp().resize(size.width, undefined, { withoutEnlargement: true }).rotate(), + suffix: size.suffix + })) + ]; + if (pngs.includes(extension)) { + resizers.forEach(element => element.resizer = element.resizer.png()); + } else if (jpgs.includes(extension)) { + resizers.forEach(element => element.resizer = element.resizer.jpeg()); + } + for (let { resizer, suffix } of resizers) { + let mediaPath: string; + await new Promise(resolve => { + const filename = InjectSize(resolved, suffix); + information.serverAccessPaths[suffix] = serverPathToFile(Directory.images, filename); + stream(normalizedUrl).pipe(resizer).pipe(fs.createWriteStream(serverPathToFile(Directory.images, filename))) + .on('close', resolve) + .on('error', reject); + }); } - resolve({ data, error: reason }); + if (isLocal) { + await new Promise(resolve => { + fs.unlink(normalizedUrl, error => resolve(error === null)); + }); + } + resolve(information); }); - }); -}; + }; -export const createIfNotExists = async (path: string) => { - if (await new Promise(resolve => fs.exists(path, resolve))) { - return true; - } - return new Promise(resolve => fs.mkdir(path, error => resolve(error === null))); -}; + const classify = (url: string) => { + const isLocal = /Dash-Web(\\|\/)src(\\|\/)server(\\|\/)public(\\|\/)files/g.test(url); + return { + isLocal, + stream: isLocal ? fs.createReadStream : request, + normalized: isLocal ? path.normalize(url) : url + }; + }; -export const Destroy = (mediaPath: string) => new Promise(resolve => fs.unlink(mediaPath, error => resolve(error === null))); + const parseExifData = async (source: string): Promise => { + return new Promise(resolve => { + new ExifImage(source, (error, data) => { + let reason: Opt = undefined; + if (error) { + reason = (error as any).code; + } + resolve({ data, error: reason }); + }); + }); + }; } \ No newline at end of file diff --git a/src/server/SharedMediaTypes.ts b/src/server/SharedMediaTypes.ts index 3d3234125..8d0f441f0 100644 --- a/src/server/SharedMediaTypes.ts +++ b/src/server/SharedMediaTypes.ts @@ -1,9 +1,8 @@ -export namespace SharedMediaTypes { - +export namespace AcceptibleMedia { export const gifs = [".gif"]; export const pngs = [".png"]; export const jpgs = [".jpg", ".jpeg"]; export const imageFormats = [...pngs, ...jpgs, ...gifs]; export const videoFormats = [".mov", ".mp4"]; - + export const applicationFormats = [".pdf"]; } \ No newline at end of file diff --git a/src/server/database.ts b/src/server/database.ts index b81fc03a4..db81245c1 100644 --- a/src/server/database.ts +++ b/src/server/database.ts @@ -293,7 +293,7 @@ export namespace Database { }; export const QueryUploadHistory = async (contentSize: number) => { - return SanitizedSingletonQuery({ contentSize }, AuxiliaryCollections.GooglePhotosUploadHistory); + return SanitizedSingletonQuery({ contentSize }, AuxiliaryCollections.GooglePhotosUploadHistory); }; export namespace GoogleAuthenticationToken { @@ -322,7 +322,7 @@ export namespace Database { } - export const LogUpload = async (information: DashUploadUtils.UploadInformation) => { + export const LogUpload = async (information: DashUploadUtils.ImageUploadInformation) => { const bundle = { _id: Utils.GenerateDeterministicGuid(String(information.contentSize!)), ...information diff --git a/src/server/index.ts b/src/server/index.ts index d02a6005e..d77923710 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -24,7 +24,7 @@ import GooglePhotosManager from "./ApiManagers/GooglePhotosManager"; import DiagnosticManager from "./ApiManagers/DiagnosticManager"; export const publicDirectory = path.resolve(__dirname, "public"); -export const filesDirectory = path.resolve(publicDirectory, "files") + "/"; +export const filesDirectory = path.resolve(publicDirectory, "files"); /** * These are the functions run before the server starts @@ -34,7 +34,7 @@ export const filesDirectory = path.resolve(publicDirectory, "files") + "/"; async function preliminaryFunctions() { await GoogleCredentialsLoader.loadCredentials(); GoogleApiServerUtils.processProjectCredentials(); - await DashUploadUtils.buildFilePartitions(); + await DashUploadUtils.buildFileDirectories(); await log_execution({ startMessage: "attempting to initialize mongodb connection", endMessage: "connection outcome determined", -- cgit v1.2.3-70-g09d2