diff options
| -rw-r--r-- | src/server/ActionUtilities.ts | 28 | ||||
| -rw-r--r-- | src/server/Initialization.ts | 110 | ||||
| -rw-r--r-- | src/server/RouteManager.ts | 131 | ||||
| -rw-r--r-- | src/server/RouteStore.ts | 1 | ||||
| -rw-r--r-- | src/server/apis/google/GoogleApiServerUtils.ts | 21 | ||||
| -rw-r--r-- | src/server/database.ts | 7 | ||||
| -rw-r--r-- | src/server/index.ts | 2089 |
7 files changed, 1240 insertions, 1147 deletions
diff --git a/src/server/ActionUtilities.ts b/src/server/ActionUtilities.ts new file mode 100644 index 000000000..9a009791b --- /dev/null +++ b/src/server/ActionUtilities.ts @@ -0,0 +1,28 @@ +import * as fs from 'fs'; +import { ExecOptions } from 'shelljs'; +import { exec } from 'child_process'; +import * as path from 'path'; + +export const command_line = (command: string, fromDirectory?: string) => { + return new Promise<string>((resolve, reject) => { + let options: ExecOptions = {}; + if (fromDirectory) { + options.cwd = path.join(__dirname, fromDirectory); + } + exec(command, options, (err, stdout) => err ? reject(err) : resolve(stdout)); + }); +}; + +export const read_text_file = (relativePath: string) => { + let target = path.join(__dirname, relativePath); + return new Promise<string>((resolve, reject) => { + fs.readFile(target, (err, data) => err ? reject(err) : resolve(data.toString())); + }); +}; + +export const write_text_file = (relativePath: string, contents: any) => { + let target = path.join(__dirname, relativePath); + return new Promise<void>((resolve, reject) => { + fs.writeFile(target, contents, (err) => err ? reject(err) : resolve()); + }); +};
\ No newline at end of file diff --git a/src/server/Initialization.ts b/src/server/Initialization.ts new file mode 100644 index 000000000..e371a3edb --- /dev/null +++ b/src/server/Initialization.ts @@ -0,0 +1,110 @@ +import * as express from 'express'; +import * as expressValidator from 'express-validator'; +import * as session from 'express-session'; +import * as passport from 'passport'; +import * as bodyParser from 'body-parser'; +import * as cookieParser from 'cookie-parser'; +import expressFlash = require('express-flash'); +import flash = require('connect-flash'); +import { Database } from './database'; +const MongoStore = require('connect-mongo')(session); +import mongoose, { ConnectionStates } from 'mongoose'; +import { RouteStore } from './RouteStore'; +import RouteManager from './RouteManager'; +import * as webpack from 'webpack'; +const config = require('../../webpack.config'); +const compiler = webpack(config); +import * as wdm from 'webpack-dev-middleware'; +import * as whm from 'webpack-hot-middleware'; +import * as fs from 'fs'; + +export interface InitializationOptions { + listenAtPort: number; + routeSetter: (server: RouteManager) => void; +} + +export default async function InitializeServer(options: InitializationOptions) { + const { listenAtPort, routeSetter } = options; + const server = injectMiddleware(express()); + const { url } = Database; + try { + await connectToDatabase(url); + } catch (e) { + console.error(`Mongoose FAILED to establish default connection at ${url}`); + console.error(e); + console.log('Since a valid database connection is required to use Dash, killing the server process.\nPlease try again later.'); + process.exit(1); + } + + // static file serving + server.use(express.static(__dirname + RouteStore.public)); + server.use(RouteStore.images, express.static(__dirname + RouteStore.public)); + + routeSetter(new RouteManager(server, determineEnvironment())); + + server.use(wdm(compiler, { publicPath: config.output.publicPath })); + server.use(whm(compiler)); + server.listen(listenAtPort, () => console.log(`server started at http://localhost:${listenAtPort}`)); + + return server; +} + +function determineEnvironment() { + const isRelease = process.env.RELEASE === "true"; + + console.log(`running server in ${isRelease ? 'release' : 'debug'} mode`); + console.log(process.env.PWD); + + let clientUtils = fs.readFileSync("./src/client/util/ClientUtils.ts.temp", "utf8"); + clientUtils = `//AUTO-GENERATED FILE: DO NOT EDIT\n${clientUtils.replace('"mode"', String(isRelease))}`; + fs.writeFileSync("./src/client/util/ClientUtils.ts", clientUtils, "utf8"); + + return isRelease; +} + +const week = 7 * 24 * 60 * 60 * 1000; +const secret = "64d6866242d3b5a5503c675b32c9605e4e90478e9b77bcf2bc"; + +function injectMiddleware(server: express.Express) { + [ + cookieParser(), + session({ + secret, + resave: true, + cookie: { maxAge: week }, + saveUninitialized: true, + store: new MongoStore({ url: Database.url }) + }), + flash(), + expressFlash(), + bodyParser.json({ limit: "10mb" }), + bodyParser.urlencoded({ extended: true }), + expressValidator(), + passport.initialize(), + passport.session(), + (req: express.Request, res: express.Response, next: express.NextFunction) => { + res.locals.user = req.user; + next(); + } + ].forEach(next => server.use(next)); + return server; +} + +async function connectToDatabase(url: string) { + const { connection } = mongoose; + process.on('SIGINT', () => { + connection.close(() => { + console.log('Mongoose default connection disconnected through app termination'); + process.exit(0); + }); + }); + if (connection.readyState === ConnectionStates.disconnected) { + return new Promise<void>((resolve, reject) => { + connection.on('error', reject); + connection.on('connected', () => { + console.log(`Mongoose established default connection at ${url}`); + resolve(); + }); + }); + } +}
\ No newline at end of file diff --git a/src/server/RouteManager.ts b/src/server/RouteManager.ts new file mode 100644 index 000000000..cf15e45c9 --- /dev/null +++ b/src/server/RouteManager.ts @@ -0,0 +1,131 @@ +import RouteSubscriber from "./RouteSubscriber"; +import { RouteStore } from "./RouteStore"; +import { DashUserModel } from "./authentication/models/user_model"; +import * as express from 'express'; +import * as qs from 'query-string'; + +export default class RouteManager { + private server: express.Express; + private _isRelease: boolean; + + public get release() { + return this._isRelease; + } + + constructor(server: express.Express, isRelease: boolean) { + this.server = server; + this._isRelease = isRelease; + } + + /** + * Please invoke this function when adding a new route to Dash's server. + * It ensures that any requests leading to or containing user-sensitive information + * does not execute unless Passport authentication detects a user logged in. + * @param method whether or not the request is a GET or a POST + * @param handler the action to invoke, recieving a DashUserModel and, as expected, the Express.Request and Express.Response + * @param onRejection an optional callback invoked on return if no user is found to be logged in + * @param subscribers the forward slash prepended path names (reference and add to RouteStore.ts) that will all invoke the given @param handler + */ + addSupervisedRoute(initializer: RouteInitializer) { + const { method, subscription, onValidation, onRejection, onError } = initializer; + const release = this._isRelease; + let abstracted = async (req: express.Request, res: express.Response) => { + const { user, originalUrl: target } = req; + if (user || isSharedDocAccess(target)) { + try { + await onValidation(user, req, res, release); + } catch (e) { + if (onError) { + onError(req, res, e, release); + } else { + _error(res, `The server encountered an internal error handling ${target}.`, e); + } + } + } else { + req.session!.target = target; + try { + await (onRejection || LoginRedirect)(req, res, release); + } catch (e) { + if (onError) { + onError(req, res, e, this._isRelease); + } else { + _error(res, `The server encountered an internal error when rejecting ${target}.`, e); + } + } + } + }; + const subscribe = (subscriber: RouteSubscriber | string) => { + let route: string; + if (typeof subscriber === "string") { + route = subscriber; + } else { + route = subscriber.build; + } + switch (method) { + case Method.GET: + this.server.get(route, abstracted); + break; + case Method.POST: + this.server.post(route, abstracted); + break; + } + }; + if (Array.isArray(subscription)) { + subscription.forEach(subscribe); + } else { + subscribe(subscription); + } + } + +} + +export enum Method { + GET, + POST +} + +export type ValidationHandler = (user: DashUserModel, req: express.Request, res: express.Response, isRelease: boolean) => any | Promise<any>; +export type RejectionHandler = (req: express.Request, res: express.Response, isRelease: boolean) => any | Promise<any>; +export type ErrorHandler = (req: express.Request, res: express.Response, error: any, isRelease: boolean) => any | Promise<any>; + +const LoginRedirect: RejectionHandler = (_req, res) => res.redirect(RouteStore.login); + +export interface RouteInitializer { + method: Method; + subscription: string | RouteSubscriber | (string | RouteSubscriber)[]; + onValidation: ValidationHandler; + onRejection?: RejectionHandler; + onError?: ErrorHandler; +} + +const isSharedDocAccess = (target: string) => { + const shared = qs.parse(qs.extract(target), { sort: false }).sharing === "true"; + const docAccess = target.startsWith("/doc/"); + return shared && docAccess; +}; + +export const STATUS = { + OK: 200, + BAD_REQUEST: 400, + EXECUTION_ERROR: 500, + PERMISSION_DENIED: 403 +}; + +export function _error(res: express.Response, message: string, error?: any) { + res.statusMessage = message; + res.status(STATUS.EXECUTION_ERROR).send(error); +} + +export function _success(res: express.Response, body: any) { + res.status(STATUS.OK).send(body); +} + +export function _invalid(res: express.Response, message: string) { + res.statusMessage = message; + res.status(STATUS.BAD_REQUEST).send(); +} + +export function _permission_denied(res: express.Response, message: string) { + res.statusMessage = message; + res.status(STATUS.BAD_REQUEST).send("Permission Denied!"); +} diff --git a/src/server/RouteStore.ts b/src/server/RouteStore.ts index 7426ffb39..de2553b2f 100644 --- a/src/server/RouteStore.ts +++ b/src/server/RouteStore.ts @@ -7,6 +7,7 @@ export enum RouteStore { corsProxy = "/corsProxy", delete = "/delete", deleteAll = "/deleteAll", + pull = "/pull", // UPLOAD AND STATIC FILE SERVING public = "/public", diff --git a/src/server/apis/google/GoogleApiServerUtils.ts b/src/server/apis/google/GoogleApiServerUtils.ts index 5714c9928..6093197f1 100644 --- a/src/server/apis/google/GoogleApiServerUtils.ts +++ b/src/server/apis/google/GoogleApiServerUtils.ts @@ -9,6 +9,8 @@ import request = require('request-promise'); import * as qs from 'query-string'; import Photos = require('googlephotos'); import { Database } from "../../database"; +const path = require("path"); + /** * Server side authentication for Google Api queries. */ @@ -76,22 +78,24 @@ export namespace GoogleApiServerUtils { }); }; - const RetrieveOAuthClient = async (information: CredentialInformation) => { - return new Promise<OAuth2Client>((resolve, reject) => { - readFile(information.credentialsPath, async (err, credentials) => { + let AuthorizationManager: OAuth2Client; + + export const LoadOAuthClient = async () => { + return new Promise<void>((resolve, reject) => { + readFile(path.join(__dirname, "../../credentials/google_docs_credentials.json"), async (err, credentials) => { if (err) { reject(err); return console.log('Error loading client secret file:', err); } const { client_secret, client_id, redirect_uris } = parseBuffer(credentials).installed; - resolve(new google.auth.OAuth2(client_id, client_secret, redirect_uris[0])); + AuthorizationManager = new google.auth.OAuth2(client_id, client_secret, redirect_uris[0]); + resolve(); }); }); }; export const GenerateAuthenticationUrl = async (information: CredentialInformation) => { - const client = await RetrieveOAuthClient(information); - return client.generateAuthUrl({ + return AuthorizationManager.generateAuthUrl({ access_type: 'offline', scope: SCOPES.map(relative => prefix + relative), }); @@ -103,14 +107,13 @@ export namespace GoogleApiServerUtils { name: string; } export const ProcessClientSideCode = async (information: CredentialInformation, authenticationCode: string): Promise<GoogleAuthenticationResult> => { - const oAuth2Client = await RetrieveOAuthClient(information); return new Promise<GoogleAuthenticationResult>((resolve, reject) => { - oAuth2Client.getToken(authenticationCode, async (err, token) => { + AuthorizationManager.getToken(authenticationCode, async (err, token) => { if (err || !token) { reject(err); return console.error('Error retrieving access token', err); } - oAuth2Client.setCredentials(token); + AuthorizationManager.setCredentials(token); const enriched = injectUserInfo(token); await Database.Auxiliary.GoogleAuthenticationToken.Write(information.userId, enriched); const { given_name, picture } = enriched.userInfo; diff --git a/src/server/database.ts b/src/server/database.ts index db86b472d..25e1e67e0 100644 --- a/src/server/database.ts +++ b/src/server/database.ts @@ -8,16 +8,19 @@ import { GoogleApiServerUtils } from './apis/google/GoogleApiServerUtils'; export namespace Database { + const schema = 'Dash'; + const port = 27017; + export const url = `mongodb://localhost:${port}/${schema}`; + class Database { public static DocumentsCollection = 'documents'; private MongoClient = mongodb.MongoClient; - private url = 'mongodb://localhost:27017/Dash'; private currentWrites: { [id: string]: Promise<void> } = {}; private db?: mongodb.Db; private onConnect: (() => void)[] = []; constructor() { - this.MongoClient.connect(this.url, (err, client) => { + this.MongoClient.connect(url, (_err, client) => { this.db = client.db(); this.onConnect.forEach(fn => fn()); }); diff --git a/src/server/index.ts b/src/server/index.ts index 2203ae2e1..ad18857b6 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -1,1255 +1,1072 @@ require('dotenv').config(); -import * as bodyParser from 'body-parser'; -import { exec, ExecOptions } from 'child_process'; -import * as cookieParser from 'cookie-parser'; -import * as express from 'express'; -import * as session from 'express-session'; -import * as expressValidator from 'express-validator'; +import { exec } from 'child_process'; import * as formidable from 'formidable'; import * as fs from 'fs'; import * as sharp from 'sharp'; import * as Pdfjs from 'pdfjs-dist'; const imageDataUri = require('image-data-uri'); import * as mobileDetect from 'mobile-detect'; -import * as passport from 'passport'; import * as path from 'path'; import * as request from 'request'; -import * as io from 'socket.io'; +import io from 'socket.io'; import { Socket } from 'socket.io'; -import * as webpack from 'webpack'; -import * as wdm from 'webpack-dev-middleware'; -import * as whm from 'webpack-hot-middleware'; import { Utils } from '../Utils'; import { getForgot, getLogin, getLogout, getReset, getSignup, postForgot, postLogin, postReset, postSignup } from './authentication/controllers/user_controller'; -import { DashUserModel } from './authentication/models/user_model'; import { Client } from './Client'; import { Database } from './database'; import { MessageStore, Transferable, Types, Diff, YoutubeQueryTypes as YoutubeQueryType, YoutubeQueryInput } from "./Message"; import { RouteStore } from './RouteStore'; import v4 = require('uuid/v4'); -const app = express(); -const config = require('../../webpack.config'); import { createCanvas } from "canvas"; -const compiler = webpack(config); -const port = 1050; // default port to listen const serverPort = 4321; -import expressFlash = require('express-flash'); -import flash = require('connect-flash'); import { Search } from './Search'; import * as Archiver from 'archiver'; var AdmZip = require('adm-zip'); import * as YoutubeApi from "./apis/youtube/youtubeApiSample"; import { Response } from 'express-serve-static-core'; import { GoogleApiServerUtils } from "./apis/google/GoogleApiServerUtils"; -const MongoStore = require('connect-mongo')(session); -const mongoose = require('mongoose'); const probe = require("probe-image-size"); const pdf = require('pdf-parse'); var findInFiles = require('find-in-files'); import { GooglePhotosUploadUtils } from './apis/google/GooglePhotosUploadUtils'; -import * as qs from 'query-string'; import { Opt } from '../new_fields/Doc'; import { DashUploadUtils } from './DashUploadUtils'; import { BatchedArray, TimeUnit } from 'array-batcher'; import { ParsedPDF } from "./PdfTypes"; import { reject } from 'bluebird'; -import { ExifData } from 'exif'; -import { Result } from '../client/northstar/model/idea/idea'; import RouteSubscriber from './RouteSubscriber'; +import InitializeServer from './Initialization'; +import { Method, _success, _permission_denied, _error, _invalid } from './RouteManager'; +import { command_line, read_text_file } from './ActionUtilities'; -const download = (url: string, dest: fs.PathLike) => request.get(url).pipe(fs.createWriteStream(dest)); let youtubeApiKey: string; -YoutubeApi.readApiKey((apiKey: string) => youtubeApiKey = apiKey); -const release = process.env.RELEASE === "true"; -if (process.env.RELEASE === "true") { - console.log("Running server in release mode"); -} else { - console.log("Running server in debug mode"); -} -console.log(process.env.PWD); -let clientUtils = fs.readFileSync("./src/client/util/ClientUtils.ts.temp", "utf8"); -clientUtils = `//AUTO-GENERATED FILE: DO NOT EDIT\n${clientUtils.replace('"mode"', String(release))}`; -fs.writeFileSync("./src/client/util/ClientUtils.ts", clientUtils, "utf8"); - -const mongoUrl = 'mongodb://localhost:27017/Dash'; -mongoose.connection.readyState === 0 && mongoose.connect(mongoUrl); -mongoose.connection.on('connected', () => console.log("connected")); - -// SESSION MANAGEMENT AND AUTHENTICATION MIDDLEWARE -// ORDER OF IMPORTS MATTERS - -app.use(cookieParser()); -app.use(session({ - secret: "64d6866242d3b5a5503c675b32c9605e4e90478e9b77bcf2bc", - resave: true, - cookie: { maxAge: 7 * 24 * 60 * 60 * 1000 }, - saveUninitialized: true, - store: new MongoStore({ url: 'mongodb://localhost:27017/Dash' }) -})); - -app.use(flash()); -app.use(expressFlash()); -app.use(bodyParser.json({ limit: "10mb" })); -app.use(bodyParser.urlencoded({ extended: true })); -app.use(expressValidator()); -app.use(passport.initialize()); -app.use(passport.session()); -app.use((req, res, next) => { - res.locals.user = req.user; - next(); -}); - -app.get("/hello", (req, res) => res.send("<p>Hello</p>")); - -enum Method { - GET, - POST +export type Hierarchy = { [id: string]: string | Hierarchy }; +export type ZipMutator = (file: Archiver.Archiver) => void | Promise<void>; + +export interface NewMediaItem { + description: string; + simpleMediaItem: { + uploadToken: string; + }; } -export type ValidationHandler = (user: DashUserModel, req: express.Request, res: express.Response) => any | Promise<any>; -export type RejectionHandler = (req: express.Request, res: express.Response) => any | Promise<any>; -export type ErrorHandler = (req: express.Request, res: express.Response, error: any) => any | Promise<any>; +(async () => { + YoutubeApi.readApiKey((apiKey: string) => youtubeApiKey = apiKey); + await GoogleApiServerUtils.LoadOAuthClient(); + + await InitializeServer({ + listenAtPort: 1050, + routeSetter: router => { + + router.addSupervisedRoute({ + method: Method.GET, + subscription: "/pull", + onValidation: (_user, _req, res) => { + exec('"C:\\Program Files\\Git\\git-bash.exe" -c "git pull"', err => { + if (err) { + res.send(err.message); + return; + } + res.redirect("/"); + }); + } + }); -const LoginRedirect: RejectionHandler = (_req, res) => res.redirect(RouteStore.login); + router.addSupervisedRoute({ + method: Method.GET, + subscription: "/buxton", + onValidation: (_user, _req, res) => { + let cwd = '../scraping/buxton'; -export interface RouteInitializer { - method: Method; - subscribers: string | RouteSubscriber | (string | RouteSubscriber)[]; - onValidation: ValidationHandler; - onRejection?: RejectionHandler; - onError?: ErrorHandler; -} + let onResolved = (stdout: string) => { console.log(stdout); res.redirect("/"); }; + let onRejected = (err: any) => { console.error(err.message); res.send(err); }; + let tryPython3 = () => command_line('python3 scraper.py', cwd).then(onResolved, onRejected); -const isSharedDocAccess = (target: string) => { - const shared = qs.parse(qs.extract(target), { sort: false }).sharing === "true"; - const docAccess = target.startsWith("/doc/"); - return shared && docAccess; -}; - -/** - * Please invoke this function when adding a new route to Dash's server. - * It ensures that any requests leading to or containing user-sensitive information - * does not execute unless Passport authentication detects a user logged in. - * @param method whether or not the request is a GET or a POST - * @param handler the action to invoke, recieving a DashUserModel and, as expected, the Express.Request and Express.Response - * @param onRejection an optional callback invoked on return if no user is found to be logged in - * @param subscribers the forward slash prepended path names (reference and add to RouteStore.ts) that will all invoke the given @param handler - */ -function addSecureRoute(initializer: RouteInitializer) { - const { method, subscribers, onValidation, onRejection, onError } = initializer; - let abstracted = async (req: express.Request, res: express.Response) => { - const { user, originalUrl: target } = req; - if (user || isSharedDocAccess(target)) { - try { - await onValidation(user, req, res); - } catch (e) { - if (onError) { - onError(req, res, e); - } else { - _error(res, `The server encountered an internal error handling ${target}.`, e); - } - } - } else { - req.session!.target = target; - try { - await (onRejection || LoginRedirect)(req, res); - } catch (e) { - if (onError) { - onError(req, res, e); - } else { - _error(res, `The server encountered an internal error when rejecting ${target}.`, e); + command_line('python scraper.py', cwd).then(onResolved, tryPython3); + }, + }); + + router.addSupervisedRoute({ + method: Method.GET, + subscription: "/version", + onValidation: (_user, _req, res) => { + exec('"C:\\Program Files\\Git\\bin\\git.exe" rev-parse HEAD', (err, stdout) => { + if (err) { + res.send(err.message); + return; + } + res.send(stdout); + }); } - } - } - }; - const subscribe = (subscriber: RouteSubscriber | string) => { - let route: string; - if (typeof subscriber === "string") { - route = subscriber; - } else { - route = subscriber.build; - } - switch (method) { - case Method.GET: - app.get(route, abstracted); - break; - case Method.POST: - app.post(route, abstracted); - break; - } - }; - if (Array.isArray(subscribers)) { - subscribers.forEach(subscribe); - } else { - subscribe(subscribers); - } -} + }); -// STATIC FILE SERVING -app.use(express.static(__dirname + RouteStore.public)); -app.use(RouteStore.images, express.static(__dirname + RouteStore.public)); + // SEARCH + const solrURL = "http://localhost:8983/solr/#/dash"; -app.get("/pull", (req, res) => - exec('"C:\\Program Files\\Git\\git-bash.exe" -c "git pull"', (err, stdout, stderr) => { - if (err) { - res.send(err.message); - return; - } - res.redirect("/"); - })); - -app.get("/buxton", (req, res) => { - let cwd = '../scraping/buxton'; - - let onResolved = (stdout: string) => { console.log(stdout); res.redirect("/"); }; - let onRejected = (err: any) => { console.error(err.message); res.send(err); }; - let tryPython3 = () => command_line('python3 scraper.py', cwd).then(onResolved, onRejected); - - command_line('python scraper.py', cwd).then(onResolved, tryPython3); -}); - -const STATUS = { - OK: 200, - BAD_REQUEST: 400, - EXECUTION_ERROR: 500, - PERMISSION_DENIED: 403 -}; - -const command_line = (command: string, fromDirectory?: string) => { - return new Promise<string>((resolve, reject) => { - let options: ExecOptions = {}; - if (fromDirectory) { - options.cwd = path.join(__dirname, fromDirectory); - } - exec(command, options, (err, stdout) => err ? reject(err) : resolve(stdout)); - }); -}; + // GETTERS -const read_text_file = (relativePath: string) => { - let target = path.join(__dirname, relativePath); - return new Promise<string>((resolve, reject) => { - fs.readFile(target, (err, data) => err ? reject(err) : resolve(data.toString())); - }); -}; + DashServer.get("/textsearch", async (req, res) => { + let q = req.query.q; + if (q === undefined) { + res.send([]); + return; + } + let results = await findInFiles.find({ 'term': q, 'flags': 'ig' }, uploadDirectory + "text", ".txt$"); + let resObj: { ids: string[], numFound: number, lines: string[] } = { ids: [], numFound: 0, lines: [] }; + for (var result in results) { + resObj.ids.push(path.basename(result, ".txt").replace(/upload_/, "")); + resObj.lines.push(results[result].line); + resObj.numFound++; + } + res.send(resObj); + }); -const write_text_file = (relativePath: string, contents: any) => { - let target = path.join(__dirname, relativePath); - return new Promise<void>((resolve, reject) => { - fs.writeFile(target, contents, (err) => err ? reject(err) : resolve()); - }); -}; + DashServer.get("/search", async (req, res) => { + const solrQuery: any = {}; + ["q", "fq", "start", "rows", "hl", "hl.fl"].forEach(key => solrQuery[key] = req.query[key]); + if (solrQuery.q === undefined) { + res.send([]); + return; + } + let results = await Search.Instance.search(solrQuery); + res.send(results); + }); -app.get("/version", (req, res) => { - exec('"C:\\Program Files\\Git\\bin\\git.exe" rev-parse HEAD', (err, stdout, stderr) => { - if (err) { - res.send(err.message); - return; - } - res.send(stdout); - }); -}); - -// SEARCH -const solrURL = "http://localhost:8983/solr/#/dash"; - -// GETTERS - -app.get("/textsearch", async (req, res) => { - let q = req.query.q; - if (q === undefined) { - res.send([]); - return; - } - let results = await findInFiles.find({ 'term': q, 'flags': 'ig' }, uploadDirectory + "text", ".txt$"); - let resObj: { ids: string[], numFound: number, lines: string[] } = { ids: [], numFound: 0, lines: [] }; - for (var result in results) { - resObj.ids.push(path.basename(result, ".txt").replace(/upload_/, "")); - resObj.lines.push(results[result].line); - resObj.numFound++; - } - res.send(resObj); -}); - -app.get("/search", async (req, res) => { - const solrQuery: any = {}; - ["q", "fq", "start", "rows", "hl", "hl.fl"].forEach(key => solrQuery[key] = req.query[key]); - if (solrQuery.q === undefined) { - res.send([]); - return; - } - let results = await Search.Instance.search(solrQuery); - res.send(results); -}); - -function msToTime(duration: number) { - let milliseconds = Math.floor((duration % 1000) / 100), - seconds = Math.floor((duration / 1000) % 60), - minutes = Math.floor((duration / (1000 * 60)) % 60), - hours = Math.floor((duration / (1000 * 60 * 60)) % 24); - - let hoursS = (hours < 10) ? "0" + hours : hours; - let minutesS = (minutes < 10) ? "0" + minutes : minutes; - let secondsS = (seconds < 10) ? "0" + seconds : seconds; - - return hoursS + ":" + minutesS + ":" + secondsS + "." + milliseconds; -} + function msToTime(duration: number) { + let milliseconds = Math.floor((duration % 1000) / 100), + seconds = Math.floor((duration / 1000) % 60), + minutes = Math.floor((duration / (1000 * 60)) % 60), + hours = Math.floor((duration / (1000 * 60 * 60)) % 24); -async function getDocs(id: string) { - const files = new Set<string>(); - const docs: { [id: string]: any } = {}; - const fn = (doc: any): string[] => { - const id = doc.id; - if (typeof id === "string" && id.endsWith("Proto")) { - //Skip protos - return []; - } - const ids: string[] = []; - for (const key in doc.fields) { - if (!doc.fields.hasOwnProperty(key)) { - continue; + let hoursS = (hours < 10) ? "0" + hours : hours; + let minutesS = (minutes < 10) ? "0" + minutes : minutes; + let secondsS = (seconds < 10) ? "0" + seconds : seconds; + + return hoursS + ":" + minutesS + ":" + secondsS + "." + milliseconds; } - const field = doc.fields[key]; - if (field === undefined || field === null) { - continue; + + async function getDocs(id: string) { + const files = new Set<string>(); + const docs: { [id: string]: any } = {}; + const fn = (doc: any): string[] => { + const id = doc.id; + if (typeof id === "string" && id.endsWith("Proto")) { + //Skip protos + return []; + } + const ids: string[] = []; + for (const key in doc.fields) { + if (!doc.fields.hasOwnProperty(key)) { + continue; + } + const field = doc.fields[key]; + if (field === undefined || field === null) { + continue; + } + + if (field.__type === "proxy" || field.__type === "prefetch_proxy") { + ids.push(field.fieldId); + } else if (field.__type === "script" || field.__type === "computed") { + if (field.captures) { + ids.push(field.captures.fieldId); + } + } else if (field.__type === "list") { + ids.push(...fn(field)); + } else if (typeof field === "string") { + const re = /"(?:dataD|d)ocumentId"\s*:\s*"([\w\-]*)"/g; + let match: string[] | null; + while ((match = re.exec(field)) !== null) { + ids.push(match[1]); + } + } else if (field.__type === "RichTextField") { + const re = /"href"\s*:\s*"(.*?)"/g; + let match: string[] | null; + while ((match = re.exec(field.Data)) !== null) { + const urlString = match[1]; + const split = new URL(urlString).pathname.split("doc/"); + if (split.length > 1) { + ids.push(split[split.length - 1]); + } + } + const re2 = /"src"\s*:\s*"(.*?)"/g; + while ((match = re2.exec(field.Data)) !== null) { + const urlString = match[1]; + const pathname = new URL(urlString).pathname; + files.add(pathname); + } + } else if (["audio", "image", "video", "pdf", "web"].includes(field.__type)) { + const url = new URL(field.url); + const pathname = url.pathname; + files.add(pathname); + } + } + + if (doc.id) { + docs[doc.id] = doc; + } + return ids; + }; + await Database.Instance.visit([id], fn); + return { id, docs, files }; } + DashServer.get("/serializeDoc/:docId", async (req, res) => { + const { docs, files } = await getDocs(req.params.docId); + res.send({ docs, files: Array.from(files) }); + }); - if (field.__type === "proxy" || field.__type === "prefetch_proxy") { - ids.push(field.fieldId); - } else if (field.__type === "script" || field.__type === "computed") { - if (field.captures) { - ids.push(field.captures.fieldId); + router.addSupervisedRoute({ + method: Method.GET, + subscription: new RouteSubscriber(RouteStore.imageHierarchyExport).add('docId'), + onValidation: async (_user, req, res) => { + const id = req.params.docId; + const hierarchy: Hierarchy = {}; + await targetedVisitorRecursive(id, hierarchy); + BuildAndDispatchZip(res, async zip => { + await hierarchyTraverserRecursive(zip, hierarchy); + }); } - } else if (field.__type === "list") { - ids.push(...fn(field)); - } else if (typeof field === "string") { - const re = /"(?:dataD|d)ocumentId"\s*:\s*"([\w\-]*)"/g; - let match: string[] | null; - while ((match = re.exec(field)) !== null) { - ids.push(match[1]); + }); + + const BuildAndDispatchZip = async (res: Response, mutator: ZipMutator): Promise<void> => { + const zip = Archiver('zip'); + zip.pipe(res); + await mutator(zip); + return zip.finalize(); + }; + + const targetedVisitorRecursive = async (seedId: string, hierarchy: Hierarchy): Promise<void> => { + const local: Hierarchy = {}; + const { title, data } = await getData(seedId); + const label = `${title} (${seedId})`; + if (Array.isArray(data)) { + hierarchy[label] = local; + await Promise.all(data.map(proxy => targetedVisitorRecursive(proxy.fieldId, local))); + } else { + hierarchy[label + path.extname(data)] = data; } - } else if (field.__type === "RichTextField") { - const re = /"href"\s*:\s*"(.*?)"/g; - let match: string[] | null; - while ((match = re.exec(field.Data)) !== null) { - const urlString = match[1]; - const split = new URL(urlString).pathname.split("doc/"); - if (split.length > 1) { - ids.push(split[split.length - 1]); + }; + + const getData = async (seedId: string): Promise<{ data: string | any[], title: string }> => { + return new Promise<{ data: string | any[], title: string }>((resolve, reject) => { + Database.Instance.getDocument(seedId, async (result: any) => { + const { data, proto, title } = result.fields; + if (data) { + if (data.url) { + resolve({ data: data.url, title }); + } else if (data.fields) { + resolve({ data: data.fields, title }); + } else { + reject(); + } + } + if (proto) { + getData(proto.fieldId).then(resolve, reject); + } + }); + }); + }; + + const hierarchyTraverserRecursive = async (file: Archiver.Archiver, hierarchy: Hierarchy, prefix = "Dash Export"): Promise<void> => { + for (const key of Object.keys(hierarchy)) { + const result = hierarchy[key]; + if (typeof result === "string") { + let path: string; + let matches: RegExpExecArray | null; + if ((matches = /\:1050\/files\/(upload\_[\da-z]{32}.*)/g.exec(result)) !== null) { + path = `${__dirname}/public/files/${matches[1]}`; + } else { + const information = await DashUploadUtils.UploadImage(result); + path = information.mediaPaths[0]; + } + file.file(path, { name: key, prefix }); + } else { + await hierarchyTraverserRecursive(file, result, `${prefix}/${key}`); } } - const re2 = /"src"\s*:\s*"(.*?)"/g; - while ((match = re2.exec(field.Data)) !== null) { - const urlString = match[1]; - const pathname = new URL(urlString).pathname; - files.add(pathname); + }; + + DashServer.get("/downloadId/:docId", async (req, res) => { + res.set('Content-disposition', `attachment;`); + res.set('Content-Type', "application/zip"); + const { id, docs, files } = await getDocs(req.params.docId); + const docString = JSON.stringify({ id, docs }); + const zip = Archiver('zip'); + zip.pipe(res); + zip.append(docString, { name: "doc.json" }); + files.forEach(val => { + zip.file(__dirname + RouteStore.public + val, { name: val.substring(1) }); + }); + zip.finalize(); + }); + + DashServer.post("/uploadDoc", (req, res) => { + let form = new formidable.IncomingForm(); + form.keepExtensions = true; + // let path = req.body.path; + const ids: { [id: string]: string } = {}; + let remap = true; + const getId = (id: string): string => { + if (!remap) return id; + if (id.endsWith("Proto")) return id; + if (id in ids) { + return ids[id]; + } else { + return ids[id] = v4(); + } + }; + const mapFn = (doc: any) => { + if (doc.id) { + doc.id = getId(doc.id); + } + for (const key in doc.fields) { + if (!doc.fields.hasOwnProperty(key)) { + continue; + } + const field = doc.fields[key]; + if (field === undefined || field === null) { + continue; + } + + if (field.__type === "proxy" || field.__type === "prefetch_proxy") { + field.fieldId = getId(field.fieldId); + } else if (field.__type === "script" || field.__type === "computed") { + if (field.captures) { + field.captures.fieldId = getId(field.captures.fieldId); + } + } else if (field.__type === "list") { + mapFn(field); + } else if (typeof field === "string") { + const re = /("(?:dataD|d)ocumentId"\s*:\s*")([\w\-]*)"/g; + doc.fields[key] = (field as any).replace(re, (match: any, p1: string, p2: string) => { + return `${p1}${getId(p2)}"`; + }); + } else if (field.__type === "RichTextField") { + const re = /("href"\s*:\s*")(.*?)"/g; + field.Data = field.Data.replace(re, (match: any, p1: string, p2: string) => { + return `${p1}${getId(p2)}"`; + }); + } + } + }; + form.parse(req, async (err, fields, files) => { + remap = fields.remap !== "false"; + let id: string = ""; + try { + for (const name in files) { + const path_2 = files[name].path; + const zip = new AdmZip(path_2); + zip.getEntries().forEach((entry: any) => { + if (!entry.entryName.startsWith("files/")) return; + let dirname = path.dirname(entry.entryName) + "/"; + let extname = path.extname(entry.entryName); + let basename = path.basename(entry.entryName).split(".")[0]; + // zip.extractEntryTo(dirname + basename + "_o" + extname, __dirname + RouteStore.public, true, false); + // zip.extractEntryTo(dirname + basename + "_s" + extname, __dirname + RouteStore.public, true, false); + // zip.extractEntryTo(dirname + basename + "_m" + extname, __dirname + RouteStore.public, true, false); + // zip.extractEntryTo(dirname + basename + "_l" + extname, __dirname + RouteStore.public, true, false); + try { + zip.extractEntryTo(entry.entryName, __dirname + RouteStore.public, true, false); + dirname = "/" + dirname; + + fs.createReadStream(__dirname + RouteStore.public + dirname + basename + extname).pipe(fs.createWriteStream(__dirname + RouteStore.public + dirname + basename + "_o" + extname)); + fs.createReadStream(__dirname + RouteStore.public + dirname + basename + extname).pipe(fs.createWriteStream(__dirname + RouteStore.public + dirname + basename + "_s" + extname)); + fs.createReadStream(__dirname + RouteStore.public + dirname + basename + extname).pipe(fs.createWriteStream(__dirname + RouteStore.public + dirname + basename + "_m" + extname)); + fs.createReadStream(__dirname + RouteStore.public + dirname + basename + extname).pipe(fs.createWriteStream(__dirname + RouteStore.public + dirname + basename + "_l" + extname)); + } catch (e) { + console.log(e); + } + }); + const json = zip.getEntry("doc.json"); + let docs: any; + try { + let data = JSON.parse(json.getData().toString("utf8")); + docs = data.docs; + id = data.id; + docs = Object.keys(docs).map(key => docs[key]); + docs.forEach(mapFn); + await Promise.all(docs.map((doc: any) => new Promise(res => Database.Instance.replace(doc.id, doc, (err, r) => { + err && console.log(err); + res(); + }, true, "newDocuments")))); + } catch (e) { console.log(e); } + fs.unlink(path_2, () => { }); + } + if (id) { + res.send(JSON.stringify(getId(id))); + } else { + res.send(JSON.stringify("error")); + } + } catch (e) { console.log(e); } + }); + }); + + DashServer.get("/whosOnline", (req, res) => { + let users: any = { active: {}, inactive: {} }; + const now = Date.now(); + + for (const user in timeMap) { + const time = timeMap[user]; + const key = ((now - time) / 1000) < (60 * 5) ? "active" : "inactive"; + users[key][user] = `Last active ${msToTime(now - time)} ago`; } - } else if (["audio", "image", "video", "pdf", "web"].includes(field.__type)) { - const url = new URL(field.url); - const pathname = url.pathname; - files.add(pathname); + + res.send(users); + }); + DashServer.get("/thumbnail/:filename", (req, res) => { + let filename = req.params.filename; + let noExt = filename.substring(0, filename.length - ".png".length); + let pagenumber = parseInt(noExt.split('-')[1]); + fs.exists(uploadDirectory + filename, (exists: boolean) => { + console.log(`${uploadDirectory + filename} ${exists ? "exists" : "does not exist"}`); + if (exists) { + let input = fs.createReadStream(uploadDirectory + filename); + probe(input, (err: any, result: any) => { + if (err) { + console.log(err); + console.log(`error on ${filename}`); + return; + } + res.send({ path: "/files/" + filename, width: result.width, height: result.height }); + }); + } + else { + LoadPage(uploadDirectory + filename.substring(0, filename.length - noExt.split('-')[1].length - ".PNG".length - 1) + ".pdf", pagenumber, res); + } + }); + }); + + function LoadPage(file: string, pageNumber: number, res: Response) { + console.log(file); + Pdfjs.getDocument(file).promise + .then((pdf: Pdfjs.PDFDocumentProxy) => { + let factory = new NodeCanvasFactory(); + console.log(pageNumber); + pdf.getPage(pageNumber).then((page: Pdfjs.PDFPageProxy) => { + console.log("reading " + page); + let viewport = page.getViewport(1 as any); + let canvasAndContext = factory.create(viewport.width, viewport.height); + let renderContext = { + canvasContext: canvasAndContext.context, + viewport: viewport, + canvasFactory: factory + }; + console.log("read " + pageNumber); + + page.render(renderContext).promise + .then(() => { + console.log("saving " + pageNumber); + let stream = canvasAndContext.canvas.createPNGStream(); + let pngFile = `${file.substring(0, file.length - ".pdf".length)}-${pageNumber}.PNG`; + let out = fs.createWriteStream(pngFile); + stream.pipe(out); + out.on("finish", () => { + console.log(`Success! Saved to ${pngFile}`); + let name = path.basename(pngFile); + res.send({ path: "/files/" + name, width: viewport.width, height: viewport.height }); + }); + }, (reason: string) => { + console.error(reason + ` ${pageNumber}`); + }); + }); + }); } - } - if (doc.id) { - docs[doc.id] = doc; - } - return ids; - }; - await Database.Instance.visit([id], fn); - return { id, docs, files }; -} -app.get("/serializeDoc/:docId", async (req, res) => { - const { docs, files } = await getDocs(req.params.docId); - res.send({ docs, files: Array.from(files) }); -}); + /** + * Anyone attempting to navigate to localhost at this port will + * first have to log in. + */ + router.addSupervisedRoute({ + method: Method.GET, + subscription: RouteStore.root, + onValidation: (_user, _req, res) => res.redirect(RouteStore.home) + }); -export type Hierarchy = { [id: string]: string | Hierarchy }; -export type ZipMutator = (file: Archiver.Archiver) => void | Promise<void>; + router.addSupervisedRoute({ + method: Method.GET, + subscription: RouteStore.getUsers, + onValidation: async (_user, _req, res) => { + const cursor = await Database.Instance.query({}, { email: 1, userDocumentId: 1 }, "users"); + const results = await cursor.toArray(); + res.send(results.map(user => ({ email: user.email, userDocumentId: user.userDocumentId }))); + }, + }); -addSecureRoute({ - method: Method.GET, - subscribers: new RouteSubscriber(RouteStore.imageHierarchyExport).add('docId'), - onValidation: async (_user, req, res) => { - const id = req.params.docId; - const hierarchy: Hierarchy = {}; - await targetedVisitorRecursive(id, hierarchy); - BuildAndDispatchZip(res, async zip => { - await hierarchyTraverserRecursive(zip, hierarchy); - }); - } -}); - -const BuildAndDispatchZip = async (res: Response, mutator: ZipMutator): Promise<void> => { - const zip = Archiver('zip'); - zip.pipe(res); - await mutator(zip); - return zip.finalize(); -}; - -const targetedVisitorRecursive = async (seedId: string, hierarchy: Hierarchy): Promise<void> => { - const local: Hierarchy = {}; - const { title, data } = await getData(seedId); - const label = `${title} (${seedId})`; - if (Array.isArray(data)) { - hierarchy[label] = local; - await Promise.all(data.map(proxy => targetedVisitorRecursive(proxy.fieldId, local))); - } else { - hierarchy[label + path.extname(data)] = data; - } -}; - -const getData = async (seedId: string): Promise<{ data: string | any[], title: string }> => { - return new Promise<{ data: string | any[], title: string }>((resolve, reject) => { - Database.Instance.getDocument(seedId, async (result: any) => { - const { data, proto, title } = result.fields; - if (data) { - if (data.url) { - resolve({ data: data.url, title }); - } else if (data.fields) { - resolve({ data: data.fields, title }); - } else { - reject(); + router.addSupervisedRoute({ + method: Method.GET, + subscription: [RouteStore.home, RouteStore.openDocumentWithId], + onValidation: (_user, req, res) => { + let detector = new mobileDetect(req.headers['user-agent'] || ""); + let filename = detector.mobile() !== null ? 'mobile/image.html' : 'index.html'; + res.sendFile(path.join(__dirname, '../../deploy/' + filename)); + }, + }); + + router.addSupervisedRoute({ + method: Method.GET, + subscription: RouteStore.getUserDocumentId, + onValidation: (user, _req, res) => res.send(user.userDocumentId), + onRejection: (_req, res) => res.send(undefined) + }); + + router.addSupervisedRoute({ + method: Method.GET, + subscription: RouteStore.getCurrUser, + onValidation: (user, _req, res) => { res.send(JSON.stringify(user)); }, + onRejection: (_req, res) => res.send(JSON.stringify({ id: "__guest__", email: "" })) + }); + + const ServicesApiKeyMap = new Map<string, string | undefined>([ + ["face", process.env.FACE], + ["vision", process.env.VISION], + ["handwriting", process.env.HANDWRITING] + ]); + + router.addSupervisedRoute({ + method: Method.GET, + subscription: new RouteSubscriber(RouteStore.cognitiveServices).add('requestedservice'), + onValidation: (_user, req, res) => { + let service = req.params.requestedservice; + res.send(ServicesApiKeyMap.get(service)); + } + }); + + class NodeCanvasFactory { + create = (width: number, height: number) => { + var canvas = createCanvas(width, height); + var context = canvas.getContext('2d'); + return { + canvas: canvas, + context: context, + }; + } + + reset = (canvasAndContext: any, width: number, height: number) => { + canvasAndContext.canvas.width = width; + canvasAndContext.canvas.height = height; + } + + destroy = (canvasAndContext: any) => { + canvasAndContext.canvas.width = 0; + canvasAndContext.canvas.height = 0; + canvasAndContext.canvas = null; + canvasAndContext.context = null; } } - if (proto) { - getData(proto.fieldId).then(resolve, reject); - } - }); - }); -}; - -const hierarchyTraverserRecursive = async (file: Archiver.Archiver, hierarchy: Hierarchy, prefix = "Dash Export"): Promise<void> => { - for (const key of Object.keys(hierarchy)) { - const result = hierarchy[key]; - if (typeof result === "string") { - let path: string; - let matches: RegExpExecArray | null; - if ((matches = /\:1050\/files\/(upload\_[\da-z]{32}.*)/g.exec(result)) !== null) { - path = `${__dirname}/public/files/${matches[1]}`; - } else { - const information = await DashUploadUtils.UploadImage(result); - path = information.mediaPaths[0]; - } - file.file(path, { name: key, prefix }); - } else { - await hierarchyTraverserRecursive(file, result, `${prefix}/${key}`); - } - } -}; - -app.get("/downloadId/:docId", async (req, res) => { - res.set('Content-disposition', `attachment;`); - res.set('Content-Type', "application/zip"); - const { id, docs, files } = await getDocs(req.params.docId); - const docString = JSON.stringify({ id, docs }); - const zip = Archiver('zip'); - zip.pipe(res); - zip.append(docString, { name: "doc.json" }); - files.forEach(val => { - zip.file(__dirname + RouteStore.public + val, { name: val.substring(1) }); - }); - zip.finalize(); -}); - -app.post("/uploadDoc", (req, res) => { - let form = new formidable.IncomingForm(); - form.keepExtensions = true; - // let path = req.body.path; - const ids: { [id: string]: string } = {}; - let remap = true; - const getId = (id: string): string => { - if (!remap) return id; - if (id.endsWith("Proto")) return id; - if (id in ids) { - return ids[id]; - } else { - return ids[id] = v4(); - } - }; - const mapFn = (doc: any) => { - if (doc.id) { - doc.id = getId(doc.id); - } - for (const key in doc.fields) { - if (!doc.fields.hasOwnProperty(key)) { - continue; - } - const field = doc.fields[key]; - if (field === undefined || field === null) { - continue; + + const pngTypes = [".png", ".PNG"]; + const jpgTypes = [".jpg", ".JPG", ".jpeg", ".JPEG"]; + const uploadDirectory = __dirname + "/public/files/"; + const pdfDirectory = uploadDirectory + "text"; + DashUploadUtils.createIfNotExists(pdfDirectory); + + interface ImageFileResponse { + name: string; + path: string; + type: string; + exif: Opt<DashUploadUtils.EnrichedExifData>; } - if (field.__type === "proxy" || field.__type === "prefetch_proxy") { - field.fieldId = getId(field.fieldId); - } else if (field.__type === "script" || field.__type === "computed") { - if (field.captures) { - field.captures.fieldId = getId(field.captures.fieldId); + router.addSupervisedRoute({ + method: Method.POST, + subscription: RouteStore.upload, + onValidation: (_user, req, res) => { + let form = new formidable.IncomingForm(); + form.uploadDir = uploadDirectory; + form.keepExtensions = true; + form.parse(req, async (_err, _fields, files) => { + let results: ImageFileResponse[] = []; + for (const key in files) { + const { type, path: location, name } = files[key]; + const filename = path.basename(location); + let uploadInformation: Opt<DashUploadUtils.UploadInformation>; + if (filename.endsWith(".pdf")) { + let dataBuffer = fs.readFileSync(uploadDirectory + filename); + const result: ParsedPDF = await pdf(dataBuffer); + await new Promise<void>(resolve => { + const path = pdfDirectory + "/" + filename.substring(0, filename.length - ".pdf".length) + ".txt"; + fs.createWriteStream(path).write(result.text, error => { + if (!error) { + resolve(); + } else { + reject(error); + } + }); + }); + } else { + uploadInformation = await DashUploadUtils.UploadImage(uploadDirectory + filename, filename); + } + const exif = uploadInformation ? uploadInformation.exifData : undefined; + results.push({ name, type, path: `/files/${filename}`, exif }); + + } + _success(res, results); + }); } - } else if (field.__type === "list") { - mapFn(field); - } else if (typeof field === "string") { - const re = /("(?:dataD|d)ocumentId"\s*:\s*")([\w\-]*)"/g; - doc.fields[key] = (field as any).replace(re, (match: any, p1: string, p2: string) => { - return `${p1}${getId(p2)}"`; - }); - } else if (field.__type === "RichTextField") { - const re = /("href"\s*:\s*")(.*?)"/g; - field.Data = field.Data.replace(re, (match: any, p1: string, p2: string) => { - return `${p1}${getId(p2)}"`; - }); - } - } - }; - form.parse(req, async (err, fields, files) => { - remap = fields.remap !== "false"; - let id: string = ""; - try { - for (const name in files) { - const path_2 = files[name].path; - const zip = new AdmZip(path_2); - zip.getEntries().forEach((entry: any) => { - if (!entry.entryName.startsWith("files/")) return; - let dirname = path.dirname(entry.entryName) + "/"; - let extname = path.extname(entry.entryName); - let basename = path.basename(entry.entryName).split(".")[0]; - // zip.extractEntryTo(dirname + basename + "_o" + extname, __dirname + RouteStore.public, true, false); - // zip.extractEntryTo(dirname + basename + "_s" + extname, __dirname + RouteStore.public, true, false); - // zip.extractEntryTo(dirname + basename + "_m" + extname, __dirname + RouteStore.public, true, false); - // zip.extractEntryTo(dirname + basename + "_l" + extname, __dirname + RouteStore.public, true, false); - try { - zip.extractEntryTo(entry.entryName, __dirname + RouteStore.public, true, false); - dirname = "/" + dirname; - - fs.createReadStream(__dirname + RouteStore.public + dirname + basename + extname).pipe(fs.createWriteStream(__dirname + RouteStore.public + dirname + basename + "_o" + extname)); - fs.createReadStream(__dirname + RouteStore.public + dirname + basename + extname).pipe(fs.createWriteStream(__dirname + RouteStore.public + dirname + basename + "_s" + extname)); - fs.createReadStream(__dirname + RouteStore.public + dirname + basename + extname).pipe(fs.createWriteStream(__dirname + RouteStore.public + dirname + basename + "_m" + extname)); - fs.createReadStream(__dirname + RouteStore.public + dirname + basename + extname).pipe(fs.createWriteStream(__dirname + RouteStore.public + dirname + basename + "_l" + extname)); - } catch (e) { - console.log(e); + }); + + router.addSupervisedRoute({ + method: Method.POST, + subscription: RouteStore.inspectImage, + onValidation: async (_user, req, res) => { + const { source } = req.body; + if (typeof source === "string") { + const uploadInformation = await DashUploadUtils.UploadImage(source); + return res.send(await DashUploadUtils.InspectImage(uploadInformation.mediaPaths[0])); } - }); - const json = zip.getEntry("doc.json"); - let docs: any; - try { - let data = JSON.parse(json.getData().toString("utf8")); - docs = data.docs; - id = data.id; - docs = Object.keys(docs).map(key => docs[key]); - docs.forEach(mapFn); - await Promise.all(docs.map((doc: any) => new Promise(res => Database.Instance.replace(doc.id, doc, (err, r) => { - err && console.log(err); - res(); - }, true, "newDocuments")))); - } catch (e) { console.log(e); } - fs.unlink(path_2, () => { }); - } - if (id) { - res.send(JSON.stringify(getId(id))); - } else { - res.send(JSON.stringify("error")); - } - } catch (e) { console.log(e); } - }); -}); - -app.get("/whosOnline", (req, res) => { - let users: any = { active: {}, inactive: {} }; - const now = Date.now(); - - for (const user in timeMap) { - const time = timeMap[user]; - const key = ((now - time) / 1000) < (60 * 5) ? "active" : "inactive"; - users[key][user] = `Last active ${msToTime(now - time)} ago`; - } - - res.send(users); -}); -app.get("/thumbnail/:filename", (req, res) => { - let filename = req.params.filename; - let noExt = filename.substring(0, filename.length - ".png".length); - let pagenumber = parseInt(noExt.split('-')[1]); - fs.exists(uploadDirectory + filename, (exists: boolean) => { - console.log(`${uploadDirectory + filename} ${exists ? "exists" : "does not exist"}`); - if (exists) { - let input = fs.createReadStream(uploadDirectory + filename); - probe(input, (err: any, result: any) => { - if (err) { - console.log(err); - console.log(`error on ${filename}`); - return; + res.send({}); } - res.send({ path: "/files/" + filename, width: result.width, height: result.height }); }); - } - else { - LoadPage(uploadDirectory + filename.substring(0, filename.length - noExt.split('-')[1].length - ".PNG".length - 1) + ".pdf", pagenumber, res); - } - }); -}); - -function LoadPage(file: string, pageNumber: number, res: Response) { - console.log(file); - Pdfjs.getDocument(file).promise - .then((pdf: Pdfjs.PDFDocumentProxy) => { - let factory = new NodeCanvasFactory(); - console.log(pageNumber); - pdf.getPage(pageNumber).then((page: Pdfjs.PDFPageProxy) => { - console.log("reading " + page); - let viewport = page.getViewport(1 as any); - let canvasAndContext = factory.create(viewport.width, viewport.height); - let renderContext = { - canvasContext: canvasAndContext.context, - viewport: viewport, - canvasFactory: factory - }; - console.log("read " + pageNumber); - - page.render(renderContext).promise - .then(() => { - console.log("saving " + pageNumber); - let stream = canvasAndContext.canvas.createPNGStream(); - let pngFile = `${file.substring(0, file.length - ".pdf".length)}-${pageNumber}.PNG`; - let out = fs.createWriteStream(pngFile); - stream.pipe(out); - out.on("finish", () => { - console.log(`Success! Saved to ${pngFile}`); - let name = path.basename(pngFile); - res.send({ path: "/files/" + name, width: viewport.width, height: viewport.height }); - }); - }, (reason: string) => { - console.error(reason + ` ${pageNumber}`); + + router.addSupervisedRoute({ + method: Method.POST, + subscription: RouteStore.dataUriToImage, + onValidation: (_user, req, res) => { + const uri = req.body.uri; + const filename = req.body.name; + if (!uri || !filename) { + res.status(401).send("incorrect parameters specified"); + return; + } + imageDataUri.outputFile(uri, uploadDirectory + filename).then((savedName: string) => { + const ext = path.extname(savedName); + let resizers = [ + { resizer: sharp().resize(100, undefined, { withoutEnlargement: true }), suffix: "_s" }, + { resizer: sharp().resize(400, undefined, { withoutEnlargement: true }), suffix: "_m" }, + { resizer: sharp().resize(900, undefined, { withoutEnlargement: true }), suffix: "_l" }, + ]; + let isImage = false; + if (pngTypes.includes(ext)) { + resizers.forEach(element => { + element.resizer = element.resizer.png(); + }); + isImage = true; + } else if (jpgTypes.includes(ext)) { + resizers.forEach(element => { + element.resizer = element.resizer.jpeg(); + }); + isImage = true; + } + if (isImage) { + resizers.forEach(resizer => { + fs.createReadStream(savedName).pipe(resizer.resizer).pipe(fs.createWriteStream(uploadDirectory + filename + resizer.suffix + ext)); + }); + } + res.send("/files/" + filename + ext); }); + } }); - }); -} -/** - * Anyone attempting to navigate to localhost at this port will - * first have to log in. - */ -addSecureRoute({ - method: Method.GET, - subscribers: RouteStore.root, - onValidation: (_user, _req, res) => res.redirect(RouteStore.home) -}); - -addSecureRoute({ - method: Method.GET, - subscribers: RouteStore.getUsers, - onValidation: async (_user, _req, res) => { - const cursor = await Database.Instance.query({}, { email: 1, userDocumentId: 1 }, "users"); - const results = await cursor.toArray(); - res.send(results.map(user => ({ email: user.email, userDocumentId: user.userDocumentId }))); - }, -}); - -addSecureRoute({ - method: Method.GET, - subscribers: [RouteStore.home, RouteStore.openDocumentWithId], - onValidation: (_user, req, res) => { - let detector = new mobileDetect(req.headers['user-agent'] || ""); - let filename = detector.mobile() !== null ? 'mobile/image.html' : 'index.html'; - res.sendFile(path.join(__dirname, '../../deploy/' + filename)); - }, -}); - -addSecureRoute({ - method: Method.GET, - subscribers: RouteStore.getUserDocumentId, - onValidation: (user, _req, res) => res.send(user.userDocumentId), - onRejection: (_req, res) => res.send(undefined) -}); - -addSecureRoute({ - method: Method.GET, - subscribers: RouteStore.getCurrUser, - onValidation: (user, _req, res) => { res.send(JSON.stringify(user)); }, - onRejection: (_req, res) => res.send(JSON.stringify({ id: "__guest__", email: "" })) -}); - -const ServicesApiKeyMap = new Map<string, string | undefined>([ - ["face", process.env.FACE], - ["vision", process.env.VISION], - ["handwriting", process.env.HANDWRITING] -]); - -addSecureRoute({ - method: Method.GET, - subscribers: new RouteSubscriber(RouteStore.cognitiveServices).add('requestedservice'), - onValidation: (_user, req, res) => { - let service = req.params.requestedservice; - res.send(ServicesApiKeyMap.get(service)); - } -}); - -class NodeCanvasFactory { - create = (width: number, height: number) => { - var canvas = createCanvas(width, height); - var context = canvas.getContext('2d'); - return { - canvas: canvas, - context: context, - }; - } - - reset = (canvasAndContext: any, width: number, height: number) => { - canvasAndContext.canvas.width = width; - canvasAndContext.canvas.height = height; - } - - destroy = (canvasAndContext: any) => { - canvasAndContext.canvas.width = 0; - canvasAndContext.canvas.height = 0; - canvasAndContext.canvas = null; - canvasAndContext.context = null; - } -} - -const pngTypes = [".png", ".PNG"]; -const jpgTypes = [".jpg", ".JPG", ".jpeg", ".JPEG"]; -const uploadDirectory = __dirname + "/public/files/"; -const pdfDirectory = uploadDirectory + "text"; -DashUploadUtils.createIfNotExists(pdfDirectory); - -interface ImageFileResponse { - name: string; - path: string; - type: string; - exif: Opt<DashUploadUtils.EnrichedExifData>; -} - -addSecureRoute({ - method: Method.POST, - subscribers: RouteStore.upload, - onValidation: (_user, req, res) => { - let form = new formidable.IncomingForm(); - form.uploadDir = uploadDirectory; - form.keepExtensions = true; - form.parse(req, async (_err, _fields, files) => { - let results: ImageFileResponse[] = []; - for (const key in files) { - const { type, path: location, name } = files[key]; - const filename = path.basename(location); - let uploadInformation: Opt<DashUploadUtils.UploadInformation>; - if (filename.endsWith(".pdf")) { - let dataBuffer = fs.readFileSync(uploadDirectory + filename); - const result: ParsedPDF = await pdf(dataBuffer); - await new Promise<void>(resolve => { - const path = pdfDirectory + "/" + filename.substring(0, filename.length - ".pdf".length) + ".txt"; - fs.createWriteStream(path).write(result.text, error => { - if (!error) { - resolve(); - } else { - reject(error); + // AUTHENTICATION + + // Sign Up + DashServer.get(RouteStore.signup, getSignup); + DashServer.post(RouteStore.signup, postSignup); + + // Log In + DashServer.get(RouteStore.login, getLogin); + DashServer.post(RouteStore.login, postLogin); + + // Log Out + DashServer.get(RouteStore.logout, getLogout); + + // FORGOT PASSWORD EMAIL HANDLING + DashServer.get(RouteStore.forgot, getForgot); + DashServer.post(RouteStore.forgot, postForgot); + + // RESET PASSWORD EMAIL HANDLING + DashServer.get(RouteStore.reset, getReset); + DashServer.post(RouteStore.reset, postReset); + + const headerCharRegex = /[^\t\x20-\x7e\x80-\xff]/; + DashServer.use(RouteStore.corsProxy, (req, res) => { + req.pipe(request(decodeURIComponent(req.url.substring(1)))).on("response", res => { + const headers = Object.keys(res.headers); + headers.forEach(headerName => { + const header = res.headers[headerName]; + if (Array.isArray(header)) { + res.headers[headerName] = header.filter(h => !headerCharRegex.test(h)); + } else if (header) { + if (headerCharRegex.test(header as any)) { + delete res.headers[headerName]; } - }); + } }); - } else { - uploadInformation = await DashUploadUtils.UploadImage(uploadDirectory + filename, filename); + }).pipe(res); + }); + + router.addSupervisedRoute({ + method: Method.GET, + subscription: RouteStore.delete, + onValidation: (_user, _req, res, isRelease) => { + if (isRelease) { + return _permission_denied(res, deletionPermissionError); + } + deleteFields().then(() => res.redirect(RouteStore.home)); } - const exif = uploadInformation ? uploadInformation.exifData : undefined; - results.push({ name, type, path: `/files/${filename}`, exif }); + }); + router.addSupervisedRoute({ + method: Method.GET, + subscription: RouteStore.deleteAll, + onValidation: (_user, _req, res, isRelease) => { + if (isRelease) { + return _permission_denied(res, deletionPermissionError); + } + deleteAll().then(() => res.redirect(RouteStore.home)); + } + }); + + const server = io(); + interface Map { + [key: string]: Client; } - _success(res, results); - }); - } -}); - -addSecureRoute({ - method: Method.POST, - subscribers: RouteStore.inspectImage, - onValidation: async (_user, req, res) => { - const { source } = req.body; - if (typeof source === "string") { - const uploadInformation = await DashUploadUtils.UploadImage(source); - return res.send(await DashUploadUtils.InspectImage(uploadInformation.mediaPaths[0])); - } - res.send({}); - } -}); - -addSecureRoute({ - method: Method.POST, - subscribers: RouteStore.dataUriToImage, - onValidation: (_user, req, res) => { - const uri = req.body.uri; - const filename = req.body.name; - if (!uri || !filename) { - res.status(401).send("incorrect parameters specified"); - return; - } - imageDataUri.outputFile(uri, uploadDirectory + filename).then((savedName: string) => { - const ext = path.extname(savedName); - let resizers = [ - { resizer: sharp().resize(100, undefined, { withoutEnlargement: true }), suffix: "_s" }, - { resizer: sharp().resize(400, undefined, { withoutEnlargement: true }), suffix: "_m" }, - { resizer: sharp().resize(900, undefined, { withoutEnlargement: true }), suffix: "_l" }, - ]; - let isImage = false; - if (pngTypes.includes(ext)) { - resizers.forEach(element => { - element.resizer = element.resizer.png(); - }); - isImage = true; - } else if (jpgTypes.includes(ext)) { - resizers.forEach(element => { - element.resizer = element.resizer.jpeg(); + let clients: Map = {}; + + let socketMap = new Map<SocketIO.Socket, string>(); + let timeMap: { [id: string]: number } = {}; + + server.on("connection", function (socket: Socket) { + socket.use((packet, next) => { + let id = socketMap.get(socket); + if (id) { + timeMap[id] = Date.now(); + } + next(); }); - isImage = true; + + Utils.Emit(socket, MessageStore.Foo, "handshooken"); + + Utils.AddServerHandler(socket, MessageStore.Bar, guid => barReceived(socket, guid)); + Utils.AddServerHandler(socket, MessageStore.SetField, (args) => setField(socket, args)); + Utils.AddServerHandlerCallback(socket, MessageStore.GetField, getField); + Utils.AddServerHandlerCallback(socket, MessageStore.GetFields, getFields); + if (!router.release) { + Utils.AddServerHandler(socket, MessageStore.DeleteAll, deleteFields); + } + + Utils.AddServerHandler(socket, MessageStore.CreateField, CreateField); + Utils.AddServerHandlerCallback(socket, MessageStore.YoutubeApiQuery, HandleYoutubeQuery); + Utils.AddServerHandler(socket, MessageStore.UpdateField, diff => UpdateField(socket, diff)); + Utils.AddServerHandler(socket, MessageStore.DeleteField, id => DeleteField(socket, id)); + Utils.AddServerHandler(socket, MessageStore.DeleteFields, ids => DeleteFields(socket, ids)); + Utils.AddServerHandlerCallback(socket, MessageStore.GetRefField, GetRefField); + Utils.AddServerHandlerCallback(socket, MessageStore.GetRefFields, GetRefFields); + }); + + async function deleteFields() { + await Database.Instance.deleteAll(); + await Search.Instance.clear(); + await Database.Instance.deleteAll('newDocuments'); } - if (isImage) { - resizers.forEach(resizer => { - fs.createReadStream(savedName).pipe(resizer.resizer).pipe(fs.createWriteStream(uploadDirectory + filename + resizer.suffix + ext)); - }); + + async function deleteAll() { + await Database.Instance.deleteAll(); + await Database.Instance.deleteAll('newDocuments'); + await Database.Instance.deleteAll('sessions'); + await Database.Instance.deleteAll('users'); + await Search.Instance.clear(); } - res.send("/files/" + filename + ext); - }); - } -}); - -// AUTHENTICATION - -// Sign Up -app.get(RouteStore.signup, getSignup); -app.post(RouteStore.signup, postSignup); - -// Log In -app.get(RouteStore.login, getLogin); -app.post(RouteStore.login, postLogin); - -// Log Out -app.get(RouteStore.logout, getLogout); - -// FORGOT PASSWORD EMAIL HANDLING -app.get(RouteStore.forgot, getForgot); -app.post(RouteStore.forgot, postForgot); - -// RESET PASSWORD EMAIL HANDLING -app.get(RouteStore.reset, getReset); -app.post(RouteStore.reset, postReset); - -const headerCharRegex = /[^\t\x20-\x7e\x80-\xff]/; -app.use(RouteStore.corsProxy, (req, res) => { - req.pipe(request(decodeURIComponent(req.url.substring(1)))).on("response", res => { - const headers = Object.keys(res.headers); - headers.forEach(headerName => { - const header = res.headers[headerName]; - if (Array.isArray(header)) { - res.headers[headerName] = header.filter(h => !headerCharRegex.test(h)); - } else if (header) { - if (headerCharRegex.test(header as any)) { - delete res.headers[headerName]; - } + + function barReceived(socket: SocketIO.Socket, guid: string) { + clients[guid] = new Client(guid.toString()); + console.log(`User ${guid} has connected`); + socketMap.set(socket, guid); } - }); - }).pipe(res); -}); - -addSecureRoute({ - method: Method.GET, - subscribers: RouteStore.delete, - onValidation: (_user, _req, res) => { - if (release) { - return _permission_denied(res, deletionPermissionError); - } - deleteFields().then(() => res.redirect(RouteStore.home)); - } -}); - -addSecureRoute({ - method: Method.GET, - subscribers: RouteStore.deleteAll, - onValidation: (_user, _req, res) => { - if (release) { - return _permission_denied(res, deletionPermissionError); - } - deleteAll().then(() => res.redirect(RouteStore.home)); - } -}); -app.use(wdm(compiler, { publicPath: config.output.publicPath })); + function getField([id, callback]: [string, (result?: Transferable) => void]) { + Database.Instance.getDocument(id, (result?: Transferable) => + callback(result ? result : undefined)); + } -app.use(whm(compiler)); + function getFields([ids, callback]: [string[], (result: Transferable[]) => void]) { + Database.Instance.getDocuments(ids, callback); + } -// start the Express server -app.listen(port, () => - console.log(`server started at http://localhost:${port}`)); + function setField(socket: Socket, newValue: Transferable) { + Database.Instance.update(newValue.id, newValue, () => + socket.broadcast.emit(MessageStore.SetField.Message, newValue)); + if (newValue.type === Types.Text) { + Search.Instance.updateDocument({ id: newValue.id, data: (newValue as any).data }); + console.log("set field"); + console.log("checking in"); + } + } -const server = io(); -interface Map { - [key: string]: Client; -} -let clients: Map = {}; + function GetRefField([id, callback]: [string, (result?: Transferable) => void]) { + Database.Instance.getDocument(id, callback, "newDocuments"); + } -let socketMap = new Map<SocketIO.Socket, string>(); -let timeMap: { [id: string]: number } = {}; + function GetRefFields([ids, callback]: [string[], (result?: Transferable[]) => void]) { + Database.Instance.getDocuments(ids, callback, "newDocuments"); + } -server.on("connection", function (socket: Socket) { - socket.use((packet, next) => { - let id = socketMap.get(socket); - if (id) { - timeMap[id] = Date.now(); - } - next(); - }); + function HandleYoutubeQuery([query, callback]: [YoutubeQueryInput, (result?: any[]) => void]) { + switch (query.type) { + case YoutubeQueryType.Channels: + YoutubeApi.authorizedGetChannel(youtubeApiKey); + break; + case YoutubeQueryType.SearchVideo: + YoutubeApi.authorizedGetVideos(youtubeApiKey, query.userInput, callback); + case YoutubeQueryType.VideoDetails: + YoutubeApi.authorizedGetVideoDetails(youtubeApiKey, query.videoIds, callback); + } + } - Utils.Emit(socket, MessageStore.Foo, "handshooken"); - - Utils.AddServerHandler(socket, MessageStore.Bar, guid => barReceived(socket, guid)); - Utils.AddServerHandler(socket, MessageStore.SetField, (args) => setField(socket, args)); - Utils.AddServerHandlerCallback(socket, MessageStore.GetField, getField); - Utils.AddServerHandlerCallback(socket, MessageStore.GetFields, getFields); - if (!release) { - Utils.AddServerHandler(socket, MessageStore.DeleteAll, deleteFields); - } - - Utils.AddServerHandler(socket, MessageStore.CreateField, CreateField); - Utils.AddServerHandlerCallback(socket, MessageStore.YoutubeApiQuery, HandleYoutubeQuery); - Utils.AddServerHandler(socket, MessageStore.UpdateField, diff => UpdateField(socket, diff)); - Utils.AddServerHandler(socket, MessageStore.DeleteField, id => DeleteField(socket, id)); - Utils.AddServerHandler(socket, MessageStore.DeleteFields, ids => DeleteFields(socket, ids)); - Utils.AddServerHandlerCallback(socket, MessageStore.GetRefField, GetRefField); - Utils.AddServerHandlerCallback(socket, MessageStore.GetRefFields, GetRefFields); -}); - -async function deleteFields() { - await Database.Instance.deleteAll(); - await Search.Instance.clear(); - await Database.Instance.deleteAll('newDocuments'); -} + const credentialsPath = path.join(__dirname, "./credentials/google_docs_credentials.json"); + + const EndpointHandlerMap = new Map<GoogleApiServerUtils.Action, GoogleApiServerUtils.ApiRouter>([ + ["create", (api, params) => api.create(params)], + ["retrieve", (api, params) => api.get(params)], + ["update", (api, params) => api.batchUpdate(params)], + ]); + + DashServer.post(RouteStore.googleDocs + "/:sector/:action", (req, res) => { + let sector: GoogleApiServerUtils.Service = req.params.sector as GoogleApiServerUtils.Service; + let action: GoogleApiServerUtils.Action = req.params.action as GoogleApiServerUtils.Action; + GoogleApiServerUtils.GetEndpoint(GoogleApiServerUtils.Service[sector], { credentialsPath, userId: req.headers.userId as string }).then(endpoint => { + let handler = EndpointHandlerMap.get(action); + if (endpoint && handler) { + let execute = handler(endpoint, req.body).then( + response => res.send(response.data), + rejection => res.send(rejection) + ); + execute.catch(exception => res.send(exception)); + return; + } + res.send(undefined); + }); + }); -async function deleteAll() { - await Database.Instance.deleteAll(); - await Database.Instance.deleteAll('newDocuments'); - await Database.Instance.deleteAll('sessions'); - await Database.Instance.deleteAll('users'); - await Search.Instance.clear(); -} + router.addSupervisedRoute({ + method: Method.GET, + subscription: RouteStore.readGoogleAccessToken, + onValidation: async (user, _req, res) => { + const userId = user.id; + const token = await Database.Auxiliary.GoogleAuthenticationToken.Fetch(userId); + const information = { credentialsPath, userId }; + if (!token) { + return res.send(await GoogleApiServerUtils.GenerateAuthenticationUrl(information)); + } + GoogleApiServerUtils.RetrieveAccessToken(information).then(token => res.send(token)); + } + }); -function barReceived(socket: SocketIO.Socket, guid: string) { - clients[guid] = new Client(guid.toString()); - console.log(`User ${guid} has connected`); - socketMap.set(socket, guid); -} + router.addSupervisedRoute({ + method: Method.POST, + subscription: RouteStore.writeGoogleAccessToken, + onValidation: async (user, req, res) => { + const userId = user.id; + const information = { credentialsPath, userId }; + res.send(await GoogleApiServerUtils.ProcessClientSideCode(information, req.body.authenticationCode)); + } + }); -function getField([id, callback]: [string, (result?: Transferable) => void]) { - Database.Instance.getDocument(id, (result?: Transferable) => - callback(result ? result : undefined)); -} + const tokenError = "Unable to successfully upload bytes for all images!"; + const mediaError = "Unable to convert all uploaded bytes to media items!"; + const userIdError = "Unable to parse the identification of the user!"; + + router.addSupervisedRoute({ + method: Method.POST, + subscription: RouteStore.googlePhotosMediaUpload, + onValidation: async (user, req, res) => { + const { media } = req.body; + const userId = user.id; + if (!userId) { + return _error(res, userIdError); + } -function getFields([ids, callback]: [string[], (result: Transferable[]) => void]) { - Database.Instance.getDocuments(ids, callback); -} + await GooglePhotosUploadUtils.initialize({ credentialsPath, userId }); + + let failed: number[] = []; + + const newMediaItems = await BatchedArray.from<GooglePhotosUploadUtils.MediaInput>(media, { batchSize: 25 }).batchedMapPatientInterval( + { magnitude: 100, unit: TimeUnit.Milliseconds }, + async (batch: GooglePhotosUploadUtils.MediaInput[]) => { + const newMediaItems: NewMediaItem[] = []; + for (let index = 0; index < batch.length; index++) { + const element = batch[index]; + const uploadToken = await GooglePhotosUploadUtils.DispatchGooglePhotosUpload(element.url); + if (!uploadToken) { + failed.push(index); + } else { + newMediaItems.push({ + description: element.description, + simpleMediaItem: { uploadToken } + }); + } + } + return newMediaItems; + } + ); -function setField(socket: Socket, newValue: Transferable) { - Database.Instance.update(newValue.id, newValue, () => - socket.broadcast.emit(MessageStore.SetField.Message, newValue)); - if (newValue.type === Types.Text) { - Search.Instance.updateDocument({ id: newValue.id, data: (newValue as any).data }); - console.log("set field"); - console.log("checking in"); - } -} + const failedCount = failed.length; + if (failedCount) { + console.error(`Unable to upload ${failedCount} image${failedCount === 1 ? "" : "s"} to Google's servers`); + } -function GetRefField([id, callback]: [string, (result?: Transferable) => void]) { - Database.Instance.getDocument(id, callback, "newDocuments"); -} + GooglePhotosUploadUtils.CreateMediaItems(newMediaItems, req.body.album).then( + result => _success(res, { results: result.newMediaItemResults, failed }), + error => _error(res, mediaError, error) + ); + } + }); -function GetRefFields([ids, callback]: [string[], (result?: Transferable[]) => void]) { - Database.Instance.getDocuments(ids, callback, "newDocuments"); -} + interface MediaItem { + baseUrl: string; + filename: string; + } + const prefix = "google_photos_"; -function HandleYoutubeQuery([query, callback]: [YoutubeQueryInput, (result?: any[]) => void]) { - switch (query.type) { - case YoutubeQueryType.Channels: - YoutubeApi.authorizedGetChannel(youtubeApiKey); - break; - case YoutubeQueryType.SearchVideo: - YoutubeApi.authorizedGetVideos(youtubeApiKey, query.userInput, callback); - case YoutubeQueryType.VideoDetails: - YoutubeApi.authorizedGetVideoDetails(youtubeApiKey, query.videoIds, callback); - } -} + const downloadError = "Encountered an error while executing downloads."; + const requestError = "Unable to execute download: the body's media items were malformed."; + const deletionPermissionError = "Cannot perform specialized delete outside of the development environment!"; -const credentialsPath = path.join(__dirname, "./credentials/google_docs_credentials.json"); - -const EndpointHandlerMap = new Map<GoogleApiServerUtils.Action, GoogleApiServerUtils.ApiRouter>([ - ["create", (api, params) => api.create(params)], - ["retrieve", (api, params) => api.get(params)], - ["update", (api, params) => api.batchUpdate(params)], -]); - -app.post(RouteStore.googleDocs + "/:sector/:action", (req, res) => { - let sector: GoogleApiServerUtils.Service = req.params.sector as GoogleApiServerUtils.Service; - let action: GoogleApiServerUtils.Action = req.params.action as GoogleApiServerUtils.Action; - GoogleApiServerUtils.GetEndpoint(GoogleApiServerUtils.Service[sector], { credentialsPath, userId: req.headers.userId as string }).then(endpoint => { - let handler = EndpointHandlerMap.get(action); - if (endpoint && handler) { - let execute = handler(endpoint, req.body).then( - response => res.send(response.data), - rejection => res.send(rejection) - ); - execute.catch(exception => res.send(exception)); - return; - } - res.send(undefined); - }); -}); - -addSecureRoute({ - method: Method.GET, - subscribers: RouteStore.readGoogleAccessToken, - onValidation: async (user, _req, res) => { - const userId = user.id; - const token = await Database.Auxiliary.GoogleAuthenticationToken.Fetch(userId); - const information = { credentialsPath, userId }; - if (!token) { - return res.send(await GoogleApiServerUtils.GenerateAuthenticationUrl(information)); - } - GoogleApiServerUtils.RetrieveAccessToken(information).then(token => res.send(token)); - } -}); - -addSecureRoute({ - method: Method.POST, - subscribers: RouteStore.writeGoogleAccessToken, - onValidation: async (user, req, res) => { - const userId = user.id; - const information = { credentialsPath, userId }; - res.send(await GoogleApiServerUtils.ProcessClientSideCode(information, req.body.authenticationCode)); - } -}); - -const tokenError = "Unable to successfully upload bytes for all images!"; -const mediaError = "Unable to convert all uploaded bytes to media items!"; -const userIdError = "Unable to parse the identification of the user!"; + DashServer.get("/deleteWithAux", async (_req, res) => { + if (release) { + return _permission_denied(res, deletionPermissionError); + } + await Database.Auxiliary.DeleteAll(); + res.redirect(RouteStore.delete); + }); -export interface NewMediaItem { - description: string; - simpleMediaItem: { - uploadToken: string; - }; -} + DashServer.get("/deleteWithGoogleCredentials", async (req, res) => { + if (release) { + return _permission_denied(res, deletionPermissionError); + } + await Database.Auxiliary.GoogleAuthenticationToken.DeleteAll(); + res.redirect(RouteStore.delete); + }); -addSecureRoute({ - method: Method.POST, - subscribers: RouteStore.googlePhotosMediaUpload, - onValidation: async (user, req, res) => { - const { media } = req.body; - const userId = user.id; - if (!userId) { - return _error(res, userIdError); - } + const UploadError = (count: number) => `Unable to upload ${count} images to Dash's server`; + DashServer.post(RouteStore.googlePhotosMediaDownload, async (req, res) => { + const contents: { mediaItems: MediaItem[] } = req.body; + let failed = 0; + if (contents) { + const completed: Opt<DashUploadUtils.UploadInformation>[] = []; + for (let item of contents.mediaItems) { + const { contentSize, ...attributes } = await DashUploadUtils.InspectImage(item.baseUrl); + const found: Opt<DashUploadUtils.UploadInformation> = await Database.Auxiliary.QueryUploadHistory(contentSize!); + if (!found) { + const upload = await DashUploadUtils.UploadInspectedImage({ contentSize, ...attributes }, item.filename, prefix).catch(error => _error(res, downloadError, error)); + if (upload) { + completed.push(upload); + await Database.Auxiliary.LogUpload(upload); + } else { + failed++; + } + } else { + completed.push(found); + } + } + if (failed) { + return _error(res, UploadError(failed)); + } + return _success(res, completed); + } + _invalid(res, requestError); + }); - await GooglePhotosUploadUtils.initialize({ credentialsPath, userId }); + const suffixMap: { [type: string]: (string | [string, string | ((json: any) => any)]) } = { + "number": "_n", + "string": "_t", + "boolean": "_b", + "image": ["_t", "url"], + "video": ["_t", "url"], + "pdf": ["_t", "url"], + "audio": ["_t", "url"], + "web": ["_t", "url"], + "date": ["_d", value => new Date(value.date).toISOString()], + "proxy": ["_i", "fieldId"], + "list": ["_l", list => { + const results = []; + for (const value of list.fields) { + const term = ToSearchTerm(value); + if (term) { + results.push(term.value); + } + } + return results.length ? results : null; + }] + }; - let failed: number[] = []; + function ToSearchTerm(val: any): { suffix: string, value: any } | undefined { + if (val === null || val === undefined) { + return; + } + const type = val.__type || typeof val; + let suffix = suffixMap[type]; + if (!suffix) { + return; + } - const newMediaItems = await BatchedArray.from<GooglePhotosUploadUtils.MediaInput>(media, { batchSize: 25 }).batchedMapPatientInterval( - { magnitude: 100, unit: TimeUnit.Milliseconds }, - async (batch: GooglePhotosUploadUtils.MediaInput[]) => { - const newMediaItems: NewMediaItem[] = []; - for (let index = 0; index < batch.length; index++) { - const element = batch[index]; - const uploadToken = await GooglePhotosUploadUtils.DispatchGooglePhotosUpload(element.url); - if (!uploadToken) { - failed.push(index); + if (Array.isArray(suffix)) { + const accessor = suffix[1]; + if (typeof accessor === "function") { + val = accessor(val); } else { - newMediaItems.push({ - description: element.description, - simpleMediaItem: { uploadToken } - }); + val = val[accessor]; } + suffix = suffix[0]; } - return newMediaItems; - } - ); - const failedCount = failed.length; - if (failedCount) { - console.error(`Unable to upload ${failedCount} image${failedCount === 1 ? "" : "s"} to Google's servers`); - } + return { suffix, value: val }; + } - GooglePhotosUploadUtils.CreateMediaItems(newMediaItems, req.body.album).then( - result => _success(res, { results: result.newMediaItemResults, failed }), - error => _error(res, mediaError, error) - ); - } -}); + function getSuffix(value: string | [string, any]): string { + return typeof value === "string" ? value : value[0]; + } -interface MediaItem { - baseUrl: string; - filename: string; -} -const prefix = "google_photos_"; - -const downloadError = "Encountered an error while executing downloads."; -const requestError = "Unable to execute download: the body's media items were malformed."; -const deletionPermissionError = "Cannot perform specialized delete outside of the development environment!"; - -app.get("/deleteWithAux", async (_req, res) => { - if (release) { - return _permission_denied(res, deletionPermissionError); - } - await Database.Auxiliary.DeleteAll(); - res.redirect(RouteStore.delete); -}); - -app.get("/deleteWithGoogleCredentials", async (req, res) => { - if (release) { - return _permission_denied(res, deletionPermissionError); - } - await Database.Auxiliary.GoogleAuthenticationToken.DeleteAll(); - res.redirect(RouteStore.delete); -}); - -const UploadError = (count: number) => `Unable to upload ${count} images to Dash's server`; -app.post(RouteStore.googlePhotosMediaDownload, async (req, res) => { - const contents: { mediaItems: MediaItem[] } = req.body; - let failed = 0; - if (contents) { - const completed: Opt<DashUploadUtils.UploadInformation>[] = []; - for (let item of contents.mediaItems) { - const { contentSize, ...attributes } = await DashUploadUtils.InspectImage(item.baseUrl); - const found: Opt<DashUploadUtils.UploadInformation> = await Database.Auxiliary.QueryUploadHistory(contentSize!); - if (!found) { - const upload = await DashUploadUtils.UploadInspectedImage({ contentSize, ...attributes }, item.filename, prefix).catch(error => _error(res, downloadError, error)); - if (upload) { - completed.push(upload); - await Database.Auxiliary.LogUpload(upload); - } else { - failed++; + function UpdateField(socket: Socket, diff: Diff) { + Database.Instance.update(diff.id, diff.diff, + () => socket.broadcast.emit(MessageStore.UpdateField.Message, diff), false, "newDocuments"); + const docfield = diff.diff.$set; + if (!docfield) { + return; + } + const update: any = { id: diff.id }; + let dynfield = false; + for (let key in docfield) { + if (!key.startsWith("fields.")) continue; + dynfield = true; + let val = docfield[key]; + key = key.substring(7); + Object.values(suffixMap).forEach(suf => update[key + getSuffix(suf)] = { set: null }); + let term = ToSearchTerm(val); + if (term !== undefined) { + let { suffix, value } = term; + update[key + suffix] = { set: value }; + } + } + if (dynfield) { + Search.Instance.updateDocument(update); } - } else { - completed.push(found); - } - } - if (failed) { - return _error(res, UploadError(failed)); - } - return _success(res, completed); - } - _invalid(res, requestError); -}); - -const _error = (res: Response, message: string, error?: any) => { - res.statusMessage = message; - res.status(STATUS.EXECUTION_ERROR).send(error); -}; - -const _success = (res: Response, body: any) => { - res.status(STATUS.OK).send(body); -}; - -const _invalid = (res: Response, message: string) => { - res.statusMessage = message; - res.status(STATUS.BAD_REQUEST).send(); -}; - -const _permission_denied = (res: Response, message: string) => { - res.statusMessage = message; - res.status(STATUS.BAD_REQUEST).send("Permission Denied!"); -}; - -const suffixMap: { [type: string]: (string | [string, string | ((json: any) => any)]) } = { - "number": "_n", - "string": "_t", - "boolean": "_b", - "image": ["_t", "url"], - "video": ["_t", "url"], - "pdf": ["_t", "url"], - "audio": ["_t", "url"], - "web": ["_t", "url"], - "date": ["_d", value => new Date(value.date).toISOString()], - "proxy": ["_i", "fieldId"], - "list": ["_l", list => { - const results = []; - for (const value of list.fields) { - const term = ToSearchTerm(value); - if (term) { - results.push(term.value); } - } - return results.length ? results : null; - }] -}; - -function ToSearchTerm(val: any): { suffix: string, value: any } | undefined { - if (val === null || val === undefined) { - return; - } - const type = val.__type || typeof val; - let suffix = suffixMap[type]; - if (!suffix) { - return; - } - - if (Array.isArray(suffix)) { - const accessor = suffix[1]; - if (typeof accessor === "function") { - val = accessor(val); - } else { - val = val[accessor]; - } - suffix = suffix[0]; - } - - return { suffix, value: val }; -} -function getSuffix(value: string | [string, any]): string { - return typeof value === "string" ? value : value[0]; -} - -function UpdateField(socket: Socket, diff: Diff) { - Database.Instance.update(diff.id, diff.diff, - () => socket.broadcast.emit(MessageStore.UpdateField.Message, diff), false, "newDocuments"); - const docfield = diff.diff.$set; - if (!docfield) { - return; - } - const update: any = { id: diff.id }; - let dynfield = false; - for (let key in docfield) { - if (!key.startsWith("fields.")) continue; - dynfield = true; - let val = docfield[key]; - key = key.substring(7); - Object.values(suffixMap).forEach(suf => update[key + getSuffix(suf)] = { set: null }); - let term = ToSearchTerm(val); - if (term !== undefined) { - let { suffix, value } = term; - update[key + suffix] = { set: value }; - } - } - if (dynfield) { - Search.Instance.updateDocument(update); - } -} - -function DeleteField(socket: Socket, id: string) { - Database.Instance.delete({ _id: id }, "newDocuments").then(() => { - socket.broadcast.emit(MessageStore.DeleteField.Message, id); - }); + function DeleteField(socket: Socket, id: string) { + Database.Instance.delete({ _id: id }, "newDocuments").then(() => { + socket.broadcast.emit(MessageStore.DeleteField.Message, id); + }); - Search.Instance.deleteDocuments([id]); -} + Search.Instance.deleteDocuments([id]); + } -function DeleteFields(socket: Socket, ids: string[]) { - Database.Instance.delete({ _id: { $in: ids } }, "newDocuments").then(() => { - socket.broadcast.emit(MessageStore.DeleteFields.Message, ids); - }); + function DeleteFields(socket: Socket, ids: string[]) { + Database.Instance.delete({ _id: { $in: ids } }, "newDocuments").then(() => { + socket.broadcast.emit(MessageStore.DeleteFields.Message, ids); + }); - Search.Instance.deleteDocuments(ids); + Search.Instance.deleteDocuments(ids); -} + } -function CreateField(newValue: any) { - Database.Instance.insert(newValue, "newDocuments"); -} + function CreateField(newValue: any) { + Database.Instance.insert(newValue, "newDocuments"); + } -server.listen(serverPort); -console.log(`listening on port ${serverPort}`); + server.listen(serverPort); + console.log(`listening on port ${serverPort}`); + } + }); +})();
\ No newline at end of file |