From 91e4ac65e0b8d1ff5c17ea0e80666038281ec5a6 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Tue, 15 Oct 2019 12:54:57 -0400 Subject: initial commit --- src/server/ActionUtilities.ts | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 src/server/ActionUtilities.ts (limited to 'src/server/ActionUtilities.ts') diff --git a/src/server/ActionUtilities.ts b/src/server/ActionUtilities.ts new file mode 100644 index 000000000..9a009791b --- /dev/null +++ b/src/server/ActionUtilities.ts @@ -0,0 +1,28 @@ +import * as fs from 'fs'; +import { ExecOptions } from 'shelljs'; +import { exec } from 'child_process'; +import * as path from 'path'; + +export const command_line = (command: string, fromDirectory?: string) => { + return new Promise((resolve, reject) => { + let options: ExecOptions = {}; + if (fromDirectory) { + options.cwd = path.join(__dirname, fromDirectory); + } + exec(command, options, (err, stdout) => err ? reject(err) : resolve(stdout)); + }); +}; + +export const read_text_file = (relativePath: string) => { + let target = path.join(__dirname, relativePath); + return new Promise((resolve, reject) => { + fs.readFile(target, (err, data) => err ? reject(err) : resolve(data.toString())); + }); +}; + +export const write_text_file = (relativePath: string, contents: any) => { + let target = path.join(__dirname, relativePath); + return new Promise((resolve, reject) => { + fs.writeFile(target, contents, (err) => err ? reject(err) : resolve()); + }); +}; \ No newline at end of file -- cgit v1.2.3-70-g09d2 From 8441282d3d07d90536464e7b084b278ea9957cf8 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Mon, 18 Nov 2019 16:59:26 -0500 Subject: log --- src/server/ActionUtilities.ts | 8 +++++++- src/server/index.ts | 5 +++-- 2 files changed, 10 insertions(+), 3 deletions(-) (limited to 'src/server/ActionUtilities.ts') diff --git a/src/server/ActionUtilities.ts b/src/server/ActionUtilities.ts index 9a009791b..b418772a6 100644 --- a/src/server/ActionUtilities.ts +++ b/src/server/ActionUtilities.ts @@ -25,4 +25,10 @@ export const write_text_file = (relativePath: string, contents: any) => { return new Promise((resolve, reject) => { fs.writeFile(target, contents, (err) => err ? reject(err) : resolve()); }); -}; \ No newline at end of file +}; + +export async function log_execution(startMessage: string, endMessage: string, contents: () => void | Promise) { + console.log('\x1b[36m%s\x1b[0m', `${startMessage}...`); + await contents(); + console.log(endMessage); +} \ No newline at end of file diff --git a/src/server/index.ts b/src/server/index.ts index d68e9faa1..fb031a254 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -21,6 +21,7 @@ import { GoogleCredentialsLoader } from './credentials/CredentialsLoader'; import DeleteManager from "./ApiManagers/DeleteManager"; import PDFManager from "./ApiManagers/PDFManager"; import UploadManager from "./ApiManagers/UploadManager"; +import { log_execution } from "./ActionUtilities"; export const publicDirectory = __dirname + "/public"; export const filesDirectory = publicDirectory + "/files/"; @@ -43,7 +44,7 @@ async function preliminaryFunctions() { // divide the public directory based on type await Promise.all(Object.keys(Partitions).map(partition => DashUploadUtils.createIfNotExists(filesDirectory + partition))); // connect to the database - await Database.tryInitializeConnection(); + await log_execution("attempting to initialize database connection", "connected", Database.tryInitializeConnection); } /** @@ -243,6 +244,6 @@ function routeSetter(router: RouteManager) { } (async function start() { - await preliminaryFunctions(); + await log_execution("starting execution of preliminary functions", "completed preliminary functions", preliminaryFunctions); await initializeServer({ listenAtPort: 1050, routeSetter }); })(); -- cgit v1.2.3-70-g09d2 From 9a72d65baaa293fa84592de9bc43d555b5cae319 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Tue, 19 Nov 2019 19:52:56 -0500 Subject: logging --- src/Utils.ts | 5 +++++ src/server/ActionUtilities.ts | 31 +++++++++++++++++++++++++++---- src/server/Initialization.ts | 8 +++++++- src/server/Websocket/Websocket.ts | 12 ++++++------ src/server/index.ts | 12 ++++++++++-- 5 files changed, 55 insertions(+), 13 deletions(-) (limited to 'src/server/ActionUtilities.ts') diff --git a/src/Utils.ts b/src/Utils.ts index 13d1be767..891a3826b 100644 --- a/src/Utils.ts +++ b/src/Utils.ts @@ -2,6 +2,7 @@ import v4 = require('uuid/v4'); import v5 = require("uuid/v5"); import { Socket } from 'socket.io'; import { Message } from './server/Message'; +import { EventEmitter } from 'events'; export namespace Utils { @@ -238,6 +239,10 @@ export namespace Utils { handler([arg, loggingCallback('S sending', fn, message.Name)]); }); } + + export function InjectLogger() { + + } } export function OmitKeys(obj: any, keys: string[], addKeyFunc?: (dup: any) => void): { omit: any, extract: any } { diff --git a/src/server/ActionUtilities.ts b/src/server/ActionUtilities.ts index b418772a6..5e88ea460 100644 --- a/src/server/ActionUtilities.ts +++ b/src/server/ActionUtilities.ts @@ -27,8 +27,31 @@ export const write_text_file = (relativePath: string, contents: any) => { }); }; -export async function log_execution(startMessage: string, endMessage: string, contents: () => void | Promise) { - console.log('\x1b[36m%s\x1b[0m', `${startMessage}...`); - await contents(); - console.log(endMessage); +export interface LogData { + startMessage: string; + endMessage: string; + action: () => void | Promise; +} + +export async function log_execution({ startMessage, endMessage, action }: LogData) { + const color = `\x1b[${30 + Math.ceil(Math.random() * 6)}m%s\x1b[0m`; + console.log(color, `${startMessage}...`); + await action(); + console.log(color, endMessage); +} + +export enum ConsoleColors { + Black = `\x1b[30m%s\x1b[0m`, + Red = `\x1b[31m%s\x1b[0m`, + Green = `\x1b[32m%s\x1b[0m`, + Yellow = `\x1b[33m%s\x1b[0m`, + Blue = `\x1b[34m%s\x1b[0m`, + Magenta = `\x1b[35m%s\x1b[0m`, + Cyan = `\x1b[36m%s\x1b[0m`, + White = `\x1b[37m%s\x1b[0m` +} + +export function logPort(listener: string, port: number) { + process.stdout.write(`${listener} listening on port `); + console.log(ConsoleColors.Yellow, port); } \ No newline at end of file diff --git a/src/server/Initialization.ts b/src/server/Initialization.ts index 6ac0787c5..357d7837e 100644 --- a/src/server/Initialization.ts +++ b/src/server/Initialization.ts @@ -19,6 +19,7 @@ import * as fs from 'fs'; import * as request from 'request'; import RouteSubscriber from './RouteSubscriber'; import { publicDirectory } from '.'; +import { ConsoleColors, logPort } from './ActionUtilities'; /* RouteSetter is a wrapper around the server that prevents the server from being exposed. */ @@ -35,6 +36,11 @@ export default async function InitializeServer(options: InitializationOptions) { server.use(express.static(publicDirectory)); server.use("/images", express.static(publicDirectory)); + server.use("*", (req, _res, next) => { + console.log(ConsoleColors.Cyan, req.originalUrl, req.user.id); + next(); + }); + server.use(wdm(compiler, { publicPath: config.output.publicPath })); server.use(whm(compiler)); @@ -44,7 +50,7 @@ export default async function InitializeServer(options: InitializationOptions) { const isRelease = determineEnvironment(); //vs. dev mode routeSetter(new RouteManager(server, isRelease)); - server.listen(listenAtPort, () => console.log(`server started at http://localhost:${listenAtPort}`)); + server.listen(listenAtPort, () => logPort("server", listenAtPort)); return isRelease; } diff --git a/src/server/Websocket/Websocket.ts b/src/server/Websocket/Websocket.ts index f6a6c8718..810b3410c 100644 --- a/src/server/Websocket/Websocket.ts +++ b/src/server/Websocket/Websocket.ts @@ -6,9 +6,9 @@ import { Database } from "../database"; import { Search } from "../Search"; import * as io from 'socket.io'; import YoutubeApi from "../apis/youtube/youtubeApiSample"; -import { readFile } from "fs"; -import { Credentials } from "google-auth-library"; import { GoogleCredentialsLoader } from "../credentials/CredentialsLoader"; +import { ConsoleColors, logPort } from "../ActionUtilities"; +import { EventEmitter } from "events"; export namespace WebSocket { @@ -28,10 +28,10 @@ export namespace WebSocket { async function preliminaryFunctions() { } - export function initialize(serverPort: number, isRelease: boolean) { + export function initialize(socketPort: number, isRelease: boolean) { const endpoint = io(); - endpoint.listen(serverPort); - console.log(`listening on port ${serverPort}`); + endpoint.listen(socketPort); + logPort("websocket", socketPort); endpoint.on("connection", function (socket: Socket) { socket.use((_packet, next) => { @@ -92,7 +92,7 @@ export namespace WebSocket { function barReceived(socket: SocketIO.Socket, guid: string) { clients[guid] = new Client(guid.toString()); - console.log(`User ${guid} has connected`); + console.log(ConsoleColors.Green, `User ${guid} has connected`); socketMap.set(socket, guid); } diff --git a/src/server/index.ts b/src/server/index.ts index 01ebf0ac1..618940c1a 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -43,7 +43,11 @@ async function preliminaryFunctions() { // divide the public directory based on type await Promise.all(Object.keys(Partitions).map(partition => DashUploadUtils.createIfNotExists(filesDirectory + partition))); // connect to the database - await Database.tryInitializeConnection(); + await log_execution({ + startMessage: "attempting to initialize mongodb connection", + endMessage: "connection outcome determined", + action: Database.tryInitializeConnection + }); } /** @@ -106,6 +110,10 @@ function routeSetter(router: RouteManager) { } (async function start() { - await log_execution("starting execution of preliminary functions", "completed preliminary functions", preliminaryFunctions); + await log_execution({ + startMessage: "starting execution of preliminary functions", + endMessage: "completed preliminary functions", + action: preliminaryFunctions + }); await initializeServer({ listenAtPort: 1050, routeSetter }); })(); -- cgit v1.2.3-70-g09d2 From 39bac937106e77679b2dc76078b812a6b6b11a94 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Thu, 21 Nov 2019 18:20:01 -0500 Subject: last cleanup --- src/server/ActionUtilities.ts | 3 ++- src/server/ApiManagers/ApiManager.ts | 4 ++-- src/server/ApiManagers/UploadManager.ts | 4 ++-- src/server/DashUploadUtils.ts | 12 ++++++++++++ src/server/index.ts | 25 ++++++++----------------- 5 files changed, 26 insertions(+), 22 deletions(-) (limited to 'src/server/ActionUtilities.ts') diff --git a/src/server/ActionUtilities.ts b/src/server/ActionUtilities.ts index 5e88ea460..a5f33833d 100644 --- a/src/server/ActionUtilities.ts +++ b/src/server/ActionUtilities.ts @@ -33,8 +33,9 @@ export interface LogData { action: () => void | Promise; } +let current = Math.ceil(Math.random() * 20); export async function log_execution({ startMessage, endMessage, action }: LogData) { - const color = `\x1b[${30 + Math.ceil(Math.random() * 6)}m%s\x1b[0m`; + const color = `\x1b[${31 + current++ % 6}m%s\x1b[0m`; console.log(color, `${startMessage}...`); await action(); console.log(color, endMessage); diff --git a/src/server/ApiManagers/ApiManager.ts b/src/server/ApiManagers/ApiManager.ts index 9fd726060..e2b01d585 100644 --- a/src/server/ApiManagers/ApiManager.ts +++ b/src/server/ApiManagers/ApiManager.ts @@ -5,7 +5,7 @@ export type Registration = (initializer: RouteInitializer) => void; export default abstract class ApiManager { protected abstract initialize(register: Registration): void; - public register(router: RouteManager) { - this.initialize(router.addSupervisedRoute); + public register(register: Registration) { + this.initialize(register); } } \ No newline at end of file diff --git a/src/server/ApiManagers/UploadManager.ts b/src/server/ApiManagers/UploadManager.ts index 01abdab54..aca63a918 100644 --- a/src/server/ApiManagers/UploadManager.ts +++ b/src/server/ApiManagers/UploadManager.ts @@ -5,7 +5,7 @@ import v4 = require('uuid/v4'); var AdmZip = require('adm-zip'); import * as path from 'path'; import { createReadStream, createWriteStream, unlink, readFileSync } from "fs"; -import { publicDirectory, filesDirectory, Partitions } from ".."; +import { publicDirectory, filesDirectory } from ".."; import { Database } from "../database"; import { DashUploadUtils } from "../DashUploadUtils"; import { Opt } from "../../new_fields/Doc"; @@ -142,7 +142,7 @@ export default class UploadManager extends ApiManager { let dataBuffer = readFileSync(filesDirectory + filename); const result: ParsedPDF = await pdf(dataBuffer); await new Promise((resolve, reject) => { - const path = filesDirectory + Partitions.pdf_text + "/" + filename.substring(0, filename.length - ".pdf".length) + ".txt"; + const path = filesDirectory + DashUploadUtils.Partitions.pdf_text + "/" + filename.substring(0, filename.length - ".pdf".length) + ".txt"; createWriteStream(path).write(result.text, error => { if (!error) { resolve(); diff --git a/src/server/DashUploadUtils.ts b/src/server/DashUploadUtils.ts index 8f5b0e1a8..8a429b81b 100644 --- a/src/server/DashUploadUtils.ts +++ b/src/server/DashUploadUtils.ts @@ -6,6 +6,7 @@ import request = require('request-promise'); import { ExifData, ExifImage } from 'exif'; import { Opt } from '../new_fields/Doc'; import { SharedMediaTypes } from './SharedMediaTypes'; +import { filesDirectory } from '.'; const uploadDirectory = path.join(__dirname, './public/files/'); @@ -89,6 +90,17 @@ export namespace DashUploadUtils { error?: string; } + export enum Partitions { + pdf_text, + images, + videos + } + + export async function buildFilePartitions() { + const pending = Object.keys(Partitions).map(sub => createIfNotExists(filesDirectory + sub)); + return Promise.all(pending); + } + /** * Based on the url's classification as local or remote, gleans * as much information as possible about the specified image diff --git a/src/server/index.ts b/src/server/index.ts index 618940c1a..9c48aca45 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -22,13 +22,8 @@ import { log_execution } from "./ActionUtilities"; import GeneralGoogleManager from "./ApiManagers/GeneralGoogleManager"; import GooglePhotosManager from "./ApiManagers/GooglePhotosManager"; -export const publicDirectory = __dirname + "/public"; -export const filesDirectory = publicDirectory + "/files/"; -export enum Partitions { - pdf_text, - images, - videos -} +export const publicDirectory = path.resolve(__dirname, "public"); +export const filesDirectory = path.resolve(publicDirectory, "files") + "/"; /** * These are the functions run before the server starts @@ -36,13 +31,9 @@ export enum Partitions { * before clients can access the server should be run or awaited here. */ async function preliminaryFunctions() { - // make project credentials globally accessible await GoogleCredentialsLoader.loadCredentials(); - // read the resulting credentials into a different namespace GoogleApiServerUtils.processProjectCredentials(); - // divide the public directory based on type - await Promise.all(Object.keys(Partitions).map(partition => DashUploadUtils.createIfNotExists(filesDirectory + partition))); - // connect to the database + await DashUploadUtils.buildFilePartitions(); await log_execution({ startMessage: "attempting to initialize mongodb connection", endMessage: "connection outcome determined", @@ -59,7 +50,7 @@ async function preliminaryFunctions() { * that will manage the registration of new routes * with the server */ -function routeSetter(router: RouteManager) { +function routeSetter({ isRelease, addSupervisedRoute }: RouteManager) { const managers = [ new UserManager(), new UploadManager(), @@ -73,16 +64,16 @@ function routeSetter(router: RouteManager) { ]; // initialize API Managers - managers.forEach(manager => manager.register(router)); + managers.forEach(manager => manager.register(addSupervisedRoute)); // initialize the web socket (bidirectional communication: if a user changes // a field on one client, that change must be broadcast to all other clients) - WebSocket.initialize(serverPort, router.isRelease); + WebSocket.initialize(serverPort, isRelease); /** * Accessing root index redirects to home */ - router.addSupervisedRoute({ + addSupervisedRoute({ method: Method.GET, subscription: "/", onValidation: ({ res }) => res.redirect("/home") @@ -94,7 +85,7 @@ function routeSetter(router: RouteManager) { res.sendFile(path.join(__dirname, '../../deploy/' + filename)); }; - router.addSupervisedRoute({ + addSupervisedRoute({ method: Method.GET, subscription: ["/home", new RouteSubscriber("doc").add("docId")], onValidation: serve, -- cgit v1.2.3-70-g09d2 From b831be86743e329cce441b3d7ae2aa5321e7fb9c Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Sat, 23 Nov 2019 17:09:13 -0500 Subject: improved user activity log --- src/server/ActionUtilities.ts | 13 ++++ src/server/ApiManagers/UserManager.ts | 46 ++++++------ src/server/Initialization.ts | 8 ++- src/server/Websocket/Websocket.ts | 22 +++--- .../authentication/controllers/user_controller.ts | 3 - views/login.pug | 2 - views/stylesheets/authentication.css | 81 ++++++++++++++++++++++ views/user_activity.pug | 19 +++++ 8 files changed, 155 insertions(+), 39 deletions(-) create mode 100644 views/user_activity.pug (limited to 'src/server/ActionUtilities.ts') diff --git a/src/server/ActionUtilities.ts b/src/server/ActionUtilities.ts index a5f33833d..7f493dd70 100644 --- a/src/server/ActionUtilities.ts +++ b/src/server/ActionUtilities.ts @@ -55,4 +55,17 @@ export enum ConsoleColors { export function logPort(listener: string, port: number) { process.stdout.write(`${listener} listening on port `); console.log(ConsoleColors.Yellow, port); +} + +export function msToTime(duration: number) { + let milliseconds = Math.floor((duration % 1000) / 100), + seconds = Math.floor((duration / 1000) % 60), + minutes = Math.floor((duration / (1000 * 60)) % 60), + hours = Math.floor((duration / (1000 * 60 * 60)) % 24); + + let hoursS = (hours < 10) ? "0" + hours : hours; + let minutesS = (minutes < 10) ? "0" + minutes : minutes; + let secondsS = (seconds < 10) ? "0" + seconds : seconds; + + return hoursS + ":" + minutesS + ":" + secondsS + "." + milliseconds; } \ No newline at end of file diff --git a/src/server/ApiManagers/UserManager.ts b/src/server/ApiManagers/UserManager.ts index 51a434fcf..4ee5a2b85 100644 --- a/src/server/ApiManagers/UserManager.ts +++ b/src/server/ApiManagers/UserManager.ts @@ -1,7 +1,14 @@ import ApiManager, { Registration } from "./ApiManager"; import { Method } from "../RouteManager"; -import { WebSocket } from "../Websocket/Websocket"; import { Database } from "../database"; +import { msToTime } from "../ActionUtilities"; + +export const timeMap: { [id: string]: number } = {}; +interface ActivityUnit { + user: string; + duration: number; +} + export default class UserManager extends ApiManager { @@ -32,35 +39,34 @@ export default class UserManager extends ApiManager { register({ method: Method.GET, - subscription: "/whosOnline", + subscription: "/activity", onValidation: ({ res }) => { - let users: any = { active: {}, inactive: {} }; const now = Date.now(); - const { timeMap } = WebSocket; + const activeTimes: ActivityUnit[] = []; + const inactiveTimes: ActivityUnit[] = []; + for (const user in timeMap) { const time = timeMap[user]; - const key = ((now - time) / 1000) < (60 * 5) ? "active" : "inactive"; - users[key][user] = `Last active ${msToTime(now - time)} ago`; + const duration = now - time; + const target = (duration / 1000) < (60 * 5) ? activeTimes : inactiveTimes; + target.push({ user, duration }); } - res.send(users); + const process = (target: { user: string, duration: number }[]) => { + const comparator = (first: ActivityUnit, second: ActivityUnit) => first.duration - second.duration; + const sorted = target.sort(comparator); + return sorted.map(({ user, duration }) => `${user} (last active ${msToTime(duration)} ago)`); + }; + + res.render("user_activity.pug", { + title: "User Activity", + active: process(activeTimes), + inactive: process(inactiveTimes) + }); } }); } -} - -function msToTime(duration: number) { - let milliseconds = Math.floor((duration % 1000) / 100), - seconds = Math.floor((duration / 1000) % 60), - minutes = Math.floor((duration / (1000 * 60)) % 60), - hours = Math.floor((duration / (1000 * 60 * 60)) % 24); - - let hoursS = (hours < 10) ? "0" + hours : hours; - let minutesS = (minutes < 10) ? "0" + minutes : minutes; - let secondsS = (seconds < 10) ? "0" + seconds : seconds; - - return hoursS + ":" + minutesS + ":" + secondsS + "." + milliseconds; } \ No newline at end of file diff --git a/src/server/Initialization.ts b/src/server/Initialization.ts index 08b476822..7fad5556d 100644 --- a/src/server/Initialization.ts +++ b/src/server/Initialization.ts @@ -20,6 +20,8 @@ import * as request from 'request'; import RouteSubscriber from './RouteSubscriber'; import { publicDirectory } from '.'; import { ConsoleColors, logPort } from './ActionUtilities'; +import { WebSocket } from './Websocket/Websocket'; +import { timeMap } from './ApiManagers/UserManager'; /* RouteSetter is a wrapper around the server that prevents the server from being exposed. */ @@ -37,7 +39,11 @@ export default async function InitializeServer(options: InitializationOptions) { server.use("/images", express.static(publicDirectory)); server.use("*", (req, _res, next) => { - console.log(ConsoleColors.Cyan, req.originalUrl, req.user.email); + const userEmail = req.user?.email; + console.log(ConsoleColors.Cyan, req.originalUrl, userEmail ?? ""); + if (userEmail) { + timeMap[userEmail] = Date.now(); + } next(); }); diff --git a/src/server/Websocket/Websocket.ts b/src/server/Websocket/Websocket.ts index 74a6b4263..fbf71f707 100644 --- a/src/server/Websocket/Websocket.ts +++ b/src/server/Websocket/Websocket.ts @@ -8,16 +8,12 @@ import * as io from 'socket.io'; import YoutubeApi from "../apis/youtube/youtubeApiSample"; import { GoogleCredentialsLoader } from "../credentials/CredentialsLoader"; import { ConsoleColors, logPort } from "../ActionUtilities"; +import { timeMap } from "../ApiManagers/UserManager"; export namespace WebSocket { - interface Map { - [key: string]: Client; - } - let clients: Map = {}; - + let clients: { [key: string]: Client } = {}; export const socketMap = new Map(); - export const timeMap: { [id: string]: number } = {}; export async function start(serverPort: number, isRelease: boolean) { await preliminaryFunctions(); @@ -31,9 +27,9 @@ export namespace WebSocket { const endpoint = io(); endpoint.on("connection", function (socket: Socket) { socket.use((_packet, next) => { - let id = socketMap.get(socket); - if (id) { - timeMap[id] = Date.now(); + let userEmail = socketMap.get(socket); + if (userEmail) { + timeMap[userEmail] = Date.now(); } next(); }); @@ -87,10 +83,10 @@ export namespace WebSocket { await Search.Instance.clear(); } - function barReceived(socket: SocketIO.Socket, guid: string) { - clients[guid] = new Client(guid.toString()); - console.log(ConsoleColors.Green, `user ${guid} has connected to the web socket`); - socketMap.set(socket, guid); + function barReceived(socket: SocketIO.Socket, userEmail: string) { + clients[userEmail] = new Client(userEmail.toString()); + console.log(ConsoleColors.Green, `user ${userEmail} has connected to the web socket`); + socketMap.set(socket, userEmail); } function getField([id, callback]: [string, (result?: Transferable) => void]) { diff --git a/src/server/authentication/controllers/user_controller.ts b/src/server/authentication/controllers/user_controller.ts index b2b9d33f6..517353479 100644 --- a/src/server/authentication/controllers/user_controller.ts +++ b/src/server/authentication/controllers/user_controller.ts @@ -3,10 +3,7 @@ import { Request, Response, NextFunction } from "express"; import * as passport from "passport"; import { IVerifyOptions } from "passport-local"; import "../config/passport"; -import * as request from "express-validator"; import flash = require("express-flash"); -import * as session from "express-session"; -import * as pug from 'pug'; import * as async from 'async'; import * as nodemailer from 'nodemailer'; import c = require("crypto"); diff --git a/views/login.pug b/views/login.pug index 9bc40a495..26da5e29e 100644 --- a/views/login.pug +++ b/views/login.pug @@ -14,11 +14,9 @@ block content .inner.login h3.auth_header Log In .form-group - //- label.col-sm-3.control-label(for='email', id='email_label') Email .col-sm-7 input.form-control(type='email', name='email', id='email', placeholder='Email', autofocus, required) .form-group - //- label.col-sm-3.control-label(for='password') Password .col-sm-7 input.form-control(type='password', name='password', id='password', placeholder='Password', required) .form-group diff --git a/views/stylesheets/authentication.css b/views/stylesheets/authentication.css index 36bb880af..ff1f4aace 100644 --- a/views/stylesheets/authentication.css +++ b/views/stylesheets/authentication.css @@ -139,4 +139,85 @@ body { padding-right: 10px; font-family: Arial, Helvetica, sans-serif; font-size: 16px; +} + +.outermost, .online-container { + display: flex; + flex-direction: row; + height: 98vh; + justify-content: center; +} + +.online-container { + background: white; + display: flex; + flex-direction: row; + height: 80%; + width: 80%; + align-self: center; + justify-content: center; + border-radius: 8px; + box-shadow: 10px 10px 10px #00000099; +} + +.partition { + width: 50%; + display: flex; + flex-direction: column; + border: 1px solid black; +} + +.inner-activity { + display: flex; + flex-direction: column; + justify-content: center; + height: 100%; + border-top: 2px solid black; + background: white; + padding: 20px; + overflow: scroll; +} + +ol { + align-self: center; +} + +li { + font-family: Arial, Helvetica, sans-serif; + border: 1px solid black; + padding: 10px; + border-radius: 5px; + margin-bottom: 5px; +} + +.duration { + font-style: italic; +} + +span.user-type { + align-self: center; + font-family: Arial, Helvetica, sans-serif; + font-weight: bold; + font-size: 20px; + margin: 50px; +} + +#active-partition { + background: green; + border-top-left-radius: 8px; + border-bottom-left-radius: 8px; +} + +#active-inner { + border-bottom-left-radius: 8px; +} + +#inactive-partition { + background: red; + border-top-right-radius: 8px; + border-bottom-right-radius: 8px; +} + +#inactive-inner { + border-bottom-right-radius: 8px; } \ No newline at end of file diff --git a/views/user_activity.pug b/views/user_activity.pug new file mode 100644 index 000000000..68e42140d --- /dev/null +++ b/views/user_activity.pug @@ -0,0 +1,19 @@ +extends ./layout + +block content + style + include ./stylesheets/authentication.css + .outermost + .online-container + .partition(id="active-partition") + span.user-type Active Users + .inner-activity(id="active-inner") + ol + each val in active + li= val + .partition(id="inactive-partition") + span.user-type Inactive Users + .inner-activity(id="inactive-inner") + ol + each val in inactive + li= val \ No newline at end of file -- cgit v1.2.3-70-g09d2 From df5584ccd40bd83f1362b32db67969e7ffbf2e3f Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Wed, 27 Nov 2019 04:03:30 -0500 Subject: improved file partitioning in server and generified upload method --- package.json | 2 + src/client/documents/Documents.ts | 1 - src/client/util/ClientDiagnostics.ts | 11 +- .../util/Import & Export/DirectoryImportBox.tsx | 19 +- src/client/views/Main.tsx | 2 +- src/client/views/collections/CollectionSubView.tsx | 22 +- src/client/views/pdf/PDFViewer.tsx | 3 +- src/server/ActionUtilities.ts | 17 +- src/server/ApiManagers/DownloadManager.ts | 4 +- src/server/ApiManagers/GooglePhotosManager.ts | 4 +- src/server/ApiManagers/PDFManager.ts | 34 +- src/server/ApiManagers/SearchManager.ts | 4 +- src/server/ApiManagers/UploadManager.ts | 47 ++- src/server/DashUploadUtils.ts | 388 +++++++++++---------- src/server/SharedMediaTypes.ts | 5 +- src/server/database.ts | 4 +- src/server/index.ts | 4 +- 17 files changed, 314 insertions(+), 257 deletions(-) (limited to 'src/server/ActionUtilities.ts') diff --git a/package.json b/package.json index 393df8574..3725d76eb 100644 --- a/package.json +++ b/package.json @@ -105,6 +105,7 @@ "@types/react-table": "^6.7.22", "@types/request": "^2.48.1", "@types/request-promise": "^4.1.42", + "@types/rimraf": "^2.0.3", "@types/sharp": "^0.22.2", "@types/shelljs": "^0.8.5", "@types/socket.io": "^2.1.2", @@ -211,6 +212,7 @@ "readline": "^1.3.0", "request": "^2.88.0", "request-promise": "^4.2.4", + "rimraf": "^3.0.0", "serializr": "^1.5.1", "sharp": "^0.22.1", "shelljs": "^0.8.3", diff --git a/src/client/documents/Documents.ts b/src/client/documents/Documents.ts index dea057b93..a2f4d23c7 100644 --- a/src/client/documents/Documents.ts +++ b/src/client/documents/Documents.ts @@ -377,7 +377,6 @@ export namespace Docs { let extension = path.extname(target); target = `${target.substring(0, target.length - extension.length)}_o${extension}`; } - // if (target !== "http://www.cs.brown.edu/") { requestImageSize(target) .then((size: any) => { let aspect = size.height / size.width; diff --git a/src/client/util/ClientDiagnostics.ts b/src/client/util/ClientDiagnostics.ts index 24f196252..7eef935fd 100644 --- a/src/client/util/ClientDiagnostics.ts +++ b/src/client/util/ClientDiagnostics.ts @@ -1,9 +1,6 @@ -import { observable, runInAction } from "mobx"; -import { MainView } from "../views/MainView"; - export namespace ClientDiagnostics { - export function start() { + export async function start() { let serverPolls = 0; const serverHandle = setInterval(async () => { @@ -17,14 +14,16 @@ export namespace ClientDiagnostics { let executed = false; - const solrHandle = setInterval(async () => { + const handle = async () => { const response = await fetch("/solrHeartbeat"); if (!(await response.json()).running) { !executed && alert("Looks like SOLR is not running on your machine."); executed = true; clearInterval(solrHandle); } - }, 1000 * 15); + }; + await handle(); + const solrHandle = setInterval(handle, 1000 * 15); } diff --git a/src/client/util/Import & Export/DirectoryImportBox.tsx b/src/client/util/Import & Export/DirectoryImportBox.tsx index f0880f193..16ae50685 100644 --- a/src/client/util/Import & Export/DirectoryImportBox.tsx +++ b/src/client/util/Import & Export/DirectoryImportBox.tsx @@ -22,18 +22,10 @@ import "./DirectoryImportBox.scss"; import { Networking } from "../../Network"; import { BatchedArray } from "array-batcher"; import * as path from 'path'; -import { DashUploadUtils } from "../../../server/DashUploadUtils"; -import { SharedMediaTypes } from "../../../server/SharedMediaTypes"; +import { AcceptibleMedia } from "../../../server/SharedMediaTypes"; const unsupported = ["text/html", "text/plain"]; -interface ImageUploadResponse { - name: string; - path: string; - type: string; - exif: any; -} - @observer export default class DirectoryImportBox extends React.Component { private selector = React.createRef(); @@ -98,7 +90,7 @@ export default class DirectoryImportBox extends React.Component let file = files.item(i); if (file && !unsupported.includes(file.type)) { const ext = path.extname(file.name).toLowerCase(); - if (SharedMediaTypes.imageFormats.includes(ext)) { + if (AcceptibleMedia.imageFormats.includes(ext)) { validated.push(file); } } @@ -114,7 +106,7 @@ export default class DirectoryImportBox extends React.Component runInAction(() => this.phase = `Internal: uploading ${this.quota - this.completed} files to Dash...`); - const uploads = await BatchedArray.from(validated, { batchSize: 15 }).batchedMapAsync(async (batch, collector) => { + const uploads = await BatchedArray.from(validated, { batchSize: 15 }).batchedMapAsync(async (batch, collector) => { const formData = new FormData(); batch.forEach(file => { @@ -127,16 +119,17 @@ export default class DirectoryImportBox extends React.Component runInAction(() => this.completed += batch.length); }); + const size = "_o"; await Promise.all(uploads.map(async upload => { const type = upload.type; - const path = Utils.prepend(upload.path); + const path = Utils.prepend(upload.clientAccessPath); const options = { nativeWidth: 300, width: 300, title: upload.name }; const document = await Docs.Get.DocumentFromType(type, path, options); - const { data, error } = upload.exif; + const { data, error } = upload.exifData; if (document) { Doc.GetProto(document).exif = error || Docs.Get.DocumentHierarchyFromJson(data); docs.push(document); diff --git a/src/client/views/Main.tsx b/src/client/views/Main.tsx index dec4a24e4..9e699978f 100644 --- a/src/client/views/Main.tsx +++ b/src/client/views/Main.tsx @@ -10,7 +10,7 @@ import { ClientDiagnostics } from "../util/ClientDiagnostics"; AssignAllExtensions(); (async () => { - ClientDiagnostics.start(); + await ClientDiagnostics.start(); const info = await CurrentUserUtils.loadCurrentUser(); DocServer.init(window.location.protocol, window.location.hostname, 4321, info.email); await Docs.Prototypes.initialize(); diff --git a/src/client/views/collections/CollectionSubView.tsx b/src/client/views/collections/CollectionSubView.tsx index 1c3ff37ee..a1bd1527e 100644 --- a/src/client/views/collections/CollectionSubView.tsx +++ b/src/client/views/collections/CollectionSubView.tsx @@ -22,6 +22,7 @@ import React = require("react"); var path = require('path'); import { GooglePhotos } from "../../apis/google_docs/GooglePhotosClientUtils"; import { ImageUtils } from "../../util/Import & Export/ImageUtils"; +import { Networking } from "../../Network"; export interface CollectionViewProps extends FieldViewProps { addDocument: (document: Doc) => boolean; @@ -271,28 +272,25 @@ export function CollectionSubView(schemaCtor: (doc: Doc) => T) { let file = item.getAsFile(); let formData = new FormData(); - if (file) { - formData.append('file', file); + if (!file || !file.type) { + continue; } - let dropFileName = file ? file.name : "-empty-"; - let prom = fetch(Utils.prepend("/upload"), { - method: 'POST', - body: formData - }).then(async (res: Response) => { - (await res.json()).map(action((file: any) => { + formData.append('file', file); + let dropFileName = file ? file.name : "-empty-"; + promises.push(Networking.PostFormDataToServer("/upload", formData).then(results => { + results.map(action((file: any) => { let full = { ...options, nativeWidth: type.indexOf("video") !== -1 ? 600 : 300, width: 300, title: dropFileName }; - let pathname = Utils.prepend(file.path); + let pathname = Utils.prepend(file.clientAccessPath); Docs.Get.DocumentFromType(type, pathname, full).then(doc => { doc && (Doc.GetProto(doc).fileUpload = path.basename(pathname).replace("upload_", "").replace(/\.[a-z0-9]*$/, "")); doc && this.props.addDocument(doc); }); })); - }); - promises.push(prom); + })); } } - if (text) { + if (text && !text.includes("https://")) { this.props.addDocument(Docs.Create.TextDocument({ ...options, documentText: "@@@" + text, width: 400, height: 315 })); return; } diff --git a/src/client/views/pdf/PDFViewer.tsx b/src/client/views/pdf/PDFViewer.tsx index b737ce221..c075a4f99 100644 --- a/src/client/views/pdf/PDFViewer.tsx +++ b/src/client/views/pdf/PDFViewer.tsx @@ -125,7 +125,8 @@ export class PDFViewer extends DocAnnotatableComponent this._showWaiting = this._showCover = true); this.props.startupLive && this.setupPdfJsViewer(); this._searchReactionDisposer = reaction(() => this.Document.search_string, searchString => { diff --git a/src/server/ActionUtilities.ts b/src/server/ActionUtilities.ts index 7f493dd70..c9fc86fea 100644 --- a/src/server/ActionUtilities.ts +++ b/src/server/ActionUtilities.ts @@ -2,6 +2,7 @@ import * as fs from 'fs'; import { ExecOptions } from 'shelljs'; import { exec } from 'child_process'; import * as path from 'path'; +import * as rimraf from "rimraf"; export const command_line = (command: string, fromDirectory?: string) => { return new Promise((resolve, reject) => { @@ -68,4 +69,18 @@ export function msToTime(duration: number) { let secondsS = (seconds < 10) ? "0" + seconds : seconds; return hoursS + ":" + minutesS + ":" + secondsS + "." + milliseconds; -} \ No newline at end of file +} + +export const createIfNotExists = async (path: string) => { + if (await new Promise(resolve => fs.exists(path, resolve))) { + return true; + } + return new Promise(resolve => fs.mkdir(path, error => resolve(error === null))); +}; + +export async function Prune(rootDirectory: string): Promise { + const error = await new Promise(resolve => rimraf(rootDirectory, resolve)); + return error === null; +} + +export const Destroy = (mediaPath: string) => new Promise(resolve => fs.unlink(mediaPath, error => resolve(error === null))); \ No newline at end of file diff --git a/src/server/ApiManagers/DownloadManager.ts b/src/server/ApiManagers/DownloadManager.ts index fc6ba0d22..5bad46eda 100644 --- a/src/server/ApiManagers/DownloadManager.ts +++ b/src/server/ApiManagers/DownloadManager.ts @@ -5,7 +5,7 @@ import * as Archiver from 'archiver'; import * as express from 'express'; import { Database } from "../database"; import * as path from "path"; -import { DashUploadUtils } from "../DashUploadUtils"; +import { DashUploadUtils, SizeSuffix } from "../DashUploadUtils"; import { publicDirectory } from ".."; export type Hierarchy = { [id: string]: string | Hierarchy }; @@ -254,7 +254,7 @@ async function writeHierarchyRecursive(file: Archiver.Archiver, hierarchy: Hiera // and dropped in the browser and thus hosted remotely) so we upload it // to our server and point the zip file to it, so it can bundle up the bytes const information = await DashUploadUtils.UploadImage(result); - path = information.mediaPaths[0]; + path = information.serverAccessPaths[SizeSuffix.Original]; } // write the file specified by the path to the directory in the // zip file given by the prefix. diff --git a/src/server/ApiManagers/GooglePhotosManager.ts b/src/server/ApiManagers/GooglePhotosManager.ts index c7af69375..5a709688b 100644 --- a/src/server/ApiManagers/GooglePhotosManager.ts +++ b/src/server/ApiManagers/GooglePhotosManager.ts @@ -86,10 +86,10 @@ export default class GooglePhotosManager extends ApiManager { const contents: { mediaItems: MediaItem[] } = req.body; let failed = 0; if (contents) { - const completed: Opt[] = []; + const completed: Opt[] = []; for (let item of contents.mediaItems) { const { contentSize, ...attributes } = await DashUploadUtils.InspectImage(item.baseUrl); - const found: Opt = await Database.Auxiliary.QueryUploadHistory(contentSize!); + const found: Opt = await Database.Auxiliary.QueryUploadHistory(contentSize!); if (!found) { const upload = await DashUploadUtils.UploadInspectedImage({ contentSize, ...attributes }, item.filename, prefix).catch(error => _error(res, downloadError, error)); if (upload) { diff --git a/src/server/ApiManagers/PDFManager.ts b/src/server/ApiManagers/PDFManager.ts index 632b4965a..4bd750aaf 100644 --- a/src/server/ApiManagers/PDFManager.ts +++ b/src/server/ApiManagers/PDFManager.ts @@ -2,12 +2,12 @@ import ApiManager, { Registration } from "./ApiManager"; import { Method } from "../RouteManager"; import RouteSubscriber from "../RouteSubscriber"; import { exists, createReadStream, createWriteStream } from "fs"; -import { filesDirectory } from ".."; import * as Pdfjs from 'pdfjs-dist'; import { createCanvas } from "canvas"; const probe = require("probe-image-size"); import * as express from "express"; import * as path from "path"; +import { Directory, serverPathToFile, clientPathToFile } from "./UploadManager"; export default class PDFManager extends ApiManager { @@ -21,21 +21,27 @@ export default class PDFManager extends ApiManager { let noExt = filename.substring(0, filename.length - ".png".length); let pagenumber = parseInt(noExt.split('-')[1]); return new Promise(resolve => { - exists(filesDirectory + filename, (exists: boolean) => { - console.log(`${filesDirectory + filename} ${exists ? "exists" : "does not exist"}`); + const path = serverPathToFile(Directory.pdf_thumbnails, filename); + exists(path, (exists: boolean) => { + console.log(`${path} ${exists ? "exists" : "does not exist"}`); if (exists) { - let input = createReadStream(filesDirectory + filename); - probe(input, (err: any, result: any) => { + let input = createReadStream(path); + probe(input, (err: any, { width, height }: any) => { if (err) { console.log(err); console.log(`error on ${filename}`); return; } - res.send({ path: "/files/" + filename, width: result.width, height: result.height }); + res.send({ + path: clientPathToFile(Directory.pdf_thumbnails, filename), + width, + height + }); }); } else { - LoadPage(filesDirectory + filename.substring(0, filename.length - noExt.split('-')[1].length - ".PNG".length - 1) + ".pdf", pagenumber, res); + const name = filename.substring(0, filename.length - noExt.split('-')[1].length - ".PNG".length - 1) + ".pdf"; + LoadPage(serverPathToFile(Directory.pdfs, name), pagenumber, res); } resolve(); }); @@ -55,8 +61,8 @@ export default class PDFManager extends ApiManager { let canvasAndContext = factory.create(viewport.width, viewport.height); let renderContext = { canvasContext: canvasAndContext.context, - viewport: viewport, - canvasFactory: factory + canvasFactory: factory, + viewport }; console.log("read " + pageNumber); @@ -64,13 +70,17 @@ export default class PDFManager extends ApiManager { .then(() => { console.log("saving " + pageNumber); let stream = canvasAndContext.canvas.createPNGStream(); - let pngFile = `${file.substring(0, file.length - ".pdf".length)}-${pageNumber}.PNG`; + let filenames = path.basename(file).split("."); + const pngFile = serverPathToFile(Directory.pdf_thumbnails, `${filenames[0]}-${pageNumber}.png`); let out = createWriteStream(pngFile); stream.pipe(out); out.on("finish", () => { console.log(`Success! Saved to ${pngFile}`); - let name = path.basename(pngFile); - res.send({ path: "/files/" + name, width: viewport.width, height: viewport.height }); + res.send({ + path: pngFile, + width: viewport.width, + height: viewport.height + }); }); }, (reason: string) => { console.error(reason + ` ${pageNumber}`); diff --git a/src/server/ApiManagers/SearchManager.ts b/src/server/ApiManagers/SearchManager.ts index 1c801715a..d3f8995b0 100644 --- a/src/server/ApiManagers/SearchManager.ts +++ b/src/server/ApiManagers/SearchManager.ts @@ -3,7 +3,7 @@ import { Method } from "../RouteManager"; import { Search } from "../Search"; var findInFiles = require('find-in-files'); import * as path from 'path'; -import { filesDirectory } from ".."; +import { pathToDirectory, Directory } from "./UploadManager"; export default class SearchManager extends ApiManager { @@ -18,7 +18,7 @@ export default class SearchManager extends ApiManager { res.send([]); return; } - let results = await findInFiles.find({ 'term': q, 'flags': 'ig' }, filesDirectory + "text", ".txt$"); + let results = await findInFiles.find({ 'term': q, 'flags': 'ig' }, pathToDirectory(Directory.text), ".txt$"); let resObj: { ids: string[], numFound: number, lines: string[] } = { ids: [], numFound: 0, lines: [] }; for (var result in results) { resObj.ids.push(path.basename(result, ".txt").replace(/upload_/, "")); diff --git a/src/server/ApiManagers/UploadManager.ts b/src/server/ApiManagers/UploadManager.ts index 2a9faacd8..2f76871a6 100644 --- a/src/server/ApiManagers/UploadManager.ts +++ b/src/server/ApiManagers/UploadManager.ts @@ -7,14 +7,33 @@ import { extname, basename, dirname } from 'path'; import { createReadStream, createWriteStream, unlink, readFileSync } from "fs"; import { publicDirectory, filesDirectory } from ".."; import { Database } from "../database"; -import { DashUploadUtils } from "../DashUploadUtils"; -import { Opt } from "../../new_fields/Doc"; -import { ParsedPDF } from "../PdfTypes"; -const pdf = require('pdf-parse'); +import { DashUploadUtils, SizeSuffix } from "../DashUploadUtils"; import * as sharp from 'sharp'; -import { SharedMediaTypes } from "../SharedMediaTypes"; +import { AcceptibleMedia } from "../SharedMediaTypes"; +import { normalize } from "path"; const imageDataUri = require('image-data-uri'); +export enum Directory { + parsed_files = "parsed_files", + images = "images", + videos = "videos", + pdfs = "pdfs", + text = "text", + pdf_thumbnails = "pdf_thumbnails" +} + +export function serverPathToFile(directory: Directory, filename: string) { + return normalize(`${filesDirectory}/${directory}/${filename}`); +} + +export function pathToDirectory(directory: Directory) { + return normalize(`${filesDirectory}/${directory}`); +} + +export function clientPathToFile(directory: Directory, filename: string) { + return `/files/${directory}/${filename}`; +} + export default class UploadManager extends ApiManager { protected initialize(register: Registration): void { @@ -129,13 +148,14 @@ export default class UploadManager extends ApiManager { subscription: "/upload", onValidation: async ({ req, res }) => { let form = new formidable.IncomingForm(); - form.uploadDir = filesDirectory; + form.uploadDir = pathToDirectory(Directory.parsed_files); form.keepExtensions = true; return new Promise(resolve => { form.parse(req, async (_err, _fields, files) => { let results: any[] = []; for (const key in files) { - results.push(DashUploadUtils.upload(files[key])); + const result = await DashUploadUtils.upload(files[key]); + result && results.push(result); } _success(res, results); resolve(); @@ -150,8 +170,8 @@ export default class UploadManager extends ApiManager { onValidation: async ({ req, res }) => { const { source } = req.body; if (typeof source === "string") { - const uploadInformation = await DashUploadUtils.UploadImage(source); - return res.send(await DashUploadUtils.InspectImage(uploadInformation.mediaPaths[0])); + const { serverAccessPaths } = await DashUploadUtils.UploadImage(source); + return res.send(await DashUploadUtils.InspectImage(serverAccessPaths[SizeSuffix.Original])); } res.send({}); } @@ -167,9 +187,9 @@ export default class UploadManager extends ApiManager { res.status(401).send("incorrect parameters specified"); return; } - return imageDataUri.outputFile(uri, filesDirectory + filename).then((savedName: string) => { + return imageDataUri.outputFile(uri, serverPathToFile(Directory.images, filename)).then((savedName: string) => { const ext = extname(savedName).toLowerCase(); - const { pngs, jpgs } = SharedMediaTypes; + const { pngs, jpgs } = AcceptibleMedia; let resizers = [ { resizer: sharp().resize(100, undefined, { withoutEnlargement: true }), suffix: "_s" }, { resizer: sharp().resize(400, undefined, { withoutEnlargement: true }), suffix: "_m" }, @@ -189,10 +209,11 @@ export default class UploadManager extends ApiManager { } if (isImage) { resizers.forEach(resizer => { - createReadStream(savedName).pipe(resizer.resizer).pipe(createWriteStream(filesDirectory + filename + resizer.suffix + ext)); + const path = serverPathToFile(Directory.images, filename + resizer.suffix + ext); + createReadStream(savedName).pipe(resizer.resizer).pipe(createWriteStream(path)); }); } - res.send("/files/" + filename + ext); + res.send(clientPathToFile(Directory.images, filename + ext)); }); } }); diff --git a/src/server/DashUploadUtils.ts b/src/server/DashUploadUtils.ts index 839aada4b..0a670ec01 100644 --- a/src/server/DashUploadUtils.ts +++ b/src/server/DashUploadUtils.ts @@ -5,18 +5,32 @@ import * as sharp from 'sharp'; import request = require('request-promise'); import { ExifData, ExifImage } from 'exif'; import { Opt } from '../new_fields/Doc'; -import { SharedMediaTypes } from './SharedMediaTypes'; +import { AcceptibleMedia } from './SharedMediaTypes'; import { filesDirectory } from '.'; import { File } from 'formidable'; -import { extname, basename } from "path"; +import { basename } from "path"; +import { ConsoleColors, createIfNotExists } from './ActionUtilities'; +import { ParsedPDF } from "../server/PdfTypes"; +const parse = require('pdf-parse'); +import { Directory, serverPathToFile, clientPathToFile } from './ApiManagers/UploadManager'; -const uploadDirectory = path.join(__dirname, './public/files/'); +export enum SizeSuffix { + Small = "_s", + Medium = "_m", + Large = "_l", + Original = "_o" +} export namespace DashUploadUtils { + function InjectSize(filename: string, size: SizeSuffix) { + const extension = path.extname(filename).toLowerCase(); + return filename.substring(0, filename.length - extension.length) + size + extension; + } + export interface Size { width: number; - suffix: string; + suffix: SizeSuffix; } export interface ImageFileResponse { @@ -27,215 +41,221 @@ export namespace DashUploadUtils { } export const Sizes: { [size: string]: Size } = { - SMALL: { width: 100, suffix: "_s" }, - MEDIUM: { width: 400, suffix: "_m" }, - LARGE: { width: 900, suffix: "_l" }, + SMALL: { width: 100, suffix: SizeSuffix.Small }, + MEDIUM: { width: 400, suffix: SizeSuffix.Medium }, + LARGE: { width: 900, suffix: SizeSuffix.Large }, }; export function validateExtension(url: string) { - return SharedMediaTypes.imageFormats.includes(path.extname(url).toLowerCase()); + return AcceptibleMedia.imageFormats.includes(path.extname(url).toLowerCase()); } const size = "content-length"; const type = "content-type"; - export interface UploadInformation { - mediaPaths: string[]; - fileNames: { [key: string]: string }; + export interface ImageUploadInformation { + clientAccessPath: string; + serverAccessPaths: { [key: string]: string }; exifData: EnrichedExifData; contentSize?: number; contentType?: string; } - export function upload(file: File): any { + export async function upload(file: File): Promise { const { type, path, name } = file; - const filename = basename(path); - const extension = extname(path).toLowerCase(); - if (extension === ".pdf") { - - } else if { - let partition: Opt; - if(imageFormats.includes(extension)) { - partition = DashUploadUtils.Partitions.images; - } else if (videoFormats.includes(extension)) { - partition = DashUploadUtils.Partitions.videos; - } - let uploadInformation: Opt; - if (partition) { - uploadInformation = await DashUploadUtils.UploadImage(`${filesDirectory}/${partition}/${filename}`, filename); - } else { - console.log(`Unable to accommodate, and ignored, the following file upload: ${filename}`); + const { imageFormats, videoFormats, applicationFormats } = AcceptibleMedia; + const types = type.split("/"); + + const category = types[0]; + const format = `.${types[1]}`; + + switch (category) { + case "image": + if (imageFormats.includes(format)) { + const { clientAccessPath } = await UploadImage(path, basename(path), format); + return { clientAccessPath, name, type }; + } + case "video": + if (videoFormats.includes(format)) { + return MoveParsedFile(path, Directory.videos); + } + case "application": + if (applicationFormats.includes(format)) { + return UploadPdf(path); + } } + console.log(ConsoleColors.Red, `Ignoring unsupported file ${name} with upload type (${type}).`); + return { clientAccessPath: undefined }; } - const exif = uploadInformation ? uploadInformation.exifData : undefined; - results.push({ name, type, path: `/files/${filename}`, exif }); -} + async function UploadPdf(absolutePath: string) { + let dataBuffer = fs.readFileSync(absolutePath); + const result: ParsedPDF = await parse(dataBuffer); + const parsedName = basename(absolutePath); + await new Promise((resolve, reject) => { + const textFilename = `${parsedName.substring(0, parsedName.length - 4)}.txt`; + const writeStream = fs.createWriteStream(serverPathToFile(Directory.text, textFilename)); + writeStream.write(result.text, error => error ? reject(error) : resolve()); + }); + return MoveParsedFile(absolutePath, Directory.pdfs); + } -const generate = (prefix: string, url: string) => `${prefix}upload_${Utils.GenerateGuid()}${sanitizeExtension(url)}`; -const sanitize = (filename: string) => filename.replace(/\s+/g, "_"); -const sanitizeExtension = (source: string) => { - let extension = path.extname(source); - extension = extension.toLowerCase(); - extension = extension.split("?")[0]; - return extension; -}; - -/** - * Uploads an image specified by the @param source to Dash's /public/files/ - * directory, and returns information generated during that upload - * - * @param {string} source is either the absolute path of an already uploaded image or - * the url of a remote image - * @param {string} filename dictates what to call the image. If not specified, - * the name {@param prefix}_upload_{GUID} - * @param {string} prefix is a string prepended to the generated image name in the - * event that @param filename is not specified - * - * @returns {UploadInformation} This method returns - * 1) the paths to the uploaded images (plural due to resizing) - * 2) the file name of each of the resized images - * 3) the size of the image, in bytes (4432130) - * 4) the content type of the image, i.e. image/(jpeg | png | ...) - */ -export const UploadImage = async (source: string, filename?: string, prefix: string = ""): Promise => { - const metadata = await InspectImage(source); - return UploadInspectedImage(metadata, filename, prefix); -}; - -export interface InspectionResults { - isLocal: boolean; - stream: any; - normalizedUrl: string; - exifData: EnrichedExifData; - contentSize?: number; - contentType?: string; -} + const generate = (prefix: string, url: string) => `${prefix}upload_${Utils.GenerateGuid()}${sanitizeExtension(url)}`; + const sanitizeExtension = (source: string) => { + let extension = path.extname(source); + extension = extension.toLowerCase(); + extension = extension.split("?")[0]; + return extension; + }; -export interface EnrichedExifData { - data: ExifData; - error?: string; -} + /** + * Uploads an image specified by the @param source to Dash's /public/files/ + * directory, and returns information generated during that upload + * + * @param {string} source is either the absolute path of an already uploaded image or + * the url of a remote image + * @param {string} filename dictates what to call the image. If not specified, + * the name {@param prefix}_upload_{GUID} + * @param {string} prefix is a string prepended to the generated image name in the + * event that @param filename is not specified + * + * @returns {ImageUploadInformation} This method returns + * 1) the paths to the uploaded images (plural due to resizing) + * 2) the file name of each of the resized images + * 3) the size of the image, in bytes (4432130) + * 4) the content type of the image, i.e. image/(jpeg | png | ...) + */ + export const UploadImage = async (source: string, filename?: string, format?: string, prefix: string = ""): Promise => { + const metadata = await InspectImage(source); + return UploadInspectedImage(metadata, filename, format, prefix); + }; -export enum Partitions { - pdf_text = "pdf_text", - images = "images", - videos = "videos" -} + export interface InspectionResults { + isLocal: boolean; + stream: any; + normalizedUrl: string; + exifData: EnrichedExifData; + contentSize?: number; + contentType?: string; + } -export async function buildFilePartitions() { - const pending = Object.keys(Partitions).map(sub => createIfNotExists(filesDirectory + sub)); - return Promise.all(pending); -} + export interface EnrichedExifData { + data: ExifData; + error?: string; + } -/** - * Based on the url's classification as local or remote, gleans - * as much information as possible about the specified image - * - * @param source is the path or url to the image in question - */ -export const InspectImage = async (source: string): Promise => { - const { isLocal, stream, normalized: normalizedUrl } = classify(source); - const exifData = await parseExifData(source); - const results = { - exifData, - isLocal, - stream, - normalizedUrl - }; - // stop here if local, since request.head() can't handle local paths, only urls on the web - if (isLocal) { - return results; + export async function buildFileDirectories() { + const pending = Object.keys(Directory).map(sub => createIfNotExists(`${filesDirectory}/${sub}`)); + return Promise.all(pending); } - const metadata = (await new Promise((resolve, reject) => { - request.head(source, async (error, res) => { - if (error) { - return reject(error); - } - resolve(res); - }); - })).headers; - return { - contentSize: parseInt(metadata[size]), - contentType: metadata[type], - ...results - }; -}; - -export const UploadInspectedImage = async (metadata: InspectionResults, filename?: string, prefix = ""): Promise => { - const { isLocal, stream, normalizedUrl, contentSize, contentType, exifData } = metadata; - const resolved = filename ? sanitize(filename) : generate(prefix, normalizedUrl); - const extension = sanitizeExtension(normalizedUrl || resolved); - let information: UploadInformation = { - mediaPaths: [], - fileNames: { clean: resolved }, - exifData, - contentSize, - contentType, - }; - const { pngs, jpgs } = SharedMediaTypes; - return new Promise(async (resolve, reject) => { - const resizers = [ - { resizer: sharp().rotate(), suffix: "_o" }, - ...Object.values(Sizes).map(size => ({ - resizer: sharp().resize(size.width, undefined, { withoutEnlargement: true }).rotate(), - suffix: size.suffix - })) - ]; - if (pngs.includes(extension)) { - resizers.forEach(element => element.resizer = element.resizer.png()); - } else if (jpgs.includes(extension)) { - resizers.forEach(element => element.resizer = element.resizer.jpeg()); - } - for (let resizer of resizers) { - const suffix = resizer.suffix; - let mediaPath: string; - await new Promise(resolve => { - const filename = resolved.substring(0, resolved.length - extension.length) + suffix + extension; - information.mediaPaths.push(mediaPath = uploadDirectory + filename); - information.fileNames[suffix] = filename; - stream(normalizedUrl).pipe(resizer.resizer).pipe(fs.createWriteStream(mediaPath)) - .on('close', resolve) - .on('error', reject); - }); + + /** + * Based on the url's classification as local or remote, gleans + * as much information as possible about the specified image + * + * @param source is the path or url to the image in question + */ + export const InspectImage = async (source: string): Promise => { + const { isLocal, stream, normalized: normalizedUrl } = classify(source); + const exifData = await parseExifData(source); + const results = { + exifData, + isLocal, + stream, + normalizedUrl + }; + // stop here if local, since request.head() can't handle local paths, only urls on the web + if (isLocal) { + return results; } - if (!isLocal) { - await new Promise(resolve => { - stream(normalizedUrl).pipe(fs.createWriteStream(uploadDirectory + resolved)).on('close', resolve); + const metadata = (await new Promise((resolve, reject) => { + request.head(source, async (error, res) => { + if (error) { + return reject(error); + } + resolve(res); }); - } - resolve(information); - }); -}; - -const classify = (url: string) => { - const isLocal = /Dash-Web(\\|\/)src(\\|\/)server(\\|\/)public(\\|\/)files/g.test(url); - return { - isLocal, - stream: isLocal ? fs.createReadStream : request, - normalized: isLocal ? path.normalize(url) : url + })).headers; + return { + contentSize: parseInt(metadata[size]), + contentType: metadata[type], + ...results + }; }; -}; - -const parseExifData = async (source: string): Promise => { - return new Promise(resolve => { - new ExifImage(source, (error, data) => { - let reason: Opt = undefined; - if (error) { - reason = (error as any).code; + + export async function MoveParsedFile(absolutePath: string, destination: Directory): Promise<{ clientAccessPath: Opt }> { + return new Promise<{ clientAccessPath: Opt }>(resolve => { + const filename = basename(absolutePath); + const destinationPath = serverPathToFile(destination, filename); + fs.rename(absolutePath, destinationPath, error => { + resolve({ clientAccessPath: error ? undefined : clientPathToFile(destination, filename) }); + }); + }); + } + + export const UploadInspectedImage = async (metadata: InspectionResults, filename?: string, format?: string, prefix = ""): Promise => { + const { isLocal, stream, normalizedUrl, contentSize, contentType, exifData } = metadata; + const resolved = filename || generate(prefix, normalizedUrl); + const extension = format || sanitizeExtension(normalizedUrl || resolved); + let information: ImageUploadInformation = { + clientAccessPath: clientPathToFile(Directory.images, resolved), + serverAccessPaths: {}, + exifData, + contentSize, + contentType, + }; + const { pngs, jpgs } = AcceptibleMedia; + return new Promise(async (resolve, reject) => { + const resizers = [ + { resizer: sharp().rotate(), suffix: SizeSuffix.Original }, + ...Object.values(Sizes).map(size => ({ + resizer: sharp().resize(size.width, undefined, { withoutEnlargement: true }).rotate(), + suffix: size.suffix + })) + ]; + if (pngs.includes(extension)) { + resizers.forEach(element => element.resizer = element.resizer.png()); + } else if (jpgs.includes(extension)) { + resizers.forEach(element => element.resizer = element.resizer.jpeg()); + } + for (let { resizer, suffix } of resizers) { + let mediaPath: string; + await new Promise(resolve => { + const filename = InjectSize(resolved, suffix); + information.serverAccessPaths[suffix] = serverPathToFile(Directory.images, filename); + stream(normalizedUrl).pipe(resizer).pipe(fs.createWriteStream(serverPathToFile(Directory.images, filename))) + .on('close', resolve) + .on('error', reject); + }); } - resolve({ data, error: reason }); + if (isLocal) { + await new Promise(resolve => { + fs.unlink(normalizedUrl, error => resolve(error === null)); + }); + } + resolve(information); }); - }); -}; + }; -export const createIfNotExists = async (path: string) => { - if (await new Promise(resolve => fs.exists(path, resolve))) { - return true; - } - return new Promise(resolve => fs.mkdir(path, error => resolve(error === null))); -}; + const classify = (url: string) => { + const isLocal = /Dash-Web(\\|\/)src(\\|\/)server(\\|\/)public(\\|\/)files/g.test(url); + return { + isLocal, + stream: isLocal ? fs.createReadStream : request, + normalized: isLocal ? path.normalize(url) : url + }; + }; -export const Destroy = (mediaPath: string) => new Promise(resolve => fs.unlink(mediaPath, error => resolve(error === null))); + const parseExifData = async (source: string): Promise => { + return new Promise(resolve => { + new ExifImage(source, (error, data) => { + let reason: Opt = undefined; + if (error) { + reason = (error as any).code; + } + resolve({ data, error: reason }); + }); + }); + }; } \ No newline at end of file diff --git a/src/server/SharedMediaTypes.ts b/src/server/SharedMediaTypes.ts index 3d3234125..8d0f441f0 100644 --- a/src/server/SharedMediaTypes.ts +++ b/src/server/SharedMediaTypes.ts @@ -1,9 +1,8 @@ -export namespace SharedMediaTypes { - +export namespace AcceptibleMedia { export const gifs = [".gif"]; export const pngs = [".png"]; export const jpgs = [".jpg", ".jpeg"]; export const imageFormats = [...pngs, ...jpgs, ...gifs]; export const videoFormats = [".mov", ".mp4"]; - + export const applicationFormats = [".pdf"]; } \ No newline at end of file diff --git a/src/server/database.ts b/src/server/database.ts index b81fc03a4..db81245c1 100644 --- a/src/server/database.ts +++ b/src/server/database.ts @@ -293,7 +293,7 @@ export namespace Database { }; export const QueryUploadHistory = async (contentSize: number) => { - return SanitizedSingletonQuery({ contentSize }, AuxiliaryCollections.GooglePhotosUploadHistory); + return SanitizedSingletonQuery({ contentSize }, AuxiliaryCollections.GooglePhotosUploadHistory); }; export namespace GoogleAuthenticationToken { @@ -322,7 +322,7 @@ export namespace Database { } - export const LogUpload = async (information: DashUploadUtils.UploadInformation) => { + export const LogUpload = async (information: DashUploadUtils.ImageUploadInformation) => { const bundle = { _id: Utils.GenerateDeterministicGuid(String(information.contentSize!)), ...information diff --git a/src/server/index.ts b/src/server/index.ts index d02a6005e..d77923710 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -24,7 +24,7 @@ import GooglePhotosManager from "./ApiManagers/GooglePhotosManager"; import DiagnosticManager from "./ApiManagers/DiagnosticManager"; export const publicDirectory = path.resolve(__dirname, "public"); -export const filesDirectory = path.resolve(publicDirectory, "files") + "/"; +export const filesDirectory = path.resolve(publicDirectory, "files"); /** * These are the functions run before the server starts @@ -34,7 +34,7 @@ export const filesDirectory = path.resolve(publicDirectory, "files") + "/"; async function preliminaryFunctions() { await GoogleCredentialsLoader.loadCredentials(); GoogleApiServerUtils.processProjectCredentials(); - await DashUploadUtils.buildFilePartitions(); + await DashUploadUtils.buildFileDirectories(); await log_execution({ startMessage: "attempting to initialize mongodb connection", endMessage: "connection outcome determined", -- cgit v1.2.3-70-g09d2