From 0b72a27ead9d1e933ae349b8a3e9e9b8702664d1 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Sat, 9 Nov 2019 16:18:23 -0500 Subject: factored out all but google resources into managers --- src/server/ApiManagers/PDFManager.ts | 107 +++++++++++++++++++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100644 src/server/ApiManagers/PDFManager.ts (limited to 'src/server/ApiManagers/PDFManager.ts') diff --git a/src/server/ApiManagers/PDFManager.ts b/src/server/ApiManagers/PDFManager.ts new file mode 100644 index 000000000..f328557b4 --- /dev/null +++ b/src/server/ApiManagers/PDFManager.ts @@ -0,0 +1,107 @@ +import ApiManager, { Registration } from "./ApiManager"; +import { Method } from "../RouteManager"; +import RouteSubscriber from "../RouteSubscriber"; +import { exists, createReadStream, createWriteStream } from "fs"; +import { filesDirectory } from ".."; +import * as Pdfjs from 'pdfjs-dist'; +import { createCanvas } from "canvas"; +const probe = require("probe-image-size"); +import * as express from "express"; +import * as path from "path"; + +export default class PDFManager extends ApiManager { + + protected initialize(register: Registration): void { + + register({ + method: Method.GET, + subscription: new RouteSubscriber("/thumbnail").add("filename"), + onValidation: ({ req, res }) => { + let filename = req.params.filename; + let noExt = filename.substring(0, filename.length - ".png".length); + let pagenumber = parseInt(noExt.split('-')[1]); + return new Promise(resolve => { + exists(filesDirectory + filename, (exists: boolean) => { + console.log(`${filesDirectory + filename} ${exists ? "exists" : "does not exist"}`); + if (exists) { + let input = createReadStream(filesDirectory + filename); + probe(input, (err: any, result: any) => { + if (err) { + console.log(err); + console.log(`error on ${filename}`); + return; + } + res.send({ path: "/files/" + filename, width: result.width, height: result.height }); + }); + } + else { + LoadPage(filesDirectory + filename.substring(0, filename.length - noExt.split('-')[1].length - ".PNG".length - 1) + ".pdf", pagenumber, res); + } + resolve(); + }); + }); + } + }); + + function LoadPage(file: string, pageNumber: number, res: express.Response) { + console.log(file); + Pdfjs.getDocument(file).promise + .then((pdf: Pdfjs.PDFDocumentProxy) => { + let factory = new NodeCanvasFactory(); + console.log(pageNumber); + pdf.getPage(pageNumber).then((page: Pdfjs.PDFPageProxy) => { + console.log("reading " + page); + let viewport = page.getViewport(1 as any); + let canvasAndContext = factory.create(viewport.width, viewport.height); + let renderContext = { + canvasContext: canvasAndContext.context, + viewport: viewport, + canvasFactory: factory + }; + console.log("read " + pageNumber); + + page.render(renderContext).promise + .then(() => { + console.log("saving " + pageNumber); + let stream = canvasAndContext.canvas.createPNGStream(); + let pngFile = `${file.substring(0, file.length - ".pdf".length)}-${pageNumber}.PNG`; + let out = createWriteStream(pngFile); + stream.pipe(out); + out.on("finish", () => { + console.log(`Success! Saved to ${pngFile}`); + let name = path.basename(pngFile); + res.send({ path: "/files/" + name, width: viewport.width, height: viewport.height }); + }); + }, (reason: string) => { + console.error(reason + ` ${pageNumber}`); + }); + }); + }); + } + + } + +} + +class NodeCanvasFactory { + create = (width: number, height: number) => { + var canvas = createCanvas(width, height); + var context = canvas.getContext('2d'); + return { + canvas, + context + }; + } + + reset = (canvasAndContext: any, width: number, height: number) => { + canvasAndContext.canvas.width = width; + canvasAndContext.canvas.height = height; + } + + destroy = (canvasAndContext: any) => { + canvasAndContext.canvas.width = 0; + canvasAndContext.canvas.height = 0; + canvasAndContext.canvas = null; + canvasAndContext.context = null; + } +} \ No newline at end of file -- cgit v1.2.3-70-g09d2 From 36ad83493d2bd58dc6fe62df6002789ccc1b06a1 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Sun, 10 Nov 2019 14:56:58 -0500 Subject: no more RouteStore --- src/Utils.ts | 3 +- src/client/apis/GoogleAuthenticationManager.tsx | 5 +-- .../apis/google_docs/GoogleApiClientUtils.ts | 7 ++-- .../apis/google_docs/GooglePhotosClientUtils.ts | 5 +-- src/client/cognitive_services/CognitiveServices.ts | 3 +- src/client/util/History.ts | 3 +- .../util/Import & Export/DirectoryImportBox.tsx | 3 +- src/client/util/Import & Export/ImageUtils.ts | 7 ++-- src/client/util/SharingManager.tsx | 3 +- src/client/views/MainView.tsx | 5 +-- src/client/views/collections/CollectionSubView.tsx | 4 +- src/client/views/nodes/ImageBox.tsx | 3 +- src/client/views/nodes/VideoBox.tsx | 3 +- src/client/views/search/SearchBox.tsx | 3 +- src/mobile/ImageUpload.tsx | 3 +- src/new_fields/RichTextUtils.ts | 3 +- src/server/ApiManagers/DeleteManager.ts | 13 +++---- src/server/ApiManagers/ExportManager.ts | 5 +-- src/server/ApiManagers/PDFManager.ts | 2 +- src/server/ApiManagers/UploadManager.ts | 15 +++----- src/server/ApiManagers/UserManager.ts | 7 ++-- src/server/ApiManagers/UtilManager.ts | 7 ++++ src/server/Initialization.ts | 28 +++++++------- src/server/RouteManager.ts | 5 +-- src/server/RouteStore.ts | 45 ---------------------- src/server/RouteSubscriber.ts | 2 +- src/server/authentication/config/passport.ts | 5 +-- .../authentication/controllers/user_controller.ts | 32 +++++++-------- .../authentication/models/current_user_utils.ts | 7 ++-- src/server/index.ts | 26 ++++++------- 30 files changed, 96 insertions(+), 166 deletions(-) delete mode 100644 src/server/RouteStore.ts (limited to 'src/server/ApiManagers/PDFManager.ts') diff --git a/src/Utils.ts b/src/Utils.ts index 9a2f01f80..abff2eaba 100644 --- a/src/Utils.ts +++ b/src/Utils.ts @@ -2,7 +2,6 @@ import v4 = require('uuid/v4'); import v5 = require("uuid/v5"); import { Socket } from 'socket.io'; import { Message } from './server/Message'; -import { RouteStore } from './server/RouteStore'; export namespace Utils { @@ -46,7 +45,7 @@ export namespace Utils { } export function CorsProxy(url: string): string { - return prepend(RouteStore.corsProxy + "/") + encodeURIComponent(url); + return prepend("/corsProxy/") + encodeURIComponent(url); } export function CopyText(text: string) { diff --git a/src/client/apis/GoogleAuthenticationManager.tsx b/src/client/apis/GoogleAuthenticationManager.tsx index 1ec9d8412..ae77c4b7b 100644 --- a/src/client/apis/GoogleAuthenticationManager.tsx +++ b/src/client/apis/GoogleAuthenticationManager.tsx @@ -4,7 +4,6 @@ import * as React from "react"; import MainViewModal from "../views/MainViewModal"; import { Opt } from "../../new_fields/Doc"; import { Networking } from "../Network"; -import { RouteStore } from "../../server/RouteStore"; import "./GoogleAuthenticationManager.scss"; const AuthenticationUrl = "https://accounts.google.com/o/oauth2/v2/auth"; @@ -31,7 +30,7 @@ export default class GoogleAuthenticationManager extends React.Component<{}> { } public fetchOrGenerateAccessToken = async () => { - let response = await Networking.FetchFromServer(RouteStore.readGoogleAccessToken); + let response = await Networking.FetchFromServer("/readGoogleAccessToken"); // if this is an authentication url, activate the UI to register the new access token if (new RegExp(AuthenticationUrl).test(response)) { this.isOpen = true; @@ -44,7 +43,7 @@ export default class GoogleAuthenticationManager extends React.Component<{}> { return; } const { access_token, avatar, name } = await Networking.PostToServer( - RouteStore.writeGoogleAccessToken, + "/writeGoogleAccessToken", { authenticationCode } ); runInAction(() => { diff --git a/src/client/apis/google_docs/GoogleApiClientUtils.ts b/src/client/apis/google_docs/GoogleApiClientUtils.ts index 183679317..26c7f8d2e 100644 --- a/src/client/apis/google_docs/GoogleApiClientUtils.ts +++ b/src/client/apis/google_docs/GoogleApiClientUtils.ts @@ -1,5 +1,4 @@ import { docs_v1, slides_v1 } from "googleapis"; -import { RouteStore } from "../../../server/RouteStore"; import { Opt } from "../../../new_fields/Doc"; import { isArray } from "util"; import { EditorState } from "prosemirror-state"; @@ -77,7 +76,7 @@ export namespace GoogleApiClientUtils { * @returns the documentId of the newly generated document, or undefined if the creation process fails. */ export const create = async (options: CreateOptions): Promise => { - const path = `${RouteStore.googleDocs}/Documents/${Actions.Create}`; + const path = `/googleDocs/Documents/${Actions.Create}`; const parameters = { requestBody: { title: options.title || `Dash Export (${new Date().toDateString()})` @@ -154,7 +153,7 @@ export namespace GoogleApiClientUtils { } export const retrieve = async (options: RetrieveOptions): Promise => { - const path = `${RouteStore.googleDocs}/Documents/${Actions.Retrieve}`; + const path = `/googleDocs/Documents/${Actions.Retrieve}`; try { const parameters = { documentId: options.documentId }; const schema: RetrievalResult = await Networking.PostToServer(path, parameters); @@ -165,7 +164,7 @@ export namespace GoogleApiClientUtils { }; export const update = async (options: UpdateOptions): Promise => { - const path = `${RouteStore.googleDocs}/Documents/${Actions.Update}`; + const path = `/googleDocs/Documents/${Actions.Update}`; const parameters = { documentId: options.documentId, requestBody: { diff --git a/src/client/apis/google_docs/GooglePhotosClientUtils.ts b/src/client/apis/google_docs/GooglePhotosClientUtils.ts index 402fc64b5..bf8897061 100644 --- a/src/client/apis/google_docs/GooglePhotosClientUtils.ts +++ b/src/client/apis/google_docs/GooglePhotosClientUtils.ts @@ -1,5 +1,4 @@ import { Utils } from "../../../Utils"; -import { RouteStore } from "../../../server/RouteStore"; import { ImageField } from "../../../new_fields/URLField"; import { Cast, StrCast } from "../../../new_fields/Types"; import { Doc, Opt, DocListCastAsync } from "../../../new_fields/Doc"; @@ -307,7 +306,7 @@ export namespace GooglePhotos { }; export const WriteMediaItemsToServer = async (body: { mediaItems: any[] }): Promise => { - const uploads = await Networking.PostToServer(RouteStore.googlePhotosMediaDownload, body); + const uploads = await Networking.PostToServer("/googlePhotosMediaDownload", body); return uploads; }; @@ -345,7 +344,7 @@ export namespace GooglePhotos { media.push({ url, description }); } if (media.length) { - const results = await Networking.PostToServer(RouteStore.googlePhotosMediaUpload, { media, album }); + const results = await Networking.PostToServer("/googlePhotosMediaUpload", { media, album }); return results; } }; diff --git a/src/client/cognitive_services/CognitiveServices.ts b/src/client/cognitive_services/CognitiveServices.ts index 08fcb4883..af5fb39fc 100644 --- a/src/client/cognitive_services/CognitiveServices.ts +++ b/src/client/cognitive_services/CognitiveServices.ts @@ -2,7 +2,6 @@ import * as request from "request-promise"; import { Doc, Field, Opt } from "../../new_fields/Doc"; import { Cast } from "../../new_fields/Types"; import { Docs } from "../documents/Documents"; -import { RouteStore } from "../../server/RouteStore"; import { Utils } from "../../Utils"; import { InkData } from "../../new_fields/InkField"; import { UndoManager } from "../util/UndoManager"; @@ -39,7 +38,7 @@ export enum Confidence { export namespace CognitiveServices { const ExecuteQuery = async (service: Service, manager: APIManager, data: D): Promise => { - return fetch(Utils.prepend(`${RouteStore.cognitiveServices}/${service}`)).then(async response => { + return fetch(Utils.prepend(`cognitiveServices/${service}`)).then(async response => { let apiKey = await response.text(); if (!apiKey) { console.log(`No API key found for ${service}: ensure index.ts has access to a .env file in your root directory`); diff --git a/src/client/util/History.ts b/src/client/util/History.ts index 899abbe40..1c51236cb 100644 --- a/src/client/util/History.ts +++ b/src/client/util/History.ts @@ -1,6 +1,5 @@ import { Doc, Opt, Field } from "../../new_fields/Doc"; import { DocServer } from "../DocServer"; -import { RouteStore } from "../../server/RouteStore"; import { MainView } from "../views/MainView"; import * as qs from 'query-string'; import { Utils, OmitKeys } from "../../Utils"; @@ -26,7 +25,7 @@ export namespace HistoryUtil { // const handlers: ((state: ParsedUrl | null) => void)[] = []; function onHistory(e: PopStateEvent) { - if (window.location.pathname !== RouteStore.home) { + if (window.location.pathname !== "/home") { const url = e.state as ParsedUrl || parseUrl(window.location); if (url) { switch (url.type) { diff --git a/src/client/util/Import & Export/DirectoryImportBox.tsx b/src/client/util/Import & Export/DirectoryImportBox.tsx index 2e0ba25eb..437e7766b 100644 --- a/src/client/util/Import & Export/DirectoryImportBox.tsx +++ b/src/client/util/Import & Export/DirectoryImportBox.tsx @@ -1,7 +1,6 @@ import "fs"; import React = require("react"); import { Doc, DocListCast, DocListCastAsync, Opt } from "../../../new_fields/Doc"; -import { RouteStore } from "../../../server/RouteStore"; import { action, observable, autorun, runInAction, computed, reaction, IReactionDisposer } from "mobx"; import { FieldViewProps, FieldView } from "../../views/nodes/FieldView"; import Measure, { ContentRect } from "react-measure"; @@ -124,7 +123,7 @@ export default class DirectoryImportBox extends React.Component formData.append(Utils.GenerateGuid(), file); }); - collector.push(...(await Networking.PostFormDataToServer(RouteStore.upload, formData))); + collector.push(...(await Networking.PostFormDataToServer("/upload", formData))); runInAction(() => this.completed += batch.length); }); diff --git a/src/client/util/Import & Export/ImageUtils.ts b/src/client/util/Import & Export/ImageUtils.ts index 914f4870a..ca80f3bca 100644 --- a/src/client/util/Import & Export/ImageUtils.ts +++ b/src/client/util/Import & Export/ImageUtils.ts @@ -1,7 +1,6 @@ -import { Doc, DocListCast, DocListCastAsync, Opt } from "../../../new_fields/Doc"; +import { Doc } from "../../../new_fields/Doc"; import { ImageField } from "../../../new_fields/URLField"; import { Cast, StrCast } from "../../../new_fields/Types"; -import { RouteStore } from "../../../server/RouteStore"; import { Docs } from "../../documents/Documents"; import { Networking } from "../../Network"; import { Id } from "../../../new_fields/FieldSymbols"; @@ -15,7 +14,7 @@ export namespace ImageUtils { return false; } const source = field.url.href; - const response = await Networking.PostToServer(RouteStore.inspectImage, { source }); + const response = await Networking.PostToServer("/inspectImage", { source }); const { error, data } = response.exifData; document.exif = error || Docs.Get.DocumentHierarchyFromJson(data); return data !== undefined; @@ -23,7 +22,7 @@ export namespace ImageUtils { export const ExportHierarchyToFileSystem = async (collection: Doc): Promise => { const a = document.createElement("a"); - a.href = Utils.prepend(`${RouteStore.imageHierarchyExport}/${collection[Id]}`); + a.href = Utils.prepend(`imageHierarchyExport/${collection[Id]}`); a.download = `Dash Export [${StrCast(collection.title)}].zip`; a.click(); }; diff --git a/src/client/util/SharingManager.tsx b/src/client/util/SharingManager.tsx index 2082d6324..cc1d628b1 100644 --- a/src/client/util/SharingManager.tsx +++ b/src/client/util/SharingManager.tsx @@ -4,7 +4,6 @@ import MainViewModal from "../views/MainViewModal"; import { Doc, Opt, DocCastAsync } from "../../new_fields/Doc"; import { DocServer } from "../DocServer"; import { Cast, StrCast } from "../../new_fields/Types"; -import { RouteStore } from "../../server/RouteStore"; import * as RequestPromise from "request-promise"; import { Utils } from "../../Utils"; import "./SharingManager.scss"; @@ -104,7 +103,7 @@ export default class SharingManager extends React.Component<{}> { } populateUsers = async () => { - let userList = await RequestPromise.get(Utils.prepend(RouteStore.getUsers)); + let userList = await RequestPromise.get(Utils.prepend("/getUsers")); const raw = JSON.parse(userList) as User[]; const evaluating = raw.map(async user => { let isCandidate = user.email !== Doc.CurrentUserEmail; diff --git a/src/client/views/MainView.tsx b/src/client/views/MainView.tsx index 39585113b..0c5a1003b 100644 --- a/src/client/views/MainView.tsx +++ b/src/client/views/MainView.tsx @@ -12,7 +12,6 @@ import { List } from '../../new_fields/List'; import { listSpec } from '../../new_fields/Schema'; import { Cast, FieldValue, StrCast } from '../../new_fields/Types'; import { CurrentUserUtils } from '../../server/authentication/models/current_user_utils'; -import { RouteStore } from '../../server/RouteStore'; import { emptyFunction, returnEmptyString, returnFalse, returnOne, returnTrue, Utils } from '../../Utils'; import GoogleAuthenticationManager from '../apis/GoogleAuthenticationManager'; import { DocServer } from '../DocServer'; @@ -74,7 +73,7 @@ export class MainView extends React.Component { this._urlState = HistoryUtil.parseUrl(window.location) || {} as any; // causes errors to be generated when modifying an observable outside of an action configure({ enforceActions: "observed" }); - if (window.location.pathname !== RouteStore.home) { + if (window.location.pathname !== "/home") { let pathname = window.location.pathname.substr(1).split("/"); if (pathname.length > 1) { let type = pathname[0]; @@ -395,7 +394,7 @@ export class MainView extends React.Component { zoomToScale={emptyFunction} getScale={returnOne}> - ; diff --git a/src/client/views/collections/CollectionSubView.tsx b/src/client/views/collections/CollectionSubView.tsx index 6e8e4fa12..306f8e052 100644 --- a/src/client/views/collections/CollectionSubView.tsx +++ b/src/client/views/collections/CollectionSubView.tsx @@ -8,7 +8,6 @@ import { listSpec } from "../../../new_fields/Schema"; import { ScriptField } from "../../../new_fields/ScriptField"; import { Cast } from "../../../new_fields/Types"; import { CurrentUserUtils } from "../../../server/authentication/models/current_user_utils"; -import { RouteStore } from "../../../server/RouteStore"; import { Utils } from "../../../Utils"; import { DocServer } from "../../DocServer"; import { DocumentType } from "../../documents/DocumentTypes"; @@ -243,7 +242,6 @@ export function CollectionSubView(schemaCtor: (doc: Doc) => T) { let promises: Promise[] = []; // tslint:disable-next-line:prefer-for-of for (let i = 0; i < e.dataTransfer.items.length; i++) { - const upload = window.location.origin + RouteStore.upload; let item = e.dataTransfer.items[i]; if (item.kind === "string" && item.type.indexOf("uri") !== -1) { let str: string; @@ -268,7 +266,7 @@ export function CollectionSubView(schemaCtor: (doc: Doc) => T) { } let dropFileName = file ? file.name : "-empty-"; - let prom = fetch(upload, { + let prom = fetch(Utils.prepend("/upload"), { method: 'POST', body: formData }).then(async (res: Response) => { diff --git a/src/client/views/nodes/ImageBox.tsx b/src/client/views/nodes/ImageBox.tsx index 9f39eccea..07fd832be 100644 --- a/src/client/views/nodes/ImageBox.tsx +++ b/src/client/views/nodes/ImageBox.tsx @@ -12,7 +12,6 @@ import { createSchema, listSpec, makeInterface } from '../../../new_fields/Schem import { ComputedField } from '../../../new_fields/ScriptField'; import { BoolCast, Cast, FieldValue, NumCast, StrCast } from '../../../new_fields/Types'; import { AudioField, ImageField } from '../../../new_fields/URLField'; -import { RouteStore } from '../../../server/RouteStore'; import { Utils, returnOne, emptyFunction } from '../../../Utils'; import { CognitiveServices, Confidence, Service, Tag } from '../../cognitive_services/CognitiveServices'; import { Docs } from '../../documents/Documents'; @@ -152,7 +151,7 @@ export class ImageBox extends DocAnnotatableComponent { if (isRelease) { return _permission_denied(res, deletionPermissionError); } await WebSocket.deleteFields(); - res.redirect(RouteStore.home); + res.redirect("/home"); } }); register({ method: Method.GET, - subscription: RouteStore.deleteAll, + subscription: "/deleteAll", onValidation: async ({ res, isRelease }) => { if (isRelease) { return _permission_denied(res, deletionPermissionError); } await WebSocket.deleteAll(); - res.redirect(RouteStore.home); + res.redirect("/home"); } }); @@ -41,7 +40,7 @@ export default class DeleteManager extends ApiManager { return _permission_denied(res, deletionPermissionError); } await Database.Auxiliary.DeleteAll(); - res.redirect(RouteStore.delete); + res.redirect("/delete"); } }); @@ -53,7 +52,7 @@ export default class DeleteManager extends ApiManager { return _permission_denied(res, deletionPermissionError); } await Database.Auxiliary.GoogleAuthenticationToken.DeleteAll(); - res.redirect(RouteStore.delete); + res.redirect("/delete"); } }); diff --git a/src/server/ApiManagers/ExportManager.ts b/src/server/ApiManagers/ExportManager.ts index d42db1056..fc6ba0d22 100644 --- a/src/server/ApiManagers/ExportManager.ts +++ b/src/server/ApiManagers/ExportManager.ts @@ -1,7 +1,6 @@ import ApiManager, { Registration } from "./ApiManager"; import { Method } from "../RouteManager"; import RouteSubscriber from "../RouteSubscriber"; -import { RouteStore } from "../RouteStore"; import * as Archiver from 'archiver'; import * as express from 'express'; import { Database } from "../database"; @@ -32,7 +31,7 @@ export default class DownloadManager extends ApiManager { */ register({ method: Method.GET, - subscription: new RouteSubscriber(RouteStore.imageHierarchyExport).add('docId'), + subscription: new RouteSubscriber("imageHierarchyExport").add('docId'), onValidation: async ({ req, res }) => { const id = req.params.docId; const hierarchy: Hierarchy = {}; @@ -43,7 +42,7 @@ export default class DownloadManager extends ApiManager { register({ method: Method.GET, - subscription: new RouteSubscriber("/downloadId").add("docId"), + subscription: new RouteSubscriber("downloadId").add("docId"), onValidation: async ({ req, res }) => { return BuildAndDispatchZip(res, async zip => { const { id, docs, files } = await getDocs(req.params.docId); diff --git a/src/server/ApiManagers/PDFManager.ts b/src/server/ApiManagers/PDFManager.ts index f328557b4..632b4965a 100644 --- a/src/server/ApiManagers/PDFManager.ts +++ b/src/server/ApiManagers/PDFManager.ts @@ -15,7 +15,7 @@ export default class PDFManager extends ApiManager { register({ method: Method.GET, - subscription: new RouteSubscriber("/thumbnail").add("filename"), + subscription: new RouteSubscriber("thumbnail").add("filename"), onValidation: ({ req, res }) => { let filename = req.params.filename; let noExt = filename.substring(0, filename.length - ".png".length); diff --git a/src/server/ApiManagers/UploadManager.ts b/src/server/ApiManagers/UploadManager.ts index 38635eda5..01abdab54 100644 --- a/src/server/ApiManagers/UploadManager.ts +++ b/src/server/ApiManagers/UploadManager.ts @@ -6,7 +6,6 @@ var AdmZip = require('adm-zip'); import * as path from 'path'; import { createReadStream, createWriteStream, unlink, readFileSync } from "fs"; import { publicDirectory, filesDirectory, Partitions } from ".."; -import { RouteStore } from "../RouteStore"; import { Database } from "../database"; import { DashUploadUtils } from "../DashUploadUtils"; import { Opt } from "../../new_fields/Doc"; @@ -85,12 +84,8 @@ export default class UploadManager extends ApiManager { let dirname = path.dirname(entry.entryName) + "/"; let extname = path.extname(entry.entryName); let basename = path.basename(entry.entryName).split(".")[0]; - // zip.extractEntryTo(dirname + basename + "_o" + extname, __dirname + RouteStore.public, true, false); - // zip.extractEntryTo(dirname + basename + "_s" + extname, __dirname + RouteStore.public, true, false); - // zip.extractEntryTo(dirname + basename + "_m" + extname, __dirname + RouteStore.public, true, false); - // zip.extractEntryTo(dirname + basename + "_l" + extname, __dirname + RouteStore.public, true, false); try { - zip.extractEntryTo(entry.entryName, __dirname + RouteStore.public, true, false); + zip.extractEntryTo(entry.entryName, publicDirectory, true, false); dirname = "/" + dirname; createReadStream(publicDirectory + dirname + basename + extname).pipe(createWriteStream(publicDirectory + dirname + basename + "_o" + extname)); @@ -131,7 +126,7 @@ export default class UploadManager extends ApiManager { register({ method: Method.POST, - subscription: RouteStore.upload, + subscription: "/upload", onValidation: async ({ req, res }) => { let form = new formidable.IncomingForm(); form.uploadDir = filesDirectory; @@ -147,7 +142,7 @@ export default class UploadManager extends ApiManager { let dataBuffer = readFileSync(filesDirectory + filename); const result: ParsedPDF = await pdf(dataBuffer); await new Promise((resolve, reject) => { - const path = filesDirectory + Partitions.PdfText + "/" + filename.substring(0, filename.length - ".pdf".length) + ".txt"; + const path = filesDirectory + Partitions.pdf_text + "/" + filename.substring(0, filename.length - ".pdf".length) + ".txt"; createWriteStream(path).write(result.text, error => { if (!error) { resolve(); @@ -171,7 +166,7 @@ export default class UploadManager extends ApiManager { register({ method: Method.POST, - subscription: RouteStore.inspectImage, + subscription: "/inspectImage", onValidation: async ({ req, res }) => { const { source } = req.body; if (typeof source === "string") { @@ -184,7 +179,7 @@ export default class UploadManager extends ApiManager { register({ method: Method.POST, - subscription: RouteStore.dataUriToImage, + subscription: "/uploadURI", onValidation: ({ req, res }) => { const uri = req.body.uri; const filename = req.body.name; diff --git a/src/server/ApiManagers/UserManager.ts b/src/server/ApiManagers/UserManager.ts index fe1ce7f2b..51a434fcf 100644 --- a/src/server/ApiManagers/UserManager.ts +++ b/src/server/ApiManagers/UserManager.ts @@ -1,7 +1,6 @@ import ApiManager, { Registration } from "./ApiManager"; import { Method } from "../RouteManager"; import { WebSocket } from "../Websocket/Websocket"; -import { RouteStore } from "../RouteStore"; import { Database } from "../database"; export default class UserManager extends ApiManager { @@ -10,7 +9,7 @@ export default class UserManager extends ApiManager { register({ method: Method.GET, - subscription: RouteStore.getUsers, + subscription: "/getUsers", onValidation: async ({ res }) => { const cursor = await Database.Instance.query({}, { email: 1, userDocumentId: 1 }, "users"); const results = await cursor.toArray(); @@ -20,13 +19,13 @@ export default class UserManager extends ApiManager { register({ method: Method.GET, - subscription: RouteStore.getUserDocumentId, + subscription: "/getUserDocumentId", onValidation: ({ res, user }) => res.send(user.userDocumentId) }); register({ method: Method.GET, - subscription: RouteStore.getCurrUser, + subscription: "/getCurrentUser", onValidation: ({ res, user }) => res.send(JSON.stringify(user)), onUnauthenticated: ({ res }) => res.send(JSON.stringify({ id: "__guest__", email: "" })) }); diff --git a/src/server/ApiManagers/UtilManager.ts b/src/server/ApiManagers/UtilManager.ts index 61cda2e9b..c1234be6c 100644 --- a/src/server/ApiManagers/UtilManager.ts +++ b/src/server/ApiManagers/UtilManager.ts @@ -2,11 +2,18 @@ import ApiManager, { Registration } from "./ApiManager"; import { Method } from "../RouteManager"; import { exec } from 'child_process'; import { command_line } from "../ActionUtilities"; +import RouteSubscriber from "../RouteSubscriber"; export default class UtilManager extends ApiManager { protected initialize(register: Registration): void { + register({ + method: Method.GET, + subscription: new RouteSubscriber("environment").add("key"), + onValidation: ({ req, res }) => res.send(process.env[req.params.key]) + }); + register({ method: Method.GET, subscription: "/pull", diff --git a/src/server/Initialization.ts b/src/server/Initialization.ts index fbb5ae7a6..306058d81 100644 --- a/src/server/Initialization.ts +++ b/src/server/Initialization.ts @@ -9,7 +9,6 @@ import flash = require('connect-flash'); import { Database } from './database'; import { getForgot, getLogin, getLogout, getReset, getSignup, postForgot, postLogin, postReset, postSignup } from './authentication/controllers/user_controller'; const MongoStore = require('connect-mongo')(session); -import { RouteStore } from './RouteStore'; import RouteManager from './RouteManager'; import * as webpack from 'webpack'; const config = require('../../webpack.config'); @@ -18,6 +17,8 @@ import * as wdm from 'webpack-dev-middleware'; import * as whm from 'webpack-hot-middleware'; import * as fs from 'fs'; import * as request from 'request'; +import RouteSubscriber from './RouteSubscriber'; +import { publicDirectory } from '.'; export type RouteSetter = (server: RouteManager) => void; export interface InitializationOptions { @@ -29,8 +30,8 @@ export default async function InitializeServer(options: InitializationOptions) { const { listenAtPort, routeSetter } = options; const server = buildWithMiddleware(express()); - server.use(express.static(__dirname + RouteStore.public)); - server.use(RouteStore.images, express.static(__dirname + RouteStore.public)); + server.use(express.static(publicDirectory)); + server.use("/images", express.static(publicDirectory)); server.use(wdm(compiler, { publicPath: config.output.publicPath })); server.use(whm(compiler)); @@ -87,24 +88,25 @@ function determineEnvironment() { } function registerAuthenticationRoutes(server: express.Express) { - server.get(RouteStore.signup, getSignup); - server.post(RouteStore.signup, postSignup); + server.get("/signup", getSignup); + server.post("/signup", postSignup); - server.get(RouteStore.login, getLogin); - server.post(RouteStore.login, postLogin); + server.get("/login", getLogin); + server.post("/login", postLogin); - server.get(RouteStore.logout, getLogout); + server.get("/logout", getLogout); - server.get(RouteStore.forgot, getForgot); - server.post(RouteStore.forgot, postForgot); + server.get("/forgotPassword", getForgot); + server.post("/forgotPassword", postForgot); - server.get(RouteStore.reset, getReset); - server.post(RouteStore.reset, postReset); + const reset = new RouteSubscriber("resetPassword").add("token").build; + server.get(reset, getReset); + server.post(reset, postReset); } function registerCorsProxy(server: express.Express) { const headerCharRegex = /[^\t\x20-\x7e\x80-\xff]/; - server.use(RouteStore.corsProxy, (req, res) => { + server.use("/corsProxy", (req, res) => { req.pipe(request(decodeURIComponent(req.url.substring(1)))).on("response", res => { const headers = Object.keys(res.headers); headers.forEach(headerName => { diff --git a/src/server/RouteManager.ts b/src/server/RouteManager.ts index c1d38327f..3aae5734a 100644 --- a/src/server/RouteManager.ts +++ b/src/server/RouteManager.ts @@ -1,5 +1,4 @@ import RouteSubscriber from "./RouteSubscriber"; -import { RouteStore } from "./RouteStore"; import { DashUserModel } from "./authentication/models/user_model"; import * as express from 'express'; @@ -67,10 +66,10 @@ export default class RouteManager { if (onUnauthenticated) { await tryExecute(onUnauthenticated, core); if (!res.headersSent) { - res.redirect(RouteStore.login); + res.redirect("/login"); } } else { - res.redirect(RouteStore.login); + res.redirect("/login"); } } setTimeout(() => { diff --git a/src/server/RouteStore.ts b/src/server/RouteStore.ts deleted file mode 100644 index a310d0c95..000000000 --- a/src/server/RouteStore.ts +++ /dev/null @@ -1,45 +0,0 @@ -// PREPEND ALL ROUTES WITH FORWARD SLASHES! - -export enum RouteStore { - // GENERAL - root = "/", - home = "/home", - corsProxy = "/corsProxy", - delete = "/delete", - deleteAll = "/deleteAll", - pull = "/pull", - - // UPLOAD AND STATIC FILE SERVING - public = "/public", - upload = "/upload", - dataUriToImage = "/uploadURI", - images = "/images", - inspectImage = "/inspectImage", - imageHierarchyExport = "/imageHierarchyExport", - - // USER AND WORKSPACES - getCurrUser = "/getCurrentUser", - getUsers = "/getUsers", - getUserDocumentId = "/getUserDocumentId", - updateCursor = "/updateCursor", - - openDocumentWithId = "/doc/:docId", - - // AUTHENTICATION - signup = "/signup", - login = "/login", - logout = "/logout", - forgot = "/forgotpassword", - reset = "/reset/:token", - - // APIS - cognitiveServices = "/cognitiveservices", - googleDocs = "/googleDocs", - readGoogleAccessToken = "/readGoogleAccessToken", - writeGoogleAccessToken = "/writeGoogleAccessToken", - googlePhotosMediaUpload = "/googlePhotosMediaUpload", - googlePhotosMediaDownload = "/googlePhotosMediaDownload", - googleDocsGet = "/googleDocsGet", - checkGoogle = "/checkGoogleAuthentication" - -} \ No newline at end of file diff --git a/src/server/RouteSubscriber.ts b/src/server/RouteSubscriber.ts index e49be8af5..a1cf7c1c4 100644 --- a/src/server/RouteSubscriber.ts +++ b/src/server/RouteSubscriber.ts @@ -3,7 +3,7 @@ export default class RouteSubscriber { private requestParameters: string[] = []; constructor(root: string) { - this._root = root; + this._root = `/${root}`; } add(...parameters: string[]) { diff --git a/src/server/authentication/config/passport.ts b/src/server/authentication/config/passport.ts index 8915a4abf..0b15c3a36 100644 --- a/src/server/authentication/config/passport.ts +++ b/src/server/authentication/config/passport.ts @@ -3,7 +3,6 @@ import * as passportLocal from 'passport-local'; import _ from "lodash"; import { default as User } from '../models/user_model'; import { Request, Response, NextFunction } from "express"; -import { RouteStore } from '../../RouteStore'; const LocalStrategy = passportLocal.Strategy; @@ -35,13 +34,13 @@ export let isAuthenticated = (req: Request, res: Response, next: NextFunction) = if (req.isAuthenticated()) { return next(); } - return res.redirect(RouteStore.login); + return res.redirect("/login"); }; export let isAuthorized = (req: Request, res: Response, next: NextFunction) => { const provider = req.path.split("/").slice(-1)[0]; - if (_.find((req.user as any).tokens, { kind: provider })) { + if (_.find((req.user).tokens, { kind: provider })) { next(); } else { res.redirect(`/auth/${provider}`); diff --git a/src/server/authentication/controllers/user_controller.ts b/src/server/authentication/controllers/user_controller.ts index f5c6e1610..b2b9d33f6 100644 --- a/src/server/authentication/controllers/user_controller.ts +++ b/src/server/authentication/controllers/user_controller.ts @@ -10,10 +10,7 @@ import * as pug from 'pug'; import * as async from 'async'; import * as nodemailer from 'nodemailer'; import c = require("crypto"); -import { RouteStore } from "../../RouteStore"; import { Utils } from "../../../Utils"; -import { Schema } from "mongoose"; -import { Opt } from "../../../new_fields/Doc"; import { MailOptions } from "nodemailer/lib/stream-transport"; /** @@ -23,8 +20,7 @@ import { MailOptions } from "nodemailer/lib/stream-transport"; */ export let getSignup = (req: Request, res: Response) => { if (req.user) { - let user = req.user; - return res.redirect(RouteStore.home); + return res.redirect("/home"); } res.render("signup.pug", { title: "Sign Up", @@ -45,7 +41,7 @@ export let postSignup = (req: Request, res: Response, next: NextFunction) => { const errors = req.validationErrors(); if (errors) { - return res.redirect(RouteStore.signup); + return res.redirect("/signup"); } const email = req.body.email as String; @@ -62,7 +58,7 @@ export let postSignup = (req: Request, res: Response, next: NextFunction) => { User.findOne({ email }, (err, existingUser) => { if (err) { return next(err); } if (existingUser) { - return res.redirect(RouteStore.login); + return res.redirect("/login"); } user.save((err: any) => { if (err) { return next(err); } @@ -81,7 +77,7 @@ let tryRedirectToTarget = (req: Request, res: Response) => { req.session.target = undefined; res.redirect(target); } else { - res.redirect(RouteStore.home); + res.redirect("/home"); } }; @@ -93,7 +89,7 @@ let tryRedirectToTarget = (req: Request, res: Response) => { export let getLogin = (req: Request, res: Response) => { if (req.user) { req.session!.target = undefined; - return res.redirect(RouteStore.home); + return res.redirect("/home"); } res.render("login.pug", { title: "Log In", @@ -115,13 +111,13 @@ export let postLogin = (req: Request, res: Response, next: NextFunction) => { if (errors) { req.flash("errors", "Unable to login at this time. Please try again."); - return res.redirect(RouteStore.signup); + return res.redirect("/signup"); } passport.authenticate("local", (err: Error, user: DashUserModel, info: IVerifyOptions) => { if (err) { next(err); return; } if (!user) { - return res.redirect(RouteStore.signup); + return res.redirect("/signup"); } req.logIn(user, (err) => { if (err) { next(err); return; } @@ -141,7 +137,7 @@ export let getLogout = (req: Request, res: Response) => { if (sess) { sess.destroy((err) => { if (err) { console.log(err); } }); } - res.redirect(RouteStore.login); + res.redirect("/login"); }; export let getForgot = function (req: Request, res: Response) { @@ -168,7 +164,7 @@ export let postForgot = function (req: Request, res: Response, next: NextFunctio User.findOne({ email }, function (err, user: DashUserModel) { if (!user) { // NO ACCOUNT WITH SUBMITTED EMAIL - res.redirect(RouteStore.forgot); + res.redirect("/forgotPassword"); return; } user.passwordResetToken = token; @@ -192,7 +188,7 @@ export let postForgot = function (req: Request, res: Response, next: NextFunctio subject: 'Dash Password Reset', text: 'You are receiving this because you (or someone else) have requested the reset of the password for your account.\n\n' + 'Please click on the following link, or paste this into your browser to complete the process:\n\n' + - 'http://' + req.headers.host + '/reset/' + token + '\n\n' + + 'http://' + req.headers.host + '/resetPassword/' + token + '\n\n' + 'If you did not request this, please ignore this email and your password will remain unchanged.\n' } as MailOptions; smtpTransport.sendMail(mailOptions, function (err: Error | null) { @@ -202,14 +198,14 @@ export let postForgot = function (req: Request, res: Response, next: NextFunctio } ], function (err) { if (err) return next(err); - res.redirect(RouteStore.forgot); + res.redirect("/forgotPassword"); }); }; export let getReset = function (req: Request, res: Response) { User.findOne({ passwordResetToken: req.params.token, passwordResetExpires: { $gt: Date.now() } }, function (err, user: DashUserModel) { if (!user || err) { - return res.redirect(RouteStore.forgot); + return res.redirect("/forgotPassword"); } res.render("reset.pug", { title: "Reset Password", @@ -239,7 +235,7 @@ export let postReset = function (req: Request, res: Response) { user.save(function (err) { if (err) { - res.redirect(RouteStore.login); + res.redirect("/login"); return; } req.logIn(user, function (err) { @@ -271,6 +267,6 @@ export let postReset = function (req: Request, res: Response) { }); } ], function (err) { - res.redirect(RouteStore.login); + res.redirect("/login"); }); }; \ No newline at end of file diff --git a/src/server/authentication/models/current_user_utils.ts b/src/server/authentication/models/current_user_utils.ts index 73cac879e..5a8815983 100644 --- a/src/server/authentication/models/current_user_utils.ts +++ b/src/server/authentication/models/current_user_utils.ts @@ -11,7 +11,6 @@ import { List } from "../../../new_fields/List"; import { listSpec } from "../../../new_fields/Schema"; import { Cast, StrCast, PromiseValue } from "../../../new_fields/Types"; import { Utils } from "../../../Utils"; -import { RouteStore } from "../../RouteStore"; import { ScriptField } from "../../../new_fields/ScriptField"; import { ButtonBox } from "../../../client/views/nodes/ButtonBox"; import { UndoManager } from "../../../client/util/UndoManager"; @@ -198,8 +197,8 @@ export class CurrentUserUtils { return doc; } - public static loadCurrentUser() { - return rp.get(Utils.prepend(RouteStore.getCurrUser)).then(response => { + public static async loadCurrentUser() { + return rp.get(Utils.prepend("/getCurrentUser")).then(response => { if (response) { const result: { id: string, email: string } = JSON.parse(response); return result; @@ -212,7 +211,7 @@ export class CurrentUserUtils { public static async loadUserDocument({ id, email }: { id: string, email: string }) { this.curr_id = id; Doc.CurrentUserEmail = email; - await rp.get(Utils.prepend(RouteStore.getUserDocumentId)).then(id => { + await rp.get(Utils.prepend("/getUserDocumentId")).then(id => { if (id && id !== "guest") { return DocServer.GetRefField(id).then(async field => { if (field instanceof Doc) { diff --git a/src/server/index.ts b/src/server/index.ts index aec301a74..8fc402cc9 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -3,7 +3,6 @@ import { GoogleApiServerUtils } from "./apis/google/GoogleApiServerUtils"; import * as mobileDetect from 'mobile-detect'; import * as path from 'path'; import { Database } from './database'; -import { RouteStore } from './RouteStore'; const serverPort = 4321; import { GooglePhotosUploadUtils } from './apis/google/GooglePhotosUploadUtils'; import { Opt } from '../new_fields/Doc'; @@ -23,7 +22,7 @@ import DeleteManager from "./ApiManagers/DeleteManager"; import PDFManager from "./ApiManagers/PDFManager"; import UploadManager from "./ApiManagers/UploadManager"; -export const publicDirectory = __dirname + RouteStore.public; +export const publicDirectory = __dirname + "/public"; export const filesDirectory = publicDirectory + "/files/"; export enum Partitions { pdf_text, @@ -73,13 +72,12 @@ function routeSetter(router: RouteManager) { WebSocket.initialize(serverPort, router.isRelease); /** - * Anyone attempting to navigate to localhost at this port will - * first have to log in. + * Accessing root index redirects to home */ router.addSupervisedRoute({ method: Method.GET, - subscription: RouteStore.root, - onValidation: ({ res }) => res.redirect(RouteStore.home) + subscription: "/", + onValidation: ({ res }) => res.redirect("/home") }); const serve: OnUnauthenticated = ({ req, res }) => { @@ -90,7 +88,7 @@ function routeSetter(router: RouteManager) { router.addSupervisedRoute({ method: Method.GET, - subscription: [RouteStore.home, new RouteSubscriber("/doc").add("docId")], + subscription: ["/home", new RouteSubscriber("doc").add("docId")], onValidation: serve, onUnauthenticated: ({ req, ...remaining }) => { const { originalUrl: target } = req; @@ -110,9 +108,9 @@ function routeSetter(router: RouteManager) { router.addSupervisedRoute({ method: Method.GET, - subscription: new RouteSubscriber(RouteStore.cognitiveServices).add('requestedservice'), + subscription: new RouteSubscriber("cognitiveServices").add('requestedService'), onValidation: ({ req, res }) => { - let service = req.params.requestedservice; + let service = req.params.requestedService; res.send(ServicesApiKeyMap.get(service)); } }); @@ -125,7 +123,7 @@ function routeSetter(router: RouteManager) { router.addSupervisedRoute({ method: Method.POST, - subscription: new RouteSubscriber(RouteStore.googleDocs).add("sector", "action"), + subscription: new RouteSubscriber("googleDocs").add("sector", "action"), onValidation: async ({ req, res, user }) => { let sector: GoogleApiServerUtils.Service = req.params.sector as GoogleApiServerUtils.Service; let action: GoogleApiServerUtils.Action = req.params.action as GoogleApiServerUtils.Action; @@ -143,7 +141,7 @@ function routeSetter(router: RouteManager) { router.addSupervisedRoute({ method: Method.GET, - subscription: RouteStore.readGoogleAccessToken, + subscription: "/readGoogleAccessToken", onValidation: async ({ user, res }) => { const userId = user.id; const token = await GoogleApiServerUtils.retrieveAccessToken(userId); @@ -156,7 +154,7 @@ function routeSetter(router: RouteManager) { router.addSupervisedRoute({ method: Method.POST, - subscription: RouteStore.writeGoogleAccessToken, + subscription: "/writeGoogleAccessToken", onValidation: async ({ user, req, res }) => { res.send(await GoogleApiServerUtils.processNewUser(user.id, req.body.authenticationCode)); } @@ -173,7 +171,7 @@ function routeSetter(router: RouteManager) { router.addSupervisedRoute({ method: Method.POST, - subscription: RouteStore.googlePhotosMediaUpload, + subscription: "/googlePhotosMediaUpload", onValidation: async ({ user, req, res }) => { const { media } = req.body; @@ -228,7 +226,7 @@ function routeSetter(router: RouteManager) { const UploadError = (count: number) => `Unable to upload ${count} images to Dash's server`; router.addSupervisedRoute({ method: Method.POST, - subscription: RouteStore.googlePhotosMediaDownload, + subscription: "/googlePhotosMediaDownload", onValidation: async ({ req, res }) => { const contents: { mediaItems: MediaItem[] } = req.body; let failed = 0; -- cgit v1.2.3-70-g09d2 From df5584ccd40bd83f1362b32db67969e7ffbf2e3f Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Wed, 27 Nov 2019 04:03:30 -0500 Subject: improved file partitioning in server and generified upload method --- package.json | 2 + src/client/documents/Documents.ts | 1 - src/client/util/ClientDiagnostics.ts | 11 +- .../util/Import & Export/DirectoryImportBox.tsx | 19 +- src/client/views/Main.tsx | 2 +- src/client/views/collections/CollectionSubView.tsx | 22 +- src/client/views/pdf/PDFViewer.tsx | 3 +- src/server/ActionUtilities.ts | 17 +- src/server/ApiManagers/DownloadManager.ts | 4 +- src/server/ApiManagers/GooglePhotosManager.ts | 4 +- src/server/ApiManagers/PDFManager.ts | 34 +- src/server/ApiManagers/SearchManager.ts | 4 +- src/server/ApiManagers/UploadManager.ts | 47 ++- src/server/DashUploadUtils.ts | 388 +++++++++++---------- src/server/SharedMediaTypes.ts | 5 +- src/server/database.ts | 4 +- src/server/index.ts | 4 +- 17 files changed, 314 insertions(+), 257 deletions(-) (limited to 'src/server/ApiManagers/PDFManager.ts') diff --git a/package.json b/package.json index 393df8574..3725d76eb 100644 --- a/package.json +++ b/package.json @@ -105,6 +105,7 @@ "@types/react-table": "^6.7.22", "@types/request": "^2.48.1", "@types/request-promise": "^4.1.42", + "@types/rimraf": "^2.0.3", "@types/sharp": "^0.22.2", "@types/shelljs": "^0.8.5", "@types/socket.io": "^2.1.2", @@ -211,6 +212,7 @@ "readline": "^1.3.0", "request": "^2.88.0", "request-promise": "^4.2.4", + "rimraf": "^3.0.0", "serializr": "^1.5.1", "sharp": "^0.22.1", "shelljs": "^0.8.3", diff --git a/src/client/documents/Documents.ts b/src/client/documents/Documents.ts index dea057b93..a2f4d23c7 100644 --- a/src/client/documents/Documents.ts +++ b/src/client/documents/Documents.ts @@ -377,7 +377,6 @@ export namespace Docs { let extension = path.extname(target); target = `${target.substring(0, target.length - extension.length)}_o${extension}`; } - // if (target !== "http://www.cs.brown.edu/") { requestImageSize(target) .then((size: any) => { let aspect = size.height / size.width; diff --git a/src/client/util/ClientDiagnostics.ts b/src/client/util/ClientDiagnostics.ts index 24f196252..7eef935fd 100644 --- a/src/client/util/ClientDiagnostics.ts +++ b/src/client/util/ClientDiagnostics.ts @@ -1,9 +1,6 @@ -import { observable, runInAction } from "mobx"; -import { MainView } from "../views/MainView"; - export namespace ClientDiagnostics { - export function start() { + export async function start() { let serverPolls = 0; const serverHandle = setInterval(async () => { @@ -17,14 +14,16 @@ export namespace ClientDiagnostics { let executed = false; - const solrHandle = setInterval(async () => { + const handle = async () => { const response = await fetch("/solrHeartbeat"); if (!(await response.json()).running) { !executed && alert("Looks like SOLR is not running on your machine."); executed = true; clearInterval(solrHandle); } - }, 1000 * 15); + }; + await handle(); + const solrHandle = setInterval(handle, 1000 * 15); } diff --git a/src/client/util/Import & Export/DirectoryImportBox.tsx b/src/client/util/Import & Export/DirectoryImportBox.tsx index f0880f193..16ae50685 100644 --- a/src/client/util/Import & Export/DirectoryImportBox.tsx +++ b/src/client/util/Import & Export/DirectoryImportBox.tsx @@ -22,18 +22,10 @@ import "./DirectoryImportBox.scss"; import { Networking } from "../../Network"; import { BatchedArray } from "array-batcher"; import * as path from 'path'; -import { DashUploadUtils } from "../../../server/DashUploadUtils"; -import { SharedMediaTypes } from "../../../server/SharedMediaTypes"; +import { AcceptibleMedia } from "../../../server/SharedMediaTypes"; const unsupported = ["text/html", "text/plain"]; -interface ImageUploadResponse { - name: string; - path: string; - type: string; - exif: any; -} - @observer export default class DirectoryImportBox extends React.Component { private selector = React.createRef(); @@ -98,7 +90,7 @@ export default class DirectoryImportBox extends React.Component let file = files.item(i); if (file && !unsupported.includes(file.type)) { const ext = path.extname(file.name).toLowerCase(); - if (SharedMediaTypes.imageFormats.includes(ext)) { + if (AcceptibleMedia.imageFormats.includes(ext)) { validated.push(file); } } @@ -114,7 +106,7 @@ export default class DirectoryImportBox extends React.Component runInAction(() => this.phase = `Internal: uploading ${this.quota - this.completed} files to Dash...`); - const uploads = await BatchedArray.from(validated, { batchSize: 15 }).batchedMapAsync(async (batch, collector) => { + const uploads = await BatchedArray.from(validated, { batchSize: 15 }).batchedMapAsync(async (batch, collector) => { const formData = new FormData(); batch.forEach(file => { @@ -127,16 +119,17 @@ export default class DirectoryImportBox extends React.Component runInAction(() => this.completed += batch.length); }); + const size = "_o"; await Promise.all(uploads.map(async upload => { const type = upload.type; - const path = Utils.prepend(upload.path); + const path = Utils.prepend(upload.clientAccessPath); const options = { nativeWidth: 300, width: 300, title: upload.name }; const document = await Docs.Get.DocumentFromType(type, path, options); - const { data, error } = upload.exif; + const { data, error } = upload.exifData; if (document) { Doc.GetProto(document).exif = error || Docs.Get.DocumentHierarchyFromJson(data); docs.push(document); diff --git a/src/client/views/Main.tsx b/src/client/views/Main.tsx index dec4a24e4..9e699978f 100644 --- a/src/client/views/Main.tsx +++ b/src/client/views/Main.tsx @@ -10,7 +10,7 @@ import { ClientDiagnostics } from "../util/ClientDiagnostics"; AssignAllExtensions(); (async () => { - ClientDiagnostics.start(); + await ClientDiagnostics.start(); const info = await CurrentUserUtils.loadCurrentUser(); DocServer.init(window.location.protocol, window.location.hostname, 4321, info.email); await Docs.Prototypes.initialize(); diff --git a/src/client/views/collections/CollectionSubView.tsx b/src/client/views/collections/CollectionSubView.tsx index 1c3ff37ee..a1bd1527e 100644 --- a/src/client/views/collections/CollectionSubView.tsx +++ b/src/client/views/collections/CollectionSubView.tsx @@ -22,6 +22,7 @@ import React = require("react"); var path = require('path'); import { GooglePhotos } from "../../apis/google_docs/GooglePhotosClientUtils"; import { ImageUtils } from "../../util/Import & Export/ImageUtils"; +import { Networking } from "../../Network"; export interface CollectionViewProps extends FieldViewProps { addDocument: (document: Doc) => boolean; @@ -271,28 +272,25 @@ export function CollectionSubView(schemaCtor: (doc: Doc) => T) { let file = item.getAsFile(); let formData = new FormData(); - if (file) { - formData.append('file', file); + if (!file || !file.type) { + continue; } - let dropFileName = file ? file.name : "-empty-"; - let prom = fetch(Utils.prepend("/upload"), { - method: 'POST', - body: formData - }).then(async (res: Response) => { - (await res.json()).map(action((file: any) => { + formData.append('file', file); + let dropFileName = file ? file.name : "-empty-"; + promises.push(Networking.PostFormDataToServer("/upload", formData).then(results => { + results.map(action((file: any) => { let full = { ...options, nativeWidth: type.indexOf("video") !== -1 ? 600 : 300, width: 300, title: dropFileName }; - let pathname = Utils.prepend(file.path); + let pathname = Utils.prepend(file.clientAccessPath); Docs.Get.DocumentFromType(type, pathname, full).then(doc => { doc && (Doc.GetProto(doc).fileUpload = path.basename(pathname).replace("upload_", "").replace(/\.[a-z0-9]*$/, "")); doc && this.props.addDocument(doc); }); })); - }); - promises.push(prom); + })); } } - if (text) { + if (text && !text.includes("https://")) { this.props.addDocument(Docs.Create.TextDocument({ ...options, documentText: "@@@" + text, width: 400, height: 315 })); return; } diff --git a/src/client/views/pdf/PDFViewer.tsx b/src/client/views/pdf/PDFViewer.tsx index b737ce221..c075a4f99 100644 --- a/src/client/views/pdf/PDFViewer.tsx +++ b/src/client/views/pdf/PDFViewer.tsx @@ -125,7 +125,8 @@ export class PDFViewer extends DocAnnotatableComponent this._showWaiting = this._showCover = true); this.props.startupLive && this.setupPdfJsViewer(); this._searchReactionDisposer = reaction(() => this.Document.search_string, searchString => { diff --git a/src/server/ActionUtilities.ts b/src/server/ActionUtilities.ts index 7f493dd70..c9fc86fea 100644 --- a/src/server/ActionUtilities.ts +++ b/src/server/ActionUtilities.ts @@ -2,6 +2,7 @@ import * as fs from 'fs'; import { ExecOptions } from 'shelljs'; import { exec } from 'child_process'; import * as path from 'path'; +import * as rimraf from "rimraf"; export const command_line = (command: string, fromDirectory?: string) => { return new Promise((resolve, reject) => { @@ -68,4 +69,18 @@ export function msToTime(duration: number) { let secondsS = (seconds < 10) ? "0" + seconds : seconds; return hoursS + ":" + minutesS + ":" + secondsS + "." + milliseconds; -} \ No newline at end of file +} + +export const createIfNotExists = async (path: string) => { + if (await new Promise(resolve => fs.exists(path, resolve))) { + return true; + } + return new Promise(resolve => fs.mkdir(path, error => resolve(error === null))); +}; + +export async function Prune(rootDirectory: string): Promise { + const error = await new Promise(resolve => rimraf(rootDirectory, resolve)); + return error === null; +} + +export const Destroy = (mediaPath: string) => new Promise(resolve => fs.unlink(mediaPath, error => resolve(error === null))); \ No newline at end of file diff --git a/src/server/ApiManagers/DownloadManager.ts b/src/server/ApiManagers/DownloadManager.ts index fc6ba0d22..5bad46eda 100644 --- a/src/server/ApiManagers/DownloadManager.ts +++ b/src/server/ApiManagers/DownloadManager.ts @@ -5,7 +5,7 @@ import * as Archiver from 'archiver'; import * as express from 'express'; import { Database } from "../database"; import * as path from "path"; -import { DashUploadUtils } from "../DashUploadUtils"; +import { DashUploadUtils, SizeSuffix } from "../DashUploadUtils"; import { publicDirectory } from ".."; export type Hierarchy = { [id: string]: string | Hierarchy }; @@ -254,7 +254,7 @@ async function writeHierarchyRecursive(file: Archiver.Archiver, hierarchy: Hiera // and dropped in the browser and thus hosted remotely) so we upload it // to our server and point the zip file to it, so it can bundle up the bytes const information = await DashUploadUtils.UploadImage(result); - path = information.mediaPaths[0]; + path = information.serverAccessPaths[SizeSuffix.Original]; } // write the file specified by the path to the directory in the // zip file given by the prefix. diff --git a/src/server/ApiManagers/GooglePhotosManager.ts b/src/server/ApiManagers/GooglePhotosManager.ts index c7af69375..5a709688b 100644 --- a/src/server/ApiManagers/GooglePhotosManager.ts +++ b/src/server/ApiManagers/GooglePhotosManager.ts @@ -86,10 +86,10 @@ export default class GooglePhotosManager extends ApiManager { const contents: { mediaItems: MediaItem[] } = req.body; let failed = 0; if (contents) { - const completed: Opt[] = []; + const completed: Opt[] = []; for (let item of contents.mediaItems) { const { contentSize, ...attributes } = await DashUploadUtils.InspectImage(item.baseUrl); - const found: Opt = await Database.Auxiliary.QueryUploadHistory(contentSize!); + const found: Opt = await Database.Auxiliary.QueryUploadHistory(contentSize!); if (!found) { const upload = await DashUploadUtils.UploadInspectedImage({ contentSize, ...attributes }, item.filename, prefix).catch(error => _error(res, downloadError, error)); if (upload) { diff --git a/src/server/ApiManagers/PDFManager.ts b/src/server/ApiManagers/PDFManager.ts index 632b4965a..4bd750aaf 100644 --- a/src/server/ApiManagers/PDFManager.ts +++ b/src/server/ApiManagers/PDFManager.ts @@ -2,12 +2,12 @@ import ApiManager, { Registration } from "./ApiManager"; import { Method } from "../RouteManager"; import RouteSubscriber from "../RouteSubscriber"; import { exists, createReadStream, createWriteStream } from "fs"; -import { filesDirectory } from ".."; import * as Pdfjs from 'pdfjs-dist'; import { createCanvas } from "canvas"; const probe = require("probe-image-size"); import * as express from "express"; import * as path from "path"; +import { Directory, serverPathToFile, clientPathToFile } from "./UploadManager"; export default class PDFManager extends ApiManager { @@ -21,21 +21,27 @@ export default class PDFManager extends ApiManager { let noExt = filename.substring(0, filename.length - ".png".length); let pagenumber = parseInt(noExt.split('-')[1]); return new Promise(resolve => { - exists(filesDirectory + filename, (exists: boolean) => { - console.log(`${filesDirectory + filename} ${exists ? "exists" : "does not exist"}`); + const path = serverPathToFile(Directory.pdf_thumbnails, filename); + exists(path, (exists: boolean) => { + console.log(`${path} ${exists ? "exists" : "does not exist"}`); if (exists) { - let input = createReadStream(filesDirectory + filename); - probe(input, (err: any, result: any) => { + let input = createReadStream(path); + probe(input, (err: any, { width, height }: any) => { if (err) { console.log(err); console.log(`error on ${filename}`); return; } - res.send({ path: "/files/" + filename, width: result.width, height: result.height }); + res.send({ + path: clientPathToFile(Directory.pdf_thumbnails, filename), + width, + height + }); }); } else { - LoadPage(filesDirectory + filename.substring(0, filename.length - noExt.split('-')[1].length - ".PNG".length - 1) + ".pdf", pagenumber, res); + const name = filename.substring(0, filename.length - noExt.split('-')[1].length - ".PNG".length - 1) + ".pdf"; + LoadPage(serverPathToFile(Directory.pdfs, name), pagenumber, res); } resolve(); }); @@ -55,8 +61,8 @@ export default class PDFManager extends ApiManager { let canvasAndContext = factory.create(viewport.width, viewport.height); let renderContext = { canvasContext: canvasAndContext.context, - viewport: viewport, - canvasFactory: factory + canvasFactory: factory, + viewport }; console.log("read " + pageNumber); @@ -64,13 +70,17 @@ export default class PDFManager extends ApiManager { .then(() => { console.log("saving " + pageNumber); let stream = canvasAndContext.canvas.createPNGStream(); - let pngFile = `${file.substring(0, file.length - ".pdf".length)}-${pageNumber}.PNG`; + let filenames = path.basename(file).split("."); + const pngFile = serverPathToFile(Directory.pdf_thumbnails, `${filenames[0]}-${pageNumber}.png`); let out = createWriteStream(pngFile); stream.pipe(out); out.on("finish", () => { console.log(`Success! Saved to ${pngFile}`); - let name = path.basename(pngFile); - res.send({ path: "/files/" + name, width: viewport.width, height: viewport.height }); + res.send({ + path: pngFile, + width: viewport.width, + height: viewport.height + }); }); }, (reason: string) => { console.error(reason + ` ${pageNumber}`); diff --git a/src/server/ApiManagers/SearchManager.ts b/src/server/ApiManagers/SearchManager.ts index 1c801715a..d3f8995b0 100644 --- a/src/server/ApiManagers/SearchManager.ts +++ b/src/server/ApiManagers/SearchManager.ts @@ -3,7 +3,7 @@ import { Method } from "../RouteManager"; import { Search } from "../Search"; var findInFiles = require('find-in-files'); import * as path from 'path'; -import { filesDirectory } from ".."; +import { pathToDirectory, Directory } from "./UploadManager"; export default class SearchManager extends ApiManager { @@ -18,7 +18,7 @@ export default class SearchManager extends ApiManager { res.send([]); return; } - let results = await findInFiles.find({ 'term': q, 'flags': 'ig' }, filesDirectory + "text", ".txt$"); + let results = await findInFiles.find({ 'term': q, 'flags': 'ig' }, pathToDirectory(Directory.text), ".txt$"); let resObj: { ids: string[], numFound: number, lines: string[] } = { ids: [], numFound: 0, lines: [] }; for (var result in results) { resObj.ids.push(path.basename(result, ".txt").replace(/upload_/, "")); diff --git a/src/server/ApiManagers/UploadManager.ts b/src/server/ApiManagers/UploadManager.ts index 2a9faacd8..2f76871a6 100644 --- a/src/server/ApiManagers/UploadManager.ts +++ b/src/server/ApiManagers/UploadManager.ts @@ -7,14 +7,33 @@ import { extname, basename, dirname } from 'path'; import { createReadStream, createWriteStream, unlink, readFileSync } from "fs"; import { publicDirectory, filesDirectory } from ".."; import { Database } from "../database"; -import { DashUploadUtils } from "../DashUploadUtils"; -import { Opt } from "../../new_fields/Doc"; -import { ParsedPDF } from "../PdfTypes"; -const pdf = require('pdf-parse'); +import { DashUploadUtils, SizeSuffix } from "../DashUploadUtils"; import * as sharp from 'sharp'; -import { SharedMediaTypes } from "../SharedMediaTypes"; +import { AcceptibleMedia } from "../SharedMediaTypes"; +import { normalize } from "path"; const imageDataUri = require('image-data-uri'); +export enum Directory { + parsed_files = "parsed_files", + images = "images", + videos = "videos", + pdfs = "pdfs", + text = "text", + pdf_thumbnails = "pdf_thumbnails" +} + +export function serverPathToFile(directory: Directory, filename: string) { + return normalize(`${filesDirectory}/${directory}/${filename}`); +} + +export function pathToDirectory(directory: Directory) { + return normalize(`${filesDirectory}/${directory}`); +} + +export function clientPathToFile(directory: Directory, filename: string) { + return `/files/${directory}/${filename}`; +} + export default class UploadManager extends ApiManager { protected initialize(register: Registration): void { @@ -129,13 +148,14 @@ export default class UploadManager extends ApiManager { subscription: "/upload", onValidation: async ({ req, res }) => { let form = new formidable.IncomingForm(); - form.uploadDir = filesDirectory; + form.uploadDir = pathToDirectory(Directory.parsed_files); form.keepExtensions = true; return new Promise(resolve => { form.parse(req, async (_err, _fields, files) => { let results: any[] = []; for (const key in files) { - results.push(DashUploadUtils.upload(files[key])); + const result = await DashUploadUtils.upload(files[key]); + result && results.push(result); } _success(res, results); resolve(); @@ -150,8 +170,8 @@ export default class UploadManager extends ApiManager { onValidation: async ({ req, res }) => { const { source } = req.body; if (typeof source === "string") { - const uploadInformation = await DashUploadUtils.UploadImage(source); - return res.send(await DashUploadUtils.InspectImage(uploadInformation.mediaPaths[0])); + const { serverAccessPaths } = await DashUploadUtils.UploadImage(source); + return res.send(await DashUploadUtils.InspectImage(serverAccessPaths[SizeSuffix.Original])); } res.send({}); } @@ -167,9 +187,9 @@ export default class UploadManager extends ApiManager { res.status(401).send("incorrect parameters specified"); return; } - return imageDataUri.outputFile(uri, filesDirectory + filename).then((savedName: string) => { + return imageDataUri.outputFile(uri, serverPathToFile(Directory.images, filename)).then((savedName: string) => { const ext = extname(savedName).toLowerCase(); - const { pngs, jpgs } = SharedMediaTypes; + const { pngs, jpgs } = AcceptibleMedia; let resizers = [ { resizer: sharp().resize(100, undefined, { withoutEnlargement: true }), suffix: "_s" }, { resizer: sharp().resize(400, undefined, { withoutEnlargement: true }), suffix: "_m" }, @@ -189,10 +209,11 @@ export default class UploadManager extends ApiManager { } if (isImage) { resizers.forEach(resizer => { - createReadStream(savedName).pipe(resizer.resizer).pipe(createWriteStream(filesDirectory + filename + resizer.suffix + ext)); + const path = serverPathToFile(Directory.images, filename + resizer.suffix + ext); + createReadStream(savedName).pipe(resizer.resizer).pipe(createWriteStream(path)); }); } - res.send("/files/" + filename + ext); + res.send(clientPathToFile(Directory.images, filename + ext)); }); } }); diff --git a/src/server/DashUploadUtils.ts b/src/server/DashUploadUtils.ts index 839aada4b..0a670ec01 100644 --- a/src/server/DashUploadUtils.ts +++ b/src/server/DashUploadUtils.ts @@ -5,18 +5,32 @@ import * as sharp from 'sharp'; import request = require('request-promise'); import { ExifData, ExifImage } from 'exif'; import { Opt } from '../new_fields/Doc'; -import { SharedMediaTypes } from './SharedMediaTypes'; +import { AcceptibleMedia } from './SharedMediaTypes'; import { filesDirectory } from '.'; import { File } from 'formidable'; -import { extname, basename } from "path"; +import { basename } from "path"; +import { ConsoleColors, createIfNotExists } from './ActionUtilities'; +import { ParsedPDF } from "../server/PdfTypes"; +const parse = require('pdf-parse'); +import { Directory, serverPathToFile, clientPathToFile } from './ApiManagers/UploadManager'; -const uploadDirectory = path.join(__dirname, './public/files/'); +export enum SizeSuffix { + Small = "_s", + Medium = "_m", + Large = "_l", + Original = "_o" +} export namespace DashUploadUtils { + function InjectSize(filename: string, size: SizeSuffix) { + const extension = path.extname(filename).toLowerCase(); + return filename.substring(0, filename.length - extension.length) + size + extension; + } + export interface Size { width: number; - suffix: string; + suffix: SizeSuffix; } export interface ImageFileResponse { @@ -27,215 +41,221 @@ export namespace DashUploadUtils { } export const Sizes: { [size: string]: Size } = { - SMALL: { width: 100, suffix: "_s" }, - MEDIUM: { width: 400, suffix: "_m" }, - LARGE: { width: 900, suffix: "_l" }, + SMALL: { width: 100, suffix: SizeSuffix.Small }, + MEDIUM: { width: 400, suffix: SizeSuffix.Medium }, + LARGE: { width: 900, suffix: SizeSuffix.Large }, }; export function validateExtension(url: string) { - return SharedMediaTypes.imageFormats.includes(path.extname(url).toLowerCase()); + return AcceptibleMedia.imageFormats.includes(path.extname(url).toLowerCase()); } const size = "content-length"; const type = "content-type"; - export interface UploadInformation { - mediaPaths: string[]; - fileNames: { [key: string]: string }; + export interface ImageUploadInformation { + clientAccessPath: string; + serverAccessPaths: { [key: string]: string }; exifData: EnrichedExifData; contentSize?: number; contentType?: string; } - export function upload(file: File): any { + export async function upload(file: File): Promise { const { type, path, name } = file; - const filename = basename(path); - const extension = extname(path).toLowerCase(); - if (extension === ".pdf") { - - } else if { - let partition: Opt; - if(imageFormats.includes(extension)) { - partition = DashUploadUtils.Partitions.images; - } else if (videoFormats.includes(extension)) { - partition = DashUploadUtils.Partitions.videos; - } - let uploadInformation: Opt; - if (partition) { - uploadInformation = await DashUploadUtils.UploadImage(`${filesDirectory}/${partition}/${filename}`, filename); - } else { - console.log(`Unable to accommodate, and ignored, the following file upload: ${filename}`); + const { imageFormats, videoFormats, applicationFormats } = AcceptibleMedia; + const types = type.split("/"); + + const category = types[0]; + const format = `.${types[1]}`; + + switch (category) { + case "image": + if (imageFormats.includes(format)) { + const { clientAccessPath } = await UploadImage(path, basename(path), format); + return { clientAccessPath, name, type }; + } + case "video": + if (videoFormats.includes(format)) { + return MoveParsedFile(path, Directory.videos); + } + case "application": + if (applicationFormats.includes(format)) { + return UploadPdf(path); + } } + console.log(ConsoleColors.Red, `Ignoring unsupported file ${name} with upload type (${type}).`); + return { clientAccessPath: undefined }; } - const exif = uploadInformation ? uploadInformation.exifData : undefined; - results.push({ name, type, path: `/files/${filename}`, exif }); -} + async function UploadPdf(absolutePath: string) { + let dataBuffer = fs.readFileSync(absolutePath); + const result: ParsedPDF = await parse(dataBuffer); + const parsedName = basename(absolutePath); + await new Promise((resolve, reject) => { + const textFilename = `${parsedName.substring(0, parsedName.length - 4)}.txt`; + const writeStream = fs.createWriteStream(serverPathToFile(Directory.text, textFilename)); + writeStream.write(result.text, error => error ? reject(error) : resolve()); + }); + return MoveParsedFile(absolutePath, Directory.pdfs); + } -const generate = (prefix: string, url: string) => `${prefix}upload_${Utils.GenerateGuid()}${sanitizeExtension(url)}`; -const sanitize = (filename: string) => filename.replace(/\s+/g, "_"); -const sanitizeExtension = (source: string) => { - let extension = path.extname(source); - extension = extension.toLowerCase(); - extension = extension.split("?")[0]; - return extension; -}; - -/** - * Uploads an image specified by the @param source to Dash's /public/files/ - * directory, and returns information generated during that upload - * - * @param {string} source is either the absolute path of an already uploaded image or - * the url of a remote image - * @param {string} filename dictates what to call the image. If not specified, - * the name {@param prefix}_upload_{GUID} - * @param {string} prefix is a string prepended to the generated image name in the - * event that @param filename is not specified - * - * @returns {UploadInformation} This method returns - * 1) the paths to the uploaded images (plural due to resizing) - * 2) the file name of each of the resized images - * 3) the size of the image, in bytes (4432130) - * 4) the content type of the image, i.e. image/(jpeg | png | ...) - */ -export const UploadImage = async (source: string, filename?: string, prefix: string = ""): Promise => { - const metadata = await InspectImage(source); - return UploadInspectedImage(metadata, filename, prefix); -}; - -export interface InspectionResults { - isLocal: boolean; - stream: any; - normalizedUrl: string; - exifData: EnrichedExifData; - contentSize?: number; - contentType?: string; -} + const generate = (prefix: string, url: string) => `${prefix}upload_${Utils.GenerateGuid()}${sanitizeExtension(url)}`; + const sanitizeExtension = (source: string) => { + let extension = path.extname(source); + extension = extension.toLowerCase(); + extension = extension.split("?")[0]; + return extension; + }; -export interface EnrichedExifData { - data: ExifData; - error?: string; -} + /** + * Uploads an image specified by the @param source to Dash's /public/files/ + * directory, and returns information generated during that upload + * + * @param {string} source is either the absolute path of an already uploaded image or + * the url of a remote image + * @param {string} filename dictates what to call the image. If not specified, + * the name {@param prefix}_upload_{GUID} + * @param {string} prefix is a string prepended to the generated image name in the + * event that @param filename is not specified + * + * @returns {ImageUploadInformation} This method returns + * 1) the paths to the uploaded images (plural due to resizing) + * 2) the file name of each of the resized images + * 3) the size of the image, in bytes (4432130) + * 4) the content type of the image, i.e. image/(jpeg | png | ...) + */ + export const UploadImage = async (source: string, filename?: string, format?: string, prefix: string = ""): Promise => { + const metadata = await InspectImage(source); + return UploadInspectedImage(metadata, filename, format, prefix); + }; -export enum Partitions { - pdf_text = "pdf_text", - images = "images", - videos = "videos" -} + export interface InspectionResults { + isLocal: boolean; + stream: any; + normalizedUrl: string; + exifData: EnrichedExifData; + contentSize?: number; + contentType?: string; + } -export async function buildFilePartitions() { - const pending = Object.keys(Partitions).map(sub => createIfNotExists(filesDirectory + sub)); - return Promise.all(pending); -} + export interface EnrichedExifData { + data: ExifData; + error?: string; + } -/** - * Based on the url's classification as local or remote, gleans - * as much information as possible about the specified image - * - * @param source is the path or url to the image in question - */ -export const InspectImage = async (source: string): Promise => { - const { isLocal, stream, normalized: normalizedUrl } = classify(source); - const exifData = await parseExifData(source); - const results = { - exifData, - isLocal, - stream, - normalizedUrl - }; - // stop here if local, since request.head() can't handle local paths, only urls on the web - if (isLocal) { - return results; + export async function buildFileDirectories() { + const pending = Object.keys(Directory).map(sub => createIfNotExists(`${filesDirectory}/${sub}`)); + return Promise.all(pending); } - const metadata = (await new Promise((resolve, reject) => { - request.head(source, async (error, res) => { - if (error) { - return reject(error); - } - resolve(res); - }); - })).headers; - return { - contentSize: parseInt(metadata[size]), - contentType: metadata[type], - ...results - }; -}; - -export const UploadInspectedImage = async (metadata: InspectionResults, filename?: string, prefix = ""): Promise => { - const { isLocal, stream, normalizedUrl, contentSize, contentType, exifData } = metadata; - const resolved = filename ? sanitize(filename) : generate(prefix, normalizedUrl); - const extension = sanitizeExtension(normalizedUrl || resolved); - let information: UploadInformation = { - mediaPaths: [], - fileNames: { clean: resolved }, - exifData, - contentSize, - contentType, - }; - const { pngs, jpgs } = SharedMediaTypes; - return new Promise(async (resolve, reject) => { - const resizers = [ - { resizer: sharp().rotate(), suffix: "_o" }, - ...Object.values(Sizes).map(size => ({ - resizer: sharp().resize(size.width, undefined, { withoutEnlargement: true }).rotate(), - suffix: size.suffix - })) - ]; - if (pngs.includes(extension)) { - resizers.forEach(element => element.resizer = element.resizer.png()); - } else if (jpgs.includes(extension)) { - resizers.forEach(element => element.resizer = element.resizer.jpeg()); - } - for (let resizer of resizers) { - const suffix = resizer.suffix; - let mediaPath: string; - await new Promise(resolve => { - const filename = resolved.substring(0, resolved.length - extension.length) + suffix + extension; - information.mediaPaths.push(mediaPath = uploadDirectory + filename); - information.fileNames[suffix] = filename; - stream(normalizedUrl).pipe(resizer.resizer).pipe(fs.createWriteStream(mediaPath)) - .on('close', resolve) - .on('error', reject); - }); + + /** + * Based on the url's classification as local or remote, gleans + * as much information as possible about the specified image + * + * @param source is the path or url to the image in question + */ + export const InspectImage = async (source: string): Promise => { + const { isLocal, stream, normalized: normalizedUrl } = classify(source); + const exifData = await parseExifData(source); + const results = { + exifData, + isLocal, + stream, + normalizedUrl + }; + // stop here if local, since request.head() can't handle local paths, only urls on the web + if (isLocal) { + return results; } - if (!isLocal) { - await new Promise(resolve => { - stream(normalizedUrl).pipe(fs.createWriteStream(uploadDirectory + resolved)).on('close', resolve); + const metadata = (await new Promise((resolve, reject) => { + request.head(source, async (error, res) => { + if (error) { + return reject(error); + } + resolve(res); }); - } - resolve(information); - }); -}; - -const classify = (url: string) => { - const isLocal = /Dash-Web(\\|\/)src(\\|\/)server(\\|\/)public(\\|\/)files/g.test(url); - return { - isLocal, - stream: isLocal ? fs.createReadStream : request, - normalized: isLocal ? path.normalize(url) : url + })).headers; + return { + contentSize: parseInt(metadata[size]), + contentType: metadata[type], + ...results + }; }; -}; - -const parseExifData = async (source: string): Promise => { - return new Promise(resolve => { - new ExifImage(source, (error, data) => { - let reason: Opt = undefined; - if (error) { - reason = (error as any).code; + + export async function MoveParsedFile(absolutePath: string, destination: Directory): Promise<{ clientAccessPath: Opt }> { + return new Promise<{ clientAccessPath: Opt }>(resolve => { + const filename = basename(absolutePath); + const destinationPath = serverPathToFile(destination, filename); + fs.rename(absolutePath, destinationPath, error => { + resolve({ clientAccessPath: error ? undefined : clientPathToFile(destination, filename) }); + }); + }); + } + + export const UploadInspectedImage = async (metadata: InspectionResults, filename?: string, format?: string, prefix = ""): Promise => { + const { isLocal, stream, normalizedUrl, contentSize, contentType, exifData } = metadata; + const resolved = filename || generate(prefix, normalizedUrl); + const extension = format || sanitizeExtension(normalizedUrl || resolved); + let information: ImageUploadInformation = { + clientAccessPath: clientPathToFile(Directory.images, resolved), + serverAccessPaths: {}, + exifData, + contentSize, + contentType, + }; + const { pngs, jpgs } = AcceptibleMedia; + return new Promise(async (resolve, reject) => { + const resizers = [ + { resizer: sharp().rotate(), suffix: SizeSuffix.Original }, + ...Object.values(Sizes).map(size => ({ + resizer: sharp().resize(size.width, undefined, { withoutEnlargement: true }).rotate(), + suffix: size.suffix + })) + ]; + if (pngs.includes(extension)) { + resizers.forEach(element => element.resizer = element.resizer.png()); + } else if (jpgs.includes(extension)) { + resizers.forEach(element => element.resizer = element.resizer.jpeg()); + } + for (let { resizer, suffix } of resizers) { + let mediaPath: string; + await new Promise(resolve => { + const filename = InjectSize(resolved, suffix); + information.serverAccessPaths[suffix] = serverPathToFile(Directory.images, filename); + stream(normalizedUrl).pipe(resizer).pipe(fs.createWriteStream(serverPathToFile(Directory.images, filename))) + .on('close', resolve) + .on('error', reject); + }); } - resolve({ data, error: reason }); + if (isLocal) { + await new Promise(resolve => { + fs.unlink(normalizedUrl, error => resolve(error === null)); + }); + } + resolve(information); }); - }); -}; + }; -export const createIfNotExists = async (path: string) => { - if (await new Promise(resolve => fs.exists(path, resolve))) { - return true; - } - return new Promise(resolve => fs.mkdir(path, error => resolve(error === null))); -}; + const classify = (url: string) => { + const isLocal = /Dash-Web(\\|\/)src(\\|\/)server(\\|\/)public(\\|\/)files/g.test(url); + return { + isLocal, + stream: isLocal ? fs.createReadStream : request, + normalized: isLocal ? path.normalize(url) : url + }; + }; -export const Destroy = (mediaPath: string) => new Promise(resolve => fs.unlink(mediaPath, error => resolve(error === null))); + const parseExifData = async (source: string): Promise => { + return new Promise(resolve => { + new ExifImage(source, (error, data) => { + let reason: Opt = undefined; + if (error) { + reason = (error as any).code; + } + resolve({ data, error: reason }); + }); + }); + }; } \ No newline at end of file diff --git a/src/server/SharedMediaTypes.ts b/src/server/SharedMediaTypes.ts index 3d3234125..8d0f441f0 100644 --- a/src/server/SharedMediaTypes.ts +++ b/src/server/SharedMediaTypes.ts @@ -1,9 +1,8 @@ -export namespace SharedMediaTypes { - +export namespace AcceptibleMedia { export const gifs = [".gif"]; export const pngs = [".png"]; export const jpgs = [".jpg", ".jpeg"]; export const imageFormats = [...pngs, ...jpgs, ...gifs]; export const videoFormats = [".mov", ".mp4"]; - + export const applicationFormats = [".pdf"]; } \ No newline at end of file diff --git a/src/server/database.ts b/src/server/database.ts index b81fc03a4..db81245c1 100644 --- a/src/server/database.ts +++ b/src/server/database.ts @@ -293,7 +293,7 @@ export namespace Database { }; export const QueryUploadHistory = async (contentSize: number) => { - return SanitizedSingletonQuery({ contentSize }, AuxiliaryCollections.GooglePhotosUploadHistory); + return SanitizedSingletonQuery({ contentSize }, AuxiliaryCollections.GooglePhotosUploadHistory); }; export namespace GoogleAuthenticationToken { @@ -322,7 +322,7 @@ export namespace Database { } - export const LogUpload = async (information: DashUploadUtils.UploadInformation) => { + export const LogUpload = async (information: DashUploadUtils.ImageUploadInformation) => { const bundle = { _id: Utils.GenerateDeterministicGuid(String(information.contentSize!)), ...information diff --git a/src/server/index.ts b/src/server/index.ts index d02a6005e..d77923710 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -24,7 +24,7 @@ import GooglePhotosManager from "./ApiManagers/GooglePhotosManager"; import DiagnosticManager from "./ApiManagers/DiagnosticManager"; export const publicDirectory = path.resolve(__dirname, "public"); -export const filesDirectory = path.resolve(publicDirectory, "files") + "/"; +export const filesDirectory = path.resolve(publicDirectory, "files"); /** * These are the functions run before the server starts @@ -34,7 +34,7 @@ export const filesDirectory = path.resolve(publicDirectory, "files") + "/"; async function preliminaryFunctions() { await GoogleCredentialsLoader.loadCredentials(); GoogleApiServerUtils.processProjectCredentials(); - await DashUploadUtils.buildFilePartitions(); + await DashUploadUtils.buildFileDirectories(); await log_execution({ startMessage: "attempting to initialize mongodb connection", endMessage: "connection outcome determined", -- cgit v1.2.3-70-g09d2 From 77ee66de66a411f79bbbc036d379d09be38d172f Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Mon, 2 Dec 2019 12:12:58 -0500 Subject: further cleanup --- src/Utils.ts | 2 - src/client/util/ClientDiagnostics.ts | 18 +-- .../util/Import & Export/DirectoryImportBox.tsx | 3 +- src/client/views/MainView.tsx | 2 +- src/client/views/collections/CollectionSubView.tsx | 6 +- src/server/ApiManagers/DeleteManager.ts | 1 - src/server/ApiManagers/GeneralGoogleManager.ts | 15 +-- src/server/ApiManagers/PDFManager.ts | 134 ++++++++++----------- src/server/ApiManagers/UploadManager.ts | 43 ++++--- src/server/DashUploadUtils.ts | 18 ++- src/server/RouteManager.ts | 14 +++ .../authentication/models/current_user_utils.ts | 3 +- src/server/credentials/test.json | 14 --- 13 files changed, 125 insertions(+), 148 deletions(-) delete mode 100644 src/server/credentials/test.json (limited to 'src/server/ApiManagers/PDFManager.ts') diff --git a/src/Utils.ts b/src/Utils.ts index b60e9e023..2543743a4 100644 --- a/src/Utils.ts +++ b/src/Utils.ts @@ -2,8 +2,6 @@ import v4 = require('uuid/v4'); import v5 = require("uuid/v5"); import { Socket } from 'socket.io'; import { Message } from './server/Message'; -import { EventEmitter } from 'events'; -import { ConsoleColors } from './server/ActionUtilities'; export namespace Utils { diff --git a/src/client/util/ClientDiagnostics.ts b/src/client/util/ClientDiagnostics.ts index 7eef935fd..0a213aa1c 100644 --- a/src/client/util/ClientDiagnostics.ts +++ b/src/client/util/ClientDiagnostics.ts @@ -12,18 +12,22 @@ export namespace ClientDiagnostics { serverPolls--; }, 1000 * 15); - let executed = false; - const handle = async () => { + let solrHandle: NodeJS.Timeout | undefined; + const handler = async () => { const response = await fetch("/solrHeartbeat"); if (!(await response.json()).running) { - !executed && alert("Looks like SOLR is not running on your machine."); - executed = true; - clearInterval(solrHandle); + if (!executed) { + alert("Looks like SOLR is not running on your machine."); + executed = true; + solrHandle && clearInterval(solrHandle); + } } }; - await handle(); - const solrHandle = setInterval(handle, 1000 * 15); + await handler(); + if (!executed) { + solrHandle = setInterval(handler, 1000 * 15); + } } diff --git a/src/client/util/Import & Export/DirectoryImportBox.tsx b/src/client/util/Import & Export/DirectoryImportBox.tsx index b5e806a97..104d9e099 100644 --- a/src/client/util/Import & Export/DirectoryImportBox.tsx +++ b/src/client/util/Import & Export/DirectoryImportBox.tsx @@ -106,7 +106,8 @@ export default class DirectoryImportBox extends React.Component runInAction(() => this.phase = `Internal: uploading ${this.quota - this.completed} files to Dash...`); - const uploads = await BatchedArray.from(validated, { batchSize: 15 }).batchedMapAsync(async (batch, collector) => { + const batched = BatchedArray.from(validated, { batchSize: 15 }); + const uploads = await batched.batchedMapAsync(async (batch, collector) => { const formData = new FormData(); batch.forEach(file => { diff --git a/src/client/views/MainView.tsx b/src/client/views/MainView.tsx index 5231075a1..85dfd8be2 100644 --- a/src/client/views/MainView.tsx +++ b/src/client/views/MainView.tsx @@ -4,7 +4,7 @@ import { faMusic, faObjectGroup, faPause, faMousePointer, faPenNib, faFileAudio, faPen, faEraser, faPlay, faPortrait, faRedoAlt, faThumbtack, faTree, faTv, faUndoAlt, faHighlighter, faMicrophone, faCompressArrowsAlt } from '@fortawesome/free-solid-svg-icons'; import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; -import { action, computed, configure, observable, reaction, runInAction, autorun } from 'mobx'; +import { action, computed, configure, observable, reaction, runInAction } from 'mobx'; import { observer } from 'mobx-react'; import "normalize.css"; import * as React from 'react'; diff --git a/src/client/views/collections/CollectionSubView.tsx b/src/client/views/collections/CollectionSubView.tsx index a1bd1527e..a1ae77fef 100644 --- a/src/client/views/collections/CollectionSubView.tsx +++ b/src/client/views/collections/CollectionSubView.tsx @@ -6,7 +6,7 @@ import { Id } from "../../../new_fields/FieldSymbols"; import { List } from "../../../new_fields/List"; import { listSpec } from "../../../new_fields/Schema"; import { ScriptField } from "../../../new_fields/ScriptField"; -import { Cast, StrCast } from "../../../new_fields/Types"; +import { Cast } from "../../../new_fields/Types"; import { CurrentUserUtils } from "../../../server/authentication/models/current_user_utils"; import { Utils } from "../../../Utils"; import { DocServer } from "../../DocServer"; @@ -279,9 +279,9 @@ export function CollectionSubView(schemaCtor: (doc: Doc) => T) { formData.append('file', file); let dropFileName = file ? file.name : "-empty-"; promises.push(Networking.PostFormDataToServer("/upload", formData).then(results => { - results.map(action((file: any) => { + results.map(action(({ clientAccessPath }: any) => { let full = { ...options, nativeWidth: type.indexOf("video") !== -1 ? 600 : 300, width: 300, title: dropFileName }; - let pathname = Utils.prepend(file.clientAccessPath); + let pathname = Utils.prepend(clientAccessPath); Docs.Get.DocumentFromType(type, pathname, full).then(doc => { doc && (Doc.GetProto(doc).fileUpload = path.basename(pathname).replace("upload_", "").replace(/\.[a-z0-9]*$/, "")); doc && this.props.addDocument(doc); diff --git a/src/server/ApiManagers/DeleteManager.ts b/src/server/ApiManagers/DeleteManager.ts index f58a28ce5..71818c673 100644 --- a/src/server/ApiManagers/DeleteManager.ts +++ b/src/server/ApiManagers/DeleteManager.ts @@ -56,7 +56,6 @@ export default class DeleteManager extends ApiManager { } }); - } } diff --git a/src/server/ApiManagers/GeneralGoogleManager.ts b/src/server/ApiManagers/GeneralGoogleManager.ts index 171912185..629684e0c 100644 --- a/src/server/ApiManagers/GeneralGoogleManager.ts +++ b/src/server/ApiManagers/GeneralGoogleManager.ts @@ -20,8 +20,7 @@ export default class GeneralGoogleManager extends ApiManager { method: Method.GET, subscription: "/readGoogleAccessToken", onValidation: async ({ user, res }) => { - const userId = user.id; - const token = await GoogleApiServerUtils.retrieveAccessToken(userId); + const token = await GoogleApiServerUtils.retrieveAccessToken(user.id); if (!token) { return res.send(GoogleApiServerUtils.generateAuthenticationUrl()); } @@ -37,18 +36,6 @@ export default class GeneralGoogleManager extends ApiManager { } }); - register({ - method: Method.GET, - subscription: "/deleteWithGoogleCredentials", - onValidation: async ({ res, isRelease }) => { - if (isRelease) { - return _permission_denied(res, deletionPermissionError); - } - await Database.Auxiliary.GoogleAuthenticationToken.DeleteAll(); - res.redirect("/delete"); - } - }); - register({ method: Method.POST, subscription: new RouteSubscriber("/googleDocs").add("sector", "action"), diff --git a/src/server/ApiManagers/PDFManager.ts b/src/server/ApiManagers/PDFManager.ts index 4bd750aaf..151b48dd9 100644 --- a/src/server/ApiManagers/PDFManager.ts +++ b/src/server/ApiManagers/PDFManager.ts @@ -4,10 +4,11 @@ import RouteSubscriber from "../RouteSubscriber"; import { exists, createReadStream, createWriteStream } from "fs"; import * as Pdfjs from 'pdfjs-dist'; import { createCanvas } from "canvas"; -const probe = require("probe-image-size"); +const imageSize = require("probe-image-size"); import * as express from "express"; import * as path from "path"; import { Directory, serverPathToFile, clientPathToFile } from "./UploadManager"; +import { ConsoleColors } from "../ActionUtilities"; export default class PDFManager extends ApiManager { @@ -16,84 +17,77 @@ export default class PDFManager extends ApiManager { register({ method: Method.GET, subscription: new RouteSubscriber("thumbnail").add("filename"), - onValidation: ({ req, res }) => { - let filename = req.params.filename; - let noExt = filename.substring(0, filename.length - ".png".length); - let pagenumber = parseInt(noExt.split('-')[1]); - return new Promise(resolve => { - const path = serverPathToFile(Directory.pdf_thumbnails, filename); - exists(path, (exists: boolean) => { - console.log(`${path} ${exists ? "exists" : "does not exist"}`); - if (exists) { - let input = createReadStream(path); - probe(input, (err: any, { width, height }: any) => { - if (err) { - console.log(err); - console.log(`error on ${filename}`); - return; - } - res.send({ - path: clientPathToFile(Directory.pdf_thumbnails, filename), - width, - height - }); - }); - } - else { - const name = filename.substring(0, filename.length - noExt.split('-')[1].length - ".PNG".length - 1) + ".pdf"; - LoadPage(serverPathToFile(Directory.pdfs, name), pagenumber, res); - } - resolve(); - }); - }); - } + onValidation: ({ req, res }) => getOrCreateThumbnail(req.params.filename, res) }); - function LoadPage(file: string, pageNumber: number, res: express.Response) { - console.log(file); - Pdfjs.getDocument(file).promise - .then((pdf: Pdfjs.PDFDocumentProxy) => { - let factory = new NodeCanvasFactory(); - console.log(pageNumber); - pdf.getPage(pageNumber).then((page: Pdfjs.PDFPageProxy) => { - console.log("reading " + page); - let viewport = page.getViewport(1 as any); - let canvasAndContext = factory.create(viewport.width, viewport.height); - let renderContext = { - canvasContext: canvasAndContext.context, - canvasFactory: factory, - viewport - }; - console.log("read " + pageNumber); + } - page.render(renderContext).promise - .then(() => { - console.log("saving " + pageNumber); - let stream = canvasAndContext.canvas.createPNGStream(); - let filenames = path.basename(file).split("."); - const pngFile = serverPathToFile(Directory.pdf_thumbnails, `${filenames[0]}-${pageNumber}.png`); - let out = createWriteStream(pngFile); - stream.pipe(out); - out.on("finish", () => { - console.log(`Success! Saved to ${pngFile}`); - res.send({ - path: pngFile, - width: viewport.width, - height: viewport.height - }); - }); - }, (reason: string) => { - console.error(reason + ` ${pageNumber}`); - }); +} + +function getOrCreateThumbnail(thumbnailName: string, res: express.Response) { + const noExtension = thumbnailName.substring(0, thumbnailName.length - ".png".length); + const pageString = noExtension.split('-')[1]; + const pageNumber = parseInt(pageString); + return new Promise(resolve => { + const path = serverPathToFile(Directory.pdf_thumbnails, thumbnailName); + exists(path, (exists: boolean) => { + if (exists) { + let existingThumbnail = createReadStream(path); + imageSize(existingThumbnail, (err: any, { width, height }: any) => { + if (err) { + console.log(ConsoleColors.Red, `In PDF thumbnail response, unable to determine dimensions of ${thumbnailName}:`); + console.log(err); + return; + } + res.send({ + path: clientPathToFile(Directory.pdf_thumbnails, thumbnailName), + width, + height }); }); - } - - } + } else { + const offset = thumbnailName.length - pageString.length - 5; + const name = thumbnailName.substring(0, offset) + ".pdf"; + const path = serverPathToFile(Directory.pdfs, name); + CreateThumbnail(path, pageNumber, res); + } + resolve(); + }); + }); +} +async function CreateThumbnail(file: string, pageNumber: number, res: express.Response) { + const documentProxy = await Pdfjs.getDocument(file).promise; + const factory = new NodeCanvasFactory(); + const page = await documentProxy.getPage(pageNumber); + const viewport = page.getViewport(1 as any); + const { canvas, context } = factory.create(viewport.width, viewport.height); + const renderContext = { + canvasContext: context, + canvasFactory: factory, + viewport + }; + await page.render(renderContext).promise; + const pngStream = canvas.createPNGStream(); + const filenames = path.basename(file).split("."); + const pngFile = serverPathToFile(Directory.pdf_thumbnails, `${filenames[0]}-${pageNumber}.png`); + const out = createWriteStream(pngFile); + pngStream.pipe(out); + out.on("finish", () => { + res.send({ + path: pngFile, + width: viewport.width, + height: viewport.height + }); + }); + out.on("error", error => { + console.log(ConsoleColors.Red, `In PDF thumbnail creation, encountered the following error when piping ${pngFile}:`); + console.log(error); + }); } class NodeCanvasFactory { + create = (width: number, height: number) => { var canvas = createCanvas(width, height); var context = canvas.getContext('2d'); diff --git a/src/server/ApiManagers/UploadManager.ts b/src/server/ApiManagers/UploadManager.ts index 2f76871a6..80ae0ad61 100644 --- a/src/server/ApiManagers/UploadManager.ts +++ b/src/server/ApiManagers/UploadManager.ts @@ -38,6 +38,27 @@ export default class UploadManager extends ApiManager { protected initialize(register: Registration): void { + register({ + method: Method.POST, + subscription: "/upload", + onValidation: async ({ req, res }) => { + let form = new formidable.IncomingForm(); + form.uploadDir = pathToDirectory(Directory.parsed_files); + form.keepExtensions = true; + return new Promise(resolve => { + form.parse(req, async (_err, _fields, files) => { + let results: any[] = []; + for (const key in files) { + const result = await DashUploadUtils.upload(files[key]); + result && results.push(result); + } + _success(res, results); + resolve(); + }); + }); + } + }); + register({ method: Method.POST, subscription: "/uploadDoc", @@ -142,28 +163,6 @@ export default class UploadManager extends ApiManager { } }); - - register({ - method: Method.POST, - subscription: "/upload", - onValidation: async ({ req, res }) => { - let form = new formidable.IncomingForm(); - form.uploadDir = pathToDirectory(Directory.parsed_files); - form.keepExtensions = true; - return new Promise(resolve => { - form.parse(req, async (_err, _fields, files) => { - let results: any[] = []; - for (const key in files) { - const result = await DashUploadUtils.upload(files[key]); - result && results.push(result); - } - _success(res, results); - resolve(); - }); - }); - } - }); - register({ method: Method.POST, subscription: "/inspectImage", diff --git a/src/server/DashUploadUtils.ts b/src/server/DashUploadUtils.ts index c831eb072..9ccc72e35 100644 --- a/src/server/DashUploadUtils.ts +++ b/src/server/DashUploadUtils.ts @@ -85,7 +85,8 @@ export namespace DashUploadUtils { return UploadPdf(path); } } - console.log(ConsoleColors.Red, `Ignoring unsupported file ${name} with upload type (${type}).`); + + console.log(ConsoleColors.Red, `Ignoring unsupported file (${name}) with upload type (${type}).`); return { clientAccessPath: undefined }; } @@ -169,17 +170,12 @@ export namespace DashUploadUtils { if (isLocal) { return results; } - const metadata = (await new Promise((resolve, reject) => { - request.head(source, async (error, res) => { - if (error) { - return reject(error); - } - resolve(res); - }); - })).headers; + const { headers } = (await new Promise((resolve, reject) => { + request.head(source, (error, res) => error ? reject(error) : resolve(res)); + })); return { - contentSize: parseInt(metadata[size]), - contentType: metadata[type], + contentSize: parseInt(headers[size]), + contentType: headers[type], ...results }; }; diff --git a/src/server/RouteManager.ts b/src/server/RouteManager.ts index 3a20d5af5..7c49485f1 100644 --- a/src/server/RouteManager.ts +++ b/src/server/RouteManager.ts @@ -26,6 +26,8 @@ export interface RouteInitializer { onError?: OnError; } +const registered = new Map>(); + export default class RouteManager { private server: express.Express; private _isRelease: boolean; @@ -89,6 +91,18 @@ export default class RouteManager { } else { route = subscriber.build; } + const existing = registered.get(route); + if (existing) { + if (existing.has(method)) { + console.log(ConsoleColors.Red, `\nDuplicate registration error: already registered ${route} with Method[${method}]`); + console.log('Please remove duplicate registrations before continuing...\n'); + process.exit(0); + } + } else { + const specific = new Set(); + specific.add(method); + registered.set(route, specific); + } switch (method) { case Method.GET: this.server.get(route, supervised); diff --git a/src/server/authentication/models/current_user_utils.ts b/src/server/authentication/models/current_user_utils.ts index 052aa54a6..ac4462f78 100644 --- a/src/server/authentication/models/current_user_utils.ts +++ b/src/server/authentication/models/current_user_utils.ts @@ -1,4 +1,4 @@ -import { action, computed, observable, reaction, runInAction } from "mobx"; +import { action, computed, observable, reaction } from "mobx"; import * as rp from 'request-promise'; import { DocServer } from "../../../client/DocServer"; import { Docs } from "../../../client/documents/Documents"; @@ -11,7 +11,6 @@ import { listSpec } from "../../../new_fields/Schema"; import { ScriptField, ComputedField } from "../../../new_fields/ScriptField"; import { Cast, PromiseValue } from "../../../new_fields/Types"; import { Utils } from "../../../Utils"; -import { ButtonBox } from "../../../client/views/nodes/ButtonBox"; import { nullAudio } from "../../../new_fields/URLField"; import { DragManager } from "../../../client/util/DragManager"; import { InkingControl } from "../../../client/views/InkingControl"; diff --git a/src/server/credentials/test.json b/src/server/credentials/test.json deleted file mode 100644 index 0a032cc2d..000000000 --- a/src/server/credentials/test.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "installed": { - "client_id": "343179513178-ud6tvmh275r2fq93u9eesrnc66t6akh9.apps.googleusercontent.com", - "project_id": "quickstart-1565056383187", - "auth_uri": "https://accounts.google.com/o/oauth2/auth", - "token_uri": "https://oauth2.googleapis.com/token", - "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", - "client_secret": "w8KIFSc0MQpmUYHed4qEzn8b", - "redirect_uris": [ - "urn:ietf:wg:oauth:2.0:oob", - "http://localhost" - ] - } -} \ No newline at end of file -- cgit v1.2.3-70-g09d2