diff options
| author | Eleanor Eng <eleanor.eng5@gmail.com> | 2019-11-16 16:37:36 -0500 |
|---|---|---|
| committer | Eleanor Eng <eleanor.eng5@gmail.com> | 2019-11-16 16:37:36 -0500 |
| commit | b5111d4f4c44b845b7dd324e4332bcdc12eda0b8 (patch) | |
| tree | 3c8ecc0af2accd5a49d6ff3ba1e46e7dab82858d /src | |
| parent | c24c16efbf798eac1003811f6c0ed98bfc65f48e (diff) | |
| parent | ab285371f6fb2a4f1e64888bafbc84b602f23416 (diff) | |
merge
Diffstat (limited to 'src')
44 files changed, 1608 insertions, 1176 deletions
diff --git a/src/Utils.ts b/src/Utils.ts index 9a2f01f80..abff2eaba 100644 --- a/src/Utils.ts +++ b/src/Utils.ts @@ -2,7 +2,6 @@ import v4 = require('uuid/v4'); import v5 = require("uuid/v5"); import { Socket } from 'socket.io'; import { Message } from './server/Message'; -import { RouteStore } from './server/RouteStore'; export namespace Utils { @@ -46,7 +45,7 @@ export namespace Utils { } export function CorsProxy(url: string): string { - return prepend(RouteStore.corsProxy + "/") + encodeURIComponent(url); + return prepend("/corsProxy/") + encodeURIComponent(url); } export function CopyText(text: string) { diff --git a/src/client/Network.ts b/src/client/Network.ts index 75ccb5e99..f9ef27267 100644 --- a/src/client/Network.ts +++ b/src/client/Network.ts @@ -1,18 +1,16 @@ import { Utils } from "../Utils"; -import { CurrentUserUtils } from "../server/authentication/models/current_user_utils"; import requestPromise = require('request-promise'); -export namespace Identified { +export namespace Networking { export async function FetchFromServer(relativeRoute: string) { - return (await fetch(relativeRoute, { headers: { userId: CurrentUserUtils.id } })).text(); + return (await fetch(relativeRoute)).text(); } export async function PostToServer(relativeRoute: string, body?: any) { let options = { uri: Utils.prepend(relativeRoute), method: "POST", - headers: { userId: CurrentUserUtils.id }, body, json: true }; @@ -22,12 +20,10 @@ export namespace Identified { export async function PostFormDataToServer(relativeRoute: string, formData: FormData) { const parameters = { method: 'POST', - headers: { userId: CurrentUserUtils.id }, - body: formData, + body: formData }; const response = await fetch(relativeRoute, parameters); - const text = await response.json(); - return text; + return response.json(); } }
\ No newline at end of file diff --git a/src/client/apis/GoogleAuthenticationManager.tsx b/src/client/apis/GoogleAuthenticationManager.tsx index 01dac3996..ae77c4b7b 100644 --- a/src/client/apis/GoogleAuthenticationManager.tsx +++ b/src/client/apis/GoogleAuthenticationManager.tsx @@ -3,8 +3,7 @@ import { observer } from "mobx-react"; import * as React from "react"; import MainViewModal from "../views/MainViewModal"; import { Opt } from "../../new_fields/Doc"; -import { Identified } from "../Network"; -import { RouteStore } from "../../server/RouteStore"; +import { Networking } from "../Network"; import "./GoogleAuthenticationManager.scss"; const AuthenticationUrl = "https://accounts.google.com/o/oauth2/v2/auth"; @@ -31,7 +30,7 @@ export default class GoogleAuthenticationManager extends React.Component<{}> { } public fetchOrGenerateAccessToken = async () => { - let response = await Identified.FetchFromServer(RouteStore.readGoogleAccessToken); + let response = await Networking.FetchFromServer("/readGoogleAccessToken"); // if this is an authentication url, activate the UI to register the new access token if (new RegExp(AuthenticationUrl).test(response)) { this.isOpen = true; @@ -39,24 +38,25 @@ export default class GoogleAuthenticationManager extends React.Component<{}> { return new Promise<string>(async resolve => { const disposer = reaction( () => this.authenticationCode, - authenticationCode => { - if (authenticationCode) { - Identified.PostToServer(RouteStore.writeGoogleAccessToken, { authenticationCode }).then( - ({ access_token, avatar, name }) => { - runInAction(() => { - this.avatar = avatar; - this.username = name; - }); - this.beginFadeout(); - disposer(); - resolve(access_token); - }, - action(() => { - this.hasBeenClicked = false; - this.success = false; - }) - ); + async authenticationCode => { + if (!authenticationCode) { + return; } + const { access_token, avatar, name } = await Networking.PostToServer( + "/writeGoogleAccessToken", + { authenticationCode } + ); + runInAction(() => { + this.avatar = avatar; + this.username = name; + }); + this.beginFadeout(); + disposer(); + resolve(access_token); + action(() => { + this.hasBeenClicked = false; + this.success = false; + }); } ); }); diff --git a/src/client/apis/google_docs/GoogleApiClientUtils.ts b/src/client/apis/google_docs/GoogleApiClientUtils.ts index 1cf01fc3d..26c7f8d2e 100644 --- a/src/client/apis/google_docs/GoogleApiClientUtils.ts +++ b/src/client/apis/google_docs/GoogleApiClientUtils.ts @@ -1,9 +1,8 @@ import { docs_v1, slides_v1 } from "googleapis"; -import { RouteStore } from "../../../server/RouteStore"; import { Opt } from "../../../new_fields/Doc"; import { isArray } from "util"; import { EditorState } from "prosemirror-state"; -import { Identified } from "../../Network"; +import { Networking } from "../../Network"; export const Pulls = "googleDocsPullCount"; export const Pushes = "googleDocsPushCount"; @@ -77,14 +76,14 @@ export namespace GoogleApiClientUtils { * @returns the documentId of the newly generated document, or undefined if the creation process fails. */ export const create = async (options: CreateOptions): Promise<CreationResult> => { - const path = `${RouteStore.googleDocs}/Documents/${Actions.Create}`; + const path = `/googleDocs/Documents/${Actions.Create}`; const parameters = { requestBody: { title: options.title || `Dash Export (${new Date().toDateString()})` } }; try { - const schema: docs_v1.Schema$Document = await Identified.PostToServer(path, parameters); + const schema: docs_v1.Schema$Document = await Networking.PostToServer(path, parameters); return schema.documentId; } catch { return undefined; @@ -154,10 +153,10 @@ export namespace GoogleApiClientUtils { } export const retrieve = async (options: RetrieveOptions): Promise<RetrievalResult> => { - const path = `${RouteStore.googleDocs}/Documents/${Actions.Retrieve}`; + const path = `/googleDocs/Documents/${Actions.Retrieve}`; try { const parameters = { documentId: options.documentId }; - const schema: RetrievalResult = await Identified.PostToServer(path, parameters); + const schema: RetrievalResult = await Networking.PostToServer(path, parameters); return schema; } catch { return undefined; @@ -165,7 +164,7 @@ export namespace GoogleApiClientUtils { }; export const update = async (options: UpdateOptions): Promise<UpdateResult> => { - const path = `${RouteStore.googleDocs}/Documents/${Actions.Update}`; + const path = `/googleDocs/Documents/${Actions.Update}`; const parameters = { documentId: options.documentId, requestBody: { @@ -173,7 +172,7 @@ export namespace GoogleApiClientUtils { } }; try { - const replies: UpdateResult = await Identified.PostToServer(path, parameters); + const replies: UpdateResult = await Networking.PostToServer(path, parameters); return replies; } catch { return undefined; diff --git a/src/client/apis/google_docs/GooglePhotosClientUtils.ts b/src/client/apis/google_docs/GooglePhotosClientUtils.ts index e93fa6eb4..bf8897061 100644 --- a/src/client/apis/google_docs/GooglePhotosClientUtils.ts +++ b/src/client/apis/google_docs/GooglePhotosClientUtils.ts @@ -1,5 +1,4 @@ import { Utils } from "../../../Utils"; -import { RouteStore } from "../../../server/RouteStore"; import { ImageField } from "../../../new_fields/URLField"; import { Cast, StrCast } from "../../../new_fields/Types"; import { Doc, Opt, DocListCastAsync } from "../../../new_fields/Doc"; @@ -13,7 +12,7 @@ import { Docs, DocumentOptions } from "../../documents/Documents"; import { NewMediaItemResult, MediaItem } from "../../../server/apis/google/SharedTypes"; import { AssertionError } from "assert"; import { DocumentView } from "../../views/nodes/DocumentView"; -import { Identified } from "../../Network"; +import { Networking } from "../../Network"; import GoogleAuthenticationManager from "../GoogleAuthenticationManager"; export namespace GooglePhotos { @@ -78,6 +77,7 @@ export namespace GooglePhotos { } export const CollectionToAlbum = async (options: AlbumCreationOptions): Promise<Opt<AlbumCreationResult>> => { + await GoogleAuthenticationManager.Instance.fetchOrGenerateAccessToken(); const { collection, title, descriptionKey, tag } = options; const dataDocument = Doc.GetProto(collection); const images = ((await DocListCastAsync(dataDocument.data)) || []).filter(doc => Cast(doc.data, ImageField)); @@ -127,6 +127,7 @@ export namespace GooglePhotos { export type CollectionConstructor = (data: Array<Doc>, options: DocumentOptions, ...args: any) => Doc; export const CollectionFromSearch = async (constructor: CollectionConstructor, requested: Opt<Partial<Query.SearchOptions>>): Promise<Doc> => { + await GoogleAuthenticationManager.Instance.fetchOrGenerateAccessToken(); let response = await Query.ContentSearch(requested); let uploads = await Transactions.WriteMediaItemsToServer(response); const children = uploads.map((upload: Transactions.UploadInformation) => { @@ -147,6 +148,7 @@ export namespace GooglePhotos { const comparator = (a: string, b: string) => (a < b) ? -1 : (a > b ? 1 : 0); export const TagChildImages = async (collection: Doc) => { + await GoogleAuthenticationManager.Instance.fetchOrGenerateAccessToken(); const idMapping = await Cast(collection.googlePhotosIdMapping, Doc); if (!idMapping) { throw new Error("Appending image metadata requires that the targeted collection have already been mapped to an album!"); @@ -304,7 +306,7 @@ export namespace GooglePhotos { }; export const WriteMediaItemsToServer = async (body: { mediaItems: any[] }): Promise<UploadInformation[]> => { - const uploads = await Identified.PostToServer(RouteStore.googlePhotosMediaDownload, body); + const uploads = await Networking.PostToServer("/googlePhotosMediaDownload", body); return uploads; }; @@ -325,6 +327,7 @@ export namespace GooglePhotos { } export const UploadImages = async (sources: Doc[], album?: AlbumReference, descriptionKey = "caption"): Promise<Opt<ImageUploadResults>> => { + await GoogleAuthenticationManager.Instance.fetchOrGenerateAccessToken(); if (album && "title" in album) { album = await Create.Album(album.title); } @@ -341,7 +344,7 @@ export namespace GooglePhotos { media.push({ url, description }); } if (media.length) { - const results = await Identified.PostToServer(RouteStore.googlePhotosMediaUpload, { media, album }); + const results = await Networking.PostToServer("/googlePhotosMediaUpload", { media, album }); return results; } }; diff --git a/src/client/cognitive_services/CognitiveServices.ts b/src/client/cognitive_services/CognitiveServices.ts index 08fcb4883..e74aef998 100644 --- a/src/client/cognitive_services/CognitiveServices.ts +++ b/src/client/cognitive_services/CognitiveServices.ts @@ -2,7 +2,6 @@ import * as request from "request-promise"; import { Doc, Field, Opt } from "../../new_fields/Doc"; import { Cast } from "../../new_fields/Types"; import { Docs } from "../documents/Documents"; -import { RouteStore } from "../../server/RouteStore"; import { Utils } from "../../Utils"; import { InkData } from "../../new_fields/InkField"; import { UndoManager } from "../util/UndoManager"; @@ -39,7 +38,7 @@ export enum Confidence { export namespace CognitiveServices { const ExecuteQuery = async <D>(service: Service, manager: APIManager<D>, data: D): Promise<any> => { - return fetch(Utils.prepend(`${RouteStore.cognitiveServices}/${service}`)).then(async response => { + return fetch(Utils.prepend(`environment/${service}`)).then(async response => { let apiKey = await response.text(); if (!apiKey) { console.log(`No API key found for ${service}: ensure index.ts has access to a .env file in your root directory`); diff --git a/src/client/util/History.ts b/src/client/util/History.ts index 899abbe40..1c51236cb 100644 --- a/src/client/util/History.ts +++ b/src/client/util/History.ts @@ -1,6 +1,5 @@ import { Doc, Opt, Field } from "../../new_fields/Doc"; import { DocServer } from "../DocServer"; -import { RouteStore } from "../../server/RouteStore"; import { MainView } from "../views/MainView"; import * as qs from 'query-string'; import { Utils, OmitKeys } from "../../Utils"; @@ -26,7 +25,7 @@ export namespace HistoryUtil { // const handlers: ((state: ParsedUrl | null) => void)[] = []; function onHistory(e: PopStateEvent) { - if (window.location.pathname !== RouteStore.home) { + if (window.location.pathname !== "/home") { const url = e.state as ParsedUrl || parseUrl(window.location); if (url) { switch (url.type) { diff --git a/src/client/util/Import & Export/DirectoryImportBox.tsx b/src/client/util/Import & Export/DirectoryImportBox.tsx index d74b51993..437e7766b 100644 --- a/src/client/util/Import & Export/DirectoryImportBox.tsx +++ b/src/client/util/Import & Export/DirectoryImportBox.tsx @@ -1,7 +1,6 @@ import "fs"; import React = require("react"); import { Doc, DocListCast, DocListCastAsync, Opt } from "../../../new_fields/Doc"; -import { RouteStore } from "../../../server/RouteStore"; import { action, observable, autorun, runInAction, computed, reaction, IReactionDisposer } from "mobx"; import { FieldViewProps, FieldView } from "../../views/nodes/FieldView"; import Measure, { ContentRect } from "react-measure"; @@ -20,9 +19,11 @@ import { listSpec } from "../../../new_fields/Schema"; import { GooglePhotos } from "../../apis/google_docs/GooglePhotosClientUtils"; import { SchemaHeaderField } from "../../../new_fields/SchemaHeaderField"; import "./DirectoryImportBox.scss"; -import { Identified } from "../../Network"; +import { Networking } from "../../Network"; import { BatchedArray } from "array-batcher"; -import { ExifData } from "exif"; +import * as path from 'path'; +import { DashUploadUtils } from "../../../server/DashUploadUtils"; +import { SharedMediaTypes } from "../../../server/SharedMediaTypes"; const unsupported = ["text/html", "text/plain"]; @@ -95,7 +96,12 @@ export default class DirectoryImportBox extends React.Component<FieldViewProps> let validated: File[] = []; for (let i = 0; i < files.length; i++) { let file = files.item(i); - file && !unsupported.includes(file.type) && validated.push(file); + if (file && !unsupported.includes(file.type)) { + const ext = path.extname(file.name).toLowerCase(); + if (SharedMediaTypes.imageFormats.includes(ext)) { + validated.push(file); + } + } } runInAction(() => { @@ -108,7 +114,7 @@ export default class DirectoryImportBox extends React.Component<FieldViewProps> runInAction(() => this.phase = `Internal: uploading ${this.quota - this.completed} files to Dash...`); - const uploads = await BatchedArray.from(validated, { batchSize: 15 }).batchedMapAsync(async batch => { + const uploads = await BatchedArray.from(validated, { batchSize: 15 }).batchedMapAsync<ImageUploadResponse>(async (batch, collector) => { const formData = new FormData(); batch.forEach(file => { @@ -117,9 +123,8 @@ export default class DirectoryImportBox extends React.Component<FieldViewProps> formData.append(Utils.GenerateGuid(), file); }); - const responses = await Identified.PostFormDataToServer(RouteStore.upload, formData); + collector.push(...(await Networking.PostFormDataToServer("/upload", formData))); runInAction(() => this.completed += batch.length); - return responses as ImageUploadResponse[]; }); await Promise.all(uploads.map(async upload => { diff --git a/src/client/util/Import & Export/ImageUtils.ts b/src/client/util/Import & Export/ImageUtils.ts index c9abf38fa..ca80f3bca 100644 --- a/src/client/util/Import & Export/ImageUtils.ts +++ b/src/client/util/Import & Export/ImageUtils.ts @@ -1,9 +1,8 @@ -import { Doc, DocListCast, DocListCastAsync, Opt } from "../../../new_fields/Doc"; +import { Doc } from "../../../new_fields/Doc"; import { ImageField } from "../../../new_fields/URLField"; import { Cast, StrCast } from "../../../new_fields/Types"; -import { RouteStore } from "../../../server/RouteStore"; import { Docs } from "../../documents/Documents"; -import { Identified } from "../../Network"; +import { Networking } from "../../Network"; import { Id } from "../../../new_fields/FieldSymbols"; import { Utils } from "../../../Utils"; @@ -15,7 +14,7 @@ export namespace ImageUtils { return false; } const source = field.url.href; - const response = await Identified.PostToServer(RouteStore.inspectImage, { source }); + const response = await Networking.PostToServer("/inspectImage", { source }); const { error, data } = response.exifData; document.exif = error || Docs.Get.DocumentHierarchyFromJson(data); return data !== undefined; @@ -23,7 +22,7 @@ export namespace ImageUtils { export const ExportHierarchyToFileSystem = async (collection: Doc): Promise<void> => { const a = document.createElement("a"); - a.href = Utils.prepend(`${RouteStore.imageHierarchyExport}/${collection[Id]}`); + a.href = Utils.prepend(`imageHierarchyExport/${collection[Id]}`); a.download = `Dash Export [${StrCast(collection.title)}].zip`; a.click(); }; diff --git a/src/client/util/SharingManager.tsx b/src/client/util/SharingManager.tsx index 2082d6324..cc1d628b1 100644 --- a/src/client/util/SharingManager.tsx +++ b/src/client/util/SharingManager.tsx @@ -4,7 +4,6 @@ import MainViewModal from "../views/MainViewModal"; import { Doc, Opt, DocCastAsync } from "../../new_fields/Doc"; import { DocServer } from "../DocServer"; import { Cast, StrCast } from "../../new_fields/Types"; -import { RouteStore } from "../../server/RouteStore"; import * as RequestPromise from "request-promise"; import { Utils } from "../../Utils"; import "./SharingManager.scss"; @@ -104,7 +103,7 @@ export default class SharingManager extends React.Component<{}> { } populateUsers = async () => { - let userList = await RequestPromise.get(Utils.prepend(RouteStore.getUsers)); + let userList = await RequestPromise.get(Utils.prepend("/getUsers")); const raw = JSON.parse(userList) as User[]; const evaluating = raw.map(async user => { let isCandidate = user.email !== Doc.CurrentUserEmail; diff --git a/src/client/views/MainView.tsx b/src/client/views/MainView.tsx index 39585113b..0c5a1003b 100644 --- a/src/client/views/MainView.tsx +++ b/src/client/views/MainView.tsx @@ -12,7 +12,6 @@ import { List } from '../../new_fields/List'; import { listSpec } from '../../new_fields/Schema'; import { Cast, FieldValue, StrCast } from '../../new_fields/Types'; import { CurrentUserUtils } from '../../server/authentication/models/current_user_utils'; -import { RouteStore } from '../../server/RouteStore'; import { emptyFunction, returnEmptyString, returnFalse, returnOne, returnTrue, Utils } from '../../Utils'; import GoogleAuthenticationManager from '../apis/GoogleAuthenticationManager'; import { DocServer } from '../DocServer'; @@ -74,7 +73,7 @@ export class MainView extends React.Component { this._urlState = HistoryUtil.parseUrl(window.location) || {} as any; // causes errors to be generated when modifying an observable outside of an action configure({ enforceActions: "observed" }); - if (window.location.pathname !== RouteStore.home) { + if (window.location.pathname !== "/home") { let pathname = window.location.pathname.substr(1).split("/"); if (pathname.length > 1) { let type = pathname[0]; @@ -395,7 +394,7 @@ export class MainView extends React.Component { zoomToScale={emptyFunction} getScale={returnOne}> </DocumentView> - <button className="mainView-logout" key="logout" onClick={() => window.location.assign(Utils.prepend(RouteStore.logout))}> + <button className="mainView-logout" key="logout" onClick={() => window.location.assign(Utils.prepend("/logout"))}> {CurrentUserUtils.GuestWorkspace ? "Exit" : "Log Out"} </button> </div></div>; diff --git a/src/client/views/collections/CollectionSubView.tsx b/src/client/views/collections/CollectionSubView.tsx index 6e8e4fa12..306f8e052 100644 --- a/src/client/views/collections/CollectionSubView.tsx +++ b/src/client/views/collections/CollectionSubView.tsx @@ -8,7 +8,6 @@ import { listSpec } from "../../../new_fields/Schema"; import { ScriptField } from "../../../new_fields/ScriptField"; import { Cast } from "../../../new_fields/Types"; import { CurrentUserUtils } from "../../../server/authentication/models/current_user_utils"; -import { RouteStore } from "../../../server/RouteStore"; import { Utils } from "../../../Utils"; import { DocServer } from "../../DocServer"; import { DocumentType } from "../../documents/DocumentTypes"; @@ -243,7 +242,6 @@ export function CollectionSubView<T>(schemaCtor: (doc: Doc) => T) { let promises: Promise<void>[] = []; // tslint:disable-next-line:prefer-for-of for (let i = 0; i < e.dataTransfer.items.length; i++) { - const upload = window.location.origin + RouteStore.upload; let item = e.dataTransfer.items[i]; if (item.kind === "string" && item.type.indexOf("uri") !== -1) { let str: string; @@ -268,7 +266,7 @@ export function CollectionSubView<T>(schemaCtor: (doc: Doc) => T) { } let dropFileName = file ? file.name : "-empty-"; - let prom = fetch(upload, { + let prom = fetch(Utils.prepend("/upload"), { method: 'POST', body: formData }).then(async (res: Response) => { diff --git a/src/client/views/nodes/ImageBox.tsx b/src/client/views/nodes/ImageBox.tsx index 9f39eccea..07fd832be 100644 --- a/src/client/views/nodes/ImageBox.tsx +++ b/src/client/views/nodes/ImageBox.tsx @@ -12,7 +12,6 @@ import { createSchema, listSpec, makeInterface } from '../../../new_fields/Schem import { ComputedField } from '../../../new_fields/ScriptField'; import { BoolCast, Cast, FieldValue, NumCast, StrCast } from '../../../new_fields/Types'; import { AudioField, ImageField } from '../../../new_fields/URLField'; -import { RouteStore } from '../../../server/RouteStore'; import { Utils, returnOne, emptyFunction } from '../../../Utils'; import { CognitiveServices, Confidence, Service, Tag } from '../../cognitive_services/CognitiveServices'; import { Docs } from '../../documents/Documents'; @@ -152,7 +151,7 @@ export class ImageBox extends DocAnnotatableComponent<FieldViewProps, ImageDocum recorder.ondataavailable = async function (e: any) { const formData = new FormData(); formData.append("file", e.data); - const res = await fetch(Utils.prepend(RouteStore.upload), { + const res = await fetch(Utils.prepend("/upload"), { method: 'POST', body: formData }); diff --git a/src/client/views/nodes/VideoBox.tsx b/src/client/views/nodes/VideoBox.tsx index aa9b28118..4884fa2bf 100644 --- a/src/client/views/nodes/VideoBox.tsx +++ b/src/client/views/nodes/VideoBox.tsx @@ -6,7 +6,6 @@ import { InkTool } from "../../../new_fields/InkField"; import { makeInterface, createSchema, listSpec } from "../../../new_fields/Schema"; import { Cast, FieldValue, NumCast, BoolCast, StrCast } from "../../../new_fields/Types"; import { VideoField } from "../../../new_fields/URLField"; -import { RouteStore } from "../../../server/RouteStore"; import { Utils, emptyFunction, returnOne } from "../../../Utils"; import { Docs, DocUtils } from "../../documents/Documents"; import { ContextMenu } from "../ContextMenu"; @@ -183,7 +182,7 @@ export class VideoBox extends DocAnnotatableComponent<FieldViewProps, VideoDocum public static async convertDataUri(imageUri: string, returnedFilename: string) { try { - let posting = Utils.prepend(RouteStore.dataUriToImage); + let posting = Utils.prepend("/uploadURI"); const returnedUri = await rp.post(posting, { body: { uri: imageUri, diff --git a/src/client/views/search/SearchBox.tsx b/src/client/views/search/SearchBox.tsx index b728ffaa9..6100c74df 100644 --- a/src/client/views/search/SearchBox.tsx +++ b/src/client/views/search/SearchBox.tsx @@ -8,7 +8,6 @@ import * as rp from 'request-promise'; import { Doc } from '../../../new_fields/Doc'; import { Id } from '../../../new_fields/FieldSymbols'; import { Cast, NumCast } from '../../../new_fields/Types'; -import { RouteStore } from '../../../server/RouteStore'; import { Utils } from '../../../Utils'; import { Docs } from '../../documents/Documents'; import { SetupDrag } from '../../util/DragManager'; @@ -90,7 +89,7 @@ export class SearchBox extends React.Component { public static async convertDataUri(imageUri: string, returnedFilename: string) { try { - let posting = Utils.prepend(RouteStore.dataUriToImage); + let posting = Utils.prepend("uploadURI"); const returnedUri = await rp.post(posting, { body: { uri: imageUri, diff --git a/src/mobile/ImageUpload.tsx b/src/mobile/ImageUpload.tsx index 33a615cbf..9fdaac66e 100644 --- a/src/mobile/ImageUpload.tsx +++ b/src/mobile/ImageUpload.tsx @@ -1,7 +1,6 @@ import * as ReactDOM from 'react-dom'; import * as rp from 'request-promise'; import { Docs } from '../client/documents/Documents'; -import { RouteStore } from '../server/RouteStore'; import "./ImageUpload.scss"; import React = require('react'); import { DocServer } from '../client/DocServer'; @@ -58,7 +57,7 @@ class Uploader extends React.Component { this.status = "getting user document"; - const res = await rp.get(Utils.prepend(RouteStore.getUserDocumentId)); + const res = await rp.get(Utils.prepend("/getUserDocumentId")); if (!res) { throw new Error("No user id returned"); } diff --git a/src/new_fields/RichTextUtils.ts b/src/new_fields/RichTextUtils.ts index 601939ed2..238b27439 100644 --- a/src/new_fields/RichTextUtils.ts +++ b/src/new_fields/RichTextUtils.ts @@ -8,7 +8,6 @@ import { Opt, Doc } from "./Doc"; import Color = require('color'); import { sinkListItem } from "prosemirror-schema-list"; import { Utils } from "../Utils"; -import { RouteStore } from "../server/RouteStore"; import { Docs } from "../client/documents/Documents"; import { schema } from "../client/util/RichTextSchema"; import { GooglePhotos } from "../client/apis/google_docs/GooglePhotosClientUtils"; @@ -17,7 +16,7 @@ import { Cast, StrCast } from "./Types"; import { Id } from "./FieldSymbols"; import { DocumentView } from "../client/views/nodes/DocumentView"; import { AssertionError } from "assert"; -import { Identified } from "../client/Network"; +import { Networking } from "../client/Network"; export namespace RichTextUtils { @@ -129,7 +128,7 @@ export namespace RichTextUtils { return { baseUrl, filename }; }); - const uploads = await Identified.PostToServer(RouteStore.googlePhotosMediaDownload, { mediaItems }); + const uploads = await Networking.PostToServer("/googlePhotosMediaDownload", { mediaItems }); if (uploads.length !== mediaItems.length) { throw new AssertionError({ expected: mediaItems.length, actual: uploads.length, message: "Error with internally uploading inlineObjects!" }); diff --git a/src/server/ApiManagers/ApiManager.ts b/src/server/ApiManagers/ApiManager.ts index 264c78a17..9fd726060 100644 --- a/src/server/ApiManagers/ApiManager.ts +++ b/src/server/ApiManagers/ApiManager.ts @@ -1,7 +1,11 @@ -import RouteManager from "../RouteManager"; +import RouteManager, { RouteInitializer } from "../RouteManager"; -export default abstract class ApiManager { +export type Registration = (initializer: RouteInitializer) => void; - public abstract register(router: RouteManager): void; +export default abstract class ApiManager { + protected abstract initialize(register: Registration): void; + public register(router: RouteManager) { + this.initialize(router.addSupervisedRoute); + } }
\ No newline at end of file diff --git a/src/server/ApiManagers/DeleteManager.ts b/src/server/ApiManagers/DeleteManager.ts new file mode 100644 index 000000000..f58a28ce5 --- /dev/null +++ b/src/server/ApiManagers/DeleteManager.ts @@ -0,0 +1,64 @@ +import ApiManager, { Registration } from "./ApiManager"; +import { Method, _permission_denied } from "../RouteManager"; +import { WebSocket } from "../Websocket/Websocket"; +import { Database } from "../database"; + +export default class DeleteManager extends ApiManager { + + protected initialize(register: Registration): void { + + register({ + method: Method.GET, + subscription: "/delete", + onValidation: async ({ res, isRelease }) => { + if (isRelease) { + return _permission_denied(res, deletionPermissionError); + } + await WebSocket.deleteFields(); + res.redirect("/home"); + } + }); + + register({ + method: Method.GET, + subscription: "/deleteAll", + onValidation: async ({ res, isRelease }) => { + if (isRelease) { + return _permission_denied(res, deletionPermissionError); + } + await WebSocket.deleteAll(); + res.redirect("/home"); + } + }); + + + register({ + method: Method.GET, + subscription: "/deleteWithAux", + onValidation: async ({ res, isRelease }) => { + if (isRelease) { + return _permission_denied(res, deletionPermissionError); + } + await Database.Auxiliary.DeleteAll(); + res.redirect("/delete"); + } + }); + + register({ + method: Method.GET, + subscription: "/deleteWithGoogleCredentials", + onValidation: async ({ res, isRelease }) => { + if (isRelease) { + return _permission_denied(res, deletionPermissionError); + } + await Database.Auxiliary.GoogleAuthenticationToken.DeleteAll(); + res.redirect("/delete"); + } + }); + + + } + +} + +const deletionPermissionError = "Cannot perform a delete operation outside of the development environment!"; diff --git a/src/server/ApiManagers/ExportManager.ts b/src/server/ApiManagers/ExportManager.ts new file mode 100644 index 000000000..fc6ba0d22 --- /dev/null +++ b/src/server/ApiManagers/ExportManager.ts @@ -0,0 +1,267 @@ +import ApiManager, { Registration } from "./ApiManager"; +import { Method } from "../RouteManager"; +import RouteSubscriber from "../RouteSubscriber"; +import * as Archiver from 'archiver'; +import * as express from 'express'; +import { Database } from "../database"; +import * as path from "path"; +import { DashUploadUtils } from "../DashUploadUtils"; +import { publicDirectory } from ".."; + +export type Hierarchy = { [id: string]: string | Hierarchy }; +export type ZipMutator = (file: Archiver.Archiver) => void | Promise<void>; +export interface DocumentElements { + data: string | any[]; + title: string; +} + +export default class DownloadManager extends ApiManager { + + protected initialize(register: Registration): void { + + /** + * Let's say someone's using Dash to organize images in collections. + * This lets them export the hierarchy they've built to their + * own file system in a useful format. + * + * This handler starts with a single document id (interesting only + * if it's that of a collection). It traverses the database, captures + * the nesting of only nested images or collections, writes + * that to a zip file and returns it to the client for download. + */ + register({ + method: Method.GET, + subscription: new RouteSubscriber("imageHierarchyExport").add('docId'), + onValidation: async ({ req, res }) => { + const id = req.params.docId; + const hierarchy: Hierarchy = {}; + await buildHierarchyRecursive(id, hierarchy); + return BuildAndDispatchZip(res, zip => writeHierarchyRecursive(zip, hierarchy)); + } + }); + + register({ + method: Method.GET, + subscription: new RouteSubscriber("downloadId").add("docId"), + onValidation: async ({ req, res }) => { + return BuildAndDispatchZip(res, async zip => { + const { id, docs, files } = await getDocs(req.params.docId); + const docString = JSON.stringify({ id, docs }); + zip.append(docString, { name: "doc.json" }); + files.forEach(val => { + zip.file(publicDirectory + val, { name: val.substring(1) }); + }); + }); + } + }); + + register({ + method: Method.GET, + subscription: new RouteSubscriber("/serializeDoc").add("docId"), + onValidation: async ({ req, res }) => { + const { docs, files } = await getDocs(req.params.docId); + res.send({ docs, files: Array.from(files) }); + } + }); + + + } + +} + +async function getDocs(id: string) { + const files = new Set<string>(); + const docs: { [id: string]: any } = {}; + const fn = (doc: any): string[] => { + const id = doc.id; + if (typeof id === "string" && id.endsWith("Proto")) { + //Skip protos + return []; + } + const ids: string[] = []; + for (const key in doc.fields) { + if (!doc.fields.hasOwnProperty(key)) { + continue; + } + const field = doc.fields[key]; + if (field === undefined || field === null) { + continue; + } + + if (field.__type === "proxy" || field.__type === "prefetch_proxy") { + ids.push(field.fieldId); + } else if (field.__type === "script" || field.__type === "computed") { + if (field.captures) { + ids.push(field.captures.fieldId); + } + } else if (field.__type === "list") { + ids.push(...fn(field)); + } else if (typeof field === "string") { + const re = /"(?:dataD|d)ocumentId"\s*:\s*"([\w\-]*)"/g; + let match: string[] | null; + while ((match = re.exec(field)) !== null) { + ids.push(match[1]); + } + } else if (field.__type === "RichTextField") { + const re = /"href"\s*:\s*"(.*?)"/g; + let match: string[] | null; + while ((match = re.exec(field.Data)) !== null) { + const urlString = match[1]; + const split = new URL(urlString).pathname.split("doc/"); + if (split.length > 1) { + ids.push(split[split.length - 1]); + } + } + const re2 = /"src"\s*:\s*"(.*?)"/g; + while ((match = re2.exec(field.Data)) !== null) { + const urlString = match[1]; + const pathname = new URL(urlString).pathname; + files.add(pathname); + } + } else if (["audio", "image", "video", "pdf", "web"].includes(field.__type)) { + const url = new URL(field.url); + const pathname = url.pathname; + files.add(pathname); + } + } + + if (doc.id) { + docs[doc.id] = doc; + } + return ids; + }; + await Database.Instance.visit([id], fn); + return { id, docs, files }; +} + +/** + * This utility function factors out the process + * of creating a zip file and sending it back to the client + * by piping it into a response. + * + * Learn more about piping and readable / writable streams here! + * https://www.freecodecamp.org/news/node-js-streams-everything-you-need-to-know-c9141306be93/ + * + * @param res the writable stream response object that will transfer the generated zip file + * @param mutator the callback function used to actually modify and insert information into the zip instance + */ +export async function BuildAndDispatchZip(res: express.Response, mutator: ZipMutator): Promise<void> { + res.set('Content-disposition', `attachment;`); + res.set('Content-Type', "application/zip"); + const zip = Archiver('zip'); + zip.pipe(res); + await mutator(zip); + return zip.finalize(); +} + +/** + * This function starts with a single document id as a seed, + * typically that of a collection, and then descends the entire tree + * of image or collection documents that are reachable from that seed. + * @param seedId the id of the root of the subtree we're trying to capture, interesting only if it's a collection + * @param hierarchy the data structure we're going to use to record the nesting of the collections and images as we descend + */ + +/* +Below is an example of the JSON hierarchy built from two images contained inside a collection titled 'a nested collection', +following the general recursive structure shown immediately below +{ + "parent folder name":{ + "first child's fild name":"first child's url" + ... + "nth child's fild name":"nth child's url" + } +} +{ + "a nested collection (865c4734-c036-4d67-a588-c71bb43d1440)":{ + "an image of a cat (ace99ffd-8ed8-4026-a5d5-a353fff57bdd).jpg":"https://upload.wikimedia.org/wikipedia/commons/thumb/3/3a/Cat03.jpg/1200px-Cat03.jpg", + "1*SGJw31T5Q9Zfsk24l2yirg.gif (9321cc9b-9b3e-4cb6-b99c-b7e667340f05).gif":"https://cdn-media-1.freecodecamp.org/images/1*SGJw31T5Q9Zfsk24l2yirg.gif" + } +} +*/ +async function buildHierarchyRecursive(seedId: string, hierarchy: Hierarchy): Promise<void> { + const { title, data } = await getData(seedId); + const label = `${title} (${seedId})`; + // is the document a collection? + if (Array.isArray(data)) { + // recurse over all documents in the collection. + const local: Hierarchy = {}; // create a child hierarchy for this level, which will get passed in as the parent of the recursive call + hierarchy[label] = local; // store it at the index in the parent, so we'll end up with a map of maps of maps + await Promise.all(data.map(proxy => buildHierarchyRecursive(proxy.fieldId, local))); + } else { + // now, data can only be a string, namely the url of the image + const filename = label + path.extname(data); // this is the file name under which the output image will be stored + hierarchy[filename] = data; + } +} + +/** + * This is a very specific utility method to help traverse the database + * to parse data and titles out of images and collections alone. + * + * We don't know if the document id given to is corresponds to a view document or a data + * document. If it's a data document, the response from the database will have + * a data field. If not, call recursively on the proto, and resolve with *its* data + * + * @param targetId the id of the Dash document whose data is being requests + * @returns the data of the document, as well as its title + */ +async function getData(targetId: string): Promise<DocumentElements> { + return new Promise<DocumentElements>((resolve, reject) => { + Database.Instance.getDocument(targetId, async (result: any) => { + const { data, proto, title } = result.fields; + if (data) { + if (data.url) { + resolve({ data: data.url, title }); + } else if (data.fields) { + resolve({ data: data.fields, title }); + } else { + reject(); + } + } else if (proto) { + getData(proto.fieldId).then(resolve, reject); + } else { + reject(); + } + }); + }); +} + +/** + * + * @param file the zip file to which we write the files + * @param hierarchy the data structure from which we read, defining the nesting of the documents in the zip + * @param prefix lets us create nested folders in the zip file by continually appending to the end + * of the prefix with each layer of recursion. + * + * Function Call #1 => "Dash Export" + * Function Call #2 => "Dash Export/a nested collection" + * Function Call #3 => "Dash Export/a nested collection/lowest level collection" + * ... + */ +async function writeHierarchyRecursive(file: Archiver.Archiver, hierarchy: Hierarchy, prefix = "Dash Export"): Promise<void> { + for (const documentTitle of Object.keys(hierarchy)) { + const result = hierarchy[documentTitle]; + // base case or leaf node, we've hit a url (image) + if (typeof result === "string") { + let path: string; + let matches: RegExpExecArray | null; + if ((matches = /\:1050\/files\/(upload\_[\da-z]{32}.*)/g.exec(result)) !== null) { + // image already exists on our server + path = `${__dirname}/public/files/${matches[1]}`; + } else { + // the image doesn't already exist on our server (may have been dragged + // and dropped in the browser and thus hosted remotely) so we upload it + // to our server and point the zip file to it, so it can bundle up the bytes + const information = await DashUploadUtils.UploadImage(result); + path = information.mediaPaths[0]; + } + // write the file specified by the path to the directory in the + // zip file given by the prefix. + file.file(path, { name: documentTitle, prefix }); + } else { + // we've hit a collection, so we have to recurse + await writeHierarchyRecursive(file, result, `${prefix}/${documentTitle}`); + } + } +}
\ No newline at end of file diff --git a/src/server/ApiManagers/GeneralGoogleManager.ts b/src/server/ApiManagers/GeneralGoogleManager.ts new file mode 100644 index 000000000..89efebf78 --- /dev/null +++ b/src/server/ApiManagers/GeneralGoogleManager.ts @@ -0,0 +1,83 @@ +import ApiManager, { Registration } from "./ApiManager"; +import { Method, _permission_denied } from "../RouteManager"; +import { GoogleApiServerUtils } from "../apis/google/GoogleApiServerUtils"; +import { Database } from "../database"; +import RouteSubscriber from "../RouteSubscriber"; + +const deletionPermissionError = "Cannot perform specialized delete outside of the development environment!"; +const ServicesApiKeyMap = new Map<string, string | undefined>([ + ["face", process.env.FACE], + ["vision", process.env.VISION], + ["handwriting", process.env.HANDWRITING] +]); +const EndpointHandlerMap = new Map<GoogleApiServerUtils.Action, GoogleApiServerUtils.ApiRouter>([ + ["create", (api, params) => api.create(params)], + ["retrieve", (api, params) => api.get(params)], + ["update", (api, params) => api.batchUpdate(params)], +]); + +export default class GeneralGoogleManager extends ApiManager { + + protected initialize(register: Registration): void { + + register({ + method: Method.GET, + subscription: "/readGoogleAccessToken", + onValidation: async ({ user, res }) => { + const userId = user.id; + const token = await GoogleApiServerUtils.retrieveAccessToken(userId); + if (!token) { + return res.send(GoogleApiServerUtils.generateAuthenticationUrl()); + } + return res.send(token); + } + }); + + register({ + method: Method.POST, + subscription: "/writeGoogleAccessToken", + onValidation: async ({ user, req, res }) => { + res.send(await GoogleApiServerUtils.processNewUser(user.id, req.body.authenticationCode)); + } + }); + + register({ + method: Method.GET, + subscription: "/deleteWithGoogleCredentials", + onValidation: async ({ res, isRelease }) => { + if (isRelease) { + return _permission_denied(res, deletionPermissionError); + } + await Database.Auxiliary.GoogleAuthenticationToken.DeleteAll(); + res.redirect("/delete"); + } + }); + + register({ + method: Method.GET, + subscription: new RouteSubscriber("/cognitiveServices").add('requestedservice'), + onValidation: ({ req, res }) => { + let service = req.params.requestedservice; + res.send(ServicesApiKeyMap.get(service)); + } + }); + + register({ + method: Method.POST, + subscription: new RouteSubscriber("/googleDocs").add("sector", "action"), + onValidation: async ({ req, res, user }) => { + let sector: GoogleApiServerUtils.Service = req.params.sector as GoogleApiServerUtils.Service; + let action: GoogleApiServerUtils.Action = req.params.action as GoogleApiServerUtils.Action; + const endpoint = await GoogleApiServerUtils.GetEndpoint(GoogleApiServerUtils.Service[sector], user.id); + let handler = EndpointHandlerMap.get(action); + if (endpoint && handler) { + handler(endpoint, req.body) + .then(response => res.send(response.data)) + .catch(exception => res.send(exception)); + return; + } + res.send(undefined); + } + }); + } +}
\ No newline at end of file diff --git a/src/server/ApiManagers/GooglePhotosManager.ts b/src/server/ApiManagers/GooglePhotosManager.ts new file mode 100644 index 000000000..1f6051c28 --- /dev/null +++ b/src/server/ApiManagers/GooglePhotosManager.ts @@ -0,0 +1,111 @@ +import ApiManager, { Registration } from "./ApiManager"; +import { Method, _error, _success, _invalid } from "../RouteManager"; +import * as path from "path"; +import { GoogleApiServerUtils } from "../apis/google/GoogleApiServerUtils"; +import { BatchedArray, TimeUnit } from "array-batcher"; +import { GooglePhotosUploadUtils } from "../apis/google/GooglePhotosUploadUtils"; +import { Opt } from "../../new_fields/Doc"; +import { DashUploadUtils } from "../DashUploadUtils"; +import { Database } from "../database"; + +const authenticationError = "Unable to authenticate Google credentials before uploading to Google Photos!"; +const mediaError = "Unable to convert all uploaded bytes to media items!"; +const UploadError = (count: number) => `Unable to upload ${count} images to Dash's server`; +const requestError = "Unable to execute download: the body's media items were malformed."; +const downloadError = "Encountered an error while executing downloads."; +interface GooglePhotosUploadFailure { + batch: number; + index: number; + url: string; + reason: string; +} +interface MediaItem { + baseUrl: string; + filename: string; +} +interface NewMediaItem { + description: string; + simpleMediaItem: { + uploadToken: string; + }; +} +const prefix = "google_photos_"; + +export default class GooglePhotosManager extends ApiManager { + + protected initialize(register: Registration): void { + + register({ + method: Method.POST, + subscription: "/googlePhotosMediaUpload", + onValidation: async ({ user, req, res }) => { + const { media } = req.body; + const token = await GoogleApiServerUtils.retrieveAccessToken(user.id); + if (!token) { + return _error(res, authenticationError); + } + let failed: GooglePhotosUploadFailure[] = []; + const batched = BatchedArray.from<GooglePhotosUploadUtils.UploadSource>(media, { batchSize: 25 }); + const newMediaItems = await batched.batchedMapPatientInterval<NewMediaItem>( + { magnitude: 100, unit: TimeUnit.Milliseconds }, + async (batch: any, collector: any, { completedBatches }: any) => { + for (let index = 0; index < batch.length; index++) { + const { url, description } = batch[index]; + const fail = (reason: string) => failed.push({ reason, batch: completedBatches + 1, index, url }); + const uploadToken = await GooglePhotosUploadUtils.DispatchGooglePhotosUpload(token, url).catch(fail); + if (!uploadToken) { + fail(`${path.extname(url)} is not an accepted extension`); + } else { + collector.push({ + description, + simpleMediaItem: { uploadToken } + }); + } + } + } + ); + const failedCount = failed.length; + if (failedCount) { + console.error(`Unable to upload ${failedCount} image${failedCount === 1 ? "" : "s"} to Google's servers`); + console.log(failed.map(({ reason, batch, index, url }) => `@${batch}.${index}: ${url} failed:\n${reason}`).join('\n\n')); + } + return GooglePhotosUploadUtils.CreateMediaItems(token, newMediaItems, req.body.album).then( + results => _success(res, { results, failed }), + error => _error(res, mediaError, error) + ); + } + }); + + register({ + method: Method.POST, + subscription: "/googlePhotosMediaDownload", + onValidation: async ({ req, res }) => { + const contents: { mediaItems: MediaItem[] } = req.body; + let failed = 0; + if (contents) { + const completed: Opt<DashUploadUtils.UploadInformation>[] = []; + for (let item of contents.mediaItems) { + const { contentSize, ...attributes } = await DashUploadUtils.InspectImage(item.baseUrl); + const found: Opt<DashUploadUtils.UploadInformation> = await Database.Auxiliary.QueryUploadHistory(contentSize!); + if (!found) { + const upload = await DashUploadUtils.UploadInspectedImage({ contentSize, ...attributes }, item.filename, prefix).catch(error => _error(res, downloadError, error)); + if (upload) { + completed.push(upload); + await Database.Auxiliary.LogUpload(upload); + } else { + failed++; + } + } else { + completed.push(found); + } + } + if (failed) { + return _error(res, UploadError(failed)); + } + return _success(res, completed); + } + _invalid(res, requestError); + } + }); + } +}
\ No newline at end of file diff --git a/src/server/ApiManagers/PDFManager.ts b/src/server/ApiManagers/PDFManager.ts new file mode 100644 index 000000000..632b4965a --- /dev/null +++ b/src/server/ApiManagers/PDFManager.ts @@ -0,0 +1,107 @@ +import ApiManager, { Registration } from "./ApiManager"; +import { Method } from "../RouteManager"; +import RouteSubscriber from "../RouteSubscriber"; +import { exists, createReadStream, createWriteStream } from "fs"; +import { filesDirectory } from ".."; +import * as Pdfjs from 'pdfjs-dist'; +import { createCanvas } from "canvas"; +const probe = require("probe-image-size"); +import * as express from "express"; +import * as path from "path"; + +export default class PDFManager extends ApiManager { + + protected initialize(register: Registration): void { + + register({ + method: Method.GET, + subscription: new RouteSubscriber("thumbnail").add("filename"), + onValidation: ({ req, res }) => { + let filename = req.params.filename; + let noExt = filename.substring(0, filename.length - ".png".length); + let pagenumber = parseInt(noExt.split('-')[1]); + return new Promise<void>(resolve => { + exists(filesDirectory + filename, (exists: boolean) => { + console.log(`${filesDirectory + filename} ${exists ? "exists" : "does not exist"}`); + if (exists) { + let input = createReadStream(filesDirectory + filename); + probe(input, (err: any, result: any) => { + if (err) { + console.log(err); + console.log(`error on ${filename}`); + return; + } + res.send({ path: "/files/" + filename, width: result.width, height: result.height }); + }); + } + else { + LoadPage(filesDirectory + filename.substring(0, filename.length - noExt.split('-')[1].length - ".PNG".length - 1) + ".pdf", pagenumber, res); + } + resolve(); + }); + }); + } + }); + + function LoadPage(file: string, pageNumber: number, res: express.Response) { + console.log(file); + Pdfjs.getDocument(file).promise + .then((pdf: Pdfjs.PDFDocumentProxy) => { + let factory = new NodeCanvasFactory(); + console.log(pageNumber); + pdf.getPage(pageNumber).then((page: Pdfjs.PDFPageProxy) => { + console.log("reading " + page); + let viewport = page.getViewport(1 as any); + let canvasAndContext = factory.create(viewport.width, viewport.height); + let renderContext = { + canvasContext: canvasAndContext.context, + viewport: viewport, + canvasFactory: factory + }; + console.log("read " + pageNumber); + + page.render(renderContext).promise + .then(() => { + console.log("saving " + pageNumber); + let stream = canvasAndContext.canvas.createPNGStream(); + let pngFile = `${file.substring(0, file.length - ".pdf".length)}-${pageNumber}.PNG`; + let out = createWriteStream(pngFile); + stream.pipe(out); + out.on("finish", () => { + console.log(`Success! Saved to ${pngFile}`); + let name = path.basename(pngFile); + res.send({ path: "/files/" + name, width: viewport.width, height: viewport.height }); + }); + }, (reason: string) => { + console.error(reason + ` ${pageNumber}`); + }); + }); + }); + } + + } + +} + +class NodeCanvasFactory { + create = (width: number, height: number) => { + var canvas = createCanvas(width, height); + var context = canvas.getContext('2d'); + return { + canvas, + context + }; + } + + reset = (canvasAndContext: any, width: number, height: number) => { + canvasAndContext.canvas.width = width; + canvasAndContext.canvas.height = height; + } + + destroy = (canvasAndContext: any) => { + canvasAndContext.canvas.width = 0; + canvasAndContext.canvas.height = 0; + canvasAndContext.canvas = null; + canvasAndContext.context = null; + } +}
\ No newline at end of file diff --git a/src/server/ApiManagers/SearchManager.ts b/src/server/ApiManagers/SearchManager.ts index 15b87204c..1c801715a 100644 --- a/src/server/ApiManagers/SearchManager.ts +++ b/src/server/ApiManagers/SearchManager.ts @@ -1,15 +1,15 @@ -import ApiManager from "./ApiManager"; -import RouteManager, { Method } from "../RouteManager"; +import ApiManager, { Registration } from "./ApiManager"; +import { Method } from "../RouteManager"; import { Search } from "../Search"; var findInFiles = require('find-in-files'); import * as path from 'path'; -import { uploadDirectory } from ".."; +import { filesDirectory } from ".."; export default class SearchManager extends ApiManager { - public register(router: RouteManager): void { + protected initialize(register: Registration): void { - router.addSupervisedRoute({ + register({ method: Method.GET, subscription: "/textsearch", onValidation: async ({ req, res }) => { @@ -18,7 +18,7 @@ export default class SearchManager extends ApiManager { res.send([]); return; } - let results = await findInFiles.find({ 'term': q, 'flags': 'ig' }, uploadDirectory + "text", ".txt$"); + let results = await findInFiles.find({ 'term': q, 'flags': 'ig' }, filesDirectory + "text", ".txt$"); let resObj: { ids: string[], numFound: number, lines: string[] } = { ids: [], numFound: 0, lines: [] }; for (var result in results) { resObj.ids.push(path.basename(result, ".txt").replace(/upload_/, "")); @@ -29,7 +29,7 @@ export default class SearchManager extends ApiManager { } }); - router.addSupervisedRoute({ + register({ method: Method.GET, subscription: "/search", onValidation: async ({ req, res }) => { diff --git a/src/server/ApiManagers/UploadManager.ts b/src/server/ApiManagers/UploadManager.ts new file mode 100644 index 000000000..01abdab54 --- /dev/null +++ b/src/server/ApiManagers/UploadManager.ts @@ -0,0 +1,222 @@ +import ApiManager, { Registration } from "./ApiManager"; +import { Method, _success } from "../RouteManager"; +import * as formidable from 'formidable'; +import v4 = require('uuid/v4'); +var AdmZip = require('adm-zip'); +import * as path from 'path'; +import { createReadStream, createWriteStream, unlink, readFileSync } from "fs"; +import { publicDirectory, filesDirectory, Partitions } from ".."; +import { Database } from "../database"; +import { DashUploadUtils } from "../DashUploadUtils"; +import { Opt } from "../../new_fields/Doc"; +import { ParsedPDF } from "../PdfTypes"; +const pdf = require('pdf-parse'); +import * as sharp from 'sharp'; +import { SharedMediaTypes } from "../SharedMediaTypes"; +const imageDataUri = require('image-data-uri'); + +export default class UploadManager extends ApiManager { + + protected initialize(register: Registration): void { + + register({ + method: Method.POST, + subscription: "/uploadDoc", + onValidation: ({ req, res }) => { + let form = new formidable.IncomingForm(); + form.keepExtensions = true; + // let path = req.body.path; + const ids: { [id: string]: string } = {}; + let remap = true; + const getId = (id: string): string => { + if (!remap) return id; + if (id.endsWith("Proto")) return id; + if (id in ids) { + return ids[id]; + } else { + return ids[id] = v4(); + } + }; + const mapFn = (doc: any) => { + if (doc.id) { + doc.id = getId(doc.id); + } + for (const key in doc.fields) { + if (!doc.fields.hasOwnProperty(key)) { + continue; + } + const field = doc.fields[key]; + if (field === undefined || field === null) { + continue; + } + + if (field.__type === "proxy" || field.__type === "prefetch_proxy") { + field.fieldId = getId(field.fieldId); + } else if (field.__type === "script" || field.__type === "computed") { + if (field.captures) { + field.captures.fieldId = getId(field.captures.fieldId); + } + } else if (field.__type === "list") { + mapFn(field); + } else if (typeof field === "string") { + const re = /("(?:dataD|d)ocumentId"\s*:\s*")([\w\-]*)"/g; + doc.fields[key] = (field as any).replace(re, (match: any, p1: string, p2: string) => { + return `${p1}${getId(p2)}"`; + }); + } else if (field.__type === "RichTextField") { + const re = /("href"\s*:\s*")(.*?)"/g; + field.Data = field.Data.replace(re, (match: any, p1: string, p2: string) => { + return `${p1}${getId(p2)}"`; + }); + } + } + }; + return new Promise<void>(resolve => { + form.parse(req, async (_err, fields, files) => { + remap = fields.remap !== "false"; + let id: string = ""; + try { + for (const name in files) { + const path_2 = files[name].path; + const zip = new AdmZip(path_2); + zip.getEntries().forEach((entry: any) => { + if (!entry.entryName.startsWith("files/")) return; + let dirname = path.dirname(entry.entryName) + "/"; + let extname = path.extname(entry.entryName); + let basename = path.basename(entry.entryName).split(".")[0]; + try { + zip.extractEntryTo(entry.entryName, publicDirectory, true, false); + dirname = "/" + dirname; + + createReadStream(publicDirectory + dirname + basename + extname).pipe(createWriteStream(publicDirectory + dirname + basename + "_o" + extname)); + createReadStream(publicDirectory + dirname + basename + extname).pipe(createWriteStream(publicDirectory + dirname + basename + "_s" + extname)); + createReadStream(publicDirectory + dirname + basename + extname).pipe(createWriteStream(publicDirectory + dirname + basename + "_m" + extname)); + createReadStream(publicDirectory + dirname + basename + extname).pipe(createWriteStream(publicDirectory + dirname + basename + "_l" + extname)); + } catch (e) { + console.log(e); + } + }); + const json = zip.getEntry("doc.json"); + let docs: any; + try { + let data = JSON.parse(json.getData().toString("utf8")); + docs = data.docs; + id = data.id; + docs = Object.keys(docs).map(key => docs[key]); + docs.forEach(mapFn); + await Promise.all(docs.map((doc: any) => new Promise(res => Database.Instance.replace(doc.id, doc, (err, r) => { + err && console.log(err); + res(); + }, true, "newDocuments")))); + } catch (e) { console.log(e); } + unlink(path_2, () => { }); + } + if (id) { + res.send(JSON.stringify(getId(id))); + } else { + res.send(JSON.stringify("error")); + } + } catch (e) { console.log(e); } + resolve(); + }); + }); + } + }); + + + register({ + method: Method.POST, + subscription: "/upload", + onValidation: async ({ req, res }) => { + let form = new formidable.IncomingForm(); + form.uploadDir = filesDirectory; + form.keepExtensions = true; + return new Promise<void>(resolve => { + form.parse(req, async (_err, _fields, files) => { + let results: DashUploadUtils.ImageFileResponse[] = []; + for (const key in files) { + const { type, path: location, name } = files[key]; + const filename = path.basename(location); + let uploadInformation: Opt<DashUploadUtils.UploadInformation>; + if (filename.endsWith(".pdf")) { + let dataBuffer = readFileSync(filesDirectory + filename); + const result: ParsedPDF = await pdf(dataBuffer); + await new Promise<void>((resolve, reject) => { + const path = filesDirectory + Partitions.pdf_text + "/" + filename.substring(0, filename.length - ".pdf".length) + ".txt"; + createWriteStream(path).write(result.text, error => { + if (!error) { + resolve(); + } else { + reject(error); + } + }); + }); + } else { + uploadInformation = await DashUploadUtils.UploadImage(filesDirectory + filename, filename); + } + const exif = uploadInformation ? uploadInformation.exifData : undefined; + results.push({ name, type, path: `/files/${filename}`, exif }); + } + _success(res, results); + resolve(); + }); + }); + } + }); + + register({ + method: Method.POST, + subscription: "/inspectImage", + onValidation: async ({ req, res }) => { + const { source } = req.body; + if (typeof source === "string") { + const uploadInformation = await DashUploadUtils.UploadImage(source); + return res.send(await DashUploadUtils.InspectImage(uploadInformation.mediaPaths[0])); + } + res.send({}); + } + }); + + register({ + method: Method.POST, + subscription: "/uploadURI", + onValidation: ({ req, res }) => { + const uri = req.body.uri; + const filename = req.body.name; + if (!uri || !filename) { + res.status(401).send("incorrect parameters specified"); + return; + } + return imageDataUri.outputFile(uri, filesDirectory + filename).then((savedName: string) => { + const ext = path.extname(savedName).toLowerCase(); + const { pngs, jpgs } = SharedMediaTypes; + let resizers = [ + { resizer: sharp().resize(100, undefined, { withoutEnlargement: true }), suffix: "_s" }, + { resizer: sharp().resize(400, undefined, { withoutEnlargement: true }), suffix: "_m" }, + { resizer: sharp().resize(900, undefined, { withoutEnlargement: true }), suffix: "_l" }, + ]; + let isImage = false; + if (pngs.includes(ext)) { + resizers.forEach(element => { + element.resizer = element.resizer.png(); + }); + isImage = true; + } else if (jpgs.includes(ext)) { + resizers.forEach(element => { + element.resizer = element.resizer.jpeg(); + }); + isImage = true; + } + if (isImage) { + resizers.forEach(resizer => { + createReadStream(savedName).pipe(resizer.resizer).pipe(createWriteStream(filesDirectory + filename + resizer.suffix + ext)); + }); + } + res.send("/files/" + filename + ext); + }); + } + }); + + } + +}
\ No newline at end of file diff --git a/src/server/ApiManagers/UserManager.ts b/src/server/ApiManagers/UserManager.ts index bb8837dc6..51a434fcf 100644 --- a/src/server/ApiManagers/UserManager.ts +++ b/src/server/ApiManagers/UserManager.ts @@ -1,11 +1,36 @@ -import ApiManager from "./ApiManager"; -import RouteManager, { Method } from "../RouteManager"; +import ApiManager, { Registration } from "./ApiManager"; +import { Method } from "../RouteManager"; import { WebSocket } from "../Websocket/Websocket"; +import { Database } from "../database"; export default class UserManager extends ApiManager { - public register(router: RouteManager): void { - router.addSupervisedRoute({ + protected initialize(register: Registration): void { + + register({ + method: Method.GET, + subscription: "/getUsers", + onValidation: async ({ res }) => { + const cursor = await Database.Instance.query({}, { email: 1, userDocumentId: 1 }, "users"); + const results = await cursor.toArray(); + res.send(results.map(user => ({ email: user.email, userDocumentId: user.userDocumentId }))); + } + }); + + register({ + method: Method.GET, + subscription: "/getUserDocumentId", + onValidation: ({ res, user }) => res.send(user.userDocumentId) + }); + + register({ + method: Method.GET, + subscription: "/getCurrentUser", + onValidation: ({ res, user }) => res.send(JSON.stringify(user)), + onUnauthenticated: ({ res }) => res.send(JSON.stringify({ id: "__guest__", email: "" })) + }); + + register({ method: Method.GET, subscription: "/whosOnline", onValidation: ({ res }) => { @@ -16,25 +41,26 @@ export default class UserManager extends ApiManager { for (const user in timeMap) { const time = timeMap[user]; const key = ((now - time) / 1000) < (60 * 5) ? "active" : "inactive"; - users[key][user] = `Last active ${this.msToTime(now - time)} ago`; + users[key][user] = `Last active ${msToTime(now - time)} ago`; } res.send(users); } }); + } - private msToTime(duration: number) { - let milliseconds = Math.floor((duration % 1000) / 100), - seconds = Math.floor((duration / 1000) % 60), - minutes = Math.floor((duration / (1000 * 60)) % 60), - hours = Math.floor((duration / (1000 * 60 * 60)) % 24); +} - let hoursS = (hours < 10) ? "0" + hours : hours; - let minutesS = (minutes < 10) ? "0" + minutes : minutes; - let secondsS = (seconds < 10) ? "0" + seconds : seconds; +function msToTime(duration: number) { + let milliseconds = Math.floor((duration % 1000) / 100), + seconds = Math.floor((duration / 1000) % 60), + minutes = Math.floor((duration / (1000 * 60)) % 60), + hours = Math.floor((duration / (1000 * 60 * 60)) % 24); - return hoursS + ":" + minutesS + ":" + secondsS + "." + milliseconds; - } + let hoursS = (hours < 10) ? "0" + hours : hours; + let minutesS = (minutes < 10) ? "0" + minutes : minutes; + let secondsS = (seconds < 10) ? "0" + seconds : seconds; + return hoursS + ":" + minutesS + ":" + secondsS + "." + milliseconds; }
\ No newline at end of file diff --git a/src/server/ApiManagers/UtilManager.ts b/src/server/ApiManagers/UtilManager.ts index 79b904e8a..c1234be6c 100644 --- a/src/server/ApiManagers/UtilManager.ts +++ b/src/server/ApiManagers/UtilManager.ts @@ -1,50 +1,63 @@ -import ApiManager from "./ApiManager"; -import RouteManager, { Method } from "../RouteManager"; +import ApiManager, { Registration } from "./ApiManager"; +import { Method } from "../RouteManager"; import { exec } from 'child_process'; import { command_line } from "../ActionUtilities"; +import RouteSubscriber from "../RouteSubscriber"; export default class UtilManager extends ApiManager { - public register(router: RouteManager): void { + protected initialize(register: Registration): void { - router.addSupervisedRoute({ + register({ + method: Method.GET, + subscription: new RouteSubscriber("environment").add("key"), + onValidation: ({ req, res }) => res.send(process.env[req.params.key]) + }); + + register({ method: Method.GET, subscription: "/pull", - onValidation: ({ res }) => { - exec('"C:\\Program Files\\Git\\git-bash.exe" -c "git pull"', err => { - if (err) { - res.send(err.message); - return; - } - res.redirect("/"); + onValidation: async ({ res }) => { + return new Promise<void>(resolve => { + exec('"C:\\Program Files\\Git\\git-bash.exe" -c "git pull"', err => { + if (err) { + res.send(err.message); + return; + } + res.redirect("/"); + resolve(); + }); }); } }); - router.addSupervisedRoute({ + register({ method: Method.GET, subscription: "/buxton", - onValidation: ({ res }) => { + onValidation: async ({ res }) => { let cwd = '../scraping/buxton'; let onResolved = (stdout: string) => { console.log(stdout); res.redirect("/"); }; let onRejected = (err: any) => { console.error(err.message); res.send(err); }; let tryPython3 = () => command_line('python3 scraper.py', cwd).then(onResolved, onRejected); - command_line('python scraper.py', cwd).then(onResolved, tryPython3); + return command_line('python scraper.py', cwd).then(onResolved, tryPython3); }, }); - router.addSupervisedRoute({ + register({ method: Method.GET, subscription: "/version", onValidation: ({ res }) => { - exec('"C:\\Program Files\\Git\\bin\\git.exe" rev-parse HEAD', (err, stdout) => { - if (err) { - res.send(err.message); - return; - } - res.send(stdout); + return new Promise<void>(resolve => { + exec('"C:\\Program Files\\Git\\bin\\git.exe" rev-parse HEAD', (err, stdout) => { + if (err) { + res.send(err.message); + return; + } + res.send(stdout); + }); + resolve(); }); } }); diff --git a/src/server/DashUploadUtils.ts b/src/server/DashUploadUtils.ts index 46d897339..8f5b0e1a8 100644 --- a/src/server/DashUploadUtils.ts +++ b/src/server/DashUploadUtils.ts @@ -5,6 +5,7 @@ import * as sharp from 'sharp'; import request = require('request-promise'); import { ExifData, ExifImage } from 'exif'; import { Opt } from '../new_fields/Doc'; +import { SharedMediaTypes } from './SharedMediaTypes'; const uploadDirectory = path.join(__dirname, './public/files/'); @@ -15,17 +16,22 @@ export namespace DashUploadUtils { suffix: string; } + export interface ImageFileResponse { + name: string; + path: string; + type: string; + exif: Opt<DashUploadUtils.EnrichedExifData>; + } + export const Sizes: { [size: string]: Size } = { SMALL: { width: 100, suffix: "_s" }, MEDIUM: { width: 400, suffix: "_m" }, LARGE: { width: 900, suffix: "_l" }, }; - const gifs = [".gif"]; - const pngs = [".png"]; - const jpgs = [".jpg", ".jpeg"]; - export const imageFormats = [...pngs, ...jpgs, ...gifs]; - const videoFormats = [".mov", ".mp4"]; + export function validateExtension(url: string) { + return SharedMediaTypes.imageFormats.includes(path.extname(url).toLowerCase()); + } const size = "content-length"; const type = "content-type"; @@ -128,6 +134,7 @@ export namespace DashUploadUtils { contentSize, contentType, }; + const { pngs, imageFormats, jpgs, videoFormats } = SharedMediaTypes; return new Promise<UploadInformation>(async (resolve, reject) => { const resizers = [ { resizer: sharp().rotate(), suffix: "_o" }, diff --git a/src/server/Initialization.ts b/src/server/Initialization.ts index 0e5b8649e..6ac0787c5 100644 --- a/src/server/Initialization.ts +++ b/src/server/Initialization.ts @@ -9,7 +9,6 @@ import flash = require('connect-flash'); import { Database } from './database'; import { getForgot, getLogin, getLogout, getReset, getSignup, postForgot, postLogin, postReset, postSignup } from './authentication/controllers/user_controller'; const MongoStore = require('connect-mongo')(session); -import { RouteStore } from './RouteStore'; import RouteManager from './RouteManager'; import * as webpack from 'webpack'; const config = require('../../webpack.config'); @@ -18,6 +17,8 @@ import * as wdm from 'webpack-dev-middleware'; import * as whm from 'webpack-hot-middleware'; import * as fs from 'fs'; import * as request from 'request'; +import RouteSubscriber from './RouteSubscriber'; +import { publicDirectory } from '.'; /* RouteSetter is a wrapper around the server that prevents the server from being exposed. */ @@ -31,8 +32,8 @@ export default async function InitializeServer(options: InitializationOptions) { const { listenAtPort, routeSetter } = options; const server = buildWithMiddleware(express()); - server.use(express.static(__dirname + RouteStore.public)); - server.use(RouteStore.images, express.static(__dirname + RouteStore.public)); + server.use(express.static(publicDirectory)); + server.use("/images", express.static(publicDirectory)); server.use(wdm(compiler, { publicPath: config.output.publicPath })); server.use(whm(compiler)); @@ -40,7 +41,7 @@ export default async function InitializeServer(options: InitializationOptions) { registerAuthenticationRoutes(server); registerCorsProxy(server); - const isRelease = determineEnvironment(); //vs. dev mode + const isRelease = determineEnvironment(); //vs. dev mode routeSetter(new RouteManager(server, isRelease)); server.listen(listenAtPort, () => console.log(`server started at http://localhost:${listenAtPort}`)); @@ -90,24 +91,25 @@ function determineEnvironment() { } function registerAuthenticationRoutes(server: express.Express) { - server.get(RouteStore.signup, getSignup); - server.post(RouteStore.signup, postSignup); + server.get("/signup", getSignup); + server.post("/signup", postSignup); - server.get(RouteStore.login, getLogin); - server.post(RouteStore.login, postLogin); + server.get("/login", getLogin); + server.post("/login", postLogin); - server.get(RouteStore.logout, getLogout); + server.get("/logout", getLogout); - server.get(RouteStore.forgot, getForgot); - server.post(RouteStore.forgot, postForgot); + server.get("/forgotPassword", getForgot); + server.post("/forgotPassword", postForgot); - server.get(RouteStore.reset, getReset); - server.post(RouteStore.reset, postReset); + const reset = new RouteSubscriber("resetPassword").add("token").build; + server.get(reset, getReset); + server.post(reset, postReset); } function registerCorsProxy(server: express.Express) { const headerCharRegex = /[^\t\x20-\x7e\x80-\xff]/; - server.use(RouteStore.corsProxy, (req, res) => { + server.use("/corsProxy", (req, res) => { req.pipe(request(decodeURIComponent(req.url.substring(1)))).on("response", res => { const headers = Object.keys(res.headers); headers.forEach(headerName => { diff --git a/src/server/RouteManager.ts b/src/server/RouteManager.ts index b3864e89c..3aae5734a 100644 --- a/src/server/RouteManager.ts +++ b/src/server/RouteManager.ts @@ -1,8 +1,6 @@ import RouteSubscriber from "./RouteSubscriber"; -import { RouteStore } from "./RouteStore"; import { DashUserModel } from "./authentication/models/user_model"; import * as express from 'express'; -import { Opt } from "../new_fields/Doc"; export enum Method { GET, @@ -41,23 +39,18 @@ export default class RouteManager { } /** - * Please invoke this function when adding a new route to Dash's server. - * It ensures that any requests leading to or containing user-sensitive information - * does not execute unless Passport authentication detects a user logged in. - * @param method whether or not the request is a GET or a POST - * @param handler the action to invoke, recieving a DashUserModel and, as expected, the Express.Request and Express.Response - * @param onRejection an optional callback invoked on return if no user is found to be logged in - * @param subscribers the forward slash prepended path names (reference and add to RouteStore.ts) that will all invoke the given @param handler + * + * @param initializer */ - addSupervisedRoute(initializer: RouteInitializer) { + addSupervisedRoute = (initializer: RouteInitializer): void => { const { method, subscription, onValidation, onUnauthenticated, onError } = initializer; const isRelease = this._isRelease; let supervised = async (req: express.Request, res: express.Response) => { const { user, originalUrl: target } = req; const core = { req, res, isRelease }; - const tryExecute = async (target: (args: any) => any | Promise<any>, args: any) => { + const tryExecute = async (toExecute: (args: any) => any | Promise<any>, args: any) => { try { - await target(args); + await toExecute(args); } catch (e) { if (onError) { onError({ ...core, error: e }); @@ -72,12 +65,16 @@ export default class RouteManager { req.session!.target = target; if (onUnauthenticated) { await tryExecute(onUnauthenticated, core); + if (!res.headersSent) { + res.redirect("/login"); + } } else { - res.redirect(RouteStore.login); + res.redirect("/login"); } } setTimeout(() => { if (!res.headersSent) { + console.log("Initiating fallback for ", target); const warning = `request to ${target} fell through - this is a fallback response`; res.send({ warning }); } @@ -116,6 +113,7 @@ export const STATUS = { }; export function _error(res: express.Response, message: string, error?: any) { + console.error(message); res.statusMessage = message; res.status(STATUS.EXECUTION_ERROR).send(error); } diff --git a/src/server/RouteStore.ts b/src/server/RouteStore.ts deleted file mode 100644 index de2553b2f..000000000 --- a/src/server/RouteStore.ts +++ /dev/null @@ -1,44 +0,0 @@ -// PREPEND ALL ROUTES WITH FORWARD SLASHES! - -export enum RouteStore { - // GENERAL - root = "/", - home = "/home", - corsProxy = "/corsProxy", - delete = "/delete", - deleteAll = "/deleteAll", - pull = "/pull", - - // UPLOAD AND STATIC FILE SERVING - public = "/public", - upload = "/upload", - dataUriToImage = "/uploadURI", - images = "/images", - inspectImage = "/inspectImage", - imageHierarchyExport = "/imageHierarchyExport", - - // USER AND WORKSPACES - getCurrUser = "/getCurrentUser", - getUsers = "/getUsers", - getUserDocumentId = "/getUserDocumentId", - updateCursor = "/updateCursor", - - openDocumentWithId = "/doc/:docId", - - // AUTHENTICATION - signup = "/signup", - login = "/login", - logout = "/logout", - forgot = "/forgotpassword", - reset = "/reset/:token", - - // APIS - cognitiveServices = "/cognitiveservices", - googleDocs = "/googleDocs", - readGoogleAccessToken = "/readGoogleAccessToken", - writeGoogleAccessToken = "/writeGoogleAccessToken", - googlePhotosMediaUpload = "/googlePhotosMediaUpload", - googlePhotosMediaDownload = "/googlePhotosMediaDownload", - googleDocsGet = "/googleDocsGet" - -}
\ No newline at end of file diff --git a/src/server/RouteSubscriber.ts b/src/server/RouteSubscriber.ts index e49be8af5..a1cf7c1c4 100644 --- a/src/server/RouteSubscriber.ts +++ b/src/server/RouteSubscriber.ts @@ -3,7 +3,7 @@ export default class RouteSubscriber { private requestParameters: string[] = []; constructor(root: string) { - this._root = root; + this._root = `/${root}`; } add(...parameters: string[]) { diff --git a/src/server/SharedMediaTypes.ts b/src/server/SharedMediaTypes.ts new file mode 100644 index 000000000..3d3234125 --- /dev/null +++ b/src/server/SharedMediaTypes.ts @@ -0,0 +1,9 @@ +export namespace SharedMediaTypes { + + export const gifs = [".gif"]; + export const pngs = [".png"]; + export const jpgs = [".jpg", ".jpeg"]; + export const imageFormats = [...pngs, ...jpgs, ...gifs]; + export const videoFormats = [".mov", ".mp4"]; + +}
\ No newline at end of file diff --git a/src/server/Websocket/Websocket.ts b/src/server/Websocket/Websocket.ts index 2461dd8d5..f6a6c8718 100644 --- a/src/server/Websocket/Websocket.ts +++ b/src/server/Websocket/Websocket.ts @@ -4,9 +4,11 @@ import { Client } from "../Client"; import { Socket } from "socket.io"; import { Database } from "../database"; import { Search } from "../Search"; -import io from 'socket.io'; +import * as io from 'socket.io'; import YoutubeApi from "../apis/youtube/youtubeApiSample"; -import { youtubeApiKey } from ".."; +import { readFile } from "fs"; +import { Credentials } from "google-auth-library"; +import { GoogleCredentialsLoader } from "../credentials/CredentialsLoader"; export namespace WebSocket { @@ -18,6 +20,14 @@ export namespace WebSocket { export const socketMap = new Map<SocketIO.Socket, string>(); export const timeMap: { [id: string]: number } = {}; + export async function start(serverPort: number, isRelease: boolean) { + await preliminaryFunctions(); + initialize(serverPort, isRelease); + } + + async function preliminaryFunctions() { + } + export function initialize(serverPort: number, isRelease: boolean) { const endpoint = io(); endpoint.listen(serverPort); @@ -54,14 +64,15 @@ export namespace WebSocket { } function HandleYoutubeQuery([query, callback]: [YoutubeQueryInput, (result?: any[]) => void]) { + const { ProjectCredentials } = GoogleCredentialsLoader; switch (query.type) { case YoutubeQueryTypes.Channels: - YoutubeApi.authorizedGetChannel(youtubeApiKey); + YoutubeApi.authorizedGetChannel(ProjectCredentials); break; case YoutubeQueryTypes.SearchVideo: - YoutubeApi.authorizedGetVideos(youtubeApiKey, query.userInput, callback); + YoutubeApi.authorizedGetVideos(ProjectCredentials, query.userInput, callback); case YoutubeQueryTypes.VideoDetails: - YoutubeApi.authorizedGetVideoDetails(youtubeApiKey, query.videoIds, callback); + YoutubeApi.authorizedGetVideoDetails(ProjectCredentials, query.videoIds, callback); } } diff --git a/src/server/apis/google/GoogleApiServerUtils.ts b/src/server/apis/google/GoogleApiServerUtils.ts index 6093197f1..b3657ee43 100644 --- a/src/server/apis/google/GoogleApiServerUtils.ts +++ b/src/server/apis/google/GoogleApiServerUtils.ts @@ -1,145 +1,282 @@ import { google } from "googleapis"; -import { createInterface } from "readline"; -import { readFile, writeFile } from "fs"; -import { OAuth2Client, Credentials } from "google-auth-library"; +import { OAuth2Client, Credentials, OAuth2ClientOptions } from "google-auth-library"; import { Opt } from "../../../new_fields/Doc"; -import { GlobalOptions } from "googleapis-common"; import { GaxiosResponse } from "gaxios"; import request = require('request-promise'); import * as qs from 'query-string'; -import Photos = require('googlephotos'); import { Database } from "../../database"; -const path = require("path"); +import { GoogleCredentialsLoader } from "../../credentials/CredentialsLoader"; /** - * Server side authentication for Google Api queries. + * Scopes give Google users fine granularity of control + * over the information they make accessible via the API. + * This is the somewhat overkill list of what Dash requests + * from the user. */ -export namespace GoogleApiServerUtils { +const scope = [ + 'documents.readonly', + 'documents', + 'presentations', + 'presentations.readonly', + 'drive', + 'drive.file', + 'photoslibrary', + 'photoslibrary.appendonly', + 'photoslibrary.sharing', + 'userinfo.profile' +].map(relative => `https://www.googleapis.com/auth/${relative}`); - // If modifying these scopes, delete token.json. - const prefix = 'https://www.googleapis.com/auth/'; - const SCOPES = [ - 'documents.readonly', - 'documents', - 'presentations', - 'presentations.readonly', - 'drive', - 'drive.file', - 'photoslibrary', - 'photoslibrary.appendonly', - 'photoslibrary.sharing', - 'userinfo.profile' - ]; - - export const parseBuffer = (data: Buffer) => JSON.parse(data.toString()); +/** + * This namespace manages server side authentication for Google API queries, either + * from the standard v1 APIs or the Google Photos REST API. + */ +export namespace GoogleApiServerUtils { + /** + * As we expand out to more Google APIs that are accessible from + * the 'googleapis' module imported above, this enum will record + * the list and provide a unified string representation of each API. + */ export enum Service { Documents = "Documents", Slides = "Slides" } - export interface CredentialInformation { - credentialsPath: string; - userId: string; + /** + * Global credentials read once from a JSON file + * before the server is started that + * allow us to build OAuth2 clients with Dash's + * application specific credentials. + */ + let installed: OAuth2ClientOptions; + + /** + * This is a global authorization client that is never + * passed around, and whose credentials are never set. + * Its job is purely to generate new authentication urls + * (users will follow to get to Google's permissions GUI) + * and to use the codes returned from that process to generate the + * initial credentials. + */ + let worker: OAuth2Client; + + /** + * This function is called once before the server is started, + * reading in Dash's project-specific credentials (client secret + * and client id) for later repeated access. It also sets up the + * global, intentionally unauthenticated worker OAuth2 client instance. + */ + export function processProjectCredentials(): void { + const { client_secret, client_id, redirect_uris } = GoogleCredentialsLoader.ProjectCredentials; + // initialize the global authorization client + installed = { + clientId: client_id, + clientSecret: client_secret, + redirectUri: redirect_uris[0] + }; + worker = generateClient(); } + /** + * A briefer format for the response from a 'googleapis' API request + */ export type ApiResponse = Promise<GaxiosResponse>; + + /** + * A generic form for a handler that executes some request on the endpoint + */ export type ApiRouter = (endpoint: Endpoint, parameters: any) => ApiResponse; + + /** + * A generic form for the asynchronous function that actually submits the + * request to the API and returns the corresporing response. Helpful when + * making an extensible endpoint definition. + */ export type ApiHandler = (parameters: any, methodOptions?: any) => ApiResponse; + + /** + * A literal union type indicating the valid actions for these 'googleapis' + * requestions + */ export type Action = "create" | "retrieve" | "update"; - export type Endpoint = { get: ApiHandler, create: ApiHandler, batchUpdate: ApiHandler }; - export type EndpointParameters = GlobalOptions & { version: "v1" }; - - export const GetEndpoint = (sector: string, paths: CredentialInformation) => { - return new Promise<Opt<Endpoint>>(resolve => { - RetrieveCredentials(paths).then(authentication => { - let routed: Opt<Endpoint>; - let parameters: EndpointParameters = { auth: authentication.client, version: "v1" }; - switch (sector) { - case Service.Documents: - routed = google.docs(parameters).documents; - break; - case Service.Slides: - routed = google.slides(parameters).presentations; - break; - } - resolve(routed); - }); - }); - }; - - export const RetrieveAccessToken = (information: CredentialInformation) => { - return new Promise<string>((resolve, reject) => { - RetrieveCredentials(information).then( - credentials => resolve(credentials.token.access_token!), - error => reject(`Error: unable to authenticate Google Photos API request.\n${error}`) - ); - }); - }; + /** + * An interface defining any entity on which one can invoke + * anuy of the following handlers. All 'googleapis' wrappers + * such as google.docs().documents and google.slides().presentations + * satisfy this interface. + */ + export interface Endpoint { + get: ApiHandler; + create: ApiHandler; + batchUpdate: ApiHandler; + } - let AuthorizationManager: OAuth2Client; + /** + * Maps the Dash user id of a given user to their single + * associated OAuth2 client, mitigating the creation + * of needless duplicate clients that would arise from + * making one new client instance per request. + */ + const authenticationClients = new Map<String, OAuth2Client>(); - export const LoadOAuthClient = async () => { - return new Promise<void>((resolve, reject) => { - readFile(path.join(__dirname, "../../credentials/google_docs_credentials.json"), async (err, credentials) => { - if (err) { - reject(err); - return console.log('Error loading client secret file:', err); - } - const { client_secret, client_id, redirect_uris } = parseBuffer(credentials).installed; - AuthorizationManager = new google.auth.OAuth2(client_id, client_secret, redirect_uris[0]); - resolve(); - }); + /** + * This function receives the target sector ("which G-Suite app's API am I interested in?") + * and the id of the Dash user making the request to the API. With this information, it generates + * an authenticated OAuth2 client and passes it into the relevant 'googleapis' wrapper. + * @param sector the particular desired G-Suite 'googleapis' API (docs, slides, etc.) + * @param userId the id of the Dash user making the request to the API + * @returns the relevant 'googleapis' wrapper, if any + */ + export async function GetEndpoint(sector: string, userId: string): Promise<Opt<Endpoint>> { + return new Promise(async resolve => { + const auth = await retrieveOAuthClient(userId); + if (!auth) { + return resolve(); + } + let routed: Opt<Endpoint>; + let parameters: any = { auth, version: "v1" }; + switch (sector) { + case Service.Documents: + routed = google.docs(parameters).documents; + break; + case Service.Slides: + routed = google.slides(parameters).presentations; + break; + } + resolve(routed); + }); + } + + /** + * Returns the lengthy string or access token that can be passed into + * the headers of an API request or into the constructor of the Photos + * client API wrapper. + * @param userId the Dash user id of the user requesting his/her associated + * access_token + * @returns the current access_token associated with the requesting + * Dash user. The access_token is valid for only an hour, and + * is then refreshed. + */ + export async function retrieveAccessToken(userId: string): Promise<string> { + return new Promise(async resolve => { + const { credentials } = await retrieveCredentials(userId); + if (!credentials) { + return resolve(); + } + resolve(credentials.access_token!); }); - }; + } - export const GenerateAuthenticationUrl = async (information: CredentialInformation) => { - return AuthorizationManager.generateAuthUrl({ - access_type: 'offline', - scope: SCOPES.map(relative => prefix + relative), + /** + * Manipulates a mapping such that, in the limit, each Dash user has + * an associated authenticated OAuth2 client at their disposal. This + * function ensures that the client's credentials always remain up to date + * @param userId the Dash user id of the user requesting account integration + * @returns returns an initialized OAuth2 client instance, likely to be passed into Google's + * npm-installed API wrappers that use authenticated client instances rather than access codes for + * security. + */ + export async function retrieveOAuthClient(userId: string): Promise<OAuth2Client> { + return new Promise(async resolve => { + const { credentials, refreshed } = await retrieveCredentials(userId); + if (!credentials) { + return resolve(); + } + let client = authenticationClients.get(userId); + if (!client) { + authenticationClients.set(userId, client = generateClient(credentials)); + } else if (refreshed) { + client.setCredentials(credentials); + } + resolve(client); }); - }; + } + + /** + * Creates a new OAuth2Client instance, and if provided, sets + * the specific credentials on the client + * @param credentials if you have access to the credentials that you'll eventually set on + * the client, just pass them in at initialization + * @returns the newly created, potentially certified, OAuth2 client instance + */ + function generateClient(credentials?: Credentials): OAuth2Client { + const client = new google.auth.OAuth2(installed); + credentials && client.setCredentials(credentials); + return client; + } + + /** + * Calls on the worker (which does not have and does not need + * any credentials) to produce a url to which the user can + * navigate to give Dash the necessary Google permissions. + * @returns the newly generated url to the authentication landing page + */ + export function generateAuthenticationUrl(): string { + return worker.generateAuthUrl({ scope, access_type: 'offline' }); + } + /** + * This is what we return to the server in processNewUser(), after the + * worker OAuth2Client has used the user-pasted authentication code + * to retrieve an access token and an info token. The avatar is the + * URL to the Google-hosted mono-color, single white letter profile 'image'. + */ export interface GoogleAuthenticationResult { access_token: string; avatar: string; name: string; } - export const ProcessClientSideCode = async (information: CredentialInformation, authenticationCode: string): Promise<GoogleAuthenticationResult> => { - return new Promise<GoogleAuthenticationResult>((resolve, reject) => { - AuthorizationManager.getToken(authenticationCode, async (err, token) => { - if (err || !token) { + + /** + * This method receives the authentication code that the + * user pasted into the overlay in the client side and uses the worker + * and the authentication code to fetch the full set of credentials that + * we'll store in the database for each user. This is called once per + * new account integration. + * @param userId the Dash user id of the user requesting account integration, used to associate the new credentials + * with a Dash user in the googleAuthentication table of the database. + * @param authenticationCode the Google-provided authentication code that the user copied + * from Google's permissions UI and pasted into the overlay. + * + * EXAMPLE CODE: 4/sgF2A5uGg4xASHf7VQDnLtdqo3mUlfQqLSce_HYz5qf1nFtHj9YTeGs + * + * @returns the information necessary to authenticate a client side google photos request + * and display basic user information in the overlay on successful authentication. + * This can be expanded as needed by adding properties to the interface GoogleAuthenticationResult. + */ + export async function processNewUser(userId: string, authenticationCode: string): Promise<GoogleAuthenticationResult> { + const credentials = await new Promise<Credentials>((resolve, reject) => { + worker.getToken(authenticationCode, async (err, credentials) => { + if (err || !credentials) { reject(err); - return console.error('Error retrieving access token', err); + return; } - AuthorizationManager.setCredentials(token); - const enriched = injectUserInfo(token); - await Database.Auxiliary.GoogleAuthenticationToken.Write(information.userId, enriched); - const { given_name, picture } = enriched.userInfo; - resolve({ - access_token: enriched.access_token!, - avatar: picture, - name: given_name - }); + resolve(credentials); }); }); - }; + const enriched = injectUserInfo(credentials); + await Database.Auxiliary.GoogleAuthenticationToken.Write(userId, enriched); + const { given_name, picture } = enriched.userInfo; + return { + access_token: enriched.access_token!, + avatar: picture, + name: given_name + }; + } /** - * It's pretty cool: the credentials id_token is split into thirds by periods. - * The middle third contains a base64-encoded JSON string with all the - * user info contained in the interface below. So, we isolate that middle third, - * base64 decode with atob and parse the JSON. - * @param credentials the client credentials returned from OAuth after the user - * has executed the authentication routine + * This type represents the union of the full set of OAuth2 credentials + * and all of a Google user's publically available information. This is the strucure + * of the JSON object we ultimately store in the googleAuthentication table of the database. */ - const injectUserInfo = (credentials: Credentials): EnrichedCredentials => { - const userInfo = JSON.parse(atob(credentials.id_token!.split(".")[1])); - return { ...credentials, userInfo }; - }; - export type EnrichedCredentials = Credentials & { userInfo: UserInfo }; + + /** + * This interface defines all of the information we + * receive from parsing the base64 encoded info-token + * for a Google user. + */ export interface UserInfo { at_hash: string; aud: string; @@ -155,70 +292,71 @@ export namespace GoogleApiServerUtils { sub: string; } - export const RetrieveCredentials = (information: CredentialInformation) => { - return new Promise<TokenResult>((resolve, reject) => { - readFile(information.credentialsPath, async (err, credentials) => { - if (err) { - reject(err); - return console.log('Error loading client secret file:', err); - } - authorize(parseBuffer(credentials), information.userId).then(resolve, reject); - }); - }); - }; - - export const RetrievePhotosEndpoint = (paths: CredentialInformation) => { - return new Promise<any>((resolve, reject) => { - RetrieveAccessToken(paths).then( - token => resolve(new Photos(token)), - reject - ); - }); - }; - - type TokenResult = { token: Credentials, client: OAuth2Client }; - /** - * Create an OAuth2 client with the given credentials, and returns the promise resolving to the authenticated client - * @param {Object} credentials The authorization client credentials. - */ - export function authorize(credentials: any, userId: string): Promise<TokenResult> { - const { client_secret, client_id, redirect_uris } = credentials.installed; - const oAuth2Client = new google.auth.OAuth2(client_id, client_secret, redirect_uris[0]); - return new Promise<TokenResult>((resolve, reject) => { - // Attempting to authorize user (${userId}) - Database.Auxiliary.GoogleAuthenticationToken.Fetch(userId).then(token => { - if (token!.expiry_date! < new Date().getTime()) { - // Token has expired, so submitting a request for a refreshed access token - return refreshToken(token!, client_id, client_secret, oAuth2Client, userId).then(resolve, reject); - } - // Authentication successful! - oAuth2Client.setCredentials(token!); - resolve({ token: token!, client: oAuth2Client }); - }); - }); + /** + * It's pretty cool: the credentials id_token is split into thirds by periods. + * The middle third contains a base64-encoded JSON string with all the + * user info contained in the interface below. So, we isolate that middle third, + * base64 decode with atob and parse the JSON. + * @param credentials the client credentials returned from OAuth after the user + * has executed the authentication routine + * @returns the full set of credentials in the structure in which they'll be stored + * in the database. + */ + function injectUserInfo(credentials: Credentials): EnrichedCredentials { + const userInfo: UserInfo = JSON.parse(atob(credentials.id_token!.split(".")[1])); + return { ...credentials, userInfo }; } - const refreshEndpoint = "https://oauth2.googleapis.com/token"; - const refreshToken = (credentials: Credentials, client_id: string, client_secret: string, oAuth2Client: OAuth2Client, userId: string) => { - return new Promise<TokenResult>(resolve => { - let headerParameters = { headers: { 'Content-Type': 'application/x-www-form-urlencoded' } }; - let queryParameters = { - refreshToken: credentials.refresh_token, - client_id, - client_secret, - grant_type: "refresh_token" - }; - let url = `${refreshEndpoint}?${qs.stringify(queryParameters)}`; - request.post(url, headerParameters).then(async response => { - let { access_token, expires_in } = JSON.parse(response); - const expiry_date = new Date().getTime() + (expires_in * 1000); - await Database.Auxiliary.GoogleAuthenticationToken.Update(userId, access_token, expiry_date); - credentials.access_token = access_token; - credentials.expiry_date = expiry_date; - oAuth2Client.setCredentials(credentials); - resolve({ token: credentials, client: oAuth2Client }); - }); + /** + * Looks in the database for any credentials object with the given user id, + * and returns them. If the credentials are found but expired, the function will + * automatically refresh the credentials and then resolve with the updated values. + * @param userId the id of the Dash user requesting his/her credentials. Eventually, each user might + * be associated with multiple different sets of Google credentials. + * @returns the credentials, or undefined if the user has no stored associated credentials, + * and a flag indicating whether or not they were refreshed during retrieval + */ + async function retrieveCredentials(userId: string): Promise<{ credentials: Opt<Credentials>, refreshed: boolean }> { + let credentials: Opt<Credentials> = await Database.Auxiliary.GoogleAuthenticationToken.Fetch(userId); + let refreshed = false; + if (!credentials) { + return { credentials: undefined, refreshed }; + } + // check for token expiry + if (credentials.expiry_date! <= new Date().getTime()) { + credentials = await refreshAccessToken(credentials, userId); + } + return { credentials, refreshed }; + } + + /** + * This function submits a request to OAuth with the local refresh token + * to revalidate the credentials for a given Google user associated with + * the Dash user id passed in. In addition to returning the credentials, it + * writes the diff to the database. + * @param credentials the credentials + * @param userId the id of the Dash user implicitly requesting that + * his/her credentials be refreshed + * @returns the updated credentials + */ + async function refreshAccessToken(credentials: Credentials, userId: string): Promise<Credentials> { + let headerParameters = { headers: { 'Content-Type': 'application/x-www-form-urlencoded' } }; + let url = `https://oauth2.googleapis.com/token?${qs.stringify({ + refreshToken: credentials.refresh_token, + grant_type: "refresh_token", + ...installed + })}`; + const { access_token, expires_in } = await new Promise<any>(async resolve => { + const response = await request.post(url, headerParameters); + resolve(JSON.parse(response)); }); - }; + // expires_in is in seconds, but we're building the new expiry date in milliseconds + const expiry_date = new Date().getTime() + (expires_in * 1000); + await Database.Auxiliary.GoogleAuthenticationToken.Update(userId, access_token, expiry_date); + // update the relevant properties + credentials.access_token = access_token; + credentials.expiry_date = expiry_date; + return credentials; + } }
\ No newline at end of file diff --git a/src/server/apis/google/GooglePhotosUploadUtils.ts b/src/server/apis/google/GooglePhotosUploadUtils.ts index 4a67e57cc..3862fedc8 100644 --- a/src/server/apis/google/GooglePhotosUploadUtils.ts +++ b/src/server/apis/google/GooglePhotosUploadUtils.ts @@ -1,85 +1,148 @@ import request = require('request-promise'); -import { GoogleApiServerUtils } from './GoogleApiServerUtils'; import * as path from 'path'; -import { MediaItemCreationResult } from './SharedTypes'; -import { NewMediaItem } from "../../index"; +import { NewMediaItemResult } from './SharedTypes'; import { BatchedArray, TimeUnit } from 'array-batcher'; import { DashUploadUtils } from '../../DashUploadUtils'; +/** + * This namespace encompasses the logic + * necessary to upload images to Google's server, + * and then initialize / create those images in the Photos + * API given the upload tokens returned from the initial + * uploading process. + * + * https://developers.google.com/photos/library/reference/rest/v1/mediaItems/batchCreate + */ export namespace GooglePhotosUploadUtils { - export interface Paths { - uploadDirectory: string; - credentialsPath: string; - tokenPath: string; - } - - export interface MediaInput { + /** + * Specifies the structure of the object + * necessary to upload bytes to Google's servers. + * The url is streamed to access the image's bytes, + * and the description is what appears in Google Photos' + * description field. + */ + export interface UploadSource { url: string; description: string; } - const prepend = (extension: string) => `https://photoslibrary.googleapis.com/v1/${extension}`; - const headers = (type: string) => ({ - 'Content-Type': `application/${type}`, - 'Authorization': Bearer, - }); + /** + * This is the format needed to pass + * into the BatchCreate API request + * to take a reference to raw uploaded bytes + * and actually create an image in Google Photos. + * + * So, to instantiate this interface you must have already dispatched an upload + * and received an upload token. + */ + export interface NewMediaItem { + description: string; + simpleMediaItem: { + uploadToken: string; + }; + } - let Bearer: string; + /** + * A utility function to streamline making + * calls to the API's url - accentuates + * the relative path in the caller. + * @param extension the desired + * subset of the API + */ + function prepend(extension: string): string { + return `https://photoslibrary.googleapis.com/v1/${extension}`; + } - export const initialize = async (information: GoogleApiServerUtils.CredentialInformation) => { - const token = await GoogleApiServerUtils.RetrieveAccessToken(information); - Bearer = `Bearer ${token}`; - }; + /** + * Factors out the creation of the API request's + * authentication elements stored in the header. + * @param type the contents of the request + * @param token the user-specific Google access token + */ + function headers(type: string, token: string) { + return { + 'Content-Type': `application/${type}`, + 'Authorization': `Bearer ${token}`, + }; + } - export const DispatchGooglePhotosUpload = async (url: string) => { - if (!DashUploadUtils.imageFormats.includes(path.extname(url))) { + /** + * This is the first step in the remote image creation process. + * Here we upload the raw bytes of the image to Google's servers by + * setting authentication and other required header properties and including + * the raw bytes to the image, to be uploaded, in the body of the request. + * @param bearerToken the user-specific Google access token, specifies the account associated + * with the eventual image creation + * @param url the url of the image to upload + * @param filename an optional name associated with the uploaded image - if not specified + * defaults to the filename (basename) in the url + */ + export const DispatchGooglePhotosUpload = async (bearerToken: string, url: string, filename?: string): Promise<any> => { + // check if the url points to a non-image or an unsupported format + if (!DashUploadUtils.validateExtension(url)) { return undefined; } - const body = await request(url, { encoding: null }); const parameters = { method: 'POST', + uri: prepend('uploads'), headers: { - ...headers('octet-stream'), - 'X-Goog-Upload-File-Name': path.basename(url), + ...headers('octet-stream', bearerToken), + 'X-Goog-Upload-File-Name': filename || path.basename(url), 'X-Goog-Upload-Protocol': 'raw' }, - uri: prepend('uploads'), - body + body: await request(url, { encoding: null }) // returns a readable stream with the unencoded binary image data }; - return new Promise<any>((resolve, reject) => request(parameters, (error, _response, body) => { + return new Promise((resolve, reject) => request(parameters, (error, _response, body) => { if (error) { - console.log(error); + // on rejection, the server logs the error and the offending image return reject(error); } resolve(body); })); }; - export const CreateMediaItems = async (newMediaItems: NewMediaItem[], album?: { id: string }): Promise<MediaItemCreationResult> => { - const newMediaItemResults = await BatchedArray.from(newMediaItems, { batchSize: 50 }).batchedMapPatientInterval( + /** + * This is the second step in the remote image creation process: having uploaded + * the raw bytes of the image and received / stored pointers (upload tokens) to those + * bytes, we can now instruct the API to finalize the creation of those images by + * submitting a batch create request with the list of upload tokens and the description + * to be associated with reach resulting new image. + * @param bearerToken the user-specific Google access token, specifies the account associated + * with the eventual image creation + * @param newMediaItems a list of objects containing a description and, effectively, the + * pointer to the uploaded bytes + * @param album if included, will add all of the newly created remote images to the album + * with the specified id + */ + export const CreateMediaItems = async (bearerToken: string, newMediaItems: NewMediaItem[], album?: { id: string }): Promise<NewMediaItemResult[]> => { + // it's important to note that the API can't handle more than 50 items in each request and + // seems to need at least some latency between requests (spamming it synchronously has led to the server returning errors)... + const batched = BatchedArray.from(newMediaItems, { batchSize: 50 }); + // ...so we execute them in delayed batches and await the entire execution + return batched.batchedMapPatientInterval( { magnitude: 100, unit: TimeUnit.Milliseconds }, - async (batch: NewMediaItem[]) => { + async (batch: NewMediaItem[], collector: any): Promise<any> => { const parameters = { method: 'POST', - headers: headers('json'), + headers: headers('json', bearerToken), uri: prepend('mediaItems:batchCreate'), body: { newMediaItems: batch } as any, json: true }; + // register the target album, if provided album && (parameters.body.albumId = album.id); - return (await new Promise<MediaItemCreationResult>((resolve, reject) => { + collector.push(...(await new Promise<NewMediaItemResult[]>((resolve, reject) => { request(parameters, (error, _response, body) => { if (error) { reject(error); } else { - resolve(body); + resolve(body.newMediaItemResults); } }); - })).newMediaItemResults; + }))); } ); - return { newMediaItemResults }; }; }
\ No newline at end of file diff --git a/src/server/authentication/config/passport.ts b/src/server/authentication/config/passport.ts index 8915a4abf..0b15c3a36 100644 --- a/src/server/authentication/config/passport.ts +++ b/src/server/authentication/config/passport.ts @@ -3,7 +3,6 @@ import * as passportLocal from 'passport-local'; import _ from "lodash"; import { default as User } from '../models/user_model'; import { Request, Response, NextFunction } from "express"; -import { RouteStore } from '../../RouteStore'; const LocalStrategy = passportLocal.Strategy; @@ -35,13 +34,13 @@ export let isAuthenticated = (req: Request, res: Response, next: NextFunction) = if (req.isAuthenticated()) { return next(); } - return res.redirect(RouteStore.login); + return res.redirect("/login"); }; export let isAuthorized = (req: Request, res: Response, next: NextFunction) => { const provider = req.path.split("/").slice(-1)[0]; - if (_.find((req.user as any).tokens, { kind: provider })) { + if (_.find((req.user).tokens, { kind: provider })) { next(); } else { res.redirect(`/auth/${provider}`); diff --git a/src/server/authentication/controllers/user_controller.ts b/src/server/authentication/controllers/user_controller.ts index f5c6e1610..b2b9d33f6 100644 --- a/src/server/authentication/controllers/user_controller.ts +++ b/src/server/authentication/controllers/user_controller.ts @@ -10,10 +10,7 @@ import * as pug from 'pug'; import * as async from 'async'; import * as nodemailer from 'nodemailer'; import c = require("crypto"); -import { RouteStore } from "../../RouteStore"; import { Utils } from "../../../Utils"; -import { Schema } from "mongoose"; -import { Opt } from "../../../new_fields/Doc"; import { MailOptions } from "nodemailer/lib/stream-transport"; /** @@ -23,8 +20,7 @@ import { MailOptions } from "nodemailer/lib/stream-transport"; */ export let getSignup = (req: Request, res: Response) => { if (req.user) { - let user = req.user; - return res.redirect(RouteStore.home); + return res.redirect("/home"); } res.render("signup.pug", { title: "Sign Up", @@ -45,7 +41,7 @@ export let postSignup = (req: Request, res: Response, next: NextFunction) => { const errors = req.validationErrors(); if (errors) { - return res.redirect(RouteStore.signup); + return res.redirect("/signup"); } const email = req.body.email as String; @@ -62,7 +58,7 @@ export let postSignup = (req: Request, res: Response, next: NextFunction) => { User.findOne({ email }, (err, existingUser) => { if (err) { return next(err); } if (existingUser) { - return res.redirect(RouteStore.login); + return res.redirect("/login"); } user.save((err: any) => { if (err) { return next(err); } @@ -81,7 +77,7 @@ let tryRedirectToTarget = (req: Request, res: Response) => { req.session.target = undefined; res.redirect(target); } else { - res.redirect(RouteStore.home); + res.redirect("/home"); } }; @@ -93,7 +89,7 @@ let tryRedirectToTarget = (req: Request, res: Response) => { export let getLogin = (req: Request, res: Response) => { if (req.user) { req.session!.target = undefined; - return res.redirect(RouteStore.home); + return res.redirect("/home"); } res.render("login.pug", { title: "Log In", @@ -115,13 +111,13 @@ export let postLogin = (req: Request, res: Response, next: NextFunction) => { if (errors) { req.flash("errors", "Unable to login at this time. Please try again."); - return res.redirect(RouteStore.signup); + return res.redirect("/signup"); } passport.authenticate("local", (err: Error, user: DashUserModel, info: IVerifyOptions) => { if (err) { next(err); return; } if (!user) { - return res.redirect(RouteStore.signup); + return res.redirect("/signup"); } req.logIn(user, (err) => { if (err) { next(err); return; } @@ -141,7 +137,7 @@ export let getLogout = (req: Request, res: Response) => { if (sess) { sess.destroy((err) => { if (err) { console.log(err); } }); } - res.redirect(RouteStore.login); + res.redirect("/login"); }; export let getForgot = function (req: Request, res: Response) { @@ -168,7 +164,7 @@ export let postForgot = function (req: Request, res: Response, next: NextFunctio User.findOne({ email }, function (err, user: DashUserModel) { if (!user) { // NO ACCOUNT WITH SUBMITTED EMAIL - res.redirect(RouteStore.forgot); + res.redirect("/forgotPassword"); return; } user.passwordResetToken = token; @@ -192,7 +188,7 @@ export let postForgot = function (req: Request, res: Response, next: NextFunctio subject: 'Dash Password Reset', text: 'You are receiving this because you (or someone else) have requested the reset of the password for your account.\n\n' + 'Please click on the following link, or paste this into your browser to complete the process:\n\n' + - 'http://' + req.headers.host + '/reset/' + token + '\n\n' + + 'http://' + req.headers.host + '/resetPassword/' + token + '\n\n' + 'If you did not request this, please ignore this email and your password will remain unchanged.\n' } as MailOptions; smtpTransport.sendMail(mailOptions, function (err: Error | null) { @@ -202,14 +198,14 @@ export let postForgot = function (req: Request, res: Response, next: NextFunctio } ], function (err) { if (err) return next(err); - res.redirect(RouteStore.forgot); + res.redirect("/forgotPassword"); }); }; export let getReset = function (req: Request, res: Response) { User.findOne({ passwordResetToken: req.params.token, passwordResetExpires: { $gt: Date.now() } }, function (err, user: DashUserModel) { if (!user || err) { - return res.redirect(RouteStore.forgot); + return res.redirect("/forgotPassword"); } res.render("reset.pug", { title: "Reset Password", @@ -239,7 +235,7 @@ export let postReset = function (req: Request, res: Response) { user.save(function (err) { if (err) { - res.redirect(RouteStore.login); + res.redirect("/login"); return; } req.logIn(user, function (err) { @@ -271,6 +267,6 @@ export let postReset = function (req: Request, res: Response) { }); } ], function (err) { - res.redirect(RouteStore.login); + res.redirect("/login"); }); };
\ No newline at end of file diff --git a/src/server/authentication/models/current_user_utils.ts b/src/server/authentication/models/current_user_utils.ts index 73cac879e..5a8815983 100644 --- a/src/server/authentication/models/current_user_utils.ts +++ b/src/server/authentication/models/current_user_utils.ts @@ -11,7 +11,6 @@ import { List } from "../../../new_fields/List"; import { listSpec } from "../../../new_fields/Schema"; import { Cast, StrCast, PromiseValue } from "../../../new_fields/Types"; import { Utils } from "../../../Utils"; -import { RouteStore } from "../../RouteStore"; import { ScriptField } from "../../../new_fields/ScriptField"; import { ButtonBox } from "../../../client/views/nodes/ButtonBox"; import { UndoManager } from "../../../client/util/UndoManager"; @@ -198,8 +197,8 @@ export class CurrentUserUtils { return doc; } - public static loadCurrentUser() { - return rp.get(Utils.prepend(RouteStore.getCurrUser)).then(response => { + public static async loadCurrentUser() { + return rp.get(Utils.prepend("/getCurrentUser")).then(response => { if (response) { const result: { id: string, email: string } = JSON.parse(response); return result; @@ -212,7 +211,7 @@ export class CurrentUserUtils { public static async loadUserDocument({ id, email }: { id: string, email: string }) { this.curr_id = id; Doc.CurrentUserEmail = email; - await rp.get(Utils.prepend(RouteStore.getUserDocumentId)).then(id => { + await rp.get(Utils.prepend("/getUserDocumentId")).then(id => { if (id && id !== "guest") { return DocServer.GetRefField(id).then(async field => { if (field instanceof Doc) { diff --git a/src/server/credentials/CredentialsLoader.ts b/src/server/credentials/CredentialsLoader.ts new file mode 100644 index 000000000..e3f4d167b --- /dev/null +++ b/src/server/credentials/CredentialsLoader.ts @@ -0,0 +1,29 @@ +import { readFile } from "fs"; + +export namespace GoogleCredentialsLoader { + + export interface InstalledCredentials { + client_id: string; + project_id: string; + auth_uri: string; + token_uri: string; + auth_provider_x509_cert_url: string; + client_secret: string; + redirect_uris: string[]; + } + + export let ProjectCredentials: InstalledCredentials; + + export async function loadCredentials() { + ProjectCredentials = await new Promise<InstalledCredentials>(resolve => { + readFile(__dirname + '/google_project_credentials.json', function processClientSecrets(err, content) { + if (err) { + console.log('Error loading client secret file: ' + err); + return; + } + resolve(JSON.parse(content.toString()).installed); + }); + }); + } + +} diff --git a/src/server/credentials/google_project_credentials.json b/src/server/credentials/google_project_credentials.json new file mode 100644 index 000000000..5d9c62eb1 --- /dev/null +++ b/src/server/credentials/google_project_credentials.json @@ -0,0 +1,14 @@ +{ + "installed": { + "client_id": "1005546247619-kqpnvh42mpa803tem8556b87umi4j9r0.apps.googleusercontent.com", + "project_id": "brown-dash", + "auth_uri": "https://accounts.google.com/o/oauth2/auth", + "token_uri": "https://oauth2.googleapis.com/token", + "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", + "client_secret": "WshLb5TH9SdFVGGbQcnYj7IU", + "redirect_uris": [ + "urn:ietf:wg:oauth:2.0:oob", + "http://localhost" + ] + } +}
\ No newline at end of file diff --git a/src/server/credentials/google_docs_credentials.json b/src/server/credentials/test.json index 955c5a3c1..0a032cc2d 100644 --- a/src/server/credentials/google_docs_credentials.json +++ b/src/server/credentials/test.json @@ -6,6 +6,9 @@ "token_uri": "https://oauth2.googleapis.com/token", "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", "client_secret": "w8KIFSc0MQpmUYHed4qEzn8b", - "redirect_uris": ["urn:ietf:wg:oauth:2.0:oob", "http://localhost"] + "redirect_uris": [ + "urn:ietf:wg:oauth:2.0:oob", + "http://localhost" + ] } }
\ No newline at end of file diff --git a/src/server/database.ts b/src/server/database.ts index 12626e594..b81fc03a4 100644 --- a/src/server/database.ts +++ b/src/server/database.ts @@ -60,6 +60,10 @@ export namespace Database { constructor() { this.MongoClient.connect(url, (_err, client) => { + if (!client) { + console.error("\nPlease start MongoDB by running 'mongod' in a terminal before continuing...\n"); + process.exit(0); + } this.db = client.db(); this.onConnect.forEach(fn => fn()); }); @@ -298,7 +302,7 @@ export namespace Database { export type StoredCredentials = Credentials & { _id: string }; - export const Fetch = async (userId: string, removeId = true) => { + export const Fetch = async (userId: string, removeId = true): Promise<Opt<StoredCredentials>> => { return SanitizedSingletonQuery<StoredCredentials>({ userId }, GoogleAuthentication, removeId); }; diff --git a/src/server/index.ts b/src/server/index.ts index 93f4238bc..773b84403 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -1,432 +1,84 @@ require('dotenv').config(); -import * as formidable from 'formidable'; -import * as fs from 'fs'; -import * as sharp from 'sharp'; -import * as Pdfjs from 'pdfjs-dist'; -const imageDataUri = require('image-data-uri'); +import { GoogleApiServerUtils } from "./apis/google/GoogleApiServerUtils"; import * as mobileDetect from 'mobile-detect'; import * as path from 'path'; import { Database } from './database'; -import { RouteStore } from './RouteStore'; -import v4 = require('uuid/v4'); -import { createCanvas } from "canvas"; const serverPort = 4321; -import { Search } from './Search'; -import * as Archiver from 'archiver'; -var AdmZip = require('adm-zip'); -import * as YoutubeApi from "./apis/youtube/youtubeApiSample"; -import { Response } from 'express-serve-static-core'; -import { GoogleApiServerUtils } from "./apis/google/GoogleApiServerUtils"; -const probe = require("probe-image-size"); -const pdf = require('pdf-parse'); -import { GooglePhotosUploadUtils } from './apis/google/GooglePhotosUploadUtils'; -import { Opt } from '../new_fields/Doc'; import { DashUploadUtils } from './DashUploadUtils'; -import { BatchedArray, TimeUnit } from 'array-batcher'; -import { ParsedPDF } from "./PdfTypes"; -import { reject } from 'bluebird'; import RouteSubscriber from './RouteSubscriber'; -import InitializeServer from './Initialization'; +import initializeServer from './Initialization'; import RouteManager, { Method, _success, _permission_denied, _error, _invalid, OnUnauthenticated } from './RouteManager'; import * as qs from 'query-string'; import UtilManager from './ApiManagers/UtilManager'; import SearchManager from './ApiManagers/SearchManager'; import UserManager from './ApiManagers/UserManager'; import { WebSocket } from './Websocket/Websocket'; - -export let youtubeApiKey: string; - -export type Hierarchy = { [id: string]: string | Hierarchy }; -export type ZipMutator = (file: Archiver.Archiver) => void | Promise<void>; - -export interface NewMediaItem { - description: string; - simpleMediaItem: { - uploadToken: string; - }; -} - -const pngTypes = [".png", ".PNG"]; -const jpgTypes = [".jpg", ".JPG", ".jpeg", ".JPEG"]; -export const uploadDirectory = __dirname + "/public/files/"; -const pdfDirectory = uploadDirectory + "text"; -const solrURL = "http://localhost:8983/solr/#/dash"; - -start(); - -async function start() { - await PreliminaryFunctions(); - await InitializeServer({ listenAtPort: 1050, routeSetter }); +import DownloadManager from './ApiManagers/ExportManager'; +import { GoogleCredentialsLoader } from './credentials/CredentialsLoader'; +import DeleteManager from "./ApiManagers/DeleteManager"; +import PDFManager from "./ApiManagers/PDFManager"; +import UploadManager from "./ApiManagers/UploadManager"; +import GeneralGoogleManager from "./ApiManagers/GeneralGoogleManager"; +import GooglePhotosManager from "./ApiManagers/GooglePhotosManager"; + +export const publicDirectory = __dirname + "/public"; +export const filesDirectory = publicDirectory + "/files/"; +export enum Partitions { + pdf_text, + images, + videos } -async function PreliminaryFunctions() { - await new Promise<void>(resolve => { - YoutubeApi.readApiKey((apiKey: string) => { - youtubeApiKey = apiKey; - resolve(); - }); - }); - await GoogleApiServerUtils.LoadOAuthClient(); - await DashUploadUtils.createIfNotExists(pdfDirectory); +/** + * These are the functions run before the server starts + * listening. Anything that must be complete + * before clients can access the server should be run or awaited here. + */ +async function preliminaryFunctions() { + // make project credentials globally accessible + await GoogleCredentialsLoader.loadCredentials(); + // read the resulting credentials into a different namespace + GoogleApiServerUtils.processProjectCredentials(); + // divide the public directory based on type + await Promise.all(Object.keys(Partitions).map(partition => DashUploadUtils.createIfNotExists(filesDirectory + partition))); + // connect to the database await Database.tryInitializeConnection(); } +/** + * Either clustered together as an API manager + * or individually referenced below, by the completion + * of this function's execution, all routes will + * be registered on the server + * @param router the instance of the route manager + * that will manage the registration of new routes + * with the server + */ function routeSetter(router: RouteManager) { - new UtilManager().register(router); - new SearchManager().register(router); - new UserManager().register(router); - + // initialize API Managers + [ + new UserManager(), + new UploadManager(), + new DownloadManager(), + new SearchManager(), + new PDFManager(), + new DeleteManager(), + new UtilManager(), + new GeneralGoogleManager(), + new GooglePhotosManager(), + ].forEach(manager => manager.register(router)); + + // initialize the web socket (bidirectional communication: if a user changes + // a field on one client, that change must be broadcast to all other clients) WebSocket.initialize(serverPort, router.isRelease); - async function getDocs(id: string) { - const files = new Set<string>(); - const docs: { [id: string]: any } = {}; - const fn = (doc: any): string[] => { - const id = doc.id; - if (typeof id === "string" && id.endsWith("Proto")) { - //Skip protos - return []; - } - const ids: string[] = []; - for (const key in doc.fields) { - if (!doc.fields.hasOwnProperty(key)) { - continue; - } - const field = doc.fields[key]; - if (field === undefined || field === null) { - continue; - } - - if (field.__type === "proxy" || field.__type === "prefetch_proxy") { - ids.push(field.fieldId); - } else if (field.__type === "script" || field.__type === "computed") { - if (field.captures) { - ids.push(field.captures.fieldId); - } - } else if (field.__type === "list") { - ids.push(...fn(field)); - } else if (typeof field === "string") { - const re = /"(?:dataD|d)ocumentId"\s*:\s*"([\w\-]*)"/g; - let match: string[] | null; - while ((match = re.exec(field)) !== null) { - ids.push(match[1]); - } - } else if (field.__type === "RichTextField") { - const re = /"href"\s*:\s*"(.*?)"/g; - let match: string[] | null; - while ((match = re.exec(field.Data)) !== null) { - const urlString = match[1]; - const split = new URL(urlString).pathname.split("doc/"); - if (split.length > 1) { - ids.push(split[split.length - 1]); - } - } - const re2 = /"src"\s*:\s*"(.*?)"/g; - while ((match = re2.exec(field.Data)) !== null) { - const urlString = match[1]; - const pathname = new URL(urlString).pathname; - files.add(pathname); - } - } else if (["audio", "image", "video", "pdf", "web"].includes(field.__type)) { - const url = new URL(field.url); - const pathname = url.pathname; - files.add(pathname); - } - } - - if (doc.id) { - docs[doc.id] = doc; - } - return ids; - }; - await Database.Instance.visit([id], fn); - return { id, docs, files }; - } - - router.addSupervisedRoute({ - method: Method.GET, - subscription: new RouteSubscriber("/serializeDoc").add("docId"), - onValidation: async ({ req, res }) => { - const { docs, files } = await getDocs(req.params.docId); - res.send({ docs, files: Array.from(files) }); - } - }); - - router.addSupervisedRoute({ - method: Method.GET, - subscription: new RouteSubscriber(RouteStore.imageHierarchyExport).add('docId'), - onValidation: async ({ req, res }) => { - const id = req.params.docId; - const hierarchy: Hierarchy = {}; - await targetedVisitorRecursive(id, hierarchy); - BuildAndDispatchZip(res, async zip => { - await hierarchyTraverserRecursive(zip, hierarchy); - }); - } - }); - - const BuildAndDispatchZip = async (res: Response, mutator: ZipMutator): Promise<void> => { - const zip = Archiver('zip'); - zip.pipe(res); - await mutator(zip); - return zip.finalize(); - }; - - const targetedVisitorRecursive = async (seedId: string, hierarchy: Hierarchy): Promise<void> => { - const local: Hierarchy = {}; - const { title, data } = await getData(seedId); - const label = `${title} (${seedId})`; - if (Array.isArray(data)) { - hierarchy[label] = local; - await Promise.all(data.map(proxy => targetedVisitorRecursive(proxy.fieldId, local))); - } else { - hierarchy[label + path.extname(data)] = data; - } - }; - - const getData = async (seedId: string): Promise<{ data: string | any[], title: string }> => { - return new Promise<{ data: string | any[], title: string }>((resolve, reject) => { - Database.Instance.getDocument(seedId, async (result: any) => { - const { data, proto, title } = result.fields; - if (data) { - if (data.url) { - resolve({ data: data.url, title }); - } else if (data.fields) { - resolve({ data: data.fields, title }); - } else { - reject(); - } - } - if (proto) { - getData(proto.fieldId).then(resolve, reject); - } - }); - }); - }; - - const hierarchyTraverserRecursive = async (file: Archiver.Archiver, hierarchy: Hierarchy, prefix = "Dash Export"): Promise<void> => { - for (const key of Object.keys(hierarchy)) { - const result = hierarchy[key]; - if (typeof result === "string") { - let path: string; - let matches: RegExpExecArray | null; - if ((matches = /\:1050\/files\/(upload\_[\da-z]{32}.*)/g.exec(result)) !== null) { - path = `${__dirname}/public/files/${matches[1]}`; - } else { - const information = await DashUploadUtils.UploadImage(result); - path = information.mediaPaths[0]; - } - file.file(path, { name: key, prefix }); - } else { - await hierarchyTraverserRecursive(file, result, `${prefix}/${key}`); - } - } - }; - - router.addSupervisedRoute({ - method: Method.GET, - subscription: new RouteSubscriber("/downloadId").add("docId"), - onValidation: async ({ req, res }) => { - res.set('Content-disposition', `attachment;`); - res.set('Content-Type', "application/zip"); - const { id, docs, files } = await getDocs(req.params.docId); - const docString = JSON.stringify({ id, docs }); - const zip = Archiver('zip'); - zip.pipe(res); - zip.append(docString, { name: "doc.json" }); - files.forEach(val => { - zip.file(__dirname + RouteStore.public + val, { name: val.substring(1) }); - }); - zip.finalize(); - } - }); - - router.addSupervisedRoute({ - method: Method.POST, - subscription: "/uploadDoc", - onValidation: ({ req, res }) => { - let form = new formidable.IncomingForm(); - form.keepExtensions = true; - // let path = req.body.path; - const ids: { [id: string]: string } = {}; - let remap = true; - const getId = (id: string): string => { - if (!remap) return id; - if (id.endsWith("Proto")) return id; - if (id in ids) { - return ids[id]; - } else { - return ids[id] = v4(); - } - }; - const mapFn = (doc: any) => { - if (doc.id) { - doc.id = getId(doc.id); - } - for (const key in doc.fields) { - if (!doc.fields.hasOwnProperty(key)) { - continue; - } - const field = doc.fields[key]; - if (field === undefined || field === null) { - continue; - } - - if (field.__type === "proxy" || field.__type === "prefetch_proxy") { - field.fieldId = getId(field.fieldId); - } else if (field.__type === "script" || field.__type === "computed") { - if (field.captures) { - field.captures.fieldId = getId(field.captures.fieldId); - } - } else if (field.__type === "list") { - mapFn(field); - } else if (typeof field === "string") { - const re = /("(?:dataD|d)ocumentId"\s*:\s*")([\w\-]*)"/g; - doc.fields[key] = (field as any).replace(re, (match: any, p1: string, p2: string) => { - return `${p1}${getId(p2)}"`; - }); - } else if (field.__type === "RichTextField") { - const re = /("href"\s*:\s*")(.*?)"/g; - field.Data = field.Data.replace(re, (match: any, p1: string, p2: string) => { - return `${p1}${getId(p2)}"`; - }); - } - } - }; - form.parse(req, async (err, fields, files) => { - remap = fields.remap !== "false"; - let id: string = ""; - try { - for (const name in files) { - const path_2 = files[name].path; - const zip = new AdmZip(path_2); - zip.getEntries().forEach((entry: any) => { - if (!entry.entryName.startsWith("files/")) return; - let dirname = path.dirname(entry.entryName) + "/"; - let extname = path.extname(entry.entryName); - let basename = path.basename(entry.entryName).split(".")[0]; - // zip.extractEntryTo(dirname + basename + "_o" + extname, __dirname + RouteStore.public, true, false); - // zip.extractEntryTo(dirname + basename + "_s" + extname, __dirname + RouteStore.public, true, false); - // zip.extractEntryTo(dirname + basename + "_m" + extname, __dirname + RouteStore.public, true, false); - // zip.extractEntryTo(dirname + basename + "_l" + extname, __dirname + RouteStore.public, true, false); - try { - zip.extractEntryTo(entry.entryName, __dirname + RouteStore.public, true, false); - dirname = "/" + dirname; - - fs.createReadStream(__dirname + RouteStore.public + dirname + basename + extname).pipe(fs.createWriteStream(__dirname + RouteStore.public + dirname + basename + "_o" + extname)); - fs.createReadStream(__dirname + RouteStore.public + dirname + basename + extname).pipe(fs.createWriteStream(__dirname + RouteStore.public + dirname + basename + "_s" + extname)); - fs.createReadStream(__dirname + RouteStore.public + dirname + basename + extname).pipe(fs.createWriteStream(__dirname + RouteStore.public + dirname + basename + "_m" + extname)); - fs.createReadStream(__dirname + RouteStore.public + dirname + basename + extname).pipe(fs.createWriteStream(__dirname + RouteStore.public + dirname + basename + "_l" + extname)); - } catch (e) { - console.log(e); - } - }); - const json = zip.getEntry("doc.json"); - let docs: any; - try { - let data = JSON.parse(json.getData().toString("utf8")); - docs = data.docs; - id = data.id; - docs = Object.keys(docs).map(key => docs[key]); - docs.forEach(mapFn); - await Promise.all(docs.map((doc: any) => new Promise(res => Database.Instance.replace(doc.id, doc, (err, r) => { - err && console.log(err); - res(); - }, true, "newDocuments")))); - } catch (e) { console.log(e); } - fs.unlink(path_2, () => { }); - } - if (id) { - res.send(JSON.stringify(getId(id))); - } else { - res.send(JSON.stringify("error")); - } - } catch (e) { console.log(e); } - }); - } - }); - - router.addSupervisedRoute({ - method: Method.GET, - subscription: new RouteSubscriber("/thumbnail").add("filename"), - onValidation: ({ req, res }) => { - let filename = req.params.filename; - let noExt = filename.substring(0, filename.length - ".png".length); - let pagenumber = parseInt(noExt.split('-')[1]); - fs.exists(uploadDirectory + filename, (exists: boolean) => { - console.log(`${uploadDirectory + filename} ${exists ? "exists" : "does not exist"}`); - if (exists) { - let input = fs.createReadStream(uploadDirectory + filename); - probe(input, (err: any, result: any) => { - if (err) { - console.log(err); - console.log(`error on ${filename}`); - return; - } - res.send({ path: "/files/" + filename, width: result.width, height: result.height }); - }); - } - else { - LoadPage(uploadDirectory + filename.substring(0, filename.length - noExt.split('-')[1].length - ".PNG".length - 1) + ".pdf", pagenumber, res); - } - }); - } - }); - - function LoadPage(file: string, pageNumber: number, res: Response) { - console.log(file); - Pdfjs.getDocument(file).promise - .then((pdf: Pdfjs.PDFDocumentProxy) => { - let factory = new NodeCanvasFactory(); - console.log(pageNumber); - pdf.getPage(pageNumber).then((page: Pdfjs.PDFPageProxy) => { - console.log("reading " + page); - let viewport = page.getViewport(1 as any); - let canvasAndContext = factory.create(viewport.width, viewport.height); - let renderContext = { - canvasContext: canvasAndContext.context, - viewport: viewport, - canvasFactory: factory - }; - console.log("read " + pageNumber); - - page.render(renderContext).promise - .then(() => { - console.log("saving " + pageNumber); - let stream = canvasAndContext.canvas.createPNGStream(); - let pngFile = `${file.substring(0, file.length - ".pdf".length)}-${pageNumber}.PNG`; - let out = fs.createWriteStream(pngFile); - stream.pipe(out); - out.on("finish", () => { - console.log(`Success! Saved to ${pngFile}`); - let name = path.basename(pngFile); - res.send({ path: "/files/" + name, width: viewport.width, height: viewport.height }); - }); - }, (reason: string) => { - console.error(reason + ` ${pageNumber}`); - }); - }); - }); - } - /** - * Anyone attempting to navigate to localhost at this port will - * first have to log in. + * Accessing root index redirects to home */ router.addSupervisedRoute({ method: Method.GET, - subscription: RouteStore.root, - onValidation: ({ res }) => res.redirect(RouteStore.home) - }); - - router.addSupervisedRoute({ - method: Method.GET, - subscription: RouteStore.getUsers, - onValidation: async ({ res }) => { - const cursor = await Database.Instance.query({}, { email: 1, userDocumentId: 1 }, "users"); - const results = await cursor.toArray(); - res.send(results.map(user => ({ email: user.email, userDocumentId: user.userDocumentId }))); - } + subscription: "/", + onValidation: ({ res }) => res.redirect("/home") }); const serve: OnUnauthenticated = ({ req, res }) => { @@ -437,7 +89,7 @@ function routeSetter(router: RouteManager) { router.addSupervisedRoute({ method: Method.GET, - subscription: [RouteStore.home, new RouteSubscriber("/doc").add("docId")], + subscription: ["/home", new RouteSubscriber("doc").add("docId")], onValidation: serve, onUnauthenticated: ({ req, ...remaining }) => { const { originalUrl: target } = req; @@ -448,344 +100,9 @@ function routeSetter(router: RouteManager) { } } }); +} - router.addSupervisedRoute({ - method: Method.GET, - subscription: RouteStore.getUserDocumentId, - onValidation: ({ res, user }) => res.send(user.userDocumentId) - }); - - router.addSupervisedRoute({ - method: Method.GET, - subscription: RouteStore.getCurrUser, - onValidation: ({ res, user }) => res.send(JSON.stringify(user)), - onUnauthenticated: ({ res }) => res.send(JSON.stringify({ id: "__guest__", email: "" })) - }); - - const ServicesApiKeyMap = new Map<string, string | undefined>([ - ["face", process.env.FACE], - ["vision", process.env.VISION], - ["handwriting", process.env.HANDWRITING] - ]); - - router.addSupervisedRoute({ - method: Method.GET, - subscription: new RouteSubscriber(RouteStore.cognitiveServices).add('requestedservice'), - onValidation: ({ req, res }) => { - let service = req.params.requestedservice; - res.send(ServicesApiKeyMap.get(service)); - } - }); - - class NodeCanvasFactory { - create = (width: number, height: number) => { - var canvas = createCanvas(width, height); - var context = canvas.getContext('2d'); - return { - canvas: canvas, - context: context, - }; - } - - reset = (canvasAndContext: any, width: number, height: number) => { - canvasAndContext.canvas.width = width; - canvasAndContext.canvas.height = height; - } - - destroy = (canvasAndContext: any) => { - canvasAndContext.canvas.width = 0; - canvasAndContext.canvas.height = 0; - canvasAndContext.canvas = null; - canvasAndContext.context = null; - } - } - - interface ImageFileResponse { - name: string; - path: string; - type: string; - exif: Opt<DashUploadUtils.EnrichedExifData>; - } - - router.addSupervisedRoute({ - method: Method.POST, - subscription: RouteStore.upload, - onValidation: ({ req, res }) => { - let form = new formidable.IncomingForm(); - form.uploadDir = uploadDirectory; - form.keepExtensions = true; - form.parse(req, async (_err, _fields, files) => { - let results: ImageFileResponse[] = []; - for (const key in files) { - const { type, path: location, name } = files[key]; - const filename = path.basename(location); - let uploadInformation: Opt<DashUploadUtils.UploadInformation>; - if (filename.endsWith(".pdf")) { - let dataBuffer = fs.readFileSync(uploadDirectory + filename); - const result: ParsedPDF = await pdf(dataBuffer); - await new Promise<void>(resolve => { - const path = pdfDirectory + "/" + filename.substring(0, filename.length - ".pdf".length) + ".txt"; - fs.createWriteStream(path).write(result.text, error => { - if (!error) { - resolve(); - } else { - reject(error); - } - }); - }); - } else { - uploadInformation = await DashUploadUtils.UploadImage(uploadDirectory + filename, filename); - } - const exif = uploadInformation ? uploadInformation.exifData : undefined; - results.push({ name, type, path: `/files/${filename}`, exif }); - - } - _success(res, results); - }); - } - }); - - router.addSupervisedRoute({ - method: Method.POST, - subscription: RouteStore.inspectImage, - onValidation: async ({ req, res }) => { - const { source } = req.body; - if (typeof source === "string") { - const uploadInformation = await DashUploadUtils.UploadImage(source); - return res.send(await DashUploadUtils.InspectImage(uploadInformation.mediaPaths[0])); - } - res.send({}); - } - }); - - router.addSupervisedRoute({ - method: Method.POST, - subscription: RouteStore.dataUriToImage, - onValidation: ({ req, res }) => { - const uri = req.body.uri; - const filename = req.body.name; - if (!uri || !filename) { - res.status(401).send("incorrect parameters specified"); - return; - } - imageDataUri.outputFile(uri, uploadDirectory + filename).then((savedName: string) => { - const ext = path.extname(savedName); - let resizers = [ - { resizer: sharp().resize(100, undefined, { withoutEnlargement: true }), suffix: "_s" }, - { resizer: sharp().resize(400, undefined, { withoutEnlargement: true }), suffix: "_m" }, - { resizer: sharp().resize(900, undefined, { withoutEnlargement: true }), suffix: "_l" }, - ]; - let isImage = false; - if (pngTypes.includes(ext)) { - resizers.forEach(element => { - element.resizer = element.resizer.png(); - }); - isImage = true; - } else if (jpgTypes.includes(ext)) { - resizers.forEach(element => { - element.resizer = element.resizer.jpeg(); - }); - isImage = true; - } - if (isImage) { - resizers.forEach(resizer => { - fs.createReadStream(savedName).pipe(resizer.resizer).pipe(fs.createWriteStream(uploadDirectory + filename + resizer.suffix + ext)); - }); - } - res.send("/files/" + filename + ext); - }); - } - }); - - router.addSupervisedRoute({ - method: Method.GET, - subscription: RouteStore.delete, - onValidation: ({ res, isRelease }) => { - if (isRelease) { - return _permission_denied(res, deletionPermissionError); - } - WebSocket.deleteFields().then(() => res.redirect(RouteStore.home)); - } - }); - - router.addSupervisedRoute({ - method: Method.GET, - subscription: RouteStore.deleteAll, - onValidation: ({ res, isRelease }) => { - if (isRelease) { - return _permission_denied(res, deletionPermissionError); - } - WebSocket.deleteAll().then(() => res.redirect(RouteStore.home)); - } - }); - - const credentialsPath = path.join(__dirname, "./credentials/google_docs_credentials.json"); - - const EndpointHandlerMap = new Map<GoogleApiServerUtils.Action, GoogleApiServerUtils.ApiRouter>([ - ["create", (api, params) => api.create(params)], - ["retrieve", (api, params) => api.get(params)], - ["update", (api, params) => api.batchUpdate(params)], - ]); - - router.addSupervisedRoute({ - method: Method.POST, - subscription: new RouteSubscriber(RouteStore.googleDocs).add("sector", "action"), - onValidation: ({ req, res }) => { - let sector: GoogleApiServerUtils.Service = req.params.sector as GoogleApiServerUtils.Service; - let action: GoogleApiServerUtils.Action = req.params.action as GoogleApiServerUtils.Action; - GoogleApiServerUtils.GetEndpoint(GoogleApiServerUtils.Service[sector], { credentialsPath, userId: req.headers.userId as string }).then(endpoint => { - let handler = EndpointHandlerMap.get(action); - if (endpoint && handler) { - let execute = handler(endpoint, req.body).then( - response => res.send(response.data), - rejection => res.send(rejection) - ); - execute.catch(exception => res.send(exception)); - return; - } - res.send(undefined); - }); - } - }); - - router.addSupervisedRoute({ - method: Method.GET, - subscription: RouteStore.readGoogleAccessToken, - onValidation: async ({ user, res }) => { - const userId = user.id; - const token = await Database.Auxiliary.GoogleAuthenticationToken.Fetch(userId); - const information = { credentialsPath, userId }; - if (!token) { - return res.send(await GoogleApiServerUtils.GenerateAuthenticationUrl(information)); - } - GoogleApiServerUtils.RetrieveAccessToken(information).then(token => res.send(token)); - } - }); - - router.addSupervisedRoute({ - method: Method.POST, - subscription: RouteStore.writeGoogleAccessToken, - onValidation: async ({ user, req, res }) => { - const userId = user.id; - const information = { credentialsPath, userId }; - res.send(await GoogleApiServerUtils.ProcessClientSideCode(information, req.body.authenticationCode)); - } - }); - - const tokenError = "Unable to successfully upload bytes for all images!"; - const mediaError = "Unable to convert all uploaded bytes to media items!"; - const userIdError = "Unable to parse the identification of the user!"; - - router.addSupervisedRoute({ - method: Method.POST, - subscription: RouteStore.googlePhotosMediaUpload, - onValidation: async ({ user, req, res }) => { - const { media } = req.body; - const userId = user.id; - if (!userId) { - return _error(res, userIdError); - } - - await GooglePhotosUploadUtils.initialize({ credentialsPath, userId }); - - let failed: number[] = []; - - const newMediaItems = await BatchedArray.from<GooglePhotosUploadUtils.MediaInput>(media, { batchSize: 25 }).batchedMapPatientInterval( - { magnitude: 100, unit: TimeUnit.Milliseconds }, - async (batch: GooglePhotosUploadUtils.MediaInput[]) => { - const newMediaItems: NewMediaItem[] = []; - for (let index = 0; index < batch.length; index++) { - const element = batch[index]; - const uploadToken = await GooglePhotosUploadUtils.DispatchGooglePhotosUpload(element.url); - if (!uploadToken) { - failed.push(index); - } else { - newMediaItems.push({ - description: element.description, - simpleMediaItem: { uploadToken } - }); - } - } - return newMediaItems; - } - ); - - const failedCount = failed.length; - if (failedCount) { - console.error(`Unable to upload ${failedCount} image${failedCount === 1 ? "" : "s"} to Google's servers`); - } - - GooglePhotosUploadUtils.CreateMediaItems(newMediaItems, req.body.album).then( - result => _success(res, { results: result.newMediaItemResults, failed }), - error => _error(res, mediaError, error) - ); - } - }); - - interface MediaItem { - baseUrl: string; - filename: string; - } - const prefix = "google_photos_"; - - const downloadError = "Encountered an error while executing downloads."; - const requestError = "Unable to execute download: the body's media items were malformed."; - const deletionPermissionError = "Cannot perform specialized delete outside of the development environment!"; - - router.addSupervisedRoute({ - method: Method.GET, - subscription: "/deleteWithAux", - onValidation: async ({ res, isRelease }) => { - if (isRelease) { - return _permission_denied(res, deletionPermissionError); - } - await Database.Auxiliary.DeleteAll(); - res.redirect(RouteStore.delete); - } - }); - - router.addSupervisedRoute({ - method: Method.GET, - subscription: "/deleteWithGoogleCredentials", - onValidation: async ({ res, isRelease }) => { - if (isRelease) { - return _permission_denied(res, deletionPermissionError); - } - await Database.Auxiliary.GoogleAuthenticationToken.DeleteAll(); - res.redirect(RouteStore.delete); - } - }); - - const UploadError = (count: number) => `Unable to upload ${count} images to Dash's server`; - router.addSupervisedRoute({ - method: Method.POST, - subscription: RouteStore.googlePhotosMediaDownload, - onValidation: async ({ req, res }) => { - const contents: { mediaItems: MediaItem[] } = req.body; - let failed = 0; - if (contents) { - const completed: Opt<DashUploadUtils.UploadInformation>[] = []; - for (let item of contents.mediaItems) { - const { contentSize, ...attributes } = await DashUploadUtils.InspectImage(item.baseUrl); - const found: Opt<DashUploadUtils.UploadInformation> = await Database.Auxiliary.QueryUploadHistory(contentSize!); - if (!found) { - const upload = await DashUploadUtils.UploadInspectedImage({ contentSize, ...attributes }, item.filename, prefix).catch(error => _error(res, downloadError, error)); - if (upload) { - completed.push(upload); - await Database.Auxiliary.LogUpload(upload); - } else { - failed++; - } - } else { - completed.push(found); - } - } - if (failed) { - return _error(res, UploadError(failed)); - } - return _success(res, completed); - } - _invalid(res, requestError); - } - }); -}
\ No newline at end of file +(async function start() { + await preliminaryFunctions(); + await initializeServer({ listenAtPort: 1050, routeSetter }); +})(); |