From 3db5ea503754b74681f44ebffeb251dfad8ee65e Mon Sep 17 00:00:00 2001 From: Mohammad Amoush Date: Tue, 29 Oct 2019 17:05:01 -0400 Subject: a few comments --- src/server/Initialization.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/server/Initialization.ts b/src/server/Initialization.ts index fbb5ae7a6..3ea8f2085 100644 --- a/src/server/Initialization.ts +++ b/src/server/Initialization.ts @@ -19,6 +19,8 @@ import * as whm from 'webpack-hot-middleware'; import * as fs from 'fs'; import * as request from 'request'; +/* RouteSetter is a wrapper around the server that prevents the server + from being exposed. */ export type RouteSetter = (server: RouteManager) => void; export interface InitializationOptions { listenAtPort: number; @@ -38,7 +40,7 @@ export default async function InitializeServer(options: InitializationOptions) { registerAuthenticationRoutes(server); registerCorsProxy(server); - const isRelease = determineEnvironment(); + const isRelease = determineEnvironment(); //vs. dev mode routeSetter(new RouteManager(server, isRelease)); server.listen(listenAtPort, () => console.log(`server started at http://localhost:${listenAtPort}`)); @@ -73,6 +75,7 @@ function buildWithMiddleware(server: express.Express) { return server; } +/* Determine if the enviroment is dev mode or release mode. */ function determineEnvironment() { const isRelease = process.env.RELEASE === "true"; -- cgit v1.2.3-70-g09d2 From ee4910b1a90284c71ebdaa5fbd9243148ae113f6 Mon Sep 17 00:00:00 2001 From: Mohammad Amoush Date: Sat, 9 Nov 2019 16:29:20 -0500 Subject: initial --- src/server/ApiManagers/GeneralGoogleManager.ts | 48 ++++++++++ src/server/ApiManagers/GooglePhotosManager.ts | 108 ++++++++++++++++++++++ src/server/index.ts | 123 ------------------------- 3 files changed, 156 insertions(+), 123 deletions(-) create mode 100644 src/server/ApiManagers/GeneralGoogleManager.ts create mode 100644 src/server/ApiManagers/GooglePhotosManager.ts (limited to 'src') diff --git a/src/server/ApiManagers/GeneralGoogleManager.ts b/src/server/ApiManagers/GeneralGoogleManager.ts new file mode 100644 index 000000000..cb37b0dce --- /dev/null +++ b/src/server/ApiManagers/GeneralGoogleManager.ts @@ -0,0 +1,48 @@ +import ApiManager, { Registration } from "./ApiManager"; +import { Method, _permission_denied } from "../RouteManager"; +import { uploadDirectory } from ".."; +import { path } from "animejs"; +import { RouteStore } from "../RouteStore"; +import { GoogleApiServerUtils } from "../apis/google/GoogleApiServerUtils"; +import { Database } from "../database"; + +const deletionPermissionError = "Cannot perform specialized delete outside of the development environment!"; + +export default class GeneralGoogleManager extends ApiManager { + + protected initialize(register: Registration): void { + + register({ + method: Method.GET, + subscription: RouteStore.readGoogleAccessToken, + onValidation: async ({ user, res }) => { + const userId = user.id; + const token = await GoogleApiServerUtils.retrieveAccessToken(userId); + if (!token) { + return res.send(GoogleApiServerUtils.generateAuthenticationUrl()); + } + return res.send(token); + } + }); + + register({ + method: Method.POST, + subscription: RouteStore.writeGoogleAccessToken, + onValidation: async ({ user, req, res }) => { + res.send(await GoogleApiServerUtils.processNewUser(user.id, req.body.authenticationCode)); + } + }); + + register({ + method: Method.GET, + subscription: "/deleteWithGoogleCredentials", + onValidation: async ({ res, isRelease }) => { + if (isRelease) { + return _permission_denied(res, deletionPermissionError); + } + await Database.Auxiliary.GoogleAuthenticationToken.DeleteAll(); + res.redirect(RouteStore.delete); + } + }); + } +} \ No newline at end of file diff --git a/src/server/ApiManagers/GooglePhotosManager.ts b/src/server/ApiManagers/GooglePhotosManager.ts new file mode 100644 index 000000000..b5e9caa38 --- /dev/null +++ b/src/server/ApiManagers/GooglePhotosManager.ts @@ -0,0 +1,108 @@ +import ApiManager, { Registration } from "./ApiManager"; +import { Method, _error, _success, _invalid } from "../RouteManager"; +import { uploadDirectory, NewMediaItem } from ".."; +import { path } from "animejs"; +import { RouteStore } from "../RouteStore"; +import { GoogleApiServerUtils } from "../apis/google/GoogleApiServerUtils"; +import { BatchedArray, TimeUnit } from "array-batcher"; +import { GooglePhotosUploadUtils } from "../apis/google/GooglePhotosUploadUtils"; +import { MediaItem } from "../apis/google/SharedTypes"; +import { Opt } from "../../new_fields/Doc"; +import { DashUploadUtils } from "../DashUploadUtils"; +import { Database } from "../database"; +import { prefix } from "@fortawesome/free-solid-svg-icons"; + +const authenticationError = "Unable to authenticate Google credentials before uploading to Google Photos!"; +const mediaError = "Unable to convert all uploaded bytes to media items!"; +const UploadError = (count: number) => `Unable to upload ${count} images to Dash's server`; +const requestError = "Unable to execute download: the body's media items were malformed."; +const downloadError = "Encountered an error while executing downloads."; +interface GooglePhotosUploadFailure { + batch: number; + index: number; + url: string; + reason: string; +} + +export default class GooglePhotosManager extends ApiManager { + + protected initialize(register: Registration): void { + + register({ + method: Method.POST, + subscription: RouteStore.googlePhotosMediaUpload, + onValidation: async ({ user, req, res }) => { + const { media } = req.body; + + const token = await GoogleApiServerUtils.retrieveAccessToken(user.id); + if (!token) { + return _error(res, authenticationError); + } + + let failed: GooglePhotosUploadFailure[] = []; + const batched = BatchedArray.from(media, { batchSize: 25 }); + const newMediaItems = await batched.batchedMapPatientInterval( + { magnitude: 100, unit: TimeUnit.Milliseconds }, + async (batch, collector, { completedBatches }) => { + for (let index = 0; index < batch.length; index++) { + const { url, description } = batch[index]; + const fail = (reason: string) => failed.push({ reason, batch: completedBatches + 1, index, url }); + const uploadToken = await GooglePhotosUploadUtils.DispatchGooglePhotosUpload(token, url).catch(fail); + if (!uploadToken) { + fail(`${path.extname(url)} is not an accepted extension`); + } else { + collector.push({ + description, + simpleMediaItem: { uploadToken } + }); + } + } + } + ); + + const failedCount = failed.length; + if (failedCount) { + console.error(`Unable to upload ${failedCount} image${failedCount === 1 ? "" : "s"} to Google's servers`); + console.log(failed.map(({ reason, batch, index, url }) => `@${batch}.${index}: ${url} failed:\n${reason}`).join('\n\n')); + } + + return GooglePhotosUploadUtils.CreateMediaItems(token, newMediaItems, req.body.album).then( + results => _success(res, { results, failed }), + error => _error(res, mediaError, error) + ); + } + }); + + register({ + method: Method.POST, + subscription: RouteStore.googlePhotosMediaDownload, + onValidation: async ({ req, res }) => { + const contents: { mediaItems: MediaItem[] } = req.body; + let failed = 0; + if (contents) { + const completed: Opt[] = []; + for (let item of contents.mediaItems) { + const { contentSize, ...attributes } = await DashUploadUtils.InspectImage(item.baseUrl); + const found: Opt = await Database.Auxiliary.QueryUploadHistory(contentSize!); + if (!found) { + const upload = await DashUploadUtils.UploadInspectedImage({ contentSize, ...attributes }, item.filename, prefix).catch(error => _error(res, downloadError, error)); + if (upload) { + completed.push(upload); + await Database.Auxiliary.LogUpload(upload); + } else { + failed++; + } + } else { + completed.push(found); + } + } + if (failed) { + return _error(res, UploadError(failed)); + } + return _success(res, completed); + } + _invalid(res, requestError); + } + }); + } +} \ No newline at end of file diff --git a/src/server/index.ts b/src/server/index.ts index 25697e71f..9a5099d0d 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -585,89 +585,11 @@ function routeSetter(router: RouteManager) { } }); - router.addSupervisedRoute({ - method: Method.GET, - subscription: RouteStore.readGoogleAccessToken, - onValidation: async ({ user, res }) => { - const userId = user.id; - const token = await GoogleApiServerUtils.retrieveAccessToken(userId); - if (!token) { - return res.send(GoogleApiServerUtils.generateAuthenticationUrl()); - } - return res.send(token); - } - }); - - router.addSupervisedRoute({ - method: Method.POST, - subscription: RouteStore.writeGoogleAccessToken, - onValidation: async ({ user, req, res }) => { - res.send(await GoogleApiServerUtils.processNewUser(user.id, req.body.authenticationCode)); - } - }); - - const authenticationError = "Unable to authenticate Google credentials before uploading to Google Photos!"; - const mediaError = "Unable to convert all uploaded bytes to media items!"; - interface GooglePhotosUploadFailure { - batch: number; - index: number; - url: string; - reason: string; - } - - router.addSupervisedRoute({ - method: Method.POST, - subscription: RouteStore.googlePhotosMediaUpload, - onValidation: async ({ user, req, res }) => { - const { media } = req.body; - - const token = await GoogleApiServerUtils.retrieveAccessToken(user.id); - if (!token) { - return _error(res, authenticationError); - } - - let failed: GooglePhotosUploadFailure[] = []; - const batched = BatchedArray.from(media, { batchSize: 25 }); - const newMediaItems = await batched.batchedMapPatientInterval( - { magnitude: 100, unit: TimeUnit.Milliseconds }, - async (batch, collector, { completedBatches }) => { - for (let index = 0; index < batch.length; index++) { - const { url, description } = batch[index]; - const fail = (reason: string) => failed.push({ reason, batch: completedBatches + 1, index, url }); - const uploadToken = await GooglePhotosUploadUtils.DispatchGooglePhotosUpload(token, url).catch(fail); - if (!uploadToken) { - fail(`${path.extname(url)} is not an accepted extension`); - } else { - collector.push({ - description, - simpleMediaItem: { uploadToken } - }); - } - } - } - ); - - const failedCount = failed.length; - if (failedCount) { - console.error(`Unable to upload ${failedCount} image${failedCount === 1 ? "" : "s"} to Google's servers`); - console.log(failed.map(({ reason, batch, index, url }) => `@${batch}.${index}: ${url} failed:\n${reason}`).join('\n\n')); - } - - return GooglePhotosUploadUtils.CreateMediaItems(token, newMediaItems, req.body.album).then( - results => _success(res, { results, failed }), - error => _error(res, mediaError, error) - ); - } - }); - interface MediaItem { baseUrl: string; filename: string; } const prefix = "google_photos_"; - - const downloadError = "Encountered an error while executing downloads."; - const requestError = "Unable to execute download: the body's media items were malformed."; const deletionPermissionError = "Cannot perform specialized delete outside of the development environment!"; router.addSupervisedRoute({ @@ -681,49 +603,4 @@ function routeSetter(router: RouteManager) { res.redirect(RouteStore.delete); } }); - - router.addSupervisedRoute({ - method: Method.GET, - subscription: "/deleteWithGoogleCredentials", - onValidation: async ({ res, isRelease }) => { - if (isRelease) { - return _permission_denied(res, deletionPermissionError); - } - await Database.Auxiliary.GoogleAuthenticationToken.DeleteAll(); - res.redirect(RouteStore.delete); - } - }); - - const UploadError = (count: number) => `Unable to upload ${count} images to Dash's server`; - router.addSupervisedRoute({ - method: Method.POST, - subscription: RouteStore.googlePhotosMediaDownload, - onValidation: async ({ req, res }) => { - const contents: { mediaItems: MediaItem[] } = req.body; - let failed = 0; - if (contents) { - const completed: Opt[] = []; - for (let item of contents.mediaItems) { - const { contentSize, ...attributes } = await DashUploadUtils.InspectImage(item.baseUrl); - const found: Opt = await Database.Auxiliary.QueryUploadHistory(contentSize!); - if (!found) { - const upload = await DashUploadUtils.UploadInspectedImage({ contentSize, ...attributes }, item.filename, prefix).catch(error => _error(res, downloadError, error)); - if (upload) { - completed.push(upload); - await Database.Auxiliary.LogUpload(upload); - } else { - failed++; - } - } else { - completed.push(found); - } - } - if (failed) { - return _error(res, UploadError(failed)); - } - return _success(res, completed); - } - _invalid(res, requestError); - } - }); } \ No newline at end of file -- cgit v1.2.3-70-g09d2 From 06a9b3477dfef93af3c2715f5512d0d883191b58 Mon Sep 17 00:00:00 2001 From: Mohammad Amoush Date: Tue, 12 Nov 2019 17:26:58 -0500 Subject: fixed everything except for async --- src/server/ApiManagers/GeneralGoogleManager.ts | 47 ++++++++++++++++++++--- src/server/ApiManagers/GooglePhotosManager.ts | 27 +++++++------ src/server/apis/google/GooglePhotosUploadUtils.ts | 2 +- src/server/index.ts | 9 +++-- 4 files changed, 62 insertions(+), 23 deletions(-) (limited to 'src') diff --git a/src/server/ApiManagers/GeneralGoogleManager.ts b/src/server/ApiManagers/GeneralGoogleManager.ts index cb37b0dce..89efebf78 100644 --- a/src/server/ApiManagers/GeneralGoogleManager.ts +++ b/src/server/ApiManagers/GeneralGoogleManager.ts @@ -1,12 +1,20 @@ import ApiManager, { Registration } from "./ApiManager"; import { Method, _permission_denied } from "../RouteManager"; -import { uploadDirectory } from ".."; -import { path } from "animejs"; -import { RouteStore } from "../RouteStore"; import { GoogleApiServerUtils } from "../apis/google/GoogleApiServerUtils"; import { Database } from "../database"; +import RouteSubscriber from "../RouteSubscriber"; const deletionPermissionError = "Cannot perform specialized delete outside of the development environment!"; +const ServicesApiKeyMap = new Map([ + ["face", process.env.FACE], + ["vision", process.env.VISION], + ["handwriting", process.env.HANDWRITING] +]); +const EndpointHandlerMap = new Map([ + ["create", (api, params) => api.create(params)], + ["retrieve", (api, params) => api.get(params)], + ["update", (api, params) => api.batchUpdate(params)], +]); export default class GeneralGoogleManager extends ApiManager { @@ -14,7 +22,7 @@ export default class GeneralGoogleManager extends ApiManager { register({ method: Method.GET, - subscription: RouteStore.readGoogleAccessToken, + subscription: "/readGoogleAccessToken", onValidation: async ({ user, res }) => { const userId = user.id; const token = await GoogleApiServerUtils.retrieveAccessToken(userId); @@ -27,7 +35,7 @@ export default class GeneralGoogleManager extends ApiManager { register({ method: Method.POST, - subscription: RouteStore.writeGoogleAccessToken, + subscription: "/writeGoogleAccessToken", onValidation: async ({ user, req, res }) => { res.send(await GoogleApiServerUtils.processNewUser(user.id, req.body.authenticationCode)); } @@ -41,7 +49,34 @@ export default class GeneralGoogleManager extends ApiManager { return _permission_denied(res, deletionPermissionError); } await Database.Auxiliary.GoogleAuthenticationToken.DeleteAll(); - res.redirect(RouteStore.delete); + res.redirect("/delete"); + } + }); + + register({ + method: Method.GET, + subscription: new RouteSubscriber("/cognitiveServices").add('requestedservice'), + onValidation: ({ req, res }) => { + let service = req.params.requestedservice; + res.send(ServicesApiKeyMap.get(service)); + } + }); + + register({ + method: Method.POST, + subscription: new RouteSubscriber("/googleDocs").add("sector", "action"), + onValidation: async ({ req, res, user }) => { + let sector: GoogleApiServerUtils.Service = req.params.sector as GoogleApiServerUtils.Service; + let action: GoogleApiServerUtils.Action = req.params.action as GoogleApiServerUtils.Action; + const endpoint = await GoogleApiServerUtils.GetEndpoint(GoogleApiServerUtils.Service[sector], user.id); + let handler = EndpointHandlerMap.get(action); + if (endpoint && handler) { + handler(endpoint, req.body) + .then(response => res.send(response.data)) + .catch(exception => res.send(exception)); + return; + } + res.send(undefined); } }); } diff --git a/src/server/ApiManagers/GooglePhotosManager.ts b/src/server/ApiManagers/GooglePhotosManager.ts index b5e9caa38..1f6051c28 100644 --- a/src/server/ApiManagers/GooglePhotosManager.ts +++ b/src/server/ApiManagers/GooglePhotosManager.ts @@ -1,16 +1,12 @@ import ApiManager, { Registration } from "./ApiManager"; import { Method, _error, _success, _invalid } from "../RouteManager"; -import { uploadDirectory, NewMediaItem } from ".."; -import { path } from "animejs"; -import { RouteStore } from "../RouteStore"; +import * as path from "path"; import { GoogleApiServerUtils } from "../apis/google/GoogleApiServerUtils"; import { BatchedArray, TimeUnit } from "array-batcher"; import { GooglePhotosUploadUtils } from "../apis/google/GooglePhotosUploadUtils"; -import { MediaItem } from "../apis/google/SharedTypes"; import { Opt } from "../../new_fields/Doc"; import { DashUploadUtils } from "../DashUploadUtils"; import { Database } from "../database"; -import { prefix } from "@fortawesome/free-solid-svg-icons"; const authenticationError = "Unable to authenticate Google credentials before uploading to Google Photos!"; const mediaError = "Unable to convert all uploaded bytes to media items!"; @@ -23,6 +19,17 @@ interface GooglePhotosUploadFailure { url: string; reason: string; } +interface MediaItem { + baseUrl: string; + filename: string; +} +interface NewMediaItem { + description: string; + simpleMediaItem: { + uploadToken: string; + }; +} +const prefix = "google_photos_"; export default class GooglePhotosManager extends ApiManager { @@ -30,20 +37,18 @@ export default class GooglePhotosManager extends ApiManager { register({ method: Method.POST, - subscription: RouteStore.googlePhotosMediaUpload, + subscription: "/googlePhotosMediaUpload", onValidation: async ({ user, req, res }) => { const { media } = req.body; - const token = await GoogleApiServerUtils.retrieveAccessToken(user.id); if (!token) { return _error(res, authenticationError); } - let failed: GooglePhotosUploadFailure[] = []; const batched = BatchedArray.from(media, { batchSize: 25 }); const newMediaItems = await batched.batchedMapPatientInterval( { magnitude: 100, unit: TimeUnit.Milliseconds }, - async (batch, collector, { completedBatches }) => { + async (batch: any, collector: any, { completedBatches }: any) => { for (let index = 0; index < batch.length; index++) { const { url, description } = batch[index]; const fail = (reason: string) => failed.push({ reason, batch: completedBatches + 1, index, url }); @@ -59,13 +64,11 @@ export default class GooglePhotosManager extends ApiManager { } } ); - const failedCount = failed.length; if (failedCount) { console.error(`Unable to upload ${failedCount} image${failedCount === 1 ? "" : "s"} to Google's servers`); console.log(failed.map(({ reason, batch, index, url }) => `@${batch}.${index}: ${url} failed:\n${reason}`).join('\n\n')); } - return GooglePhotosUploadUtils.CreateMediaItems(token, newMediaItems, req.body.album).then( results => _success(res, { results, failed }), error => _error(res, mediaError, error) @@ -75,7 +78,7 @@ export default class GooglePhotosManager extends ApiManager { register({ method: Method.POST, - subscription: RouteStore.googlePhotosMediaDownload, + subscription: "/googlePhotosMediaDownload", onValidation: async ({ req, res }) => { const contents: { mediaItems: MediaItem[] } = req.body; let failed = 0; diff --git a/src/server/apis/google/GooglePhotosUploadUtils.ts b/src/server/apis/google/GooglePhotosUploadUtils.ts index 0abed3f1d..27532d7f0 100644 --- a/src/server/apis/google/GooglePhotosUploadUtils.ts +++ b/src/server/apis/google/GooglePhotosUploadUtils.ts @@ -122,7 +122,7 @@ export namespace GooglePhotosUploadUtils { // ...so we execute them in delayed batches and await the entire execution return batched.batchedMapPatientInterval( { magnitude: 100, unit: TimeUnit.Milliseconds }, - async (batch: NewMediaItem[], collector) => { + async (batch: NewMediaItem[], collector: any) => { const parameters = { method: 'POST', headers: headers('json', bearerToken), diff --git a/src/server/index.ts b/src/server/index.ts index 59752d6de..773b84403 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -4,10 +4,7 @@ import * as mobileDetect from 'mobile-detect'; import * as path from 'path'; import { Database } from './database'; const serverPort = 4321; -import { GooglePhotosUploadUtils } from './apis/google/GooglePhotosUploadUtils'; -import { Opt } from '../new_fields/Doc'; import { DashUploadUtils } from './DashUploadUtils'; -import { BatchedArray, TimeUnit } from 'array-batcher'; import RouteSubscriber from './RouteSubscriber'; import initializeServer from './Initialization'; import RouteManager, { Method, _success, _permission_denied, _error, _invalid, OnUnauthenticated } from './RouteManager'; @@ -21,6 +18,8 @@ import { GoogleCredentialsLoader } from './credentials/CredentialsLoader'; import DeleteManager from "./ApiManagers/DeleteManager"; import PDFManager from "./ApiManagers/PDFManager"; import UploadManager from "./ApiManagers/UploadManager"; +import GeneralGoogleManager from "./ApiManagers/GeneralGoogleManager"; +import GooglePhotosManager from "./ApiManagers/GooglePhotosManager"; export const publicDirectory = __dirname + "/public"; export const filesDirectory = publicDirectory + "/files/"; @@ -64,7 +63,9 @@ function routeSetter(router: RouteManager) { new SearchManager(), new PDFManager(), new DeleteManager(), - new UtilManager() + new UtilManager(), + new GeneralGoogleManager(), + new GooglePhotosManager(), ].forEach(manager => manager.register(router)); // initialize the web socket (bidirectional communication: if a user changes -- cgit v1.2.3-70-g09d2 From ab285371f6fb2a4f1e64888bafbc84b602f23416 Mon Sep 17 00:00:00 2001 From: Mohammad Amoush Date: Tue, 12 Nov 2019 18:04:41 -0500 Subject: only one more problem... --- src/server/apis/google/GooglePhotosUploadUtils.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/server/apis/google/GooglePhotosUploadUtils.ts b/src/server/apis/google/GooglePhotosUploadUtils.ts index 27532d7f0..3862fedc8 100644 --- a/src/server/apis/google/GooglePhotosUploadUtils.ts +++ b/src/server/apis/google/GooglePhotosUploadUtils.ts @@ -122,7 +122,7 @@ export namespace GooglePhotosUploadUtils { // ...so we execute them in delayed batches and await the entire execution return batched.batchedMapPatientInterval( { magnitude: 100, unit: TimeUnit.Milliseconds }, - async (batch: NewMediaItem[], collector: any) => { + async (batch: NewMediaItem[], collector: any): Promise => { const parameters = { method: 'POST', headers: headers('json', bearerToken), -- cgit v1.2.3-70-g09d2