aboutsummaryrefslogtreecommitdiff
path: root/src/server/ApiManagers
diff options
context:
space:
mode:
Diffstat (limited to 'src/server/ApiManagers')
-rw-r--r--src/server/ApiManagers/ApiManager.ts11
-rw-r--r--src/server/ApiManagers/DeleteManager.ts63
-rw-r--r--src/server/ApiManagers/DiagnosticManager.ts30
-rw-r--r--src/server/ApiManagers/DownloadManager.ts267
-rw-r--r--src/server/ApiManagers/GeneralGoogleManager.ts58
-rw-r--r--src/server/ApiManagers/GooglePhotosManager.ts115
-rw-r--r--src/server/ApiManagers/PDFManager.ts111
-rw-r--r--src/server/ApiManagers/SearchManager.ts49
-rw-r--r--src/server/ApiManagers/UploadManager.ts222
-rw-r--r--src/server/ApiManagers/UserManager.ts71
-rw-r--r--src/server/ApiManagers/UtilManager.ts67
11 files changed, 1064 insertions, 0 deletions
diff --git a/src/server/ApiManagers/ApiManager.ts b/src/server/ApiManagers/ApiManager.ts
new file mode 100644
index 000000000..e2b01d585
--- /dev/null
+++ b/src/server/ApiManagers/ApiManager.ts
@@ -0,0 +1,11 @@
+import RouteManager, { RouteInitializer } from "../RouteManager";
+
+export type Registration = (initializer: RouteInitializer) => void;
+
+export default abstract class ApiManager {
+ protected abstract initialize(register: Registration): void;
+
+ public register(register: Registration) {
+ this.initialize(register);
+ }
+} \ No newline at end of file
diff --git a/src/server/ApiManagers/DeleteManager.ts b/src/server/ApiManagers/DeleteManager.ts
new file mode 100644
index 000000000..71818c673
--- /dev/null
+++ b/src/server/ApiManagers/DeleteManager.ts
@@ -0,0 +1,63 @@
+import ApiManager, { Registration } from "./ApiManager";
+import { Method, _permission_denied } from "../RouteManager";
+import { WebSocket } from "../Websocket/Websocket";
+import { Database } from "../database";
+
+export default class DeleteManager extends ApiManager {
+
+ protected initialize(register: Registration): void {
+
+ register({
+ method: Method.GET,
+ subscription: "/delete",
+ onValidation: async ({ res, isRelease }) => {
+ if (isRelease) {
+ return _permission_denied(res, deletionPermissionError);
+ }
+ await WebSocket.deleteFields();
+ res.redirect("/home");
+ }
+ });
+
+ register({
+ method: Method.GET,
+ subscription: "/deleteAll",
+ onValidation: async ({ res, isRelease }) => {
+ if (isRelease) {
+ return _permission_denied(res, deletionPermissionError);
+ }
+ await WebSocket.deleteAll();
+ res.redirect("/home");
+ }
+ });
+
+
+ register({
+ method: Method.GET,
+ subscription: "/deleteWithAux",
+ onValidation: async ({ res, isRelease }) => {
+ if (isRelease) {
+ return _permission_denied(res, deletionPermissionError);
+ }
+ await Database.Auxiliary.DeleteAll();
+ res.redirect("/delete");
+ }
+ });
+
+ register({
+ method: Method.GET,
+ subscription: "/deleteWithGoogleCredentials",
+ onValidation: async ({ res, isRelease }) => {
+ if (isRelease) {
+ return _permission_denied(res, deletionPermissionError);
+ }
+ await Database.Auxiliary.GoogleAuthenticationToken.DeleteAll();
+ res.redirect("/delete");
+ }
+ });
+
+ }
+
+}
+
+const deletionPermissionError = "Cannot perform a delete operation outside of the development environment!";
diff --git a/src/server/ApiManagers/DiagnosticManager.ts b/src/server/ApiManagers/DiagnosticManager.ts
new file mode 100644
index 000000000..104985481
--- /dev/null
+++ b/src/server/ApiManagers/DiagnosticManager.ts
@@ -0,0 +1,30 @@
+import ApiManager, { Registration } from "./ApiManager";
+import { Method } from "../RouteManager";
+import request = require('request-promise');
+
+export default class DiagnosticManager extends ApiManager {
+
+ protected initialize(register: Registration): void {
+
+ register({
+ method: Method.GET,
+ subscription: "/serverHeartbeat",
+ onValidation: ({ res }) => res.send(true)
+ });
+
+ register({
+ method: Method.GET,
+ subscription: "/solrHeartbeat",
+ onValidation: async ({ res }) => {
+ try {
+ await request("http://localhost:8983");
+ res.send({ running: true });
+ } catch (e) {
+ res.send({ running: false });
+ }
+ }
+ });
+
+ }
+
+} \ No newline at end of file
diff --git a/src/server/ApiManagers/DownloadManager.ts b/src/server/ApiManagers/DownloadManager.ts
new file mode 100644
index 000000000..5bad46eda
--- /dev/null
+++ b/src/server/ApiManagers/DownloadManager.ts
@@ -0,0 +1,267 @@
+import ApiManager, { Registration } from "./ApiManager";
+import { Method } from "../RouteManager";
+import RouteSubscriber from "../RouteSubscriber";
+import * as Archiver from 'archiver';
+import * as express from 'express';
+import { Database } from "../database";
+import * as path from "path";
+import { DashUploadUtils, SizeSuffix } from "../DashUploadUtils";
+import { publicDirectory } from "..";
+
+export type Hierarchy = { [id: string]: string | Hierarchy };
+export type ZipMutator = (file: Archiver.Archiver) => void | Promise<void>;
+export interface DocumentElements {
+ data: string | any[];
+ title: string;
+}
+
+export default class DownloadManager extends ApiManager {
+
+ protected initialize(register: Registration): void {
+
+ /**
+ * Let's say someone's using Dash to organize images in collections.
+ * This lets them export the hierarchy they've built to their
+ * own file system in a useful format.
+ *
+ * This handler starts with a single document id (interesting only
+ * if it's that of a collection). It traverses the database, captures
+ * the nesting of only nested images or collections, writes
+ * that to a zip file and returns it to the client for download.
+ */
+ register({
+ method: Method.GET,
+ subscription: new RouteSubscriber("imageHierarchyExport").add('docId'),
+ onValidation: async ({ req, res }) => {
+ const id = req.params.docId;
+ const hierarchy: Hierarchy = {};
+ await buildHierarchyRecursive(id, hierarchy);
+ return BuildAndDispatchZip(res, zip => writeHierarchyRecursive(zip, hierarchy));
+ }
+ });
+
+ register({
+ method: Method.GET,
+ subscription: new RouteSubscriber("downloadId").add("docId"),
+ onValidation: async ({ req, res }) => {
+ return BuildAndDispatchZip(res, async zip => {
+ const { id, docs, files } = await getDocs(req.params.docId);
+ const docString = JSON.stringify({ id, docs });
+ zip.append(docString, { name: "doc.json" });
+ files.forEach(val => {
+ zip.file(publicDirectory + val, { name: val.substring(1) });
+ });
+ });
+ }
+ });
+
+ register({
+ method: Method.GET,
+ subscription: new RouteSubscriber("/serializeDoc").add("docId"),
+ onValidation: async ({ req, res }) => {
+ const { docs, files } = await getDocs(req.params.docId);
+ res.send({ docs, files: Array.from(files) });
+ }
+ });
+
+
+ }
+
+}
+
+async function getDocs(id: string) {
+ const files = new Set<string>();
+ const docs: { [id: string]: any } = {};
+ const fn = (doc: any): string[] => {
+ const id = doc.id;
+ if (typeof id === "string" && id.endsWith("Proto")) {
+ //Skip protos
+ return [];
+ }
+ const ids: string[] = [];
+ for (const key in doc.fields) {
+ if (!doc.fields.hasOwnProperty(key)) {
+ continue;
+ }
+ const field = doc.fields[key];
+ if (field === undefined || field === null) {
+ continue;
+ }
+
+ if (field.__type === "proxy" || field.__type === "prefetch_proxy") {
+ ids.push(field.fieldId);
+ } else if (field.__type === "script" || field.__type === "computed") {
+ if (field.captures) {
+ ids.push(field.captures.fieldId);
+ }
+ } else if (field.__type === "list") {
+ ids.push(...fn(field));
+ } else if (typeof field === "string") {
+ const re = /"(?:dataD|d)ocumentId"\s*:\s*"([\w\-]*)"/g;
+ let match: string[] | null;
+ while ((match = re.exec(field)) !== null) {
+ ids.push(match[1]);
+ }
+ } else if (field.__type === "RichTextField") {
+ const re = /"href"\s*:\s*"(.*?)"/g;
+ let match: string[] | null;
+ while ((match = re.exec(field.Data)) !== null) {
+ const urlString = match[1];
+ const split = new URL(urlString).pathname.split("doc/");
+ if (split.length > 1) {
+ ids.push(split[split.length - 1]);
+ }
+ }
+ const re2 = /"src"\s*:\s*"(.*?)"/g;
+ while ((match = re2.exec(field.Data)) !== null) {
+ const urlString = match[1];
+ const pathname = new URL(urlString).pathname;
+ files.add(pathname);
+ }
+ } else if (["audio", "image", "video", "pdf", "web"].includes(field.__type)) {
+ const url = new URL(field.url);
+ const pathname = url.pathname;
+ files.add(pathname);
+ }
+ }
+
+ if (doc.id) {
+ docs[doc.id] = doc;
+ }
+ return ids;
+ };
+ await Database.Instance.visit([id], fn);
+ return { id, docs, files };
+}
+
+/**
+ * This utility function factors out the process
+ * of creating a zip file and sending it back to the client
+ * by piping it into a response.
+ *
+ * Learn more about piping and readable / writable streams here!
+ * https://www.freecodecamp.org/news/node-js-streams-everything-you-need-to-know-c9141306be93/
+ *
+ * @param res the writable stream response object that will transfer the generated zip file
+ * @param mutator the callback function used to actually modify and insert information into the zip instance
+ */
+export async function BuildAndDispatchZip(res: express.Response, mutator: ZipMutator): Promise<void> {
+ res.set('Content-disposition', `attachment;`);
+ res.set('Content-Type', "application/zip");
+ const zip = Archiver('zip');
+ zip.pipe(res);
+ await mutator(zip);
+ return zip.finalize();
+}
+
+/**
+ * This function starts with a single document id as a seed,
+ * typically that of a collection, and then descends the entire tree
+ * of image or collection documents that are reachable from that seed.
+ * @param seedId the id of the root of the subtree we're trying to capture, interesting only if it's a collection
+ * @param hierarchy the data structure we're going to use to record the nesting of the collections and images as we descend
+ */
+
+/*
+Below is an example of the JSON hierarchy built from two images contained inside a collection titled 'a nested collection',
+following the general recursive structure shown immediately below
+{
+ "parent folder name":{
+ "first child's fild name":"first child's url"
+ ...
+ "nth child's fild name":"nth child's url"
+ }
+}
+{
+ "a nested collection (865c4734-c036-4d67-a588-c71bb43d1440)":{
+ "an image of a cat (ace99ffd-8ed8-4026-a5d5-a353fff57bdd).jpg":"https://upload.wikimedia.org/wikipedia/commons/thumb/3/3a/Cat03.jpg/1200px-Cat03.jpg",
+ "1*SGJw31T5Q9Zfsk24l2yirg.gif (9321cc9b-9b3e-4cb6-b99c-b7e667340f05).gif":"https://cdn-media-1.freecodecamp.org/images/1*SGJw31T5Q9Zfsk24l2yirg.gif"
+ }
+}
+*/
+async function buildHierarchyRecursive(seedId: string, hierarchy: Hierarchy): Promise<void> {
+ const { title, data } = await getData(seedId);
+ const label = `${title} (${seedId})`;
+ // is the document a collection?
+ if (Array.isArray(data)) {
+ // recurse over all documents in the collection.
+ const local: Hierarchy = {}; // create a child hierarchy for this level, which will get passed in as the parent of the recursive call
+ hierarchy[label] = local; // store it at the index in the parent, so we'll end up with a map of maps of maps
+ await Promise.all(data.map(proxy => buildHierarchyRecursive(proxy.fieldId, local)));
+ } else {
+ // now, data can only be a string, namely the url of the image
+ const filename = label + path.extname(data); // this is the file name under which the output image will be stored
+ hierarchy[filename] = data;
+ }
+}
+
+/**
+ * This is a very specific utility method to help traverse the database
+ * to parse data and titles out of images and collections alone.
+ *
+ * We don't know if the document id given to is corresponds to a view document or a data
+ * document. If it's a data document, the response from the database will have
+ * a data field. If not, call recursively on the proto, and resolve with *its* data
+ *
+ * @param targetId the id of the Dash document whose data is being requests
+ * @returns the data of the document, as well as its title
+ */
+async function getData(targetId: string): Promise<DocumentElements> {
+ return new Promise<DocumentElements>((resolve, reject) => {
+ Database.Instance.getDocument(targetId, async (result: any) => {
+ const { data, proto, title } = result.fields;
+ if (data) {
+ if (data.url) {
+ resolve({ data: data.url, title });
+ } else if (data.fields) {
+ resolve({ data: data.fields, title });
+ } else {
+ reject();
+ }
+ } else if (proto) {
+ getData(proto.fieldId).then(resolve, reject);
+ } else {
+ reject();
+ }
+ });
+ });
+}
+
+/**
+ *
+ * @param file the zip file to which we write the files
+ * @param hierarchy the data structure from which we read, defining the nesting of the documents in the zip
+ * @param prefix lets us create nested folders in the zip file by continually appending to the end
+ * of the prefix with each layer of recursion.
+ *
+ * Function Call #1 => "Dash Export"
+ * Function Call #2 => "Dash Export/a nested collection"
+ * Function Call #3 => "Dash Export/a nested collection/lowest level collection"
+ * ...
+ */
+async function writeHierarchyRecursive(file: Archiver.Archiver, hierarchy: Hierarchy, prefix = "Dash Export"): Promise<void> {
+ for (const documentTitle of Object.keys(hierarchy)) {
+ const result = hierarchy[documentTitle];
+ // base case or leaf node, we've hit a url (image)
+ if (typeof result === "string") {
+ let path: string;
+ let matches: RegExpExecArray | null;
+ if ((matches = /\:1050\/files\/(upload\_[\da-z]{32}.*)/g.exec(result)) !== null) {
+ // image already exists on our server
+ path = `${__dirname}/public/files/${matches[1]}`;
+ } else {
+ // the image doesn't already exist on our server (may have been dragged
+ // and dropped in the browser and thus hosted remotely) so we upload it
+ // to our server and point the zip file to it, so it can bundle up the bytes
+ const information = await DashUploadUtils.UploadImage(result);
+ path = information.serverAccessPaths[SizeSuffix.Original];
+ }
+ // write the file specified by the path to the directory in the
+ // zip file given by the prefix.
+ file.file(path, { name: documentTitle, prefix });
+ } else {
+ // we've hit a collection, so we have to recurse
+ await writeHierarchyRecursive(file, result, `${prefix}/${documentTitle}`);
+ }
+ }
+} \ No newline at end of file
diff --git a/src/server/ApiManagers/GeneralGoogleManager.ts b/src/server/ApiManagers/GeneralGoogleManager.ts
new file mode 100644
index 000000000..629684e0c
--- /dev/null
+++ b/src/server/ApiManagers/GeneralGoogleManager.ts
@@ -0,0 +1,58 @@
+import ApiManager, { Registration } from "./ApiManager";
+import { Method, _permission_denied } from "../RouteManager";
+import { GoogleApiServerUtils } from "../apis/google/GoogleApiServerUtils";
+import { Database } from "../database";
+import RouteSubscriber from "../RouteSubscriber";
+
+const deletionPermissionError = "Cannot perform specialized delete outside of the development environment!";
+
+const EndpointHandlerMap = new Map<GoogleApiServerUtils.Action, GoogleApiServerUtils.ApiRouter>([
+ ["create", (api, params) => api.create(params)],
+ ["retrieve", (api, params) => api.get(params)],
+ ["update", (api, params) => api.batchUpdate(params)],
+]);
+
+export default class GeneralGoogleManager extends ApiManager {
+
+ protected initialize(register: Registration): void {
+
+ register({
+ method: Method.GET,
+ subscription: "/readGoogleAccessToken",
+ onValidation: async ({ user, res }) => {
+ const token = await GoogleApiServerUtils.retrieveAccessToken(user.id);
+ if (!token) {
+ return res.send(GoogleApiServerUtils.generateAuthenticationUrl());
+ }
+ return res.send(token);
+ }
+ });
+
+ register({
+ method: Method.POST,
+ subscription: "/writeGoogleAccessToken",
+ onValidation: async ({ user, req, res }) => {
+ res.send(await GoogleApiServerUtils.processNewUser(user.id, req.body.authenticationCode));
+ }
+ });
+
+ register({
+ method: Method.POST,
+ subscription: new RouteSubscriber("/googleDocs").add("sector", "action"),
+ onValidation: async ({ req, res, user }) => {
+ let sector: GoogleApiServerUtils.Service = req.params.sector as GoogleApiServerUtils.Service;
+ let action: GoogleApiServerUtils.Action = req.params.action as GoogleApiServerUtils.Action;
+ const endpoint = await GoogleApiServerUtils.GetEndpoint(GoogleApiServerUtils.Service[sector], user.id);
+ let handler = EndpointHandlerMap.get(action);
+ if (endpoint && handler) {
+ handler(endpoint, req.body)
+ .then(response => res.send(response.data))
+ .catch(exception => res.send(exception));
+ return;
+ }
+ res.send(undefined);
+ }
+ });
+
+ }
+} \ No newline at end of file
diff --git a/src/server/ApiManagers/GooglePhotosManager.ts b/src/server/ApiManagers/GooglePhotosManager.ts
new file mode 100644
index 000000000..4a0c0b936
--- /dev/null
+++ b/src/server/ApiManagers/GooglePhotosManager.ts
@@ -0,0 +1,115 @@
+import ApiManager, { Registration } from "./ApiManager";
+import { Method, _error, _success, _invalid } from "../RouteManager";
+import * as path from "path";
+import { GoogleApiServerUtils } from "../apis/google/GoogleApiServerUtils";
+import { BatchedArray, TimeUnit } from "array-batcher";
+import { GooglePhotosUploadUtils } from "../apis/google/GooglePhotosUploadUtils";
+import { Opt } from "../../new_fields/Doc";
+import { DashUploadUtils, InjectSize, SizeSuffix } from "../DashUploadUtils";
+import { Database } from "../database";
+
+const authenticationError = "Unable to authenticate Google credentials before uploading to Google Photos!";
+const mediaError = "Unable to convert all uploaded bytes to media items!";
+const UploadError = (count: number) => `Unable to upload ${count} images to Dash's server`;
+const requestError = "Unable to execute download: the body's media items were malformed.";
+const downloadError = "Encountered an error while executing downloads.";
+interface GooglePhotosUploadFailure {
+ batch: number;
+ index: number;
+ url: string;
+ reason: string;
+}
+interface MediaItem {
+ baseUrl: string;
+ filename: string;
+}
+interface NewMediaItem {
+ description: string;
+ simpleMediaItem: {
+ uploadToken: string;
+ };
+}
+const prefix = "google_photos_";
+
+/**
+ * This manager handles the creation of routes for google photos functionality.
+ */
+export default class GooglePhotosManager extends ApiManager {
+
+ protected initialize(register: Registration): void {
+
+ register({
+ method: Method.POST,
+ subscription: "/googlePhotosMediaUpload",
+ onValidation: async ({ user, req, res }) => {
+ const { media } = req.body;
+ const token = await GoogleApiServerUtils.retrieveAccessToken(user.id);
+ if (!token) {
+ return _error(res, authenticationError);
+ }
+ let failed: GooglePhotosUploadFailure[] = [];
+ const batched = BatchedArray.from<GooglePhotosUploadUtils.UploadSource>(media, { batchSize: 25 });
+ const newMediaItems = await batched.batchedMapPatientInterval<NewMediaItem>(
+ { magnitude: 100, unit: TimeUnit.Milliseconds },
+ async (batch: any, collector: any, { completedBatches }: any) => {
+ for (let index = 0; index < batch.length; index++) {
+ const { url, description } = batch[index];
+ const fail = (reason: string) => failed.push({ reason, batch: completedBatches + 1, index, url });
+ const uploadToken = await GooglePhotosUploadUtils.DispatchGooglePhotosUpload(token, InjectSize(url, SizeSuffix.Original)).catch(fail);
+ if (!uploadToken) {
+ fail(`${path.extname(url)} is not an accepted extension`);
+ } else {
+ collector.push({
+ description,
+ simpleMediaItem: { uploadToken }
+ });
+ }
+ }
+ }
+ );
+ const failedCount = failed.length;
+ if (failedCount) {
+ console.error(`Unable to upload ${failedCount} image${failedCount === 1 ? "" : "s"} to Google's servers`);
+ console.log(failed.map(({ reason, batch, index, url }) => `@${batch}.${index}: ${url} failed:\n${reason}`).join('\n\n'));
+ }
+ return GooglePhotosUploadUtils.CreateMediaItems(token, newMediaItems, req.body.album).then(
+ results => _success(res, { results, failed }),
+ error => _error(res, mediaError, error)
+ );
+ }
+ });
+
+ register({
+ method: Method.POST,
+ subscription: "/googlePhotosMediaDownload",
+ onValidation: async ({ req, res }) => {
+ const contents: { mediaItems: MediaItem[] } = req.body;
+ let failed = 0;
+ if (contents) {
+ const completed: Opt<DashUploadUtils.ImageUploadInformation>[] = [];
+ for (let item of contents.mediaItems) {
+ const { contentSize, ...attributes } = await DashUploadUtils.InspectImage(item.baseUrl);
+ const found: Opt<DashUploadUtils.ImageUploadInformation> = await Database.Auxiliary.QueryUploadHistory(contentSize!);
+ if (!found) {
+ const upload = await DashUploadUtils.UploadInspectedImage({ contentSize, ...attributes }, item.filename, prefix).catch(error => _error(res, downloadError, error));
+ if (upload) {
+ completed.push(upload);
+ await Database.Auxiliary.LogUpload(upload);
+ } else {
+ failed++;
+ }
+ } else {
+ completed.push(found);
+ }
+ }
+ if (failed) {
+ return _error(res, UploadError(failed));
+ }
+ return _success(res, completed);
+ }
+ _invalid(res, requestError);
+ }
+ });
+
+ }
+} \ No newline at end of file
diff --git a/src/server/ApiManagers/PDFManager.ts b/src/server/ApiManagers/PDFManager.ts
new file mode 100644
index 000000000..151b48dd9
--- /dev/null
+++ b/src/server/ApiManagers/PDFManager.ts
@@ -0,0 +1,111 @@
+import ApiManager, { Registration } from "./ApiManager";
+import { Method } from "../RouteManager";
+import RouteSubscriber from "../RouteSubscriber";
+import { exists, createReadStream, createWriteStream } from "fs";
+import * as Pdfjs from 'pdfjs-dist';
+import { createCanvas } from "canvas";
+const imageSize = require("probe-image-size");
+import * as express from "express";
+import * as path from "path";
+import { Directory, serverPathToFile, clientPathToFile } from "./UploadManager";
+import { ConsoleColors } from "../ActionUtilities";
+
+export default class PDFManager extends ApiManager {
+
+ protected initialize(register: Registration): void {
+
+ register({
+ method: Method.GET,
+ subscription: new RouteSubscriber("thumbnail").add("filename"),
+ onValidation: ({ req, res }) => getOrCreateThumbnail(req.params.filename, res)
+ });
+
+ }
+
+}
+
+function getOrCreateThumbnail(thumbnailName: string, res: express.Response) {
+ const noExtension = thumbnailName.substring(0, thumbnailName.length - ".png".length);
+ const pageString = noExtension.split('-')[1];
+ const pageNumber = parseInt(pageString);
+ return new Promise<void>(resolve => {
+ const path = serverPathToFile(Directory.pdf_thumbnails, thumbnailName);
+ exists(path, (exists: boolean) => {
+ if (exists) {
+ let existingThumbnail = createReadStream(path);
+ imageSize(existingThumbnail, (err: any, { width, height }: any) => {
+ if (err) {
+ console.log(ConsoleColors.Red, `In PDF thumbnail response, unable to determine dimensions of ${thumbnailName}:`);
+ console.log(err);
+ return;
+ }
+ res.send({
+ path: clientPathToFile(Directory.pdf_thumbnails, thumbnailName),
+ width,
+ height
+ });
+ });
+ } else {
+ const offset = thumbnailName.length - pageString.length - 5;
+ const name = thumbnailName.substring(0, offset) + ".pdf";
+ const path = serverPathToFile(Directory.pdfs, name);
+ CreateThumbnail(path, pageNumber, res);
+ }
+ resolve();
+ });
+ });
+}
+
+async function CreateThumbnail(file: string, pageNumber: number, res: express.Response) {
+ const documentProxy = await Pdfjs.getDocument(file).promise;
+ const factory = new NodeCanvasFactory();
+ const page = await documentProxy.getPage(pageNumber);
+ const viewport = page.getViewport(1 as any);
+ const { canvas, context } = factory.create(viewport.width, viewport.height);
+ const renderContext = {
+ canvasContext: context,
+ canvasFactory: factory,
+ viewport
+ };
+ await page.render(renderContext).promise;
+ const pngStream = canvas.createPNGStream();
+ const filenames = path.basename(file).split(".");
+ const pngFile = serverPathToFile(Directory.pdf_thumbnails, `${filenames[0]}-${pageNumber}.png`);
+ const out = createWriteStream(pngFile);
+ pngStream.pipe(out);
+ out.on("finish", () => {
+ res.send({
+ path: pngFile,
+ width: viewport.width,
+ height: viewport.height
+ });
+ });
+ out.on("error", error => {
+ console.log(ConsoleColors.Red, `In PDF thumbnail creation, encountered the following error when piping ${pngFile}:`);
+ console.log(error);
+ });
+}
+
+class NodeCanvasFactory {
+
+ create = (width: number, height: number) => {
+ var canvas = createCanvas(width, height);
+ var context = canvas.getContext('2d');
+ return {
+ canvas,
+ context
+ };
+ }
+
+ reset = (canvasAndContext: any, width: number, height: number) => {
+ canvasAndContext.canvas.width = width;
+ canvasAndContext.canvas.height = height;
+ }
+
+ destroy = (canvasAndContext: any) => {
+ canvasAndContext.canvas.width = 0;
+ canvasAndContext.canvas.height = 0;
+ canvasAndContext.canvas = null;
+ canvasAndContext.context = null;
+ }
+} \ No newline at end of file
diff --git a/src/server/ApiManagers/SearchManager.ts b/src/server/ApiManagers/SearchManager.ts
new file mode 100644
index 000000000..d3f8995b0
--- /dev/null
+++ b/src/server/ApiManagers/SearchManager.ts
@@ -0,0 +1,49 @@
+import ApiManager, { Registration } from "./ApiManager";
+import { Method } from "../RouteManager";
+import { Search } from "../Search";
+var findInFiles = require('find-in-files');
+import * as path from 'path';
+import { pathToDirectory, Directory } from "./UploadManager";
+
+export default class SearchManager extends ApiManager {
+
+ protected initialize(register: Registration): void {
+
+ register({
+ method: Method.GET,
+ subscription: "/textsearch",
+ onValidation: async ({ req, res }) => {
+ let q = req.query.q;
+ if (q === undefined) {
+ res.send([]);
+ return;
+ }
+ let results = await findInFiles.find({ 'term': q, 'flags': 'ig' }, pathToDirectory(Directory.text), ".txt$");
+ let resObj: { ids: string[], numFound: number, lines: string[] } = { ids: [], numFound: 0, lines: [] };
+ for (var result in results) {
+ resObj.ids.push(path.basename(result, ".txt").replace(/upload_/, ""));
+ resObj.lines.push(results[result].line);
+ resObj.numFound++;
+ }
+ res.send(resObj);
+ }
+ });
+
+ register({
+ method: Method.GET,
+ subscription: "/search",
+ onValidation: async ({ req, res }) => {
+ const solrQuery: any = {};
+ ["q", "fq", "start", "rows", "hl", "hl.fl"].forEach(key => solrQuery[key] = req.query[key]);
+ if (solrQuery.q === undefined) {
+ res.send([]);
+ return;
+ }
+ let results = await Search.Instance.search(solrQuery);
+ res.send(results);
+ }
+ });
+
+ }
+
+} \ No newline at end of file
diff --git a/src/server/ApiManagers/UploadManager.ts b/src/server/ApiManagers/UploadManager.ts
new file mode 100644
index 000000000..80ae0ad61
--- /dev/null
+++ b/src/server/ApiManagers/UploadManager.ts
@@ -0,0 +1,222 @@
+import ApiManager, { Registration } from "./ApiManager";
+import { Method, _success } from "../RouteManager";
+import * as formidable from 'formidable';
+import v4 = require('uuid/v4');
+var AdmZip = require('adm-zip');
+import { extname, basename, dirname } from 'path';
+import { createReadStream, createWriteStream, unlink, readFileSync } from "fs";
+import { publicDirectory, filesDirectory } from "..";
+import { Database } from "../database";
+import { DashUploadUtils, SizeSuffix } from "../DashUploadUtils";
+import * as sharp from 'sharp';
+import { AcceptibleMedia } from "../SharedMediaTypes";
+import { normalize } from "path";
+const imageDataUri = require('image-data-uri');
+
+export enum Directory {
+ parsed_files = "parsed_files",
+ images = "images",
+ videos = "videos",
+ pdfs = "pdfs",
+ text = "text",
+ pdf_thumbnails = "pdf_thumbnails"
+}
+
+export function serverPathToFile(directory: Directory, filename: string) {
+ return normalize(`${filesDirectory}/${directory}/${filename}`);
+}
+
+export function pathToDirectory(directory: Directory) {
+ return normalize(`${filesDirectory}/${directory}`);
+}
+
+export function clientPathToFile(directory: Directory, filename: string) {
+ return `/files/${directory}/${filename}`;
+}
+
+export default class UploadManager extends ApiManager {
+
+ protected initialize(register: Registration): void {
+
+ register({
+ method: Method.POST,
+ subscription: "/upload",
+ onValidation: async ({ req, res }) => {
+ let form = new formidable.IncomingForm();
+ form.uploadDir = pathToDirectory(Directory.parsed_files);
+ form.keepExtensions = true;
+ return new Promise<void>(resolve => {
+ form.parse(req, async (_err, _fields, files) => {
+ let results: any[] = [];
+ for (const key in files) {
+ const result = await DashUploadUtils.upload(files[key]);
+ result && results.push(result);
+ }
+ _success(res, results);
+ resolve();
+ });
+ });
+ }
+ });
+
+ register({
+ method: Method.POST,
+ subscription: "/uploadDoc",
+ onValidation: ({ req, res }) => {
+ let form = new formidable.IncomingForm();
+ form.keepExtensions = true;
+ // let path = req.body.path;
+ const ids: { [id: string]: string } = {};
+ let remap = true;
+ const getId = (id: string): string => {
+ if (!remap) return id;
+ if (id.endsWith("Proto")) return id;
+ if (id in ids) {
+ return ids[id];
+ } else {
+ return ids[id] = v4();
+ }
+ };
+ const mapFn = (doc: any) => {
+ if (doc.id) {
+ doc.id = getId(doc.id);
+ }
+ for (const key in doc.fields) {
+ if (!doc.fields.hasOwnProperty(key)) {
+ continue;
+ }
+ const field = doc.fields[key];
+ if (field === undefined || field === null) {
+ continue;
+ }
+
+ if (field.__type === "proxy" || field.__type === "prefetch_proxy") {
+ field.fieldId = getId(field.fieldId);
+ } else if (field.__type === "script" || field.__type === "computed") {
+ if (field.captures) {
+ field.captures.fieldId = getId(field.captures.fieldId);
+ }
+ } else if (field.__type === "list") {
+ mapFn(field);
+ } else if (typeof field === "string") {
+ const re = /("(?:dataD|d)ocumentId"\s*:\s*")([\w\-]*)"/g;
+ doc.fields[key] = (field as any).replace(re, (match: any, p1: string, p2: string) => {
+ return `${p1}${getId(p2)}"`;
+ });
+ } else if (field.__type === "RichTextField") {
+ const re = /("href"\s*:\s*")(.*?)"/g;
+ field.Data = field.Data.replace(re, (match: any, p1: string, p2: string) => {
+ return `${p1}${getId(p2)}"`;
+ });
+ }
+ }
+ };
+ return new Promise<void>(resolve => {
+ form.parse(req, async (_err, fields, files) => {
+ remap = fields.remap !== "false";
+ let id: string = "";
+ try {
+ for (const name in files) {
+ const path_2 = files[name].path;
+ const zip = new AdmZip(path_2);
+ zip.getEntries().forEach((entry: any) => {
+ if (!entry.entryName.startsWith("files/")) return;
+ let directory = dirname(entry.entryName) + "/";
+ let extension = extname(entry.entryName);
+ let base = basename(entry.entryName).split(".")[0];
+ try {
+ zip.extractEntryTo(entry.entryName, publicDirectory, true, false);
+ directory = "/" + directory;
+
+ createReadStream(publicDirectory + directory + base + extension).pipe(createWriteStream(publicDirectory + directory + base + "_o" + extension));
+ createReadStream(publicDirectory + directory + base + extension).pipe(createWriteStream(publicDirectory + directory + base + "_s" + extension));
+ createReadStream(publicDirectory + directory + base + extension).pipe(createWriteStream(publicDirectory + directory + base + "_m" + extension));
+ createReadStream(publicDirectory + directory + base + extension).pipe(createWriteStream(publicDirectory + directory + base + "_l" + extension));
+ } catch (e) {
+ console.log(e);
+ }
+ });
+ const json = zip.getEntry("doc.json");
+ let docs: any;
+ try {
+ let data = JSON.parse(json.getData().toString("utf8"));
+ docs = data.docs;
+ id = data.id;
+ docs = Object.keys(docs).map(key => docs[key]);
+ docs.forEach(mapFn);
+ await Promise.all(docs.map((doc: any) => new Promise(res => Database.Instance.replace(doc.id, doc, (err, r) => {
+ err && console.log(err);
+ res();
+ }, true, "newDocuments"))));
+ } catch (e) { console.log(e); }
+ unlink(path_2, () => { });
+ }
+ if (id) {
+ res.send(JSON.stringify(getId(id)));
+ } else {
+ res.send(JSON.stringify("error"));
+ }
+ } catch (e) { console.log(e); }
+ resolve();
+ });
+ });
+ }
+ });
+
+ register({
+ method: Method.POST,
+ subscription: "/inspectImage",
+ onValidation: async ({ req, res }) => {
+ const { source } = req.body;
+ if (typeof source === "string") {
+ const { serverAccessPaths } = await DashUploadUtils.UploadImage(source);
+ return res.send(await DashUploadUtils.InspectImage(serverAccessPaths[SizeSuffix.Original]));
+ }
+ res.send({});
+ }
+ });
+
+ register({
+ method: Method.POST,
+ subscription: "/uploadURI",
+ onValidation: ({ req, res }) => {
+ const uri = req.body.uri;
+ const filename = req.body.name;
+ if (!uri || !filename) {
+ res.status(401).send("incorrect parameters specified");
+ return;
+ }
+ return imageDataUri.outputFile(uri, serverPathToFile(Directory.images, filename)).then((savedName: string) => {
+ const ext = extname(savedName).toLowerCase();
+ const { pngs, jpgs } = AcceptibleMedia;
+ let resizers = [
+ { resizer: sharp().resize(100, undefined, { withoutEnlargement: true }), suffix: "_s" },
+ { resizer: sharp().resize(400, undefined, { withoutEnlargement: true }), suffix: "_m" },
+ { resizer: sharp().resize(900, undefined, { withoutEnlargement: true }), suffix: "_l" },
+ ];
+ let isImage = false;
+ if (pngs.includes(ext)) {
+ resizers.forEach(element => {
+ element.resizer = element.resizer.png();
+ });
+ isImage = true;
+ } else if (jpgs.includes(ext)) {
+ resizers.forEach(element => {
+ element.resizer = element.resizer.jpeg();
+ });
+ isImage = true;
+ }
+ if (isImage) {
+ resizers.forEach(resizer => {
+ const path = serverPathToFile(Directory.images, filename + resizer.suffix + ext);
+ createReadStream(savedName).pipe(resizer.resizer).pipe(createWriteStream(path));
+ });
+ }
+ res.send(clientPathToFile(Directory.images, filename + ext));
+ });
+ }
+ });
+
+ }
+
+} \ No newline at end of file
diff --git a/src/server/ApiManagers/UserManager.ts b/src/server/ApiManagers/UserManager.ts
new file mode 100644
index 000000000..0f7d14320
--- /dev/null
+++ b/src/server/ApiManagers/UserManager.ts
@@ -0,0 +1,71 @@
+import ApiManager, { Registration } from "./ApiManager";
+import { Method } from "../RouteManager";
+import { Database } from "../database";
+import { msToTime } from "../ActionUtilities";
+
+export const timeMap: { [id: string]: number } = {};
+interface ActivityUnit {
+ user: string;
+ duration: number;
+}
+
+export default class UserManager extends ApiManager {
+
+ protected initialize(register: Registration): void {
+
+ register({
+ method: Method.GET,
+ subscription: "/getUsers",
+ onValidation: async ({ res }) => {
+ const cursor = await Database.Instance.query({}, { email: 1, userDocumentId: 1 }, "users");
+ const results = await cursor.toArray();
+ res.send(results.map(user => ({ email: user.email, userDocumentId: user.userDocumentId })));
+ }
+ });
+
+ register({
+ method: Method.GET,
+ subscription: "/getUserDocumentId",
+ onValidation: ({ res, user }) => res.send(user.userDocumentId)
+ });
+
+ register({
+ method: Method.GET,
+ subscription: "/getCurrentUser",
+ onValidation: ({ res, user }) => res.send(JSON.stringify(user)),
+ onUnauthenticated: ({ res }) => res.send(JSON.stringify({ id: "__guest__", email: "" }))
+ });
+
+ register({
+ method: Method.GET,
+ subscription: "/activity",
+ onValidation: ({ res }) => {
+ const now = Date.now();
+
+ const activeTimes: ActivityUnit[] = [];
+ const inactiveTimes: ActivityUnit[] = [];
+
+ for (const user in timeMap) {
+ const time = timeMap[user];
+ const duration = now - time;
+ const target = (duration / 1000) < (60 * 5) ? activeTimes : inactiveTimes;
+ target.push({ user, duration });
+ }
+
+ const process = (target: { user: string, duration: number }[]) => {
+ const comparator = (first: ActivityUnit, second: ActivityUnit) => first.duration - second.duration;
+ const sorted = target.sort(comparator);
+ return sorted.map(({ user, duration }) => `${user} (${msToTime(duration)})`);
+ };
+
+ res.render("user_activity.pug", {
+ title: "User Activity",
+ active: process(activeTimes),
+ inactive: process(inactiveTimes)
+ });
+ }
+ });
+
+ }
+
+} \ No newline at end of file
diff --git a/src/server/ApiManagers/UtilManager.ts b/src/server/ApiManagers/UtilManager.ts
new file mode 100644
index 000000000..c1234be6c
--- /dev/null
+++ b/src/server/ApiManagers/UtilManager.ts
@@ -0,0 +1,67 @@
+import ApiManager, { Registration } from "./ApiManager";
+import { Method } from "../RouteManager";
+import { exec } from 'child_process';
+import { command_line } from "../ActionUtilities";
+import RouteSubscriber from "../RouteSubscriber";
+
+export default class UtilManager extends ApiManager {
+
+ protected initialize(register: Registration): void {
+
+ register({
+ method: Method.GET,
+ subscription: new RouteSubscriber("environment").add("key"),
+ onValidation: ({ req, res }) => res.send(process.env[req.params.key])
+ });
+
+ register({
+ method: Method.GET,
+ subscription: "/pull",
+ onValidation: async ({ res }) => {
+ return new Promise<void>(resolve => {
+ exec('"C:\\Program Files\\Git\\git-bash.exe" -c "git pull"', err => {
+ if (err) {
+ res.send(err.message);
+ return;
+ }
+ res.redirect("/");
+ resolve();
+ });
+ });
+ }
+ });
+
+ register({
+ method: Method.GET,
+ subscription: "/buxton",
+ onValidation: async ({ res }) => {
+ let cwd = '../scraping/buxton';
+
+ let onResolved = (stdout: string) => { console.log(stdout); res.redirect("/"); };
+ let onRejected = (err: any) => { console.error(err.message); res.send(err); };
+ let tryPython3 = () => command_line('python3 scraper.py', cwd).then(onResolved, onRejected);
+
+ return command_line('python scraper.py', cwd).then(onResolved, tryPython3);
+ },
+ });
+
+ register({
+ method: Method.GET,
+ subscription: "/version",
+ onValidation: ({ res }) => {
+ return new Promise<void>(resolve => {
+ exec('"C:\\Program Files\\Git\\bin\\git.exe" rev-parse HEAD', (err, stdout) => {
+ if (err) {
+ res.send(err.message);
+ return;
+ }
+ res.send(stdout);
+ });
+ resolve();
+ });
+ }
+ });
+
+ }
+
+} \ No newline at end of file
generated by cgit v1.2.3-70-g09d2 (git 2.46.0) at 2025-10-06 21:28:46 +0000