aboutsummaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to 'src')
-rw-r--r--src/client/documents/Documents.ts1
-rw-r--r--src/client/util/ClientDiagnostics.ts11
-rw-r--r--src/client/util/Import & Export/DirectoryImportBox.tsx19
-rw-r--r--src/client/views/Main.tsx2
-rw-r--r--src/client/views/collections/CollectionSubView.tsx22
-rw-r--r--src/client/views/pdf/PDFViewer.tsx3
-rw-r--r--src/server/ActionUtilities.ts17
-rw-r--r--src/server/ApiManagers/DownloadManager.ts4
-rw-r--r--src/server/ApiManagers/GooglePhotosManager.ts4
-rw-r--r--src/server/ApiManagers/PDFManager.ts34
-rw-r--r--src/server/ApiManagers/SearchManager.ts4
-rw-r--r--src/server/ApiManagers/UploadManager.ts47
-rw-r--r--src/server/DashUploadUtils.ts388
-rw-r--r--src/server/SharedMediaTypes.ts5
-rw-r--r--src/server/database.ts4
-rw-r--r--src/server/index.ts4
16 files changed, 312 insertions, 257 deletions
diff --git a/src/client/documents/Documents.ts b/src/client/documents/Documents.ts
index dea057b93..a2f4d23c7 100644
--- a/src/client/documents/Documents.ts
+++ b/src/client/documents/Documents.ts
@@ -377,7 +377,6 @@ export namespace Docs {
let extension = path.extname(target);
target = `${target.substring(0, target.length - extension.length)}_o${extension}`;
}
- // if (target !== "http://www.cs.brown.edu/") {
requestImageSize(target)
.then((size: any) => {
let aspect = size.height / size.width;
diff --git a/src/client/util/ClientDiagnostics.ts b/src/client/util/ClientDiagnostics.ts
index 24f196252..7eef935fd 100644
--- a/src/client/util/ClientDiagnostics.ts
+++ b/src/client/util/ClientDiagnostics.ts
@@ -1,9 +1,6 @@
-import { observable, runInAction } from "mobx";
-import { MainView } from "../views/MainView";
-
export namespace ClientDiagnostics {
- export function start() {
+ export async function start() {
let serverPolls = 0;
const serverHandle = setInterval(async () => {
@@ -17,14 +14,16 @@ export namespace ClientDiagnostics {
let executed = false;
- const solrHandle = setInterval(async () => {
+ const handle = async () => {
const response = await fetch("/solrHeartbeat");
if (!(await response.json()).running) {
!executed && alert("Looks like SOLR is not running on your machine.");
executed = true;
clearInterval(solrHandle);
}
- }, 1000 * 15);
+ };
+ await handle();
+ const solrHandle = setInterval(handle, 1000 * 15);
}
diff --git a/src/client/util/Import & Export/DirectoryImportBox.tsx b/src/client/util/Import & Export/DirectoryImportBox.tsx
index f0880f193..16ae50685 100644
--- a/src/client/util/Import & Export/DirectoryImportBox.tsx
+++ b/src/client/util/Import & Export/DirectoryImportBox.tsx
@@ -22,18 +22,10 @@ import "./DirectoryImportBox.scss";
import { Networking } from "../../Network";
import { BatchedArray } from "array-batcher";
import * as path from 'path';
-import { DashUploadUtils } from "../../../server/DashUploadUtils";
-import { SharedMediaTypes } from "../../../server/SharedMediaTypes";
+import { AcceptibleMedia } from "../../../server/SharedMediaTypes";
const unsupported = ["text/html", "text/plain"];
-interface ImageUploadResponse {
- name: string;
- path: string;
- type: string;
- exif: any;
-}
-
@observer
export default class DirectoryImportBox extends React.Component<FieldViewProps> {
private selector = React.createRef<HTMLInputElement>();
@@ -98,7 +90,7 @@ export default class DirectoryImportBox extends React.Component<FieldViewProps>
let file = files.item(i);
if (file && !unsupported.includes(file.type)) {
const ext = path.extname(file.name).toLowerCase();
- if (SharedMediaTypes.imageFormats.includes(ext)) {
+ if (AcceptibleMedia.imageFormats.includes(ext)) {
validated.push(file);
}
}
@@ -114,7 +106,7 @@ export default class DirectoryImportBox extends React.Component<FieldViewProps>
runInAction(() => this.phase = `Internal: uploading ${this.quota - this.completed} files to Dash...`);
- const uploads = await BatchedArray.from(validated, { batchSize: 15 }).batchedMapAsync<ImageUploadResponse>(async (batch, collector) => {
+ const uploads = await BatchedArray.from(validated, { batchSize: 15 }).batchedMapAsync<any>(async (batch, collector) => {
const formData = new FormData();
batch.forEach(file => {
@@ -127,16 +119,17 @@ export default class DirectoryImportBox extends React.Component<FieldViewProps>
runInAction(() => this.completed += batch.length);
});
+ const size = "_o";
await Promise.all(uploads.map(async upload => {
const type = upload.type;
- const path = Utils.prepend(upload.path);
+ const path = Utils.prepend(upload.clientAccessPath);
const options = {
nativeWidth: 300,
width: 300,
title: upload.name
};
const document = await Docs.Get.DocumentFromType(type, path, options);
- const { data, error } = upload.exif;
+ const { data, error } = upload.exifData;
if (document) {
Doc.GetProto(document).exif = error || Docs.Get.DocumentHierarchyFromJson(data);
docs.push(document);
diff --git a/src/client/views/Main.tsx b/src/client/views/Main.tsx
index dec4a24e4..9e699978f 100644
--- a/src/client/views/Main.tsx
+++ b/src/client/views/Main.tsx
@@ -10,7 +10,7 @@ import { ClientDiagnostics } from "../util/ClientDiagnostics";
AssignAllExtensions();
(async () => {
- ClientDiagnostics.start();
+ await ClientDiagnostics.start();
const info = await CurrentUserUtils.loadCurrentUser();
DocServer.init(window.location.protocol, window.location.hostname, 4321, info.email);
await Docs.Prototypes.initialize();
diff --git a/src/client/views/collections/CollectionSubView.tsx b/src/client/views/collections/CollectionSubView.tsx
index 1c3ff37ee..a1bd1527e 100644
--- a/src/client/views/collections/CollectionSubView.tsx
+++ b/src/client/views/collections/CollectionSubView.tsx
@@ -22,6 +22,7 @@ import React = require("react");
var path = require('path');
import { GooglePhotos } from "../../apis/google_docs/GooglePhotosClientUtils";
import { ImageUtils } from "../../util/Import & Export/ImageUtils";
+import { Networking } from "../../Network";
export interface CollectionViewProps extends FieldViewProps {
addDocument: (document: Doc) => boolean;
@@ -271,28 +272,25 @@ export function CollectionSubView<T>(schemaCtor: (doc: Doc) => T) {
let file = item.getAsFile();
let formData = new FormData();
- if (file) {
- formData.append('file', file);
+ if (!file || !file.type) {
+ continue;
}
- let dropFileName = file ? file.name : "-empty-";
- let prom = fetch(Utils.prepend("/upload"), {
- method: 'POST',
- body: formData
- }).then(async (res: Response) => {
- (await res.json()).map(action((file: any) => {
+ formData.append('file', file);
+ let dropFileName = file ? file.name : "-empty-";
+ promises.push(Networking.PostFormDataToServer("/upload", formData).then(results => {
+ results.map(action((file: any) => {
let full = { ...options, nativeWidth: type.indexOf("video") !== -1 ? 600 : 300, width: 300, title: dropFileName };
- let pathname = Utils.prepend(file.path);
+ let pathname = Utils.prepend(file.clientAccessPath);
Docs.Get.DocumentFromType(type, pathname, full).then(doc => {
doc && (Doc.GetProto(doc).fileUpload = path.basename(pathname).replace("upload_", "").replace(/\.[a-z0-9]*$/, ""));
doc && this.props.addDocument(doc);
});
}));
- });
- promises.push(prom);
+ }));
}
}
- if (text) {
+ if (text && !text.includes("https://")) {
this.props.addDocument(Docs.Create.TextDocument({ ...options, documentText: "@@@" + text, width: 400, height: 315 }));
return;
}
diff --git a/src/client/views/pdf/PDFViewer.tsx b/src/client/views/pdf/PDFViewer.tsx
index b737ce221..c075a4f99 100644
--- a/src/client/views/pdf/PDFViewer.tsx
+++ b/src/client/views/pdf/PDFViewer.tsx
@@ -125,7 +125,8 @@ export class PDFViewer extends DocAnnotatableComponent<IViewerProps, PdfDocument
!this.props.Document.lockedTransform && (this.props.Document.lockedTransform = true);
// change the address to be the file address of the PNG version of each page
// file address of the pdf
- this._coverPath = JSON.parse(await rp.get(Utils.prepend(`/thumbnail${this.props.url.substring("files/".length, this.props.url.length - ".pdf".length)}-${(this.Document.curPage || 1)}.PNG`)));
+ const path = Utils.prepend(`/thumbnail${this.props.url.substring("files/pdfs/".length, this.props.url.length - ".pdf".length)}-${(this.Document.curPage || 1)}.png`);
+ this._coverPath = JSON.parse(await rp.get(path));
runInAction(() => this._showWaiting = this._showCover = true);
this.props.startupLive && this.setupPdfJsViewer();
this._searchReactionDisposer = reaction(() => this.Document.search_string, searchString => {
diff --git a/src/server/ActionUtilities.ts b/src/server/ActionUtilities.ts
index 7f493dd70..c9fc86fea 100644
--- a/src/server/ActionUtilities.ts
+++ b/src/server/ActionUtilities.ts
@@ -2,6 +2,7 @@ import * as fs from 'fs';
import { ExecOptions } from 'shelljs';
import { exec } from 'child_process';
import * as path from 'path';
+import * as rimraf from "rimraf";
export const command_line = (command: string, fromDirectory?: string) => {
return new Promise<string>((resolve, reject) => {
@@ -68,4 +69,18 @@ export function msToTime(duration: number) {
let secondsS = (seconds < 10) ? "0" + seconds : seconds;
return hoursS + ":" + minutesS + ":" + secondsS + "." + milliseconds;
-} \ No newline at end of file
+}
+
+export const createIfNotExists = async (path: string) => {
+ if (await new Promise<boolean>(resolve => fs.exists(path, resolve))) {
+ return true;
+ }
+ return new Promise<boolean>(resolve => fs.mkdir(path, error => resolve(error === null)));
+};
+
+export async function Prune(rootDirectory: string): Promise<boolean> {
+ const error = await new Promise<Error>(resolve => rimraf(rootDirectory, resolve));
+ return error === null;
+}
+
+export const Destroy = (mediaPath: string) => new Promise<boolean>(resolve => fs.unlink(mediaPath, error => resolve(error === null))); \ No newline at end of file
diff --git a/src/server/ApiManagers/DownloadManager.ts b/src/server/ApiManagers/DownloadManager.ts
index fc6ba0d22..5bad46eda 100644
--- a/src/server/ApiManagers/DownloadManager.ts
+++ b/src/server/ApiManagers/DownloadManager.ts
@@ -5,7 +5,7 @@ import * as Archiver from 'archiver';
import * as express from 'express';
import { Database } from "../database";
import * as path from "path";
-import { DashUploadUtils } from "../DashUploadUtils";
+import { DashUploadUtils, SizeSuffix } from "../DashUploadUtils";
import { publicDirectory } from "..";
export type Hierarchy = { [id: string]: string | Hierarchy };
@@ -254,7 +254,7 @@ async function writeHierarchyRecursive(file: Archiver.Archiver, hierarchy: Hiera
// and dropped in the browser and thus hosted remotely) so we upload it
// to our server and point the zip file to it, so it can bundle up the bytes
const information = await DashUploadUtils.UploadImage(result);
- path = information.mediaPaths[0];
+ path = information.serverAccessPaths[SizeSuffix.Original];
}
// write the file specified by the path to the directory in the
// zip file given by the prefix.
diff --git a/src/server/ApiManagers/GooglePhotosManager.ts b/src/server/ApiManagers/GooglePhotosManager.ts
index c7af69375..5a709688b 100644
--- a/src/server/ApiManagers/GooglePhotosManager.ts
+++ b/src/server/ApiManagers/GooglePhotosManager.ts
@@ -86,10 +86,10 @@ export default class GooglePhotosManager extends ApiManager {
const contents: { mediaItems: MediaItem[] } = req.body;
let failed = 0;
if (contents) {
- const completed: Opt<DashUploadUtils.UploadInformation>[] = [];
+ const completed: Opt<DashUploadUtils.ImageUploadInformation>[] = [];
for (let item of contents.mediaItems) {
const { contentSize, ...attributes } = await DashUploadUtils.InspectImage(item.baseUrl);
- const found: Opt<DashUploadUtils.UploadInformation> = await Database.Auxiliary.QueryUploadHistory(contentSize!);
+ const found: Opt<DashUploadUtils.ImageUploadInformation> = await Database.Auxiliary.QueryUploadHistory(contentSize!);
if (!found) {
const upload = await DashUploadUtils.UploadInspectedImage({ contentSize, ...attributes }, item.filename, prefix).catch(error => _error(res, downloadError, error));
if (upload) {
diff --git a/src/server/ApiManagers/PDFManager.ts b/src/server/ApiManagers/PDFManager.ts
index 632b4965a..4bd750aaf 100644
--- a/src/server/ApiManagers/PDFManager.ts
+++ b/src/server/ApiManagers/PDFManager.ts
@@ -2,12 +2,12 @@ import ApiManager, { Registration } from "./ApiManager";
import { Method } from "../RouteManager";
import RouteSubscriber from "../RouteSubscriber";
import { exists, createReadStream, createWriteStream } from "fs";
-import { filesDirectory } from "..";
import * as Pdfjs from 'pdfjs-dist';
import { createCanvas } from "canvas";
const probe = require("probe-image-size");
import * as express from "express";
import * as path from "path";
+import { Directory, serverPathToFile, clientPathToFile } from "./UploadManager";
export default class PDFManager extends ApiManager {
@@ -21,21 +21,27 @@ export default class PDFManager extends ApiManager {
let noExt = filename.substring(0, filename.length - ".png".length);
let pagenumber = parseInt(noExt.split('-')[1]);
return new Promise<void>(resolve => {
- exists(filesDirectory + filename, (exists: boolean) => {
- console.log(`${filesDirectory + filename} ${exists ? "exists" : "does not exist"}`);
+ const path = serverPathToFile(Directory.pdf_thumbnails, filename);
+ exists(path, (exists: boolean) => {
+ console.log(`${path} ${exists ? "exists" : "does not exist"}`);
if (exists) {
- let input = createReadStream(filesDirectory + filename);
- probe(input, (err: any, result: any) => {
+ let input = createReadStream(path);
+ probe(input, (err: any, { width, height }: any) => {
if (err) {
console.log(err);
console.log(`error on ${filename}`);
return;
}
- res.send({ path: "/files/" + filename, width: result.width, height: result.height });
+ res.send({
+ path: clientPathToFile(Directory.pdf_thumbnails, filename),
+ width,
+ height
+ });
});
}
else {
- LoadPage(filesDirectory + filename.substring(0, filename.length - noExt.split('-')[1].length - ".PNG".length - 1) + ".pdf", pagenumber, res);
+ const name = filename.substring(0, filename.length - noExt.split('-')[1].length - ".PNG".length - 1) + ".pdf";
+ LoadPage(serverPathToFile(Directory.pdfs, name), pagenumber, res);
}
resolve();
});
@@ -55,8 +61,8 @@ export default class PDFManager extends ApiManager {
let canvasAndContext = factory.create(viewport.width, viewport.height);
let renderContext = {
canvasContext: canvasAndContext.context,
- viewport: viewport,
- canvasFactory: factory
+ canvasFactory: factory,
+ viewport
};
console.log("read " + pageNumber);
@@ -64,13 +70,17 @@ export default class PDFManager extends ApiManager {
.then(() => {
console.log("saving " + pageNumber);
let stream = canvasAndContext.canvas.createPNGStream();
- let pngFile = `${file.substring(0, file.length - ".pdf".length)}-${pageNumber}.PNG`;
+ let filenames = path.basename(file).split(".");
+ const pngFile = serverPathToFile(Directory.pdf_thumbnails, `${filenames[0]}-${pageNumber}.png`);
let out = createWriteStream(pngFile);
stream.pipe(out);
out.on("finish", () => {
console.log(`Success! Saved to ${pngFile}`);
- let name = path.basename(pngFile);
- res.send({ path: "/files/" + name, width: viewport.width, height: viewport.height });
+ res.send({
+ path: pngFile,
+ width: viewport.width,
+ height: viewport.height
+ });
});
}, (reason: string) => {
console.error(reason + ` ${pageNumber}`);
diff --git a/src/server/ApiManagers/SearchManager.ts b/src/server/ApiManagers/SearchManager.ts
index 1c801715a..d3f8995b0 100644
--- a/src/server/ApiManagers/SearchManager.ts
+++ b/src/server/ApiManagers/SearchManager.ts
@@ -3,7 +3,7 @@ import { Method } from "../RouteManager";
import { Search } from "../Search";
var findInFiles = require('find-in-files');
import * as path from 'path';
-import { filesDirectory } from "..";
+import { pathToDirectory, Directory } from "./UploadManager";
export default class SearchManager extends ApiManager {
@@ -18,7 +18,7 @@ export default class SearchManager extends ApiManager {
res.send([]);
return;
}
- let results = await findInFiles.find({ 'term': q, 'flags': 'ig' }, filesDirectory + "text", ".txt$");
+ let results = await findInFiles.find({ 'term': q, 'flags': 'ig' }, pathToDirectory(Directory.text), ".txt$");
let resObj: { ids: string[], numFound: number, lines: string[] } = { ids: [], numFound: 0, lines: [] };
for (var result in results) {
resObj.ids.push(path.basename(result, ".txt").replace(/upload_/, ""));
diff --git a/src/server/ApiManagers/UploadManager.ts b/src/server/ApiManagers/UploadManager.ts
index 2a9faacd8..2f76871a6 100644
--- a/src/server/ApiManagers/UploadManager.ts
+++ b/src/server/ApiManagers/UploadManager.ts
@@ -7,14 +7,33 @@ import { extname, basename, dirname } from 'path';
import { createReadStream, createWriteStream, unlink, readFileSync } from "fs";
import { publicDirectory, filesDirectory } from "..";
import { Database } from "../database";
-import { DashUploadUtils } from "../DashUploadUtils";
-import { Opt } from "../../new_fields/Doc";
-import { ParsedPDF } from "../PdfTypes";
-const pdf = require('pdf-parse');
+import { DashUploadUtils, SizeSuffix } from "../DashUploadUtils";
import * as sharp from 'sharp';
-import { SharedMediaTypes } from "../SharedMediaTypes";
+import { AcceptibleMedia } from "../SharedMediaTypes";
+import { normalize } from "path";
const imageDataUri = require('image-data-uri');
+export enum Directory {
+ parsed_files = "parsed_files",
+ images = "images",
+ videos = "videos",
+ pdfs = "pdfs",
+ text = "text",
+ pdf_thumbnails = "pdf_thumbnails"
+}
+
+export function serverPathToFile(directory: Directory, filename: string) {
+ return normalize(`${filesDirectory}/${directory}/${filename}`);
+}
+
+export function pathToDirectory(directory: Directory) {
+ return normalize(`${filesDirectory}/${directory}`);
+}
+
+export function clientPathToFile(directory: Directory, filename: string) {
+ return `/files/${directory}/${filename}`;
+}
+
export default class UploadManager extends ApiManager {
protected initialize(register: Registration): void {
@@ -129,13 +148,14 @@ export default class UploadManager extends ApiManager {
subscription: "/upload",
onValidation: async ({ req, res }) => {
let form = new formidable.IncomingForm();
- form.uploadDir = filesDirectory;
+ form.uploadDir = pathToDirectory(Directory.parsed_files);
form.keepExtensions = true;
return new Promise<void>(resolve => {
form.parse(req, async (_err, _fields, files) => {
let results: any[] = [];
for (const key in files) {
- results.push(DashUploadUtils.upload(files[key]));
+ const result = await DashUploadUtils.upload(files[key]);
+ result && results.push(result);
}
_success(res, results);
resolve();
@@ -150,8 +170,8 @@ export default class UploadManager extends ApiManager {
onValidation: async ({ req, res }) => {
const { source } = req.body;
if (typeof source === "string") {
- const uploadInformation = await DashUploadUtils.UploadImage(source);
- return res.send(await DashUploadUtils.InspectImage(uploadInformation.mediaPaths[0]));
+ const { serverAccessPaths } = await DashUploadUtils.UploadImage(source);
+ return res.send(await DashUploadUtils.InspectImage(serverAccessPaths[SizeSuffix.Original]));
}
res.send({});
}
@@ -167,9 +187,9 @@ export default class UploadManager extends ApiManager {
res.status(401).send("incorrect parameters specified");
return;
}
- return imageDataUri.outputFile(uri, filesDirectory + filename).then((savedName: string) => {
+ return imageDataUri.outputFile(uri, serverPathToFile(Directory.images, filename)).then((savedName: string) => {
const ext = extname(savedName).toLowerCase();
- const { pngs, jpgs } = SharedMediaTypes;
+ const { pngs, jpgs } = AcceptibleMedia;
let resizers = [
{ resizer: sharp().resize(100, undefined, { withoutEnlargement: true }), suffix: "_s" },
{ resizer: sharp().resize(400, undefined, { withoutEnlargement: true }), suffix: "_m" },
@@ -189,10 +209,11 @@ export default class UploadManager extends ApiManager {
}
if (isImage) {
resizers.forEach(resizer => {
- createReadStream(savedName).pipe(resizer.resizer).pipe(createWriteStream(filesDirectory + filename + resizer.suffix + ext));
+ const path = serverPathToFile(Directory.images, filename + resizer.suffix + ext);
+ createReadStream(savedName).pipe(resizer.resizer).pipe(createWriteStream(path));
});
}
- res.send("/files/" + filename + ext);
+ res.send(clientPathToFile(Directory.images, filename + ext));
});
}
});
diff --git a/src/server/DashUploadUtils.ts b/src/server/DashUploadUtils.ts
index 839aada4b..0a670ec01 100644
--- a/src/server/DashUploadUtils.ts
+++ b/src/server/DashUploadUtils.ts
@@ -5,18 +5,32 @@ import * as sharp from 'sharp';
import request = require('request-promise');
import { ExifData, ExifImage } from 'exif';
import { Opt } from '../new_fields/Doc';
-import { SharedMediaTypes } from './SharedMediaTypes';
+import { AcceptibleMedia } from './SharedMediaTypes';
import { filesDirectory } from '.';
import { File } from 'formidable';
-import { extname, basename } from "path";
+import { basename } from "path";
+import { ConsoleColors, createIfNotExists } from './ActionUtilities';
+import { ParsedPDF } from "../server/PdfTypes";
+const parse = require('pdf-parse');
+import { Directory, serverPathToFile, clientPathToFile } from './ApiManagers/UploadManager';
-const uploadDirectory = path.join(__dirname, './public/files/');
+export enum SizeSuffix {
+ Small = "_s",
+ Medium = "_m",
+ Large = "_l",
+ Original = "_o"
+}
export namespace DashUploadUtils {
+ function InjectSize(filename: string, size: SizeSuffix) {
+ const extension = path.extname(filename).toLowerCase();
+ return filename.substring(0, filename.length - extension.length) + size + extension;
+ }
+
export interface Size {
width: number;
- suffix: string;
+ suffix: SizeSuffix;
}
export interface ImageFileResponse {
@@ -27,215 +41,221 @@ export namespace DashUploadUtils {
}
export const Sizes: { [size: string]: Size } = {
- SMALL: { width: 100, suffix: "_s" },
- MEDIUM: { width: 400, suffix: "_m" },
- LARGE: { width: 900, suffix: "_l" },
+ SMALL: { width: 100, suffix: SizeSuffix.Small },
+ MEDIUM: { width: 400, suffix: SizeSuffix.Medium },
+ LARGE: { width: 900, suffix: SizeSuffix.Large },
};
export function validateExtension(url: string) {
- return SharedMediaTypes.imageFormats.includes(path.extname(url).toLowerCase());
+ return AcceptibleMedia.imageFormats.includes(path.extname(url).toLowerCase());
}
const size = "content-length";
const type = "content-type";
- export interface UploadInformation {
- mediaPaths: string[];
- fileNames: { [key: string]: string };
+ export interface ImageUploadInformation {
+ clientAccessPath: string;
+ serverAccessPaths: { [key: string]: string };
exifData: EnrichedExifData;
contentSize?: number;
contentType?: string;
}
- export function upload(file: File): any {
+ export async function upload(file: File): Promise<any> {
const { type, path, name } = file;
- const filename = basename(path);
- const extension = extname(path).toLowerCase();
- if (extension === ".pdf") {
-
- } else if {
- let partition: Opt<string>;
- if(imageFormats.includes(extension)) {
- partition = DashUploadUtils.Partitions.images;
- } else if (videoFormats.includes(extension)) {
- partition = DashUploadUtils.Partitions.videos;
- }
- let uploadInformation: Opt<DashUploadUtils.UploadInformation>;
- if (partition) {
- uploadInformation = await DashUploadUtils.UploadImage(`${filesDirectory}/${partition}/${filename}`, filename);
- } else {
- console.log(`Unable to accommodate, and ignored, the following file upload: ${filename}`);
+ const { imageFormats, videoFormats, applicationFormats } = AcceptibleMedia;
+ const types = type.split("/");
+
+ const category = types[0];
+ const format = `.${types[1]}`;
+
+ switch (category) {
+ case "image":
+ if (imageFormats.includes(format)) {
+ const { clientAccessPath } = await UploadImage(path, basename(path), format);
+ return { clientAccessPath, name, type };
+ }
+ case "video":
+ if (videoFormats.includes(format)) {
+ return MoveParsedFile(path, Directory.videos);
+ }
+ case "application":
+ if (applicationFormats.includes(format)) {
+ return UploadPdf(path);
+ }
}
+ console.log(ConsoleColors.Red, `Ignoring unsupported file ${name} with upload type (${type}).`);
+ return { clientAccessPath: undefined };
}
- const exif = uploadInformation ? uploadInformation.exifData : undefined;
- results.push({ name, type, path: `/files/${filename}`, exif });
-}
+ async function UploadPdf(absolutePath: string) {
+ let dataBuffer = fs.readFileSync(absolutePath);
+ const result: ParsedPDF = await parse(dataBuffer);
+ const parsedName = basename(absolutePath);
+ await new Promise<void>((resolve, reject) => {
+ const textFilename = `${parsedName.substring(0, parsedName.length - 4)}.txt`;
+ const writeStream = fs.createWriteStream(serverPathToFile(Directory.text, textFilename));
+ writeStream.write(result.text, error => error ? reject(error) : resolve());
+ });
+ return MoveParsedFile(absolutePath, Directory.pdfs);
+ }
-const generate = (prefix: string, url: string) => `${prefix}upload_${Utils.GenerateGuid()}${sanitizeExtension(url)}`;
-const sanitize = (filename: string) => filename.replace(/\s+/g, "_");
-const sanitizeExtension = (source: string) => {
- let extension = path.extname(source);
- extension = extension.toLowerCase();
- extension = extension.split("?")[0];
- return extension;
-};
-
-/**
- * Uploads an image specified by the @param source to Dash's /public/files/
- * directory, and returns information generated during that upload
- *
- * @param {string} source is either the absolute path of an already uploaded image or
- * the url of a remote image
- * @param {string} filename dictates what to call the image. If not specified,
- * the name {@param prefix}_upload_{GUID}
- * @param {string} prefix is a string prepended to the generated image name in the
- * event that @param filename is not specified
- *
- * @returns {UploadInformation} This method returns
- * 1) the paths to the uploaded images (plural due to resizing)
- * 2) the file name of each of the resized images
- * 3) the size of the image, in bytes (4432130)
- * 4) the content type of the image, i.e. image/(jpeg | png | ...)
- */
-export const UploadImage = async (source: string, filename?: string, prefix: string = ""): Promise<UploadInformation> => {
- const metadata = await InspectImage(source);
- return UploadInspectedImage(metadata, filename, prefix);
-};
-
-export interface InspectionResults {
- isLocal: boolean;
- stream: any;
- normalizedUrl: string;
- exifData: EnrichedExifData;
- contentSize?: number;
- contentType?: string;
-}
+ const generate = (prefix: string, url: string) => `${prefix}upload_${Utils.GenerateGuid()}${sanitizeExtension(url)}`;
+ const sanitizeExtension = (source: string) => {
+ let extension = path.extname(source);
+ extension = extension.toLowerCase();
+ extension = extension.split("?")[0];
+ return extension;
+ };
-export interface EnrichedExifData {
- data: ExifData;
- error?: string;
-}
+ /**
+ * Uploads an image specified by the @param source to Dash's /public/files/
+ * directory, and returns information generated during that upload
+ *
+ * @param {string} source is either the absolute path of an already uploaded image or
+ * the url of a remote image
+ * @param {string} filename dictates what to call the image. If not specified,
+ * the name {@param prefix}_upload_{GUID}
+ * @param {string} prefix is a string prepended to the generated image name in the
+ * event that @param filename is not specified
+ *
+ * @returns {ImageUploadInformation} This method returns
+ * 1) the paths to the uploaded images (plural due to resizing)
+ * 2) the file name of each of the resized images
+ * 3) the size of the image, in bytes (4432130)
+ * 4) the content type of the image, i.e. image/(jpeg | png | ...)
+ */
+ export const UploadImage = async (source: string, filename?: string, format?: string, prefix: string = ""): Promise<ImageUploadInformation> => {
+ const metadata = await InspectImage(source);
+ return UploadInspectedImage(metadata, filename, format, prefix);
+ };
-export enum Partitions {
- pdf_text = "pdf_text",
- images = "images",
- videos = "videos"
-}
+ export interface InspectionResults {
+ isLocal: boolean;
+ stream: any;
+ normalizedUrl: string;
+ exifData: EnrichedExifData;
+ contentSize?: number;
+ contentType?: string;
+ }
-export async function buildFilePartitions() {
- const pending = Object.keys(Partitions).map(sub => createIfNotExists(filesDirectory + sub));
- return Promise.all(pending);
-}
+ export interface EnrichedExifData {
+ data: ExifData;
+ error?: string;
+ }
-/**
- * Based on the url's classification as local or remote, gleans
- * as much information as possible about the specified image
- *
- * @param source is the path or url to the image in question
- */
-export const InspectImage = async (source: string): Promise<InspectionResults> => {
- const { isLocal, stream, normalized: normalizedUrl } = classify(source);
- const exifData = await parseExifData(source);
- const results = {
- exifData,
- isLocal,
- stream,
- normalizedUrl
- };
- // stop here if local, since request.head() can't handle local paths, only urls on the web
- if (isLocal) {
- return results;
+ export async function buildFileDirectories() {
+ const pending = Object.keys(Directory).map(sub => createIfNotExists(`${filesDirectory}/${sub}`));
+ return Promise.all(pending);
}
- const metadata = (await new Promise<any>((resolve, reject) => {
- request.head(source, async (error, res) => {
- if (error) {
- return reject(error);
- }
- resolve(res);
- });
- })).headers;
- return {
- contentSize: parseInt(metadata[size]),
- contentType: metadata[type],
- ...results
- };
-};
-
-export const UploadInspectedImage = async (metadata: InspectionResults, filename?: string, prefix = ""): Promise<UploadInformation> => {
- const { isLocal, stream, normalizedUrl, contentSize, contentType, exifData } = metadata;
- const resolved = filename ? sanitize(filename) : generate(prefix, normalizedUrl);
- const extension = sanitizeExtension(normalizedUrl || resolved);
- let information: UploadInformation = {
- mediaPaths: [],
- fileNames: { clean: resolved },
- exifData,
- contentSize,
- contentType,
- };
- const { pngs, jpgs } = SharedMediaTypes;
- return new Promise<UploadInformation>(async (resolve, reject) => {
- const resizers = [
- { resizer: sharp().rotate(), suffix: "_o" },
- ...Object.values(Sizes).map(size => ({
- resizer: sharp().resize(size.width, undefined, { withoutEnlargement: true }).rotate(),
- suffix: size.suffix
- }))
- ];
- if (pngs.includes(extension)) {
- resizers.forEach(element => element.resizer = element.resizer.png());
- } else if (jpgs.includes(extension)) {
- resizers.forEach(element => element.resizer = element.resizer.jpeg());
- }
- for (let resizer of resizers) {
- const suffix = resizer.suffix;
- let mediaPath: string;
- await new Promise<void>(resolve => {
- const filename = resolved.substring(0, resolved.length - extension.length) + suffix + extension;
- information.mediaPaths.push(mediaPath = uploadDirectory + filename);
- information.fileNames[suffix] = filename;
- stream(normalizedUrl).pipe(resizer.resizer).pipe(fs.createWriteStream(mediaPath))
- .on('close', resolve)
- .on('error', reject);
- });
+
+ /**
+ * Based on the url's classification as local or remote, gleans
+ * as much information as possible about the specified image
+ *
+ * @param source is the path or url to the image in question
+ */
+ export const InspectImage = async (source: string): Promise<InspectionResults> => {
+ const { isLocal, stream, normalized: normalizedUrl } = classify(source);
+ const exifData = await parseExifData(source);
+ const results = {
+ exifData,
+ isLocal,
+ stream,
+ normalizedUrl
+ };
+ // stop here if local, since request.head() can't handle local paths, only urls on the web
+ if (isLocal) {
+ return results;
}
- if (!isLocal) {
- await new Promise<void>(resolve => {
- stream(normalizedUrl).pipe(fs.createWriteStream(uploadDirectory + resolved)).on('close', resolve);
+ const metadata = (await new Promise<any>((resolve, reject) => {
+ request.head(source, async (error, res) => {
+ if (error) {
+ return reject(error);
+ }
+ resolve(res);
});
- }
- resolve(information);
- });
-};
-
-const classify = (url: string) => {
- const isLocal = /Dash-Web(\\|\/)src(\\|\/)server(\\|\/)public(\\|\/)files/g.test(url);
- return {
- isLocal,
- stream: isLocal ? fs.createReadStream : request,
- normalized: isLocal ? path.normalize(url) : url
+ })).headers;
+ return {
+ contentSize: parseInt(metadata[size]),
+ contentType: metadata[type],
+ ...results
+ };
};
-};
-
-const parseExifData = async (source: string): Promise<EnrichedExifData> => {
- return new Promise<EnrichedExifData>(resolve => {
- new ExifImage(source, (error, data) => {
- let reason: Opt<string> = undefined;
- if (error) {
- reason = (error as any).code;
+
+ export async function MoveParsedFile(absolutePath: string, destination: Directory): Promise<{ clientAccessPath: Opt<string> }> {
+ return new Promise<{ clientAccessPath: Opt<string> }>(resolve => {
+ const filename = basename(absolutePath);
+ const destinationPath = serverPathToFile(destination, filename);
+ fs.rename(absolutePath, destinationPath, error => {
+ resolve({ clientAccessPath: error ? undefined : clientPathToFile(destination, filename) });
+ });
+ });
+ }
+
+ export const UploadInspectedImage = async (metadata: InspectionResults, filename?: string, format?: string, prefix = ""): Promise<ImageUploadInformation> => {
+ const { isLocal, stream, normalizedUrl, contentSize, contentType, exifData } = metadata;
+ const resolved = filename || generate(prefix, normalizedUrl);
+ const extension = format || sanitizeExtension(normalizedUrl || resolved);
+ let information: ImageUploadInformation = {
+ clientAccessPath: clientPathToFile(Directory.images, resolved),
+ serverAccessPaths: {},
+ exifData,
+ contentSize,
+ contentType,
+ };
+ const { pngs, jpgs } = AcceptibleMedia;
+ return new Promise<ImageUploadInformation>(async (resolve, reject) => {
+ const resizers = [
+ { resizer: sharp().rotate(), suffix: SizeSuffix.Original },
+ ...Object.values(Sizes).map(size => ({
+ resizer: sharp().resize(size.width, undefined, { withoutEnlargement: true }).rotate(),
+ suffix: size.suffix
+ }))
+ ];
+ if (pngs.includes(extension)) {
+ resizers.forEach(element => element.resizer = element.resizer.png());
+ } else if (jpgs.includes(extension)) {
+ resizers.forEach(element => element.resizer = element.resizer.jpeg());
+ }
+ for (let { resizer, suffix } of resizers) {
+ let mediaPath: string;
+ await new Promise<void>(resolve => {
+ const filename = InjectSize(resolved, suffix);
+ information.serverAccessPaths[suffix] = serverPathToFile(Directory.images, filename);
+ stream(normalizedUrl).pipe(resizer).pipe(fs.createWriteStream(serverPathToFile(Directory.images, filename)))
+ .on('close', resolve)
+ .on('error', reject);
+ });
}
- resolve({ data, error: reason });
+ if (isLocal) {
+ await new Promise<boolean>(resolve => {
+ fs.unlink(normalizedUrl, error => resolve(error === null));
+ });
+ }
+ resolve(information);
});
- });
-};
+ };
-export const createIfNotExists = async (path: string) => {
- if (await new Promise<boolean>(resolve => fs.exists(path, resolve))) {
- return true;
- }
- return new Promise<boolean>(resolve => fs.mkdir(path, error => resolve(error === null)));
-};
+ const classify = (url: string) => {
+ const isLocal = /Dash-Web(\\|\/)src(\\|\/)server(\\|\/)public(\\|\/)files/g.test(url);
+ return {
+ isLocal,
+ stream: isLocal ? fs.createReadStream : request,
+ normalized: isLocal ? path.normalize(url) : url
+ };
+ };
-export const Destroy = (mediaPath: string) => new Promise<boolean>(resolve => fs.unlink(mediaPath, error => resolve(error === null)));
+ const parseExifData = async (source: string): Promise<EnrichedExifData> => {
+ return new Promise<EnrichedExifData>(resolve => {
+ new ExifImage(source, (error, data) => {
+ let reason: Opt<string> = undefined;
+ if (error) {
+ reason = (error as any).code;
+ }
+ resolve({ data, error: reason });
+ });
+ });
+ };
} \ No newline at end of file
diff --git a/src/server/SharedMediaTypes.ts b/src/server/SharedMediaTypes.ts
index 3d3234125..8d0f441f0 100644
--- a/src/server/SharedMediaTypes.ts
+++ b/src/server/SharedMediaTypes.ts
@@ -1,9 +1,8 @@
-export namespace SharedMediaTypes {
-
+export namespace AcceptibleMedia {
export const gifs = [".gif"];
export const pngs = [".png"];
export const jpgs = [".jpg", ".jpeg"];
export const imageFormats = [...pngs, ...jpgs, ...gifs];
export const videoFormats = [".mov", ".mp4"];
-
+ export const applicationFormats = [".pdf"];
} \ No newline at end of file
diff --git a/src/server/database.ts b/src/server/database.ts
index b81fc03a4..db81245c1 100644
--- a/src/server/database.ts
+++ b/src/server/database.ts
@@ -293,7 +293,7 @@ export namespace Database {
};
export const QueryUploadHistory = async (contentSize: number) => {
- return SanitizedSingletonQuery<DashUploadUtils.UploadInformation>({ contentSize }, AuxiliaryCollections.GooglePhotosUploadHistory);
+ return SanitizedSingletonQuery<DashUploadUtils.ImageUploadInformation>({ contentSize }, AuxiliaryCollections.GooglePhotosUploadHistory);
};
export namespace GoogleAuthenticationToken {
@@ -322,7 +322,7 @@ export namespace Database {
}
- export const LogUpload = async (information: DashUploadUtils.UploadInformation) => {
+ export const LogUpload = async (information: DashUploadUtils.ImageUploadInformation) => {
const bundle = {
_id: Utils.GenerateDeterministicGuid(String(information.contentSize!)),
...information
diff --git a/src/server/index.ts b/src/server/index.ts
index d02a6005e..d77923710 100644
--- a/src/server/index.ts
+++ b/src/server/index.ts
@@ -24,7 +24,7 @@ import GooglePhotosManager from "./ApiManagers/GooglePhotosManager";
import DiagnosticManager from "./ApiManagers/DiagnosticManager";
export const publicDirectory = path.resolve(__dirname, "public");
-export const filesDirectory = path.resolve(publicDirectory, "files") + "/";
+export const filesDirectory = path.resolve(publicDirectory, "files");
/**
* These are the functions run before the server starts
@@ -34,7 +34,7 @@ export const filesDirectory = path.resolve(publicDirectory, "files") + "/";
async function preliminaryFunctions() {
await GoogleCredentialsLoader.loadCredentials();
GoogleApiServerUtils.processProjectCredentials();
- await DashUploadUtils.buildFilePartitions();
+ await DashUploadUtils.buildFileDirectories();
await log_execution({
startMessage: "attempting to initialize mongodb connection",
endMessage: "connection outcome determined",
generated by cgit v1.2.3-70-g09d2 (git 2.46.0) at 2025-10-06 14:24:04 +0000