aboutsummaryrefslogtreecommitdiff
path: root/src/server/ApiManagers/UploadManager.ts
diff options
context:
space:
mode:
Diffstat (limited to 'src/server/ApiManagers/UploadManager.ts')
-rw-r--r--src/server/ApiManagers/UploadManager.ts346
1 files changed, 128 insertions, 218 deletions
diff --git a/src/server/ApiManagers/UploadManager.ts b/src/server/ApiManagers/UploadManager.ts
index 2306b6589..4cb3d8baf 100644
--- a/src/server/ApiManagers/UploadManager.ts
+++ b/src/server/ApiManagers/UploadManager.ts
@@ -1,50 +1,27 @@
+import * as AdmZip from 'adm-zip';
import * as formidable from 'formidable';
-import { createReadStream, createWriteStream, unlink, writeFile } from 'fs';
-import * as path from 'path';
+import * as fs from 'fs';
+import { createReadStream, createWriteStream, unlink } from 'fs';
+import * as imageDataUri from 'image-data-uri';
import Jimp from 'jimp';
-import { filesDirectory, publicDirectory } from '..';
+import * as path from 'path';
+import * as uuid from 'uuid';
import { retrocycle } from '../../decycler/decycler';
+import { DashVersion } from '../../fields/DocSymbols';
import { DashUploadUtils, InjectSize, SizeSuffix } from '../DashUploadUtils';
-import { Database } from '../database';
import { Method, _success } from '../RouteManager';
-import RouteSubscriber from '../RouteSubscriber';
import { AcceptableMedia, Upload } from '../SharedMediaTypes';
+import { clientPathToFile, Directory, pathToDirectory, publicDirectory, serverPathToFile } from '../SocketData';
+import { Database } from '../database';
import ApiManager, { Registration } from './ApiManager';
import { SolrManager } from './SearchManager';
-import * as uuid from 'uuid';
-import { DashVersion } from '../../fields/DocSymbols';
-import * as AdmZip from 'adm-zip';
-import * as imageDataUri from 'image-data-uri';
-import * as fs from 'fs';
-
-export enum Directory {
- parsed_files = 'parsed_files',
- images = 'images',
- videos = 'videos',
- pdfs = 'pdfs',
- text = 'text',
- audio = 'audio',
- csv = 'csv',
-}
-
-export function serverPathToFile(directory: Directory, filename: string) {
- return path.normalize(`${filesDirectory}/${directory}/${filename}`);
-}
-
-export function pathToDirectory(directory: Directory) {
- return path.normalize(`${filesDirectory}/${directory}`);
-}
-
-export function clientPathToFile(directory: Directory, filename: string) {
- return `/files/${directory}/${filename}`;
-}
export default class UploadManager extends ApiManager {
protected initialize(register: Registration): void {
register({
method: Method.POST,
subscription: '/ping',
- secureHandler: async ({ req, res }) => {
+ secureHandler: async ({ /* req, */ res }) => {
_success(res, { message: DashVersion, date: new Date() });
},
});
@@ -78,31 +55,33 @@ export default class UploadManager extends ApiManager {
form.on('progress', e => fileguids.split(';').map(guid => DashUploadUtils.uploadProgress.set(guid, `read:(${Math.round((100 * +e) / +filesize)}%) ${e} of ${filesize}`)));
return new Promise<void>(resolve => {
form.parse(req, async (_err, _fields, files) => {
- const results: Upload.FileResponse[] = [];
if (_err?.message) {
- results.push({
- source: {
- filepath: '',
- originalFilename: 'none',
- newFilename: 'none',
- mimetype: 'text',
- size: 0,
- hashAlgorithm: 'md5',
- toJSON: () => ({ name: 'none', size: 0, length: 0, mtime: new Date(), filepath: '', originalFilename: 'none', newFilename: 'none', mimetype: 'text' }),
+ _success(res, [
+ {
+ source: {
+ filepath: '',
+ originalFilename: 'none',
+ newFilename: 'none',
+ mimetype: 'text',
+ size: 0,
+ hashAlgorithm: 'md5',
+ toJSON: () => ({ name: 'none', size: 0, length: 0, mtime: new Date(), filepath: '', originalFilename: 'none', newFilename: 'none', mimetype: 'text' }),
+ },
+ result: { name: 'failed upload', message: `${_err.message}` },
},
- result: { name: 'failed upload', message: `${_err.message}` },
- });
- }
- fileguids.split(';').map(guid => DashUploadUtils.uploadProgress.set(guid, `resampling images`));
+ ]);
+ } else {
+ fileguids.split(';').map(guid => DashUploadUtils.uploadProgress.set(guid, `resampling images`));
+ const results = (
+ await Promise.all(
+ Array.from(Object.keys(files)).map(
+ async key => (!files[key] ? undefined : DashUploadUtils.upload(files[key]![0] /* , key */)) // key is the guid used by the client to track upload progress.
+ )
+ )
+ ).filter(result => result && !(result.result instanceof Error));
- for (const key in files) {
- const f = files[key];
- if (f) {
- const result = await DashUploadUtils.upload(f[0], key); // key is the guid used by the client to track upload progress.
- result && !(result.result instanceof Error) && results.push(result);
- }
+ _success(res, results);
}
- _success(res, results);
resolve();
});
});
@@ -113,17 +92,14 @@ export default class UploadManager extends ApiManager {
method: Method.POST,
subscription: '/uploadYoutubeVideo',
secureHandler: async ({ req, res }) => {
- //req.readableBuffer.head.data
- return new Promise<void>(async resolve => {
- req.addListener('data', async args => {
- const payload = String.fromCharCode.apply(String, args);
- const { videoId, overwriteId } = JSON.parse(payload);
- const results: Upload.FileResponse[] = [];
- const result = await DashUploadUtils.uploadYoutube(videoId, overwriteId ?? videoId);
- result && results.push(result);
- _success(res, results);
- resolve();
- });
+ // req.readableBuffer.head.data
+ req.addListener('data', async args => {
+ const payload = String.fromCharCode(...args); // .apply(String, args);
+ const { videoId, overwriteId } = JSON.parse(payload);
+ const results: Upload.FileResponse[] = [];
+ const result = await DashUploadUtils.uploadYoutube(videoId, overwriteId ?? videoId);
+ result && results.push(result);
+ _success(res, results);
});
},
});
@@ -132,49 +108,10 @@ export default class UploadManager extends ApiManager {
method: Method.POST,
subscription: '/queryYoutubeProgress',
secureHandler: async ({ req, res }) => {
- return new Promise<void>(async resolve => {
- req.addListener('data', args => {
- const payload = String.fromCharCode.apply(String, args);
- const videoId = JSON.parse(payload).videoId;
- _success(res, { progress: DashUploadUtils.QueryYoutubeProgress(videoId, req.user) });
- resolve();
- });
- });
- },
- });
-
- register({
- method: Method.POST,
- subscription: new RouteSubscriber('youtubeScreenshot'),
- secureHandler: async ({ req, res }) => {
- const { id, timecode } = req.body;
- const convert = (raw: string) => {
- const number = Math.floor(Number(raw));
- const seconds = number % 60;
- const minutes = (number - seconds) / 60;
- return `${minutes}m${seconds}s`;
- };
- const suffix = timecode ? `&t=${convert(timecode)}` : ``;
- const targetUrl = `https://www.youtube.com/watch?v=${id}${suffix}`;
- const buffer = await captureYoutubeScreenshot(targetUrl);
- if (!buffer) {
- return res.send();
- }
- const resolvedName = `youtube_capture_${id}_${suffix}.png`;
- const resolvedPath = serverPathToFile(Directory.images, resolvedName);
- return new Promise<void>(resolve => {
- writeFile(resolvedPath, buffer, async error => {
- if (error) {
- return res.send();
- }
- await DashUploadUtils.outputResizedImages(resolvedPath, resolvedName, pathToDirectory(Directory.images));
- res.send({
- accessPaths: {
- agnostic: DashUploadUtils.getAccessPaths(Directory.images, resolvedName),
- },
- } as Upload.FileInformation);
- resolve();
- });
+ req.addListener('data', args => {
+ const payload = String.fromCharCode(...args); // .apply(String, args);
+ const { videoId } = JSON.parse(payload);
+ _success(res, { progress: DashUploadUtils.QueryYoutubeProgress(videoId) });
});
},
});
@@ -186,7 +123,8 @@ export default class UploadManager extends ApiManager {
const { sources } = req.body;
if (Array.isArray(sources)) {
const results = await Promise.all(sources.map(source => DashUploadUtils.UploadImage(source)));
- return res.send(results);
+ res.send(results);
+ return;
}
res.send();
},
@@ -203,20 +141,22 @@ export default class UploadManager extends ApiManager {
const getId = (id: string): string => {
if (!remap || id.endsWith('Proto')) return id;
if (id in ids) return ids[id];
- return (ids[id] = uuid.v4());
+ ids[id] = uuid.v4();
+ return ids[id];
};
- const mapFn = (doc: any) => {
+ const mapFn = (docIn: any) => {
+ const doc = docIn;
if (doc.id) {
doc.id = getId(doc.id);
}
+ // eslint-disable-next-line no-restricted-syntax
for (const key in doc.fields) {
- if (!doc.fields.hasOwnProperty(key)) {
- continue;
- }
+ // eslint-disable-next-line no-continue
+ if (!Object.prototype.hasOwnProperty.call(doc.fields, key)) continue;
+
const field = doc.fields[key];
- if (field === undefined || field === null) {
- continue;
- }
+ // eslint-disable-next-line no-continue
+ if (field === undefined || field === null) continue;
if (field.__type === 'Doc') {
mapFn(field);
@@ -229,78 +169,80 @@ export default class UploadManager extends ApiManager {
} else if (field.__type === 'list') {
mapFn(field);
} else if (typeof field === 'string') {
- const re = /("(?:dataD|d)ocumentId"\s*:\s*")([\w\-]*)"/g;
- doc.fields[key] = (field as any).replace(re, (match: any, p1: string, p2: string) => {
- return `${p1}${getId(p2)}"`;
- });
+ const re = /("(?:dataD|d)ocumentId"\s*:\s*")([\w-]*)"/g;
+ doc.fields[key] = (field as any).replace(re, (match: any, p1: string, p2: string) => `${p1}${getId(p2)}"`);
} else if (field.__type === 'RichTextField') {
const re = /("href"\s*:\s*")(.*?)"/g;
- field.Data = field.Data.replace(re, (match: any, p1: string, p2: string) => {
- return `${p1}${getId(p2)}"`;
- });
+ field.Data = field.Data.replace(re, (match: any, p1: string, p2: string) => `${p1}${getId(p2)}"`);
}
}
};
return new Promise<void>(resolve => {
form.parse(req, async (_err, fields, files) => {
- remap = Object.keys(fields).some(key => key === 'remap' && !fields.remap?.includes('false')); //.remap !== 'false'; // bcz: looking to see if the field 'remap' is set to 'false'
+ remap = Object.keys(fields).some(key => key === 'remap' && !fields.remap?.includes('false')); // .remap !== 'false'; // bcz: looking to see if the field 'remap' is set to 'false'
let id: string = '';
let docids: string[] = [];
let linkids: string[] = [];
try {
- for (const name in files) {
- const f = files[name];
- if (!f) continue;
- const path_2 = f[0]; // what about the rest of the array? are we guaranteed only one value is set?
- const zip = new AdmZip(path_2.filepath);
- zip.getEntries().forEach((entry: any) => {
- let entryName = entry.entryName.replace(/%%%/g, '/');
- if (!entryName.startsWith('files/')) {
- return;
- }
- const extension = path.extname(entryName);
- const pathname = publicDirectory + '/' + entry.entryName;
- const targetname = publicDirectory + '/' + entryName;
- try {
- zip.extractEntryTo(entry.entryName, publicDirectory, true, false);
- createReadStream(pathname).pipe(createWriteStream(targetname));
- Jimp.read(pathname).then(img => {
- DashUploadUtils.imageResampleSizes(extension).forEach(({ width, suffix }) => {
- const outputPath = InjectSize(targetname, suffix);
- if (!width) createReadStream(pathname).pipe(createWriteStream(outputPath));
- else img = img.resize(width, Jimp.AUTO).write(outputPath);
+ // eslint-disable-next-line no-restricted-syntax
+ for (const name in Object.keys(files)) {
+ if (Object.prototype.hasOwnProperty.call(files, name)) {
+ const f = files[name];
+ // eslint-disable-next-line no-continue
+ if (!f) continue;
+ const path2 = f[0]; // what about the rest of the array? are we guaranteed only one value is set?
+ const zip = new AdmZip(path2.filepath);
+ zip.getEntries().forEach((entry: any) => {
+ const entryName = entry.entryName.replace(/%%%/g, '/');
+ if (!entryName.startsWith('files/')) {
+ return;
+ }
+ const extension = path.extname(entryName);
+ const pathname = publicDirectory + '/' + entry.entryName;
+ const targetname = publicDirectory + '/' + entryName;
+ try {
+ zip.extractEntryTo(entry.entryName, publicDirectory, true, false);
+ createReadStream(pathname).pipe(createWriteStream(targetname));
+ Jimp.read(pathname).then(imgIn => {
+ let img = imgIn;
+ DashUploadUtils.imageResampleSizes(extension).forEach(({ width, suffix }) => {
+ const outputPath = InjectSize(targetname, suffix);
+ if (!width) createReadStream(pathname).pipe(createWriteStream(outputPath));
+ else img = img.resize(width, Jimp.AUTO).write(outputPath);
+ });
+ unlink(pathname, () => {});
});
- unlink(pathname, () => {});
- });
- } catch (e) {
- console.log(e);
- }
- });
- const json = zip.getEntry('docs.json');
- if (json) {
- try {
- const data = JSON.parse(json.getData().toString('utf8'), retrocycle());
- const { docs, links } = data;
- id = getId(data.id);
- const rdocs = Object.keys(docs).map(key => docs[key]);
- const ldocs = Object.keys(links).map(key => links[key]);
- [...rdocs, ...ldocs].forEach(mapFn);
- docids = rdocs.map(doc => doc.id);
- linkids = ldocs.map(link => link.id);
- await Promise.all(
- [...rdocs, ...ldocs].map(
- doc =>
- new Promise<void>(res => {
- // overwrite mongo doc with json doc contents
- Database.Instance.replace(doc.id, doc, (err, r) => res(err && console.log(err)), true);
- })
- )
- );
- } catch (e) {
- console.log(e);
+ } catch (e) {
+ console.log(e);
+ }
+ });
+ const json = zip.getEntry('docs.json');
+ if (json) {
+ try {
+ const data = JSON.parse(json.getData().toString('utf8'), retrocycle());
+ const { docs, links } = data;
+ id = getId(data.id);
+ const rdocs = Object.keys(docs).map(key => docs[key]);
+ const ldocs = Object.keys(links).map(key => links[key]);
+ [...rdocs, ...ldocs].forEach(mapFn);
+ docids = rdocs.map(doc => doc.id);
+ linkids = ldocs.map(link => link.id);
+ // eslint-disable-next-line no-await-in-loop
+ await Promise.all(
+ [...rdocs, ...ldocs].map(
+ doc =>
+ new Promise<void>(dbRes => {
+ // overwrite mongo doc with json doc contents
+ Database.Instance.replace(doc.id, doc, err => dbRes(err && console.log(err)), true);
+ })
+ )
+ );
+ } catch (e) {
+ console.log(e);
+ }
}
+ unlink(path2.filepath, () => {});
}
- unlink(path_2.filepath, () => {});
}
SolrManager.update();
res.send(JSON.stringify({ id, docids, linkids } || 'error'));
@@ -319,9 +261,8 @@ export default class UploadManager extends ApiManager {
secureHandler: async ({ req, res }) => {
const { source } = req.body;
if (typeof source === 'string') {
- return res.send(await DashUploadUtils.InspectImage(source));
- }
- res.send({});
+ res.send(await DashUploadUtils.InspectImage(source));
+ } else res.send({});
},
});
@@ -329,7 +270,7 @@ export default class UploadManager extends ApiManager {
method: Method.POST,
subscription: '/uploadURI',
secureHandler: ({ req, res }) => {
- const uri: any = req.body.uri;
+ const { uri } = req.body;
const filename = req.body.name;
const origSuffix = req.body.nosuffix ? SizeSuffix.None : SizeSuffix.Original;
const deleteFiles = req.body.replaceRootFilename;
@@ -338,23 +279,24 @@ export default class UploadManager extends ApiManager {
return;
}
if (deleteFiles) {
- const path = serverPathToFile(Directory.images, '');
+ const serverPath = serverPathToFile(Directory.images, '');
const regex = new RegExp(`${deleteFiles}.*`);
- fs.readdirSync(path)
+ fs.readdirSync(serverPath)
.filter((f: any) => regex.test(f))
- .map((f: any) => fs.unlinkSync(path + f));
+ .map((f: any) => fs.unlinkSync(serverPath + f));
}
- return imageDataUri.outputFile(uri, serverPathToFile(Directory.images, InjectSize(filename, origSuffix))).then((savedName: string) => {
+ imageDataUri.outputFile(uri, serverPathToFile(Directory.images, InjectSize(filename, origSuffix))).then((savedName: string) => {
const ext = path.extname(savedName).toLowerCase();
if (AcceptableMedia.imageFormats.includes(ext)) {
- Jimp.read(savedName).then(img =>
+ Jimp.read(savedName).then(imgIn => {
+ let img = imgIn;
(!origSuffix ? [{ width: 400, suffix: SizeSuffix.Medium }] : Object.values(DashUploadUtils.Sizes)) //
.forEach(({ width, suffix }) => {
const outputPath = serverPathToFile(Directory.images, InjectSize(filename, suffix) + ext);
if (!width) createReadStream(savedName).pipe(createWriteStream(outputPath));
else img = img.resize(width, Jimp.AUTO).write(outputPath);
- })
- );
+ });
+ });
}
res.send(clientPathToFile(Directory.images, filename + ext));
});
@@ -362,35 +304,3 @@ export default class UploadManager extends ApiManager {
});
}
}
-function delay(ms: number) {
- return new Promise(resolve => setTimeout(resolve, ms));
-}
-/**
- * On success, returns a buffer containing the bytes of a screenshot
- * of the video (optionally, at a timecode) specified by @param targetUrl.
- *
- * On failure, returns undefined.
- */
-async function captureYoutubeScreenshot(targetUrl: string) {
- // const browser = await launch({ args: ['--no-sandbox', '--disable-setuid-sandbox'] });
- // const page = await browser.newPage();
- // // await page.setViewport({ width: 1920, height: 1080 });
-
- // // await page.goto(targetUrl, { waitUntil: 'domcontentloaded' as any });
-
- // const videoPlayer = await page.$('.html5-video-player');
- // videoPlayer && await page.focus("video");
- // await delay(7000);
- // const ad = await page.$('.ytp-ad-skip-button-text');
- // await ad?.click();
- // await videoPlayer?.click();
- // await delay(1000);
- // // hide youtube player controls.
- // await page.evaluate(() => (document.querySelector('.ytp-chrome-bottom') as HTMLElement).style.display = 'none');
-
- // const buffer = await videoPlayer?.screenshot({ encoding: "binary" });
- // await browser.close();
-
- // return buffer;
- return null;
-}
generated by cgit v1.2.3-70-g09d2 (git 2.46.0) at 2025-10-06 18:48:49 +0000