import * as formidable from 'formidable'; import { createReadStream, createWriteStream, unlink, writeFile } from 'fs'; import { basename, dirname, extname, normalize } from 'path'; import * as sharp from 'sharp'; import { filesDirectory, publicDirectory } from '..'; import { retrocycle } from '../../decycler/decycler'; import { DashUploadUtils, InjectSize, SizeSuffix } from '../DashUploadUtils'; import { Database } from '../database'; import { Method, _success } from '../RouteManager'; import RouteSubscriber from '../RouteSubscriber'; import { AcceptableMedia, Upload } from '../SharedMediaTypes'; import ApiManager, { Registration } from './ApiManager'; import { SolrManager } from './SearchManager'; import v4 = require('uuid/v4'); import { DashVersion } from '../../fields/DocSymbols'; const AdmZip = require('adm-zip'); const imageDataUri = require('image-data-uri'); const fs = require('fs'); export enum Directory { parsed_files = 'parsed_files', images = 'images', videos = 'videos', pdfs = 'pdfs', text = 'text', pdf_thumbnails = 'pdf_thumbnails', audio = 'audio', csv = 'csv', } export function serverPathToFile(directory: Directory, filename: string) { return normalize(`${filesDirectory}/${directory}/${filename}`); } export function pathToDirectory(directory: Directory) { return normalize(`${filesDirectory}/${directory}`); } export function clientPathToFile(directory: Directory, filename: string) { return `/files/${directory}/${filename}`; } export default class UploadManager extends ApiManager { protected initialize(register: Registration): void { register({ method: Method.POST, subscription: '/ping', secureHandler: async ({ req, res }) => { _success(res, { message: DashVersion, date: new Date() }); }, }); register({ method: Method.POST, subscription: '/concatVideos', secureHandler: async ({ req, res }) => { // req.body contains the array of server paths to the videos _success(res, await DashUploadUtils.concatVideos(req.body)); }, }); register({ method: Method.POST, subscription: '/uploadFormData', secureHandler: async ({ req, res }) => { const form = new formidable.IncomingForm(); let fileguids = ''; let filesize = ''; form.on('field', (e: string, value: string) => { if (e === 'fileguids') { (fileguids = value).split(';').map(guid => DashUploadUtils.uploadProgress.set(guid, 'reading file')); } if (e === 'filesize') { filesize = value; } }); form.on('progress', e => fileguids.split(';').map(guid => DashUploadUtils.uploadProgress.set(guid, `read:(${Math.round((100 * +e) / +filesize)}%) ${e} of ${filesize}`))); form.keepExtensions = true; form.uploadDir = pathToDirectory(Directory.parsed_files); return new Promise(resolve => { form.parse(req, async (_err, _fields, files) => { const results: Upload.FileResponse[] = []; if (_err?.message) { results.push({ source: { size: 0, path: 'none', name: 'none', type: 'none', toJSON: () => ({ name: 'none', path: '' }), }, result: { name: 'failed upload', message: `${_err.message}` }, }); } for (const key in files) { const f = files[key]; if (!Array.isArray(f)) { const result = await DashUploadUtils.upload(f, key); // key is the guid used by the client to track upload progress. result && !(result.result instanceof Error) && results.push(result); } } _success(res, results); resolve(); }); }); }, }); register({ method: Method.POST, subscription: '/uploadYoutubeVideo', secureHandler: async ({ req, res }) => { //req.readableBuffer.head.data return new Promise(async resolve => { req.addListener('data', async args => { const payload = String.fromCharCode.apply(String, args); const { videoId, overwriteId } = JSON.parse(payload); const results: Upload.FileResponse[] = []; const result = await DashUploadUtils.uploadYoutube(videoId, overwriteId ?? videoId); result && results.push(result); _success(res, results); resolve(); }); }); }, }); register({ method: Method.POST, subscription: '/queryYoutubeProgress', secureHandler: async ({ req, res }) => { return new Promise(async resolve => { req.addListener('data', args => { const payload = String.fromCharCode.apply(String, args); const videoId = JSON.parse(payload).videoId; _success(res, { progress: DashUploadUtils.QueryYoutubeProgress(videoId, req.user) }); resolve(); }); }); }, }); register({ method: Method.POST, subscription: new RouteSubscriber('youtubeScreenshot'), secureHandler: async ({ req, res }) => { const { id, timecode } = req.body; const convert = (raw: string) => { const number = Math.floor(Number(raw)); const seconds = number % 60; const minutes = (number - seconds) / 60; return `${minutes}m${seconds}s`; }; const suffix = timecode ? `&t=${convert(timecode)}` : ``; const targetUrl = `https://www.youtube.com/watch?v=${id}${suffix}`; const buffer = await captureYoutubeScreenshot(targetUrl); if (!buffer) { return res.send(); } const resolvedName = `youtube_capture_${id}_${suffix}.png`; const resolvedPath = serverPathToFile(Directory.images, resolvedName); return new Promise(resolve => { writeFile(resolvedPath, buffer, async error => { if (error) { return res.send(); } await DashUploadUtils.outputResizedImages(() => createReadStream(resolvedPath), resolvedName, pathToDirectory(Directory.images)); res.send({ accessPaths: { agnostic: DashUploadUtils.getAccessPaths(Directory.images, resolvedName), }, } as Upload.FileInformation); resolve(); }); }); }, }); register({ method: Method.POST, subscription: '/uploadRemoteImage', secureHandler: async ({ req, res }) => { const { sources } = req.body; if (Array.isArray(sources)) { const results = await Promise.all(sources.map(source => DashUploadUtils.UploadImage(source))); return res.send(results); } res.send(); }, }); register({ method: Method.POST, subscription: '/uploadDoc', secureHandler: ({ req, res }) => { const form = new formidable.IncomingForm(); form.keepExtensions = true; // let path = req.body.path; const ids: { [id: string]: string } = {}; let remap = true; const getId = (id: string): string => { if (!remap || id.endsWith('Proto')) return id; if (id in ids) return ids[id]; return (ids[id] = v4()); }; const mapFn = (doc: any) => { if (doc.id) { doc.id = getId(doc.id); } for (const key in doc.fields) { if (!doc.fields.hasOwnProperty(key)) { continue; } const field = doc.fields[key]; if (field === undefined || field === null) { continue; } if (field.__type === 'Doc') { mapFn(field); } else if (field.__type === 'proxy' || field.__type === 'prefetch_proxy') { field.fieldId = getId(field.fieldId); } else if (field.__type === 'script' || field.__type === 'computed') { if (field.captures) { field.captures.fieldId = getId(field.captures.fieldId); } } else if (field.__type === 'list') { mapFn(field); } else if (typeof field === 'string') { const re = /("(?:dataD|d)ocumentId"\s*:\s*")([\w\-]*)"/g; doc.fields[key] = (field as any).replace(re, (match: any, p1: string, p2: string) => { return `${p1}${getId(p2)}"`; }); } else if (field.__type === 'RichTextField') { const re = /("href"\s*:\s*")(.*?)"/g; field.Data = field.Data.replace(re, (match: any, p1: string, p2: string) => { return `${p1}${getId(p2)}"`; }); } } }; return new Promise(resolve => { form.parse(req, async (_err, fields, files) => { remap = fields.remap !== 'false'; let id: string = ''; let docids: string[] = []; let linkids: string[] = []; try { for (const name in files) { const f = files[name]; const path_2 = Array.isArray(f) ? '' : f.path; const zip = new AdmZip(path_2); zip.getEntries().forEach((entry: any) => { let entryName = entry.entryName.replace(/%%%/g, '/'); if (!entryName.startsWith('files/')) { return; } const extension = extname(entryName); const pathname = publicDirectory + '/' + entry.entryName; const targetname = publicDirectory + '/' + entryName; try { zip.extractEntryTo(entry.entryName, publicDirectory, true, false); createReadStream(pathname).pipe(createWriteStream(targetname)); if (extension !== '.pdf') { const { pngs, jpgs } = AcceptableMedia; const resizers = [ { resizer: sharp().resize(100, undefined, { withoutEnlargement: true }), suffix: SizeSuffix.Small }, { resizer: sharp().resize(400, undefined, { withoutEnlargement: true }), suffix: SizeSuffix.Medium }, { resizer: sharp().resize(900, undefined, { withoutEnlargement: true }), suffix: SizeSuffix.Large }, ]; let isImage = false; if (pngs.includes(extension)) { resizers.forEach(element => { element.resizer = element.resizer.png(); }); isImage = true; } else if (jpgs.includes(extension)) { resizers.forEach(element => { element.resizer = element.resizer.jpeg(); }); isImage = true; } if (isImage) { resizers.forEach(resizer => { createReadStream(pathname) .on('error', e => console.log('Resizing read:' + e)) .pipe(resizer.resizer) .on('error', e => console.log('Resizing write: ' + e)) .pipe(createWriteStream(targetname.replace('_o' + extension, resizer.suffix + extension)).on('error', e => console.log('Resizing write: ' + e))); }); } } unlink(pathname, () => {}); } catch (e) { console.log(e); } }); const json = zip.getEntry('docs.json'); try { const data = JSON.parse(json.getData().toString('utf8'), retrocycle()); const { docs, links } = data; id = getId(data.id); const rdocs = Object.keys(docs).map(key => docs[key]); const ldocs = Object.keys(links).map(key => links[key]); [...rdocs, ...ldocs].forEach(mapFn); docids = rdocs.map(doc => doc.id); linkids = ldocs.map(link => link.id); await Promise.all( [...rdocs, ...ldocs].map( doc => new Promise(res => { // overwrite mongo doc with json doc contents Database.Instance.replace(doc.id, doc, (err, r) => res(err && console.log(err)), true); }) ) ); } catch (e) { console.log(e); } unlink(path_2, () => {}); } SolrManager.update(); res.send(JSON.stringify({ id, docids, linkids } || 'error')); } catch (e) { console.log(e); } resolve(); }); }); }, }); register({ method: Method.POST, subscription: '/inspectImage', secureHandler: async ({ req, res }) => { const { source } = req.body; if (typeof source === 'string') { return res.send(await DashUploadUtils.InspectImage(source)); } res.send({}); }, }); register({ method: Method.POST, subscription: '/uploadURI', secureHandler: ({ req, res }) => { const uri = req.body.uri; const filename = req.body.name; const origSuffix = req.body.nosuffix ? SizeSuffix.None : SizeSuffix.Original; const deleteFiles = req.body.replaceRootFilename; if (!uri || !filename) { res.status(401).send('incorrect parameters specified'); return; } if (deleteFiles) { const path = serverPathToFile(Directory.images, ''); const regex = new RegExp(`${deleteFiles}.*`); fs.readdirSync(path) .filter((f: any) => regex.test(f)) .map((f: any) => fs.unlinkSync(path + f)); } return imageDataUri.outputFile(uri, serverPathToFile(Directory.images, InjectSize(filename, origSuffix))).then((savedName: string) => { const ext = extname(savedName).toLowerCase(); const { pngs, jpgs } = AcceptableMedia; const resizers = !origSuffix ? [{ resizer: sharp().resize(400, undefined, { withoutEnlargement: true }), suffix: SizeSuffix.Medium }] : [ { resizer: sharp().resize(100, undefined, { withoutEnlargement: true }), suffix: SizeSuffix.Small }, { resizer: sharp().resize(400, undefined, { withoutEnlargement: true }), suffix: SizeSuffix.Medium }, { resizer: sharp().resize(900, undefined, { withoutEnlargement: true }), suffix: SizeSuffix.Large }, ]; let isImage = false; if (pngs.includes(ext)) { resizers.forEach(element => { element.resizer = element.resizer.png(); }); isImage = true; } else if (jpgs.includes(ext)) { resizers.forEach(element => { element.resizer = element.resizer.jpeg(); }); isImage = true; } if (isImage) { resizers.forEach(resizer => { const path = serverPathToFile(Directory.images, InjectSize(filename, resizer.suffix) + ext); createReadStream(savedName) .on('error', e => console.log('Resizing read:' + e)) .pipe(resizer.resizer) .on('error', e => console.log('Resizing write: ' + e)) .pipe(createWriteStream(path).on('error', e => console.log('Resizing write: ' + e))); }); } res.send(clientPathToFile(Directory.images, filename + ext)); }); }, }); } } function delay(ms: number) { return new Promise(resolve => setTimeout(resolve, ms)); } /** * On success, returns a buffer containing the bytes of a screenshot * of the video (optionally, at a timecode) specified by @param targetUrl. * * On failure, returns undefined. */ async function captureYoutubeScreenshot(targetUrl: string) { // const browser = await launch({ args: ['--no-sandbox', '--disable-setuid-sandbox'] }); // const page = await browser.newPage(); // // await page.setViewport({ width: 1920, height: 1080 }); // // await page.goto(targetUrl, { waitUntil: 'domcontentloaded' as any }); // const videoPlayer = await page.$('.html5-video-player'); // videoPlayer && await page.focus("video"); // await delay(7000); // const ad = await page.$('.ytp-ad-skip-button-text'); // await ad?.click(); // await videoPlayer?.click(); // await delay(1000); // // hide youtube player controls. // await page.evaluate(() => (document.querySelector('.ytp-chrome-bottom') as HTMLElement).style.display = 'none'); // const buffer = await videoPlayer?.screenshot({ encoding: "binary" }); // await browser.close(); // return buffer; return null; }