import * as AdmZip from 'adm-zip'; import * as formidable from 'formidable'; import * as fs from 'fs'; import { unlink } from 'fs'; import * as imageDataUri from 'image-data-uri'; import * as path from 'path'; import * as uuid from 'uuid'; import { retrocycle } from '../../decycler/decycler'; import { DashVersion } from '../../fields/DocSymbols'; import { DashUploadUtils, InjectSize, SizeSuffix, workerResample } from '../DashUploadUtils'; import { Method, _success } from '../RouteManager'; import { AcceptableMedia, Upload } from '../SharedMediaTypes'; import { Directory, clientPathToFile, pathToDirectory, publicDirectory, serverPathToFile } from '../SocketData'; import { Database } from '../database'; import ApiManager, { Registration } from './ApiManager'; export default class UploadManager extends ApiManager { protected initialize(register: Registration): void { register({ method: Method.POST, subscription: '/ping', secureHandler: async ({ /* req, */ res }) => { _success(res, { message: DashVersion, date: new Date() }); }, }); register({ method: Method.POST, subscription: '/concatVideos', secureHandler: async ({ req, res }) => { // req.body contains the array of server paths to the videos _success(res, await DashUploadUtils.concatVideos(req.body)); }, }); register({ method: Method.POST, subscription: '/uploadFormData', secureHandler: async ({ req, res }) => { const form = new formidable.IncomingForm({ keepExtensions: true, uploadDir: pathToDirectory(Directory.parsed_files) }); let fileguids = ''; let filesize = ''; form.on('field', (e: string, value: string) => { if (e === 'fileguids') { (fileguids = value).split(';').map(guid => DashUploadUtils.uploadProgress.set(guid, 'reading file')); } if (e === 'filesize') { filesize = value; } }); fileguids.split(';').map(guid => DashUploadUtils.uploadProgress.set(guid, `upload starting`)); form.on('progress', e => fileguids.split(';').map(guid => DashUploadUtils.uploadProgress.set(guid, `read:(${Math.round((100 * +e) / +filesize)}%) ${e} of ${filesize}`))); return new Promise(resolve => { form.parse(req, async (_err, _fields, files) => { if (_err?.message) { _success(res, [ { source: { filepath: '', originalFilename: 'none', newFilename: 'none', mimetype: 'text', size: 0, hashAlgorithm: 'md5', toJSON: () => ({ name: 'none', size: 0, length: 0, mtime: new Date(), filepath: '', originalFilename: 'none', newFilename: 'none', mimetype: 'text' }), }, result: { name: 'failed upload', message: `${_err.message}` }, }, ]); } else { fileguids.split(';').map(guid => DashUploadUtils.uploadProgress.set(guid, `resampling images`)); const results = ( await Promise.all( Array.from(Object.keys(files)).map( async key => (!files[key] ? undefined : DashUploadUtils.upload(files[key]![0] /* , key */)) // key is the guid used by the client to track upload progress. ) ) ).filter(result => result && !(result.result instanceof Error)); _success(res, results); } resolve(); }); }); }, }); register({ method: Method.POST, subscription: '/uploadYoutubeVideo', secureHandler: async ({ req, res }) => { // req.readableBuffer.head.data req.addListener('data', async args => { const payload = String.fromCharCode(...args); // .apply(String, args); const { videoId, overwriteId } = JSON.parse(payload); const results: Upload.FileResponse[] = []; const result = await DashUploadUtils.uploadYoutube(videoId, overwriteId ?? videoId); result && results.push(result); _success(res, results); }); }, }); register({ method: Method.POST, subscription: '/queryYoutubeProgress', secureHandler: async ({ req, res }) => { req.addListener('data', args => { const payload = String.fromCharCode(...args); // .apply(String, args); const { videoId } = JSON.parse(payload); _success(res, { progress: DashUploadUtils.QueryYoutubeProgress(videoId) }); }); }, }); register({ method: Method.POST, subscription: '/uploadRemoteImage', secureHandler: async ({ req, res }) => { const { sources } = req.body; if (Array.isArray(sources)) { const results = await Promise.all(sources.map(source => DashUploadUtils.UploadImage(source))); res.send(results); return; } res.send(); }, }); register({ method: Method.POST, subscription: '/uploadDoc', secureHandler: ({ req, res }) => { const form = new formidable.IncomingForm({ keepExtensions: true }); // let path = req.body.path; const ids: { [id: string]: string } = {}; let remap = true; const getId = (id: string): string => { if (!remap || id.endsWith('Proto')) return id; if (id in ids) return ids[id]; ids[id] = uuid.v4(); return ids[id]; }; const mapFn = (docIn: { id: string; fields: any[] }) => { const doc = docIn; if (doc.id) { doc.id = getId(doc.id); } // eslint-disable-next-line no-restricted-syntax for (const key in doc.fields) { // eslint-disable-next-line no-continue if (!Object.prototype.hasOwnProperty.call(doc.fields, key)) continue; const field = doc.fields[key]; // eslint-disable-next-line no-continue if (field === undefined || field === null) continue; if (field.__type === 'Doc') { mapFn(field); } else if (field.__type === 'proxy' || field.__type === 'prefetch_proxy') { field.fieldId = getId(field.fieldId); } else if (field.__type === 'script' || field.__type === 'computed') { if (field.captures) { field.captures.fieldId = getId(field.captures.fieldId); } } else if (field.__type === 'list') { mapFn(field); } else if (typeof field === 'string') { const re = /("(?:dataD|d)ocumentId"\s*:\s*")([\w-]*)"/g; doc.fields[key] = field.replace(re, (match: string, p1: string, p2: string) => `${p1}${getId(p2)}"`); } else if (field.__type === 'RichTextField') { const re = /("href"\s*:\s*")(.*?)"/g; field.Data = field.Data.replace(re, (match: string, p1: string, p2: string) => `${p1}${getId(p2)}"`); } } }; return new Promise(resolve => { form.parse(req, async (_err, fields, files) => { remap = Object.keys(fields).some(key => key === 'remap' && !fields.remap?.includes('false')); // .remap !== 'false'; // bcz: looking to see if the field 'remap' is set to 'false' let id: string = ''; let docids: string[] = []; let linkids: string[] = []; try { // eslint-disable-next-line no-restricted-syntax for (const name in files) { if (Object.prototype.hasOwnProperty.call(files, name)) { const f = files[name]; // eslint-disable-next-line no-continue if (!f) continue; const path2 = f[0]; // what about the rest of the array? are we guaranteed only one value is set? const zip = new AdmZip(path2.filepath); zip.getEntries().forEach(entry => { const entryName = entry.entryName.replace(/%%%/g, '/'); if (entryName.startsWith('files/')) { const pathname = publicDirectory + '/' + entry.entryName; const targetname = publicDirectory + '/' + entryName; try { zip.extractEntryTo(entry.entryName, publicDirectory, true, false); const extension = path.extname(targetname).toLowerCase(); const basefilename = targetname.substring(0, targetname.length - extension.length); workerResample(pathname, basefilename.replace(/_o$/, '') + extension, SizeSuffix.Original, true); } catch (e) { console.log(e); } } }); const json = zip.getEntry('docs.json'); if (json) { try { const data = JSON.parse(json.getData().toString('utf8'), retrocycle()); const { docs, links } = data; id = getId(data.id); const rdocs = Object.keys(docs).map(key => docs[key]); const ldocs = Object.keys(links).map(key => links[key]); [...rdocs, ...ldocs].forEach(mapFn); docids = rdocs.map(doc => doc.id); linkids = ldocs.map(link => link.id); // eslint-disable-next-line no-await-in-loop await Promise.all( [...rdocs, ...ldocs].map( doc => new Promise(dbRes => { // overwrite mongo doc with json doc contents Database.Instance.replace(doc.id, doc, err => dbRes(err && console.log(err)), true); }) ) ); } catch (e) { console.log(e); } } unlink(path2.filepath, () => {}); } } res.send(JSON.stringify({ id, docids, linkids }) || 'error'); } catch (e) { console.log(e); } resolve(); }); }); }, }); register({ method: Method.POST, subscription: '/inspectImage', secureHandler: async ({ req, res }) => { const { source } = req.body; if (typeof source === 'string') { res.send(await DashUploadUtils.InspectImage(source)); } else res.send({}); }, }); register({ method: Method.POST, subscription: '/uploadURI', secureHandler: ({ req, res }) => { const { uri } = req.body; const filename = req.body.name; const origSuffix = req.body.nosuffix ? SizeSuffix.None : SizeSuffix.Original; const deleteFiles = req.body.replaceRootFilename; if (!uri || !filename) { res.status(401).send('incorrect parameters specified'); return; } if (deleteFiles) { const serverPath = serverPathToFile(Directory.images, ''); const regex = new RegExp(`${deleteFiles}.*`); fs.readdirSync(serverPath) .filter(f => regex.test(f)) .map(f => fs.unlinkSync(serverPath + f)); } imageDataUri.outputFile(uri, serverPathToFile(Directory.images, InjectSize(filename, origSuffix))).then((savedName: string) => { const ext = path.extname(savedName).toLowerCase(); const outputPath = serverPathToFile(Directory.images, filename + ext); if (AcceptableMedia.imageFormats.includes(ext)) { workerResample(savedName, outputPath, origSuffix, false); } res.send(clientPathToFile(Directory.images, filename + ext)); }); }, }); } }