diff options
Diffstat (limited to 'src/server/DashUploadUtils.ts')
| -rw-r--r-- | src/server/DashUploadUtils.ts | 161 |
1 files changed, 77 insertions, 84 deletions
diff --git a/src/server/DashUploadUtils.ts b/src/server/DashUploadUtils.ts index 19cb3f240..cd9b635bd 100644 --- a/src/server/DashUploadUtils.ts +++ b/src/server/DashUploadUtils.ts @@ -5,8 +5,7 @@ import { File } from 'formidable'; import { createReadStream, createWriteStream, existsSync, readFileSync, rename, unlinkSync, writeFile } from 'fs'; import * as path from 'path'; import { basename } from 'path'; -import * as sharp from 'sharp'; -import { Readable, Stream } from 'stream'; +import Jimp from 'jimp'; import { filesDirectory, publicDirectory } from '.'; import { Opt } from '../fields/Doc'; import { ParsedPDF } from '../server/PdfTypes'; @@ -15,8 +14,8 @@ import { createIfNotExists } from './ActionUtilities'; import { clientPathToFile, Directory, pathToDirectory, serverPathToFile } from './ApiManagers/UploadManager'; import { resolvedServerUrl } from './server_Initialization'; import { AcceptableMedia, Upload } from './SharedMediaTypes'; -import request = require('request-promise'); -import formidable = require('formidable'); +import * as request from 'request-promise'; +import * as formidable from 'formidable'; import { AzureManager } from './ApiManagers/AzureManager'; import axios from 'axios'; const spawn = require('child_process').spawn; @@ -26,6 +25,7 @@ const ffmpeg = require('fluent-ffmpeg'); const fs = require('fs'); const requestImageSize = require('../client/util/request-image-size'); const md5File = require('md5-file'); +const autorotate = require('jpeg-autorotate'); export enum SizeSuffix { Small = '_s', @@ -55,9 +55,9 @@ export namespace DashUploadUtils { } export const Sizes: { [size: string]: Size } = { - SMALL: { width: 100, suffix: SizeSuffix.Small }, + LARGE: { width: 800, suffix: SizeSuffix.Large }, MEDIUM: { width: 400, suffix: SizeSuffix.Medium }, - LARGE: { width: 900, suffix: SizeSuffix.Large }, + SMALL: { width: 100, suffix: SizeSuffix.Small }, }; export function validateExtension(url: string) { @@ -120,14 +120,14 @@ export namespace DashUploadUtils { }; } - function resolveExistingFile(name: string, pat: string, directory: Directory, type?: string, duration?: number, rawText?: string) { - const data = { size: 0, path: path.basename(pat), name, type: type ?? '' }; - const file = { ...data, toJSON: () => ({ ...data, filename: data.path.replace(/.*\//, ''), mtime: duration?.toString(), mime: '', toJson: () => undefined as any }) }; + function resolveExistingFile(name: string, pat: string, directory: Directory, mimetype?: string | null, duration?: number, rawText?: string): Upload.FileResponse<Upload.FileInformation> { + const data = { size: 0, filepath: pat, name, type: mimetype ?? '', originalFilename: name, newFilename: path.basename(pat), mimetype: mimetype || null, hashAlgorithm: false as any }; + const file = { ...data, toJSON: () => ({ ...data, length: 0, filename: data.filepath.replace(/.*\//, ''), mtime: new Date(), mimetype: mimetype || null, toJson: () => undefined as any }) }; return { - source: file, + source: file || null, result: { accessPaths: { - agnostic: getAccessPaths(directory, data.path), + agnostic: getAccessPaths(directory, data.filepath), }, rawText, duration, @@ -145,8 +145,8 @@ export namespace DashUploadUtils { export function uploadYoutube(videoId: string, overwriteId: string): Promise<Upload.FileResponse> { return new Promise<Upload.FileResponse<Upload.FileInformation>>((res, rej) => { const name = videoId; - const path = name.replace(/^-/, '__') + '.mp4'; - const finalPath = serverPathToFile(Directory.videos, path); + const filepath = name.replace(/^-/, '__') + '.mp4'; + const finalPath = serverPathToFile(Directory.videos, filepath); if (existsSync(finalPath)) { uploadProgress.set(overwriteId, 'computing duration'); exec(`yt-dlp -o ${finalPath} "https://www.youtube.com/watch?v=${videoId}" --get-duration`, (error: any, stdout: any, stderr: any) => { @@ -156,7 +156,7 @@ export namespace DashUploadUtils { }); } else { uploadProgress.set(overwriteId, 'starting download'); - const ytdlp = spawn(`yt-dlp`, ['-o', path, `https://www.youtube.com/watch?v=${videoId}`, '--max-filesize', '100M', '-f', 'mp4']); + const ytdlp = spawn(`yt-dlp`, ['-o', filepath, `https://www.youtube.com/watch?v=${videoId}`, '--max-filesize', '100M', '-f', 'mp4']); ytdlp.stdout.on('data', (data: any) => uploadProgress.set(overwriteId, data.toString())); @@ -171,20 +171,22 @@ export namespace DashUploadUtils { res({ source: { size: 0, - path, - name, - type: '', - toJSON: () => ({ name, path }), + filepath: name, + originalFilename: name, + newFilename: name, + mimetype: 'video', + hashAlgorithm: 'md5', + toJSON: () => ({ newFilename: name, filepath, mimetype: 'video', mtime: new Date(), size: 0, length: 0, originalFilename: name }), }, result: { name: 'failed youtube query', message: `Could not archive video. ${code ? errors : uploadProgress.get(videoId)}` }, }); } else { uploadProgress.set(overwriteId, 'computing duration'); - exec(`yt-dlp-o ${path} "https://www.youtube.com/watch?v=${videoId}" --get-duration`, (error: any, stdout: any, stderr: any) => { + exec(`yt-dlp-o ${filepath} "https://www.youtube.com/watch?v=${videoId}" --get-duration`, (error: any, stdout: any, stderr: any) => { const time = Array.from(stdout.trim().split(':')).reverse(); const duration = (time.length > 2 ? Number(time[2]) * 1000 * 60 : 0) + (time.length > 1 ? Number(time[1]) * 60 : 0) + (time.length > 0 ? Number(time[0]) : 0); - const data = { size: 0, path, name, type: 'video/mp4' }; - const file = { ...data, toJSON: () => ({ ...data, filename: data.path.replace(/.*\//, ''), mtime: duration.toString(), mime: '', toJson: () => undefined as any }) }; + const data = { size: 0, filepath, name, mimetype: '', originalFilename: name, newFilename: name, hashAlgorithm: 'md5' as 'md5', type: 'video/mp4' }; + const file = { ...data, toJSON: () => ({ ...data, length: 0, filename: data.filepath.replace(/.*\//, ''), mtime: new Date(), mime: '', hashAlgorithm: 'md5' as 'md5', toJson: () => undefined as any }) }; res(MoveParsedFile(file, Directory.videos)); }); } @@ -195,18 +197,18 @@ export namespace DashUploadUtils { export async function upload(file: File, overwriteGuid?: string): Promise<Upload.FileResponse> { const isAzureOn = usingAzure(); - const { type, path, name } = file; + const { mimetype: type, filepath, originalFilename } = file; const types = type?.split('/') ?? []; - uploadProgress.set(overwriteGuid ?? name, 'uploading'); // If the client sent a guid it uses to track upload progress, use that guid. Otherwise, use the file's name. + // uploadProgress.set(overwriteGuid ?? name, 'uploading'); // If the client sent a guid it uses to track upload progress, use that guid. Otherwise, use the file's name. const category = types[0]; let format = `.${types[1]}`; - console.log(green(`Processing upload of file (${name}) and format (${format}) with upload type (${type}) in category (${category}).`)); + console.log(green(`Processing upload of file (${originalFilename}) and format (${format}) with upload type (${type}) in category (${category}).`)); switch (category) { case 'image': if (imageFormats.includes(format)) { - const result = await UploadImage(path, basename(path)); + const result = await UploadImage(filepath, basename(filepath)); return { source: file, result }; } fs.unlink(path, () => {}); @@ -215,19 +217,19 @@ export namespace DashUploadUtils { if (format.includes('x-matroska')) { console.log('case video'); await new Promise(res => - ffmpeg(file.path) + ffmpeg(file.filepath) .videoCodec('copy') // this will copy the data instead of reencode it - .save(file.path.replace('.mkv', '.mp4')) + .save(file.filepath.replace('.mkv', '.mp4')) .on('end', res) .on('error', (e: any) => console.log(e)) ); - file.path = file.path.replace('.mkv', '.mp4'); + file.filepath = file.filepath.replace('.mkv', '.mp4'); format = '.mp4'; } if (format.includes('quicktime')) { let abort = false; await new Promise<void>(res => - ffmpeg.ffprobe(file.path, (err: any, metadata: any) => { + ffmpeg.ffprobe(file.filepath, (err: any, metadata: any) => { if (metadata.streams.some((stream: any) => stream.codec_name === 'hevc')) { abort = true; } @@ -275,24 +277,26 @@ export namespace DashUploadUtils { } } - console.log(red(`Ignoring unsupported file (${name}) with upload type (${type}).`)); + console.log(red(`Ignoring unsupported file (${originalFilename}) with upload type (${type}).`)); fs.unlink(path, () => {}); - return { source: file, result: new Error(`Could not upload unsupported file (${name}) with upload type (${type}).`) }; + return { source: file, result: new Error(`Could not upload unsupported file (${originalFilename}) with upload type (${type}).`) }; } async function UploadPdf(file: File) { - const fileKey = (await md5File(file.path)) + '.pdf'; + const fileKey = (await md5File(file.filepath)) + '.pdf'; const textFilename = `${fileKey.substring(0, fileKey.length - 4)}.txt`; if (fExists(fileKey, Directory.pdfs) && fExists(textFilename, Directory.text)) { - fs.unlink(file.path, () => {}); + fs.unlink(file.filepath, () => {}); return new Promise<Upload.FileResponse>(res => { const textFilename = `${fileKey.substring(0, fileKey.length - 4)}.txt`; const readStream = createReadStream(serverPathToFile(Directory.text, textFilename)); var rawText = ''; - readStream.on('data', chunk => (rawText += chunk.toString())).on('end', () => res(resolveExistingFile(file.name, fileKey, Directory.pdfs, file.type, undefined, rawText))); + readStream + .on('data', chunk => (rawText += chunk.toString())) // + .on('end', () => res(resolveExistingFile(file.originalFilename ?? '', fileKey, Directory.pdfs, file.mimetype, undefined, rawText))); }); } - const dataBuffer = readFileSync(file.path); + const dataBuffer = readFileSync(file.filepath); const result: ParsedPDF | any = await parse(dataBuffer).catch((e: any) => e); if (!result.code) { await new Promise<void>((resolve, reject) => { @@ -301,11 +305,11 @@ export namespace DashUploadUtils { }); return MoveParsedFile(file, Directory.pdfs, undefined, result?.text, undefined, fileKey); } - return { source: file, result: { name: 'faile pdf pupload', message: `Could not upload (${file.name}).${result.message}` } }; + return { source: file, result: { name: 'faile pdf pupload', message: `Could not upload (${file.originalFilename}).${result.message}` } }; } async function UploadCsv(file: File) { - const { path: sourcePath } = file; + const { filepath: sourcePath } = file; // read the file as a string const data = readFileSync(sourcePath, 'utf8'); // split the string into an array of lines @@ -367,7 +371,7 @@ export namespace DashUploadUtils { } export interface ImageResizer { - resizer?: sharp.Sharp; + width: number; suffix: SizeSuffix; } @@ -463,12 +467,12 @@ export namespace DashUploadUtils { * to appear in the new location */ export async function MoveParsedFile(file: formidable.File, destination: Directory, suffix: string | undefined = undefined, text?: string, duration?: number, targetName?: string): Promise<Upload.FileResponse> { - const { path: sourcePath } = file; - let name = targetName ?? path.basename(sourcePath); + const { filepath } = file; + let name = targetName ?? path.basename(filepath); suffix && (name += suffix); return new Promise(resolve => { const destinationPath = serverPathToFile(destination, name); - rename(sourcePath, destinationPath, error => { + rename(filepath, destinationPath, error => { resolve({ source: file, result: error @@ -540,7 +544,7 @@ export namespace DashUploadUtils { writtenFiles = {}; } } else { - writtenFiles = await outputResizedImages(() => request(requestable), resolved, pathToDirectory(Directory.images)); + writtenFiles = await outputResizedImages(metadata.source, resolved, pathToDirectory(Directory.images)); } for (const suffix of Object.keys(writtenFiles)) { information.accessPaths[suffix] = getAccessPaths(images, writtenFiles[suffix]); @@ -595,57 +599,46 @@ export namespace DashUploadUtils { * @param outputDirectory the directory to output to, usually Directory.Images * @returns a map with suffixes as keys and resized filenames as values. */ - export async function outputResizedImages(streamProvider: () => Stream | Promise<Stream>, outputFileName: string, outputDirectory: string) { + export async function outputResizedImages(sourcePath: string, outputFileName: string, outputDirectory: string) { const writtenFiles: { [suffix: string]: string } = {}; - for (const { resizer, suffix } of resizers(path.extname(outputFileName))) { - const outputPath = path.resolve(outputDirectory, (writtenFiles[suffix] = InjectSize(outputFileName, suffix))); - await new Promise<void>(async (resolve, reject) => { - const source = streamProvider(); - let readStream = source instanceof Promise ? await source : source; - let error = false; - if (resizer) { - readStream = readStream.pipe(resizer.withMetadata()).on('error', async args => { - error = true; - if (error) { - const source2 = streamProvider(); - let readStream2: Stream | undefined; - readStream2 = source2 instanceof Promise ? await source2 : source2; - readStream2?.pipe(createWriteStream(outputPath)).on('error', resolve).on('close', resolve); - } - }); - } - !error && readStream?.pipe(createWriteStream(outputPath)).on('error', resolve).on('close', resolve); + const sizes = imageResampleSizes(path.extname(outputFileName)); + const outputPath = (suffix: SizeSuffix) => path.resolve(outputDirectory, (writtenFiles[suffix] = InjectSize(outputFileName, suffix))); + await Promise.all( + sizes.filter(({ width }) => !width).map(({ suffix }) => + new Promise<void>(res => createReadStream(sourcePath).pipe(createWriteStream(outputPath(suffix))).on('close', res)) + )); // prettier-ignore + + const fileIn = fs.readFileSync(sourcePath); + let buffer: any; + try { + const { buffer2 } = await autorotate.rotate(fileIn, { quality: 30 }); + buffer = buffer2; + } catch (e) {} + return Jimp.read(buffer ?? fileIn) + .then(async img => { + await Promise.all( sizes.filter(({ width }) => width) .map(({ width, suffix }) => + img = img.resize(width, Jimp.AUTO).write(outputPath(suffix)) + )); // prettier-ignore + return writtenFiles; + }) + .catch(e => { + console.log('ERROR' + e); + return writtenFiles; }); - } - return writtenFiles; } /** * define the resizers to use * @param ext the extension - * @returns an array of resizer functions from sharp + * @returns an array of resize descriptions */ - function resizers(ext: string): DashUploadUtils.ImageResizer[] { + export function imageResampleSizes(ext: string): DashUploadUtils.ImageResizer[] { return [ - { suffix: SizeSuffix.Original }, - ...Object.values(DashUploadUtils.Sizes).map(({ suffix, width }) => { - let initial: sharp.Sharp | undefined = sharp({ failOnError: false }).resize(width, undefined, { withoutEnlargement: true }); - if (pngs.includes(ext)) { - initial = initial.png(pngOptions); - } else if (jpgs.includes(ext)) { - initial = initial.jpeg(); - } else if (webps.includes(ext)) { - initial = initial.webp(); - } else if (tiffs.includes(ext)) { - initial = initial.tiff(); - } else if (ext === '.gif') { - initial = undefined; - } - return { - resizer: suffix === '_o' ? undefined : initial, - suffix, - }; - }), + { suffix: SizeSuffix.Original, width: 0 }, + ...[...(AcceptableMedia.imageFormats.includes(ext.toLowerCase()) ? Object.values(DashUploadUtils.Sizes) : [])].map(({ suffix, width }) => ({ + width, + suffix, + })), ]; } } |