diff options
Diffstat (limited to 'src/server/DashUploadUtils.ts')
| -rw-r--r-- | src/server/DashUploadUtils.ts | 175 |
1 files changed, 123 insertions, 52 deletions
diff --git a/src/server/DashUploadUtils.ts b/src/server/DashUploadUtils.ts index e94ef8534..4870d218b 100644 --- a/src/server/DashUploadUtils.ts +++ b/src/server/DashUploadUtils.ts @@ -2,7 +2,7 @@ import { green, red } from 'colors'; import { ExifImage } from 'exif'; import * as exifr from 'exifr'; import { File } from 'formidable'; -import { createWriteStream, existsSync, readFileSync, rename, unlinkSync, writeFile } from 'fs'; +import { createReadStream, createWriteStream, existsSync, readFileSync, rename, unlinkSync, writeFile } from 'fs'; import * as path from 'path'; import { basename } from 'path'; import * as sharp from 'sharp'; @@ -17,11 +17,13 @@ import { resolvedServerUrl } from './server_Initialization'; import { AcceptableMedia, Upload } from './SharedMediaTypes'; import request = require('request-promise'); import formidable = require('formidable'); +const spawn = require('child_process').spawn; const { exec } = require('child_process'); const parse = require('pdf-parse'); const ffmpeg = require('fluent-ffmpeg'); const fs = require('fs'); const requestImageSize = require('../client/util/request-image-size'); +const md5File = require('md5-file'); export enum SizeSuffix { Small = '_s', @@ -99,32 +101,73 @@ export namespace DashUploadUtils { }; } + function resolveExistingFile(name: string, pat: string, directory: Directory, type?: string, duration?: number, rawText?: string) { + const data = { size: 0, path: path.basename(pat), name, type: type ?? '' }; + const file = { ...data, toJSON: () => ({ ...data, filename: data.path.replace(/.*\//, ''), mtime: duration?.toString(), mime: '', toJson: () => undefined as any }) }; + return { + source: file, + result: { + accessPaths: { + agnostic: getAccessPaths(directory, data.path), + }, + rawText, + duration, + }, + }; + } + + export function QueryYoutubeProgress(videoId: string) { + return uploadProgress.get(videoId) ?? 'failed'; + } + + let uploadProgress = new Map<string, string>(); + export function uploadYoutube(videoId: string): Promise<Upload.FileResponse> { return new Promise<Upload.FileResponse<Upload.FileInformation>>((res, rej) => { console.log('Uploading YouTube video: ' + videoId); - exec('youtube-dl -o ' + (videoId + '.mp4') + ' ' + videoId + ' -f "mp4[filesize<5M]/bestvideo[filesize<5M]+bestaudio/bestvideo+bestaudio"', (error: any, stdout: any, stderr: any) => { - if (error) { - console.log(`error: Error: ${error.message}`); - res({ - source: { - size: 0, - path: videoId, - name: videoId, - type: '', - toJSON: () => ({ name: videoId, path: videoId }), - }, - result: { name: 'failed youtube query', message: `Could not upload YouTube video (${videoId}). Error: ${error.message}` }, - }); - } else { - exec('youtube-dl -o ' + (videoId + '.mp4') + ' ' + videoId + ' --get-duration', (error: any, stdout: any, stderr: any) => { - const time = Array.from(stdout.trim().split(':')).reverse(); - const duration = (time.length > 2 ? Number(time[2]) * 1000 * 60 : 0) + (time.length > 1 ? Number(time[1]) * 60 : 0) + (time.length > 0 ? Number(time[0]) : 0); - const data = { size: 0, path: videoId + '.mp4', name: videoId, type: 'video/mp4' }; - const file = { ...data, toJSON: () => ({ ...data, filename: data.path.replace(/.*\//, ''), mtime: duration.toString(), mime: '', toJson: () => undefined as any }) }; - res(MoveParsedFile(file, Directory.videos)); - }); - } - }); + const name = videoId; + const path = name.replace(/^-/, '__') + '.mp4'; + const finalPath = serverPathToFile(Directory.videos, path); + if (existsSync(finalPath)) { + uploadProgress.set(videoId, 'computing duration'); + exec(`yt-dlp -o ${finalPath} "https://www.youtube.com/watch?v=${videoId}" --get-duration`, (error: any, stdout: any, stderr: any) => { + const time = Array.from(stdout.trim().split(':')).reverse(); + const duration = (time.length > 2 ? Number(time[2]) * 1000 * 60 : 0) + (time.length > 1 ? Number(time[1]) * 60 : 0) + (time.length > 0 ? Number(time[0]) : 0); + res(resolveExistingFile(name, finalPath, Directory.videos, 'video/mp4', duration, undefined)); + }); + } else { + uploadProgress.set(videoId, 'starting download'); + const ytdlp = spawn(`yt-dlp`, ['-o', path, `https://www.youtube.com/watch?v=${videoId}`, '--max-filesize', '100M', '-f', 'mp4']); + + ytdlp.stdout.on('data', (data: any) => !uploadProgress.get(videoId)?.includes('Aborting.') && uploadProgress.set(videoId, data.toString())); + + let errors = ''; + ytdlp.stderr.on('data', (data: any) => (errors = data.toString())); + + ytdlp.on('exit', function (code: any) { + if (code || uploadProgress.get(videoId)?.includes('Aborting.')) { + res({ + source: { + size: 0, + path, + name, + type: '', + toJSON: () => ({ name, path }), + }, + result: { name: 'failed youtube query', message: `Could not archive video. ${code ? errors : uploadProgress.get(videoId)}` }, + }); + } else { + uploadProgress.set(videoId, 'computing duration'); + exec(`yt-dlp-o ${path} "https://www.youtube.com/watch?v=${videoId}" --get-duration`, (error: any, stdout: any, stderr: any) => { + const time = Array.from(stdout.trim().split(':')).reverse(); + const duration = (time.length > 2 ? Number(time[2]) * 1000 * 60 : 0) + (time.length > 1 ? Number(time[1]) * 60 : 0) + (time.length > 0 ? Number(time[0]) : 0); + const data = { size: 0, path, name, type: 'video/mp4' }; + const file = { ...data, toJSON: () => ({ ...data, filename: data.path.replace(/.*\//, ''), mtime: duration.toString(), mime: '', toJson: () => undefined as any }) }; + res(MoveParsedFile(file, Directory.videos)); + }); + } + }); + } }); } @@ -142,6 +185,8 @@ export namespace DashUploadUtils { const result = await UploadImage(path, basename(path)); return { source: file, result }; } + fs.unlink(path, () => {}); + return { source: file, result: { name: 'Unsupported image format', message: `Could not upload unsupported file (${name}). Please convert to an .jpg` } }; case 'video': if (format.includes('x-matroska')) { console.log('case video'); @@ -164,24 +209,30 @@ export namespace DashUploadUtils { res(); }) ); - if (abort) return { source: file, result: { name: 'Unsupported video format', message: `Could not upload unsupported file (${name}). Please convert to an .mp4` } }; - // bcz: instead of aborting, we could convert the file using the code below to an mp4. Problem is that this takes a long time and will clog up the server. - // await new Promise(res => - // ffmpeg(file.path) - // .videoCodec('libx264') // this will copy the data instead of reencode it - // .audioCodec('mp2') - // .save(file.path.replace('.MOV', '.mp4').replace('.mov', '.mp4')) - // .on('end', res) - // ); - // file.path = file.path.replace('.mov', '.mp4').replace('.MOV', '.mp4'); - // format = '.mp4'; + if (abort) { + // bcz: instead of aborting, we could convert the file using the code below to an mp4. Problem is that this takes a long time and will clog up the server. + // await new Promise(res => + // ffmpeg(file.path) + // .videoCodec('libx264') // this will copy the data instead of reencode it + // .audioCodec('mp2') + // .save(file.path.replace('.MOV', '.mp4').replace('.mov', '.mp4')) + // .on('end', res) + // ); + // file.path = file.path.replace('.mov', '.mp4').replace('.MOV', '.mp4'); + // format = '.mp4'; + fs.unlink(path, () => {}); + return { source: file, result: { name: 'Unsupported video format', message: `Could not upload unsupported file (${name}). Please convert to an .mp4` } }; + } } if (videoFormats.includes(format)) { return MoveParsedFile(file, Directory.videos); } + fs.unlink(path, () => {}); + return { source: file, result: { name: 'Unsupported video format', message: `Could not upload unsupported file (${name}). Please convert to an .mp4` } }; case 'application': if (applicationFormats.includes(format)) { - return UploadPdf(file); + const val = UploadPdf(file); + if (val) return val; } case 'audio': const components = format.split(';'); @@ -191,6 +242,8 @@ export namespace DashUploadUtils { if (audioFormats.includes(format)) { return UploadAudio(file, format); } + fs.unlink(path, () => {}); + return { source: file, result: { name: 'Unsupported audio format', message: `Could not upload unsupported file (${name}). Please convert to an .mp3` } }; case 'text': if (types[1] == 'csv') { return UploadCsv(file); @@ -198,20 +251,31 @@ export namespace DashUploadUtils { } console.log(red(`Ignoring unsupported file (${name}) with upload type (${type}).`)); + fs.unlink(path, () => {}); return { source: file, result: new Error(`Could not upload unsupported file (${name}) with upload type (${type}).`) }; } async function UploadPdf(file: File) { - const { path: sourcePath } = file; - const dataBuffer = readFileSync(sourcePath); - const result: ParsedPDF = await parse(dataBuffer); - await new Promise<void>((resolve, reject) => { - const name = path.basename(sourcePath); - const textFilename = `${name.substring(0, name.length - 4)}.txt`; - const writeStream = createWriteStream(serverPathToFile(Directory.text, textFilename)); - writeStream.write(result.text, error => (error ? reject(error) : resolve())); - }); - return MoveParsedFile(file, Directory.pdfs, undefined, result.text); + const fileKey = (await md5File(file.path)) + '.pdf'; + const textFilename = `${fileKey.substring(0, fileKey.length - 4)}.txt`; + if (fExists(fileKey, Directory.pdfs) && fExists(textFilename, Directory.text)) { + return new Promise<Upload.FileResponse>(res => { + const textFilename = `${fileKey.substring(0, fileKey.length - 4)}.txt`; + const readStream = createReadStream(serverPathToFile(Directory.text, textFilename)); + var rawText = ''; + readStream.on('data', chunk => (rawText += chunk.toString())).on('end', () => res(resolveExistingFile(file.name, fileKey, Directory.pdfs, file.type, undefined, rawText))); + }); + } + const dataBuffer = readFileSync(file.path); + const result: ParsedPDF | any = await parse(dataBuffer).catch((e: any) => e); + if (!result.code) { + await new Promise<void>((resolve, reject) => { + const writeStream = createWriteStream(serverPathToFile(Directory.text, textFilename)); + writeStream.write(result?.text, error => (error ? reject(error) : resolve())); + }); + return MoveParsedFile(file, Directory.pdfs, undefined, result?.text, undefined, fileKey); + } + return { source: file, result: { name: 'faile pdf pupload', message: `Could not upload (${file.name}).${result.message}` } }; } async function UploadCsv(file: File) { @@ -250,7 +314,7 @@ export namespace DashUploadUtils { export const UploadImage = async (source: string, filename?: string, prefix: string = ''): Promise<Upload.ImageInformation | Error> => { const metadata = await InspectImage(source); if (metadata instanceof Error) { - return metadata; + return { name: metadata.name, message: metadata.message }; } return UploadInspectedImage(metadata, filename || metadata.filename, prefix); }; @@ -330,10 +394,12 @@ export namespace DashUploadUtils { exifData, requestable: resolvedUrl, }; + // Use the request library to parse out file level image information in the headers const { headers } = await new Promise<any>((resolve, reject) => { - request.head(resolvedUrl, (error, res) => (error ? reject(error) : resolve(res))); - }).catch(console.error); + return request.head(resolvedUrl, (error, res) => (error ? reject(error) : resolve(res))); + }).catch(e => console.log(e)); + try { // Compute the native width and height ofthe image with an npm module const { width: nativeWidth, height: nativeHeight } = await requestImageSize(resolvedUrl); @@ -363,9 +429,9 @@ export namespace DashUploadUtils { * @param suffix If the file doesn't have a suffix and you want to provide it one * to appear in the new location */ - export async function MoveParsedFile(file: formidable.File, destination: Directory, suffix: string | undefined = undefined, text?: string, duration?: number): Promise<Upload.FileResponse> { + export async function MoveParsedFile(file: formidable.File, destination: Directory, suffix: string | undefined = undefined, text?: string, duration?: number, targetName?: string): Promise<Upload.FileResponse> { const { path: sourcePath } = file; - let name = path.basename(sourcePath); + let name = targetName ?? path.basename(sourcePath); suffix && (name += suffix); return new Promise(resolve => { const destinationPath = serverPathToFile(destination, name); @@ -386,6 +452,11 @@ export namespace DashUploadUtils { }); } + export function fExists(name: string, destination: Directory) { + const destinationPath = serverPathToFile(destination, name); + return existsSync(destinationPath); + } + export function getAccessPaths(directory: Directory, fileName: string) { return { client: clientPathToFile(directory, fileName), @@ -438,7 +509,7 @@ export namespace DashUploadUtils { }); }); //data && bufferConverterRec(data); - return { data: await exifr.parse(image), error }; + return error ? { data: undefined, error } : { data: await exifr.parse(image), error }; }; const { pngs, jpgs, webps, tiffs } = AcceptableMedia; |