diff options
Diffstat (limited to 'src/server/DashUploadUtils.ts')
| -rw-r--r-- | src/server/DashUploadUtils.ts | 349 |
1 files changed, 227 insertions, 122 deletions
diff --git a/src/server/DashUploadUtils.ts b/src/server/DashUploadUtils.ts index cae35da60..4870d218b 100644 --- a/src/server/DashUploadUtils.ts +++ b/src/server/DashUploadUtils.ts @@ -2,35 +2,35 @@ import { green, red } from 'colors'; import { ExifImage } from 'exif'; import * as exifr from 'exifr'; import { File } from 'formidable'; -import { createWriteStream, existsSync, readFileSync, rename, unlinkSync, writeFile } from 'fs'; +import { createReadStream, createWriteStream, existsSync, readFileSync, rename, unlinkSync, writeFile } from 'fs'; import * as path from 'path'; -import { basename } from "path"; +import { basename } from 'path'; import * as sharp from 'sharp'; import { Stream } from 'stream'; import { filesDirectory, publicDirectory } from '.'; import { Opt } from '../fields/Doc'; -import { ParsedPDF } from "../server/PdfTypes"; +import { ParsedPDF } from '../server/PdfTypes'; import { Utils } from '../Utils'; import { createIfNotExists } from './ActionUtilities'; import { clientPathToFile, Directory, pathToDirectory, serverPathToFile } from './ApiManagers/UploadManager'; -import { resolvedServerUrl } from "./server_Initialization"; +import { resolvedServerUrl } from './server_Initialization'; import { AcceptableMedia, Upload } from './SharedMediaTypes'; import request = require('request-promise'); import formidable = require('formidable'); -import { file } from 'jszip'; -import { csvParser } from './DataVizUtils'; -const { exec } = require("child_process"); +const spawn = require('child_process').spawn; +const { exec } = require('child_process'); const parse = require('pdf-parse'); -const ffmpeg = require("fluent-ffmpeg"); -const fs = require("fs"); -const requestImageSize = require("../client/util/request-image-size"); +const ffmpeg = require('fluent-ffmpeg'); +const fs = require('fs'); +const requestImageSize = require('../client/util/request-image-size'); +const md5File = require('md5-file'); export enum SizeSuffix { - Small = "_s", - Medium = "_m", - Large = "_l", - Original = "_o", - None = "" + Small = '_s', + Medium = '_m', + Large = '_l', + Original = '_o', + None = '', } export function InjectSize(filename: string, size: SizeSuffix) { @@ -43,7 +43,6 @@ function isLocal() { } export namespace DashUploadUtils { - export interface Size { width: number; suffix: SizeSuffix; @@ -59,19 +58,19 @@ export namespace DashUploadUtils { return AcceptableMedia.imageFormats.includes(path.extname(url).toLowerCase()); } - const size = "content-length"; - const type = "content-type"; + const size = 'content-length'; + const type = 'content-type'; + + const { imageFormats, videoFormats, applicationFormats, audioFormats } = AcceptableMedia; //TODO:glr - const { imageFormats, videoFormats, applicationFormats, audioFormats } = AcceptableMedia; //TODO:glr - export async function concatVideos(filePaths: string[]): Promise<Upload.AccessPathInfo> { // make a list of paths to create the ordered text file for ffmpeg const inputListName = 'concat.txt'; const textFilePath = path.join(filesDirectory, inputListName); // make a list of paths to create the ordered text file for ffmpeg - const filePathsText = filePaths.map(filePath => `file '${filePath}'`).join('\n'); + const filePathsText = filePaths.map(filePath => `file '${filePath}'`).join('\n'); // write the text file to the file system - writeFile(textFilePath, filePathsText, (err) => console.log(err)); + writeFile(textFilePath, filePathsText, err => console.log(err)); // make output file name based on timestamp const outputFileName = `output-${Utils.GenerateGuid()}.mp4`; @@ -81,117 +80,214 @@ export namespace DashUploadUtils { // concatenate the videos await new Promise((resolve, reject) => { var merge = ffmpeg(); - merge.input(textFilePath) - .inputOptions(['-f concat', '-safe 0']) + merge + .input(textFilePath) + .inputOptions(['-f concat', '-safe 0']) .outputOptions('-c copy') //.videoCodec("copy") .save(outputFilePath) - .on("error", reject) - .on("end", resolve); - }) - - // delete concat.txt from the file system - unlinkSync(textFilePath); - // delete the old segment videos from the server - filePaths.forEach(filePath => unlinkSync(filePath)); - - // return the path(s) to the output file - return { - accessPaths: getAccessPaths(Directory.videos, outputFileName) - } + .on('error', reject) + .on('end', resolve); + }); + + // delete concat.txt from the file system + unlinkSync(textFilePath); + // delete the old segment videos from the server + filePaths.forEach(filePath => unlinkSync(filePath)); + + // return the path(s) to the output file + return { + accessPaths: getAccessPaths(Directory.videos, outputFileName), + }; + } + + function resolveExistingFile(name: string, pat: string, directory: Directory, type?: string, duration?: number, rawText?: string) { + const data = { size: 0, path: path.basename(pat), name, type: type ?? '' }; + const file = { ...data, toJSON: () => ({ ...data, filename: data.path.replace(/.*\//, ''), mtime: duration?.toString(), mime: '', toJson: () => undefined as any }) }; + return { + source: file, + result: { + accessPaths: { + agnostic: getAccessPaths(directory, data.path), + }, + rawText, + duration, + }, + }; } + export function QueryYoutubeProgress(videoId: string) { + return uploadProgress.get(videoId) ?? 'failed'; + } + + let uploadProgress = new Map<string, string>(); + export function uploadYoutube(videoId: string): Promise<Upload.FileResponse> { - console.log("UPLOAD " + videoId); return new Promise<Upload.FileResponse<Upload.FileInformation>>((res, rej) => { - exec('youtube-dl -o ' + (videoId + ".mp4") + ' https://www.youtube.com/watch?v=' + videoId + ' -f "best[filesize<50M]"', - (error: any, stdout: any, stderr: any) => { - if (error) console.log(`error: ${error.message}`); - else if (stderr) console.log(`stderr: ${stderr}`); - else { - console.log(`stdout: ${stdout}`); - const data = { size: 0, path: videoId + ".mp4", name: videoId, type: "video/mp4" }; - const file = { ...data, toJSON: () => ({ ...data, filename: data.path.replace(/.*\//, ""), mtime: null, length: 0, mime: "", toJson: () => undefined as any }) }; - res(MoveParsedFile(file, Directory.videos)); + console.log('Uploading YouTube video: ' + videoId); + const name = videoId; + const path = name.replace(/^-/, '__') + '.mp4'; + const finalPath = serverPathToFile(Directory.videos, path); + if (existsSync(finalPath)) { + uploadProgress.set(videoId, 'computing duration'); + exec(`yt-dlp -o ${finalPath} "https://www.youtube.com/watch?v=${videoId}" --get-duration`, (error: any, stdout: any, stderr: any) => { + const time = Array.from(stdout.trim().split(':')).reverse(); + const duration = (time.length > 2 ? Number(time[2]) * 1000 * 60 : 0) + (time.length > 1 ? Number(time[1]) * 60 : 0) + (time.length > 0 ? Number(time[0]) : 0); + res(resolveExistingFile(name, finalPath, Directory.videos, 'video/mp4', duration, undefined)); + }); + } else { + uploadProgress.set(videoId, 'starting download'); + const ytdlp = spawn(`yt-dlp`, ['-o', path, `https://www.youtube.com/watch?v=${videoId}`, '--max-filesize', '100M', '-f', 'mp4']); + + ytdlp.stdout.on('data', (data: any) => !uploadProgress.get(videoId)?.includes('Aborting.') && uploadProgress.set(videoId, data.toString())); + + let errors = ''; + ytdlp.stderr.on('data', (data: any) => (errors = data.toString())); + + ytdlp.on('exit', function (code: any) { + if (code || uploadProgress.get(videoId)?.includes('Aborting.')) { + res({ + source: { + size: 0, + path, + name, + type: '', + toJSON: () => ({ name, path }), + }, + result: { name: 'failed youtube query', message: `Could not archive video. ${code ? errors : uploadProgress.get(videoId)}` }, + }); + } else { + uploadProgress.set(videoId, 'computing duration'); + exec(`yt-dlp-o ${path} "https://www.youtube.com/watch?v=${videoId}" --get-duration`, (error: any, stdout: any, stderr: any) => { + const time = Array.from(stdout.trim().split(':')).reverse(); + const duration = (time.length > 2 ? Number(time[2]) * 1000 * 60 : 0) + (time.length > 1 ? Number(time[1]) * 60 : 0) + (time.length > 0 ? Number(time[0]) : 0); + const data = { size: 0, path, name, type: 'video/mp4' }; + const file = { ...data, toJSON: () => ({ ...data, filename: data.path.replace(/.*\//, ''), mtime: duration.toString(), mime: '', toJson: () => undefined as any }) }; + res(MoveParsedFile(file, Directory.videos)); + }); } }); + } }); } export async function upload(file: File): Promise<Upload.FileResponse> { const { type, path, name } = file; - const types = type?.split("/") ?? []; + const types = type?.split('/') ?? []; const category = types[0]; let format = `.${types[1]}`; console.log(green(`Processing upload of file (${name}) and format (${format}) with upload type (${type}) in category (${category}).`)); - + switch (category) { - case "image": + case 'image': if (imageFormats.includes(format)) { const result = await UploadImage(path, basename(path)); return { source: file, result }; } - case "video": - if (format.includes("x-matroska")) { - console.log("case video"); - await new Promise(res => ffmpeg(file.path) - .videoCodec("copy") // this will copy the data instead of reencode it - .save(file.path.replace(".mkv", ".mp4")) - .on('end', res)); - file.path = file.path.replace(".mkv", ".mp4"); - format = ".mp4"; + fs.unlink(path, () => {}); + return { source: file, result: { name: 'Unsupported image format', message: `Could not upload unsupported file (${name}). Please convert to an .jpg` } }; + case 'video': + if (format.includes('x-matroska')) { + console.log('case video'); + await new Promise(res => + ffmpeg(file.path) + .videoCodec('copy') // this will copy the data instead of reencode it + .save(file.path.replace('.mkv', '.mp4')) + .on('end', res) + ); + file.path = file.path.replace('.mkv', '.mp4'); + format = '.mp4'; + } + if (format.includes('quicktime')) { + let abort = false; + await new Promise<void>(res => + ffmpeg.ffprobe(file.path, (err: any, metadata: any) => { + if (metadata.streams.some((stream: any) => stream.codec_name === 'hevc')) { + abort = true; + } + res(); + }) + ); + if (abort) { + // bcz: instead of aborting, we could convert the file using the code below to an mp4. Problem is that this takes a long time and will clog up the server. + // await new Promise(res => + // ffmpeg(file.path) + // .videoCodec('libx264') // this will copy the data instead of reencode it + // .audioCodec('mp2') + // .save(file.path.replace('.MOV', '.mp4').replace('.mov', '.mp4')) + // .on('end', res) + // ); + // file.path = file.path.replace('.mov', '.mp4').replace('.MOV', '.mp4'); + // format = '.mp4'; + fs.unlink(path, () => {}); + return { source: file, result: { name: 'Unsupported video format', message: `Could not upload unsupported file (${name}). Please convert to an .mp4` } }; + } } if (videoFormats.includes(format)) { return MoveParsedFile(file, Directory.videos); } - case "application": + fs.unlink(path, () => {}); + return { source: file, result: { name: 'Unsupported video format', message: `Could not upload unsupported file (${name}). Please convert to an .mp4` } }; + case 'application': if (applicationFormats.includes(format)) { - return UploadPdf(file); + const val = UploadPdf(file); + if (val) return val; } - case "audio": - const components = format.split(";"); + case 'audio': + const components = format.split(';'); if (components.length > 1) { format = components[0]; } if (audioFormats.includes(format)) { return UploadAudio(file, format); } - case "text": - if (types[1] == "csv") { + fs.unlink(path, () => {}); + return { source: file, result: { name: 'Unsupported audio format', message: `Could not upload unsupported file (${name}). Please convert to an .mp3` } }; + case 'text': + if (types[1] == 'csv') { return UploadCsv(file); } - } console.log(red(`Ignoring unsupported file (${name}) with upload type (${type}).`)); + fs.unlink(path, () => {}); return { source: file, result: new Error(`Could not upload unsupported file (${name}) with upload type (${type}).`) }; } async function UploadPdf(file: File) { - const { path: sourcePath } = file; - const dataBuffer = readFileSync(sourcePath); - const result: ParsedPDF = await parse(dataBuffer); - await new Promise<void>((resolve, reject) => { - const name = path.basename(sourcePath); - const textFilename = `${name.substring(0, name.length - 4)}.txt`; - const writeStream = createWriteStream(serverPathToFile(Directory.text, textFilename)); - writeStream.write(result.text, error => error ? reject(error) : resolve()); - }); - return MoveParsedFile(file, Directory.pdfs, undefined, result.text); + const fileKey = (await md5File(file.path)) + '.pdf'; + const textFilename = `${fileKey.substring(0, fileKey.length - 4)}.txt`; + if (fExists(fileKey, Directory.pdfs) && fExists(textFilename, Directory.text)) { + return new Promise<Upload.FileResponse>(res => { + const textFilename = `${fileKey.substring(0, fileKey.length - 4)}.txt`; + const readStream = createReadStream(serverPathToFile(Directory.text, textFilename)); + var rawText = ''; + readStream.on('data', chunk => (rawText += chunk.toString())).on('end', () => res(resolveExistingFile(file.name, fileKey, Directory.pdfs, file.type, undefined, rawText))); + }); + } + const dataBuffer = readFileSync(file.path); + const result: ParsedPDF | any = await parse(dataBuffer).catch((e: any) => e); + if (!result.code) { + await new Promise<void>((resolve, reject) => { + const writeStream = createWriteStream(serverPathToFile(Directory.text, textFilename)); + writeStream.write(result?.text, error => (error ? reject(error) : resolve())); + }); + return MoveParsedFile(file, Directory.pdfs, undefined, result?.text, undefined, fileKey); + } + return { source: file, result: { name: 'faile pdf pupload', message: `Could not upload (${file.name}).${result.message}` } }; } async function UploadCsv(file: File) { - const { path: sourcePath } = file; - // read the file as a string + const { path: sourcePath } = file; + // read the file as a string const data = readFileSync(sourcePath, 'utf8'); // split the string into an array of lines return MoveParsedFile(file, Directory.csv, undefined, data); // console.log(csvParser(data)); - } - const manualSuffixes = [".webm"]; + const manualSuffixes = ['.webm']; async function UploadAudio(file: File, format: string) { const suffix = manualSuffixes.includes(format) ? format : undefined; @@ -200,37 +296,37 @@ export namespace DashUploadUtils { /** * Uploads an image specified by the @param source to Dash's /public/files/ - * directory, and returns information generated during that upload - * + * directory, and returns information generated during that upload + * * @param {string} source is either the absolute path of an already uploaded image or * the url of a remote image * @param {string} filename dictates what to call the image. If not specified, * the name {@param prefix}_upload_{GUID} * @param {string} prefix is a string prepended to the generated image name in the * event that @param filename is not specified - * + * * @returns {ImageUploadInformation | Error} This method returns * 1) the paths to the uploaded images (plural due to resizing) * 2) the exif data embedded in the image, or the error explaining why exif couldn't be parsed * 3) the size of the image, in bytes (4432130) * 4) the content type of the image, i.e. image/(jpeg | png | ...) */ - export const UploadImage = async (source: string, filename?: string, prefix: string = ""): Promise<Upload.ImageInformation | Error> => { + export const UploadImage = async (source: string, filename?: string, prefix: string = ''): Promise<Upload.ImageInformation | Error> => { const metadata = await InspectImage(source); if (metadata instanceof Error) { - return metadata; + return { name: metadata.name, message: metadata.message }; } return UploadInspectedImage(metadata, filename || metadata.filename, prefix); }; export async function buildFileDirectories() { if (!existsSync(publicDirectory)) { - console.error("\nPlease ensure that the following directory exists...\n"); + console.error('\nPlease ensure that the following directory exists...\n'); console.log(publicDirectory); process.exit(0); } if (!existsSync(filesDirectory)) { - console.error("\nPlease ensure that the following directory exists...\n"); + console.error('\nPlease ensure that the following directory exists...\n'); console.log(filesDirectory); process.exit(0); } @@ -252,7 +348,7 @@ export namespace DashUploadUtils { /** * Based on the url's classification as local or remote, gleans * as much information as possible about the specified image - * + * * @param source is the path or url to the image in question */ export const InspectImage = async (source: string): Promise<Upload.InspectionResults | Error> => { @@ -265,9 +361,9 @@ export namespace DashUploadUtils { */ if ((rawMatches = /^data:image\/([a-z]+);base64,(.*)/.exec(source)) !== null) { const [ext, data] = rawMatches.slice(1, 3); - const resolved = filename = `upload_${Utils.GenerateGuid()}.${ext}`; + const resolved = (filename = `upload_${Utils.GenerateGuid()}.${ext}`); const error = await new Promise<Error | null>(resolve => { - writeFile(serverPathToFile(Directory.images, resolved), data, "base64", resolve); + writeFile(serverPathToFile(Directory.images, resolved), data, 'base64', resolve); }); if (error !== null) { return error; @@ -276,12 +372,12 @@ export namespace DashUploadUtils { } let resolvedUrl: string; /** - * + * * At this point, we want to take whatever url we have and make sure it's requestable. * Anything that's hosted by some other website already is, but if the url is a local file url * (locates the file on this server machine), we have to resolve the client side url by cutting out the * basename subtree (i.e. /images/<some_guid>.<ext>) and put it on the end of the server's url. - * + * * This can always be localhost, regardless of whether this is on the server or not, since we (the server, not the client) * will be the ones making the request, and from the perspective of dash-release or dash-web, localhost:<port> refers to the same thing * as the full dash-release.eastus.cloudapp.azure.com:<port>. @@ -290,18 +386,20 @@ export namespace DashUploadUtils { if (matches === null) { resolvedUrl = source; } else { - resolvedUrl = `${resolvedServerUrl}/${matches[1].split("\\").join("/")}`; + resolvedUrl = `${resolvedServerUrl}/${matches[1].split('\\').join('/')}`; } // See header comments: not all image files have exif data (I believe only JPG is the only format that can have it) const exifData = await parseExifData(resolvedUrl); const results = { exifData, - requestable: resolvedUrl + requestable: resolvedUrl, }; + // Use the request library to parse out file level image information in the headers - const { headers } = (await new Promise<any>((resolve, reject) => { - request.head(resolvedUrl, (error, res) => error ? reject(error) : resolve(res)); - }).catch(console.error)); + const { headers } = await new Promise<any>((resolve, reject) => { + return request.head(resolvedUrl, (error, res) => (error ? reject(error) : resolve(res))); + }).catch(e => console.log(e)); + try { // Compute the native width and height ofthe image with an npm module const { width: nativeWidth, height: nativeHeight } = await requestImageSize(resolvedUrl); @@ -313,7 +411,7 @@ export namespace DashUploadUtils { nativeWidth, nativeHeight, filename, - ...results + ...results, }; } catch (e: any) { console.log(e); @@ -331,42 +429,50 @@ export namespace DashUploadUtils { * @param suffix If the file doesn't have a suffix and you want to provide it one * to appear in the new location */ - export async function MoveParsedFile(file: formidable.File, destination: Directory, suffix: string | undefined = undefined, text?: string): Promise<Upload.FileResponse> { + export async function MoveParsedFile(file: formidable.File, destination: Directory, suffix: string | undefined = undefined, text?: string, duration?: number, targetName?: string): Promise<Upload.FileResponse> { const { path: sourcePath } = file; - let name = path.basename(sourcePath); + let name = targetName ?? path.basename(sourcePath); suffix && (name += suffix); return new Promise(resolve => { const destinationPath = serverPathToFile(destination, name); rename(sourcePath, destinationPath, error => { resolve({ source: file, - result: error ? error : { - accessPaths: { - agnostic: getAccessPaths(destination, name) - }, - rawText: text - } + result: error + ? error + : { + accessPaths: { + agnostic: getAccessPaths(destination, name), + }, + rawText: text, + duration, + }, }); }); }); } + export function fExists(name: string, destination: Directory) { + const destinationPath = serverPathToFile(destination, name); + return existsSync(destinationPath); + } + export function getAccessPaths(directory: Directory, fileName: string) { return { client: clientPathToFile(directory, fileName), - server: serverPathToFile(directory, fileName) + server: serverPathToFile(directory, fileName), }; } - export const UploadInspectedImage = async (metadata: Upload.InspectionResults, filename?: string, prefix = "", cleanUp = true): Promise<Upload.ImageInformation> => { + export const UploadInspectedImage = async (metadata: Upload.InspectionResults, filename?: string, prefix = '', cleanUp = true): Promise<Upload.ImageInformation> => { const { requestable, source, ...remaining } = metadata; - const resolved = filename || `${prefix}upload_${Utils.GenerateGuid()}.${remaining.contentType.split("/")[1].toLowerCase()}`; + const resolved = filename || `${prefix}upload_${Utils.GenerateGuid()}.${remaining.contentType.split('/')[1].toLowerCase()}`; const { images } = Directory; const information: Upload.ImageInformation = { accessPaths: { - agnostic: getAccessPaths(images, resolved) + agnostic: getAccessPaths(images, resolved), }, - ...metadata + ...metadata, }; const writtenFiles = await outputResizedImages(() => request(requestable), resolved, pathToDirectory(Directory.images)); for (const suffix of Object.keys(writtenFiles)) { @@ -383,9 +489,9 @@ export namespace DashUploadUtils { const val: any = layer[key]; if (val instanceof Buffer) { layer[key] = val.toString(); - } else if (Array.isArray(val) && typeof val[0] === "number") { + } else if (Array.isArray(val) && typeof val[0] === 'number') { layer[key] = Buffer.from(val).toString(); - } else if (typeof val === "object") { + } else if (typeof val === 'object') { bufferConverterRec(val); } } @@ -403,27 +509,27 @@ export namespace DashUploadUtils { }); }); //data && bufferConverterRec(data); - return { data: await exifr.parse(image), error }; + return error ? { data: undefined, error } : { data: await exifr.parse(image), error }; }; const { pngs, jpgs, webps, tiffs } = AcceptableMedia; const pngOptions = { compressionLevel: 9, adaptiveFiltering: true, - force: true + force: true, }; export async function outputResizedImages(streamProvider: () => Stream | Promise<Stream>, outputFileName: string, outputDirectory: string) { const writtenFiles: { [suffix: string]: string } = {}; for (const { resizer, suffix } of resizers(path.extname(outputFileName))) { - const outputPath = path.resolve(outputDirectory, writtenFiles[suffix] = InjectSize(outputFileName, suffix)); + const outputPath = path.resolve(outputDirectory, (writtenFiles[suffix] = InjectSize(outputFileName, suffix))); await new Promise<void>(async (resolve, reject) => { const source = streamProvider(); let readStream: Stream = source instanceof Promise ? await source : source; if (resizer) { readStream = readStream.pipe(resizer.withMetadata()); } - readStream.pipe(createWriteStream(outputPath)).on("close", resolve).on("error", reject); + readStream.pipe(createWriteStream(outputPath)).on('close', resolve).on('error', reject); }); } return writtenFiles; @@ -442,15 +548,14 @@ export namespace DashUploadUtils { initial = initial.webp(); } else if (tiffs.includes(ext)) { initial = initial.tiff(); - } else if (ext === ".gif") { + } else if (ext === '.gif') { initial = undefined; } return { resizer: initial, - suffix + suffix, }; - }) + }), ]; } - -}
\ No newline at end of file +} |