aboutsummaryrefslogtreecommitdiff
path: root/src/server/DashUploadUtils.ts
diff options
context:
space:
mode:
Diffstat (limited to 'src/server/DashUploadUtils.ts')
-rw-r--r--src/server/DashUploadUtils.ts236
1 files changed, 129 insertions, 107 deletions
diff --git a/src/server/DashUploadUtils.ts b/src/server/DashUploadUtils.ts
index cae35da60..ef7192ecc 100644
--- a/src/server/DashUploadUtils.ts
+++ b/src/server/DashUploadUtils.ts
@@ -4,33 +4,33 @@ import * as exifr from 'exifr';
import { File } from 'formidable';
import { createWriteStream, existsSync, readFileSync, rename, unlinkSync, writeFile } from 'fs';
import * as path from 'path';
-import { basename } from "path";
+import { basename } from 'path';
import * as sharp from 'sharp';
import { Stream } from 'stream';
import { filesDirectory, publicDirectory } from '.';
import { Opt } from '../fields/Doc';
-import { ParsedPDF } from "../server/PdfTypes";
+import { ParsedPDF } from '../server/PdfTypes';
import { Utils } from '../Utils';
import { createIfNotExists } from './ActionUtilities';
import { clientPathToFile, Directory, pathToDirectory, serverPathToFile } from './ApiManagers/UploadManager';
-import { resolvedServerUrl } from "./server_Initialization";
+import { resolvedServerUrl } from './server_Initialization';
import { AcceptableMedia, Upload } from './SharedMediaTypes';
import request = require('request-promise');
import formidable = require('formidable');
import { file } from 'jszip';
import { csvParser } from './DataVizUtils';
-const { exec } = require("child_process");
+const { exec } = require('child_process');
const parse = require('pdf-parse');
-const ffmpeg = require("fluent-ffmpeg");
-const fs = require("fs");
-const requestImageSize = require("../client/util/request-image-size");
+const ffmpeg = require('fluent-ffmpeg');
+const fs = require('fs');
+const requestImageSize = require('../client/util/request-image-size');
export enum SizeSuffix {
- Small = "_s",
- Medium = "_m",
- Large = "_l",
- Original = "_o",
- None = ""
+ Small = '_s',
+ Medium = '_m',
+ Large = '_l',
+ Original = '_o',
+ None = '',
}
export function InjectSize(filename: string, size: SizeSuffix) {
@@ -43,7 +43,6 @@ function isLocal() {
}
export namespace DashUploadUtils {
-
export interface Size {
width: number;
suffix: SizeSuffix;
@@ -59,19 +58,19 @@ export namespace DashUploadUtils {
return AcceptableMedia.imageFormats.includes(path.extname(url).toLowerCase());
}
- const size = "content-length";
- const type = "content-type";
+ const size = 'content-length';
+ const type = 'content-type';
+
+ const { imageFormats, videoFormats, applicationFormats, audioFormats } = AcceptableMedia; //TODO:glr
- const { imageFormats, videoFormats, applicationFormats, audioFormats } = AcceptableMedia; //TODO:glr
-
export async function concatVideos(filePaths: string[]): Promise<Upload.AccessPathInfo> {
// make a list of paths to create the ordered text file for ffmpeg
const inputListName = 'concat.txt';
const textFilePath = path.join(filesDirectory, inputListName);
// make a list of paths to create the ordered text file for ffmpeg
- const filePathsText = filePaths.map(filePath => `file '${filePath}'`).join('\n');
+ const filePathsText = filePaths.map(filePath => `file '${filePath}'`).join('\n');
// write the text file to the file system
- writeFile(textFilePath, filePathsText, (err) => console.log(err));
+ writeFile(textFilePath, filePathsText, err => console.log(err));
// make output file name based on timestamp
const outputFileName = `output-${Utils.GenerateGuid()}.mp4`;
@@ -81,87 +80,110 @@ export namespace DashUploadUtils {
// concatenate the videos
await new Promise((resolve, reject) => {
var merge = ffmpeg();
- merge.input(textFilePath)
- .inputOptions(['-f concat', '-safe 0'])
+ merge
+ .input(textFilePath)
+ .inputOptions(['-f concat', '-safe 0'])
.outputOptions('-c copy')
//.videoCodec("copy")
.save(outputFilePath)
- .on("error", reject)
- .on("end", resolve);
- })
-
- // delete concat.txt from the file system
- unlinkSync(textFilePath);
- // delete the old segment videos from the server
- filePaths.forEach(filePath => unlinkSync(filePath));
-
- // return the path(s) to the output file
- return {
- accessPaths: getAccessPaths(Directory.videos, outputFileName)
- }
+ .on('error', reject)
+ .on('end', resolve);
+ });
+
+ // delete concat.txt from the file system
+ unlinkSync(textFilePath);
+ // delete the old segment videos from the server
+ filePaths.forEach(filePath => unlinkSync(filePath));
+
+ // return the path(s) to the output file
+ return {
+ accessPaths: getAccessPaths(Directory.videos, outputFileName),
+ };
}
export function uploadYoutube(videoId: string): Promise<Upload.FileResponse> {
- console.log("UPLOAD " + videoId);
+ console.log('UPLOAD ' + videoId);
return new Promise<Upload.FileResponse<Upload.FileInformation>>((res, rej) => {
- exec('youtube-dl -o ' + (videoId + ".mp4") + ' https://www.youtube.com/watch?v=' + videoId + ' -f "best[filesize<50M]"',
- (error: any, stdout: any, stderr: any) => {
- if (error) console.log(`error: ${error.message}`);
- else if (stderr) console.log(`stderr: ${stderr}`);
- else {
- console.log(`stdout: ${stdout}`);
- const data = { size: 0, path: videoId + ".mp4", name: videoId, type: "video/mp4" };
- const file = { ...data, toJSON: () => ({ ...data, filename: data.path.replace(/.*\//, ""), mtime: null, length: 0, mime: "", toJson: () => undefined as any }) };
- res(MoveParsedFile(file, Directory.videos));
- }
- });
+ exec('youtube-dl -o ' + (videoId + '.mp4') + ' https://www.youtube.com/watch?v=' + videoId + ' -f "best[filesize<50M]"', (error: any, stdout: any, stderr: any) => {
+ if (error) console.log(`error: ${error.message}`);
+ else if (stderr) console.log(`stderr: ${stderr}`);
+ else {
+ console.log(`stdout: ${stdout}`);
+ const data = { size: 0, path: videoId + '.mp4', name: videoId, type: 'video/mp4' };
+ const file = { ...data, toJSON: () => ({ ...data, filename: data.path.replace(/.*\//, ''), mtime: null, length: 0, mime: '', toJson: () => undefined as any }) };
+ res(MoveParsedFile(file, Directory.videos));
+ }
+ });
});
}
export async function upload(file: File): Promise<Upload.FileResponse> {
const { type, path, name } = file;
- const types = type?.split("/") ?? [];
+ const types = type?.split('/') ?? [];
const category = types[0];
let format = `.${types[1]}`;
console.log(green(`Processing upload of file (${name}) and format (${format}) with upload type (${type}) in category (${category}).`));
-
+
switch (category) {
- case "image":
+ case 'image':
if (imageFormats.includes(format)) {
const result = await UploadImage(path, basename(path));
return { source: file, result };
}
- case "video":
- if (format.includes("x-matroska")) {
- console.log("case video");
- await new Promise(res => ffmpeg(file.path)
- .videoCodec("copy") // this will copy the data instead of reencode it
- .save(file.path.replace(".mkv", ".mp4"))
- .on('end', res));
- file.path = file.path.replace(".mkv", ".mp4");
- format = ".mp4";
+ case 'video':
+ if (format.includes('x-matroska')) {
+ console.log('case video');
+ await new Promise(res =>
+ ffmpeg(file.path)
+ .videoCodec('copy') // this will copy the data instead of reencode it
+ .save(file.path.replace('.mkv', '.mp4'))
+ .on('end', res)
+ );
+ file.path = file.path.replace('.mkv', '.mp4');
+ format = '.mp4';
+ }
+ if (format.includes('quicktime')) {
+ let abort = false;
+ await new Promise<void>(res =>
+ ffmpeg.ffprobe(file.path, (err: any, metadata: any) => {
+ if (metadata.streams.some((stream: any) => stream.codec_name === 'hevc')) {
+ abort = true;
+ }
+ res();
+ })
+ );
+ if (abort) return { source: file, result: { name: 'Unsupported video format', message: `Could not upload unsupported file (${name}). Please convert to an .mp4` } };
+ // bcz: instead of aborting, we could convert the file using the code below to an mp4. Problem is that this takes a long time and will clog up the server.
+ // await new Promise(res =>
+ // ffmpeg(file.path)
+ // .videoCodec('libx264') // this will copy the data instead of reencode it
+ // .audioCodec('mp2')
+ // .save(file.path.replace('.MOV', '.mp4').replace('.mov', '.mp4'))
+ // .on('end', res)
+ // );
+ // file.path = file.path.replace('.mov', '.mp4').replace('.MOV', '.mp4');
+ // format = '.mp4';
}
if (videoFormats.includes(format)) {
return MoveParsedFile(file, Directory.videos);
}
- case "application":
+ case 'application':
if (applicationFormats.includes(format)) {
return UploadPdf(file);
}
- case "audio":
- const components = format.split(";");
+ case 'audio':
+ const components = format.split(';');
if (components.length > 1) {
format = components[0];
}
if (audioFormats.includes(format)) {
return UploadAudio(file, format);
}
- case "text":
- if (types[1] == "csv") {
+ case 'text':
+ if (types[1] == 'csv') {
return UploadCsv(file);
}
-
}
console.log(red(`Ignoring unsupported file (${name}) with upload type (${type}).`));
@@ -176,22 +198,21 @@ export namespace DashUploadUtils {
const name = path.basename(sourcePath);
const textFilename = `${name.substring(0, name.length - 4)}.txt`;
const writeStream = createWriteStream(serverPathToFile(Directory.text, textFilename));
- writeStream.write(result.text, error => error ? reject(error) : resolve());
+ writeStream.write(result.text, error => (error ? reject(error) : resolve()));
});
return MoveParsedFile(file, Directory.pdfs, undefined, result.text);
}
async function UploadCsv(file: File) {
- const { path: sourcePath } = file;
- // read the file as a string
+ const { path: sourcePath } = file;
+ // read the file as a string
const data = readFileSync(sourcePath, 'utf8');
// split the string into an array of lines
return MoveParsedFile(file, Directory.csv, undefined, data);
// console.log(csvParser(data));
-
}
- const manualSuffixes = [".webm"];
+ const manualSuffixes = ['.webm'];
async function UploadAudio(file: File, format: string) {
const suffix = manualSuffixes.includes(format) ? format : undefined;
@@ -200,22 +221,22 @@ export namespace DashUploadUtils {
/**
* Uploads an image specified by the @param source to Dash's /public/files/
- * directory, and returns information generated during that upload
- *
+ * directory, and returns information generated during that upload
+ *
* @param {string} source is either the absolute path of an already uploaded image or
* the url of a remote image
* @param {string} filename dictates what to call the image. If not specified,
* the name {@param prefix}_upload_{GUID}
* @param {string} prefix is a string prepended to the generated image name in the
* event that @param filename is not specified
- *
+ *
* @returns {ImageUploadInformation | Error} This method returns
* 1) the paths to the uploaded images (plural due to resizing)
* 2) the exif data embedded in the image, or the error explaining why exif couldn't be parsed
* 3) the size of the image, in bytes (4432130)
* 4) the content type of the image, i.e. image/(jpeg | png | ...)
*/
- export const UploadImage = async (source: string, filename?: string, prefix: string = ""): Promise<Upload.ImageInformation | Error> => {
+ export const UploadImage = async (source: string, filename?: string, prefix: string = ''): Promise<Upload.ImageInformation | Error> => {
const metadata = await InspectImage(source);
if (metadata instanceof Error) {
return metadata;
@@ -225,12 +246,12 @@ export namespace DashUploadUtils {
export async function buildFileDirectories() {
if (!existsSync(publicDirectory)) {
- console.error("\nPlease ensure that the following directory exists...\n");
+ console.error('\nPlease ensure that the following directory exists...\n');
console.log(publicDirectory);
process.exit(0);
}
if (!existsSync(filesDirectory)) {
- console.error("\nPlease ensure that the following directory exists...\n");
+ console.error('\nPlease ensure that the following directory exists...\n');
console.log(filesDirectory);
process.exit(0);
}
@@ -252,7 +273,7 @@ export namespace DashUploadUtils {
/**
* Based on the url's classification as local or remote, gleans
* as much information as possible about the specified image
- *
+ *
* @param source is the path or url to the image in question
*/
export const InspectImage = async (source: string): Promise<Upload.InspectionResults | Error> => {
@@ -265,9 +286,9 @@ export namespace DashUploadUtils {
*/
if ((rawMatches = /^data:image\/([a-z]+);base64,(.*)/.exec(source)) !== null) {
const [ext, data] = rawMatches.slice(1, 3);
- const resolved = filename = `upload_${Utils.GenerateGuid()}.${ext}`;
+ const resolved = (filename = `upload_${Utils.GenerateGuid()}.${ext}`);
const error = await new Promise<Error | null>(resolve => {
- writeFile(serverPathToFile(Directory.images, resolved), data, "base64", resolve);
+ writeFile(serverPathToFile(Directory.images, resolved), data, 'base64', resolve);
});
if (error !== null) {
return error;
@@ -276,12 +297,12 @@ export namespace DashUploadUtils {
}
let resolvedUrl: string;
/**
- *
+ *
* At this point, we want to take whatever url we have and make sure it's requestable.
* Anything that's hosted by some other website already is, but if the url is a local file url
* (locates the file on this server machine), we have to resolve the client side url by cutting out the
* basename subtree (i.e. /images/<some_guid>.<ext>) and put it on the end of the server's url.
- *
+ *
* This can always be localhost, regardless of whether this is on the server or not, since we (the server, not the client)
* will be the ones making the request, and from the perspective of dash-release or dash-web, localhost:<port> refers to the same thing
* as the full dash-release.eastus.cloudapp.azure.com:<port>.
@@ -290,18 +311,18 @@ export namespace DashUploadUtils {
if (matches === null) {
resolvedUrl = source;
} else {
- resolvedUrl = `${resolvedServerUrl}/${matches[1].split("\\").join("/")}`;
+ resolvedUrl = `${resolvedServerUrl}/${matches[1].split('\\').join('/')}`;
}
// See header comments: not all image files have exif data (I believe only JPG is the only format that can have it)
const exifData = await parseExifData(resolvedUrl);
const results = {
exifData,
- requestable: resolvedUrl
+ requestable: resolvedUrl,
};
// Use the request library to parse out file level image information in the headers
- const { headers } = (await new Promise<any>((resolve, reject) => {
- request.head(resolvedUrl, (error, res) => error ? reject(error) : resolve(res));
- }).catch(console.error));
+ const { headers } = await new Promise<any>((resolve, reject) => {
+ request.head(resolvedUrl, (error, res) => (error ? reject(error) : resolve(res)));
+ }).catch(console.error);
try {
// Compute the native width and height ofthe image with an npm module
const { width: nativeWidth, height: nativeHeight } = await requestImageSize(resolvedUrl);
@@ -313,7 +334,7 @@ export namespace DashUploadUtils {
nativeWidth,
nativeHeight,
filename,
- ...results
+ ...results,
};
} catch (e: any) {
console.log(e);
@@ -340,12 +361,14 @@ export namespace DashUploadUtils {
rename(sourcePath, destinationPath, error => {
resolve({
source: file,
- result: error ? error : {
- accessPaths: {
- agnostic: getAccessPaths(destination, name)
- },
- rawText: text
- }
+ result: error
+ ? error
+ : {
+ accessPaths: {
+ agnostic: getAccessPaths(destination, name),
+ },
+ rawText: text,
+ },
});
});
});
@@ -354,19 +377,19 @@ export namespace DashUploadUtils {
export function getAccessPaths(directory: Directory, fileName: string) {
return {
client: clientPathToFile(directory, fileName),
- server: serverPathToFile(directory, fileName)
+ server: serverPathToFile(directory, fileName),
};
}
- export const UploadInspectedImage = async (metadata: Upload.InspectionResults, filename?: string, prefix = "", cleanUp = true): Promise<Upload.ImageInformation> => {
+ export const UploadInspectedImage = async (metadata: Upload.InspectionResults, filename?: string, prefix = '', cleanUp = true): Promise<Upload.ImageInformation> => {
const { requestable, source, ...remaining } = metadata;
- const resolved = filename || `${prefix}upload_${Utils.GenerateGuid()}.${remaining.contentType.split("/")[1].toLowerCase()}`;
+ const resolved = filename || `${prefix}upload_${Utils.GenerateGuid()}.${remaining.contentType.split('/')[1].toLowerCase()}`;
const { images } = Directory;
const information: Upload.ImageInformation = {
accessPaths: {
- agnostic: getAccessPaths(images, resolved)
+ agnostic: getAccessPaths(images, resolved),
},
- ...metadata
+ ...metadata,
};
const writtenFiles = await outputResizedImages(() => request(requestable), resolved, pathToDirectory(Directory.images));
for (const suffix of Object.keys(writtenFiles)) {
@@ -383,9 +406,9 @@ export namespace DashUploadUtils {
const val: any = layer[key];
if (val instanceof Buffer) {
layer[key] = val.toString();
- } else if (Array.isArray(val) && typeof val[0] === "number") {
+ } else if (Array.isArray(val) && typeof val[0] === 'number') {
layer[key] = Buffer.from(val).toString();
- } else if (typeof val === "object") {
+ } else if (typeof val === 'object') {
bufferConverterRec(val);
}
}
@@ -410,20 +433,20 @@ export namespace DashUploadUtils {
const pngOptions = {
compressionLevel: 9,
adaptiveFiltering: true,
- force: true
+ force: true,
};
export async function outputResizedImages(streamProvider: () => Stream | Promise<Stream>, outputFileName: string, outputDirectory: string) {
const writtenFiles: { [suffix: string]: string } = {};
for (const { resizer, suffix } of resizers(path.extname(outputFileName))) {
- const outputPath = path.resolve(outputDirectory, writtenFiles[suffix] = InjectSize(outputFileName, suffix));
+ const outputPath = path.resolve(outputDirectory, (writtenFiles[suffix] = InjectSize(outputFileName, suffix)));
await new Promise<void>(async (resolve, reject) => {
const source = streamProvider();
let readStream: Stream = source instanceof Promise ? await source : source;
if (resizer) {
readStream = readStream.pipe(resizer.withMetadata());
}
- readStream.pipe(createWriteStream(outputPath)).on("close", resolve).on("error", reject);
+ readStream.pipe(createWriteStream(outputPath)).on('close', resolve).on('error', reject);
});
}
return writtenFiles;
@@ -442,15 +465,14 @@ export namespace DashUploadUtils {
initial = initial.webp();
} else if (tiffs.includes(ext)) {
initial = initial.tiff();
- } else if (ext === ".gif") {
+ } else if (ext === '.gif') {
initial = undefined;
}
return {
resizer: initial,
- suffix
+ suffix,
};
- })
+ }),
];
}
-
-} \ No newline at end of file
+}
generated by cgit v1.2.3-70-g09d2 (git 2.46.0) at 2025-10-06 10:48:08 +0000