fixes for file uploads: restored progress for youtube videos, fixed info display on loadingBoxes, stop client from asking for progress on failed uploads. attempt to catch stream errors on webpages. fixed dataFieldView in text to print out layout document's field, not field of document that contains the field definition.

1 files changed, 14 insertions, 10 deletions

diff --git a/src/server/DashUploadUtils.ts b/src/server/DashUploadUtils.ts
index ed2430e3a..1f23ae8c1 100644
--- a/src/server/DashUploadUtils.ts
+++ b/src/server/DashUploadUtils.ts
@@ -135,36 +135,39 @@ export namespace DashUploadUtils {
         };
     }
 
-    export function QueryYoutubeProgress(videoId: string) {
-        console.log("PROGRESS:" + videoId)
+    export function QueryYoutubeProgress(videoId: string, user?: Express.User) {
+        console.log(`PROGRESS:${videoId}`, (user as any)?.email);
         return uploadProgress.get(videoId) ?? 'pending data upload';
     }
 
     let uploadProgress = new Map<string, string>();
 
-    export function uploadYoutube(videoId: string): Promise<Upload.FileResponse> {
+    export function uploadYoutube(videoId: string, overwriteId: string): Promise<Upload.FileResponse> {
         return new Promise<Upload.FileResponse<Upload.FileInformation>>((res, rej) => {
             const name = videoId;
             const path = name.replace(/^-/, '__') + '.mp4';
             const finalPath = serverPathToFile(Directory.videos, path);
             if (existsSync(finalPath)) {
-                uploadProgress.set(videoId, 'computing duration');
+                uploadProgress.set(overwriteId, 'computing duration');
                 exec(`yt-dlp -o ${finalPath} "https://www.youtube.com/watch?v=${videoId}" --get-duration`, (error: any, stdout: any, stderr: any) => {
                     const time = Array.from(stdout.trim().split(':')).reverse();
                     const duration = (time.length > 2 ? Number(time[2]) * 1000 * 60 : 0) + (time.length > 1 ? Number(time[1]) * 60 : 0) + (time.length > 0 ? Number(time[0]) : 0);
                     res(resolveExistingFile(name, finalPath, Directory.videos, 'video/mp4', duration, undefined));
                 });
             } else {
-                uploadProgress.set(videoId, 'starting download');
+                uploadProgress.set(overwriteId, 'starting download');
                 const ytdlp = spawn(`yt-dlp`, ['-o', path, `https://www.youtube.com/watch?v=${videoId}`, '--max-filesize', '100M', '-f', 'mp4']);
 
-                ytdlp.stdout.on('data', (data: any) => !uploadProgress.get(videoId)?.includes('Aborting.') && uploadProgress.set(videoId, data.toString()));
+                ytdlp.stdout.on('data', (data: any) => uploadProgress.set(overwriteId, data.toString()));
 
                 let errors = '';
-                ytdlp.stderr.on('data', (data: any) => (errors = data.toString()));
+                ytdlp.stderr.on('data', (data: any) => {
+                    uploadProgress.set(overwriteId, 'error:' + data.toString());
+                    errors = data.toString();
+                });
 
                 ytdlp.on('exit', function (code: any) {
-                    if (code || uploadProgress.get(videoId)?.includes('Aborting.')) {
+                    if (code) {
                         res({
                             source: {
                                 size: 0,
@@ -176,7 +179,7 @@ export namespace DashUploadUtils {
                             result: { name: 'failed youtube query', message: `Could not archive video.  ${code ? errors : uploadProgress.get(videoId)}` },
                         });
                     } else {
-                        uploadProgress.set(videoId, 'computing duration');
+                        uploadProgress.set(overwriteId, 'computing duration');
                         exec(`yt-dlp-o ${path} "https://www.youtube.com/watch?v=${videoId}" --get-duration`, (error: any, stdout: any, stderr: any) => {
                             const time = Array.from(stdout.trim().split(':')).reverse();
                             const duration = (time.length > 2 ? Number(time[2]) * 1000 * 60 : 0) + (time.length > 1 ? Number(time[1]) * 60 : 0) + (time.length > 0 ? Number(time[0]) : 0);
@@ -194,7 +197,7 @@ export namespace DashUploadUtils {
         const isAzureOn = usingAzure();
         const { type, path, name } = file;
         const types = type?.split('/') ?? [];
-        console.log("UPLOADING:"+ (overwriteGuid ?? name));
+        console.log('UPLOADING:' + (overwriteGuid ?? name));
         uploadProgress.set(overwriteGuid ?? name, 'uploading'); // If the client sent a guid it uses to track upload progress, use that guid. Otherwise, use the file's name.
 
         const category = types[0];
@@ -282,6 +285,7 @@ export namespace DashUploadUtils {
         const fileKey = (await md5File(file.path)) + '.pdf';
         const textFilename = `${fileKey.substring(0, fileKey.length - 4)}.txt`;
         if (fExists(fileKey, Directory.pdfs) && fExists(textFilename, Directory.text)) {
+            fs.unlink(file.path, () => {});
             return new Promise<Upload.FileResponse>(res => {
                 const textFilename = `${fileKey.substring(0, fileKey.length - 4)}.txt`;
                 const readStream = createReadStream(serverPathToFile(Directory.text, textFilename));