fixes for file uploads: restored progress for youtube videos, fixed info display on loadingBoxes, stop client from asking for progress on failed uploads. attempt to catch stream errors on webpages. fixed dataFieldView in text to print out layout document's field, not field of document that contains the field definition.

8 files changed, 38 insertions, 28 deletions

diff --git a/src/client/Network.ts b/src/client/Network.ts
index 39bf69e32..631ec9122 100644
--- a/src/client/Network.ts
+++ b/src/client/Network.ts
@@ -74,10 +74,10 @@ export namespace Networking {
         return response.json();
     }
 
-    export async function UploadYoutubeToServer<T extends Upload.FileInformation = Upload.FileInformation>(videoId: string): Promise<Upload.FileResponse<T>[]> {
+    export async function UploadYoutubeToServer<T extends Upload.FileInformation = Upload.FileInformation>(videoId: string, overwriteId?: string): Promise<Upload.FileResponse<T>[]> {
         const parameters = {
             method: 'POST',
-            body: JSON.stringify({ videoId }),
+            body: JSON.stringify({ videoId, overwriteId }),
             json: true,
         };
         const response = await fetch('/uploadYoutubeVideo', parameters);
diff --git a/src/client/documents/Documents.ts b/src/client/documents/Documents.ts
index 90c090f29..bb60cb329 100644
--- a/src/client/documents/Documents.ts
+++ b/src/client/documents/Documents.ts
@@ -1871,7 +1871,7 @@ export namespace DocUtils {
 
     export function uploadYoutubeVideoLoading(videoId: string, options: DocumentOptions, overwriteDoc?: Doc) {
         const generatedDocuments: Doc[] = [];
-        Networking.UploadYoutubeToServer(videoId).then(upfiles => {
+        Networking.UploadYoutubeToServer(videoId, overwriteDoc?.[Id]).then(upfiles => {
             const {
                 source: { name, type },
                 result,
diff --git a/src/client/views/nodes/LoadingBox.scss b/src/client/views/nodes/LoadingBox.scss
index d4a7e18f2..61a505420 100644
--- a/src/client/views/nodes/LoadingBox.scss
+++ b/src/client/views/nodes/LoadingBox.scss
@@ -21,6 +21,7 @@
 
 .textContainer {
     margin: 5px;
+    display: block;
 }
 
 .textContainer {
@@ -31,6 +32,8 @@
 .headerText {
     text-align: center;
     font-weight: bold;
+    height: auto;
+    width: 100%;
 }
 
 .spinner {
diff --git a/src/client/views/nodes/LoadingBox.tsx b/src/client/views/nodes/LoadingBox.tsx
index 0b02626e9..58c70bdd7 100644
--- a/src/client/views/nodes/LoadingBox.tsx
+++ b/src/client/views/nodes/LoadingBox.tsx
@@ -46,7 +46,7 @@ export class LoadingBox extends ViewBoxAnnotatableComponent<FieldViewProps>() {
             const updateFunc = async () => {
                 const result = await Networking.QueryYoutubeProgress(StrCast(this.rootDoc[Id])); // We use the guid of the overwriteDoc to track file uploads.
                 runInAction(() => (this.progress = result.progress));
-                this._timer = setTimeout(updateFunc, 1000);
+                !this.rootDoc.loadingError && (this._timer = setTimeout(updateFunc, 1000));
             };
             this._timer = setTimeout(updateFunc, 1000);
         }
@@ -59,8 +59,8 @@ export class LoadingBox extends ViewBoxAnnotatableComponent<FieldViewProps>() {
         return (
             <div className="loadingBoxContainer" style={{ background: !this.rootDoc.loadingError ? '' : 'red' }}>
                 <div className="textContainer">
-                    <p className="headerText">{StrCast(this.rootDoc.loadingError, 'Loading ' + (this.progress.replace('[download]', '') || '(can take several minutes)'))}</p>
                     <span className="text">{StrCast(this.rootDoc.title)}</span>
+                    <p className="headerText">{StrCast(this.rootDoc.loadingError, 'Loading ' + (this.progress.replace('[download]', '') || '(can take several minutes)'))}</p>
                     {this.rootDoc.loadingError ? null : <ReactLoading type={'spinningBubbles'} color={'blue'} height={100} width={100} />}
                 </div>
             </div>
diff --git a/src/client/views/nodes/formattedText/RichTextRules.ts b/src/client/views/nodes/formattedText/RichTextRules.ts
index 5d92f63ef..3e2afd2ce 100644
--- a/src/client/views/nodes/formattedText/RichTextRules.ts
+++ b/src/client/views/nodes/formattedText/RichTextRules.ts
@@ -279,12 +279,9 @@ export class RichTextRules {
                     const num = value.match(/^[0-9.]$/);
                     this.Document[DocData][fieldKey] = value === 'true' ? true : value === 'false' ? false : num ? Number(value) : value;
                 }
-                const target = DocServer.FindDocByTitle(docTitle) ?? this.Document;
-                if (target) {
-                    const fieldView = state.schema.nodes.dashField.create({ fieldKey, docId: target[Id], hideKey: false });
-                    return state.tr.setSelection(new TextSelection(state.doc.resolve(start), state.doc.resolve(end))).replaceSelectionWith(fieldView, true);
-                }
-                return state.tr;
+                const target = DocServer.FindDocByTitle(docTitle);
+                const fieldView = state.schema.nodes.dashField.create({ fieldKey, docId: target?.[Id], hideKey: false });
+                return state.tr.setSelection(new TextSelection(state.doc.resolve(start), state.doc.resolve(end))).replaceSelectionWith(fieldView, true);
             }),
 
             // create a text display of a metadata field on this or another document, or create a hyperlink portal to another document
diff --git a/src/server/ApiManagers/UploadManager.ts b/src/server/ApiManagers/UploadManager.ts
index ebc9deab7..c264803bd 100644
--- a/src/server/ApiManagers/UploadManager.ts
+++ b/src/server/ApiManagers/UploadManager.ts
@@ -102,11 +102,10 @@ export default class UploadManager extends ApiManager {
                 //req.readableBuffer.head.data
                 return new Promise<void>(async resolve => {
                     req.addListener('data', async args => {
-                        console.log(args);
                         const payload = String.fromCharCode.apply(String, args);
-                        const videoId = JSON.parse(payload).videoId;
+                        const { videoId, overwriteId } = JSON.parse(payload);
                         const results: Upload.FileResponse[] = [];
-                        const result = await DashUploadUtils.uploadYoutube(videoId);
+                        const result = await DashUploadUtils.uploadYoutube(videoId, overwriteId ?? videoId);
                         result && results.push(result);
                         _success(res, results);
                         resolve();
@@ -123,7 +122,7 @@ export default class UploadManager extends ApiManager {
                     req.addListener('data', args => {
                         const payload = String.fromCharCode.apply(String, args);
                         const videoId = JSON.parse(payload).videoId;
-                        _success(res, { progress: DashUploadUtils.QueryYoutubeProgress(videoId) });
+                        _success(res, { progress: DashUploadUtils.QueryYoutubeProgress(videoId, req.user) });
                         resolve();
                     });
                 });
diff --git a/src/server/DashUploadUtils.ts b/src/server/DashUploadUtils.ts
index ed2430e3a..1f23ae8c1 100644
--- a/src/server/DashUploadUtils.ts
+++ b/src/server/DashUploadUtils.ts
@@ -135,36 +135,39 @@ export namespace DashUploadUtils {
         };
     }
 
-    export function QueryYoutubeProgress(videoId: string) {
-        console.log("PROGRESS:" + videoId)
+    export function QueryYoutubeProgress(videoId: string, user?: Express.User) {
+        console.log(`PROGRESS:${videoId}`, (user as any)?.email);
         return uploadProgress.get(videoId) ?? 'pending data upload';
     }
 
     let uploadProgress = new Map<string, string>();
 
-    export function uploadYoutube(videoId: string): Promise<Upload.FileResponse> {
+    export function uploadYoutube(videoId: string, overwriteId: string): Promise<Upload.FileResponse> {
         return new Promise<Upload.FileResponse<Upload.FileInformation>>((res, rej) => {
             const name = videoId;
             const path = name.replace(/^-/, '__') + '.mp4';
             const finalPath = serverPathToFile(Directory.videos, path);
             if (existsSync(finalPath)) {
-                uploadProgress.set(videoId, 'computing duration');
+                uploadProgress.set(overwriteId, 'computing duration');
                 exec(`yt-dlp -o ${finalPath} "https://www.youtube.com/watch?v=${videoId}" --get-duration`, (error: any, stdout: any, stderr: any) => {
                     const time = Array.from(stdout.trim().split(':')).reverse();
                     const duration = (time.length > 2 ? Number(time[2]) * 1000 * 60 : 0) + (time.length > 1 ? Number(time[1]) * 60 : 0) + (time.length > 0 ? Number(time[0]) : 0);
                     res(resolveExistingFile(name, finalPath, Directory.videos, 'video/mp4', duration, undefined));
                 });
             } else {
-                uploadProgress.set(videoId, 'starting download');
+                uploadProgress.set(overwriteId, 'starting download');
                 const ytdlp = spawn(`yt-dlp`, ['-o', path, `https://www.youtube.com/watch?v=${videoId}`, '--max-filesize', '100M', '-f', 'mp4']);
 
-                ytdlp.stdout.on('data', (data: any) => !uploadProgress.get(videoId)?.includes('Aborting.') && uploadProgress.set(videoId, data.toString()));
+                ytdlp.stdout.on('data', (data: any) => uploadProgress.set(overwriteId, data.toString()));
 
                 let errors = '';
-                ytdlp.stderr.on('data', (data: any) => (errors = data.toString()));
+                ytdlp.stderr.on('data', (data: any) => {
+                    uploadProgress.set(overwriteId, 'error:' + data.toString());
+                    errors = data.toString();
+                });
 
                 ytdlp.on('exit', function (code: any) {
-                    if (code || uploadProgress.get(videoId)?.includes('Aborting.')) {
+                    if (code) {
                         res({
                             source: {
                                 size: 0,
@@ -176,7 +179,7 @@ export namespace DashUploadUtils {
                             result: { name: 'failed youtube query', message: `Could not archive video.  ${code ? errors : uploadProgress.get(videoId)}` },
                         });
                     } else {
-                        uploadProgress.set(videoId, 'computing duration');
+                        uploadProgress.set(overwriteId, 'computing duration');
                         exec(`yt-dlp-o ${path} "https://www.youtube.com/watch?v=${videoId}" --get-duration`, (error: any, stdout: any, stderr: any) => {
                             const time = Array.from(stdout.trim().split(':')).reverse();
                             const duration = (time.length > 2 ? Number(time[2]) * 1000 * 60 : 0) + (time.length > 1 ? Number(time[1]) * 60 : 0) + (time.length > 0 ? Number(time[0]) : 0);
@@ -194,7 +197,7 @@ export namespace DashUploadUtils {
         const isAzureOn = usingAzure();
         const { type, path, name } = file;
         const types = type?.split('/') ?? [];
-        console.log("UPLOADING:"+ (overwriteGuid ?? name));
+        console.log('UPLOADING:' + (overwriteGuid ?? name));
         uploadProgress.set(overwriteGuid ?? name, 'uploading'); // If the client sent a guid it uses to track upload progress, use that guid. Otherwise, use the file's name.
 
         const category = types[0];
@@ -282,6 +285,7 @@ export namespace DashUploadUtils {
         const fileKey = (await md5File(file.path)) + '.pdf';
         const textFilename = `${fileKey.substring(0, fileKey.length - 4)}.txt`;
         if (fExists(fileKey, Directory.pdfs) && fExists(textFilename, Directory.text)) {
+            fs.unlink(file.path, () => {});
             return new Promise<Upload.FileResponse>(res => {
                 const textFilename = `${fileKey.substring(0, fileKey.length - 4)}.txt`;
                 const readStream = createReadStream(serverPathToFile(Directory.text, textFilename));
diff --git a/src/server/server_Initialization.ts b/src/server/server_Initialization.ts
index 64f90d76c..839091194 100644
--- a/src/server/server_Initialization.ts
+++ b/src/server/server_Initialization.ts
@@ -194,14 +194,20 @@ function proxyServe(req: any, requrl: string, response: any) {
                         .replace(/target="_blank"/g, '');
                     response.send(header?.includes('gzip') ? zlib.gzipSync(htmlText) : htmlText);
                 } else {
-                    req.pipe(request(requrl)).pipe(response);
+                    req.pipe(request(requrl))
+                        .on('error', (e: any) => console.log('requrl ', e))
+                        .pipe(response)
+                        .on('error', (e: any) => console.log('response pipe error', e));
                     console.log('EMPTY body:' + req.url);
                 }
             } catch (e) {
                 console.log('ERROR?: ', e);
             }
         } else {
-            req.pipe(htmlBodyMemoryStream).pipe(response);
+            req.pipe(htmlBodyMemoryStream)
+                .on('error', (e: any) => console.log('html body memorystream error', e))
+                .pipe(response)
+                .on('error', (e: any) => console.log('html body memory stream response error', e));
         }
     };
     const retrieveHTTPBody = () => {
@@ -237,7 +243,8 @@ function proxyServe(req: any, requrl: string, response: any) {
                 response.headers = response._headers = res.headers;
             })
             .on('end', sendModifiedBody)
-            .pipe(htmlBodyMemoryStream);
+            .pipe(htmlBodyMemoryStream)
+            .on('error', (e: any) => console.log('http body pipe error', e));
     };
     retrieveHTTPBody();
 }