From 945c0cd12fb2a792d10800f81b52cbac8aa12a41 Mon Sep 17 00:00:00 2001
From: mehekj <mehek.jethani@gmail.com>
Date: Tue, 20 Sep 2022 11:52:44 -0400
Subject: display error instead of infinitely loading after refresh

---
 src/client/views/nodes/LoadingBox.tsx | 9 +++++++++
 1 file changed, 9 insertions(+)

(limited to 'src/client/views/nodes/LoadingBox.tsx')

diff --git a/src/client/views/nodes/LoadingBox.tsx b/src/client/views/nodes/LoadingBox.tsx
index acf728ebb..220cd0880 100644
--- a/src/client/views/nodes/LoadingBox.tsx
+++ b/src/client/views/nodes/LoadingBox.tsx
@@ -1,6 +1,8 @@
+import { observable } from 'mobx';
 import { observer } from 'mobx-react';
 import * as React from 'react';
 import ReactLoading from 'react-loading';
+import { Doc } from '../../../fields/Doc';
 import { StrCast } from '../../../fields/Types';
 import { ViewBoxAnnotatableComponent } from '../DocComponent';
 import { FieldView, FieldViewProps } from './FieldView';
@@ -33,6 +35,13 @@ export class LoadingBox extends ViewBoxAnnotatableComponent<FieldViewProps>() {
         return FieldView.LayoutString(LoadingBox, fieldKey);
     }
 
+    componentDidMount() {
+        if (!Doc.CurrentlyLoading || !Doc.CurrentlyLoading.includes(this.rootDoc)) {
+            this.rootDoc.isLoading = false;
+            this.rootDoc.errorMessage = 'Upload was interrupted, please try again';
+        }
+    }
+
     render() {
         return (
             <div className="loadingBoxContainer" style={{ background: this.rootDoc.isLoading ? '' : 'red' }}>
-- 
cgit v1.2.3-70-g09d2


From 48e0885b405cbfe728b1eda78efc19e15a104426 Mon Sep 17 00:00:00 2001
From: bobzel <zzzman@gmail.com>
Date: Thu, 22 Sep 2022 16:00:17 -0400
Subject: fixed error messages on loading to not generate temporary error
 message when there is no error.

---
 src/client/documents/Documents.ts                  | 22 ++++++----------------
 src/client/views/collections/CollectionSubView.tsx |  2 --
 src/client/views/nodes/LoadingBox.tsx              | 12 +++++-------
 3 files changed, 11 insertions(+), 25 deletions(-)

(limited to 'src/client/views/nodes/LoadingBox.tsx')

diff --git a/src/client/documents/Documents.ts b/src/client/documents/Documents.ts
index 029c3033d..f046af684 100644
--- a/src/client/documents/Documents.ts
+++ b/src/client/documents/Documents.ts
@@ -908,7 +908,7 @@ export namespace Docs {
         export function ColorDocument(options: DocumentOptions = {}) {
             return InstanceFromProto(Prototypes.get(DocumentType.COLOR), '', options);
         }
-        export function LoadingDocument(file: File | string, options: DocumentOptions, ytString?: string) {
+        export function LoadingDocument(file: File | string, options: DocumentOptions) {
             return InstanceFromProto(Prototypes.get(DocumentType.LOADING), undefined, { _height: 150, _width: 200, title: typeof file == 'string' ? file : file.name, ...options }, undefined, '');
         }
 
@@ -1758,7 +1758,6 @@ export namespace DocUtils {
         const full = { ...options, _width: 400, title: name };
         const pathname = Utils.prepend(result.accessPaths.agnostic.client);
         const doc = await DocUtils.DocumentFromType(type, pathname, full, rootDoc);
-        rootDoc && (rootDoc.isLoading = undefined) && Doc.removeCurrentlyLoading(rootDoc);
         if (doc) {
             const proto = Doc.GetProto(doc);
             proto.text = result.rawText;
@@ -1790,6 +1789,9 @@ export namespace DocUtils {
             if (Upload.isVideoInformation(result)) {
                 proto['data-duration'] = result.duration;
             }
+            if (rootDoc) {
+                Doc.removeCurrentlyLoading(rootDoc);
+            }
             generatedDocuments.push(doc);
         }
     }
@@ -1818,17 +1820,6 @@ export namespace DocUtils {
         return tbox;
     }
 
-    export async function uploadYoutubeVideo(videoId: string, options: DocumentOptions) {
-        const generatedDocuments: Doc[] = [];
-        for (const {
-            source: { name, type },
-            result,
-        } of await Networking.UploadYoutubeToServer(videoId)) {
-            name && processFileupload(generatedDocuments, name, type, result, options);
-        }
-        return generatedDocuments;
-    }
-
     export function uploadYoutubeVideoLoading(videoId: string, options: DocumentOptions, overwriteDoc?: Doc) {
         const generatedDocuments: Doc[] = [];
         Networking.UploadYoutubeToServer(videoId).then(upfiles => {
@@ -1839,7 +1830,7 @@ export namespace DocUtils {
             if ((result as any).message) {
                 if (overwriteDoc) {
                     overwriteDoc.isLoading = false;
-                    overwriteDoc.errorMessage = (result as any).message;
+                    overwriteDoc.loadingError = (result as any).message;
                     Doc.removeCurrentlyLoading(overwriteDoc);
                 }
             } else name && processFileupload(generatedDocuments, name, type, result, options, overwriteDoc);
@@ -1867,8 +1858,7 @@ export namespace DocUtils {
             } = upfiles.lastElement() ?? { source: { name: '', type: '' }, result: { message: 'upload failed' } };
             if ((result as any).message) {
                 if (overwriteDoc) {
-                    overwriteDoc.isLoading = false;
-                    overwriteDoc.errorMessage = (result as any).message;
+                    overwriteDoc.loadingError = (result as any).message;
                     Doc.removeCurrentlyLoading(overwriteDoc);
                 }
             } else name && type && processFileupload(generatedDocuments, name, type, result, options, overwriteDoc);
diff --git a/src/client/views/collections/CollectionSubView.tsx b/src/client/views/collections/CollectionSubView.tsx
index 3ae965af0..30759b766 100644
--- a/src/client/views/collections/CollectionSubView.tsx
+++ b/src/client/views/collections/CollectionSubView.tsx
@@ -464,14 +464,12 @@ export function CollectionSubView<X>(moreProps?: X) {
             if (typeof files === 'string') {
                 const loading = Docs.Create.LoadingDocument(files, options);
                 generatedDocuments.push(loading);
-                loading.isLoading = true;
                 Doc.addCurrentlyLoading(loading);
                 DocUtils.uploadYoutubeVideoLoading(files, {}, loading);
             } else {
                 generatedDocuments.push(
                     ...files.map(file => {
                         const loading = Docs.Create.LoadingDocument(file, options);
-                        loading.isLoading = true;
                         Doc.addCurrentlyLoading(loading);
                         DocUtils.uploadFileToDoc(file, {}, loading);
                         return loading;
diff --git a/src/client/views/nodes/LoadingBox.tsx b/src/client/views/nodes/LoadingBox.tsx
index 220cd0880..53390328f 100644
--- a/src/client/views/nodes/LoadingBox.tsx
+++ b/src/client/views/nodes/LoadingBox.tsx
@@ -1,4 +1,3 @@
-import { observable } from 'mobx';
 import { observer } from 'mobx-react';
 import * as React from 'react';
 import ReactLoading from 'react-loading';
@@ -36,19 +35,18 @@ export class LoadingBox extends ViewBoxAnnotatableComponent<FieldViewProps>() {
     }
 
     componentDidMount() {
-        if (!Doc.CurrentlyLoading || !Doc.CurrentlyLoading.includes(this.rootDoc)) {
-            this.rootDoc.isLoading = false;
-            this.rootDoc.errorMessage = 'Upload was interrupted, please try again';
+        if (!Doc.CurrentlyLoading?.includes(this.rootDoc)) {
+            this.rootDoc.loadingError = 'Upload interrupted, please try again';
         }
     }
 
     render() {
         return (
-            <div className="loadingBoxContainer" style={{ background: this.rootDoc.isLoading ? '' : 'red' }}>
+            <div className="loadingBoxContainer" style={{ background: !this.rootDoc.loadingError ? '' : 'red' }}>
                 <div className="textContainer">
-                    <p className="headerText">{this.rootDoc.isLoading ? 'Loading (can take several minutes):' : StrCast(this.rootDoc.errorMessage, 'Error Loading File:')}</p>
+                    <p className="headerText">{StrCast(this.rootDoc.loadingError, 'Loading (can take several minutes):')}</p>
                     <span className="text">{StrCast(this.rootDoc.title)}</span>
-                    {!this.rootDoc.isLoading ? null : <ReactLoading type={'spinningBubbles'} color={'blue'} height={100} width={100} />}
+                    {this.rootDoc.loadingError ? null : <ReactLoading type={'spinningBubbles'} color={'blue'} height={100} width={100} />}
                 </div>
             </div>
         );
-- 
cgit v1.2.3-70-g09d2


From 357247845341ef5e92d74883aeea093351d18490 Mon Sep 17 00:00:00 2001
From: bobzel <zzzman@gmail.com>
Date: Fri, 23 Sep 2022 15:29:22 -0400
Subject: added progress messages to youtube uploads

---
 src/client/Network.ts                   |  9 +++++++++
 src/client/views/nodes/LoadingBox.tsx   | 16 ++++++++++++++-
 src/server/ApiManagers/UploadManager.ts | 15 ++++++++++++++
 src/server/DashUploadUtils.ts           | 35 +++++++++------------------------
 4 files changed, 48 insertions(+), 27 deletions(-)

(limited to 'src/client/views/nodes/LoadingBox.tsx')

diff --git a/src/client/Network.ts b/src/client/Network.ts
index 996eb35d8..19eff3b3b 100644
--- a/src/client/Network.ts
+++ b/src/client/Network.ts
@@ -62,4 +62,13 @@ export namespace Networking {
         const response = await fetch('/uploadYoutubeVideo', parameters);
         return response.json();
     }
+    export async function QueryYoutubeProgress(videoId: string): Promise<{ progress: string }> {
+        const parameters = {
+            method: 'POST',
+            body: JSON.stringify({ videoId }),
+            json: true,
+        };
+        const response = await fetch('/queryYoutubeProgress', parameters);
+        return response.json();
+    }
 }
diff --git a/src/client/views/nodes/LoadingBox.tsx b/src/client/views/nodes/LoadingBox.tsx
index 53390328f..8c5255f80 100644
--- a/src/client/views/nodes/LoadingBox.tsx
+++ b/src/client/views/nodes/LoadingBox.tsx
@@ -1,8 +1,10 @@
+import { action, observable, runInAction } from 'mobx';
 import { observer } from 'mobx-react';
 import * as React from 'react';
 import ReactLoading from 'react-loading';
 import { Doc } from '../../../fields/Doc';
 import { StrCast } from '../../../fields/Types';
+import { Networking } from '../../Network';
 import { ViewBoxAnnotatableComponent } from '../DocComponent';
 import { FieldView, FieldViewProps } from './FieldView';
 import './LoadingBox.scss';
@@ -34,17 +36,29 @@ export class LoadingBox extends ViewBoxAnnotatableComponent<FieldViewProps>() {
         return FieldView.LayoutString(LoadingBox, fieldKey);
     }
 
+    _timer: any;
+    @observable progress = '';
     componentDidMount() {
         if (!Doc.CurrentlyLoading?.includes(this.rootDoc)) {
             this.rootDoc.loadingError = 'Upload interrupted, please try again';
+        } else {
+            const updateFunc = async () => {
+                const result = await Networking.QueryYoutubeProgress(StrCast(this.rootDoc.title));
+                runInAction(() => (this.progress = result.progress));
+                this._timer = setTimeout(updateFunc, 1000);
+            };
+            this._timer = setTimeout(updateFunc, 1000);
         }
     }
+    componentWillUnmount() {
+        clearTimeout(this._timer);
+    }
 
     render() {
         return (
             <div className="loadingBoxContainer" style={{ background: !this.rootDoc.loadingError ? '' : 'red' }}>
                 <div className="textContainer">
-                    <p className="headerText">{StrCast(this.rootDoc.loadingError, 'Loading (can take several minutes):')}</p>
+                    <p className="headerText">{StrCast(this.rootDoc.loadingError, 'Loading ' + (this.progress.replace('[download]', '') || '(can take several minutes)'))}</p>
                     <span className="text">{StrCast(this.rootDoc.title)}</span>
                     {this.rootDoc.loadingError ? null : <ReactLoading type={'spinningBubbles'} color={'blue'} height={100} width={100} />}
                 </div>
diff --git a/src/server/ApiManagers/UploadManager.ts b/src/server/ApiManagers/UploadManager.ts
index 76cf36d44..fe4c475c9 100644
--- a/src/server/ApiManagers/UploadManager.ts
+++ b/src/server/ApiManagers/UploadManager.ts
@@ -106,6 +106,21 @@ export default class UploadManager extends ApiManager {
             },
         });
 
+        register({
+            method: Method.POST,
+            subscription: '/queryYoutubeProgress',
+            secureHandler: async ({ req, res }) => {
+                return new Promise<void>(async resolve => {
+                    req.addListener('data', args => {
+                        const payload = String.fromCharCode.apply(String, args);
+                        const videoId = JSON.parse(payload).videoId;
+                        _success(res, { progress: DashUploadUtils.QueryYoutubeProgress(videoId) });
+                        resolve();
+                    });
+                });
+            },
+        });
+
         register({
             method: Method.POST,
             subscription: new RouteSubscriber('youtubeScreenshot'),
diff --git a/src/server/DashUploadUtils.ts b/src/server/DashUploadUtils.ts
index 2c549cc9f..45e88d8cc 100644
--- a/src/server/DashUploadUtils.ts
+++ b/src/server/DashUploadUtils.ts
@@ -117,6 +117,12 @@ export namespace DashUploadUtils {
         };
     }
 
+    export function QueryYoutubeProgress(videoId: string) {
+        return uploadProgress.get(videoId) ?? 'failed';
+    }
+
+    let uploadProgress = new Map<string, string>();
+
     export function uploadYoutube(videoId: string): Promise<Upload.FileResponse> {
         return new Promise<Upload.FileResponse<Upload.FileInformation>>((res, rej) => {
             console.log('Uploading YouTube video: ' + videoId);
@@ -124,6 +130,7 @@ export namespace DashUploadUtils {
             const path = name.replace(/^-/, '__') + '.mp4';
             const finalPath = serverPathToFile(Directory.videos, path);
             if (existsSync(finalPath)) {
+                uploadProgress.set(videoId, 'computing duration');
                 exec(`yt-dlp -o ${finalPath} "https://www.youtube.com/watch?v=${videoId}" --get-duration`, (error: any, stdout: any, stderr: any) => {
                     const time = Array.from(stdout.trim().split(':')).reverse();
                     const duration = (time.length > 2 ? Number(time[2]) * 1000 * 60 : 0) + (time.length > 1 ? Number(time[1]) * 60 : 0) + (time.length > 0 ? Number(time[0]) : 0);
@@ -132,9 +139,7 @@ export namespace DashUploadUtils {
             } else {
                 const ytdlp = spawn(`yt-dlp`, ['-o', path, videoId, '-f', 'mp4']);
 
-                ytdlp.stdout.on('data', function (data: any) {
-                    console.log('stdout: ' + data.toString()); // bcz: somehow channel this back to the client...
-                });
+                ytdlp.stdout.on('data', (data: any) => uploadProgress.set(videoId, data.toString()));
 
                 let errors = '';
                 ytdlp.stderr.on('data', (data: any) => (errors = data.toString()));
@@ -153,6 +158,7 @@ export namespace DashUploadUtils {
                             result: { name: 'failed youtube query', message: `Could not upload video.  ${errors}` },
                         });
                     } else {
+                        uploadProgress.set(videoId, 'computing duration');
                         exec(`yt-dlp-o ${path} "https://www.youtube.com/watch?v=${videoId}" --get-duration`, (error: any, stdout: any, stderr: any) => {
                             const time = Array.from(stdout.trim().split(':')).reverse();
                             const duration = (time.length > 2 ? Number(time[2]) * 1000 * 60 : 0) + (time.length > 1 ? Number(time[1]) * 60 : 0) + (time.length > 0 ? Number(time[0]) : 0);
@@ -162,29 +168,6 @@ export namespace DashUploadUtils {
                         });
                     }
                 });
-                // exec(`yt-dlp -o ${path} "https://www.youtube.com/watch?v=${videoId}" -f "mp4"`, (error: any, stdout: any, stderr: any) => {
-                //     if (error) {
-                //         console.log(`error: Error: ${error.message}`);
-                //         res({
-                //             source: {
-                //                 size: 0,
-                //                 path,
-                //                 name,
-                //                 type: '',
-                //                 toJSON: () => ({ name, path }),
-                //             },
-                //             result: { name: 'failed youtube query', message: `Could not upload YouTube video (${videoId}).  Error: ${error.message}` },
-                //         });
-                //     } else {
-                //         exec(`yt-dlp-o ${path} "https://www.youtube.com/watch?v=${videoId}" --get-duration`, (error: any, stdout: any, stderr: any) => {
-                //             const time = Array.from(stdout.trim().split(':')).reverse();
-                //             const duration = (time.length > 2 ? Number(time[2]) * 1000 * 60 : 0) + (time.length > 1 ? Number(time[1]) * 60 : 0) + (time.length > 0 ? Number(time[0]) : 0);
-                //             const data = { size: 0, path, name, type: 'video/mp4' };
-                //             const file = { ...data, toJSON: () => ({ ...data, filename: data.path.replace(/.*\//, ''), mtime: duration.toString(), mime: '', toJson: () => undefined as any }) };
-                //             res(MoveParsedFile(file, Directory.videos));
-                //         });
-                //     }
-                // });
             }
         });
     }
-- 
cgit v1.2.3-70-g09d2

