From 463a1dff3c08c717ce9b94f92bc12a87ac5344fe Mon Sep 17 00:00:00 2001
From: bobzel <zzzman@gmail.com>
Date: Tue, 29 Aug 2023 00:49:45 -0400
Subject: fixed uploading images to not resize _o image and to not crash if
 image size is too big for resizer.

---
 src/server/DashUploadUtils.ts | 19 ++++++++++++++-----
 1 file changed, 14 insertions(+), 5 deletions(-)

(limited to 'src/server')

diff --git a/src/server/DashUploadUtils.ts b/src/server/DashUploadUtils.ts
index e5e15ce99..2bafd6b7a 100644
--- a/src/server/DashUploadUtils.ts
+++ b/src/server/DashUploadUtils.ts
@@ -385,7 +385,7 @@ export namespace DashUploadUtils {
             const resolved = (filename = `upload_${Utils.GenerateGuid()}.${ext}`);
             if (usingAzure()) {
                 const response = await AzureManager.UploadBase64ImageBlob(resolved, data);
-                source =  `${AzureManager.BASE_STRING}/${resolved}`;
+                source = `${AzureManager.BASE_STRING}/${resolved}`;
             } else {
                 const error = await new Promise<Error | null>(resolve => {
                     writeFile(serverPathToFile(Directory.images, resolved), data, 'base64', resolve);
@@ -596,11 +596,20 @@ export namespace DashUploadUtils {
             const outputPath = path.resolve(outputDirectory, (writtenFiles[suffix] = InjectSize(outputFileName, suffix)));
             await new Promise<void>(async (resolve, reject) => {
                 const source = streamProvider();
-                let readStream: Stream = source instanceof Promise ? await source : source;
+                let readStream = source instanceof Promise ? await source : source;
+                let error = false;
                 if (resizer) {
-                    readStream = readStream.pipe(resizer.withMetadata());
+                    readStream = readStream.pipe(resizer.withMetadata()).on('error', async args => {
+                        error = true;
+                        if (error) {
+                            const source2 = streamProvider();
+                            let readStream2: Stream | undefined;
+                            readStream2 = source2 instanceof Promise ? await source2 : source2;
+                            readStream2?.pipe(createWriteStream(outputPath)).on('error', resolve).on('close', resolve);
+                        }
+                    });
                 }
-                readStream.pipe(createWriteStream(outputPath)).on('close', resolve).on('error', reject);
+                !error && readStream?.pipe(createWriteStream(outputPath)).on('error', resolve).on('close', resolve);
             });
         }
         return writtenFiles;
@@ -628,7 +637,7 @@ export namespace DashUploadUtils {
                     initial = undefined;
                 }
                 return {
-                    resizer: initial,
+                    resizer: suffix === '_o' ? undefined : initial,
                     suffix,
                 };
             }),
-- 
cgit v1.2.3-70-g09d2


From d762e61b4189a7c1b6bc5f13ac35d48d18328788 Mon Sep 17 00:00:00 2001
From: geireann <geireann.lindfield@gmail.com>
Date: Sat, 2 Sep 2023 10:10:04 -0400
Subject: changed upload progress to say pending instead of failed.

---
 src/server/DashUploadUtils.ts | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

(limited to 'src/server')

diff --git a/src/server/DashUploadUtils.ts b/src/server/DashUploadUtils.ts
index 2bafd6b7a..ed2430e3a 100644
--- a/src/server/DashUploadUtils.ts
+++ b/src/server/DashUploadUtils.ts
@@ -136,7 +136,8 @@ export namespace DashUploadUtils {
     }
 
     export function QueryYoutubeProgress(videoId: string) {
-        return uploadProgress.get(videoId) ?? 'failed';
+        console.log("PROGRESS:" + videoId)
+        return uploadProgress.get(videoId) ?? 'pending data upload';
     }
 
     let uploadProgress = new Map<string, string>();
@@ -193,6 +194,7 @@ export namespace DashUploadUtils {
         const isAzureOn = usingAzure();
         const { type, path, name } = file;
         const types = type?.split('/') ?? [];
+        console.log("UPLOADING:"+ (overwriteGuid ?? name));
         uploadProgress.set(overwriteGuid ?? name, 'uploading'); // If the client sent a guid it uses to track upload progress, use that guid. Otherwise, use the file's name.
 
         const category = types[0];
-- 
cgit v1.2.3-70-g09d2


From 2ab7a3130b4b548f26ba8e650a24f4695e3133dc Mon Sep 17 00:00:00 2001
From: bobzel <zzzman@gmail.com>
Date: Sat, 2 Sep 2023 15:30:31 -0400
Subject: restored some cors server code to allow bing and google searches to
 work.  adjusted webBox to change nativeWidth when navigating to a wider page
 (eg., from google search to target). enabled scripts for google and bing
 pages.  enabled entering web page url through documents title.

---
 src/client/views/nodes/WebBox.tsx   | 34 ++++++++++++++++++++++++----------
 src/server/server_Initialization.ts | 22 +++++++++++++++-------
 2 files changed, 39 insertions(+), 17 deletions(-)

(limited to 'src/server')

diff --git a/src/client/views/nodes/WebBox.tsx b/src/client/views/nodes/WebBox.tsx
index 1435dff7c..f5db2ccfd 100644
--- a/src/client/views/nodes/WebBox.tsx
+++ b/src/client/views/nodes/WebBox.tsx
@@ -1,5 +1,5 @@
 import { FontAwesomeIcon } from '@fortawesome/react-fontawesome';
-import { action, computed, IReactionDisposer, observable, ObservableMap, reaction, runInAction } from 'mobx';
+import { action, computed, IReactionDisposer, observable, ObservableMap, reaction, runInAction, trace } from 'mobx';
 import { observer } from 'mobx-react';
 import * as WebRequest from 'web-request';
 import { Doc, DocListCast, Field, Opt } from '../../../fields/Doc';
@@ -196,8 +196,13 @@ export class WebBox extends ViewBoxAnnotatableComponent<ViewBoxAnnotatableProps
         });
         this._disposers.urlchange = reaction(
             () => WebCast(this.rootDoc.data),
+            url => this.submitURL(false, false)
+        );
+        this._disposers.titling = reaction(
+            () => StrCast(this.rootDoc.title),
             url => {
-                this.submitURL(url.url.href, false, false);
+                url.startsWith('www') && this.setData('http://' + url);
+                url.startsWith('http') && this.setData(url);
             }
         );
 
@@ -416,6 +421,7 @@ export class WebBox extends ViewBoxAnnotatableComponent<ViewBoxAnnotatableProps
     }
 
     _iframetimeout: any = undefined;
+    @observable _warning = 0;
     @action
     iframeLoaded = (e: any) => {
         const iframe = this._iframe;
@@ -429,6 +435,7 @@ export class WebBox extends ViewBoxAnnotatableComponent<ViewBoxAnnotatableProps
         try {
             href = iframe?.contentWindow?.location.href;
         } catch (e) {
+            runInAction(() => this._warning++);
             href = undefined;
         }
         let requrlraw = decodeURIComponent(href?.replace(Utils.prepend('') + '/corsProxy/', '') ?? this._url.toString());
@@ -461,12 +468,19 @@ export class WebBox extends ViewBoxAnnotatableComponent<ViewBoxAnnotatableProps
             //     { passive: false }
             // );
             const initHeights = () => {
-                this._scrollHeight = Math.max(this._scrollHeight, (iframeContent.body.children[0] as any)?.scrollHeight || 0);
+                this._scrollHeight = Math.max(this._scrollHeight, iframeContent.body.scrollHeight || 0);
                 if (this._scrollHeight) {
                     this.rootDoc.nativeHeight = Math.min(NumCast(this.rootDoc.nativeHeight), this._scrollHeight);
                     this.layoutDoc.height = Math.min(this.layoutDoc[Height](), (this.layoutDoc[Width]() * NumCast(this.rootDoc.nativeHeight)) / NumCast(this.rootDoc.nativeWidth));
                 }
             };
+            const swidth = Math.max(NumCast(this.layoutDoc.nativeWidth), iframeContent.body.scrollWidth || 0);
+            if (swidth) {
+                const aspectResize = swidth / NumCast(this.rootDoc.nativeWidth);
+                this.rootDoc.nativeWidth = swidth;
+                this.rootDoc.nativeHeight = NumCast(this.rootDoc.nativeHeight) * aspectResize;
+                this.layoutDoc.height = this.layoutDoc[Height]() * aspectResize;
+            }
             initHeights();
             this._iframetimeout && clearTimeout(this._iframetimeout);
             this._iframetimeout = setTimeout(
@@ -544,7 +558,6 @@ export class WebBox extends ViewBoxAnnotatableComponent<ViewBoxAnnotatableProps
 
     goTo = (scrollTop: number, duration: number, easeFunc: 'linear' | 'ease' | undefined) => {
         if (this._outerRef.current) {
-            const iframeHeight = Math.max(scrollTop, this._scrollHeight - this.panelHeight());
             if (duration) {
                 smoothScroll(duration, [this._outerRef.current], scrollTop, easeFunc);
                 this.setDashScrollTop(scrollTop, duration);
@@ -609,9 +622,7 @@ export class WebBox extends ViewBoxAnnotatableComponent<ViewBoxAnnotatableProps
         );
     };
     @action
-    submitURL = (newUrl?: string, preview?: boolean, dontUpdateIframe?: boolean) => {
-        if (!newUrl) return;
-        if (!newUrl.startsWith('http')) newUrl = 'http://' + newUrl;
+    submitURL = (preview?: boolean, dontUpdateIframe?: boolean) => {
         try {
             if (!preview) {
                 if (this._webPageHasBeenRendered) {
@@ -667,9 +678,9 @@ export class WebBox extends ViewBoxAnnotatableComponent<ViewBoxAnnotatableProps
             !Doc.noviceMode &&
                 funcs.push({ description: (this.layoutDoc[this.fieldKey + '_useCors'] ? "Don't Use" : 'Use') + ' Cors', event: () => (this.layoutDoc[this.fieldKey + '_useCors'] = !this.layoutDoc[this.fieldKey + '_useCors']), icon: 'snowflake' });
             funcs.push({
-                description: (this.layoutDoc.allowScripts ? 'Prevent' : 'Allow') + ' Scripts',
+                description: (this.dataDoc[this.fieldKey + '_allowScripts'] ? 'Prevent' : 'Allow') + ' Scripts',
                 event: () => {
-                    this.layoutDoc.allowScripts = !this.layoutDoc.allowScripts;
+                    this.dataDoc[this.fieldKey + '_allowScripts'] = !this.dataDoc[this.fieldKey + '_allowScripts'];
                     if (this._iframe) {
                         runInAction(() => (this._hackHide = true));
                         setTimeout(action(() => (this._hackHide = false)));
@@ -743,8 +754,11 @@ export class WebBox extends ViewBoxAnnotatableComponent<ViewBoxAnnotatableProps
         }
         if (field instanceof WebField) {
             const url = this.layoutDoc[this.fieldKey + '_useCors'] ? Utils.CorsProxy(this._webUrl) : this._webUrl;
+            const scripts = this.dataDoc[this.fieldKey + '_allowScripts'] || this._webUrl.includes('wikipedia.org') || this._webUrl.includes('google.com') || this._webUrl.startsWith('https://bing');
+            //if (!scripts) console.log('No scripts for: ' + url);
             return (
                 <iframe
+                    key={this._warning}
                     className="webBox-iframe"
                     ref={action((r: HTMLIFrameElement | null) => (this._iframe = r))}
                     src={url}
@@ -752,7 +766,7 @@ export class WebBox extends ViewBoxAnnotatableComponent<ViewBoxAnnotatableProps
                     scrolling="no" // ugh.. on windows, I get an inner scroll bar for the iframe's body even though the scrollHeight should be set to the full height of the document.
                     // the 'allow-top-navigation' and 'allow-top-navigation-by-user-activation' attributes are left out to prevent iframes from redirecting the top-level Dash page
                     // sandbox={"allow-forms allow-modals allow-orientation-lock allow-pointer-lock allow-popups allow-popups-to-escape-sandbox allow-presentation allow-same-origin allow-scripts"} />;
-                    sandbox={`${this.layoutDoc.allowScripts ? 'allow-scripts' : ''} allow-forms allow-modals allow-orientation-lock allow-pointer-lock allow-popups allow-popups-to-escape-sandbox allow-presentation allow-same-origin`}
+                    sandbox={`${scripts ? 'allow-scripts' : ''} allow-forms allow-modals allow-orientation-lock allow-pointer-lock allow-popups allow-popups-to-escape-sandbox allow-presentation allow-same-origin`}
                 />
             );
         }
diff --git a/src/server/server_Initialization.ts b/src/server/server_Initialization.ts
index ee32de152..438cde8cf 100644
--- a/src/server/server_Initialization.ts
+++ b/src/server/server_Initialization.ts
@@ -149,15 +149,23 @@ function registerAuthenticationRoutes(server: express.Express) {
 
 function registerCorsProxy(server: express.Express) {
     server.use('/corsProxy', async (req, res) => {
-        //const referer = req.headers.referer ? decodeURIComponent(req.headers.referer) : '';
-        let requrl = decodeURIComponent(req.url.substring(1));
-        const qsplit = requrl.split('?q=');
-        const newqsplit = requrl.split('&q=');
+        const referer = req.headers.referer ? decodeURIComponent(req.headers.referer) : '';
+        let requrlraw = decodeURIComponent(req.url.substring(1));
+        const qsplit = requrlraw.split('?q=');
+        const newqsplit = requrlraw.split('&q=');
         if (qsplit.length > 1 && newqsplit.length > 1) {
             const lastq = newqsplit[newqsplit.length - 1];
-            requrl = qsplit[0] + '?q=' + lastq.split('&')[0] + '&' + qsplit[1].split('&')[1];
+            requrlraw = qsplit[0] + '?q=' + lastq.split('&')[0] + '&' + qsplit[1].split('&')[1];
+        }
+        const requrl = requrlraw.startsWith('/') ? referer + requrlraw : requrlraw;
+        // cors weirdness here...
+        // if the referer is a cors page and the cors() route (I think) redirected to /corsProxy/<path> and the requested url path was relative,
+        // then we redirect again to the cors referer and just add the relative path.
+        if (!requrl.startsWith('http') && req.originalUrl.startsWith('/corsProxy') && referer?.includes('corsProxy')) {
+            res.redirect(referer + (referer.endsWith('/') ? '' : '/') + requrl);
+        } else {
+            proxyServe(req, requrl, res);
         }
-        proxyServe(req, requrl, res);
     });
 }
 
@@ -178,7 +186,7 @@ function proxyServe(req: any, requrl: string, response: any) {
                         .replace('<head>', '<head> <style>[id ^= "google"] { display: none; } </style>')
                         // .replace('<script', '<noscript')
                         // .replace('</script', '</noscript')
-                        // .replace(/href="https?([^"]*)"/g, httpsToCors)
+                        .replace(/href="https?([^"]*)"/g, httpsToCors)
                         .replace(/data-srcset="[^"]*"/g, '')
                         .replace(/srcset="[^"]*"/g, '')
                         .replace(/target="_blank"/g, '');
-- 
cgit v1.2.3-70-g09d2


From d1c661ed9504bfe348f438cb455230cccc070f1a Mon Sep 17 00:00:00 2001
From: bobzel <zzzman@gmail.com>
Date: Tue, 5 Sep 2023 22:47:19 -0400
Subject: small fixes for web pages to get more to load

---
 src/server/server_Initialization.ts | 40 +++++++++++++++++++++++--------------
 1 file changed, 25 insertions(+), 15 deletions(-)

(limited to 'src/server')

diff --git a/src/server/server_Initialization.ts b/src/server/server_Initialization.ts
index 438cde8cf..64f90d76c 100644
--- a/src/server/server_Initialization.ts
+++ b/src/server/server_Initialization.ts
@@ -149,6 +149,9 @@ function registerAuthenticationRoutes(server: express.Express) {
 
 function registerCorsProxy(server: express.Express) {
     server.use('/corsProxy', async (req, res) => {
+        res.setHeader('Access-Control-Allow-Origin', '*');
+        res.header('Access-Control-Allow-Methods', 'GET, PUT, PATCH, POST, DELETE');
+        res.header('Access-Control-Allow-Headers', req.header('access-control-request-headers'));
         const referer = req.headers.referer ? decodeURIComponent(req.headers.referer) : '';
         let requrlraw = decodeURIComponent(req.url.substring(1));
         const qsplit = requrlraw.split('?q=');
@@ -171,26 +174,25 @@ function registerCorsProxy(server: express.Express) {
 
 function proxyServe(req: any, requrl: string, response: any) {
     const htmlBodyMemoryStream = new (require('memorystream'))();
-    var retrieveHTTPBody: any;
     var wasinBrFormat = false;
     const sendModifiedBody = () => {
         const header = response.headers['content-encoding'];
-        const httpsToCors = (match: any, href: string, offset: any, string: any) => `href="${resolvedServerUrl + '/corsProxy/http' + href}"`;
-        if (header?.includes('gzip')) {
+        const refToCors = (match: any, tag: string, sym: string, href: string, offset: any, string: any) => `${tag}=${sym + resolvedServerUrl}/corsProxy/${href + sym}`;
+        const relpathToCors = (match: any, href: string, offset: any, string: any) => `="${resolvedServerUrl + '/corsProxy/' + decodeURIComponent(req.originalUrl.split('/corsProxy/')[1].match(/https?:\/\/[^\/]*/)?.[0] ?? '') + '/' + href}"`;
+        if (header) {
             try {
                 const bodyStream = htmlBodyMemoryStream.read();
                 if (bodyStream) {
-                    const htmlInputText = wasinBrFormat ? Buffer.from(brotli.decompress(bodyStream)) : zlib.gunzipSync(bodyStream);
+                    const htmlInputText = wasinBrFormat ? Buffer.from(brotli.decompress(bodyStream)) : header.includes('gzip') ? zlib.gunzipSync(bodyStream) : bodyStream;
                     const htmlText = htmlInputText
                         .toString('utf8')
                         .replace('<head>', '<head> <style>[id ^= "google"] { display: none; } </style>')
-                        // .replace('<script', '<noscript')
-                        // .replace('</script', '</noscript')
-                        .replace(/href="https?([^"]*)"/g, httpsToCors)
+                        .replace(/(src|href)=([\'\"])(https?[^\2\n]*)\1/g, refToCors) // replace src or href='http(s)://...' or href="http(s)://.."
+                        //.replace(/= *"\/([^"]*)"/g, relpathToCors)
                         .replace(/data-srcset="[^"]*"/g, '')
                         .replace(/srcset="[^"]*"/g, '')
                         .replace(/target="_blank"/g, '');
-                    response.send(zlib.gzipSync(htmlText));
+                    response.send(header?.includes('gzip') ? zlib.gzipSync(htmlText) : htmlText);
                 } else {
                     req.pipe(request(requrl)).pipe(response);
                     console.log('EMPTY body:' + req.url);
@@ -202,10 +204,17 @@ function proxyServe(req: any, requrl: string, response: any) {
             req.pipe(htmlBodyMemoryStream).pipe(response);
         }
     };
-    retrieveHTTPBody = () => {
-        req.headers.cookie = '';
+    const retrieveHTTPBody = () => {
+        //req.headers.cookie = '';
         req.pipe(request(requrl))
-            .on('error', (e: any) => console.log(`Malformed CORS url: ${requrl}`, e))
+            .on('error', (e: any) => {
+                console.log(`CORS url error: ${requrl}`, e);
+                response.send(`<html><body bgcolor="red" link="006666" alink="8B4513" vlink="006666">
+                                    <title>Error</title>
+                                    <div align="center"><h1>Failed to load: ${requrl} </h1></div>
+                                    <p>${e}</p>
+                                </body></html>`);
+            })
             .on('response', (res: any) => {
                 res.headers;
                 const headers = Object.keys(res.headers);
@@ -235,9 +244,10 @@ function proxyServe(req: any, requrl: string, response: any) {
 
 function registerEmbeddedBrowseRelativePathHandler(server: express.Express) {
     server.use('*', (req, res) => {
+        // res.setHeader('Access-Control-Allow-Origin', '*');
+        // res.header('Access-Control-Allow-Methods', 'GET, PUT, PATCH, POST, DELETE');
+        // res.header('Access-Control-Allow-Headers', req.header('access-control-request-headers'));
         const relativeUrl = req.originalUrl;
-        // if (req.originalUrl === '/css/main.css' || req.originalUrl === '/favicon.ico') res.end();
-        // else
         if (!res.headersSent && req.headers.referer?.includes('corsProxy')) {
             if (!req.user) res.redirect('/home'); // When no user is logged in, we interpret a relative URL as being a reference to something they don't have access to and redirect to /home
             // a request for something by a proxied referrer means it must be a relative reference.  So construct a proxied absolute reference here.
@@ -247,8 +257,8 @@ function registerEmbeddedBrowseRelativePathHandler(server: express.Express) {
                 const actualReferUrl = proxiedRefererUrl.replace(dashServerUrl, ''); // the url of the referer without the proxy (e.g., : https://en.wikipedia.org/wiki/Engelbart)
                 const absoluteTargetBaseUrl = actualReferUrl.match(/https?:\/\/[^\/]*/)![0]; // the base of the original url (e.g.,  https://en.wikipedia.org)
                 const redirectedProxiedUrl = dashServerUrl + encodeURIComponent(absoluteTargetBaseUrl + relativeUrl); // the new proxied full url (e.g., http://localhost:<port>/corsProxy/https://en.wikipedia.org/<somethingelse>)
-                if (relativeUrl.startsWith('//')) res.redirect('http:' + relativeUrl);
-                else res.redirect(redirectedProxiedUrl);
+                const redirectUrl = relativeUrl.startsWith('//') ? 'http:' + relativeUrl : redirectedProxiedUrl;
+                res.redirect(redirectUrl);
             } catch (e) {
                 console.log('Error embed: ', e);
             }
-- 
cgit v1.2.3-70-g09d2


From 6499ac4b30c8f6ada2b0c7d74cd5f2a73f9bf180 Mon Sep 17 00:00:00 2001
From: bobzel <zzzman@gmail.com>
Date: Sun, 10 Sep 2023 15:58:09 -0400
Subject: fixes for file uploads: restored progress for youtube videos, fixed
 info display on loadingBoxes,  stop client from asking for progress on failed
 uploads. attempt to catch stream errors on webpages.  fixed dataFieldView in
 text to print out layout document's field, not field of document that
 contains the field definition.

---
 src/client/Network.ts                              |  4 ++--
 src/client/documents/Documents.ts                  |  2 +-
 src/client/views/nodes/LoadingBox.scss             |  3 +++
 src/client/views/nodes/LoadingBox.tsx              |  4 ++--
 .../views/nodes/formattedText/RichTextRules.ts     |  9 +++-----
 src/server/ApiManagers/UploadManager.ts            |  7 +++----
 src/server/DashUploadUtils.ts                      | 24 +++++++++++++---------
 src/server/server_Initialization.ts                | 13 +++++++++---
 8 files changed, 38 insertions(+), 28 deletions(-)

(limited to 'src/server')

diff --git a/src/client/Network.ts b/src/client/Network.ts
index 39bf69e32..631ec9122 100644
--- a/src/client/Network.ts
+++ b/src/client/Network.ts
@@ -74,10 +74,10 @@ export namespace Networking {
         return response.json();
     }
 
-    export async function UploadYoutubeToServer<T extends Upload.FileInformation = Upload.FileInformation>(videoId: string): Promise<Upload.FileResponse<T>[]> {
+    export async function UploadYoutubeToServer<T extends Upload.FileInformation = Upload.FileInformation>(videoId: string, overwriteId?: string): Promise<Upload.FileResponse<T>[]> {
         const parameters = {
             method: 'POST',
-            body: JSON.stringify({ videoId }),
+            body: JSON.stringify({ videoId, overwriteId }),
             json: true,
         };
         const response = await fetch('/uploadYoutubeVideo', parameters);
diff --git a/src/client/documents/Documents.ts b/src/client/documents/Documents.ts
index 90c090f29..bb60cb329 100644
--- a/src/client/documents/Documents.ts
+++ b/src/client/documents/Documents.ts
@@ -1871,7 +1871,7 @@ export namespace DocUtils {
 
     export function uploadYoutubeVideoLoading(videoId: string, options: DocumentOptions, overwriteDoc?: Doc) {
         const generatedDocuments: Doc[] = [];
-        Networking.UploadYoutubeToServer(videoId).then(upfiles => {
+        Networking.UploadYoutubeToServer(videoId, overwriteDoc?.[Id]).then(upfiles => {
             const {
                 source: { name, type },
                 result,
diff --git a/src/client/views/nodes/LoadingBox.scss b/src/client/views/nodes/LoadingBox.scss
index d4a7e18f2..61a505420 100644
--- a/src/client/views/nodes/LoadingBox.scss
+++ b/src/client/views/nodes/LoadingBox.scss
@@ -21,6 +21,7 @@
 
 .textContainer {
     margin: 5px;
+    display: block;
 }
 
 .textContainer {
@@ -31,6 +32,8 @@
 .headerText {
     text-align: center;
     font-weight: bold;
+    height: auto;
+    width: 100%;
 }
 
 .spinner {
diff --git a/src/client/views/nodes/LoadingBox.tsx b/src/client/views/nodes/LoadingBox.tsx
index 0b02626e9..58c70bdd7 100644
--- a/src/client/views/nodes/LoadingBox.tsx
+++ b/src/client/views/nodes/LoadingBox.tsx
@@ -46,7 +46,7 @@ export class LoadingBox extends ViewBoxAnnotatableComponent<FieldViewProps>() {
             const updateFunc = async () => {
                 const result = await Networking.QueryYoutubeProgress(StrCast(this.rootDoc[Id])); // We use the guid of the overwriteDoc to track file uploads.
                 runInAction(() => (this.progress = result.progress));
-                this._timer = setTimeout(updateFunc, 1000);
+                !this.rootDoc.loadingError && (this._timer = setTimeout(updateFunc, 1000));
             };
             this._timer = setTimeout(updateFunc, 1000);
         }
@@ -59,8 +59,8 @@ export class LoadingBox extends ViewBoxAnnotatableComponent<FieldViewProps>() {
         return (
             <div className="loadingBoxContainer" style={{ background: !this.rootDoc.loadingError ? '' : 'red' }}>
                 <div className="textContainer">
-                    <p className="headerText">{StrCast(this.rootDoc.loadingError, 'Loading ' + (this.progress.replace('[download]', '') || '(can take several minutes)'))}</p>
                     <span className="text">{StrCast(this.rootDoc.title)}</span>
+                    <p className="headerText">{StrCast(this.rootDoc.loadingError, 'Loading ' + (this.progress.replace('[download]', '') || '(can take several minutes)'))}</p>
                     {this.rootDoc.loadingError ? null : <ReactLoading type={'spinningBubbles'} color={'blue'} height={100} width={100} />}
                 </div>
             </div>
diff --git a/src/client/views/nodes/formattedText/RichTextRules.ts b/src/client/views/nodes/formattedText/RichTextRules.ts
index 5d92f63ef..3e2afd2ce 100644
--- a/src/client/views/nodes/formattedText/RichTextRules.ts
+++ b/src/client/views/nodes/formattedText/RichTextRules.ts
@@ -279,12 +279,9 @@ export class RichTextRules {
                     const num = value.match(/^[0-9.]$/);
                     this.Document[DocData][fieldKey] = value === 'true' ? true : value === 'false' ? false : num ? Number(value) : value;
                 }
-                const target = DocServer.FindDocByTitle(docTitle) ?? this.Document;
-                if (target) {
-                    const fieldView = state.schema.nodes.dashField.create({ fieldKey, docId: target[Id], hideKey: false });
-                    return state.tr.setSelection(new TextSelection(state.doc.resolve(start), state.doc.resolve(end))).replaceSelectionWith(fieldView, true);
-                }
-                return state.tr;
+                const target = DocServer.FindDocByTitle(docTitle);
+                const fieldView = state.schema.nodes.dashField.create({ fieldKey, docId: target?.[Id], hideKey: false });
+                return state.tr.setSelection(new TextSelection(state.doc.resolve(start), state.doc.resolve(end))).replaceSelectionWith(fieldView, true);
             }),
 
             // create a text display of a metadata field on this or another document, or create a hyperlink portal to another document
diff --git a/src/server/ApiManagers/UploadManager.ts b/src/server/ApiManagers/UploadManager.ts
index ebc9deab7..c264803bd 100644
--- a/src/server/ApiManagers/UploadManager.ts
+++ b/src/server/ApiManagers/UploadManager.ts
@@ -102,11 +102,10 @@ export default class UploadManager extends ApiManager {
                 //req.readableBuffer.head.data
                 return new Promise<void>(async resolve => {
                     req.addListener('data', async args => {
-                        console.log(args);
                         const payload = String.fromCharCode.apply(String, args);
-                        const videoId = JSON.parse(payload).videoId;
+                        const { videoId, overwriteId } = JSON.parse(payload);
                         const results: Upload.FileResponse[] = [];
-                        const result = await DashUploadUtils.uploadYoutube(videoId);
+                        const result = await DashUploadUtils.uploadYoutube(videoId, overwriteId ?? videoId);
                         result && results.push(result);
                         _success(res, results);
                         resolve();
@@ -123,7 +122,7 @@ export default class UploadManager extends ApiManager {
                     req.addListener('data', args => {
                         const payload = String.fromCharCode.apply(String, args);
                         const videoId = JSON.parse(payload).videoId;
-                        _success(res, { progress: DashUploadUtils.QueryYoutubeProgress(videoId) });
+                        _success(res, { progress: DashUploadUtils.QueryYoutubeProgress(videoId, req.user) });
                         resolve();
                     });
                 });
diff --git a/src/server/DashUploadUtils.ts b/src/server/DashUploadUtils.ts
index ed2430e3a..1f23ae8c1 100644
--- a/src/server/DashUploadUtils.ts
+++ b/src/server/DashUploadUtils.ts
@@ -135,36 +135,39 @@ export namespace DashUploadUtils {
         };
     }
 
-    export function QueryYoutubeProgress(videoId: string) {
-        console.log("PROGRESS:" + videoId)
+    export function QueryYoutubeProgress(videoId: string, user?: Express.User) {
+        console.log(`PROGRESS:${videoId}`, (user as any)?.email);
         return uploadProgress.get(videoId) ?? 'pending data upload';
     }
 
     let uploadProgress = new Map<string, string>();
 
-    export function uploadYoutube(videoId: string): Promise<Upload.FileResponse> {
+    export function uploadYoutube(videoId: string, overwriteId: string): Promise<Upload.FileResponse> {
         return new Promise<Upload.FileResponse<Upload.FileInformation>>((res, rej) => {
             const name = videoId;
             const path = name.replace(/^-/, '__') + '.mp4';
             const finalPath = serverPathToFile(Directory.videos, path);
             if (existsSync(finalPath)) {
-                uploadProgress.set(videoId, 'computing duration');
+                uploadProgress.set(overwriteId, 'computing duration');
                 exec(`yt-dlp -o ${finalPath} "https://www.youtube.com/watch?v=${videoId}" --get-duration`, (error: any, stdout: any, stderr: any) => {
                     const time = Array.from(stdout.trim().split(':')).reverse();
                     const duration = (time.length > 2 ? Number(time[2]) * 1000 * 60 : 0) + (time.length > 1 ? Number(time[1]) * 60 : 0) + (time.length > 0 ? Number(time[0]) : 0);
                     res(resolveExistingFile(name, finalPath, Directory.videos, 'video/mp4', duration, undefined));
                 });
             } else {
-                uploadProgress.set(videoId, 'starting download');
+                uploadProgress.set(overwriteId, 'starting download');
                 const ytdlp = spawn(`yt-dlp`, ['-o', path, `https://www.youtube.com/watch?v=${videoId}`, '--max-filesize', '100M', '-f', 'mp4']);
 
-                ytdlp.stdout.on('data', (data: any) => !uploadProgress.get(videoId)?.includes('Aborting.') && uploadProgress.set(videoId, data.toString()));
+                ytdlp.stdout.on('data', (data: any) => uploadProgress.set(overwriteId, data.toString()));
 
                 let errors = '';
-                ytdlp.stderr.on('data', (data: any) => (errors = data.toString()));
+                ytdlp.stderr.on('data', (data: any) => {
+                    uploadProgress.set(overwriteId, 'error:' + data.toString());
+                    errors = data.toString();
+                });
 
                 ytdlp.on('exit', function (code: any) {
-                    if (code || uploadProgress.get(videoId)?.includes('Aborting.')) {
+                    if (code) {
                         res({
                             source: {
                                 size: 0,
@@ -176,7 +179,7 @@ export namespace DashUploadUtils {
                             result: { name: 'failed youtube query', message: `Could not archive video.  ${code ? errors : uploadProgress.get(videoId)}` },
                         });
                     } else {
-                        uploadProgress.set(videoId, 'computing duration');
+                        uploadProgress.set(overwriteId, 'computing duration');
                         exec(`yt-dlp-o ${path} "https://www.youtube.com/watch?v=${videoId}" --get-duration`, (error: any, stdout: any, stderr: any) => {
                             const time = Array.from(stdout.trim().split(':')).reverse();
                             const duration = (time.length > 2 ? Number(time[2]) * 1000 * 60 : 0) + (time.length > 1 ? Number(time[1]) * 60 : 0) + (time.length > 0 ? Number(time[0]) : 0);
@@ -194,7 +197,7 @@ export namespace DashUploadUtils {
         const isAzureOn = usingAzure();
         const { type, path, name } = file;
         const types = type?.split('/') ?? [];
-        console.log("UPLOADING:"+ (overwriteGuid ?? name));
+        console.log('UPLOADING:' + (overwriteGuid ?? name));
         uploadProgress.set(overwriteGuid ?? name, 'uploading'); // If the client sent a guid it uses to track upload progress, use that guid. Otherwise, use the file's name.
 
         const category = types[0];
@@ -282,6 +285,7 @@ export namespace DashUploadUtils {
         const fileKey = (await md5File(file.path)) + '.pdf';
         const textFilename = `${fileKey.substring(0, fileKey.length - 4)}.txt`;
         if (fExists(fileKey, Directory.pdfs) && fExists(textFilename, Directory.text)) {
+            fs.unlink(file.path, () => {});
             return new Promise<Upload.FileResponse>(res => {
                 const textFilename = `${fileKey.substring(0, fileKey.length - 4)}.txt`;
                 const readStream = createReadStream(serverPathToFile(Directory.text, textFilename));
diff --git a/src/server/server_Initialization.ts b/src/server/server_Initialization.ts
index 64f90d76c..839091194 100644
--- a/src/server/server_Initialization.ts
+++ b/src/server/server_Initialization.ts
@@ -194,14 +194,20 @@ function proxyServe(req: any, requrl: string, response: any) {
                         .replace(/target="_blank"/g, '');
                     response.send(header?.includes('gzip') ? zlib.gzipSync(htmlText) : htmlText);
                 } else {
-                    req.pipe(request(requrl)).pipe(response);
+                    req.pipe(request(requrl))
+                        .on('error', (e: any) => console.log('requrl ', e))
+                        .pipe(response)
+                        .on('error', (e: any) => console.log('response pipe error', e));
                     console.log('EMPTY body:' + req.url);
                 }
             } catch (e) {
                 console.log('ERROR?: ', e);
             }
         } else {
-            req.pipe(htmlBodyMemoryStream).pipe(response);
+            req.pipe(htmlBodyMemoryStream)
+                .on('error', (e: any) => console.log('html body memorystream error', e))
+                .pipe(response)
+                .on('error', (e: any) => console.log('html body memory stream response error', e));
         }
     };
     const retrieveHTTPBody = () => {
@@ -237,7 +243,8 @@ function proxyServe(req: any, requrl: string, response: any) {
                 response.headers = response._headers = res.headers;
             })
             .on('end', sendModifiedBody)
-            .pipe(htmlBodyMemoryStream);
+            .pipe(htmlBodyMemoryStream)
+            .on('error', (e: any) => console.log('http body pipe error', e));
     };
     retrieveHTTPBody();
 }
-- 
cgit v1.2.3-70-g09d2


From b42680588ed48d57f497ff15a18a693bb7ecb9b8 Mon Sep 17 00:00:00 2001
From: bobzel <zzzman@gmail.com>
Date: Sun, 10 Sep 2023 16:54:19 -0400
Subject: instrumented pdf uploads

---
 src/server/ApiManagers/UploadManager.ts |  3 +++
 src/server/DashUploadUtils.ts           | 12 +++++++++++-
 2 files changed, 14 insertions(+), 1 deletion(-)

(limited to 'src/server')

diff --git a/src/server/ApiManagers/UploadManager.ts b/src/server/ApiManagers/UploadManager.ts
index c264803bd..854911277 100644
--- a/src/server/ApiManagers/UploadManager.ts
+++ b/src/server/ApiManagers/UploadManager.ts
@@ -64,11 +64,13 @@ export default class UploadManager extends ApiManager {
             subscription: '/uploadFormData',
             secureHandler: async ({ req, res }) => {
                 const form = new formidable.IncomingForm();
+                console.log('RECEIVED form: ');
                 form.keepExtensions = true;
                 form.uploadDir = pathToDirectory(Directory.parsed_files);
                 return new Promise<void>(resolve => {
                     form.parse(req, async (_err, _fields, files) => {
                         const results: Upload.FileResponse[] = [];
+                        console.log('Parsed files: ');
                         if (_err?.message) {
                             results.push({
                                 source: {
@@ -84,6 +86,7 @@ export default class UploadManager extends ApiManager {
                         for (const key in files) {
                             const f = files[key];
                             if (!Array.isArray(f)) {
+                                console.log('uploading file...', f.name);
                                 const result = await DashUploadUtils.upload(f, key); // key is the guid used by the client to track upload progress.
                                 result && !(result.result instanceof Error) && results.push(result);
                             }
diff --git a/src/server/DashUploadUtils.ts b/src/server/DashUploadUtils.ts
index 1f23ae8c1..5187d0d6f 100644
--- a/src/server/DashUploadUtils.ts
+++ b/src/server/DashUploadUtils.ts
@@ -282,6 +282,7 @@ export namespace DashUploadUtils {
     }
 
     async function UploadPdf(file: File) {
+        console.log('upload pdf', file.name);
         const fileKey = (await md5File(file.path)) + '.pdf';
         const textFilename = `${fileKey.substring(0, fileKey.length - 4)}.txt`;
         if (fExists(fileKey, Directory.pdfs) && fExists(textFilename, Directory.text)) {
@@ -290,16 +291,25 @@ export namespace DashUploadUtils {
                 const textFilename = `${fileKey.substring(0, fileKey.length - 4)}.txt`;
                 const readStream = createReadStream(serverPathToFile(Directory.text, textFilename));
                 var rawText = '';
-                readStream.on('data', chunk => (rawText += chunk.toString())).on('end', () => res(resolveExistingFile(file.name, fileKey, Directory.pdfs, file.type, undefined, rawText)));
+                console.log('reading stream', file);
+                readStream
+                    .on('data', chunk => {
+                        console.log('Got chunk:' + chunk.size);
+                        rawText += chunk.toString();
+                    })
+                    .on('end', () => res(resolveExistingFile(file.name, fileKey, Directory.pdfs, file.type, undefined, rawText)));
             });
         }
         const dataBuffer = readFileSync(file.path);
         const result: ParsedPDF | any = await parse(dataBuffer).catch((e: any) => e);
         if (!result.code) {
             await new Promise<void>((resolve, reject) => {
+                console.log('Got text write stream', file.name);
                 const writeStream = createWriteStream(serverPathToFile(Directory.text, textFilename));
                 writeStream.write(result?.text, error => (error ? reject(error) : resolve()));
+                console.log('write text stream', file.name);
             });
+            console.log('moving', file.name);
             return MoveParsedFile(file, Directory.pdfs, undefined, result?.text, undefined, fileKey);
         }
         return { source: file, result: { name: 'faile pdf pupload', message: `Could not upload (${file.name}).${result.message}` } };
-- 
cgit v1.2.3-70-g09d2


From 2e32ffaa398f077c5ddc81e950de010199c67c8d Mon Sep 17 00:00:00 2001
From: bobzel <zzzman@gmail.com>
Date: Sun, 10 Sep 2023 18:11:23 -0400
Subject: added progress for pdf uploads

---
 src/client/Network.ts                   |  5 ++++-
 src/server/ApiManagers/UploadManager.ts | 10 +++++++---
 src/server/DashUploadUtils.ts           | 15 ++-------------
 3 files changed, 13 insertions(+), 17 deletions(-)

(limited to 'src/server')

diff --git a/src/client/Network.ts b/src/client/Network.ts
index 631ec9122..e9f93fad3 100644
--- a/src/client/Network.ts
+++ b/src/client/Network.ts
@@ -58,11 +58,14 @@ export namespace Networking {
                     ])
                 );
             }
+            formData.set('fileguids', fileguidpairs.map(pair => pair.guid).join(';'));
             // If the fileguidpair has a guid to use (From the overwriteDoc) use that guid. Otherwise, generate a new guid.
             fileguidpairs.forEach(fileguidpair => formData.append(fileguidpair.guid ?? Utils.GenerateGuid(), fileguidpair.file));
         } else {
             // Handle the case where fileguidpairs is a single file.
-            formData.append(fileguidpairs.guid ?? Utils.GenerateGuid(), fileguidpairs.file);
+            const guids = fileguidpairs.guid ?? Utils.GenerateGuid();
+            formData.set('fileguids', guids);
+            formData.append(guids, fileguidpairs.file);
         }
         const parameters = {
             method: 'POST',
diff --git a/src/server/ApiManagers/UploadManager.ts b/src/server/ApiManagers/UploadManager.ts
index 854911277..4509425bb 100644
--- a/src/server/ApiManagers/UploadManager.ts
+++ b/src/server/ApiManagers/UploadManager.ts
@@ -64,13 +64,18 @@ export default class UploadManager extends ApiManager {
             subscription: '/uploadFormData',
             secureHandler: async ({ req, res }) => {
                 const form = new formidable.IncomingForm();
-                console.log('RECEIVED form: ');
+                let fileguids = '';
+                form.on('field', (e: string, value: string) => {
+                    if (e === 'fileguids') {
+                        (fileguids = value).split(';').map(guid => DashUploadUtils.uploadProgress.set(guid, 'reading file'));
+                    }
+                });
+                form.on('progress', e => fileguids.split(';').map(guid => DashUploadUtils.uploadProgress.set(guid, 'read: ' + e)));
                 form.keepExtensions = true;
                 form.uploadDir = pathToDirectory(Directory.parsed_files);
                 return new Promise<void>(resolve => {
                     form.parse(req, async (_err, _fields, files) => {
                         const results: Upload.FileResponse[] = [];
-                        console.log('Parsed files: ');
                         if (_err?.message) {
                             results.push({
                                 source: {
@@ -86,7 +91,6 @@ export default class UploadManager extends ApiManager {
                         for (const key in files) {
                             const f = files[key];
                             if (!Array.isArray(f)) {
-                                console.log('uploading file...', f.name);
                                 const result = await DashUploadUtils.upload(f, key); // key is the guid used by the client to track upload progress.
                                 result && !(result.result instanceof Error) && results.push(result);
                             }
diff --git a/src/server/DashUploadUtils.ts b/src/server/DashUploadUtils.ts
index 5187d0d6f..15678bd8d 100644
--- a/src/server/DashUploadUtils.ts
+++ b/src/server/DashUploadUtils.ts
@@ -140,7 +140,7 @@ export namespace DashUploadUtils {
         return uploadProgress.get(videoId) ?? 'pending data upload';
     }
 
-    let uploadProgress = new Map<string, string>();
+    export let uploadProgress = new Map<string, string>();
 
     export function uploadYoutube(videoId: string, overwriteId: string): Promise<Upload.FileResponse> {
         return new Promise<Upload.FileResponse<Upload.FileInformation>>((res, rej) => {
@@ -197,7 +197,6 @@ export namespace DashUploadUtils {
         const isAzureOn = usingAzure();
         const { type, path, name } = file;
         const types = type?.split('/') ?? [];
-        console.log('UPLOADING:' + (overwriteGuid ?? name));
         uploadProgress.set(overwriteGuid ?? name, 'uploading'); // If the client sent a guid it uses to track upload progress, use that guid. Otherwise, use the file's name.
 
         const category = types[0];
@@ -282,7 +281,6 @@ export namespace DashUploadUtils {
     }
 
     async function UploadPdf(file: File) {
-        console.log('upload pdf', file.name);
         const fileKey = (await md5File(file.path)) + '.pdf';
         const textFilename = `${fileKey.substring(0, fileKey.length - 4)}.txt`;
         if (fExists(fileKey, Directory.pdfs) && fExists(textFilename, Directory.text)) {
@@ -291,25 +289,16 @@ export namespace DashUploadUtils {
                 const textFilename = `${fileKey.substring(0, fileKey.length - 4)}.txt`;
                 const readStream = createReadStream(serverPathToFile(Directory.text, textFilename));
                 var rawText = '';
-                console.log('reading stream', file);
-                readStream
-                    .on('data', chunk => {
-                        console.log('Got chunk:' + chunk.size);
-                        rawText += chunk.toString();
-                    })
-                    .on('end', () => res(resolveExistingFile(file.name, fileKey, Directory.pdfs, file.type, undefined, rawText)));
+                readStream.on('data', chunk => (rawText += chunk.toString())).on('end', () => res(resolveExistingFile(file.name, fileKey, Directory.pdfs, file.type, undefined, rawText)));
             });
         }
         const dataBuffer = readFileSync(file.path);
         const result: ParsedPDF | any = await parse(dataBuffer).catch((e: any) => e);
         if (!result.code) {
             await new Promise<void>((resolve, reject) => {
-                console.log('Got text write stream', file.name);
                 const writeStream = createWriteStream(serverPathToFile(Directory.text, textFilename));
                 writeStream.write(result?.text, error => (error ? reject(error) : resolve()));
-                console.log('write text stream', file.name);
             });
-            console.log('moving', file.name);
             return MoveParsedFile(file, Directory.pdfs, undefined, result?.text, undefined, fileKey);
         }
         return { source: file, result: { name: 'faile pdf pupload', message: `Could not upload (${file.name}).${result.message}` } };
-- 
cgit v1.2.3-70-g09d2


From 19f6201748b9b269f7973514ce187023d88f0710 Mon Sep 17 00:00:00 2001
From: bobzel <zzzman@gmail.com>
Date: Sun, 10 Sep 2023 18:49:58 -0400
Subject: turning off command line progress printout to make log more readable.

---
 src/server/DashUploadUtils.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'src/server')

diff --git a/src/server/DashUploadUtils.ts b/src/server/DashUploadUtils.ts
index 15678bd8d..19cb3f240 100644
--- a/src/server/DashUploadUtils.ts
+++ b/src/server/DashUploadUtils.ts
@@ -136,7 +136,7 @@ export namespace DashUploadUtils {
     }
 
     export function QueryYoutubeProgress(videoId: string, user?: Express.User) {
-        console.log(`PROGRESS:${videoId}`, (user as any)?.email);
+        // console.log(`PROGRESS:${videoId}`, (user as any)?.email);
         return uploadProgress.get(videoId) ?? 'pending data upload';
     }
 
-- 
cgit v1.2.3-70-g09d2


From e7657b4f34de651358952a15c1375a61df908f29 Mon Sep 17 00:00:00 2001
From: bobzel <zzzman@gmail.com>
Date: Sun, 10 Sep 2023 19:06:49 -0400
Subject: added upload percent feedback

---
 src/client/Network.ts                   | 2 ++
 src/server/ApiManagers/UploadManager.ts | 6 +++++-
 2 files changed, 7 insertions(+), 1 deletion(-)

(limited to 'src/server')

diff --git a/src/client/Network.ts b/src/client/Network.ts
index e9f93fad3..89b31fdca 100644
--- a/src/client/Network.ts
+++ b/src/client/Network.ts
@@ -59,12 +59,14 @@ export namespace Networking {
                 );
             }
             formData.set('fileguids', fileguidpairs.map(pair => pair.guid).join(';'));
+            formData.set('filesize', fileguidpairs.reduce((sum, pair) => sum + pair.file.size, 0).toString());
             // If the fileguidpair has a guid to use (From the overwriteDoc) use that guid. Otherwise, generate a new guid.
             fileguidpairs.forEach(fileguidpair => formData.append(fileguidpair.guid ?? Utils.GenerateGuid(), fileguidpair.file));
         } else {
             // Handle the case where fileguidpairs is a single file.
             const guids = fileguidpairs.guid ?? Utils.GenerateGuid();
             formData.set('fileguids', guids);
+            formData.set('filesize', fileguidpairs.file.size.toString());
             formData.append(guids, fileguidpairs.file);
         }
         const parameters = {
diff --git a/src/server/ApiManagers/UploadManager.ts b/src/server/ApiManagers/UploadManager.ts
index 4509425bb..826db0de8 100644
--- a/src/server/ApiManagers/UploadManager.ts
+++ b/src/server/ApiManagers/UploadManager.ts
@@ -65,12 +65,16 @@ export default class UploadManager extends ApiManager {
             secureHandler: async ({ req, res }) => {
                 const form = new formidable.IncomingForm();
                 let fileguids = '';
+                let filesize = '';
                 form.on('field', (e: string, value: string) => {
                     if (e === 'fileguids') {
                         (fileguids = value).split(';').map(guid => DashUploadUtils.uploadProgress.set(guid, 'reading file'));
                     }
+                    if (e === 'filesize') {
+                        filesize = value;
+                    }
                 });
-                form.on('progress', e => fileguids.split(';').map(guid => DashUploadUtils.uploadProgress.set(guid, 'read: ' + e)));
+                form.on('progress', e => fileguids.split(';').map(guid => DashUploadUtils.uploadProgress.set(guid, `read:(${+e / +filesize}%) ${e} of ' +${filesize}`)));
                 form.keepExtensions = true;
                 form.uploadDir = pathToDirectory(Directory.parsed_files);
                 return new Promise<void>(resolve => {
-- 
cgit v1.2.3-70-g09d2


From 6e126660d3b4a68d7e5fa6cd7844821270b42a51 Mon Sep 17 00:00:00 2001
From: bobzel <zzzman@gmail.com>
Date: Sun, 10 Sep 2023 19:10:09 -0400
Subject: from last

---
 src/server/ApiManagers/UploadManager.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'src/server')

diff --git a/src/server/ApiManagers/UploadManager.ts b/src/server/ApiManagers/UploadManager.ts
index 826db0de8..58d15e90a 100644
--- a/src/server/ApiManagers/UploadManager.ts
+++ b/src/server/ApiManagers/UploadManager.ts
@@ -74,7 +74,7 @@ export default class UploadManager extends ApiManager {
                         filesize = value;
                     }
                 });
-                form.on('progress', e => fileguids.split(';').map(guid => DashUploadUtils.uploadProgress.set(guid, `read:(${+e / +filesize}%) ${e} of ' +${filesize}`)));
+                form.on('progress', e => fileguids.split(';').map(guid => DashUploadUtils.uploadProgress.set(guid, `read:(${Math.round((100 * +e) / +filesize)}%) ${e} of ' +${filesize}`)));
                 form.keepExtensions = true;
                 form.uploadDir = pathToDirectory(Directory.parsed_files);
                 return new Promise<void>(resolve => {
-- 
cgit v1.2.3-70-g09d2


From 6941f48934ae1762d349ccf1f93c107a4b60f575 Mon Sep 17 00:00:00 2001
From: bobzel <zzzman@gmail.com>
Date: Sun, 10 Sep 2023 19:12:42 -0400
Subject: ugh...from last

---
 src/server/ApiManagers/UploadManager.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'src/server')

diff --git a/src/server/ApiManagers/UploadManager.ts b/src/server/ApiManagers/UploadManager.ts
index 58d15e90a..56a9d9b6b 100644
--- a/src/server/ApiManagers/UploadManager.ts
+++ b/src/server/ApiManagers/UploadManager.ts
@@ -74,7 +74,7 @@ export default class UploadManager extends ApiManager {
                         filesize = value;
                     }
                 });
-                form.on('progress', e => fileguids.split(';').map(guid => DashUploadUtils.uploadProgress.set(guid, `read:(${Math.round((100 * +e) / +filesize)}%) ${e} of ' +${filesize}`)));
+                form.on('progress', e => fileguids.split(';').map(guid => DashUploadUtils.uploadProgress.set(guid, `read:(${Math.round((100 * +e) / +filesize)}%) ${e} of ${filesize}`)));
                 form.keepExtensions = true;
                 form.uploadDir = pathToDirectory(Directory.parsed_files);
                 return new Promise<void>(resolve => {
-- 
cgit v1.2.3-70-g09d2


From 34c64c47444e89bf900227a4f3c56d9023f5d4f3 Mon Sep 17 00:00:00 2001
From: geireann <geireann.lindfield@gmail.com>
Date: Thu, 21 Sep 2023 14:46:26 -0400
Subject: fixed zip export to include media properly.   removed ClientUtils
 which wasn't being used.

---
 src/fields/Doc.ts                   | 29 +++++++++++++++++++----------
 src/server/server_Initialization.ts | 10 +++++-----
 2 files changed, 24 insertions(+), 15 deletions(-)

(limited to 'src/server')

diff --git a/src/fields/Doc.ts b/src/fields/Doc.ts
index 56b97e42f..27da65342 100644
--- a/src/fields/Doc.ts
+++ b/src/fields/Doc.ts
@@ -853,16 +853,24 @@ export namespace Doc {
     export async function Zip(doc: Doc, zipFilename = 'dashExport.zip') {
         const { clone, map, linkMap } = await Doc.MakeClone(doc);
         const proms = new Set<string>();
-        function replacer(key: any, value: any) {
+        function replacer(key: any, value: any) {            
             if (key && ['branchOf', 'cloneOf', 'cursors'].includes(key)) return undefined;
-            if (value instanceof ImageField) {
-                const extension = value.url.href.replace(/.*\./, '');
-                proms.add(value.url.href.replace('.' + extension, '_o.' + extension));
-                return SerializationHelper.Serialize(value);
+            if (value.__type  === 'image') {
+                const extension = value.url.replace(/.*\./, '');
+                proms.add(value.url.replace('.' + extension, '_o.' + extension));
+                return SerializationHelper.Serialize(new ImageField(value.url));
             }
-            if (value instanceof PdfField || value instanceof AudioField || value instanceof VideoField) {
-                proms.add(value.url.href);
-                return SerializationHelper.Serialize(value);
+            if (value.__type  === 'pdf') {
+                proms.add(value.url);
+                return SerializationHelper.Serialize(new PdfField(value.url));
+            }
+            if (value.__type  === 'audio') {
+                proms.add(value.url);
+                return SerializationHelper.Serialize(new AudioField(value.url));
+            }
+            if (value.__type  === 'video') {
+                proms.add(value.url);
+                return SerializationHelper.Serialize(new VideoField(value.url));
             }
             if (
                 value instanceof Doc ||
@@ -889,14 +897,15 @@ export namespace Doc {
 
         const zip = new JSZip();
         var count = 0;
-        const promArr = Array.from(proms).filter(url => url.startsWith(window.location.origin));
+        const promArr = Array.from(proms).filter(url => url?.startsWith("/files")).map(url => url.replace("/",""))// window.location.origin));
+        console.log(promArr.length);
         if (!promArr.length) {
             zip.file('docs.json', jsonDocs);
             zip.generateAsync({ type: 'blob' }).then(content => saveAs(content, zipFilename));
         } else
             promArr.forEach((url, i) => {
                 // loading a file and add it in a zip file
-                JSZipUtils.getBinaryContent(url, (err: any, data: any) => {
+                JSZipUtils.getBinaryContent(window.location.origin+"/"+url, (err: any, data: any) => {
                     if (err) throw err; // or handle the error
                     // // Generate a directory within the Zip file structure
                     // const assets = zip.folder("assets");
diff --git a/src/server/server_Initialization.ts b/src/server/server_Initialization.ts
index 839091194..ccb709453 100644
--- a/src/server/server_Initialization.ts
+++ b/src/server/server_Initialization.ts
@@ -121,11 +121,11 @@ function determineEnvironment() {
     const label = isRelease ? 'release' : 'development';
     console.log(`\nrunning server in ${color(label)} mode`);
 
-    // swilkins: I don't think we need to read from ClientUtils.RELEASE anymore. Should be able to invoke process.env.RELEASE
-    // on the client side, thanks to dotenv in webpack.config.js
-    let clientUtils = fs.readFileSync('./src/client/util/ClientUtils.ts.temp', 'utf8');
-    clientUtils = `//AUTO-GENERATED FILE: DO NOT EDIT\n${clientUtils.replace('"mode"', String(isRelease))}`;
-    fs.writeFileSync('./src/client/util/ClientUtils.ts', clientUtils, 'utf8');
+    // // swilkins: I don't think we need to read from ClientUtils.RELEASE anymore. Should be able to invoke process.env.RELEASE
+    // // on the client side, thanks to dotenv in webpack.config.js
+    // let clientUtils = fs.readFileSync('./src/client/util/ClientUtils.ts.temp', 'utf8');
+    // clientUtils = `//AUTO-GENERATED FILE: DO NOT EDIT\n${clientUtils.replace('"mode"', String(isRelease))}`;
+    // fs.writeFileSync('./src/client/util/ClientUtils.ts', clientUtils, 'utf8');
 
     return isRelease;
 }
-- 
cgit v1.2.3-70-g09d2


From cab10c69667926b162d0613433e873f9911ddafa Mon Sep 17 00:00:00 2001
From: geireann <geireann.lindfield@gmail.com>
Date: Thu, 21 Sep 2023 15:11:13 -0400
Subject: fixed image importing via zip file to do resizing.

---
 src/server/ApiManagers/UploadManager.ts | 30 +++++++++++++++++++++++++++---
 1 file changed, 27 insertions(+), 3 deletions(-)

(limited to 'src/server')

diff --git a/src/server/ApiManagers/UploadManager.ts b/src/server/ApiManagers/UploadManager.ts
index 56a9d9b6b..c337144c8 100644
--- a/src/server/ApiManagers/UploadManager.ts
+++ b/src/server/ApiManagers/UploadManager.ts
@@ -262,9 +262,33 @@ export default class UploadManager extends ApiManager {
                                         zip.extractEntryTo(entry.entryName, publicDirectory, true, false);
                                         createReadStream(pathname).pipe(createWriteStream(targetname));
                                         if (extension !== '.pdf') {
-                                            createReadStream(pathname).pipe(createWriteStream(targetname.replace('_o' + extension, '_s' + extension)));
-                                            createReadStream(pathname).pipe(createWriteStream(targetname.replace('_o' + extension, '_m' + extension)));
-                                            createReadStream(pathname).pipe(createWriteStream(targetname.replace('_o' + extension, '_l' + extension)));
+                                            const { pngs, jpgs } = AcceptableMedia;
+                                            const resizers = [
+                                                    { resizer: sharp().resize(100, undefined, { withoutEnlargement: true }), suffix: SizeSuffix.Small },
+                                                    { resizer: sharp().resize(400, undefined, { withoutEnlargement: true }), suffix: SizeSuffix.Medium },
+                                                    { resizer: sharp().resize(900, undefined, { withoutEnlargement: true }), suffix: SizeSuffix.Large },
+                                                ];
+                                            let isImage = false;
+                                            if (pngs.includes(extension)) {
+                                                resizers.forEach(element => {
+                                                    element.resizer = element.resizer.png();
+                                                });
+                                                isImage = true;
+                                            } else if (jpgs.includes(extension)) {
+                                                resizers.forEach(element => {
+                                                    element.resizer = element.resizer.jpeg();
+                                                });
+                                                isImage = true;
+                                            }
+                                            if (isImage) {
+                                                resizers.forEach(resizer => {
+                                                    createReadStream(pathname)
+                                                        .on('error', e => console.log('Resizing read:' + e))
+                                                        .pipe(resizer.resizer)
+                                                        .on('error', e => console.log('Resizing write: ' + e))
+                                                        .pipe(createWriteStream(targetname.replace('_o' + extension, resizer.suffix + extension)).on('error', e => console.log('Resizing write: ' + e)));
+                                                });
+                                            }
                                         }
                                         unlink(pathname, () => {});
                                     } catch (e) {
-- 
cgit v1.2.3-70-g09d2


From 51d74db190ba898af9f62906ccfd1da097330fb4 Mon Sep 17 00:00:00 2001
From: Sophie Zhang <sophie_zhang@brown.edu>
Date: Fri, 22 Sep 2023 03:14:03 -0400
Subject: fix: fix report manager for videos, only add _l suffix for images

---
 src/client/util/reportManager/ReportManager.tsx    | 28 +++++++---------------
 .../util/reportManager/reportManagerUtils.ts       |  3 ---
 src/server/ApiManagers/UploadManager.ts            |  9 +++----
 3 files changed, 14 insertions(+), 26 deletions(-)

(limited to 'src/server')

diff --git a/src/client/util/reportManager/ReportManager.tsx b/src/client/util/reportManager/ReportManager.tsx
index 802a2f09f..e8c69f909 100644
--- a/src/client/util/reportManager/ReportManager.tsx
+++ b/src/client/util/reportManager/ReportManager.tsx
@@ -333,25 +333,15 @@ export class ReportManager extends React.Component<{}> {
                         fillWidth
                     />
                 </div>
-                <Dropzone
-                    onDrop={this.onDrop}
-                    accept={{
-                        'image/*': ['.png', '.jpg', '.jpeg', '.gif'],
-                        'video/*': ['.mp4', '.mpeg', '.webm', '.mov'],
-                        'audio/mpeg': ['.mp3'],
-                        'audio/wav': ['.wav'],
-                        'audio/ogg': ['.ogg'],
-                    }}>
-                    {({ getRootProps, getInputProps }) => (
-                        <div {...getRootProps({ className: 'dropzone' })} style={{ borderColor: isDarkMode(StrCast(Doc.UserDoc().userBackgroundColor)) ? darkColors.border : lightColors.border }}>
-                            <input {...getInputProps()} />
-                            <div className="dropzone-instructions">
-                                <AiOutlineUpload size={25} />
-                                <p>Drop or select media that shows the bug (optional)</p>
-                            </div>
-                        </div>
-                    )}
-                </Dropzone>
+                <input
+                    type="file"
+                    accept="image/*, video/*, audio/*"
+                    multiple
+                    onChange={e => {
+                        if (!e.target.files) return;
+                        this.setFormData({ ...this.formData, mediaFiles: [...this.formData.mediaFiles, ...Array.from(e.target.files).map(file => ({ _id: v4(), file }))] });
+                    }}
+                />
                 {this.formData.mediaFiles.length > 0 && <ul className="file-list">{this.formData.mediaFiles.map(file => this.getMediaPreview(file))}</ul>}
                 {this.submitting ? (
                     <Button
diff --git a/src/client/util/reportManager/reportManagerUtils.ts b/src/client/util/reportManager/reportManagerUtils.ts
index 948229442..22e5eebbb 100644
--- a/src/client/util/reportManager/reportManagerUtils.ts
+++ b/src/client/util/reportManager/reportManagerUtils.ts
@@ -88,7 +88,6 @@ export const fileLinktoServerLink = (fileLink: string): string => {
 
     const regex = 'public';
     const publicIndex = fileLink.indexOf(regex) + regex.length;
-
     let finalUrl: string = '';
     if (fileLink.includes('.png') || fileLink.includes('.jpg') || fileLink.includes('.jpeg') || fileLink.includes('.gif')) {
         finalUrl = `${serverUrl}${fileLink.substring(publicIndex + 1).replace('.', '_l.')}`;
@@ -114,11 +113,9 @@ export const getServerPath = (link: any): string => {
  * @returns the server paths or undefined on error
  */
 export const uploadFilesToServer = async (mediaFiles: FileData[]): Promise<string[] | undefined> => {
-    console.log(mediaFiles);
     try {
         // need to always upload to browndash
         const links = await Networking.UploadFilesToServer(mediaFiles.map(file => ({ file: file.file })));
-        console.log('Raw links', links);
         return (links ?? []).map(getServerPath).map(fileLinktoServerLink);
     } catch (err) {
         if (err instanceof Error) {
diff --git a/src/server/ApiManagers/UploadManager.ts b/src/server/ApiManagers/UploadManager.ts
index c337144c8..e322bde2d 100644
--- a/src/server/ApiManagers/UploadManager.ts
+++ b/src/server/ApiManagers/UploadManager.ts
@@ -79,6 +79,7 @@ export default class UploadManager extends ApiManager {
                 form.uploadDir = pathToDirectory(Directory.parsed_files);
                 return new Promise<void>(resolve => {
                     form.parse(req, async (_err, _fields, files) => {
+                        console.log('filesss', files);
                         const results: Upload.FileResponse[] = [];
                         if (_err?.message) {
                             results.push({
@@ -264,10 +265,10 @@ export default class UploadManager extends ApiManager {
                                         if (extension !== '.pdf') {
                                             const { pngs, jpgs } = AcceptableMedia;
                                             const resizers = [
-                                                    { resizer: sharp().resize(100, undefined, { withoutEnlargement: true }), suffix: SizeSuffix.Small },
-                                                    { resizer: sharp().resize(400, undefined, { withoutEnlargement: true }), suffix: SizeSuffix.Medium },
-                                                    { resizer: sharp().resize(900, undefined, { withoutEnlargement: true }), suffix: SizeSuffix.Large },
-                                                ];
+                                                { resizer: sharp().resize(100, undefined, { withoutEnlargement: true }), suffix: SizeSuffix.Small },
+                                                { resizer: sharp().resize(400, undefined, { withoutEnlargement: true }), suffix: SizeSuffix.Medium },
+                                                { resizer: sharp().resize(900, undefined, { withoutEnlargement: true }), suffix: SizeSuffix.Large },
+                                            ];
                                             let isImage = false;
                                             if (pngs.includes(extension)) {
                                                 resizers.forEach(element => {
-- 
cgit v1.2.3-70-g09d2


From 780bfc84ab965abdae2e1bb1678db3ddbdb1453b Mon Sep 17 00:00:00 2001
From: Sophie Zhang <sophie_zhang@brown.edu>
Date: Fri, 22 Sep 2023 03:16:29 -0400
Subject: remove prints

---
 src/client/util/reportManager/ReportManager.tsx | 1 -
 src/server/ApiManagers/UploadManager.ts         | 1 -
 2 files changed, 2 deletions(-)

(limited to 'src/server')

diff --git a/src/client/util/reportManager/ReportManager.tsx b/src/client/util/reportManager/ReportManager.tsx
index e8c69f909..b25d51b41 100644
--- a/src/client/util/reportManager/ReportManager.tsx
+++ b/src/client/util/reportManager/ReportManager.tsx
@@ -126,7 +126,6 @@ export class ReportManager extends React.Component<{}> {
         let formattedLinks: string[] = [];
         if (this.formData.mediaFiles.length > 0) {
             const links = await uploadFilesToServer(this.formData.mediaFiles);
-            console.log('Links', links);
             if (links) {
                 formattedLinks = links;
             }
diff --git a/src/server/ApiManagers/UploadManager.ts b/src/server/ApiManagers/UploadManager.ts
index e322bde2d..ea5d8cb33 100644
--- a/src/server/ApiManagers/UploadManager.ts
+++ b/src/server/ApiManagers/UploadManager.ts
@@ -79,7 +79,6 @@ export default class UploadManager extends ApiManager {
                 form.uploadDir = pathToDirectory(Directory.parsed_files);
                 return new Promise<void>(resolve => {
                     form.parse(req, async (_err, _fields, files) => {
-                        console.log('filesss', files);
                         const results: Upload.FileResponse[] = [];
                         if (_err?.message) {
                             results.push({
-- 
cgit v1.2.3-70-g09d2