From 83690c334c8d343d6d7560a6aa7f8e7be9cd3639 Mon Sep 17 00:00:00 2001
From: Michael Foiani <sotech117@michaels-mbp-5.devices.brown.edu>
Date: Tue, 14 Jun 2022 17:48:45 -0400
Subject: done for day - get the functionality down. will make code more
 readable and fix some small bugs tomorrow

---
 src/client/views/nodes/VideoBox.tsx | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'src/client/views/nodes/VideoBox.tsx')

diff --git a/src/client/views/nodes/VideoBox.tsx b/src/client/views/nodes/VideoBox.tsx
index b14a1f0f6..cf3282d2f 100644
--- a/src/client/views/nodes/VideoBox.tsx
+++ b/src/client/views/nodes/VideoBox.tsx
@@ -366,7 +366,7 @@ export class VideoBox extends ViewBoxAnnotatableComponent<ViewBoxAnnotatableProp
                   this._disposers.reactionDisposer = reaction(() => NumCast(this.layoutDoc._currentTimecode),
                         time => {
                               !this._playing && (vref.currentTime = time);
-                              console.log("vref time = " + vref.currentTime)
+                              // console.log("vref time = " + vref.currentTime)
                         }, { fireImmediately: true });
             }
       }
-- 
cgit v1.2.3-70-g09d2


From 57cf1f2417858c203e1018d08e1626b1094ae68d Mon Sep 17 00:00:00 2001
From: Michael Foiani <sotech117@Michaels-MacBook-Pro-5.local>
Date: Tue, 14 Jun 2022 21:47:41 -0400
Subject: change to using mobx to see when videoBox is paused or played

---
 src/client/util/RecordingApi.ts     | 47 +++++++++++++++++++++--------------
 src/client/views/nodes/VideoBox.tsx | 49 +++++++++++++++++++++----------------
 2 files changed, 57 insertions(+), 39 deletions(-)

(limited to 'src/client/views/nodes/VideoBox.tsx')

diff --git a/src/client/util/RecordingApi.ts b/src/client/util/RecordingApi.ts
index 4bbaa9a79..48ea12fd9 100644
--- a/src/client/util/RecordingApi.ts
+++ b/src/client/util/RecordingApi.ts
@@ -61,7 +61,7 @@ export class RecordingApi {
         this.tabChangeDisposeFunc = null;
 
         // for now, set playFFView
-        this.playFFView = null;
+        // this.playFFView = null;
         this.timers = null;
     }
 
@@ -232,19 +232,11 @@ export class RecordingApi {
 
     // TODO: extract this into different class with pause and resume recording
     // TODO: store the FFview with the movements
-    private playFFView: CollectionFreeFormView | null;
-    private timers: NodeJS.Timeout[] | null;
-
-    public setPlayFFView = (view: CollectionFreeFormView): void => {
-        this.playFFView = view
-    }
+    private timers: NodeJS.Timeout[] | null; 
 
     // pausing movements will dispose all timers that are planned to replay the movements
     // play movemvents will recreate them when the user resumes the presentation
     public pauseMovements = (): undefined | Error => {
-        if (this.playFFView === null) {
-            // return new Error('[recordingApi.ts] pauseMovements() failed: no view')
-        }
 
         if (!this._isPlaying) {
             //return new Error('[recordingApi.ts] pauseMovements() failed: not playing')
@@ -257,21 +249,41 @@ export class RecordingApi {
         // this.videoBox = null;
     }
 
-    private videoBox: VideoBox | null = null;
+    private videoBoxDisposeFunc: IReactionDisposer | null = null;
+
+    setVideoBox = (videoBox: VideoBox) => {
+        console.log('setVideoBox', videoBox);
+        if (this.videoBoxDisposeFunc !== null) { console.warn('setVideoBox on already videoBox'); this.videoBoxDisposeFunc(); }
+        this.videoBoxDisposeFunc =
+            reaction(() => ({ playing: videoBox._playing, timeViewed: videoBox.player?.currentTime || 0 }),
+                ({ playing, timeViewed }) => 
+                    playing ? this.playMovements(videoBox.presentation, timeViewed) : this.pauseMovements()
+                );
+    }
+
+    removeVideoBox = () => {
+        if (this.videoBoxDisposeFunc == null) { console.warn('removeVideoBox on null videoBox'); return; }
+        this.videoBoxDisposeFunc();
+    }
+
 
     // by calling pause on the VideoBox, the pauseMovements will be called
-    public pauseVideoAndMovements = (): boolean => {
-        this.videoBox?.Pause()
+    public pauseVideoAndMovements = () => {
+        // this.videoBox?.Pause()
 
         this.pauseMovements()
-        return this.videoBox == null
+        // return this.videoBox == null
     }
 
+
+
     public _isPlaying = false;
 
     public playMovements = (presentation: Presentation, timeViewed: number = 0, videoBox?: VideoBox): undefined | Error => {
-        if (presentation.movements === null) {å //|| this.playFFView === null) {
-            return new Error('[recordingApi.ts] followMovements() failed: no presentation data or no view')
+        console.log('playMovements', presentation, timeViewed, videoBox);
+
+        if (presentation.movements === null) { //|| this.playFFView === null) {
+            return new Error('[recordingApi.ts] followMovements() failed: no presentation data')
         }
         if (this._isPlaying) return;
 
@@ -279,13 +291,12 @@ export class RecordingApi {
         Doc.UserDoc().presentationMode = 'watching';
 
         // TODO: consider this bug at the end of the clip on seek
-        this.videoBox = videoBox || null;
+        // this.videoBox = videoBox || null;
 
         // only get the movements that are remaining in the video time left
         const filteredMovements = presentation.movements.filter(movement => movement.time > timeViewed * 1000)
 
         // helper to replay a movement
-        const document = this.playFFView
         let preScale = -1;
         const zoomAndPan = (movement: Movement, document: CollectionFreeFormView) => {
             const { panX, panY, scale } = movement;
diff --git a/src/client/views/nodes/VideoBox.tsx b/src/client/views/nodes/VideoBox.tsx
index cf3282d2f..ed9bcf29b 100644
--- a/src/client/views/nodes/VideoBox.tsx
+++ b/src/client/views/nodes/VideoBox.tsx
@@ -144,13 +144,23 @@ export class VideoBox extends ViewBoxAnnotatableComponent<ViewBoxAnnotatableProp
                         this.layoutDoc._height = NumCast(this.layoutDoc._width) / youtubeaspect;
                   }
             }
-            this.player && this.setPlayheadTime(0);
+          this.player && this.setPlayheadTime(0);
+          
+          // if presentation data exists, pass it to the recordingPi
+          if (this.presentation != null) {
+            RecordingApi.Instance.setVideoBox(this);  
+          }
       }
 
       componentWillUnmount() {
-            this.removeCurrentlyPlaying();
-            this.Pause();
-            Object.keys(this._disposers).forEach(d => this._disposers[d]?.());
+        this.removeCurrentlyPlaying();
+        this.Pause();
+          Object.keys(this._disposers).forEach(d => this._disposers[d]?.());
+          
+          // dispose the recordingApi's observer
+          if (this.presentation != null) {
+            RecordingApi.Instance.removeVideoBox();  
+          }
       }
 
 
@@ -162,13 +172,13 @@ export class VideoBox extends ViewBoxAnnotatableComponent<ViewBoxAnnotatableProp
             // }
 
             // if presentation isn't null, call followmovements on the recording api
-            if (this.presentation) {
-                  console.log("presentation isn't null")
-                  const err = RecordingApi.Instance.playMovements(this.presentation, this.player?.currentTime || 0, this);
-                  err && console.log(err)
-            } else {
-                  console.log("presentation is null")
-            }
+            // if (this.presentation) {
+            //       console.log("presentation isn't null")
+            //       const err = RecordingApi.Instance.playMovements(this.presentation, this.player?.currentTime || 0, this);
+            //       err && console.log(err)
+            // } else {
+            //       console.log("presentation is null")
+            // }
 
             this._playing = true;
             const eleTime = this.player?.currentTime || 0;
@@ -211,11 +221,11 @@ export class VideoBox extends ViewBoxAnnotatableComponent<ViewBoxAnnotatableProp
 
       // pauses video
       @action public Pause = (update: boolean = true) => {
-            if (this.presentation) {
-                  console.log('VideoBox : Pause');
-                  const err = RecordingApi.Instance.pauseMovements();
-                  err && console.log(err);
-            }
+            // if (this.presentation) {
+            //       console.log('VideoBox : Pause');
+            //       const err = RecordingApi.Instance.pauseMovements();
+            //       err && console.log(err);
+            // }
 
             this._playing = false;
             this.removeCurrentlyPlaying();
@@ -437,12 +447,9 @@ export class VideoBox extends ViewBoxAnnotatableComponent<ViewBoxAnnotatableProp
                               <video key="video" autoPlay={this._screenCapture} ref={this.setVideoRef} style={this._fullScreen ? this.fullScreenSize() : {}}
                                     onCanPlay={this.videoLoad}
                                     controls={VideoBox._nativeControls}
-                                    onPlay={() => {
-                                          // console.log("PLAY from CONTENT")
-                                          //this.Play()
-                                    }}
+                                    onPlay={() => this.Play()}
                                     onSeeked={this.updateTimecode}
-                                    // onPause={() => this.Pause() }
+                                    onPause={() => this.Pause() }
                                     onClick={e => e.preventDefault()}>
                                     <source src={field.url.href} type="video/mp4" />
                                     Not supported.
-- 
cgit v1.2.3-70-g09d2


From fafc1fc678433a95240e8ecefa1b31b7912d5993 Mon Sep 17 00:00:00 2001
From: Michael Foiani <sotech117@michaels-mbp-5.devices.brown.edu>
Date: Wed, 15 Jun 2022 15:18:14 -0400
Subject: refactor code, fix bugs, make first movement on screen the beginning
 of the presentation.

---
 src/client/util/RecordingApi.ts     | 195 +++++++++++++++++++++++++-----------
 src/client/views/nodes/VideoBox.tsx | 150 +++++++++++++--------------
 2 files changed, 210 insertions(+), 135 deletions(-)

(limited to 'src/client/views/nodes/VideoBox.tsx')

diff --git a/src/client/util/RecordingApi.ts b/src/client/util/RecordingApi.ts
index ae5431a03..87cb85497 100644
--- a/src/client/util/RecordingApi.ts
+++ b/src/client/util/RecordingApi.ts
@@ -57,7 +57,7 @@ export class RecordingApi {
         this.absoluteStart = -1;
 
         // used for tracking movements in the view frame
-        this.recordingFFViews = new Map();
+        this.recordingFFViews = null;
         this.tabChangeDisposeFunc = null;
 
         // for now, set playFFView
@@ -127,6 +127,10 @@ export class RecordingApi {
     }
 
     public initTabTracker = () => {
+        if (this.recordingFFViews === null) {
+            this.recordingFFViews = new Map();
+        }
+
         // init the dispose funcs on the page
         const docList = DocListCast(CollectionDockingView.Instance.props.Document.data);
         this.updateRecordingFFViewsFromTabs(docList);
@@ -142,6 +146,8 @@ export class RecordingApi {
     }
 
     public start = (meta?: Object) => {
+        this.initTabTracker();
+
         // update the presentation mode
         Doc.UserDoc().presentationMode = 'recording';
 
@@ -190,6 +196,9 @@ export class RecordingApi {
             this.tabChangeDisposeFunc?.();
             // update the presentation mode now that we are done tracking
             Doc.UserDoc().presentationMode = 'none';
+
+            this.recordingFFViews = null;
+            this.tabChangeDisposeFunc = null;
         }
 
         // clear presenation data
@@ -250,13 +259,28 @@ export class RecordingApi {
     private videoBoxDisposeFunc: IReactionDisposer | null = null;
     private videoBox: VideoBox | null = null;
 
-    setVideoBox = (videoBox: VideoBox) => {
+    setVideoBox = async (videoBox: VideoBox) => {
         console.log('setVideoBox', videoBox);
-        if (this.videoBoxDisposeFunc !== null) { console.warn('setVideoBox on already videoBox'); this.videoBoxDisposeFunc(); }
+        if (videoBox !== null) { console.warn('setVideoBox on already videoBox'); }
+        if (this.videoBoxDisposeFunc !== null) { console.warn('setVideoBox on already videoBox dispose func'); this.videoBoxDisposeFunc(); }
+
+
+        const { presentation } = videoBox;
+        if (presentation == null) { console.warn('setVideoBox on null videoBox presentation'); return; }
+        
+        let docIdtoDoc: Map<string, Doc> = new Map();
+        try {
+            docIdtoDoc = await this.loadPresentation(presentation);
+        } catch {
+            console.error('[recordingApi.ts] setVideoBox(): error loading presentation - no replay movements');
+            throw 'error loading docs from server';
+        }
+
+        
         this.videoBoxDisposeFunc =
             reaction(() => ({ playing: videoBox._playing, timeViewed: videoBox.player?.currentTime || 0 }),
                 ({ playing, timeViewed }) => 
-                    playing ? this.playMovements(videoBox.presentation, timeViewed) : this.pauseMovements()
+                    playing ? this.playMovements(presentation, docIdtoDoc, timeViewed) : this.pauseMovements()
             );
         this.videoBox = videoBox;
     }
@@ -283,10 +307,81 @@ export class RecordingApi {
 
     public _isPlaying = false;
 
-    public playMovements = (presentation: Presentation, timeViewed: number = 0, videoBox?: VideoBox): undefined | Error => {
-        console.log('playMovements', presentation, timeViewed, videoBox);
+    loadPresentation = async (presentation: Presentation) => {
+        const { movements } = presentation;
+        if (movements === null) {
+            throw '[recordingApi.ts] followMovements() failed: no presentation data';
+        }
+
+        // generate a set of all unique docIds
+        const docIds = new Set<string>();
+        for (const {docId} of movements) {
+            if (!docIds.has(docId)) docIds.add(docId);
+        }
+
+        const docIdtoDoc = new Map<string, Doc>();
+
+        let refFields = await DocServer.GetRefFields([...docIds.keys()]);
+        for (const docId in refFields) {
+            if (!refFields[docId]) {
+                throw `one field was undefined`;
+            }
+            docIdtoDoc.set(docId, refFields[docId] as Doc);
+        }
+        console.log('loadPresentation refFields', refFields, docIdtoDoc);
+
+        return docIdtoDoc;
+    }
+
+    // returns undefined if the docView isn't open on the screen
+    getCollectionFFView = (docId: string) => {
+        const isInView = DocumentManager.Instance.getDocumentViewById(docId);
+        if (isInView) { return isInView.ComponentView as CollectionFreeFormView; }
+    }
+
+    // will open the doc in a tab then return the CollectionFFView that holds it
+    openTab = (docId: string, docIdtoDoc: Map<string, Doc>) => {
+        const doc = docIdtoDoc.get(docId);
+        if (doc == undefined) {
+            console.error(`docIdtoDoc did not contain docId ${docId}`)
+            return undefined;
+        }
+        CollectionDockingView.AddSplit(doc, 'right');
+        const docView = DocumentManager.Instance.getDocumentViewById(docId);
+        return docView?.ComponentView as CollectionFreeFormView;
+    }
+
+    // helper to replay a movement
+    private preScale = -1;
+    zoomAndPan = (movement: Movement, document: CollectionFreeFormView) => {
+        const { panX, panY, scale } = movement;
+        (scale !== 0 && this.preScale !== scale) && document.zoomSmoothlyAboutPt([panX, panY], scale, 0);
+        document.Document._panX = panX;
+        document.Document._panY = panY;
+
+        this.preScale = scale;
+    }
+
+    getFirstMovements = (movements: Movement[], timeViewed: number): Map<string, Movement> => {
+        if (movements === null) return new Map();
+        // generate a set of all unique docIds
+        const docIdtoFirstMove = new Map();
+        for (const move of movements) {
+            const { docId } = move;
+            if (!docIdtoFirstMove.has(docId)) docIdtoFirstMove.set(docId, move);
+        }
+        return docIdtoFirstMove;
+    }
+
+    endPlayingPresentation = () => {
+        this.preScale = -1;
+        RecordingApi.Instance._isPlaying = false;
+    }
+
+    public playMovements = (presentation: Presentation, docIdtoDoc: Map<string, Doc>, timeViewed: number = 0) => {
+        console.log('playMovements', presentation, timeViewed, docIdtoDoc);
 
-        if (presentation.movements === null) { //|| this.playFFView === null) {
+        if (presentation.movements === null || presentation.movements.length === 0) { //|| this.playFFView === null) {
             return new Error('[recordingApi.ts] followMovements() failed: no presentation data')
         }
         if (this._isPlaying) return;
@@ -294,65 +389,45 @@ export class RecordingApi {
         this._isPlaying = true;
         Doc.UserDoc().presentationMode = 'watching';
 
-        // TODO: consider this bug at the end of the clip on seek
-        // this.videoBox = videoBox || null;
-
         // only get the movements that are remaining in the video time left
         const filteredMovements = presentation.movements.filter(movement => movement.time > timeViewed * 1000)
 
-        // helper to replay a movement
-        let preScale = -1;
-        const zoomAndPan = (movement: Movement, document: CollectionFreeFormView) => {
-            const { panX, panY, scale } = movement;
-            (scale !== 0 && preScale !== scale) && document.zoomSmoothlyAboutPt([panX, panY], scale, 0);
-            document.Document._panX = panX;
-            document.Document._panY = panY;
-
-            preScale = scale;
+        const handleFirstMovements = () => {
+            // if the first movement is a closed tab, open it
+            const firstMovement = filteredMovements[0];
+            const isClosed = this.getCollectionFFView(firstMovement.docId) === undefined;
+            if (isClosed) this.openTab(firstMovement.docId, docIdtoDoc);
+
+            // for the open tabs, set it to the first move
+            const docIdtoFirstMove = this.getFirstMovements(filteredMovements, timeViewed);
+            for (const [docId, firstMove] of docIdtoFirstMove) {
+                const colFFView = this.getCollectionFFView(docId);
+                if (colFFView) this.zoomAndPan(firstMove, colFFView);
+            }
         }
-        // set the first frame to be at the start of the pres
-        // zoomAndPan(filteredMovements[0]);
-
-        // generate a set of all unique docIds
-        const docIds = new Set(filteredMovements.map(movement => movement.docId))
-        // TODO: optimize only ons-first load
-        // TODO: make async await
-        // TODO: make sure the cahce still hs the id
-        // TODO: if they are open, set them to their first move
-        // this will load the cache, so getCachedREfFields won't have to reach server
-        DocServer.GetRefFields([...docIds]).then(refFields => { 
-            console.log('refFields', refFields)
-
-            const openTab = (docId: string) : DocumentView | undefined => {
-                const isInView = DocumentManager.Instance.getDocumentViewById(docId);
-                if (isInView) { return isInView; }
-
-                const doc = DocServer.GetCachedRefField(docId) as Doc;
-                if (doc == undefined) {
-                    console.warn('Doc server cache did not contain docId', docId)
-                    return undefined;
+        handleFirstMovements();
+        
+
+        // make timers that will execute each movement at the correct replay time
+        this.timers = filteredMovements.map(movement => {
+            const timeDiff = movement.time - timeViewed * 1000
+
+            return setTimeout(() => {
+                const collectionFFView = this.getCollectionFFView(movement.docId);
+                if (collectionFFView) {
+                    this.zoomAndPan(movement, collectionFFView);
+                } else {
+                    // tab wasn't open - open it and play the movement
+                    const openedColFFView = this.openTab(movement.docId, docIdtoDoc);
+                    openedColFFView && this.zoomAndPan(movement, openedColFFView);
                 }
-                CollectionDockingView.AddSplit(doc, 'right');
-                return DocumentManager.Instance.getDocumentView(doc);
-            }
 
-            // make timers that will execute each movement at the correct replay time
-            this.timers = filteredMovements.map(movement => {
-                const timeDiff = movement.time - timeViewed * 1000
-                return setTimeout(() => {
-                    // open tab if it is not already open
-                    const view = openTab(movement.docId);
-                    if (view) { 
-                        const collectionFFView = view.ComponentView;
-                        console.log(collectionFFView instanceof CollectionFreeFormView)
-                        // replay the movement
-                        zoomAndPan(movement, collectionFFView as CollectionFreeFormView);
-                    }
-                    // if last movement, presentation is done -> set the instance var
-                    if (movement === filteredMovements[filteredMovements.length - 1]) RecordingApi.Instance._isPlaying = false;
-                }, timeDiff)
-            });
-        })
+                // if last movement, presentation is done -> cleanup :)
+                if (movement === filteredMovements[filteredMovements.length - 1]) {
+                    this.endPlayingPresentation();
+                }
+            }, timeDiff);
+        });
     }
 
     // method that concatenates an array of presentatations into one
diff --git a/src/client/views/nodes/VideoBox.tsx b/src/client/views/nodes/VideoBox.tsx
index ed9bcf29b..34df03954 100644
--- a/src/client/views/nodes/VideoBox.tsx
+++ b/src/client/views/nodes/VideoBox.tsx
@@ -28,7 +28,7 @@ import { AnchorMenu } from "../pdf/AnchorMenu";
 import { StyleProp } from "../StyleProvider";
 import { FieldView, FieldViewProps } from './FieldView';
 import "./VideoBox.scss";
-import { RecordingApi } from "../../util/RecordingApi";
+import { Presentation, RecordingApi } from "../../util/RecordingApi";
 import { List } from "../../../fields/List";
 import { RecordingBox } from "./RecordingBox";
 const path = require('path');
@@ -48,80 +48,80 @@ const path = require('path');
 
 @observer
 export class VideoBox extends ViewBoxAnnotatableComponent<ViewBoxAnnotatableProps & FieldViewProps>() {
-      public static LayoutString(fieldKey: string) { return FieldView.LayoutString(VideoBox, fieldKey); }
-      /**
-       * Uploads an image buffer to the server and stores with specified filename.  by default the image
-       * is stored at multiple resolutions each retrieved by using the filename appended with _o, _s, _m, _l (indicating original, small, medium, or large)
-       * @param imageUri the bytes of the image
-       * @param returnedFilename the base filename to store the image on the server
-       * @param nosuffix optionally suppress creating multiple resolution images
-       */
-      public static async convertDataUri(imageUri: string, returnedFilename: string, nosuffix = false, replaceRootFilename?: string) {
-            try {
-                  const posting = Utils.prepend("/uploadURI");
-                  const returnedUri = await rp.post(posting, {
-                        body: {
-                              uri: imageUri,
-                              name: returnedFilename,
-                              nosuffix,
-                              replaceRootFilename
-                        },
-                        json: true,
-                  });
-                  return returnedUri;
-
-            } catch (e) {
-                  console.log("VideoBox :" + e);
-            }
-      }
-
-      static _youtubeIframeCounter: number = 0;
-      static heightPercent = 80; // height of video relative to videoBox when timeline is open
-      private _disposers: { [name: string]: IReactionDisposer } = {};
-      private _youtubePlayer: YT.Player | undefined = undefined;
-      private _videoRef: HTMLVideoElement | null = null; // <video> ref
-      private _contentRef: HTMLDivElement | null = null; // ref to div that wraps video and controls for full screen
-      private _youtubeIframeId: number = -1;
-      private _youtubeContentCreated = false;
-      private _audioPlayer: HTMLAudioElement | null = null;
-      private _mainCont: React.RefObject<HTMLDivElement> = React.createRef(); // outermost div
-      private _annotationLayer: React.RefObject<HTMLDivElement> = React.createRef();
-      private _playRegionTimer: any = null; // timeout for playback
-      @observable _stackedTimeline: any; // CollectionStackedTimeline ref
-      @observable static _nativeControls: boolean; // default html controls
-      @observable _marqueeing: number[] | undefined; // coords for marquee selection
-      @observable _savedAnnotations = new ObservableMap<number, HTMLDivElement[]>();
-      @observable _screenCapture = false;
-      @observable _clicking = false; // used for transition between showing/hiding timeline
-      @observable _forceCreateYouTubeIFrame = false;
-      @observable _playTimer?: NodeJS.Timeout = undefined;
-      @observable _fullScreen = false;
-      @observable _playing = false;
-      @observable _finished: boolean = false; // has playback reached end of clip
-      @observable _volume: number = 1;
-      @observable _muted: boolean = false;
-
-      @computed get links() { return DocListCast(this.dataDoc.links); }
-      @computed get heightPercent() { return NumCast(this.layoutDoc._timelineHeightPercent, 100); } // current percent of video relative to VideoBox height
-      // @computed get rawDuration() { return NumCast(this.dataDoc[this.fieldKey + "-duration"]); }
-      @observable rawDuration: number = 0;
-
-
-      @computed get youtubeVideoId() {
-            const field = Cast(this.dataDoc[this.props.fieldKey], VideoField);
-            return field && field.url.href.indexOf("youtube") !== -1 ? ((arr: string[]) => arr[arr.length - 1])(field.url.href.split("/")) : "";
-      }
-
-
-      // returns the path of the audio file
-      @computed get audiopath() {
-            const field = Cast(this.props.Document[this.props.fieldKey + '-audio'], AudioField, null);
-            const vfield = Cast(this.dataDoc[this.fieldKey], VideoField, null);
-            return field?.url.href ?? vfield?.url.href ?? "";
-      }
-
-      // returns the presentation data if it exists, null otherwise
-      @computed get presentation() {
+    public static LayoutString(fieldKey: string) { return FieldView.LayoutString(VideoBox, fieldKey); }
+    /**
+     * Uploads an image buffer to the server and stores with specified filename.  by default the image
+     * is stored at multiple resolutions each retrieved by using the filename appended with _o, _s, _m, _l (indicating original, small, medium, or large)
+     * @param imageUri the bytes of the image
+     * @param returnedFilename the base filename to store the image on the server
+     * @param nosuffix optionally suppress creating multiple resolution images
+     */
+    public static async convertDataUri(imageUri: string, returnedFilename: string, nosuffix = false, replaceRootFilename?: string) {
+        try {
+            const posting = Utils.prepend("/uploadURI");
+            const returnedUri = await rp.post(posting, {
+                body: {
+                    uri: imageUri,
+                    name: returnedFilename,
+                    nosuffix,
+                    replaceRootFilename
+                },
+                json: true,
+            });
+            return returnedUri;
+
+        } catch (e) {
+            console.log("VideoBox :" + e);
+        }
+    }
+
+    static _youtubeIframeCounter: number = 0;
+    static heightPercent = 80; // height of video relative to videoBox when timeline is open
+    private _disposers: { [name: string]: IReactionDisposer } = {};
+    private _youtubePlayer: YT.Player | undefined = undefined;
+    private _videoRef: HTMLVideoElement | null = null; // <video> ref
+    private _contentRef: HTMLDivElement | null = null; // ref to div that wraps video and controls for full screen
+    private _youtubeIframeId: number = -1;
+    private _youtubeContentCreated = false;
+    private _audioPlayer: HTMLAudioElement | null = null;
+    private _mainCont: React.RefObject<HTMLDivElement> = React.createRef(); // outermost div
+    private _annotationLayer: React.RefObject<HTMLDivElement> = React.createRef();
+    private _playRegionTimer: any = null; // timeout for playback
+    @observable _stackedTimeline: any; // CollectionStackedTimeline ref
+    @observable static _nativeControls: boolean; // default html controls
+    @observable _marqueeing: number[] | undefined; // coords for marquee selection
+    @observable _savedAnnotations = new ObservableMap<number, HTMLDivElement[]>();
+    @observable _screenCapture = false;
+    @observable _clicking = false; // used for transition between showing/hiding timeline
+    @observable _forceCreateYouTubeIFrame = false;
+    @observable _playTimer?: NodeJS.Timeout = undefined;
+    @observable _fullScreen = false;
+    @observable _playing = false;
+    @observable _finished: boolean = false; // has playback reached end of clip
+    @observable _volume: number = 1;
+    @observable _muted: boolean = false;
+
+    @computed get links() { return DocListCast(this.dataDoc.links); }
+    @computed get heightPercent() { return NumCast(this.layoutDoc._timelineHeightPercent, 100); } // current percent of video relative to VideoBox height
+    // @computed get rawDuration() { return NumCast(this.dataDoc[this.fieldKey + "-duration"]); }
+    @observable rawDuration: number = 0;
+
+
+    @computed get youtubeVideoId() {
+        const field = Cast(this.dataDoc[this.props.fieldKey], VideoField);
+        return field && field.url.href.indexOf("youtube") !== -1 ? ((arr: string[]) => arr[arr.length - 1])(field.url.href.split("/")) : "";
+    }
+
+
+    // returns the path of the audio file
+    @computed get audiopath() {
+        const field = Cast(this.props.Document[this.props.fieldKey + '-audio'], AudioField, null);
+        const vfield = Cast(this.dataDoc[this.fieldKey], VideoField, null);
+        return field?.url.href ?? vfield?.url.href ?? "";
+    }
+
+    // returns the presentation data if it exists, null otherwise
+    @computed get presentation(): Presentation | null {
             const data = this.dataDoc[this.fieldKey + '-presentation'];
             return data ? JSON.parse(data) : null;
       }
-- 
cgit v1.2.3-70-g09d2


From 98fba8bdb0fe81d6f71d0ae6018fcaaf7d8897df Mon Sep 17 00:00:00 2001
From: Michael Foiani <sotech117@michaels-mbp-5.devices.brown.edu>
Date: Wed, 15 Jun 2022 15:39:55 -0400
Subject: Refactor RecordingApi into two main files - TrackMovements and
 ReplayMovements. TrackMovements is for recording presentations and
 ReplayMovments is for replaying them.

---
 src/client/util/RecordingApi.ts                    | 461 ---------------------
 src/client/util/ReplayMovements.ts                 | 204 +++++++++
 src/client/util/TrackMovements.ts                  | 257 ++++++++++++
 src/client/views/Main.tsx                          |   6 +-
 .../collectionFreeForm/CollectionFreeFormView.tsx  |  16 +-
 .../views/nodes/RecordingBox/RecordingBox.tsx      |   2 +-
 .../views/nodes/RecordingBox/RecordingView.tsx     |  10 +-
 src/client/views/nodes/VideoBox.tsx                |   8 +-
 8 files changed, 477 insertions(+), 487 deletions(-)
 delete mode 100644 src/client/util/RecordingApi.ts
 create mode 100644 src/client/util/ReplayMovements.ts
 create mode 100644 src/client/util/TrackMovements.ts

(limited to 'src/client/views/nodes/VideoBox.tsx')

diff --git a/src/client/util/RecordingApi.ts b/src/client/util/RecordingApi.ts
deleted file mode 100644
index 87cb85497..000000000
--- a/src/client/util/RecordingApi.ts
+++ /dev/null
@@ -1,461 +0,0 @@
-import { CollectionFreeFormView } from "../views/collections/collectionFreeForm";
-import { IReactionDisposer, observable, observe, reaction } from "mobx";
-import { NumCast } from "../../fields/Types";
-import { Doc, DocListCast } from "../../fields/Doc";
-import { VideoBox } from "../views/nodes/VideoBox";
-import { isArray } from "lodash";
-import { SelectionManager } from "./SelectionManager";
-import { DocumentDecorations } from "../views/DocumentDecorations";
-import { DocumentManager } from "./DocumentManager";
-import { CollectionDockingView } from "../views/collections/CollectionDockingView";
-import { Id } from "../../fields/FieldSymbols";
-import { returnAll } from "../../Utils";
-import { ContextExclusionPlugin } from "webpack";
-import { DocServer } from "../DocServer";
-import { DocumentView } from "../views/nodes/DocumentView";
-
-type Movement = {
-    time: number,
-    panX: number,
-    panY: number,
-    scale: number,
-    docId: string,
-}
-
-
-export type Presentation = {
-    movements: Movement[] | null,
-    totalTime: number,
-    meta: Object | Object[],
-}
-
-export class RecordingApi {
-
-    private static get NULL_PRESENTATION(): Presentation {
-        return { movements: null, meta: {}, totalTime: -1, }
-    }
-
-    // instance variables
-    private currentPresentation: Presentation;
-    private tracking: boolean;
-    private absoluteStart: number;
-    // instance variable for holding the FFViews and their disposers
-    private recordingFFViews: Map<string, IReactionDisposer> | null;
-    private tabChangeDisposeFunc: IReactionDisposer | null;
-
-
-    // create static instance and getter for global use
-    @observable static _instance: RecordingApi;
-    public static get Instance(): RecordingApi { return RecordingApi._instance }
-    public constructor() {
-        // init the global instance
-        RecordingApi._instance = this;
-
-        // init the instance variables
-        this.currentPresentation = RecordingApi.NULL_PRESENTATION
-        this.tracking = false;
-        this.absoluteStart = -1;
-
-        // used for tracking movements in the view frame
-        this.recordingFFViews = null;
-        this.tabChangeDisposeFunc = null;
-
-        // for now, set playFFView
-        // this.playFFView = null;
-        this.timers = null;
-    }
-
-    // little helper :)
-    private get nullPresentation(): boolean {
-        return this.currentPresentation.movements === null
-    }
-
-    private addRecordingFFView(doc: Doc, key: string = doc[Id]): void {
-        console.info('adding dispose func : docId', key, 'doc', doc);
-
-        if (this.recordingFFViews === null) { console.warn('addFFView on null RecordingApi'); return; }
-        if (this.recordingFFViews.has(key)) { console.warn('addFFView : key already in map'); return; }
-
-        const disposeFunc = reaction(
-            () => ({ x: NumCast(doc.panX, -1), y: NumCast(doc.panY, -1), scale: NumCast(doc.viewScale, 0)}),
-            (res) => (res.x !== -1 && res.y !== -1 && this.tracking) && this.trackMovements(res.x, res.y, key, res.scale),
-        );
-        this.recordingFFViews?.set(key, disposeFunc);
-    }
-
-    private removeRecordingFFView = (key: string) => {
-        console.info('removing dispose func : docId', key);
-        if (this.recordingFFViews === null) { console.warn('removeFFView on null RecordingApi'); return; }
-        this.recordingFFViews.get(key)?.();
-        this.recordingFFViews.delete(key);
-    }
-
-    // in the case where only one tab was changed (updates not across dashboards), set only one to true
-    private updateRecordingFFViewsFromTabs = (tabbedDocs: Doc[], onlyOne = false) => {
-        if (this.recordingFFViews === null) return;
-
-        // so that the size comparisons are correct, we must filter to only the FFViews
-        const isFFView = (doc: Doc) => doc && 'viewType' in doc && doc.viewType === 'freeform';
-        const tabbedFFViews = new Set<string>();
-        for (const DashDoc of tabbedDocs) {
-            if (isFFView(DashDoc)) tabbedFFViews.add(DashDoc[Id]);
-        }
-
-
-        // new tab was added - need to add it
-        if (tabbedFFViews.size > this.recordingFFViews.size) {
-            for (const DashDoc of tabbedDocs) {
-                if (!this.recordingFFViews.has(DashDoc[Id])) {
-                    if (isFFView(DashDoc)) {
-                        this.addRecordingFFView(DashDoc);
-
-                        // only one max change, so return
-                        if (onlyOne) return;
-                    }
-                }
-            }
-        }
-        // tab was removed - need to remove it from recordingFFViews
-        else if (tabbedFFViews.size < this.recordingFFViews.size) {
-            for (const [key] of this.recordingFFViews) {
-                if (!tabbedFFViews.has(key)) { 
-                    this.removeRecordingFFView(key);
-                    if (onlyOne) return;
-                } 
-            }
-        }
-    }
-
-    public initTabTracker = () => {
-        if (this.recordingFFViews === null) {
-            this.recordingFFViews = new Map();
-        }
-
-        // init the dispose funcs on the page
-        const docList = DocListCast(CollectionDockingView.Instance.props.Document.data);
-        this.updateRecordingFFViewsFromTabs(docList);
-
-        // create a reaction to monitor changes in tabs
-        this.tabChangeDisposeFunc =
-            reaction(() => CollectionDockingView.Instance.props.Document.data,
-                (change) => {
-                    // TODO: consider changing between dashboards
-                    console.log('change in tabs', change);
-                    this.updateRecordingFFViewsFromTabs(DocListCast(change), true);
-                });
-    }
-
-    public start = (meta?: Object) => {
-        this.initTabTracker();
-
-        // update the presentation mode
-        Doc.UserDoc().presentationMode = 'recording';
-
-        // (1a) get start date for presenation
-        const startDate = new Date();
-        // (1b) set start timestamp to absolute timestamp
-        this.absoluteStart = startDate.getTime();
-
-        // (2) assign meta content if it exists
-        this.currentPresentation.meta = meta || {}
-        // (3) assign start date to currentPresenation
-        this.currentPresentation.movements = []
-        // (4) set tracking true to allow trackMovements
-        this.tracking = true
-    }
-
-    /* stops the video and returns the presentatation; if no presentation, returns undefined */
-    public yieldPresentation(clearData: boolean = true): Presentation | null {
-        // if no presentation or done tracking, return null
-        if (this.nullPresentation || !this.tracking) return null;
-
-        // set the previus recording view to the play view
-        // this.playFFView = this.recordingFFView;
-
-        // ensure we add the endTime now that they are done recording    
-        const cpy = { ...this.currentPresentation, totalTime: new Date().getTime() - this.absoluteStart };
-
-        // reset the current presentation
-        clearData && this.clear();
-
-        console.log('yieldPresentation', cpy);
-        return cpy;
-    }
-
-    public finish = (): void => {
-        // make is tracking false
-        this.tracking = false
-        // reset the RecordingApi instance
-        this.clear();
-    }
-
-    public clear = (): void => {
-        // clear the disposeFunc if we are done (not tracking)
-        if (!this.tracking) {
-            this.removeAllRecordingFFViews();
-            this.tabChangeDisposeFunc?.();
-            // update the presentation mode now that we are done tracking
-            Doc.UserDoc().presentationMode = 'none';
-
-            this.recordingFFViews = null;
-            this.tabChangeDisposeFunc = null;
-        }
-
-        // clear presenation data
-        this.currentPresentation = RecordingApi.NULL_PRESENTATION
-        // clear absoluteStart
-        this.absoluteStart = -1
-    }
-
-    private removeAllRecordingFFViews = () => {
-        if (this.recordingFFViews === null) { console.warn('removeAllFFViews on null RecordingApi'); return; }
-
-        for (const [id, disposeFunc] of this.recordingFFViews) {
-            console.log('calling dispose func : docId', id);
-            disposeFunc();
-            this.recordingFFViews.delete(id);
-        }
-    }
-
-    private trackMovements = (panX: number, panY: number, docId: string, scale: number = 0) => {
-        // ensure we are recording to track
-        if (!this.tracking) {
-            console.error('[recordingApi.ts] trackMovements(): tracking is false')
-            return;
-        }
-        // check to see if the presetation is init - if not, we are between segments
-        // TODO: make this more clear - tracking should be "live tracking", not always true when the recording api being used (between start and yieldPres)
-        // bacuse tracking should be false inbetween segments high key 
-        if (this.nullPresentation) {
-            console.warn('[recordingApi.ts] trackMovements(): trying to store movemetns between segments')
-            return;
-        }
-
-        // get the time
-        const time = new Date().getTime() - this.absoluteStart
-        // make new movement object
-        const movement: Movement = { time, panX, panY, scale, docId }
-
-        // add that movement to the current presentation data's movement array
-        this.currentPresentation.movements && this.currentPresentation.movements.push(movement)
-    }
-
-    // TODO: extract this into different class with pause and resume recording
-    // TODO: store the FFview with the movements
-    private timers: NodeJS.Timeout[] | null; 
-
-    // pausing movements will dispose all timers that are planned to replay the movements
-    // play movemvents will recreate them when the user resumes the presentation
-    public pauseMovements = (): undefined | Error => {
-
-        if (!this._isPlaying) { console.warn('[recordingApi.ts] pauseMovements(): already on paused'); return;}
-        this._isPlaying = false
-        // TODO: set userdoc presentMode to browsing
-        this.timers?.map(timer => clearTimeout(timer))
-
-        // this.videoBox = null;
-    }
-
-    private videoBoxDisposeFunc: IReactionDisposer | null = null;
-    private videoBox: VideoBox | null = null;
-
-    setVideoBox = async (videoBox: VideoBox) => {
-        console.log('setVideoBox', videoBox);
-        if (videoBox !== null) { console.warn('setVideoBox on already videoBox'); }
-        if (this.videoBoxDisposeFunc !== null) { console.warn('setVideoBox on already videoBox dispose func'); this.videoBoxDisposeFunc(); }
-
-
-        const { presentation } = videoBox;
-        if (presentation == null) { console.warn('setVideoBox on null videoBox presentation'); return; }
-        
-        let docIdtoDoc: Map<string, Doc> = new Map();
-        try {
-            docIdtoDoc = await this.loadPresentation(presentation);
-        } catch {
-            console.error('[recordingApi.ts] setVideoBox(): error loading presentation - no replay movements');
-            throw 'error loading docs from server';
-        }
-
-        
-        this.videoBoxDisposeFunc =
-            reaction(() => ({ playing: videoBox._playing, timeViewed: videoBox.player?.currentTime || 0 }),
-                ({ playing, timeViewed }) => 
-                    playing ? this.playMovements(presentation, docIdtoDoc, timeViewed) : this.pauseMovements()
-            );
-        this.videoBox = videoBox;
-    }
-
-    removeVideoBox = () => {
-        if (this.videoBoxDisposeFunc == null) { console.warn('removeVideoBox on null videoBox'); return; }
-        this.videoBoxDisposeFunc();
-
-        this.videoBox = null;
-        this.videoBoxDisposeFunc = null;
-    }
-
-
-    // by calling pause on the VideoBox, the pauseMovements will be called
-    public pauseFromInteraction = () => {
-        Doc.UserDoc().presentationMode = 'none';
-        this.videoBox?.Pause();
-
-        this.pauseMovements();
-        // return this.videoBox == null
-    }
-
-
-
-    public _isPlaying = false;
-
-    loadPresentation = async (presentation: Presentation) => {
-        const { movements } = presentation;
-        if (movements === null) {
-            throw '[recordingApi.ts] followMovements() failed: no presentation data';
-        }
-
-        // generate a set of all unique docIds
-        const docIds = new Set<string>();
-        for (const {docId} of movements) {
-            if (!docIds.has(docId)) docIds.add(docId);
-        }
-
-        const docIdtoDoc = new Map<string, Doc>();
-
-        let refFields = await DocServer.GetRefFields([...docIds.keys()]);
-        for (const docId in refFields) {
-            if (!refFields[docId]) {
-                throw `one field was undefined`;
-            }
-            docIdtoDoc.set(docId, refFields[docId] as Doc);
-        }
-        console.log('loadPresentation refFields', refFields, docIdtoDoc);
-
-        return docIdtoDoc;
-    }
-
-    // returns undefined if the docView isn't open on the screen
-    getCollectionFFView = (docId: string) => {
-        const isInView = DocumentManager.Instance.getDocumentViewById(docId);
-        if (isInView) { return isInView.ComponentView as CollectionFreeFormView; }
-    }
-
-    // will open the doc in a tab then return the CollectionFFView that holds it
-    openTab = (docId: string, docIdtoDoc: Map<string, Doc>) => {
-        const doc = docIdtoDoc.get(docId);
-        if (doc == undefined) {
-            console.error(`docIdtoDoc did not contain docId ${docId}`)
-            return undefined;
-        }
-        CollectionDockingView.AddSplit(doc, 'right');
-        const docView = DocumentManager.Instance.getDocumentViewById(docId);
-        return docView?.ComponentView as CollectionFreeFormView;
-    }
-
-    // helper to replay a movement
-    private preScale = -1;
-    zoomAndPan = (movement: Movement, document: CollectionFreeFormView) => {
-        const { panX, panY, scale } = movement;
-        (scale !== 0 && this.preScale !== scale) && document.zoomSmoothlyAboutPt([panX, panY], scale, 0);
-        document.Document._panX = panX;
-        document.Document._panY = panY;
-
-        this.preScale = scale;
-    }
-
-    getFirstMovements = (movements: Movement[], timeViewed: number): Map<string, Movement> => {
-        if (movements === null) return new Map();
-        // generate a set of all unique docIds
-        const docIdtoFirstMove = new Map();
-        for (const move of movements) {
-            const { docId } = move;
-            if (!docIdtoFirstMove.has(docId)) docIdtoFirstMove.set(docId, move);
-        }
-        return docIdtoFirstMove;
-    }
-
-    endPlayingPresentation = () => {
-        this.preScale = -1;
-        RecordingApi.Instance._isPlaying = false;
-    }
-
-    public playMovements = (presentation: Presentation, docIdtoDoc: Map<string, Doc>, timeViewed: number = 0) => {
-        console.log('playMovements', presentation, timeViewed, docIdtoDoc);
-
-        if (presentation.movements === null || presentation.movements.length === 0) { //|| this.playFFView === null) {
-            return new Error('[recordingApi.ts] followMovements() failed: no presentation data')
-        }
-        if (this._isPlaying) return;
-
-        this._isPlaying = true;
-        Doc.UserDoc().presentationMode = 'watching';
-
-        // only get the movements that are remaining in the video time left
-        const filteredMovements = presentation.movements.filter(movement => movement.time > timeViewed * 1000)
-
-        const handleFirstMovements = () => {
-            // if the first movement is a closed tab, open it
-            const firstMovement = filteredMovements[0];
-            const isClosed = this.getCollectionFFView(firstMovement.docId) === undefined;
-            if (isClosed) this.openTab(firstMovement.docId, docIdtoDoc);
-
-            // for the open tabs, set it to the first move
-            const docIdtoFirstMove = this.getFirstMovements(filteredMovements, timeViewed);
-            for (const [docId, firstMove] of docIdtoFirstMove) {
-                const colFFView = this.getCollectionFFView(docId);
-                if (colFFView) this.zoomAndPan(firstMove, colFFView);
-            }
-        }
-        handleFirstMovements();
-        
-
-        // make timers that will execute each movement at the correct replay time
-        this.timers = filteredMovements.map(movement => {
-            const timeDiff = movement.time - timeViewed * 1000
-
-            return setTimeout(() => {
-                const collectionFFView = this.getCollectionFFView(movement.docId);
-                if (collectionFFView) {
-                    this.zoomAndPan(movement, collectionFFView);
-                } else {
-                    // tab wasn't open - open it and play the movement
-                    const openedColFFView = this.openTab(movement.docId, docIdtoDoc);
-                    openedColFFView && this.zoomAndPan(movement, openedColFFView);
-                }
-
-                // if last movement, presentation is done -> cleanup :)
-                if (movement === filteredMovements[filteredMovements.length - 1]) {
-                    this.endPlayingPresentation();
-                }
-            }, timeDiff);
-        });
-    }
-
-    // method that concatenates an array of presentatations into one
-    public concatPresentations = (presentations: Presentation[]): Presentation => {
-        // these three will lead to the combined presentation
-        let combinedMovements: Movement[] = [];
-        let sumTime = 0;
-        let combinedMetas: any[] = [];
-
-        presentations.forEach((presentation) => {
-            const { movements, totalTime, meta } = presentation;
-
-            // update movements if they had one
-            if (movements) {
-                // add the summed time to the movements
-                const addedTimeMovements = movements.map(move => { return { ...move, time: move.time + sumTime } });
-                // concat the movements already in the combined presentation with these new ones
-                combinedMovements.push(...addedTimeMovements);
-            }
-
-            // update the totalTime
-            sumTime += totalTime;
-
-            // concatenate the metas
-            combinedMetas.push(meta);
-        });
-
-        // return the combined presentation with the updated total summed time
-		return { movements: combinedMovements, totalTime: sumTime, meta: combinedMetas };
-    }
-}
diff --git a/src/client/util/ReplayMovements.ts b/src/client/util/ReplayMovements.ts
new file mode 100644
index 000000000..e46810b52
--- /dev/null
+++ b/src/client/util/ReplayMovements.ts
@@ -0,0 +1,204 @@
+import { CollectionFreeFormView } from "../views/collections/collectionFreeForm";
+import { IReactionDisposer, observable, observe, reaction } from "mobx";
+import { Doc } from "../../fields/Doc";
+import { VideoBox } from "../views/nodes/VideoBox";
+import { DocumentManager } from "./DocumentManager";
+import { CollectionDockingView } from "../views/collections/CollectionDockingView";
+import { DocServer } from "../DocServer";
+import { Movement, Presentation } from "./TrackMovements";
+
+export class ReplayMovements {
+    private timers: NodeJS.Timeout[] | null; 
+    private videoBoxDisposeFunc: IReactionDisposer | null;
+    private videoBox: VideoBox | null;
+    private isPlaying: boolean;
+
+
+    // create static instance and getter for global use
+    @observable static _instance: ReplayMovements;
+    static get Instance(): ReplayMovements { return ReplayMovements._instance }
+    constructor() {
+        // init the global instance
+        ReplayMovements._instance = this;
+
+        // instance vars for replaying
+        this.timers = null;
+        this.videoBoxDisposeFunc = null;
+        this.videoBox = null;
+        this.isPlaying = false;
+    }
+
+    // pausing movements will dispose all timers that are planned to replay the movements
+    // play movemvents will recreate them when the user resumes the presentation
+    pauseMovements = (): undefined | Error => {
+        if (!this.isPlaying) {
+            console.warn('[recordingApi.ts] pauseMovements(): already on paused');
+            return;
+        }
+
+        this.isPlaying = false
+        // TODO: set userdoc presentMode to browsing
+        this.timers?.map(timer => clearTimeout(timer))
+    }
+
+    setVideoBox = async (videoBox: VideoBox) => {
+        console.log('setVideoBox', videoBox);
+        if (videoBox !== null) { console.warn('setVideoBox on already videoBox'); }
+        if (this.videoBoxDisposeFunc !== null) { console.warn('setVideoBox on already videoBox dispose func'); this.videoBoxDisposeFunc(); }
+
+
+        const { presentation } = videoBox;
+        if (presentation == null) { console.warn('setVideoBox on null videoBox presentation'); return; }
+        
+        let docIdtoDoc: Map<string, Doc> = new Map();
+        try {
+            docIdtoDoc = await this.loadPresentation(presentation);
+        } catch {
+            console.error('[recordingApi.ts] setVideoBox(): error loading presentation - no replay movements');
+            throw 'error loading docs from server';
+        }
+
+        
+        this.videoBoxDisposeFunc =
+            reaction(() => ({ playing: videoBox._playing, timeViewed: videoBox.player?.currentTime || 0 }),
+                ({ playing, timeViewed }) => 
+                    playing ? this.playMovements(presentation, docIdtoDoc, timeViewed) : this.pauseMovements()
+            );
+        this.videoBox = videoBox;
+    }
+
+    removeVideoBox = () => {
+        if (this.videoBoxDisposeFunc == null) { console.warn('removeVideoBox on null videoBox'); return; }
+        this.videoBoxDisposeFunc();
+
+        this.videoBox = null;
+        this.videoBoxDisposeFunc = null;
+    }
+
+    // should be called from interacting with the screen
+    pauseFromInteraction = () => {
+        Doc.UserDoc().presentationMode = 'none';
+        this.videoBox?.Pause();
+
+        this.pauseMovements();
+    }
+
+    loadPresentation = async (presentation: Presentation) => {
+        const { movements } = presentation;
+        if (movements === null) {
+            throw '[recordingApi.ts] followMovements() failed: no presentation data';
+        }
+
+        // generate a set of all unique docIds
+        const docIds = new Set<string>();
+        for (const {docId} of movements) {
+            if (!docIds.has(docId)) docIds.add(docId);
+        }
+
+        const docIdtoDoc = new Map<string, Doc>();
+
+        let refFields = await DocServer.GetRefFields([...docIds.keys()]);
+        for (const docId in refFields) {
+            if (!refFields[docId]) {
+                throw `one field was undefined`;
+            }
+            docIdtoDoc.set(docId, refFields[docId] as Doc);
+        }
+        console.log('loadPresentation refFields', refFields, docIdtoDoc);
+
+        return docIdtoDoc;
+    }
+
+    // returns undefined if the docView isn't open on the screen
+    getCollectionFFView = (docId: string) => {
+        const isInView = DocumentManager.Instance.getDocumentViewById(docId);
+        if (isInView) { return isInView.ComponentView as CollectionFreeFormView; }
+    }
+
+    // will open the doc in a tab then return the CollectionFFView that holds it
+    openTab = (docId: string, docIdtoDoc: Map<string, Doc>) => {
+        const doc = docIdtoDoc.get(docId);
+        if (doc == undefined) {
+            console.error(`docIdtoDoc did not contain docId ${docId}`)
+            return undefined;
+        }
+        CollectionDockingView.AddSplit(doc, 'right');
+        const docView = DocumentManager.Instance.getDocumentViewById(docId);
+        return docView?.ComponentView as CollectionFreeFormView;
+    }
+
+    // helper to replay a movement
+    zoomAndPan = (movement: Movement, document: CollectionFreeFormView) => {
+        const { panX, panY, scale } = movement;
+        scale !== 0 && document.zoomSmoothlyAboutPt([panX, panY], scale, 0);
+        document.Document._panX = panX;
+        document.Document._panY = panY;
+    }
+
+    getFirstMovements = (movements: Movement[], timeViewed: number): Map<string, Movement> => {
+        if (movements === null) return new Map();
+        // generate a set of all unique docIds
+        const docIdtoFirstMove = new Map();
+        for (const move of movements) {
+            const { docId } = move;
+            if (!docIdtoFirstMove.has(docId)) docIdtoFirstMove.set(docId, move);
+        }
+        return docIdtoFirstMove;
+    }
+
+    endPlayingPresentation = () => {
+        this.isPlaying = false;
+    }
+
+    public playMovements = (presentation: Presentation, docIdtoDoc: Map<string, Doc>, timeViewed: number = 0) => {
+        console.log('playMovements', presentation, timeViewed, docIdtoDoc);
+
+        if (presentation.movements === null || presentation.movements.length === 0) { //|| this.playFFView === null) {
+            return new Error('[recordingApi.ts] followMovements() failed: no presentation data')
+        }
+        if (this.isPlaying) return;
+
+        this.isPlaying = true;
+        Doc.UserDoc().presentationMode = 'watching';
+
+        // only get the movements that are remaining in the video time left
+        const filteredMovements = presentation.movements.filter(movement => movement.time > timeViewed * 1000)
+
+        const handleFirstMovements = () => {
+            // if the first movement is a closed tab, open it
+            const firstMovement = filteredMovements[0];
+            const isClosed = this.getCollectionFFView(firstMovement.docId) === undefined;
+            if (isClosed) this.openTab(firstMovement.docId, docIdtoDoc);
+
+            // for the open tabs, set it to the first move
+            const docIdtoFirstMove = this.getFirstMovements(filteredMovements, timeViewed);
+            for (const [docId, firstMove] of docIdtoFirstMove) {
+                const colFFView = this.getCollectionFFView(docId);
+                if (colFFView) this.zoomAndPan(firstMove, colFFView);
+            }
+        }
+        handleFirstMovements();
+        
+
+        // make timers that will execute each movement at the correct replay time
+        this.timers = filteredMovements.map(movement => {
+            const timeDiff = movement.time - timeViewed * 1000
+
+            return setTimeout(() => {
+                const collectionFFView = this.getCollectionFFView(movement.docId);
+                if (collectionFFView) {
+                    this.zoomAndPan(movement, collectionFFView);
+                } else {
+                    // tab wasn't open - open it and play the movement
+                    const openedColFFView = this.openTab(movement.docId, docIdtoDoc);
+                    openedColFFView && this.zoomAndPan(movement, openedColFFView);
+                }
+
+                // if last movement, presentation is done -> cleanup :)
+                if (movement === filteredMovements[filteredMovements.length - 1]) {
+                    this.endPlayingPresentation();
+                }
+            }, timeDiff);
+        });
+    }
+}
diff --git a/src/client/util/TrackMovements.ts b/src/client/util/TrackMovements.ts
new file mode 100644
index 000000000..342bb440e
--- /dev/null
+++ b/src/client/util/TrackMovements.ts
@@ -0,0 +1,257 @@
+import { IReactionDisposer, observable, observe, reaction } from "mobx";
+import { NumCast } from "../../fields/Types";
+import { Doc, DocListCast } from "../../fields/Doc";
+import { CollectionDockingView } from "../views/collections/CollectionDockingView";
+import { Id } from "../../fields/FieldSymbols";
+
+export type Movement = {
+    time: number,
+    panX: number,
+    panY: number,
+    scale: number,
+    docId: string,
+}
+
+export type Presentation = {
+    movements: Movement[] | null,
+    totalTime: number,
+    meta: Object | Object[],
+}
+
+export class TrackMovements {
+
+    private static get NULL_PRESENTATION(): Presentation {
+        return { movements: null, meta: {}, totalTime: -1, }
+    }
+
+    // instance variables
+    private currentPresentation: Presentation;
+    private tracking: boolean;
+    private absoluteStart: number;
+    // instance variable for holding the FFViews and their disposers
+    private recordingFFViews: Map<string, IReactionDisposer> | null;
+    private tabChangeDisposeFunc: IReactionDisposer | null;
+
+
+    // create static instance and getter for global use
+    @observable static _instance: TrackMovements;
+    static get Instance(): TrackMovements { return TrackMovements._instance }
+    constructor() {
+        // init the global instance
+        TrackMovements._instance = this;
+
+        // init the instance variables
+        this.currentPresentation = TrackMovements.NULL_PRESENTATION
+        this.tracking = false;
+        this.absoluteStart = -1;
+
+        // used for tracking movements in the view frame
+        this.recordingFFViews = null;
+        this.tabChangeDisposeFunc = null;
+    }
+
+    // little helper :)
+    private get nullPresentation(): boolean {
+        return this.currentPresentation.movements === null
+    }
+
+    private addRecordingFFView(doc: Doc, key: string = doc[Id]): void {
+        console.info('adding dispose func : docId', key, 'doc', doc);
+
+        if (this.recordingFFViews === null) { console.warn('addFFView on null RecordingApi'); return; }
+        if (this.recordingFFViews.has(key)) { console.warn('addFFView : key already in map'); return; }
+
+        const disposeFunc = reaction(
+            () => ({ x: NumCast(doc.panX, -1), y: NumCast(doc.panY, -1), scale: NumCast(doc.viewScale, 0)}),
+            (res) => (res.x !== -1 && res.y !== -1 && this.tracking) && this.trackMovement(res.x, res.y, key, res.scale),
+        );
+        this.recordingFFViews?.set(key, disposeFunc);
+    }
+
+    private removeRecordingFFView = (key: string) => {
+        console.info('removing dispose func : docId', key);
+        if (this.recordingFFViews === null) { console.warn('removeFFView on null RecordingApi'); return; }
+        this.recordingFFViews.get(key)?.();
+        this.recordingFFViews.delete(key);
+    }
+
+    // in the case where only one tab was changed (updates not across dashboards), set only one to true
+    private updateRecordingFFViewsFromTabs = (tabbedDocs: Doc[], onlyOne = false) => {
+        if (this.recordingFFViews === null) return;
+
+        // so that the size comparisons are correct, we must filter to only the FFViews
+        const isFFView = (doc: Doc) => doc && 'viewType' in doc && doc.viewType === 'freeform';
+        const tabbedFFViews = new Set<string>();
+        for (const DashDoc of tabbedDocs) {
+            if (isFFView(DashDoc)) tabbedFFViews.add(DashDoc[Id]);
+        }
+
+
+        // new tab was added - need to add it
+        if (tabbedFFViews.size > this.recordingFFViews.size) {
+            for (const DashDoc of tabbedDocs) {
+                if (!this.recordingFFViews.has(DashDoc[Id])) {
+                    if (isFFView(DashDoc)) {
+                        this.addRecordingFFView(DashDoc);
+
+                        // only one max change, so return
+                        if (onlyOne) return;
+                    }
+                }
+            }
+        }
+        // tab was removed - need to remove it from recordingFFViews
+        else if (tabbedFFViews.size < this.recordingFFViews.size) {
+            for (const [key] of this.recordingFFViews) {
+                if (!tabbedFFViews.has(key)) { 
+                    this.removeRecordingFFView(key);
+                    if (onlyOne) return;
+                } 
+            }
+        }
+    }
+
+    private initTabTracker = () => {
+        if (this.recordingFFViews === null) {
+            this.recordingFFViews = new Map();
+        }
+
+        // init the dispose funcs on the page
+        const docList = DocListCast(CollectionDockingView.Instance.props.Document.data);
+        this.updateRecordingFFViewsFromTabs(docList);
+
+        // create a reaction to monitor changes in tabs
+        this.tabChangeDisposeFunc =
+            reaction(() => CollectionDockingView.Instance.props.Document.data,
+                (change) => {
+                    // TODO: consider changing between dashboards
+                    console.log('change in tabs', change);
+                    this.updateRecordingFFViewsFromTabs(DocListCast(change), true);
+                });
+    }
+
+    start = (meta?: Object) => {
+        this.initTabTracker();
+
+        // update the presentation mode
+        Doc.UserDoc().presentationMode = 'recording';
+
+        // (1a) get start date for presenation
+        const startDate = new Date();
+        // (1b) set start timestamp to absolute timestamp
+        this.absoluteStart = startDate.getTime();
+
+        // (2) assign meta content if it exists
+        this.currentPresentation.meta = meta || {}
+        // (3) assign start date to currentPresenation
+        this.currentPresentation.movements = []
+        // (4) set tracking true to allow trackMovements
+        this.tracking = true
+    }
+
+    /* stops the video and returns the presentatation; if no presentation, returns undefined */
+    yieldPresentation(clearData: boolean = true): Presentation | null {
+        // if no presentation or done tracking, return null
+        if (this.nullPresentation || !this.tracking) return null;
+
+        // set the previus recording view to the play view
+        // this.playFFView = this.recordingFFView;
+
+        // ensure we add the endTime now that they are done recording    
+        const cpy = { ...this.currentPresentation, totalTime: new Date().getTime() - this.absoluteStart };
+
+        // reset the current presentation
+        clearData && this.clear();
+
+        console.log('yieldPresentation', cpy);
+        return cpy;
+    }
+
+    finish = (): void => {
+        // make is tracking false
+        this.tracking = false
+        // reset the RecordingApi instance
+        this.clear();
+    }
+
+    private clear = (): void => {
+        // clear the disposeFunc if we are done (not tracking)
+        if (!this.tracking) {
+            this.removeAllRecordingFFViews();
+            this.tabChangeDisposeFunc?.();
+            // update the presentation mode now that we are done tracking
+            Doc.UserDoc().presentationMode = 'none';
+
+            this.recordingFFViews = null;
+            this.tabChangeDisposeFunc = null;
+        }
+
+        // clear presenation data
+        this.currentPresentation = TrackMovements.NULL_PRESENTATION
+        // clear absoluteStart
+        this.absoluteStart = -1
+    }
+
+    removeAllRecordingFFViews = () => {
+        if (this.recordingFFViews === null) { console.warn('removeAllFFViews on null RecordingApi'); return; }
+
+        for (const [id, disposeFunc] of this.recordingFFViews) {
+            console.log('calling dispose func : docId', id);
+            disposeFunc();
+            this.recordingFFViews.delete(id);
+        }
+    }
+
+    private trackMovement = (panX: number, panY: number, docId: string, scale: number = 0) => {
+        // ensure we are recording to track
+        if (!this.tracking) {
+            console.error('[recordingApi.ts] trackMovements(): tracking is false')
+            return;
+        }
+        // check to see if the presetation is init - if not, we are between segments
+        // TODO: make this more clear - tracking should be "live tracking", not always true when the recording api being used (between start and yieldPres)
+        // bacuse tracking should be false inbetween segments high key 
+        if (this.nullPresentation) {
+            console.warn('[recordingApi.ts] trackMovements(): trying to store movemetns between segments')
+            return;
+        }
+
+        // get the time
+        const time = new Date().getTime() - this.absoluteStart
+        // make new movement object
+        const movement: Movement = { time, panX, panY, scale, docId }
+
+        // add that movement to the current presentation data's movement array
+        this.currentPresentation.movements && this.currentPresentation.movements.push(movement)
+    }
+
+
+    // method that concatenates an array of presentatations into one
+    public concatPresentations = (presentations: Presentation[]): Presentation => {
+        // these three will lead to the combined presentation
+        let combinedMovements: Movement[] = [];
+        let sumTime = 0;
+        let combinedMetas: any[] = [];
+
+        presentations.forEach((presentation) => {
+            const { movements, totalTime, meta } = presentation;
+
+            // update movements if they had one
+            if (movements) {
+                // add the summed time to the movements
+                const addedTimeMovements = movements.map(move => { return { ...move, time: move.time + sumTime } });
+                // concat the movements already in the combined presentation with these new ones
+                combinedMovements.push(...addedTimeMovements);
+            }
+
+            // update the totalTime
+            sumTime += totalTime;
+
+            // concatenate the metas
+            combinedMetas.push(meta);
+        });
+
+        // return the combined presentation with the updated total summed time
+		return { movements: combinedMovements, totalTime: sumTime, meta: combinedMetas };
+    }
+}
diff --git a/src/client/views/Main.tsx b/src/client/views/Main.tsx
index 517fe097c..c1b67ba19 100644
--- a/src/client/views/Main.tsx
+++ b/src/client/views/Main.tsx
@@ -8,7 +8,8 @@ import { AssignAllExtensions } from "../../extensions/General/Extensions";
 import { Docs } from "../documents/Documents";
 import { CurrentUserUtils } from "../util/CurrentUserUtils";
 import { LinkManager } from "../util/LinkManager";
-import { RecordingApi } from "../util/RecordingApi";
+import { ReplayMovements } from '../util/ReplayMovements';
+import { TrackMovements } from "../util/TrackMovements";
 import { CollectionView } from "./collections/CollectionView";
 import { MainView } from "./MainView";
 
@@ -37,6 +38,7 @@ AssignAllExtensions();
     const expires = "expires=" + d.toUTCString();
     document.cookie = `loadtime=${loading};${expires};path=/`;
     new LinkManager();
-    new RecordingApi;
+    new TrackMovements();
+    new ReplayMovements();
     ReactDOM.render(<MainView />, document.getElementById('root'));
 })();
\ No newline at end of file
diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx
index 7db3b1482..a661cebb8 100644
--- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx
+++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx
@@ -58,7 +58,7 @@ import React = require("react");
 import { FieldView, FieldViewProps } from "../../nodes/FieldView";
 import { InkTranscription } from "../../InkTranscription";
 import e = require("connect-flash");
-import { RecordingApi } from "../../../util/RecordingApi";
+import { ReplayMovements } from "../../../util/ReplayMovements";
 
 export const panZoomSchema = createSchema({
     _panX: "number",
@@ -1005,20 +1005,8 @@ export class CollectionFreeFormView extends CollectionSubView<Partial<collection
 
     @action
     setPan(panX: number, panY: number, panTime: number = 0, clamp: boolean = false) {
-        // set the current respective FFview to the tab being panned.
-        // if (Doc.UserDoc()?.presentationMode === 'recording') {
-        //     // RecordingApi.Instance.setRecordingFFView(this);
-        //     console.log('setRecordingFFView', this);
-        // }
-        // TODO: make this based off the specific recording FFView
-        // (Doc.UserDoc()?.presentationMode === 'none' || Doc.UserDoc()?.presentationMode === 'watching') && RecordingApi.Instance.setPlayFFView(this);
-        // if (Doc.UserDoc()?.presentationMode === 'watching') {
-        //     RecordingApi.Instance.pauseVideoAndMovements();
-        //     Doc.UserDoc().presentationMode = 'none';
-        //     // RecordingApi.Instance.pauseMovements()
-        // }
         // this is the easiest way to do this -> will talk with Bob about using mobx to do this to remove this line of code.
-        if (Doc.UserDoc()?.presentationMode === 'watching') RecordingApi.Instance.pauseFromInteraction();
+        if (Doc.UserDoc()?.presentationMode === 'watching') ReplayMovements.Instance.pauseFromInteraction();
 
         if (!this.isAnnotationOverlay && clamp) {
             // this section wraps the pan position, horizontally and/or vertically whenever the content is panned out of the viewing bounds
diff --git a/src/client/views/nodes/RecordingBox/RecordingBox.tsx b/src/client/views/nodes/RecordingBox/RecordingBox.tsx
index a28677525..0ff7c4292 100644
--- a/src/client/views/nodes/RecordingBox/RecordingBox.tsx
+++ b/src/client/views/nodes/RecordingBox/RecordingBox.tsx
@@ -8,7 +8,7 @@ import { FieldView } from "../FieldView";
 import { VideoBox } from "../VideoBox";
 import { RecordingView } from './RecordingView';
 import { DocumentType } from "../../../documents/DocumentTypes";
-import { Presentation } from "../../../util/RecordingApi";
+import { Presentation } from "../../../util/TrackMovements";
 import { Doc } from "../../../../fields/Doc";
 import { Id } from "../../../../fields/FieldSymbols";
 
diff --git a/src/client/views/nodes/RecordingBox/RecordingView.tsx b/src/client/views/nodes/RecordingBox/RecordingView.tsx
index ec9838bdd..83ed6914e 100644
--- a/src/client/views/nodes/RecordingBox/RecordingView.tsx
+++ b/src/client/views/nodes/RecordingBox/RecordingView.tsx
@@ -8,7 +8,7 @@ import { IconContext } from "react-icons";
 import { Networking } from '../../../Network';
 import { Upload } from '../../../../server/SharedMediaTypes';
 import { returnFalse, returnTrue, setupMoveUpEvents } from '../../../../Utils';
-import { Presentation, RecordingApi } from '../../../util/RecordingApi';
+import { Presentation, TrackMovements } from '../../../util/TrackMovements';
 
 export interface MediaSegment {
     videoChunks: any[],
@@ -64,7 +64,7 @@ export function RecordingView(props: IRecordingViewProps) {
     useEffect(() => {
         if (finished) {
             // make the total presentation that'll match the concatted video
-            let concatPres = trackScreen && RecordingApi.Instance.concatPresentations(videos.map(v => v.presentation as Presentation));
+            let concatPres = trackScreen && TrackMovements.Instance.concatPresentations(videos.map(v => v.presentation as Presentation));
 
             // this async function uses the server to create the concatted video and then sets the result to it's accessPaths
             (async () => {
@@ -135,7 +135,7 @@ export function RecordingView(props: IRecordingViewProps) {
         videoRecorder.current.onstart = (event: any) => {
             setRecording(true);
             // start the recording api when the video recorder starts
-            trackScreen && RecordingApi.Instance.start();
+            trackScreen && TrackMovements.Instance.start();
         };
 
         videoRecorder.current.onstop = () => {
@@ -149,7 +149,7 @@ export function RecordingView(props: IRecordingViewProps) {
                 };
 
                 // depending on if a presenation exists, add it to the video
-                const presentation = RecordingApi.Instance.yieldPresentation();
+                const presentation = TrackMovements.Instance.yieldPresentation();
                 setVideos(videos => [...videos, (presentation != null && trackScreen) ? { ...nextVideo, presentation } : nextVideo]);
             }
 
@@ -174,7 +174,7 @@ export function RecordingView(props: IRecordingViewProps) {
         stream instanceof MediaStream && stream.getTracks().forEach(track => track.stop());
 
         // finish/clear the recoringApi
-        RecordingApi.Instance.finish();
+        TrackMovements.Instance.finish();
 
         // this will call upon progessbar to update videos to be in the correct order
         setFinished(true);
diff --git a/src/client/views/nodes/VideoBox.tsx b/src/client/views/nodes/VideoBox.tsx
index 34df03954..5a221fea4 100644
--- a/src/client/views/nodes/VideoBox.tsx
+++ b/src/client/views/nodes/VideoBox.tsx
@@ -28,9 +28,9 @@ import { AnchorMenu } from "../pdf/AnchorMenu";
 import { StyleProp } from "../StyleProvider";
 import { FieldView, FieldViewProps } from './FieldView';
 import "./VideoBox.scss";
-import { Presentation, RecordingApi } from "../../util/RecordingApi";
-import { List } from "../../../fields/List";
+import { Presentation } from "../../util/TrackMovements";
 import { RecordingBox } from "./RecordingBox";
+import { ReplayMovements } from "../../util/ReplayMovements";
 const path = require('path');
 
 
@@ -148,7 +148,7 @@ export class VideoBox extends ViewBoxAnnotatableComponent<ViewBoxAnnotatableProp
           
           // if presentation data exists, pass it to the recordingPi
           if (this.presentation != null) {
-            RecordingApi.Instance.setVideoBox(this);  
+            ReplayMovements.Instance.setVideoBox(this);  
           }
       }
 
@@ -159,7 +159,7 @@ export class VideoBox extends ViewBoxAnnotatableComponent<ViewBoxAnnotatableProp
           
           // dispose the recordingApi's observer
           if (this.presentation != null) {
-            RecordingApi.Instance.removeVideoBox();  
+            ReplayMovements.Instance.removeVideoBox();  
           }
       }
 
-- 
cgit v1.2.3-70-g09d2


From c7058615587ebc6a8806b61a51fe0439d3daa7ad Mon Sep 17 00:00:00 2001
From: Michael Foiani <sotech117@michaels-mbp-5.devices.brown.edu>
Date: Wed, 15 Jun 2022 15:55:32 -0400
Subject: remove comments in videobox

---
 src/client/views/nodes/VideoBox.tsx | 24 ++----------------------
 1 file changed, 2 insertions(+), 22 deletions(-)

(limited to 'src/client/views/nodes/VideoBox.tsx')

diff --git a/src/client/views/nodes/VideoBox.tsx b/src/client/views/nodes/VideoBox.tsx
index 5a221fea4..c2da1f456 100644
--- a/src/client/views/nodes/VideoBox.tsx
+++ b/src/client/views/nodes/VideoBox.tsx
@@ -146,7 +146,7 @@ export class VideoBox extends ViewBoxAnnotatableComponent<ViewBoxAnnotatableProp
             }
           this.player && this.setPlayheadTime(0);
           
-          // if presentation data exists, pass it to the recordingPi
+          // if presentation data exists, pass it to ReplayMovments
           if (this.presentation != null) {
             ReplayMovements.Instance.setVideoBox(this);  
           }
@@ -157,7 +157,7 @@ export class VideoBox extends ViewBoxAnnotatableComponent<ViewBoxAnnotatableProp
         this.Pause();
           Object.keys(this._disposers).forEach(d => this._disposers[d]?.());
           
-          // dispose the recordingApi's observer
+          // tell ReplayMovements that this VideoBox is no longer active
           if (this.presentation != null) {
             ReplayMovements.Instance.removeVideoBox();  
           }
@@ -166,20 +166,6 @@ export class VideoBox extends ViewBoxAnnotatableComponent<ViewBoxAnnotatableProp
 
       // plays video
       @action public Play = (update: boolean = true) => {
-            // if (Doc.UserDoc().presentationMode === 'watching' && !this._playing) {
-            //     console.log('VideoBox : Play : presentation mode', this._playing);
-            //     return;
-            // }
-
-            // if presentation isn't null, call followmovements on the recording api
-            // if (this.presentation) {
-            //       console.log("presentation isn't null")
-            //       const err = RecordingApi.Instance.playMovements(this.presentation, this.player?.currentTime || 0, this);
-            //       err && console.log(err)
-            // } else {
-            //       console.log("presentation is null")
-            // }
-
             this._playing = true;
             const eleTime = this.player?.currentTime || 0;
             if (this.timeline) {
@@ -221,12 +207,6 @@ export class VideoBox extends ViewBoxAnnotatableComponent<ViewBoxAnnotatableProp
 
       // pauses video
       @action public Pause = (update: boolean = true) => {
-            // if (this.presentation) {
-            //       console.log('VideoBox : Pause');
-            //       const err = RecordingApi.Instance.pauseMovements();
-            //       err && console.log(err);
-            // }
-
             this._playing = false;
             this.removeCurrentlyPlaying();
             try {
-- 
cgit v1.2.3-70-g09d2


From 144fcd73ac26ce649a4b6b9d5186761b74a91fff Mon Sep 17 00:00:00 2001
From: Michael Foiani <sotech117@michaels-mbp-5.devices.brown.edu>
Date: Wed, 15 Jun 2022 16:18:10 -0400
Subject: fix the glitchy videoBox (i think), now a fairly smooth experience :)

---
 src/client/util/ReplayMovements.ts  |  3 ++-
 src/client/views/nodes/VideoBox.tsx | 41 +++++++++++++++++++++----------------
 2 files changed, 25 insertions(+), 19 deletions(-)

(limited to 'src/client/views/nodes/VideoBox.tsx')

diff --git a/src/client/util/ReplayMovements.ts b/src/client/util/ReplayMovements.ts
index 4122908cc..6a5746c0b 100644
--- a/src/client/util/ReplayMovements.ts
+++ b/src/client/util/ReplayMovements.ts
@@ -35,6 +35,7 @@ export class ReplayMovements {
             // console.warn('[recordingApi.ts] pauseMovements(): already on paused');
             return;
         }
+        Doc.UserDoc().presentationMode = 'none';
 
         this.isPlaying = false
         // TODO: set userdoc presentMode to browsing
@@ -77,7 +78,6 @@ export class ReplayMovements {
 
     // should be called from interacting with the screen
     pauseFromInteraction = () => {
-        Doc.UserDoc().presentationMode = 'none';
         this.videoBox?.Pause();
 
         this.pauseMovements();
@@ -148,6 +148,7 @@ export class ReplayMovements {
 
     endPlayingPresentation = () => {
         this.isPlaying = false;
+        Doc.UserDoc().presentationMode = 'none';
     }
 
     public playMovements = (presentation: Presentation, docIdtoDoc: Map<string, Doc>, timeViewed: number = 0) => {
diff --git a/src/client/views/nodes/VideoBox.tsx b/src/client/views/nodes/VideoBox.tsx
index c2da1f456..a50f57617 100644
--- a/src/client/views/nodes/VideoBox.tsx
+++ b/src/client/views/nodes/VideoBox.tsx
@@ -206,7 +206,8 @@ export class VideoBox extends ViewBoxAnnotatableComponent<ViewBoxAnnotatableProp
       }
 
       // pauses video
-      @action public Pause = (update: boolean = true) => {
+    @action public Pause = (update: boolean = true) => {
+          if (!this._playing) return;
             this._playing = false;
             this.removeCurrentlyPlaying();
             try {
@@ -491,7 +492,6 @@ export class VideoBox extends ViewBoxAnnotatableComponent<ViewBoxAnnotatableProp
       // for play button
 
       onPlayDown = () => {
-            console.log("PLAY DOWN");
             this._playing ? this.Pause() : this.Play();
       }
 
@@ -573,11 +573,16 @@ export class VideoBox extends ViewBoxAnnotatableComponent<ViewBoxAnnotatableProp
             docs.forEach(doc => doc._timecodeToHide = (doc._timecodeToShow = curTime) + 1);
             return this.addDocument(doc);
       }
+    
+    @action
+    private setPlaying = (b: boolean) => {
+        this._playing = b;
+    }
 
 
       // play back the audio from seekTimeInSeconds, fullPlay tells whether clip is being played to end vs link range
       @action
-      playFrom = (seekTimeInSeconds: number, endTime?: number, fullPlay: boolean = false) => {
+      playFrom = async (seekTimeInSeconds: number, endTime?: number, fullPlay: boolean = false) => {
             clearTimeout(this._playRegionTimer);
             if (Number.isNaN(this.player?.duration)) {
                   setTimeout(() => this.playFrom(seekTimeInSeconds, endTime), 500);
@@ -591,21 +596,21 @@ export class VideoBox extends ViewBoxAnnotatableComponent<ViewBoxAnnotatableProp
                   if (seekTimeInSeconds >= 0 && (this.timeline?.trimStart || 0) <= end && seekTimeInSeconds <= (this.timeline?.trimEnd || this.rawDuration)) {
                         this.player.currentTime = start;
                         this._audioPlayer && (this._audioPlayer.currentTime = seekTimeInSeconds);
-                        this.player.play();
-                        this._audioPlayer?.play();
-                        this._playing = true;
-                        this.addCurrentlyPlaying();
-                        this._playRegionTimer = setTimeout(
-                              () => {
-                                    // need to keep track of if end of clip is reached so on next play, clip restarts
-                                    if (fullPlay) {
-                                          Doc.UserDoc().presentationMode = 'none';
-                                          this._finished = true;
-                                    }
-                                    // removes from currently playing if playback has reached end of range marker
-                                    else this.removeCurrentlyPlaying();
-                                    this.Pause();
-                              }, playRegionDuration * 1000);
+                        this.player.play().then(() => {
+                            this._audioPlayer?.play();
+                            this.setPlaying(true);
+                            this.addCurrentlyPlaying();
+                            this._playRegionTimer = setTimeout(
+                                () => {
+                                        // need to keep track of if end of clip is reached so on next play, clip restarts
+                                        if (fullPlay) {
+                                            this._finished = true;
+                                        }
+                                        // removes from currently playing if playback has reached end of range marker
+                                        else this.removeCurrentlyPlaying();
+                                        this.Pause();
+                                }, playRegionDuration * 1000);
+                        });
                   } else {
                         this.Pause();
                   }
-- 
cgit v1.2.3-70-g09d2


From 1241c614324124e61cd1c2b0f8159122ee36ffd8 Mon Sep 17 00:00:00 2001
From: Michael Foiani <sotech117@michaels-mbp-5.devices.brown.edu>
Date: Wed, 15 Jun 2022 17:16:40 -0400
Subject: think i got basic videoplayer good enough for demo. should meet with
 mehek to fix some bugs i found

---
 src/client/views/nodes/VideoBox.tsx | 58 +++++++++++++++++++++++++------------
 1 file changed, 40 insertions(+), 18 deletions(-)

(limited to 'src/client/views/nodes/VideoBox.tsx')

diff --git a/src/client/views/nodes/VideoBox.tsx b/src/client/views/nodes/VideoBox.tsx
index a50f57617..794d363f1 100644
--- a/src/client/views/nodes/VideoBox.tsx
+++ b/src/client/views/nodes/VideoBox.tsx
@@ -150,6 +150,8 @@ export class VideoBox extends ViewBoxAnnotatableComponent<ViewBoxAnnotatableProp
           if (this.presentation != null) {
             ReplayMovements.Instance.setVideoBox(this);  
           }
+
+          this._playing = false;
       }
 
       componentWillUnmount() {
@@ -165,15 +167,19 @@ export class VideoBox extends ViewBoxAnnotatableComponent<ViewBoxAnnotatableProp
 
 
       // plays video
-      @action public Play = (update: boolean = true) => {
-            this._playing = true;
+    @action public Play = (update: boolean = true) => {
+        // BUGFIXES: only run if video is not already playing & dont set isplaying to true until that async call
+        if (this._playing) return;
+        // requires a promise for native videoplayer to be set to true
+        if (!this.player) this._playing = true;
             const eleTime = this.player?.currentTime || 0;
             if (this.timeline) {
                   let start = eleTime >= this.timeline.trimEnd || eleTime <= this.timeline.trimStart ? this.timeline.trimStart : eleTime;
 
                   if (this._finished) {
-                        // restarts video if reached end on previous play
-                        this._finished = false;
+                    // restarts video if reached end on previous play
+                      this._finished = false;
+                      console.log('trim start', this.timeline.trimStart);
                         start = this.timeline.trimStart;
                   }
 
@@ -200,6 +206,7 @@ export class VideoBox extends ViewBoxAnnotatableComponent<ViewBoxAnnotatableProp
             this.player && (this.player.currentTime = time);
             this._audioPlayer && (this._audioPlayer.currentTime = time);
             // TODO: revisit this and clean it
+            console.log('seek', time);
             if ((this.player?.currentTime || -1) < this.rawDuration) {
                   this._finished = false;
             }
@@ -335,8 +342,15 @@ export class VideoBox extends ViewBoxAnnotatableComponent<ViewBoxAnnotatableProp
 
       // updates video time
       @action
-      updateTimecode = () => {
-            this.player && (this.layoutDoc._currentTimecode = this.player.currentTime);
+      updateTimecode = (e? : any) => {
+          // BUGFIX: if seeking when paused, no longer finished
+          // ok this is constantly fired all the time
+        //   if (e && !this._playing && this.player && this.player.paused) {
+        //       console.log('updateTimecode: seeking when paused', e);
+        //       this.player.currentTime = e.target.currentTime;
+        //       this._finished = false;
+        //   }
+          this.player && (this.layoutDoc._currentTimecode = this.player.currentTime);
             try {
                   this._youtubePlayer && (this.layoutDoc._currentTimecode = this._youtubePlayer.getCurrentTime?.());
             } catch (e) {
@@ -491,7 +505,8 @@ export class VideoBox extends ViewBoxAnnotatableComponent<ViewBoxAnnotatableProp
 
       // for play button
 
-      onPlayDown = () => {
+    onPlayDown = () => {
+          console.log('onPlayDown', this._playing);
             this._playing ? this.Pause() : this.Play();
       }
 
@@ -593,25 +608,32 @@ export class VideoBox extends ViewBoxAnnotatableComponent<ViewBoxAnnotatableProp
                   const start = Math.max(this.timeline?.trimStart ?? 0, seekTimeInSeconds);
                   const playRegionDuration = end - start;
                   // checks if times are within clip range
-                  if (seekTimeInSeconds >= 0 && (this.timeline?.trimStart || 0) <= end && seekTimeInSeconds <= (this.timeline?.trimEnd || this.rawDuration)) {
-                        this.player.currentTime = start;
-                        this._audioPlayer && (this._audioPlayer.currentTime = seekTimeInSeconds);
+                if (seekTimeInSeconds >= 0 && (this.timeline?.trimStart || 0) <= end && seekTimeInSeconds <= (this.timeline?.trimEnd || this.rawDuration)) {
+                    this.player.currentTime = start;
+                    this._audioPlayer && (this._audioPlayer.currentTime = seekTimeInSeconds);
+                    // BUGFIX: video.play is async, so we need to wait for it to finish
+                    console.log('playFrom');
                         this.player.play().then(() => {
                             this._audioPlayer?.play();
                             this.setPlaying(true);
                             this.addCurrentlyPlaying();
                             this._playRegionTimer = setTimeout(
                                 () => {
-                                        // need to keep track of if end of clip is reached so on next play, clip restarts
-                                        if (fullPlay) {
-                                            this._finished = true;
-                                        }
-                                        // removes from currently playing if playback has reached end of range marker
-                                        else this.removeCurrentlyPlaying();
-                                        this.Pause();
+                                    // need to keep track of if end of clip is reached so on next play, clip restarts
+                                    if (fullPlay) {
+                                        this._finished = true;
+                                        // if (this.player) {
+                                        //     this.player.currentTime = 0;
+                                        //     this.updateTimecode();
+                                        // }
+                                        
+                                    }
+                                    // removes from currently playing if playback has reached end of range marker
+                                    else this.removeCurrentlyPlaying();
+                                    this.Pause();
                                 }, playRegionDuration * 1000);
                         });
-                  } else {
+                } else {
                         this.Pause();
                   }
             }
-- 
cgit v1.2.3-70-g09d2


From 64200804a1e63981593c89abecedd4ae769da3b6 Mon Sep 17 00:00:00 2001
From: Michael Foiani <sotech117@Michaels-MacBook-Pro-5.local>
Date: Wed, 15 Jun 2022 22:15:03 -0400
Subject: got basic demo working

---
 src/client/views/nodes/VideoBox.tsx | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

(limited to 'src/client/views/nodes/VideoBox.tsx')

diff --git a/src/client/views/nodes/VideoBox.tsx b/src/client/views/nodes/VideoBox.tsx
index 794d363f1..b18d3ad1a 100644
--- a/src/client/views/nodes/VideoBox.tsx
+++ b/src/client/views/nodes/VideoBox.tsx
@@ -345,11 +345,11 @@ export class VideoBox extends ViewBoxAnnotatableComponent<ViewBoxAnnotatableProp
       updateTimecode = (e? : any) => {
           // BUGFIX: if seeking when paused, no longer finished
           // ok this is constantly fired all the time
-        //   if (e && !this._playing && this.player && this.player.paused) {
-        //       console.log('updateTimecode: seeking when paused', e);
-        //       this.player.currentTime = e.target.currentTime;
-        //       this._finished = false;
-        //   }
+          if (e && !this._playing && this.player && this.player.paused) {
+              // console.log('updateTimecode: seeking when paused', e);
+              this.player.currentTime = e.target.currentTime;
+              this._finished = false;
+          }
           this.player && (this.layoutDoc._currentTimecode = this.player.currentTime);
             try {
                   this._youtubePlayer && (this.layoutDoc._currentTimecode = this._youtubePlayer.getCurrentTime?.());
-- 
cgit v1.2.3-70-g09d2


From 0370192be9c6c723f198fbf28d2a63ceef0e70d4 Mon Sep 17 00:00:00 2001
From: Michael Foiani <sotech117@Michaels-MacBook-Pro-5.local>
Date: Thu, 23 Jun 2022 12:36:45 -0400
Subject: small bug fixes and debugging

---
 src/client/util/ReplayMovements.ts  | 9 ++++++---
 src/client/views/nodes/VideoBox.tsx | 6 +++---
 2 files changed, 9 insertions(+), 6 deletions(-)

(limited to 'src/client/views/nodes/VideoBox.tsx')

diff --git a/src/client/util/ReplayMovements.ts b/src/client/util/ReplayMovements.ts
index 6a5746c0b..86bc4c5de 100644
--- a/src/client/util/ReplayMovements.ts
+++ b/src/client/util/ReplayMovements.ts
@@ -122,8 +122,10 @@ export class ReplayMovements {
             console.error(`docIdtoDoc did not contain docId ${docId}`)
             return undefined;
         }
+        // console.log('openTab', docId, doc);
         CollectionDockingView.AddSplit(doc, 'right');
-        const docView = DocumentManager.Instance.getDocumentViewById(docId);
+        const docView = DocumentManager.Instance.getDocumentView(doc);
+        // BUG - this returns undefined if the doc is already open
         return docView?.ComponentView as CollectionFreeFormView;
     }
 
@@ -135,7 +137,7 @@ export class ReplayMovements {
         document.Document._panY = panY;
     }
 
-    getFirstMovements = (movements: Movement[], timeViewed: number): Map<string, Movement> => {
+    getFirstMovements = (movements: Movement[]): Map<string, Movement> => {
         if (movements === null) return new Map();
         // generate a set of all unique docIds
         const docIdtoFirstMove = new Map();
@@ -172,7 +174,7 @@ export class ReplayMovements {
             if (isClosed) this.openTab(firstMovement.docId, docIdtoDoc);
 
             // for the open tabs, set it to the first move
-            const docIdtoFirstMove = this.getFirstMovements(filteredMovements, timeViewed);
+            const docIdtoFirstMove = this.getFirstMovements(filteredMovements);
             for (const [docId, firstMove] of docIdtoFirstMove) {
                 const colFFView = this.getCollectionFFView(docId);
                 if (colFFView) this.zoomAndPan(firstMove, colFFView);
@@ -192,6 +194,7 @@ export class ReplayMovements {
                 } else {
                     // tab wasn't open - open it and play the movement
                     const openedColFFView = this.openTab(movement.docId, docIdtoDoc);
+                    console.log('openedColFFView', openedColFFView);
                     openedColFFView && this.zoomAndPan(movement, openedColFFView);
                 }
 
diff --git a/src/client/views/nodes/VideoBox.tsx b/src/client/views/nodes/VideoBox.tsx
index b18d3ad1a..680c5f2a9 100644
--- a/src/client/views/nodes/VideoBox.tsx
+++ b/src/client/views/nodes/VideoBox.tsx
@@ -132,7 +132,9 @@ export class VideoBox extends ViewBoxAnnotatableComponent<ViewBoxAnnotatableProp
       public get player(): HTMLVideoElement | null { return this._videoRef; }
 
 
-      componentDidMount() {
+    componentDidMount() {
+        this._playing = false;
+        
             this.props.setContentView?.(this); // this tells the DocumentView that this VideoBox is the "content" of the document.  this allows the DocumentView to indirectly call getAnchor() on the VideoBox when making a link.
             if (this.youtubeVideoId) {
                   const youtubeaspect = 400 / 315;
@@ -150,8 +152,6 @@ export class VideoBox extends ViewBoxAnnotatableComponent<ViewBoxAnnotatableProp
           if (this.presentation != null) {
             ReplayMovements.Instance.setVideoBox(this);  
           }
-
-          this._playing = false;
       }
 
       componentWillUnmount() {
-- 
cgit v1.2.3-70-g09d2

