thumbnail preview of video when hovering over timeline

5 files changed, 104 insertions, 7 deletions

diff --git a/src/.DS_Store b/src/.DS_Store
index 4bf9cdac7..4751acf44 100644
--- a/src/.DS_Store
+++ b/src/.DS_Store
diff --git a/src/client/views/collections/CollectionStackedTimeline.scss b/src/client/views/collections/CollectionStackedTimeline.scss
index e8b6817b4..580cbccda 100644
--- a/src/client/views/collections/CollectionStackedTimeline.scss
+++ b/src/client/views/collections/CollectionStackedTimeline.scss
@@ -8,6 +8,16 @@
     background-color: $white;
     border: 2px solid $dark-gray;
     border-width: 0 2px 0 2px;
+
+    &:hover {
+        .collectionStackedTimeline-hover {
+            display: block;
+        }
+    }
+}
+
+.timeline-container:hover + .videoBox-thumbnail {
+    display: block;
 }
 
 ::-webkit-scrollbar {
@@ -61,15 +71,23 @@
         border-width: 1px;
     }
 
-    .collectionStackedTimeline-current {
+    .collectionStackedTimeline-current, .collectionStackedTimeline-hover {
         width: 1px;
         height: 100%;
-        background-color: $pink;
         position: absolute;
         top: 0px;
         pointer-events: none;
     }
 
+    .collectionStackedTimeline-current {
+        background-color: $pink;
+    }
+
+    .collectionStackedTimeline-hover {
+        display: none;
+        background-color: $medium-blue;
+    }
+
     .collectionStackedTimeline-marker-timeline {
         position: absolute;
         top: 2.5%;
@@ -108,3 +126,11 @@
         pointer-events: none;
     }
 }
+
+.videoBox-thumbnail {
+    position: absolute;
+    z-index: 10000;
+    transform: translate(-50%, 100%);
+    height: 100%;
+    display: none;
+}
\ No newline at end of file
diff --git a/src/client/views/collections/CollectionStackedTimeline.tsx b/src/client/views/collections/CollectionStackedTimeline.tsx
index ca0b9d3d3..2b78f5764 100644
--- a/src/client/views/collections/CollectionStackedTimeline.tsx
+++ b/src/client/views/collections/CollectionStackedTimeline.tsx
@@ -43,6 +43,8 @@ import {
 } from "../nodes/DocumentView";
 import { LabelBox } from "../nodes/LabelBox";
 import "./CollectionStackedTimeline.scss";
+import { VideoBox } from "../nodes/VideoBox";
+import { ImageField } from "../../../fields/URLField";
 
 export type CollectionStackedTimelineProps = {
     Play: () => void;
@@ -86,9 +88,12 @@ export class CollectionStackedTimeline extends CollectionSubView<CollectionStack
     @observable _trimEnd: number = 0; // trim controls end pos
 
     @observable _zoomFactor: number = 1;
-
     @observable _scroll: number = 0;
 
+    @observable _hoverTime: number = 0;
+
+    @observable _thumbnail: string | undefined;
+
     // ensures that clip doesn't get trimmed so small that controls cannot be adjusted anymore 
     get minTrimLength() { return Math.max(this._timeline?.getBoundingClientRect() ? 0.05 * this.clipDuration : 0, 0.5); }
 
@@ -315,11 +320,28 @@ export class CollectionStackedTimeline extends CollectionSubView<CollectionStack
     }
 
 
+    @action
+    onHover = (e: React.MouseEvent): void => {
+        e.stopPropagation();
+        const rect = this._timeline?.getBoundingClientRect();
+        const clientX = e.clientX;
+        if (rect) {
+            this._hoverTime = Math.min(this.toTimeline(clientX - rect.x, rect.width), this.clipEnd);
+            if (this.dataDoc.thumbnails) {
+                const nearest = Math.floor(this._hoverTime / this.props.rawDuration * VideoBox.numThumbnails);
+                const thumbnails = Cast(this.dataDoc.thumbnails, listSpec("string"), []);
+                const src = new ImageField(thumbnails[nearest]).url.href.replace(".png", "_s.png");
+                this._thumbnail = src;
+                console.log(src);
+            }
+        }
+    }
+
+
     // for dragging trim start handle
     @action
     trimLeft = (e: React.PointerEvent): void => {
         const rect = this._timeline?.getBoundingClientRect();
-        const clientX = e.movementX;
         setupMoveUpEvents(
             this,
             e,
@@ -346,7 +368,6 @@ export class CollectionStackedTimeline extends CollectionSubView<CollectionStack
     @action
     trimRight = (e: React.PointerEvent): void => {
         const rect = this._timeline?.getBoundingClientRect();
-        const clientX = e.movementX;
         setupMoveUpEvents(
             this,
             e,
@@ -631,6 +652,7 @@ export class CollectionStackedTimeline extends CollectionSubView<CollectionStack
                 style={{ width: this.props.PanelWidth() }}
                 onWheel={e => e.stopPropagation()}
                 onScroll={this.setScroll}
+                onMouseMove={(e) => this.isContentActive() && this.onHover(e)}
                 ref={wrapper => this._timelineWrapper = wrapper}>
                 <div
                     className="collectionStackedTimeline"
@@ -702,6 +724,13 @@ export class CollectionStackedTimeline extends CollectionSubView<CollectionStack
                     {/* {this.renderDictation} */}
 
                     <div
+                        className="collectionStackedTimeline-hover"
+                        style={{
+                            left: `${((this._hoverTime - this.clipStart) / this.clipDuration) * 100}%`,
+                        }}
+                    />
+
+                    <div
                         className="collectionStackedTimeline-current"
                         style={{
                             left: `${((this.currentTime - this.clipStart) / this.clipDuration) * 100}%`,
@@ -743,6 +772,7 @@ export class CollectionStackedTimeline extends CollectionSubView<CollectionStack
                     )}
                 </div>
             </div>
+            {this._thumbnail && <img className="videoBox-thumbnail" style={{ left: `calc(${((this._hoverTime - this.clipStart) / this.clipDuration) * 100}%` }} src={this._thumbnail} />}
         </div >);
     }
 }
diff --git a/src/client/views/nodes/AudioBox.tsx b/src/client/views/nodes/AudioBox.tsx
index 787ad939d..59c37753a 100644
--- a/src/client/views/nodes/AudioBox.tsx
+++ b/src/client/views/nodes/AudioBox.tsx
@@ -591,6 +591,8 @@ export class AudioBox extends ViewBoxAnnotatableComponent<ViewBoxAnnotatableProp
                         />
                     </div>}
 
+                {this.miniPlayer && <div>/</div>}
+
                 <div className="timecode-duration">
                     {this.timeline && formatTime(Math.round(this.timeline.clipDuration))}
                 </div>
diff --git a/src/client/views/nodes/VideoBox.tsx b/src/client/views/nodes/VideoBox.tsx
index 236c0d5fd..172b0b16f 100644
--- a/src/client/views/nodes/VideoBox.tsx
+++ b/src/client/views/nodes/VideoBox.tsx
@@ -28,6 +28,8 @@ import { AnchorMenu } from "../pdf/AnchorMenu";
 import { StyleProp } from "../StyleProvider";
 import { FieldView, FieldViewProps } from './FieldView';
 import "./VideoBox.scss";
+import { Image } from "wikijs";
+import { List } from "../../../fields/List";
 const path = require('path');
 
 
@@ -85,6 +87,7 @@ export class VideoBox extends ViewBoxAnnotatableComponent<ViewBoxAnnotatableProp
 
     static _youtubeIframeCounter: number = 0;
     static heightPercent = 80; // height of video relative to videoBox when timeline is open
+    static numThumbnails = 20;
     private _disposers: { [name: string]: IReactionDisposer } = {};
     private _youtubePlayer: YT.Player | undefined = undefined;
     private _videoRef: HTMLVideoElement | null = null; // <video> ref
@@ -370,6 +373,42 @@ export class VideoBox extends ViewBoxAnnotatableComponent<ViewBoxAnnotatableProp
     }
 
 
+    // extracts video thumbnails and saves them as field of doc
+    getVideoThumbnails = () => {
+        this.layoutDoc.cloneO
+        const video = document.createElement('video');
+        const thumbnails: string[] = [];
+        video.onloadedmetadata = () => {
+            video.currentTime = 0;
+        };
+        video.onseeked = () => {
+            const canvas = document.createElement('canvas');
+            canvas.height = video.videoHeight;
+            canvas.width = video.videoWidth;
+            const ctx = canvas.getContext('2d');
+            ctx?.drawImage(video, 0, 0, canvas.width, canvas.height);
+            const imgUrl = canvas.toDataURL();
+            const retitled = StrCast(this.rootDoc.title).replace(/[ -\.:]/g, "");
+            const encodedFilename = encodeURIComponent("thumbnail" + retitled + "_" + video.currentTime.toString().replace(/\./, "_"));
+            const filename = basename(encodedFilename);
+            VideoBox.convertDataUri(imgUrl, filename).then((returnedFilename: string) => {
+                returnedFilename && thumbnails.push(returnedFilename);
+                const newTime = video.currentTime + video.duration / VideoBox.numThumbnails;
+                if (newTime < video.duration) {
+                    video.currentTime = newTime;
+                    console.log(thumbnails.length);
+                }
+                else {
+                    this.dataDoc.thumbnails = new List<string>(thumbnails);
+                }
+            });
+        };
+
+        const field = Cast(this.dataDoc[this.fieldKey], VideoField);
+        field && (video.src = field.url.href);
+    }
+
+
     // sets video element ref
     @action
     setVideoRef = (vref: HTMLVideoElement | null) => {
@@ -381,6 +420,8 @@ export class VideoBox extends ViewBoxAnnotatableComponent<ViewBoxAnnotatableProp
             this._disposers.reactionDisposer?.();
             this._disposers.reactionDisposer = reaction(() => NumCast(this.layoutDoc._currentTimecode),
                 time => !this._playing && (vref.currentTime = time), { fireImmediately: true });
+
+            !this.dataDoc.thumbnails && this.getVideoThumbnails();
         }
     }
 
@@ -395,11 +436,9 @@ export class VideoBox extends ViewBoxAnnotatableComponent<ViewBoxAnnotatableProp
                     document.addEventListener('pointermove', this.controlsFade);
                     this._controlsVisible = true;
                     this._controlsFadeTimer = setTimeout(action(() => this._controlsVisible = false), 3000)
-                    console.log("added");
                 }
                 else {
                     document.removeEventListener('pointermove', this.controlsFade);
-                    console.log("removed");
                 }
             });
         }