Merge branch 'master' of https://github.com/brown-dash/Dash-Web into joanne-tutorialagent

1 files changed, 57 insertions, 7 deletions

diff --git a/src/client/views/nodes/VideoBox.tsx b/src/client/views/nodes/VideoBox.tsx
index fa099178c..f994bdbb5 100644
--- a/src/client/views/nodes/VideoBox.tsx
+++ b/src/client/views/nodes/VideoBox.tsx
@@ -30,6 +30,7 @@ import { StyleProp } from '../StyleProp';
 import { DocumentView } from './DocumentView';
 import { FieldView, FieldViewProps } from './FieldView';
 import { FocusViewOptions } from './FocusViewOptions';
+import { gptImageLabel } from '../../apis/gpt/GPT';
 import './VideoBox.scss';
 
 /**
@@ -109,6 +110,52 @@ export class VideoBox extends ViewBoxAnnotatableComponent<FieldViewProps>() {
         return this._videoRef;
     }
 
+    autoTag = async () => {
+        if (this.Document.$tags_chat) return;
+        try {
+            if (!this.player) throw new Error('Video element not available.');
+
+            // 1) Extract a frame at the video's midpoint
+            const videoDuration = this.player.duration;
+            const snapshotTime = videoDuration / 2;
+
+            // Seek the video element to the midpoint
+            await new Promise<void>(resolve => {
+                const onSeeked = () => {
+                    this.player!.removeEventListener('seeked', onSeeked);
+                    resolve();
+                };
+                this.player!.addEventListener('seeked', onSeeked);
+                this.player!.currentTime = snapshotTime;
+            });
+
+            // 2) Draw the frame onto a canvas and get a base64 representation
+            const canvas = document.createElement('canvas');
+            canvas.width = this.player.videoWidth;
+            canvas.height = this.player.videoHeight;
+            const ctx = canvas.getContext('2d');
+            if (!ctx) throw new Error('Failed to create canvas context.');
+            ctx.drawImage(this.player, 0, 0, canvas.width, canvas.height);
+            const base64Image = canvas.toDataURL('image/png');
+
+            // 3) Send the image data to GPT for classification and descriptive tags
+            const label = await gptImageLabel(
+                base64Image,
+                `Classify this video frame as either a PERSON or LANDSCAPE.
+            Then provide five additional descriptive tags (single words) separated by spaces.
+            Finally, add one detailed summary phrase using underscores.`
+            ).then(raw => raw.trim().toUpperCase());
+
+            // 4) Normalize and store labels in the Document's tags
+            const aspect = this.player!.videoWidth / (this.player!.videoHeight || 1);
+            this.Document.$tags_chat = new List<string>([...label.split(/\s+/), `ASPECT_${aspect}`]);
+            // 5) Turn on tag display in layout
+            this.Document._layout_showTags = true;
+        } catch (err) {
+            console.error('Video autoTag failed:', err);
+        }
+    };
+
     componentDidMount() {
         this.unmounting = false;
         this._props.setContentViewBox?.(this); // this tells the DocumentView that this VideoBox is the "content" of the document.  this allows the DocumentView to indirectly call getAnchor() on the VideoBox when making a link.
@@ -338,12 +385,17 @@ export class VideoBox extends ViewBoxAnnotatableComponent<FieldViewProps>() {
     getAnchor = (addAsAnnotation: boolean, pinProps?: PinProps) => {
         const timecode = Cast(this.layoutDoc._layout_currentTimecode, 'number', null);
         const marquee = AnchorMenu.Instance.GetAnchor?.(undefined, addAsAnnotation);
+        const docAnchor = () =>
+            Docs.Create.ConfigDocument({
+                title: '#' + timecode,
+                _timecodeToShow: timecode,
+                annotationOn: this.Document,
+            });
         if (!addAsAnnotation && marquee) marquee.backgroundColor = 'transparent';
-        const anchor =
-            addAsAnnotation && marquee
-                ? CollectionStackedTimeline.createAnchor(this.Document, this.dataDoc, this.annotationKey, timecode || undefined, undefined, marquee, addAsAnnotation) || this.Document
-                : Docs.Create.ConfigDocument({ title: '#' + timecode, _timecodeToShow: timecode, annotationOn: this.Document });
+        const visibleAnchor = () => addAsAnnotation && marquee && (CollectionStackedTimeline.createAnchor(this.Document, this.dataDoc, this.annotationKey, timecode || undefined, undefined, marquee, addAsAnnotation) || this.Document);
+        const anchor = visibleAnchor() || docAnchor();
         PinDocView(anchor, { pinDocLayout: pinProps?.pinDocLayout, pinData: { ...(pinProps?.pinData ?? {}), temporal: true, pannable: true } }, this.Document);
+        addAsAnnotation && this.addDocument(anchor);
         return anchor;
     };
 
@@ -376,9 +428,7 @@ export class VideoBox extends ViewBoxAnnotatableComponent<FieldViewProps>() {
             }
             return this._stackedTimeline.getView(doc, options);
         }
-        return new Promise<Opt<DocumentView>>(res => {
-            DocumentView.addViewRenderedCb(doc, dv => res(dv));
-        });
+        return new Promise<Opt<DocumentView>>(res => DocumentView.addViewRenderedCb(doc, res));
     };
 
     // extracts video thumbnails and saves them as field of doc