adding autotagging

5 files changed, 36 insertions, 8 deletions

diff --git a/src/client/views/ViewBoxInterface.ts b/src/client/views/ViewBoxInterface.ts
index 0ddac8914..5726a78f3 100644
--- a/src/client/views/ViewBoxInterface.ts
+++ b/src/client/views/ViewBoxInterface.ts
@@ -24,6 +24,7 @@ export abstract class ViewBoxInterface<P> extends ObservableReactComponent<React
     promoteCollection?: () => void; // moves contents of collection to parent
     hasChildDocs?: () => Doc[];
     docEditorView?: () => void;
+    autoTag?: () => void; // auto tag the document
     showSmartDraw?: (x: number, y: number, regenerate?: boolean) => void;
     updateIcon?: (usePanelDimensions?: boolean) => Promise<void>; // updates the icon representation of the document
     getAnchor?: (addAsAnnotation: boolean, pinData?: PinProps) => Doc; // returns an Anchor Doc that represents the current state of the doc's componentview (e.g., the current playhead location of a an audio/video box)
diff --git a/src/client/views/collections/collectionFreeForm/ImageLabelBox.tsx b/src/client/views/collections/collectionFreeForm/ImageLabelBox.tsx
index ff9fb14e7..28d3ccd48 100644
--- a/src/client/views/collections/collectionFreeForm/ImageLabelBox.tsx
+++ b/src/client/views/collections/collectionFreeForm/ImageLabelBox.tsx
@@ -158,7 +158,7 @@ export class ImageLabelBox extends ViewBoxBaseComponent<FieldViewProps>() {
         this._currentLabel = e.target.value;
     });
 
-    classifyImagesInBox = async () => {
+    classifyImagesInBox = async (prompt? : string) => {
         this.startLoading();
 
         // Converts the images into a Base64 format, afterwhich the information is sent to GPT to label them.
@@ -168,7 +168,7 @@ export class ImageLabelBox extends ViewBoxBaseComponent<FieldViewProps>() {
                 const url = ImageCastWithSuffix(doc[Doc.LayoutDataKey(doc)], '_o') ?? '';
                 return imageUrlToBase64(url).then(hrefBase64 =>
                 !hrefBase64 ? undefined : 
-                    gptImageLabel(hrefBase64,'Give three labels to describe this image.').then(labels => 
+                    gptImageLabel(hrefBase64, prompt ?? 'Give three labels to describe this image.').then(labels => 
                         ({ doc, labels }))) ; // prettier-ignore
             }
         });
diff --git a/src/client/views/nodes/ImageBox.tsx b/src/client/views/nodes/ImageBox.tsx
index 999d7089b..38f0390db 100644
--- a/src/client/views/nodes/ImageBox.tsx
+++ b/src/client/views/nodes/ImageBox.tsx
@@ -119,6 +119,13 @@ export class ImageBox extends ViewBoxAnnotatableComponent<FieldViewProps>() {
         this._dropDisposer?.();
         ele && (this._dropDisposer = DragManager.MakeDropTarget(ele, this.drop.bind(this), this.Document));
     };
+
+    autoTag = () => {
+        //Doc.getDescription(this.Document).then(desc => this.desc = desc)
+    }
+
+
+
     getAnchor = (addAsAnnotation: boolean, pinProps?: PinProps) => {
         const visibleAnchor = this._getAnchor?.(this._savedAnnotations, true); // use marquee anchor, otherwise, save zoom/pan as anchor
         const anchor =
diff --git a/src/client/views/nodes/scrapbook/ScrapbookBox.tsx b/src/client/views/nodes/scrapbook/ScrapbookBox.tsx
index 6cfe9a62c..55d0df585 100644
--- a/src/client/views/nodes/scrapbook/ScrapbookBox.tsx
+++ b/src/client/views/nodes/scrapbook/ScrapbookBox.tsx
@@ -68,7 +68,21 @@ export class ScrapbookBox extends ViewBoxAnnotatableComponent<FieldViewProps>()
             placeholder2.y = 200;
             placeholder2._width = 250;
             //placeholder2.overrideFields = new List<string>(['x', 'y', '_width']); // shouldn't need to do this for layout fields since the placeholder already overrides its protos
-            this.dataDoc[this.fieldKey] = new List<Doc>([placeholder, placeholder2]);
+            
+            const sidebar = Docs.Create.TextDocument('sidebar');
+            sidebar.accepts_docType = DocumentType.RTF;
+            sidebar.accepts_textType = 'sidebar';
+            const placeholder3 = new Doc();
+            placeholder3.proto = sidebar;
+            placeholder3.original = sidebar;
+            placeholder3.x = 280;
+            placeholder3.y = -50;
+            placeholder3._width = 50; 
+            placeholder3._height = 200;
+
+            this.dataDoc[this.fieldKey] = new List<Doc>([placeholder, placeholder2, placeholder3]);
+
+          
         }
     }
 
diff --git a/src/client/views/search/FaceRecognitionHandler.tsx b/src/client/views/search/FaceRecognitionHandler.tsx
index cb837e3ab..436cca8cf 100644
--- a/src/client/views/search/FaceRecognitionHandler.tsx
+++ b/src/client/views/search/FaceRecognitionHandler.tsx
@@ -9,6 +9,8 @@ import { ImageField } from '../../../fields/URLField';
 import { DocumentType } from '../../documents/DocumentTypes';
 import { Docs } from '../../documents/Documents';
 import { DocumentManager } from '../../util/DocumentManager';
+import { reaction } from 'mobx';
+import { DocumentView } from '../nodes/DocumentView';
 
 /**
  * A singleton class that handles face recognition and manages face Doc collections for each face found.
@@ -33,7 +35,7 @@ export class FaceRecognitionHandler {
     // eslint-disable-next-line no-use-before-define
     static _instance: FaceRecognitionHandler;
     private _apiModelReady = false;
-    private _pendingAPIModelReadyDocs: Doc[] = [];
+    private _pendingAPIModelReadyDocs: DocumentView[] = [];
 
     public static get Instance() {
         return FaceRecognitionHandler._instance ?? new FaceRecognitionHandler();
@@ -126,7 +128,7 @@ export class FaceRecognitionHandler {
     constructor() {
         FaceRecognitionHandler._instance = this;
         this.loadAPIModels().then(() => this._pendingAPIModelReadyDocs.forEach(this.classifyFacesInImage));
-        DocumentManager.Instance.AddAnyViewRenderedCB(dv => FaceRecognitionHandler.Instance.classifyFacesInImage(dv.Document));
+        DocumentManager.Instance.AddAnyViewRenderedCB(dv => FaceRecognitionHandler.Instance.classifyFacesInImage(dv));
     }
 
     /**
@@ -199,14 +201,18 @@ export class FaceRecognitionHandler {
      * match them to existing unique faces, otherwise new unique face(s) are created.
      * @param imgDoc The document being analyzed.
      */
-    private classifyFacesInImage = async (imgDoc: Doc) => {
+    private classifyFacesInImage = async (imgDocView: DocumentView) => {
+        const imgDoc = imgDocView.Document;
         if (!Doc.UserDoc().recognizeFaceImages) return;
         const activeDashboard = Doc.ActiveDashboard;
         if (!this._apiModelReady || !activeDashboard) {
-            this._pendingAPIModelReadyDocs.push(imgDoc);
+            this._pendingAPIModelReadyDocs.push(imgDocView);
         } else if (imgDoc.type === DocumentType.LOADING && !imgDoc.loadingError) {
-            setTimeout(() => this.classifyFacesInImage(imgDoc), 1000);
+            setTimeout(() => this.classifyFacesInImage(imgDocView), 1000);
         } else {
+            reaction(() => ({sel:imgDocView.isSelected()}), ({sel}) => !sel && imgDoc.type == 'text'
+            && imgDocView.ComponentView?.autoTag?.(), {fireImmediately: true} 
+        )
             const imgUrl = ImageCast(imgDoc[Doc.LayoutDataKey(imgDoc)]);
             if (imgUrl && !DocListCast(Doc.MyFaceCollection?.examinedFaceDocs).includes(imgDoc[DocData])) {
                 // only examine Docs that have an image and that haven't already been examined.