feat: added view of labels on docs in freeform

6 files changed, 161 insertions, 20 deletions

diff --git a/src/client/views/KeywordBox.tsx b/src/client/views/KeywordBox.tsx
new file mode 100644
index 000000000..3faddeb64
--- /dev/null
+++ b/src/client/views/KeywordBox.tsx
@@ -0,0 +1,69 @@
+import { action, makeObservable } from 'mobx';
+import { observer } from 'mobx-react';
+import React from 'react';
+import { Doc } from '../../fields/Doc';
+import { DocData } from '../../fields/DocSymbols';
+import { List } from '../../fields/List';
+import { ObservableReactComponent } from './ObservableReactComponent';
+
+interface KeywordBoxProps {
+    _doc: Doc;
+    _isEditing: boolean;
+}
+
+@observer
+export class KeywordBox extends ObservableReactComponent<KeywordBoxProps> {
+    constructor(props: any) {
+        super(props);
+        makeObservable(this);
+    }
+
+    @action
+    setToEditing = () => {
+        this._props._isEditing = true;
+    };
+
+    @action
+    setToView = () => {
+        this._props._isEditing = false;
+    };
+
+    submitLabel = () => {};
+
+    onInputChange = () => {};
+
+    render() {
+        const keywordsList = this._props._doc![DocData].data_labels;
+        return (
+            <div className="keywords-container">
+                {(keywordsList as List<string>).map(label => {
+                    return (
+                        <div className="keyword" onClick={this.setToEditing}>
+                            {label}
+                        </div>
+                    );
+                })}
+                {this._props._isEditing ? (
+                    <div>
+                        <input
+                            defaultValue=""
+                            autoComplete="off"
+                            onChange={this.onInputChange}
+                            onKeyDown={e => {
+                                e.key === 'Enter' ? this.submitLabel() : null;
+                                e.stopPropagation();
+                            }}
+                            type="text"
+                            placeholder="Input keywords for document..."
+                            aria-label="keyword-input"
+                            className="keyword-input"
+                            style={{ width: '100%', borderRadius: '5px' }}
+                        />
+                    </div>
+                ) : (
+                    <div></div>
+                )}
+            </div>
+        );
+    }
+}
diff --git a/src/client/views/StyleProvider.scss b/src/client/views/StyleProvider.scss
index ce00f6101..1e2af9a3a 100644
--- a/src/client/views/StyleProvider.scss
+++ b/src/client/views/StyleProvider.scss
@@ -53,3 +53,16 @@
 .styleProvider-treeView-icon {
     opacity: 0;
 }
+
+.keywords-container {
+    display: flex;
+    flex-wrap: wrap;
+}
+
+.keyword {
+    padding: 5px 10px;
+    background-color: lightblue;
+    border: 1px solid black;
+    border-radius: 5px;
+    white-space: nowrap;
+}
diff --git a/src/client/views/StyleProvider.tsx b/src/client/views/StyleProvider.tsx
index b7f8a3170..fb509516a 100644
--- a/src/client/views/StyleProvider.tsx
+++ b/src/client/views/StyleProvider.tsx
@@ -12,7 +12,9 @@ import { BsArrowDown, BsArrowDownUp, BsArrowUp } from 'react-icons/bs';
 import { FaFilter } from 'react-icons/fa';
 import { ClientUtils, DashColor, lightOrDark } from '../../ClientUtils';
 import { Doc, Opt, StrListCast } from '../../fields/Doc';
+import { DocData } from '../../fields/DocSymbols';
 import { Id } from '../../fields/FieldSymbols';
+import { List } from '../../fields/List';
 import { ScriptField } from '../../fields/ScriptField';
 import { BoolCast, Cast, DocCast, ImageCast, NumCast, ScriptCast, StrCast } from '../../fields/Types';
 import { AudioAnnoState } from '../../server/SharedMediaTypes';
@@ -23,6 +25,7 @@ import { SnappingManager } from '../util/SnappingManager';
 import { undoBatch, UndoManager } from '../util/UndoManager';
 import { TreeSort } from './collections/TreeSort';
 import { Colors } from './global/globalEnums';
+import { KeywordBox } from './KeywordBox';
 import { DocumentView, DocumentViewProps } from './nodes/DocumentView';
 import { FieldViewProps } from './nodes/FieldView';
 import { StyleProp } from './StyleProp';
@@ -367,12 +370,18 @@ export function DefaultStyleProvider(doc: Opt<Doc>, props: Opt<FieldViewProps &
                     </Tooltip>
                 );
             };
+            const keywords = () => {
+                if (doc && doc![DocData].data_labels && doc![DocData].showLabels) {
+                    return (<KeywordBox _isEditing={false} _doc={doc}></KeywordBox>)
+                }
+            }
             return (
                 <>
                     {paint()}
                     {lock()}
                     {filter()}
                     {audio()}
+                    {keywords()}
                 </>
             );
         }
diff --git a/src/client/views/collections/collectionFreeForm/ImageLabelBox.tsx b/src/client/views/collections/collectionFreeForm/ImageLabelBox.tsx
index 571a4504f..cfb81e1a0 100644
--- a/src/client/views/collections/collectionFreeForm/ImageLabelBox.tsx
+++ b/src/client/views/collections/collectionFreeForm/ImageLabelBox.tsx
@@ -5,7 +5,7 @@ import { observer } from 'mobx-react';
 import React from 'react';
 import { Doc, NumListCast, Opt } from '../../../../fields/Doc';
 import { Docs } from '../../../documents/Documents';
-import { CollectionViewType, DocumentType } from '../../../documents/DocumentTypes';
+import { DocumentType } from '../../../documents/DocumentTypes';
 import { ViewBoxBaseComponent } from '../../DocComponent';
 import { FieldView, FieldViewProps } from '../../nodes/FieldView';
 import { MarqueeOptionsMenu } from './MarqueeOptionsMenu';
@@ -24,6 +24,9 @@ import { List } from '../../../../fields/List';
 import { DragManager } from '../../../util/DragManager';
 import { OpenWhere } from '../../nodes/OpenWhere';
 import similarity from 'compute-cosine-similarity';
+import { DocumentView } from '../../nodes/DocumentView';
+
+export class ImageInformationItem {}
 
 export class ImageLabelBoxData {
     static _instance: ImageLabelBoxData;
@@ -101,7 +104,6 @@ export class ImageLabelBox extends ViewBoxBaseComponent<FieldViewProps>() {
         makeObservable(this);
         ring.register();
         ImageLabelBox.Instance = this;
-        console.log('Image Box Has Been Initialized');
     }
 
     // ImageLabelBox.Instance.setData()
@@ -127,7 +129,6 @@ export class ImageLabelBox extends ViewBoxBaseComponent<FieldViewProps>() {
     @action
     startLoading = () => {
         this._loading = true;
-        console.log('Start loading has been called!');
     };
 
     @action
@@ -138,6 +139,11 @@ export class ImageLabelBox extends ViewBoxBaseComponent<FieldViewProps>() {
     @action
     toggleDisplayInformation = () => {
         this._displayImageInformation = !this._displayImageInformation;
+        if (this._displayImageInformation) {
+            this._selectedImages.forEach(doc => (doc[DocData].showLabels = true));
+        } else {
+            this._selectedImages.forEach(doc => (doc[DocData].showLabels = false));
+        }
     };
 
     @action
@@ -155,33 +161,58 @@ export class ImageLabelBox extends ViewBoxBaseComponent<FieldViewProps>() {
     classifyImagesInBox = async () => {
         this.startLoading();
 
+        // const imageInfos = this._selectedImages.map(async doc => {
+        //     if (!doc[DocData].data_labels) {
+        //         const [name, type] = ImageCast(doc[Doc.LayoutFieldKey(doc)]).url.href.split('.');
+        //         return CollectionCardView.imageUrlToBase64(`${name}_o.${type}`).then(hrefBase64 =>
+        //         !hrefBase64 ? undefined :
+        //             gptImageLabel(hrefBase64).then(labels =>
+        //                 Promise.all(labels.split('\n').map(label => gptGetEmbedding(label))).then(embeddings =>
+        //                     ({ doc, embeddings, labels }))) ); // prettier-ignore
+        //     }
+        // }); // Converts the images into a Base64 format, afterwhich the information is sent to GPT to label them.
+
         const imageInfos = this._selectedImages.map(async doc => {
             if (!doc[DocData].data_labels) {
                 const [name, type] = ImageCast(doc[Doc.LayoutFieldKey(doc)]).url.href.split('.');
                 return CollectionCardView.imageUrlToBase64(`${name}_o.${type}`).then(hrefBase64 =>
                 !hrefBase64 ? undefined : 
                     gptImageLabel(hrefBase64).then(labels => 
-                        Promise.all(labels.split('\n').map(label => gptGetEmbedding(label))).then(embeddings => 
-                            ({ doc, embeddings, labels }))) ); // prettier-ignore
+                
+                            ({ doc, labels }))) ; // prettier-ignore
             }
-        }); // Converts the images into a Base64 format, afterwhich the information is sent to GPT to label them.
+        });
 
         (await Promise.all(imageInfos)).forEach(imageInfo => {
-            if (imageInfo && imageInfo.embeddings && Array.isArray(imageInfo.embeddings)) {
-                imageInfo.doc[DocData].data_labels = imageInfo.labels;
+            if (imageInfo) {
+                imageInfo.doc[DocData].data_labels = new List<string>();
 
                 const labels = imageInfo.labels.split('\n');
                 labels.forEach(label => {
-                    label = label.replace(/^\d+\.\s*/, '').trim();
+                    label = label.replace(/^\d+\.\s*|-|\*/, '').trim();
                     imageInfo.doc[DocData][`${label}`] = true;
-                });
-
-                numberRange(3).forEach(n => {
-                    imageInfo.doc[`data_labels_embedding_${n + 1}`] = new List<number>(imageInfo.embeddings[n]);
+                    (imageInfo.doc[DocData].data_labels as List<string>).push(label);
                 });
             }
         }); // Add the labels as fields to each image.
 
+        // (await Promise.all(imageInfos)).forEach(imageInfo => {
+        //     if (imageInfo && imageInfo.embeddings && Array.isArray(imageInfo.embeddings)) {
+        //         imageInfo.doc[DocData].data_labels = new List<string>();
+
+        //         const labels = imageInfo.labels.split('\n');
+        //         labels.forEach(label => {
+        //             label = label.replace(/^\d+\.\s*|-|\*/, '').trim();
+        //             imageInfo.doc[DocData][`${label}`] = true;
+        //             (imageInfo.doc[DocData].data_labels as List<string>).push(label);
+        //         });
+
+        //         numberRange(5).forEach(n => {
+        //             imageInfo.doc[`data_labels_embedding_${n + 1}`] = new List<number>(imageInfo.embeddings[n]);
+        //         });
+        //     }
+        // }); // Add the labels as fields to each image.
+
         this.endLoading();
     };
 
@@ -189,7 +220,13 @@ export class ImageLabelBox extends ViewBoxBaseComponent<FieldViewProps>() {
      * Groups images to most similar labels.
      */
     groupImagesInBox = action(async () => {
-        console.log('Calling!');
+        this._selectedImages.forEach(doc => {
+            (doc[DocData].data_labels as List<string>).forEach(async (label, index) => {
+                const embedding = await gptGetEmbedding(label);
+                doc[`data_labels_embedding_${index + 1}`] = new List<number>(embedding);
+            });
+        });
+
         const labelToEmbedding = new Map<string, number[]>();
         // Create embeddings for the labels.
         await Promise.all(this._labelGroups.map(async label => gptGetEmbedding(label).then(labelEmbedding => labelToEmbedding.set(label, labelEmbedding))));
@@ -197,7 +234,7 @@ export class ImageLabelBox extends ViewBoxBaseComponent<FieldViewProps>() {
         // For each image, loop through the labels, and calculate similarity. Associate it with the
         // most similar one.
         this._selectedImages.forEach(doc => {
-            const embedLists = numberRange(3).map(n => Array.from(NumListCast(doc[`data_labels_embedding_${n + 1}`])));
+            const embedLists = numberRange(5).map(n => Array.from(NumListCast(doc[`data_labels_embedding_${n + 1}`])));
             const bestEmbedScore = (embedding: Opt<number[]>) => Math.max(...embedLists.map((l, index) => (embedding && (1 - index * 0.1) * similarity(Array.from(embedding), l)!) || 0));
             const {label: mostSimilarLabelCollect} = 
                 this._labelGroups.map(label => ({ label, similarityScore: bestEmbedScore(labelToEmbedding.get(label)) }))
@@ -293,10 +330,14 @@ export class ImageLabelBox extends ViewBoxBaseComponent<FieldViewProps>() {
                         {this._selectedImages.map(doc => {
                             const [name, type] = ImageCast(doc[Doc.LayoutFieldKey(doc)]).url.href.split('.');
                             return (
-                                <div className="image-information" style={{ borderColor: SettingsManager.userColor }} key={Utils.GenerateGuid()} onClick={() => this._props.addDocTab(doc, OpenWhere.addRightKeyvalue)}>
-                                    <img src={`${name}_o.${type}`}></img>
-                                    <div className="image-information-labels">
-                                        {(doc[DocData].data_labels as string).split('\n').map(label => {
+                                <div className="image-information" style={{ borderColor: SettingsManager.userColor }} key={Utils.GenerateGuid()}>
+                                    <img
+                                        src={`${name}_o.${type}`}
+                                        onClick={async () => {
+                                            await DocumentView.showDocument(doc, { willZoomCentered: true });
+                                        }}></img>
+                                    <div className="image-information-labels" onClick={() => this._props.addDocTab(doc, OpenWhere.addRightKeyvalue)}>
+                                        {(doc[DocData].data_labels as List<string>).map(label => {
                                             return (
                                                 <div key={Utils.GenerateGuid()} className="image-label" style={{ backgroundColor: SettingsManager.userVariantColor, borderColor: SettingsManager.userColor }}>
                                                     {label}
diff --git a/src/client/views/nodes/DocumentView.tsx b/src/client/views/nodes/DocumentView.tsx
index 7a1f94948..9ff96c692 100644
--- a/src/client/views/nodes/DocumentView.tsx
+++ b/src/client/views/nodes/DocumentView.tsx
@@ -573,7 +573,7 @@ export class DocumentViewInternal extends DocComponent<FieldViewProps & Document
             if (this._props.renderDepth === 0) {
                 appearanceItems.splice(0, 0, { description: 'Open in Lightbox', event: () => DocumentView.SetLightboxDoc(this.Document), icon: 'external-link-alt' });
             }
-            appearanceItems.push({ description: 'Pin', event: () => this._props.pinToPres(this.Document, {}), icon: 'eye' });
+            appearanceItems.push({ description: 'Pin', event: () => this._props.pinToPres(this.Document, {}), icon: 'map-pin' });
             if (this.Document._layout_isFlashcard) {
                 appearanceItems.push({ description: 'Create ChatCard', event: () => this.askGPT(), icon: 'id-card' });
             }
diff --git a/src/client/views/nodes/ImageBox.tsx b/src/client/views/nodes/ImageBox.tsx
index 3da878a4f..fb90f907f 100644
--- a/src/client/views/nodes/ImageBox.tsx
+++ b/src/client/views/nodes/ImageBox.tsx
@@ -1,5 +1,6 @@
 import { FontAwesomeIcon } from '@fortawesome/react-fontawesome';
 import { Tooltip } from '@mui/material';
+import zIndex from '@mui/material/styles/zIndex';
 import { Colors } from 'browndash-components';
 import { action, computed, IReactionDisposer, makeObservable, observable, ObservableMap, reaction } from 'mobx';
 import { observer } from 'mobx-react';
@@ -10,6 +11,7 @@ import { Doc, DocListCast, Opt } from '../../../fields/Doc';
 import { DocData } from '../../../fields/DocSymbols';
 import { Id } from '../../../fields/FieldSymbols';
 import { InkTool } from '../../../fields/InkField';
+import { List } from '../../../fields/List';
 import { ObjectField } from '../../../fields/ObjectField';
 import { Cast, ImageCast, NumCast, StrCast } from '../../../fields/Types';
 import { ImageField } from '../../../fields/URLField';
@@ -281,6 +283,13 @@ export class ImageBox extends ViewBoxAnnotatableComponent<FieldViewProps>() {
                 }),
                 icon: 'pencil-alt',
             });
+            funcs.push({
+                description: 'Toggle Keywords',
+                event: () => {
+                    this.Document[DocData].showLabels = !this.Document[DocData].showLabels;
+                },
+                icon: 'eye',
+            });
             ContextMenu.Instance?.addItem({ description: 'Options...', subitems: funcs, icon: 'asterisk' });
         }
     };