feat: added view of labels on docs in freeform

1 files changed, 60 insertions, 19 deletions

diff --git a/src/client/views/collections/collectionFreeForm/ImageLabelBox.tsx b/src/client/views/collections/collectionFreeForm/ImageLabelBox.tsx
index 571a4504f..cfb81e1a0 100644
--- a/src/client/views/collections/collectionFreeForm/ImageLabelBox.tsx
+++ b/src/client/views/collections/collectionFreeForm/ImageLabelBox.tsx
@@ -5,7 +5,7 @@ import { observer } from 'mobx-react';
 import React from 'react';
 import { Doc, NumListCast, Opt } from '../../../../fields/Doc';
 import { Docs } from '../../../documents/Documents';
-import { CollectionViewType, DocumentType } from '../../../documents/DocumentTypes';
+import { DocumentType } from '../../../documents/DocumentTypes';
 import { ViewBoxBaseComponent } from '../../DocComponent';
 import { FieldView, FieldViewProps } from '../../nodes/FieldView';
 import { MarqueeOptionsMenu } from './MarqueeOptionsMenu';
@@ -24,6 +24,9 @@ import { List } from '../../../../fields/List';
 import { DragManager } from '../../../util/DragManager';
 import { OpenWhere } from '../../nodes/OpenWhere';
 import similarity from 'compute-cosine-similarity';
+import { DocumentView } from '../../nodes/DocumentView';
+
+export class ImageInformationItem {}
 
 export class ImageLabelBoxData {
     static _instance: ImageLabelBoxData;
@@ -101,7 +104,6 @@ export class ImageLabelBox extends ViewBoxBaseComponent<FieldViewProps>() {
         makeObservable(this);
         ring.register();
         ImageLabelBox.Instance = this;
-        console.log('Image Box Has Been Initialized');
     }
 
     // ImageLabelBox.Instance.setData()
@@ -127,7 +129,6 @@ export class ImageLabelBox extends ViewBoxBaseComponent<FieldViewProps>() {
     @action
     startLoading = () => {
         this._loading = true;
-        console.log('Start loading has been called!');
     };
 
     @action
@@ -138,6 +139,11 @@ export class ImageLabelBox extends ViewBoxBaseComponent<FieldViewProps>() {
     @action
     toggleDisplayInformation = () => {
         this._displayImageInformation = !this._displayImageInformation;
+        if (this._displayImageInformation) {
+            this._selectedImages.forEach(doc => (doc[DocData].showLabels = true));
+        } else {
+            this._selectedImages.forEach(doc => (doc[DocData].showLabels = false));
+        }
     };
 
     @action
@@ -155,33 +161,58 @@ export class ImageLabelBox extends ViewBoxBaseComponent<FieldViewProps>() {
     classifyImagesInBox = async () => {
         this.startLoading();
 
+        // const imageInfos = this._selectedImages.map(async doc => {
+        //     if (!doc[DocData].data_labels) {
+        //         const [name, type] = ImageCast(doc[Doc.LayoutFieldKey(doc)]).url.href.split('.');
+        //         return CollectionCardView.imageUrlToBase64(`${name}_o.${type}`).then(hrefBase64 =>
+        //         !hrefBase64 ? undefined :
+        //             gptImageLabel(hrefBase64).then(labels =>
+        //                 Promise.all(labels.split('\n').map(label => gptGetEmbedding(label))).then(embeddings =>
+        //                     ({ doc, embeddings, labels }))) ); // prettier-ignore
+        //     }
+        // }); // Converts the images into a Base64 format, afterwhich the information is sent to GPT to label them.
+
         const imageInfos = this._selectedImages.map(async doc => {
             if (!doc[DocData].data_labels) {
                 const [name, type] = ImageCast(doc[Doc.LayoutFieldKey(doc)]).url.href.split('.');
                 return CollectionCardView.imageUrlToBase64(`${name}_o.${type}`).then(hrefBase64 =>
                 !hrefBase64 ? undefined : 
                     gptImageLabel(hrefBase64).then(labels => 
-                        Promise.all(labels.split('\n').map(label => gptGetEmbedding(label))).then(embeddings => 
-                            ({ doc, embeddings, labels }))) ); // prettier-ignore
+                
+                            ({ doc, labels }))) ; // prettier-ignore
             }
-        }); // Converts the images into a Base64 format, afterwhich the information is sent to GPT to label them.
+        });
 
         (await Promise.all(imageInfos)).forEach(imageInfo => {
-            if (imageInfo && imageInfo.embeddings && Array.isArray(imageInfo.embeddings)) {
-                imageInfo.doc[DocData].data_labels = imageInfo.labels;
+            if (imageInfo) {
+                imageInfo.doc[DocData].data_labels = new List<string>();
 
                 const labels = imageInfo.labels.split('\n');
                 labels.forEach(label => {
-                    label = label.replace(/^\d+\.\s*/, '').trim();
+                    label = label.replace(/^\d+\.\s*|-|\*/, '').trim();
                     imageInfo.doc[DocData][`${label}`] = true;
-                });
-
-                numberRange(3).forEach(n => {
-                    imageInfo.doc[`data_labels_embedding_${n + 1}`] = new List<number>(imageInfo.embeddings[n]);
+                    (imageInfo.doc[DocData].data_labels as List<string>).push(label);
                 });
             }
         }); // Add the labels as fields to each image.
 
+        // (await Promise.all(imageInfos)).forEach(imageInfo => {
+        //     if (imageInfo && imageInfo.embeddings && Array.isArray(imageInfo.embeddings)) {
+        //         imageInfo.doc[DocData].data_labels = new List<string>();
+
+        //         const labels = imageInfo.labels.split('\n');
+        //         labels.forEach(label => {
+        //             label = label.replace(/^\d+\.\s*|-|\*/, '').trim();
+        //             imageInfo.doc[DocData][`${label}`] = true;
+        //             (imageInfo.doc[DocData].data_labels as List<string>).push(label);
+        //         });
+
+        //         numberRange(5).forEach(n => {
+        //             imageInfo.doc[`data_labels_embedding_${n + 1}`] = new List<number>(imageInfo.embeddings[n]);
+        //         });
+        //     }
+        // }); // Add the labels as fields to each image.
+
         this.endLoading();
     };
 
@@ -189,7 +220,13 @@ export class ImageLabelBox extends ViewBoxBaseComponent<FieldViewProps>() {
      * Groups images to most similar labels.
      */
     groupImagesInBox = action(async () => {
-        console.log('Calling!');
+        this._selectedImages.forEach(doc => {
+            (doc[DocData].data_labels as List<string>).forEach(async (label, index) => {
+                const embedding = await gptGetEmbedding(label);
+                doc[`data_labels_embedding_${index + 1}`] = new List<number>(embedding);
+            });
+        });
+
         const labelToEmbedding = new Map<string, number[]>();
         // Create embeddings for the labels.
         await Promise.all(this._labelGroups.map(async label => gptGetEmbedding(label).then(labelEmbedding => labelToEmbedding.set(label, labelEmbedding))));
@@ -197,7 +234,7 @@ export class ImageLabelBox extends ViewBoxBaseComponent<FieldViewProps>() {
         // For each image, loop through the labels, and calculate similarity. Associate it with the
         // most similar one.
         this._selectedImages.forEach(doc => {
-            const embedLists = numberRange(3).map(n => Array.from(NumListCast(doc[`data_labels_embedding_${n + 1}`])));
+            const embedLists = numberRange(5).map(n => Array.from(NumListCast(doc[`data_labels_embedding_${n + 1}`])));
             const bestEmbedScore = (embedding: Opt<number[]>) => Math.max(...embedLists.map((l, index) => (embedding && (1 - index * 0.1) * similarity(Array.from(embedding), l)!) || 0));
             const {label: mostSimilarLabelCollect} = 
                 this._labelGroups.map(label => ({ label, similarityScore: bestEmbedScore(labelToEmbedding.get(label)) }))
@@ -293,10 +330,14 @@ export class ImageLabelBox extends ViewBoxBaseComponent<FieldViewProps>() {
                         {this._selectedImages.map(doc => {
                             const [name, type] = ImageCast(doc[Doc.LayoutFieldKey(doc)]).url.href.split('.');
                             return (
-                                <div className="image-information" style={{ borderColor: SettingsManager.userColor }} key={Utils.GenerateGuid()} onClick={() => this._props.addDocTab(doc, OpenWhere.addRightKeyvalue)}>
-                                    <img src={`${name}_o.${type}`}></img>
-                                    <div className="image-information-labels">
-                                        {(doc[DocData].data_labels as string).split('\n').map(label => {
+                                <div className="image-information" style={{ borderColor: SettingsManager.userColor }} key={Utils.GenerateGuid()}>
+                                    <img
+                                        src={`${name}_o.${type}`}
+                                        onClick={async () => {
+                                            await DocumentView.showDocument(doc, { willZoomCentered: true });
+                                        }}></img>
+                                    <div className="image-information-labels" onClick={() => this._props.addDocTab(doc, OpenWhere.addRightKeyvalue)}>
+                                        {(doc[DocData].data_labels as List<string>).map(label => {
                                             return (
                                                 <div key={Utils.GenerateGuid()} className="image-label" style={{ backgroundColor: SettingsManager.userVariantColor, borderColor: SettingsManager.userColor }}>
                                                     {label}