From 8aee62b8623e23f6478960291857ee47f50f9aaf Mon Sep 17 00:00:00 2001
From: IEatChili <nanunguyen99@gmail.com>
Date: Thu, 13 Jun 2024 16:28:24 -0400
Subject: feat: more ui updates

---
 .../collectionFreeForm/ImageLabelBox.tsx           | 81 +++++++++++++++++++---
 1 file changed, 70 insertions(+), 11 deletions(-)

(limited to 'src/client/views/collections/collectionFreeForm/ImageLabelBox.tsx')

diff --git a/src/client/views/collections/collectionFreeForm/ImageLabelBox.tsx b/src/client/views/collections/collectionFreeForm/ImageLabelBox.tsx
index f5530ccc4..571a4504f 100644
--- a/src/client/views/collections/collectionFreeForm/ImageLabelBox.tsx
+++ b/src/client/views/collections/collectionFreeForm/ImageLabelBox.tsx
@@ -3,9 +3,9 @@ import { Colors, IconButton } from 'browndash-components';
 import { action, computed, makeObservable, observable, reaction } from 'mobx';
 import { observer } from 'mobx-react';
 import React from 'react';
-import { Doc } from '../../../../fields/Doc';
+import { Doc, NumListCast, Opt } from '../../../../fields/Doc';
 import { Docs } from '../../../documents/Documents';
-import { DocumentType } from '../../../documents/DocumentTypes';
+import { CollectionViewType, DocumentType } from '../../../documents/DocumentTypes';
 import { ViewBoxBaseComponent } from '../../DocComponent';
 import { FieldView, FieldViewProps } from '../../nodes/FieldView';
 import { MarqueeOptionsMenu } from './MarqueeOptionsMenu';
@@ -21,6 +21,9 @@ import { CollectionCardView } from '../CollectionCardDeckView';
 import { gptGetEmbedding, gptImageLabel } from '../../../apis/gpt/GPT';
 import { numberRange, Utils } from '../../../../Utils';
 import { List } from '../../../../fields/List';
+import { DragManager } from '../../../util/DragManager';
+import { OpenWhere } from '../../nodes/OpenWhere';
+import similarity from 'compute-cosine-similarity';
 
 export class ImageLabelBoxData {
     static _instance: ImageLabelBoxData;
@@ -63,11 +66,26 @@ export class ImageLabelBox extends ViewBoxBaseComponent<FieldViewProps>() {
         return FieldView.LayoutString(ImageLabelBox, fieldKey);
     }
 
+    private _dropDisposer?: DragManager.DragDropDisposer;
     public static Instance: ImageLabelBox;
     private _inputRef = React.createRef<HTMLInputElement>();
     @observable _loading: boolean = false;
     private _currentLabel: string = '';
 
+    protected createDropTarget = (ele: HTMLDivElement) => {
+        this._dropDisposer?.();
+        ele && (this._dropDisposer = DragManager.MakeDropTarget(ele, this.onInternalDrop.bind(this), this.layoutDoc));
+    };
+
+    protected onInternalDrop(e: Event, de: DragManager.DropEvent): boolean {
+        const { docDragData } = de.complete;
+        if (docDragData) {
+            ImageLabelBoxData.Instance.setData(ImageLabelBoxData.Instance._docs.concat(docDragData.droppedDocuments));
+            return false;
+        }
+        return false;
+    }
+
     @computed get _labelGroups() {
         return ImageLabelBoxData.Instance._labelGroups;
     }
@@ -102,7 +120,7 @@ export class ImageLabelBox extends ViewBoxBaseComponent<FieldViewProps>() {
 
     @action
     groupImages = () => {
-        MarqueeOptionsMenu.Instance.groupImages();
+        this.groupImagesInBox();
         MainView.Instance.closeFlyout();
     };
 
@@ -122,6 +140,14 @@ export class ImageLabelBox extends ViewBoxBaseComponent<FieldViewProps>() {
         this._displayImageInformation = !this._displayImageInformation;
     };
 
+    @action
+    submitLabel = () => {
+        const input = document.getElementById('new-label') as HTMLInputElement;
+        ImageLabelBoxData.Instance.addLabel(this._currentLabel);
+        this._currentLabel = '';
+        input.value = '';
+    };
+
     onInputChange = action((e: React.ChangeEvent<HTMLInputElement>) => {
         this._currentLabel = e.target.value;
     });
@@ -143,6 +169,13 @@ export class ImageLabelBox extends ViewBoxBaseComponent<FieldViewProps>() {
         (await Promise.all(imageInfos)).forEach(imageInfo => {
             if (imageInfo && imageInfo.embeddings && Array.isArray(imageInfo.embeddings)) {
                 imageInfo.doc[DocData].data_labels = imageInfo.labels;
+
+                const labels = imageInfo.labels.split('\n');
+                labels.forEach(label => {
+                    label = label.replace(/^\d+\.\s*/, '').trim();
+                    imageInfo.doc[DocData][`${label}`] = true;
+                });
+
                 numberRange(3).forEach(n => {
                     imageInfo.doc[`data_labels_embedding_${n + 1}`] = new List<number>(imageInfo.embeddings[n]);
                 });
@@ -152,6 +185,32 @@ export class ImageLabelBox extends ViewBoxBaseComponent<FieldViewProps>() {
         this.endLoading();
     };
 
+    /**
+     * Groups images to most similar labels.
+     */
+    groupImagesInBox = action(async () => {
+        console.log('Calling!');
+        const labelToEmbedding = new Map<string, number[]>();
+        // Create embeddings for the labels.
+        await Promise.all(this._labelGroups.map(async label => gptGetEmbedding(label).then(labelEmbedding => labelToEmbedding.set(label, labelEmbedding))));
+
+        // For each image, loop through the labels, and calculate similarity. Associate it with the
+        // most similar one.
+        this._selectedImages.forEach(doc => {
+            const embedLists = numberRange(3).map(n => Array.from(NumListCast(doc[`data_labels_embedding_${n + 1}`])));
+            const bestEmbedScore = (embedding: Opt<number[]>) => Math.max(...embedLists.map((l, index) => (embedding && (1 - index * 0.1) * similarity(Array.from(embedding), l)!) || 0));
+            const {label: mostSimilarLabelCollect} = 
+                this._labelGroups.map(label => ({ label, similarityScore: bestEmbedScore(labelToEmbedding.get(label)) }))
+                           .reduce((prev, cur) => cur.similarityScore < 0.3 || cur.similarityScore <= prev.similarityScore ? prev: cur,
+                                   { label: '', similarityScore: 0, }); // prettier-ignore
+            doc[DocData].data_label = mostSimilarLabelCollect; // The label most similar to the image's contents.
+        });
+
+        if (this._selectedImages) {
+            MarqueeOptionsMenu.Instance.groupImages();
+        }
+    });
+
     render() {
         if (this._loading) {
             return (
@@ -163,14 +222,14 @@ export class ImageLabelBox extends ViewBoxBaseComponent<FieldViewProps>() {
 
         if (this._selectedImages.length === 0) {
             return (
-                <div className="searchBox-container" style={{ pointerEvents: 'all', color: SnappingManager.userColor, background: SnappingManager.userBackgroundColor }}>
+                <div className="searchBox-container" style={{ pointerEvents: 'all', color: SnappingManager.userColor, background: SnappingManager.userBackgroundColor }} ref={ele => this.createDropTarget(ele)}>
                     <p style={{ fontSize: 'large' }}>In order to classify and sort images, marquee select the desired images and press the 'Classify and Sort Images' button. Then, add the desired groups for the images to be put in.</p>
                 </div>
             );
         }
 
         return (
-            <div className="searchBox-container" style={{ pointerEvents: 'all', color: SnappingManager.userColor, background: SnappingManager.userBackgroundColor }}>
+            <div className="searchBox-container" style={{ pointerEvents: 'all', color: SnappingManager.userColor, background: SnappingManager.userBackgroundColor }} ref={ele => this.createDropTarget(ele)}>
                 <div className="searchBox-bar" style={{ pointerEvents: 'all', color: SnappingManager.userColor, background: SnappingManager.userBackgroundColor }}>
                     <IconButton
                         tooltip={'See image information'}
@@ -183,12 +242,12 @@ export class ImageLabelBox extends ViewBoxBaseComponent<FieldViewProps>() {
                         defaultValue=""
                         autoComplete="off"
                         onChange={this.onInputChange}
-                        // onKeyDown={e => {
-                        //     e.key === 'Enter' ? this.submitSearch() : null;
-                        //     e.stopPropagation();
-                        // }}
+                        onKeyDown={e => {
+                            e.key === 'Enter' ? this.submitLabel() : null;
+                            e.stopPropagation();
+                        }}
                         type="text"
-                        placeholder="Input labels for image groupings..."
+                        placeholder="Input groups for images to be put into..."
                         aria-label="label-input"
                         id="new-label"
                         className="searchBox-input"
@@ -234,7 +293,7 @@ export class ImageLabelBox extends ViewBoxBaseComponent<FieldViewProps>() {
                         {this._selectedImages.map(doc => {
                             const [name, type] = ImageCast(doc[Doc.LayoutFieldKey(doc)]).url.href.split('.');
                             return (
-                                <div className="image-information" style={{ borderColor: SettingsManager.userColor }} key={Utils.GenerateGuid()}>
+                                <div className="image-information" style={{ borderColor: SettingsManager.userColor }} key={Utils.GenerateGuid()} onClick={() => this._props.addDocTab(doc, OpenWhere.addRightKeyvalue)}>
                                     <img src={`${name}_o.${type}`}></img>
                                     <div className="image-information-labels">
                                         {(doc[DocData].data_labels as string).split('\n').map(label => {
-- 
cgit v1.2.3-70-g09d2

