From 8aee62b8623e23f6478960291857ee47f50f9aaf Mon Sep 17 00:00:00 2001 From: IEatChili Date: Thu, 13 Jun 2024 16:28:24 -0400 Subject: feat: more ui updates --- .../collectionFreeForm/ImageLabelBox.tsx | 81 +++++++++++++++++++--- .../collections/collectionFreeForm/MarqueeView.tsx | 22 +----- src/client/views/nodes/ImageBox.tsx | 4 ++ 3 files changed, 76 insertions(+), 31 deletions(-) (limited to 'src') diff --git a/src/client/views/collections/collectionFreeForm/ImageLabelBox.tsx b/src/client/views/collections/collectionFreeForm/ImageLabelBox.tsx index f5530ccc4..571a4504f 100644 --- a/src/client/views/collections/collectionFreeForm/ImageLabelBox.tsx +++ b/src/client/views/collections/collectionFreeForm/ImageLabelBox.tsx @@ -3,9 +3,9 @@ import { Colors, IconButton } from 'browndash-components'; import { action, computed, makeObservable, observable, reaction } from 'mobx'; import { observer } from 'mobx-react'; import React from 'react'; -import { Doc } from '../../../../fields/Doc'; +import { Doc, NumListCast, Opt } from '../../../../fields/Doc'; import { Docs } from '../../../documents/Documents'; -import { DocumentType } from '../../../documents/DocumentTypes'; +import { CollectionViewType, DocumentType } from '../../../documents/DocumentTypes'; import { ViewBoxBaseComponent } from '../../DocComponent'; import { FieldView, FieldViewProps } from '../../nodes/FieldView'; import { MarqueeOptionsMenu } from './MarqueeOptionsMenu'; @@ -21,6 +21,9 @@ import { CollectionCardView } from '../CollectionCardDeckView'; import { gptGetEmbedding, gptImageLabel } from '../../../apis/gpt/GPT'; import { numberRange, Utils } from '../../../../Utils'; import { List } from '../../../../fields/List'; +import { DragManager } from '../../../util/DragManager'; +import { OpenWhere } from '../../nodes/OpenWhere'; +import similarity from 'compute-cosine-similarity'; export class ImageLabelBoxData { static _instance: ImageLabelBoxData; @@ -63,11 +66,26 @@ export class ImageLabelBox extends ViewBoxBaseComponent() { return FieldView.LayoutString(ImageLabelBox, fieldKey); } + private _dropDisposer?: DragManager.DragDropDisposer; public static Instance: ImageLabelBox; private _inputRef = React.createRef(); @observable _loading: boolean = false; private _currentLabel: string = ''; + protected createDropTarget = (ele: HTMLDivElement) => { + this._dropDisposer?.(); + ele && (this._dropDisposer = DragManager.MakeDropTarget(ele, this.onInternalDrop.bind(this), this.layoutDoc)); + }; + + protected onInternalDrop(e: Event, de: DragManager.DropEvent): boolean { + const { docDragData } = de.complete; + if (docDragData) { + ImageLabelBoxData.Instance.setData(ImageLabelBoxData.Instance._docs.concat(docDragData.droppedDocuments)); + return false; + } + return false; + } + @computed get _labelGroups() { return ImageLabelBoxData.Instance._labelGroups; } @@ -102,7 +120,7 @@ export class ImageLabelBox extends ViewBoxBaseComponent() { @action groupImages = () => { - MarqueeOptionsMenu.Instance.groupImages(); + this.groupImagesInBox(); MainView.Instance.closeFlyout(); }; @@ -122,6 +140,14 @@ export class ImageLabelBox extends ViewBoxBaseComponent() { this._displayImageInformation = !this._displayImageInformation; }; + @action + submitLabel = () => { + const input = document.getElementById('new-label') as HTMLInputElement; + ImageLabelBoxData.Instance.addLabel(this._currentLabel); + this._currentLabel = ''; + input.value = ''; + }; + onInputChange = action((e: React.ChangeEvent) => { this._currentLabel = e.target.value; }); @@ -143,6 +169,13 @@ export class ImageLabelBox extends ViewBoxBaseComponent() { (await Promise.all(imageInfos)).forEach(imageInfo => { if (imageInfo && imageInfo.embeddings && Array.isArray(imageInfo.embeddings)) { imageInfo.doc[DocData].data_labels = imageInfo.labels; + + const labels = imageInfo.labels.split('\n'); + labels.forEach(label => { + label = label.replace(/^\d+\.\s*/, '').trim(); + imageInfo.doc[DocData][`${label}`] = true; + }); + numberRange(3).forEach(n => { imageInfo.doc[`data_labels_embedding_${n + 1}`] = new List(imageInfo.embeddings[n]); }); @@ -152,6 +185,32 @@ export class ImageLabelBox extends ViewBoxBaseComponent() { this.endLoading(); }; + /** + * Groups images to most similar labels. + */ + groupImagesInBox = action(async () => { + console.log('Calling!'); + const labelToEmbedding = new Map(); + // Create embeddings for the labels. + await Promise.all(this._labelGroups.map(async label => gptGetEmbedding(label).then(labelEmbedding => labelToEmbedding.set(label, labelEmbedding)))); + + // For each image, loop through the labels, and calculate similarity. Associate it with the + // most similar one. + this._selectedImages.forEach(doc => { + const embedLists = numberRange(3).map(n => Array.from(NumListCast(doc[`data_labels_embedding_${n + 1}`]))); + const bestEmbedScore = (embedding: Opt) => Math.max(...embedLists.map((l, index) => (embedding && (1 - index * 0.1) * similarity(Array.from(embedding), l)!) || 0)); + const {label: mostSimilarLabelCollect} = + this._labelGroups.map(label => ({ label, similarityScore: bestEmbedScore(labelToEmbedding.get(label)) })) + .reduce((prev, cur) => cur.similarityScore < 0.3 || cur.similarityScore <= prev.similarityScore ? prev: cur, + { label: '', similarityScore: 0, }); // prettier-ignore + doc[DocData].data_label = mostSimilarLabelCollect; // The label most similar to the image's contents. + }); + + if (this._selectedImages) { + MarqueeOptionsMenu.Instance.groupImages(); + } + }); + render() { if (this._loading) { return ( @@ -163,14 +222,14 @@ export class ImageLabelBox extends ViewBoxBaseComponent() { if (this._selectedImages.length === 0) { return ( -
+
this.createDropTarget(ele)}>

In order to classify and sort images, marquee select the desired images and press the 'Classify and Sort Images' button. Then, add the desired groups for the images to be put in.

); } return ( -
+
this.createDropTarget(ele)}>
() { defaultValue="" autoComplete="off" onChange={this.onInputChange} - // onKeyDown={e => { - // e.key === 'Enter' ? this.submitSearch() : null; - // e.stopPropagation(); - // }} + onKeyDown={e => { + e.key === 'Enter' ? this.submitLabel() : null; + e.stopPropagation(); + }} type="text" - placeholder="Input labels for image groupings..." + placeholder="Input groups for images to be put into..." aria-label="label-input" id="new-label" className="searchBox-input" @@ -234,7 +293,7 @@ export class ImageLabelBox extends ViewBoxBaseComponent() { {this._selectedImages.map(doc => { const [name, type] = ImageCast(doc[Doc.LayoutFieldKey(doc)]).url.href.split('.'); return ( -
+
this._props.addDocTab(doc, OpenWhere.addRightKeyvalue)}>
{(doc[DocData].data_labels as string).split('\n').map(label => { diff --git a/src/client/views/collections/collectionFreeForm/MarqueeView.tsx b/src/client/views/collections/collectionFreeForm/MarqueeView.tsx index 81f2a94c1..f03a9d62d 100644 --- a/src/client/views/collections/collectionFreeForm/MarqueeView.tsx +++ b/src/client/views/collections/collectionFreeForm/MarqueeView.tsx @@ -455,26 +455,8 @@ export class MarqueeView extends ObservableReactComponent { - const labelGroups = ImageLabelBox.Instance!._labelGroups; - const labelToEmbedding = new Map(); - // Create embeddings for the labels. - await Promise.all(labelGroups.map(async label => gptGetEmbedding(label).then(labelEmbedding => labelToEmbedding.set(label, labelEmbedding)))); - - // For each image, loop through the labels, and calculate similarity. Associate it with the - // most similar one. - this._selectedDocs.forEach(doc => { - const embedLists = numberRange(3).map(n => Array.from(NumListCast(doc[`data_labels_embedding_${n + 1}`]))); - const bestEmbedScore = (embedding: Opt) => Math.max(...embedLists.map((l, index) => (embedding && (1 - index * 0.1) * similarity(Array.from(embedding), l)!) || 0)); - const {label: mostSimilarLabelCollect} = - labelGroups.map(label => ({ label, similarityScore: bestEmbedScore(labelToEmbedding.get(label)) })) - .reduce((prev, cur) => cur.similarityScore < 0.3 || cur.similarityScore <= prev.similarityScore ? prev: cur, - { label: '', similarityScore: 0, }); // prettier-ignore - doc[DocData].data_label = mostSimilarLabelCollect; // The label most similar to the image's contents. - }); - if (this._selectedDocs) { - this._props.Document._type_collection = CollectionViewType.Time; // Change the collection view to a Time view. - this._props.Document.pivotField = 'data_label'; // Sets the pivot to be the 'data_label'. - } + this._props.Document._type_collection = CollectionViewType.Time; // Change the collection view to a Time view. + this._props.Document.pivotField = 'data_label'; // Sets the pivot to be the 'data_label'. }); @undoBatch diff --git a/src/client/views/nodes/ImageBox.tsx b/src/client/views/nodes/ImageBox.tsx index e4b3a1b9b..3da878a4f 100644 --- a/src/client/views/nodes/ImageBox.tsx +++ b/src/client/views/nodes/ImageBox.tsx @@ -173,6 +173,10 @@ export class ImageBox extends ViewBoxAnnotatableComponent() { Doc.SetNativeHeight(this.dataDoc, Doc.NativeHeight(targetDoc), this.fieldKey); } } + const layoutDoc = de.complete.docDragData?.draggedDocuments[0]; + const targetField = Doc.LayoutFieldKey(layoutDoc); + const targetDoc = layoutDoc[DocData]; + console.log(targetDoc[targetField]); added === false && e.preventDefault(); added !== undefined && e.stopPropagation(); return added; -- cgit v1.2.3-70-g09d2