diff options
3 files changed, 76 insertions, 31 deletions
diff --git a/src/client/views/collections/collectionFreeForm/ImageLabelBox.tsx b/src/client/views/collections/collectionFreeForm/ImageLabelBox.tsx index f5530ccc4..571a4504f 100644 --- a/src/client/views/collections/collectionFreeForm/ImageLabelBox.tsx +++ b/src/client/views/collections/collectionFreeForm/ImageLabelBox.tsx @@ -3,9 +3,9 @@ import { Colors, IconButton } from 'browndash-components'; import { action, computed, makeObservable, observable, reaction } from 'mobx'; import { observer } from 'mobx-react'; import React from 'react'; -import { Doc } from '../../../../fields/Doc'; +import { Doc, NumListCast, Opt } from '../../../../fields/Doc'; import { Docs } from '../../../documents/Documents'; -import { DocumentType } from '../../../documents/DocumentTypes'; +import { CollectionViewType, DocumentType } from '../../../documents/DocumentTypes'; import { ViewBoxBaseComponent } from '../../DocComponent'; import { FieldView, FieldViewProps } from '../../nodes/FieldView'; import { MarqueeOptionsMenu } from './MarqueeOptionsMenu'; @@ -21,6 +21,9 @@ import { CollectionCardView } from '../CollectionCardDeckView'; import { gptGetEmbedding, gptImageLabel } from '../../../apis/gpt/GPT'; import { numberRange, Utils } from '../../../../Utils'; import { List } from '../../../../fields/List'; +import { DragManager } from '../../../util/DragManager'; +import { OpenWhere } from '../../nodes/OpenWhere'; +import similarity from 'compute-cosine-similarity'; export class ImageLabelBoxData { static _instance: ImageLabelBoxData; @@ -63,11 +66,26 @@ export class ImageLabelBox extends ViewBoxBaseComponent<FieldViewProps>() { return FieldView.LayoutString(ImageLabelBox, fieldKey); } + private _dropDisposer?: DragManager.DragDropDisposer; public static Instance: ImageLabelBox; private _inputRef = React.createRef<HTMLInputElement>(); @observable _loading: boolean = false; private _currentLabel: string = ''; + protected createDropTarget = (ele: HTMLDivElement) => { + this._dropDisposer?.(); + ele && (this._dropDisposer = DragManager.MakeDropTarget(ele, this.onInternalDrop.bind(this), this.layoutDoc)); + }; + + protected onInternalDrop(e: Event, de: DragManager.DropEvent): boolean { + const { docDragData } = de.complete; + if (docDragData) { + ImageLabelBoxData.Instance.setData(ImageLabelBoxData.Instance._docs.concat(docDragData.droppedDocuments)); + return false; + } + return false; + } + @computed get _labelGroups() { return ImageLabelBoxData.Instance._labelGroups; } @@ -102,7 +120,7 @@ export class ImageLabelBox extends ViewBoxBaseComponent<FieldViewProps>() { @action groupImages = () => { - MarqueeOptionsMenu.Instance.groupImages(); + this.groupImagesInBox(); MainView.Instance.closeFlyout(); }; @@ -122,6 +140,14 @@ export class ImageLabelBox extends ViewBoxBaseComponent<FieldViewProps>() { this._displayImageInformation = !this._displayImageInformation; }; + @action + submitLabel = () => { + const input = document.getElementById('new-label') as HTMLInputElement; + ImageLabelBoxData.Instance.addLabel(this._currentLabel); + this._currentLabel = ''; + input.value = ''; + }; + onInputChange = action((e: React.ChangeEvent<HTMLInputElement>) => { this._currentLabel = e.target.value; }); @@ -143,6 +169,13 @@ export class ImageLabelBox extends ViewBoxBaseComponent<FieldViewProps>() { (await Promise.all(imageInfos)).forEach(imageInfo => { if (imageInfo && imageInfo.embeddings && Array.isArray(imageInfo.embeddings)) { imageInfo.doc[DocData].data_labels = imageInfo.labels; + + const labels = imageInfo.labels.split('\n'); + labels.forEach(label => { + label = label.replace(/^\d+\.\s*/, '').trim(); + imageInfo.doc[DocData][`${label}`] = true; + }); + numberRange(3).forEach(n => { imageInfo.doc[`data_labels_embedding_${n + 1}`] = new List<number>(imageInfo.embeddings[n]); }); @@ -152,6 +185,32 @@ export class ImageLabelBox extends ViewBoxBaseComponent<FieldViewProps>() { this.endLoading(); }; + /** + * Groups images to most similar labels. + */ + groupImagesInBox = action(async () => { + console.log('Calling!'); + const labelToEmbedding = new Map<string, number[]>(); + // Create embeddings for the labels. + await Promise.all(this._labelGroups.map(async label => gptGetEmbedding(label).then(labelEmbedding => labelToEmbedding.set(label, labelEmbedding)))); + + // For each image, loop through the labels, and calculate similarity. Associate it with the + // most similar one. + this._selectedImages.forEach(doc => { + const embedLists = numberRange(3).map(n => Array.from(NumListCast(doc[`data_labels_embedding_${n + 1}`]))); + const bestEmbedScore = (embedding: Opt<number[]>) => Math.max(...embedLists.map((l, index) => (embedding && (1 - index * 0.1) * similarity(Array.from(embedding), l)!) || 0)); + const {label: mostSimilarLabelCollect} = + this._labelGroups.map(label => ({ label, similarityScore: bestEmbedScore(labelToEmbedding.get(label)) })) + .reduce((prev, cur) => cur.similarityScore < 0.3 || cur.similarityScore <= prev.similarityScore ? prev: cur, + { label: '', similarityScore: 0, }); // prettier-ignore + doc[DocData].data_label = mostSimilarLabelCollect; // The label most similar to the image's contents. + }); + + if (this._selectedImages) { + MarqueeOptionsMenu.Instance.groupImages(); + } + }); + render() { if (this._loading) { return ( @@ -163,14 +222,14 @@ export class ImageLabelBox extends ViewBoxBaseComponent<FieldViewProps>() { if (this._selectedImages.length === 0) { return ( - <div className="searchBox-container" style={{ pointerEvents: 'all', color: SnappingManager.userColor, background: SnappingManager.userBackgroundColor }}> + <div className="searchBox-container" style={{ pointerEvents: 'all', color: SnappingManager.userColor, background: SnappingManager.userBackgroundColor }} ref={ele => this.createDropTarget(ele)}> <p style={{ fontSize: 'large' }}>In order to classify and sort images, marquee select the desired images and press the 'Classify and Sort Images' button. Then, add the desired groups for the images to be put in.</p> </div> ); } return ( - <div className="searchBox-container" style={{ pointerEvents: 'all', color: SnappingManager.userColor, background: SnappingManager.userBackgroundColor }}> + <div className="searchBox-container" style={{ pointerEvents: 'all', color: SnappingManager.userColor, background: SnappingManager.userBackgroundColor }} ref={ele => this.createDropTarget(ele)}> <div className="searchBox-bar" style={{ pointerEvents: 'all', color: SnappingManager.userColor, background: SnappingManager.userBackgroundColor }}> <IconButton tooltip={'See image information'} @@ -183,12 +242,12 @@ export class ImageLabelBox extends ViewBoxBaseComponent<FieldViewProps>() { defaultValue="" autoComplete="off" onChange={this.onInputChange} - // onKeyDown={e => { - // e.key === 'Enter' ? this.submitSearch() : null; - // e.stopPropagation(); - // }} + onKeyDown={e => { + e.key === 'Enter' ? this.submitLabel() : null; + e.stopPropagation(); + }} type="text" - placeholder="Input labels for image groupings..." + placeholder="Input groups for images to be put into..." aria-label="label-input" id="new-label" className="searchBox-input" @@ -234,7 +293,7 @@ export class ImageLabelBox extends ViewBoxBaseComponent<FieldViewProps>() { {this._selectedImages.map(doc => { const [name, type] = ImageCast(doc[Doc.LayoutFieldKey(doc)]).url.href.split('.'); return ( - <div className="image-information" style={{ borderColor: SettingsManager.userColor }} key={Utils.GenerateGuid()}> + <div className="image-information" style={{ borderColor: SettingsManager.userColor }} key={Utils.GenerateGuid()} onClick={() => this._props.addDocTab(doc, OpenWhere.addRightKeyvalue)}> <img src={`${name}_o.${type}`}></img> <div className="image-information-labels"> {(doc[DocData].data_labels as string).split('\n').map(label => { diff --git a/src/client/views/collections/collectionFreeForm/MarqueeView.tsx b/src/client/views/collections/collectionFreeForm/MarqueeView.tsx index 81f2a94c1..f03a9d62d 100644 --- a/src/client/views/collections/collectionFreeForm/MarqueeView.tsx +++ b/src/client/views/collections/collectionFreeForm/MarqueeView.tsx @@ -455,26 +455,8 @@ export class MarqueeView extends ObservableReactComponent<SubCollectionViewProps */ @undoBatch groupImages = action(async () => { - const labelGroups = ImageLabelBox.Instance!._labelGroups; - const labelToEmbedding = new Map<string, number[]>(); - // Create embeddings for the labels. - await Promise.all(labelGroups.map(async label => gptGetEmbedding(label).then(labelEmbedding => labelToEmbedding.set(label, labelEmbedding)))); - - // For each image, loop through the labels, and calculate similarity. Associate it with the - // most similar one. - this._selectedDocs.forEach(doc => { - const embedLists = numberRange(3).map(n => Array.from(NumListCast(doc[`data_labels_embedding_${n + 1}`]))); - const bestEmbedScore = (embedding: Opt<number[]>) => Math.max(...embedLists.map((l, index) => (embedding && (1 - index * 0.1) * similarity(Array.from(embedding), l)!) || 0)); - const {label: mostSimilarLabelCollect} = - labelGroups.map(label => ({ label, similarityScore: bestEmbedScore(labelToEmbedding.get(label)) })) - .reduce((prev, cur) => cur.similarityScore < 0.3 || cur.similarityScore <= prev.similarityScore ? prev: cur, - { label: '', similarityScore: 0, }); // prettier-ignore - doc[DocData].data_label = mostSimilarLabelCollect; // The label most similar to the image's contents. - }); - if (this._selectedDocs) { - this._props.Document._type_collection = CollectionViewType.Time; // Change the collection view to a Time view. - this._props.Document.pivotField = 'data_label'; // Sets the pivot to be the 'data_label'. - } + this._props.Document._type_collection = CollectionViewType.Time; // Change the collection view to a Time view. + this._props.Document.pivotField = 'data_label'; // Sets the pivot to be the 'data_label'. }); @undoBatch diff --git a/src/client/views/nodes/ImageBox.tsx b/src/client/views/nodes/ImageBox.tsx index e4b3a1b9b..3da878a4f 100644 --- a/src/client/views/nodes/ImageBox.tsx +++ b/src/client/views/nodes/ImageBox.tsx @@ -173,6 +173,10 @@ export class ImageBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { Doc.SetNativeHeight(this.dataDoc, Doc.NativeHeight(targetDoc), this.fieldKey); } } + const layoutDoc = de.complete.docDragData?.draggedDocuments[0]; + const targetField = Doc.LayoutFieldKey(layoutDoc); + const targetDoc = layoutDoc[DocData]; + console.log(targetDoc[targetField]); added === false && e.preventDefault(); added !== undefined && e.stopPropagation(); return added; |