From 8aee62b8623e23f6478960291857ee47f50f9aaf Mon Sep 17 00:00:00 2001 From: IEatChili Date: Thu, 13 Jun 2024 16:28:24 -0400 Subject: feat: more ui updates --- src/client/views/nodes/ImageBox.tsx | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'src/client/views/nodes/ImageBox.tsx') diff --git a/src/client/views/nodes/ImageBox.tsx b/src/client/views/nodes/ImageBox.tsx index e4b3a1b9b..3da878a4f 100644 --- a/src/client/views/nodes/ImageBox.tsx +++ b/src/client/views/nodes/ImageBox.tsx @@ -173,6 +173,10 @@ export class ImageBox extends ViewBoxAnnotatableComponent() { Doc.SetNativeHeight(this.dataDoc, Doc.NativeHeight(targetDoc), this.fieldKey); } } + const layoutDoc = de.complete.docDragData?.draggedDocuments[0]; + const targetField = Doc.LayoutFieldKey(layoutDoc); + const targetDoc = layoutDoc[DocData]; + console.log(targetDoc[targetField]); added === false && e.preventDefault(); added !== undefined && e.stopPropagation(); return added; -- cgit v1.2.3-70-g09d2 From 376ff1626b24cbac12b27ad072690424549f05c7 Mon Sep 17 00:00:00 2001 From: IEatChili Date: Tue, 18 Jun 2024 14:33:47 -0400 Subject: feat: added view of labels on docs in freeform --- src/client/views/KeywordBox.tsx | 69 +++++++++++++++++++ src/client/views/StyleProvider.scss | 13 ++++ src/client/views/StyleProvider.tsx | 9 +++ .../collectionFreeForm/ImageLabelBox.tsx | 79 ++++++++++++++++------ src/client/views/nodes/DocumentView.tsx | 2 +- src/client/views/nodes/ImageBox.tsx | 9 +++ 6 files changed, 161 insertions(+), 20 deletions(-) create mode 100644 src/client/views/KeywordBox.tsx (limited to 'src/client/views/nodes/ImageBox.tsx') diff --git a/src/client/views/KeywordBox.tsx b/src/client/views/KeywordBox.tsx new file mode 100644 index 000000000..3faddeb64 --- /dev/null +++ b/src/client/views/KeywordBox.tsx @@ -0,0 +1,69 @@ +import { action, makeObservable } from 'mobx'; +import { observer } from 'mobx-react'; +import React from 'react'; +import { Doc } from '../../fields/Doc'; +import { DocData } from '../../fields/DocSymbols'; +import { List } from '../../fields/List'; +import { ObservableReactComponent } from './ObservableReactComponent'; + +interface KeywordBoxProps { + _doc: Doc; + _isEditing: boolean; +} + +@observer +export class KeywordBox extends ObservableReactComponent { + constructor(props: any) { + super(props); + makeObservable(this); + } + + @action + setToEditing = () => { + this._props._isEditing = true; + }; + + @action + setToView = () => { + this._props._isEditing = false; + }; + + submitLabel = () => {}; + + onInputChange = () => {}; + + render() { + const keywordsList = this._props._doc![DocData].data_labels; + return ( +
+ {(keywordsList as List).map(label => { + return ( +
+ {label} +
+ ); + })} + {this._props._isEditing ? ( +
+ { + e.key === 'Enter' ? this.submitLabel() : null; + e.stopPropagation(); + }} + type="text" + placeholder="Input keywords for document..." + aria-label="keyword-input" + className="keyword-input" + style={{ width: '100%', borderRadius: '5px' }} + /> +
+ ) : ( +
+ )} +
+ ); + } +} diff --git a/src/client/views/StyleProvider.scss b/src/client/views/StyleProvider.scss index ce00f6101..1e2af9a3a 100644 --- a/src/client/views/StyleProvider.scss +++ b/src/client/views/StyleProvider.scss @@ -53,3 +53,16 @@ .styleProvider-treeView-icon { opacity: 0; } + +.keywords-container { + display: flex; + flex-wrap: wrap; +} + +.keyword { + padding: 5px 10px; + background-color: lightblue; + border: 1px solid black; + border-radius: 5px; + white-space: nowrap; +} diff --git a/src/client/views/StyleProvider.tsx b/src/client/views/StyleProvider.tsx index b7f8a3170..fb509516a 100644 --- a/src/client/views/StyleProvider.tsx +++ b/src/client/views/StyleProvider.tsx @@ -12,7 +12,9 @@ import { BsArrowDown, BsArrowDownUp, BsArrowUp } from 'react-icons/bs'; import { FaFilter } from 'react-icons/fa'; import { ClientUtils, DashColor, lightOrDark } from '../../ClientUtils'; import { Doc, Opt, StrListCast } from '../../fields/Doc'; +import { DocData } from '../../fields/DocSymbols'; import { Id } from '../../fields/FieldSymbols'; +import { List } from '../../fields/List'; import { ScriptField } from '../../fields/ScriptField'; import { BoolCast, Cast, DocCast, ImageCast, NumCast, ScriptCast, StrCast } from '../../fields/Types'; import { AudioAnnoState } from '../../server/SharedMediaTypes'; @@ -23,6 +25,7 @@ import { SnappingManager } from '../util/SnappingManager'; import { undoBatch, UndoManager } from '../util/UndoManager'; import { TreeSort } from './collections/TreeSort'; import { Colors } from './global/globalEnums'; +import { KeywordBox } from './KeywordBox'; import { DocumentView, DocumentViewProps } from './nodes/DocumentView'; import { FieldViewProps } from './nodes/FieldView'; import { StyleProp } from './StyleProp'; @@ -367,12 +370,18 @@ export function DefaultStyleProvider(doc: Opt, props: Opt ); }; + const keywords = () => { + if (doc && doc![DocData].data_labels && doc![DocData].showLabels) { + return () + } + } return ( <> {paint()} {lock()} {filter()} {audio()} + {keywords()} ); } diff --git a/src/client/views/collections/collectionFreeForm/ImageLabelBox.tsx b/src/client/views/collections/collectionFreeForm/ImageLabelBox.tsx index 571a4504f..cfb81e1a0 100644 --- a/src/client/views/collections/collectionFreeForm/ImageLabelBox.tsx +++ b/src/client/views/collections/collectionFreeForm/ImageLabelBox.tsx @@ -5,7 +5,7 @@ import { observer } from 'mobx-react'; import React from 'react'; import { Doc, NumListCast, Opt } from '../../../../fields/Doc'; import { Docs } from '../../../documents/Documents'; -import { CollectionViewType, DocumentType } from '../../../documents/DocumentTypes'; +import { DocumentType } from '../../../documents/DocumentTypes'; import { ViewBoxBaseComponent } from '../../DocComponent'; import { FieldView, FieldViewProps } from '../../nodes/FieldView'; import { MarqueeOptionsMenu } from './MarqueeOptionsMenu'; @@ -24,6 +24,9 @@ import { List } from '../../../../fields/List'; import { DragManager } from '../../../util/DragManager'; import { OpenWhere } from '../../nodes/OpenWhere'; import similarity from 'compute-cosine-similarity'; +import { DocumentView } from '../../nodes/DocumentView'; + +export class ImageInformationItem {} export class ImageLabelBoxData { static _instance: ImageLabelBoxData; @@ -101,7 +104,6 @@ export class ImageLabelBox extends ViewBoxBaseComponent() { makeObservable(this); ring.register(); ImageLabelBox.Instance = this; - console.log('Image Box Has Been Initialized'); } // ImageLabelBox.Instance.setData() @@ -127,7 +129,6 @@ export class ImageLabelBox extends ViewBoxBaseComponent() { @action startLoading = () => { this._loading = true; - console.log('Start loading has been called!'); }; @action @@ -138,6 +139,11 @@ export class ImageLabelBox extends ViewBoxBaseComponent() { @action toggleDisplayInformation = () => { this._displayImageInformation = !this._displayImageInformation; + if (this._displayImageInformation) { + this._selectedImages.forEach(doc => (doc[DocData].showLabels = true)); + } else { + this._selectedImages.forEach(doc => (doc[DocData].showLabels = false)); + } }; @action @@ -155,33 +161,58 @@ export class ImageLabelBox extends ViewBoxBaseComponent() { classifyImagesInBox = async () => { this.startLoading(); + // const imageInfos = this._selectedImages.map(async doc => { + // if (!doc[DocData].data_labels) { + // const [name, type] = ImageCast(doc[Doc.LayoutFieldKey(doc)]).url.href.split('.'); + // return CollectionCardView.imageUrlToBase64(`${name}_o.${type}`).then(hrefBase64 => + // !hrefBase64 ? undefined : + // gptImageLabel(hrefBase64).then(labels => + // Promise.all(labels.split('\n').map(label => gptGetEmbedding(label))).then(embeddings => + // ({ doc, embeddings, labels }))) ); // prettier-ignore + // } + // }); // Converts the images into a Base64 format, afterwhich the information is sent to GPT to label them. + const imageInfos = this._selectedImages.map(async doc => { if (!doc[DocData].data_labels) { const [name, type] = ImageCast(doc[Doc.LayoutFieldKey(doc)]).url.href.split('.'); return CollectionCardView.imageUrlToBase64(`${name}_o.${type}`).then(hrefBase64 => !hrefBase64 ? undefined : gptImageLabel(hrefBase64).then(labels => - Promise.all(labels.split('\n').map(label => gptGetEmbedding(label))).then(embeddings => - ({ doc, embeddings, labels }))) ); // prettier-ignore + + ({ doc, labels }))) ; // prettier-ignore } - }); // Converts the images into a Base64 format, afterwhich the information is sent to GPT to label them. + }); (await Promise.all(imageInfos)).forEach(imageInfo => { - if (imageInfo && imageInfo.embeddings && Array.isArray(imageInfo.embeddings)) { - imageInfo.doc[DocData].data_labels = imageInfo.labels; + if (imageInfo) { + imageInfo.doc[DocData].data_labels = new List(); const labels = imageInfo.labels.split('\n'); labels.forEach(label => { - label = label.replace(/^\d+\.\s*/, '').trim(); + label = label.replace(/^\d+\.\s*|-|\*/, '').trim(); imageInfo.doc[DocData][`${label}`] = true; - }); - - numberRange(3).forEach(n => { - imageInfo.doc[`data_labels_embedding_${n + 1}`] = new List(imageInfo.embeddings[n]); + (imageInfo.doc[DocData].data_labels as List).push(label); }); } }); // Add the labels as fields to each image. + // (await Promise.all(imageInfos)).forEach(imageInfo => { + // if (imageInfo && imageInfo.embeddings && Array.isArray(imageInfo.embeddings)) { + // imageInfo.doc[DocData].data_labels = new List(); + + // const labels = imageInfo.labels.split('\n'); + // labels.forEach(label => { + // label = label.replace(/^\d+\.\s*|-|\*/, '').trim(); + // imageInfo.doc[DocData][`${label}`] = true; + // (imageInfo.doc[DocData].data_labels as List).push(label); + // }); + + // numberRange(5).forEach(n => { + // imageInfo.doc[`data_labels_embedding_${n + 1}`] = new List(imageInfo.embeddings[n]); + // }); + // } + // }); // Add the labels as fields to each image. + this.endLoading(); }; @@ -189,7 +220,13 @@ export class ImageLabelBox extends ViewBoxBaseComponent() { * Groups images to most similar labels. */ groupImagesInBox = action(async () => { - console.log('Calling!'); + this._selectedImages.forEach(doc => { + (doc[DocData].data_labels as List).forEach(async (label, index) => { + const embedding = await gptGetEmbedding(label); + doc[`data_labels_embedding_${index + 1}`] = new List(embedding); + }); + }); + const labelToEmbedding = new Map(); // Create embeddings for the labels. await Promise.all(this._labelGroups.map(async label => gptGetEmbedding(label).then(labelEmbedding => labelToEmbedding.set(label, labelEmbedding)))); @@ -197,7 +234,7 @@ export class ImageLabelBox extends ViewBoxBaseComponent() { // For each image, loop through the labels, and calculate similarity. Associate it with the // most similar one. this._selectedImages.forEach(doc => { - const embedLists = numberRange(3).map(n => Array.from(NumListCast(doc[`data_labels_embedding_${n + 1}`]))); + const embedLists = numberRange(5).map(n => Array.from(NumListCast(doc[`data_labels_embedding_${n + 1}`]))); const bestEmbedScore = (embedding: Opt) => Math.max(...embedLists.map((l, index) => (embedding && (1 - index * 0.1) * similarity(Array.from(embedding), l)!) || 0)); const {label: mostSimilarLabelCollect} = this._labelGroups.map(label => ({ label, similarityScore: bestEmbedScore(labelToEmbedding.get(label)) })) @@ -293,10 +330,14 @@ export class ImageLabelBox extends ViewBoxBaseComponent() { {this._selectedImages.map(doc => { const [name, type] = ImageCast(doc[Doc.LayoutFieldKey(doc)]).url.href.split('.'); return ( -
this._props.addDocTab(doc, OpenWhere.addRightKeyvalue)}> - -
- {(doc[DocData].data_labels as string).split('\n').map(label => { +
+ { + await DocumentView.showDocument(doc, { willZoomCentered: true }); + }}> +
this._props.addDocTab(doc, OpenWhere.addRightKeyvalue)}> + {(doc[DocData].data_labels as List).map(label => { return (
{label} diff --git a/src/client/views/nodes/DocumentView.tsx b/src/client/views/nodes/DocumentView.tsx index 7a1f94948..9ff96c692 100644 --- a/src/client/views/nodes/DocumentView.tsx +++ b/src/client/views/nodes/DocumentView.tsx @@ -573,7 +573,7 @@ export class DocumentViewInternal extends DocComponent DocumentView.SetLightboxDoc(this.Document), icon: 'external-link-alt' }); } - appearanceItems.push({ description: 'Pin', event: () => this._props.pinToPres(this.Document, {}), icon: 'eye' }); + appearanceItems.push({ description: 'Pin', event: () => this._props.pinToPres(this.Document, {}), icon: 'map-pin' }); if (this.Document._layout_isFlashcard) { appearanceItems.push({ description: 'Create ChatCard', event: () => this.askGPT(), icon: 'id-card' }); } diff --git a/src/client/views/nodes/ImageBox.tsx b/src/client/views/nodes/ImageBox.tsx index 3da878a4f..fb90f907f 100644 --- a/src/client/views/nodes/ImageBox.tsx +++ b/src/client/views/nodes/ImageBox.tsx @@ -1,5 +1,6 @@ import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; import { Tooltip } from '@mui/material'; +import zIndex from '@mui/material/styles/zIndex'; import { Colors } from 'browndash-components'; import { action, computed, IReactionDisposer, makeObservable, observable, ObservableMap, reaction } from 'mobx'; import { observer } from 'mobx-react'; @@ -10,6 +11,7 @@ import { Doc, DocListCast, Opt } from '../../../fields/Doc'; import { DocData } from '../../../fields/DocSymbols'; import { Id } from '../../../fields/FieldSymbols'; import { InkTool } from '../../../fields/InkField'; +import { List } from '../../../fields/List'; import { ObjectField } from '../../../fields/ObjectField'; import { Cast, ImageCast, NumCast, StrCast } from '../../../fields/Types'; import { ImageField } from '../../../fields/URLField'; @@ -281,6 +283,13 @@ export class ImageBox extends ViewBoxAnnotatableComponent() { }), icon: 'pencil-alt', }); + funcs.push({ + description: 'Toggle Keywords', + event: () => { + this.Document[DocData].showLabels = !this.Document[DocData].showLabels; + }, + icon: 'eye', + }); ContextMenu.Instance?.addItem({ description: 'Options...', subitems: funcs, icon: 'asterisk' }); } }; -- cgit v1.2.3-70-g09d2 From 3190f1eb07a47a5e1ccdd20e346b47094118292d Mon Sep 17 00:00:00 2001 From: IEatChili Date: Wed, 26 Jun 2024 13:58:20 -0400 Subject: feat: worked more on keyword input for docs --- src/client/views/DocumentButtonBar.tsx | 19 +++ src/client/views/DocumentDecorations.tsx | 1 + src/client/views/KeywordBox.tsx | 168 +++++++++++++++++---- src/client/views/StyleProvider.scss | 31 +++- src/client/views/StyleProvider.tsx | 6 +- .../collectionFreeForm/ImageLabelBox.tsx | 53 ++----- .../collections/collectionFreeForm/MarqueeView.tsx | 28 +++- src/client/views/nodes/ImageBox.tsx | 7 - 8 files changed, 233 insertions(+), 80 deletions(-) (limited to 'src/client/views/nodes/ImageBox.tsx') diff --git a/src/client/views/DocumentButtonBar.tsx b/src/client/views/DocumentButtonBar.tsx index 487868169..a75c7098c 100644 --- a/src/client/views/DocumentButtonBar.tsx +++ b/src/client/views/DocumentButtonBar.tsx @@ -31,6 +31,7 @@ import { DocumentLinksButton } from './nodes/DocumentLinksButton'; import { DocumentView } from './nodes/DocumentView'; import { OpenWhere } from './nodes/OpenWhere'; import { DashFieldView } from './nodes/formattedText/DashFieldView'; +import { DocData } from '../../fields/DocSymbols'; @observer export class DocumentButtonBar extends ObservableReactComponent<{ views: () => (DocumentView | undefined)[]; stack?: any }> { @@ -282,6 +283,23 @@ export class DocumentButtonBar extends ObservableReactComponent<{ views: () => ( ); } + @computed + get keywordButton() { + const targetDoc = this.view0?.Document; + return !targetDoc ? null : ( + Open keyword menu
}> +
{ + targetDoc[DocData].showLabels = !targetDoc[DocData].showLabels; + }}> + +
+ + ); + } + @observable _isRecording = false; _stopFunc: () => void = emptyFunction; @computed @@ -452,6 +470,7 @@ export class DocumentButtonBar extends ObservableReactComponent<{ views: () => (
{this.pinButton}
{this.recordButton}
{this.calendarButton}
+
{this.keywordButton}
{!Doc.UserDoc().documentLinksButton_fullMenu ? null :
{this.shareButton}
}
{this.menuButton}
diff --git a/src/client/views/DocumentDecorations.tsx b/src/client/views/DocumentDecorations.tsx index 93c3e3338..20bf8fd9f 100644 --- a/src/client/views/DocumentDecorations.tsx +++ b/src/client/views/DocumentDecorations.tsx @@ -88,6 +88,7 @@ export class DocumentDecorations extends ObservableReactComponent center.x+x || this.Bounds.r < center.x+x || this.Bounds.y > center.y+y || this.Bounds.b < center.y+y ))); + })); // prettier-ignore } diff --git a/src/client/views/KeywordBox.tsx b/src/client/views/KeywordBox.tsx index 3faddeb64..8c69f446d 100644 --- a/src/client/views/KeywordBox.tsx +++ b/src/client/views/KeywordBox.tsx @@ -1,64 +1,170 @@ -import { action, makeObservable } from 'mobx'; +import { Colors, IconButton } from 'browndash-components'; +import { action, computed, makeObservable, observable, reaction } from 'mobx'; import { observer } from 'mobx-react'; import React from 'react'; import { Doc } from '../../fields/Doc'; import { DocData } from '../../fields/DocSymbols'; import { List } from '../../fields/List'; +import { DragManager, SetupDrag } from '../util/DragManager'; +import { SnappingManager } from '../util/SnappingManager'; +import { DocumentView } from './nodes/DocumentView'; import { ObservableReactComponent } from './ObservableReactComponent'; +interface KeywordItemProps { + doc: Doc; + label: string; + setToEditing: () => void; + isEditing: boolean; +} + +@observer +export class KeywordItem extends ObservableReactComponent { + constructor(props: any) { + super(props); + makeObservable(this); + this.ref = React.createRef(); + } + + private _dropDisposer?: DragManager.DragDropDisposer; + private ref: React.RefObject; + + protected createDropTarget = (ele: HTMLDivElement) => { + this._dropDisposer?.(); + SetupDrag(this.ref, () => undefined); + //ele && (this._dropDisposer = DragManager. (ele, this.onInternalDrop.bind(this), this.layoutDoc)); + //ele && (this._dropDisposer = DragManager.MakeDropTarget(ele, this.onInternalDrop.bind(this), this.layoutDoc)); + }; + + @action + removeLabel = () => { + if (this._props.doc[DocData].data_labels) { + this._props.doc[DocData].data_labels = (this._props.doc[DocData].data_labels as List).filter(label => label !== this._props.label) as List; + } + }; + + render() { + return ( +
{}} ref={this.ref}> + {this._props.label} + {this.props.isEditing && } +
+ ); + } +} + interface KeywordBoxProps { - _doc: Doc; - _isEditing: boolean; + doc: Doc; + isEditing: boolean; } @observer export class KeywordBox extends ObservableReactComponent { + @observable _currentInput: string = ''; + //private disposer: () => void; + constructor(props: any) { super(props); makeObservable(this); } + // componentDidMount(): void { + // reaction( + // () => ({ + // isDragging: SnappingManager.IsDragging, + // selectedDoc: DocumentView.SelectedDocs().lastElement(), + // isEditing: this._props.isEditing, + // }), + // ({ isDragging, selectedDoc, isEditing }) => { + // if (isDragging || selectedDoc !== this._props.doc || !isEditing) { + // this.setToView(); + // } + // } + // ); + // } + + // componentWillUnmount() { + // this.disposer(); + // } + @action setToEditing = () => { - this._props._isEditing = true; + this._props.isEditing = true; }; @action setToView = () => { - this._props._isEditing = false; + this._props.isEditing = false; }; - submitLabel = () => {}; + submitLabel = () => { + if (this._currentInput.trim()) { + if (!this._props.doc[DocData].data_labels) { + this._props.doc[DocData].data_labels = new List(); + } - onInputChange = () => {}; + (this._props.doc![DocData].data_labels! as List).push(this._currentInput.trim()); + this._currentInput = ''; // Clear the input box + } + }; + + @action + onInputChange = (e: React.ChangeEvent) => { + this._currentInput = e.target.value; + }; render() { - const keywordsList = this._props._doc![DocData].data_labels; + const keywordsList = this._props.doc[DocData].data_labels ? this._props.doc[DocData].data_labels : new List(); + const seldoc = DocumentView.SelectedDocs().lastElement(); + if (SnappingManager.IsDragging || !(seldoc === this._props.doc) || !this._props.isEditing) { + setTimeout( + action(() => { + if ((keywordsList as List).length === 0) { + this._props.doc[DocData].showLabels = false; + } + this.setToView(); + }) + ); + } + return ( -
- {(keywordsList as List).map(label => { - return ( -
- {label} +
+
+ {(keywordsList as List).map(label => { + return ; + })} +
+ {this._props.isEditing ? ( +
+
+ { + e.key === 'Enter' ? this.submitLabel() : null; + e.stopPropagation(); + }} + type="text" + placeholder="Input keywords for document..." + aria-label="keyword-input" + className="keyword-input" + style={{ width: '100%', borderRadius: '5px' }} + /> +
+
+ { + if ((keywordsList as List).length === 0) { + this._props.doc[DocData].showLabels = false; + } else { + this.setToView(); + } + }} + icon={'x'} + style={{ width: '4px' }} + />
- ); - })} - {this._props._isEditing ? ( -
- { - e.key === 'Enter' ? this.submitLabel() : null; - e.stopPropagation(); - }} - type="text" - placeholder="Input keywords for document..." - aria-label="keyword-input" - className="keyword-input" - style={{ width: '100%', borderRadius: '5px' }} - />
) : (
diff --git a/src/client/views/StyleProvider.scss b/src/client/views/StyleProvider.scss index 1e2af9a3a..7cc06f922 100644 --- a/src/client/views/StyleProvider.scss +++ b/src/client/views/StyleProvider.scss @@ -57,12 +57,41 @@ .keywords-container { display: flex; flex-wrap: wrap; + flex-direction: column; + padding-bottom: 4px; + border: 1px solid; + border-radius: 4px; +} + +.keywords-list { + display: flex; + flex-wrap: wrap; } .keyword { - padding: 5px 10px; + padding: 5px 5px; background-color: lightblue; border: 1px solid black; border-radius: 5px; white-space: nowrap; + display: flex; + align-items: center; +} + +.keyword-editing-box { + margin-top: 8px; +} + +.keyword-input-box { + // display: flex; + // align-items: center; + // align-content: center; + margin: auto; + align-self: center; + width: 90%; +} + +.keyword-buttons { + margin-left: auto; + width: 10%; } diff --git a/src/client/views/StyleProvider.tsx b/src/client/views/StyleProvider.tsx index fb509516a..f4d73cd1d 100644 --- a/src/client/views/StyleProvider.tsx +++ b/src/client/views/StyleProvider.tsx @@ -371,8 +371,10 @@ export function DefaultStyleProvider(doc: Opt, props: Opt { - if (doc && doc![DocData].data_labels && doc![DocData].showLabels) { - return () + if (doc && doc![DocData].showLabels && (!doc[DocData].data_labels || (doc[DocData].data_labels as List).length === 0)){ + return () + } else if (doc && doc![DocData].data_labels && doc![DocData].showLabels) { + return () } } return ( diff --git a/src/client/views/collections/collectionFreeForm/ImageLabelBox.tsx b/src/client/views/collections/collectionFreeForm/ImageLabelBox.tsx index cfb81e1a0..fec4d3e12 100644 --- a/src/client/views/collections/collectionFreeForm/ImageLabelBox.tsx +++ b/src/client/views/collections/collectionFreeForm/ImageLabelBox.tsx @@ -123,7 +123,6 @@ export class ImageLabelBox extends ViewBoxBaseComponent() { @action groupImages = () => { this.groupImagesInBox(); - MainView.Instance.closeFlyout(); }; @action @@ -161,16 +160,7 @@ export class ImageLabelBox extends ViewBoxBaseComponent() { classifyImagesInBox = async () => { this.startLoading(); - // const imageInfos = this._selectedImages.map(async doc => { - // if (!doc[DocData].data_labels) { - // const [name, type] = ImageCast(doc[Doc.LayoutFieldKey(doc)]).url.href.split('.'); - // return CollectionCardView.imageUrlToBase64(`${name}_o.${type}`).then(hrefBase64 => - // !hrefBase64 ? undefined : - // gptImageLabel(hrefBase64).then(labels => - // Promise.all(labels.split('\n').map(label => gptGetEmbedding(label))).then(embeddings => - // ({ doc, embeddings, labels }))) ); // prettier-ignore - // } - // }); // Converts the images into a Base64 format, afterwhich the information is sent to GPT to label them. + // Converts the images into a Base64 format, afterwhich the information is sent to GPT to label them. const imageInfos = this._selectedImages.map(async doc => { if (!doc[DocData].data_labels) { @@ -178,8 +168,7 @@ export class ImageLabelBox extends ViewBoxBaseComponent() { return CollectionCardView.imageUrlToBase64(`${name}_o.${type}`).then(hrefBase64 => !hrefBase64 ? undefined : gptImageLabel(hrefBase64).then(labels => - - ({ doc, labels }))) ; // prettier-ignore + ({ doc, labels }))) ; // prettier-ignore } }); @@ -194,24 +183,7 @@ export class ImageLabelBox extends ViewBoxBaseComponent() { (imageInfo.doc[DocData].data_labels as List).push(label); }); } - }); // Add the labels as fields to each image. - - // (await Promise.all(imageInfos)).forEach(imageInfo => { - // if (imageInfo && imageInfo.embeddings && Array.isArray(imageInfo.embeddings)) { - // imageInfo.doc[DocData].data_labels = new List(); - - // const labels = imageInfo.labels.split('\n'); - // labels.forEach(label => { - // label = label.replace(/^\d+\.\s*|-|\*/, '').trim(); - // imageInfo.doc[DocData][`${label}`] = true; - // (imageInfo.doc[DocData].data_labels as List).push(label); - // }); - - // numberRange(5).forEach(n => { - // imageInfo.doc[`data_labels_embedding_${n + 1}`] = new List(imageInfo.embeddings[n]); - // }); - // } - // }); // Add the labels as fields to each image. + }); this.endLoading(); }; @@ -220,12 +192,15 @@ export class ImageLabelBox extends ViewBoxBaseComponent() { * Groups images to most similar labels. */ groupImagesInBox = action(async () => { - this._selectedImages.forEach(doc => { - (doc[DocData].data_labels as List).forEach(async (label, index) => { + this.startLoading(); + + for (const doc of this._selectedImages) { + for (let index = 0; index < (doc[DocData].data_labels as List).length; index++) { + const label = (doc[DocData].data_labels as List)[index]; const embedding = await gptGetEmbedding(label); doc[`data_labels_embedding_${index + 1}`] = new List(embedding); - }); - }); + } + } const labelToEmbedding = new Map(); // Create embeddings for the labels. @@ -234,8 +209,8 @@ export class ImageLabelBox extends ViewBoxBaseComponent() { // For each image, loop through the labels, and calculate similarity. Associate it with the // most similar one. this._selectedImages.forEach(doc => { - const embedLists = numberRange(5).map(n => Array.from(NumListCast(doc[`data_labels_embedding_${n + 1}`]))); - const bestEmbedScore = (embedding: Opt) => Math.max(...embedLists.map((l, index) => (embedding && (1 - index * 0.1) * similarity(Array.from(embedding), l)!) || 0)); + const embedLists = numberRange((doc[DocData].data_labels as List).length).map(n => Array.from(NumListCast(doc[`data_labels_embedding_${n + 1}`]))); + const bestEmbedScore = (embedding: Opt) => Math.max(...embedLists.map((l, index) => (embedding && similarity(Array.from(embedding), l)!) || 0)); const {label: mostSimilarLabelCollect} = this._labelGroups.map(label => ({ label, similarityScore: bestEmbedScore(labelToEmbedding.get(label)) })) .reduce((prev, cur) => cur.similarityScore < 0.3 || cur.similarityScore <= prev.similarityScore ? prev: cur, @@ -243,9 +218,13 @@ export class ImageLabelBox extends ViewBoxBaseComponent() { doc[DocData].data_label = mostSimilarLabelCollect; // The label most similar to the image's contents. }); + this.endLoading(); + if (this._selectedImages) { MarqueeOptionsMenu.Instance.groupImages(); } + + MainView.Instance.closeFlyout(); }); render() { diff --git a/src/client/views/collections/collectionFreeForm/MarqueeView.tsx b/src/client/views/collections/collectionFreeForm/MarqueeView.tsx index f03a9d62d..197681f62 100644 --- a/src/client/views/collections/collectionFreeForm/MarqueeView.tsx +++ b/src/client/views/collections/collectionFreeForm/MarqueeView.tsx @@ -455,8 +455,32 @@ export class MarqueeView extends ObservableReactComponent { - this._props.Document._type_collection = CollectionViewType.Time; // Change the collection view to a Time view. - this._props.Document.pivotField = 'data_label'; // Sets the pivot to be the 'data_label'. + const labelGroups: string[] = ImageLabelBoxData.Instance._labelGroups; + const labelToCollection: Map = new Map(); + const selectedImages = ImageLabelBoxData.Instance._docs; + + // Create new collections associated with each label and get the embeddings for the labels. + let x_offset = 0; + for (const label of labelGroups) { + const newCollection = this.getCollection([], undefined, false); + newCollection._freeform_panX = this.Bounds.left + this.Bounds.width / 2; + newCollection._freeform_panY = this.Bounds.top + this.Bounds.height / 2; + console.log(newCollection._x); + labelToCollection.set(label, newCollection); + this._props.addDocument?.(newCollection); + //newCollection._x = (newCollection._x as number) + x_offset; + //x_offset += newCollection._width as number; + } + + for (const doc of selectedImages) { + if (doc[DocData].data_label) { + Doc.AddDocToList(labelToCollection.get(doc[DocData].data_label as string)!, undefined, doc); + this._props.removeDocument?.(doc); + } + } + + //this._props.Document._type_collection = CollectionViewType.Time; // Change the collection view to a Time view. + //this._props.Document.pivotField = 'data_label'; // Sets the pivot to be the 'data_label'. }); @undoBatch diff --git a/src/client/views/nodes/ImageBox.tsx b/src/client/views/nodes/ImageBox.tsx index fb90f907f..1c90fae9e 100644 --- a/src/client/views/nodes/ImageBox.tsx +++ b/src/client/views/nodes/ImageBox.tsx @@ -283,13 +283,6 @@ export class ImageBox extends ViewBoxAnnotatableComponent() { }), icon: 'pencil-alt', }); - funcs.push({ - description: 'Toggle Keywords', - event: () => { - this.Document[DocData].showLabels = !this.Document[DocData].showLabels; - }, - icon: 'eye', - }); ContextMenu.Instance?.addItem({ description: 'Options...', subitems: funcs, icon: 'asterisk' }); } }; -- cgit v1.2.3-70-g09d2