diff options
| -rw-r--r-- | src/client/views/KeywordBox.tsx | 69 | ||||
| -rw-r--r-- | src/client/views/StyleProvider.scss | 13 | ||||
| -rw-r--r-- | src/client/views/StyleProvider.tsx | 9 | ||||
| -rw-r--r-- | src/client/views/collections/collectionFreeForm/ImageLabelBox.tsx | 79 | ||||
| -rw-r--r-- | src/client/views/nodes/DocumentView.tsx | 2 | ||||
| -rw-r--r-- | src/client/views/nodes/ImageBox.tsx | 9 |
6 files changed, 161 insertions, 20 deletions
diff --git a/src/client/views/KeywordBox.tsx b/src/client/views/KeywordBox.tsx new file mode 100644 index 000000000..3faddeb64 --- /dev/null +++ b/src/client/views/KeywordBox.tsx @@ -0,0 +1,69 @@ +import { action, makeObservable } from 'mobx'; +import { observer } from 'mobx-react'; +import React from 'react'; +import { Doc } from '../../fields/Doc'; +import { DocData } from '../../fields/DocSymbols'; +import { List } from '../../fields/List'; +import { ObservableReactComponent } from './ObservableReactComponent'; + +interface KeywordBoxProps { + _doc: Doc; + _isEditing: boolean; +} + +@observer +export class KeywordBox extends ObservableReactComponent<KeywordBoxProps> { + constructor(props: any) { + super(props); + makeObservable(this); + } + + @action + setToEditing = () => { + this._props._isEditing = true; + }; + + @action + setToView = () => { + this._props._isEditing = false; + }; + + submitLabel = () => {}; + + onInputChange = () => {}; + + render() { + const keywordsList = this._props._doc![DocData].data_labels; + return ( + <div className="keywords-container"> + {(keywordsList as List<string>).map(label => { + return ( + <div className="keyword" onClick={this.setToEditing}> + {label} + </div> + ); + })} + {this._props._isEditing ? ( + <div> + <input + defaultValue="" + autoComplete="off" + onChange={this.onInputChange} + onKeyDown={e => { + e.key === 'Enter' ? this.submitLabel() : null; + e.stopPropagation(); + }} + type="text" + placeholder="Input keywords for document..." + aria-label="keyword-input" + className="keyword-input" + style={{ width: '100%', borderRadius: '5px' }} + /> + </div> + ) : ( + <div></div> + )} + </div> + ); + } +} diff --git a/src/client/views/StyleProvider.scss b/src/client/views/StyleProvider.scss index ce00f6101..1e2af9a3a 100644 --- a/src/client/views/StyleProvider.scss +++ b/src/client/views/StyleProvider.scss @@ -53,3 +53,16 @@ .styleProvider-treeView-icon { opacity: 0; } + +.keywords-container { + display: flex; + flex-wrap: wrap; +} + +.keyword { + padding: 5px 10px; + background-color: lightblue; + border: 1px solid black; + border-radius: 5px; + white-space: nowrap; +} diff --git a/src/client/views/StyleProvider.tsx b/src/client/views/StyleProvider.tsx index b7f8a3170..fb509516a 100644 --- a/src/client/views/StyleProvider.tsx +++ b/src/client/views/StyleProvider.tsx @@ -12,7 +12,9 @@ import { BsArrowDown, BsArrowDownUp, BsArrowUp } from 'react-icons/bs'; import { FaFilter } from 'react-icons/fa'; import { ClientUtils, DashColor, lightOrDark } from '../../ClientUtils'; import { Doc, Opt, StrListCast } from '../../fields/Doc'; +import { DocData } from '../../fields/DocSymbols'; import { Id } from '../../fields/FieldSymbols'; +import { List } from '../../fields/List'; import { ScriptField } from '../../fields/ScriptField'; import { BoolCast, Cast, DocCast, ImageCast, NumCast, ScriptCast, StrCast } from '../../fields/Types'; import { AudioAnnoState } from '../../server/SharedMediaTypes'; @@ -23,6 +25,7 @@ import { SnappingManager } from '../util/SnappingManager'; import { undoBatch, UndoManager } from '../util/UndoManager'; import { TreeSort } from './collections/TreeSort'; import { Colors } from './global/globalEnums'; +import { KeywordBox } from './KeywordBox'; import { DocumentView, DocumentViewProps } from './nodes/DocumentView'; import { FieldViewProps } from './nodes/FieldView'; import { StyleProp } from './StyleProp'; @@ -367,12 +370,18 @@ export function DefaultStyleProvider(doc: Opt<Doc>, props: Opt<FieldViewProps & </Tooltip> ); }; + const keywords = () => { + if (doc && doc![DocData].data_labels && doc![DocData].showLabels) { + return (<KeywordBox _isEditing={false} _doc={doc}></KeywordBox>) + } + } return ( <> {paint()} {lock()} {filter()} {audio()} + {keywords()} </> ); } diff --git a/src/client/views/collections/collectionFreeForm/ImageLabelBox.tsx b/src/client/views/collections/collectionFreeForm/ImageLabelBox.tsx index 571a4504f..cfb81e1a0 100644 --- a/src/client/views/collections/collectionFreeForm/ImageLabelBox.tsx +++ b/src/client/views/collections/collectionFreeForm/ImageLabelBox.tsx @@ -5,7 +5,7 @@ import { observer } from 'mobx-react'; import React from 'react'; import { Doc, NumListCast, Opt } from '../../../../fields/Doc'; import { Docs } from '../../../documents/Documents'; -import { CollectionViewType, DocumentType } from '../../../documents/DocumentTypes'; +import { DocumentType } from '../../../documents/DocumentTypes'; import { ViewBoxBaseComponent } from '../../DocComponent'; import { FieldView, FieldViewProps } from '../../nodes/FieldView'; import { MarqueeOptionsMenu } from './MarqueeOptionsMenu'; @@ -24,6 +24,9 @@ import { List } from '../../../../fields/List'; import { DragManager } from '../../../util/DragManager'; import { OpenWhere } from '../../nodes/OpenWhere'; import similarity from 'compute-cosine-similarity'; +import { DocumentView } from '../../nodes/DocumentView'; + +export class ImageInformationItem {} export class ImageLabelBoxData { static _instance: ImageLabelBoxData; @@ -101,7 +104,6 @@ export class ImageLabelBox extends ViewBoxBaseComponent<FieldViewProps>() { makeObservable(this); ring.register(); ImageLabelBox.Instance = this; - console.log('Image Box Has Been Initialized'); } // ImageLabelBox.Instance.setData() @@ -127,7 +129,6 @@ export class ImageLabelBox extends ViewBoxBaseComponent<FieldViewProps>() { @action startLoading = () => { this._loading = true; - console.log('Start loading has been called!'); }; @action @@ -138,6 +139,11 @@ export class ImageLabelBox extends ViewBoxBaseComponent<FieldViewProps>() { @action toggleDisplayInformation = () => { this._displayImageInformation = !this._displayImageInformation; + if (this._displayImageInformation) { + this._selectedImages.forEach(doc => (doc[DocData].showLabels = true)); + } else { + this._selectedImages.forEach(doc => (doc[DocData].showLabels = false)); + } }; @action @@ -155,33 +161,58 @@ export class ImageLabelBox extends ViewBoxBaseComponent<FieldViewProps>() { classifyImagesInBox = async () => { this.startLoading(); + // const imageInfos = this._selectedImages.map(async doc => { + // if (!doc[DocData].data_labels) { + // const [name, type] = ImageCast(doc[Doc.LayoutFieldKey(doc)]).url.href.split('.'); + // return CollectionCardView.imageUrlToBase64(`${name}_o.${type}`).then(hrefBase64 => + // !hrefBase64 ? undefined : + // gptImageLabel(hrefBase64).then(labels => + // Promise.all(labels.split('\n').map(label => gptGetEmbedding(label))).then(embeddings => + // ({ doc, embeddings, labels }))) ); // prettier-ignore + // } + // }); // Converts the images into a Base64 format, afterwhich the information is sent to GPT to label them. + const imageInfos = this._selectedImages.map(async doc => { if (!doc[DocData].data_labels) { const [name, type] = ImageCast(doc[Doc.LayoutFieldKey(doc)]).url.href.split('.'); return CollectionCardView.imageUrlToBase64(`${name}_o.${type}`).then(hrefBase64 => !hrefBase64 ? undefined : gptImageLabel(hrefBase64).then(labels => - Promise.all(labels.split('\n').map(label => gptGetEmbedding(label))).then(embeddings => - ({ doc, embeddings, labels }))) ); // prettier-ignore + + ({ doc, labels }))) ; // prettier-ignore } - }); // Converts the images into a Base64 format, afterwhich the information is sent to GPT to label them. + }); (await Promise.all(imageInfos)).forEach(imageInfo => { - if (imageInfo && imageInfo.embeddings && Array.isArray(imageInfo.embeddings)) { - imageInfo.doc[DocData].data_labels = imageInfo.labels; + if (imageInfo) { + imageInfo.doc[DocData].data_labels = new List<string>(); const labels = imageInfo.labels.split('\n'); labels.forEach(label => { - label = label.replace(/^\d+\.\s*/, '').trim(); + label = label.replace(/^\d+\.\s*|-|\*/, '').trim(); imageInfo.doc[DocData][`${label}`] = true; - }); - - numberRange(3).forEach(n => { - imageInfo.doc[`data_labels_embedding_${n + 1}`] = new List<number>(imageInfo.embeddings[n]); + (imageInfo.doc[DocData].data_labels as List<string>).push(label); }); } }); // Add the labels as fields to each image. + // (await Promise.all(imageInfos)).forEach(imageInfo => { + // if (imageInfo && imageInfo.embeddings && Array.isArray(imageInfo.embeddings)) { + // imageInfo.doc[DocData].data_labels = new List<string>(); + + // const labels = imageInfo.labels.split('\n'); + // labels.forEach(label => { + // label = label.replace(/^\d+\.\s*|-|\*/, '').trim(); + // imageInfo.doc[DocData][`${label}`] = true; + // (imageInfo.doc[DocData].data_labels as List<string>).push(label); + // }); + + // numberRange(5).forEach(n => { + // imageInfo.doc[`data_labels_embedding_${n + 1}`] = new List<number>(imageInfo.embeddings[n]); + // }); + // } + // }); // Add the labels as fields to each image. + this.endLoading(); }; @@ -189,7 +220,13 @@ export class ImageLabelBox extends ViewBoxBaseComponent<FieldViewProps>() { * Groups images to most similar labels. */ groupImagesInBox = action(async () => { - console.log('Calling!'); + this._selectedImages.forEach(doc => { + (doc[DocData].data_labels as List<string>).forEach(async (label, index) => { + const embedding = await gptGetEmbedding(label); + doc[`data_labels_embedding_${index + 1}`] = new List<number>(embedding); + }); + }); + const labelToEmbedding = new Map<string, number[]>(); // Create embeddings for the labels. await Promise.all(this._labelGroups.map(async label => gptGetEmbedding(label).then(labelEmbedding => labelToEmbedding.set(label, labelEmbedding)))); @@ -197,7 +234,7 @@ export class ImageLabelBox extends ViewBoxBaseComponent<FieldViewProps>() { // For each image, loop through the labels, and calculate similarity. Associate it with the // most similar one. this._selectedImages.forEach(doc => { - const embedLists = numberRange(3).map(n => Array.from(NumListCast(doc[`data_labels_embedding_${n + 1}`]))); + const embedLists = numberRange(5).map(n => Array.from(NumListCast(doc[`data_labels_embedding_${n + 1}`]))); const bestEmbedScore = (embedding: Opt<number[]>) => Math.max(...embedLists.map((l, index) => (embedding && (1 - index * 0.1) * similarity(Array.from(embedding), l)!) || 0)); const {label: mostSimilarLabelCollect} = this._labelGroups.map(label => ({ label, similarityScore: bestEmbedScore(labelToEmbedding.get(label)) })) @@ -293,10 +330,14 @@ export class ImageLabelBox extends ViewBoxBaseComponent<FieldViewProps>() { {this._selectedImages.map(doc => { const [name, type] = ImageCast(doc[Doc.LayoutFieldKey(doc)]).url.href.split('.'); return ( - <div className="image-information" style={{ borderColor: SettingsManager.userColor }} key={Utils.GenerateGuid()} onClick={() => this._props.addDocTab(doc, OpenWhere.addRightKeyvalue)}> - <img src={`${name}_o.${type}`}></img> - <div className="image-information-labels"> - {(doc[DocData].data_labels as string).split('\n').map(label => { + <div className="image-information" style={{ borderColor: SettingsManager.userColor }} key={Utils.GenerateGuid()}> + <img + src={`${name}_o.${type}`} + onClick={async () => { + await DocumentView.showDocument(doc, { willZoomCentered: true }); + }}></img> + <div className="image-information-labels" onClick={() => this._props.addDocTab(doc, OpenWhere.addRightKeyvalue)}> + {(doc[DocData].data_labels as List<string>).map(label => { return ( <div key={Utils.GenerateGuid()} className="image-label" style={{ backgroundColor: SettingsManager.userVariantColor, borderColor: SettingsManager.userColor }}> {label} diff --git a/src/client/views/nodes/DocumentView.tsx b/src/client/views/nodes/DocumentView.tsx index 7a1f94948..9ff96c692 100644 --- a/src/client/views/nodes/DocumentView.tsx +++ b/src/client/views/nodes/DocumentView.tsx @@ -573,7 +573,7 @@ export class DocumentViewInternal extends DocComponent<FieldViewProps & Document if (this._props.renderDepth === 0) { appearanceItems.splice(0, 0, { description: 'Open in Lightbox', event: () => DocumentView.SetLightboxDoc(this.Document), icon: 'external-link-alt' }); } - appearanceItems.push({ description: 'Pin', event: () => this._props.pinToPres(this.Document, {}), icon: 'eye' }); + appearanceItems.push({ description: 'Pin', event: () => this._props.pinToPres(this.Document, {}), icon: 'map-pin' }); if (this.Document._layout_isFlashcard) { appearanceItems.push({ description: 'Create ChatCard', event: () => this.askGPT(), icon: 'id-card' }); } diff --git a/src/client/views/nodes/ImageBox.tsx b/src/client/views/nodes/ImageBox.tsx index 3da878a4f..fb90f907f 100644 --- a/src/client/views/nodes/ImageBox.tsx +++ b/src/client/views/nodes/ImageBox.tsx @@ -1,5 +1,6 @@ import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; import { Tooltip } from '@mui/material'; +import zIndex from '@mui/material/styles/zIndex'; import { Colors } from 'browndash-components'; import { action, computed, IReactionDisposer, makeObservable, observable, ObservableMap, reaction } from 'mobx'; import { observer } from 'mobx-react'; @@ -10,6 +11,7 @@ import { Doc, DocListCast, Opt } from '../../../fields/Doc'; import { DocData } from '../../../fields/DocSymbols'; import { Id } from '../../../fields/FieldSymbols'; import { InkTool } from '../../../fields/InkField'; +import { List } from '../../../fields/List'; import { ObjectField } from '../../../fields/ObjectField'; import { Cast, ImageCast, NumCast, StrCast } from '../../../fields/Types'; import { ImageField } from '../../../fields/URLField'; @@ -281,6 +283,13 @@ export class ImageBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { }), icon: 'pencil-alt', }); + funcs.push({ + description: 'Toggle Keywords', + event: () => { + this.Document[DocData].showLabels = !this.Document[DocData].showLabels; + }, + icon: 'eye', + }); ContextMenu.Instance?.addItem({ description: 'Options...', subitems: funcs, icon: 'asterisk' }); } }; |