From b440901843a930c6c87ec23c59f90f1349c25b50 Mon Sep 17 00:00:00 2001 From: IEatChili Date: Thu, 6 Jun 2024 15:23:12 -0400 Subject: feat: updated ui --- src/client/documents/DocumentTypes.ts | 1 + src/client/documents/Documents.ts | 4 + src/client/util/CurrentUserUtils.ts | 7 + src/client/views/Main.tsx | 2 + src/client/views/MainView.tsx | 1 + .../collectionFreeForm/ImageLabelBox.scss | 26 ++++ .../collectionFreeForm/ImageLabelBox.tsx | 150 +++++++++++++++++++++ .../collectionFreeForm/ImageLabelHandler.tsx | 2 +- .../collections/collectionFreeForm/MarqueeView.tsx | 56 +++++--- src/client/views/linking/LinkPopup.tsx | 1 - src/fields/Doc.ts | 1 + 11 files changed, 230 insertions(+), 21 deletions(-) create mode 100644 src/client/views/collections/collectionFreeForm/ImageLabelBox.scss create mode 100644 src/client/views/collections/collectionFreeForm/ImageLabelBox.tsx (limited to 'src') diff --git a/src/client/documents/DocumentTypes.ts b/src/client/documents/DocumentTypes.ts index 8f95068db..a9ea889b3 100644 --- a/src/client/documents/DocumentTypes.ts +++ b/src/client/documents/DocumentTypes.ts @@ -16,6 +16,7 @@ export enum DocumentType { SCREENSHOT = 'screenshot', FONTICON = 'fonticonbox', SEARCH = 'search', // search query + IMAGEGROUPER = 'imagegrouper', LABEL = 'label', // simple text label BUTTON = 'button', // onClick button WEBCAM = 'webcam', // webcam diff --git a/src/client/documents/Documents.ts b/src/client/documents/Documents.ts index a67e6b4f6..449347403 100644 --- a/src/client/documents/Documents.ts +++ b/src/client/documents/Documents.ts @@ -784,6 +784,10 @@ export namespace Docs { return InstanceFromProto(Prototypes.get(DocumentType.SEARCH), new List([]), options); } + export function ImageGrouperDocument(options: DocumentOptions = {}) { + return InstanceFromProto(Prototypes.get(DocumentType.IMAGEGROUPER), undefined, options); + } + export function LoadingDocument(file: File | string, options: DocumentOptions) { return InstanceFromProto(Prototypes.get(DocumentType.LOADING), undefined, { _height: 150, _width: 200, title: typeof file === 'string' ? file : file.name, ...options }, undefined, ''); } diff --git a/src/client/util/CurrentUserUtils.ts b/src/client/util/CurrentUserUtils.ts index e095bc659..486b6815a 100644 --- a/src/client/util/CurrentUserUtils.ts +++ b/src/client/util/CurrentUserUtils.ts @@ -458,6 +458,7 @@ pie title Minerals in my tap water { title: "Shared", toolTip: "Shared Docs", target: Doc.MySharedDocs, ignoreClick: true, icon: "users", funcs: {badgeValue: badgeValue}}, { title: "Trails", toolTip: "Trails ⌘R", target: Doc.UserDoc(), ignoreClick: true, icon: "pres-trail", funcs: {target: getActiveDashTrails}}, { title: "User Doc", toolTip: "User Doc", target: this.setupUserDocView(doc, "myUserDocView"), ignoreClick: true, icon: "address-card",funcs: {hidden: "IsNoviceMode()"} }, + { title: "Image Grouper", toolTip: "Image Grouper", target: this.setupImageGrouper(doc, "myImageGrouper"), ignoreClick: true, icon: "folder-open", hidden: true } ].map(tuple => ({...tuple, scripts:{onClick: 'selectMainMenu(this)'}})); } @@ -493,6 +494,12 @@ pie title Minerals in my tap water _lockedPosition: true, _type_collection: CollectionViewType.Schema }); } + static setupImageGrouper(doc: Doc, field: string) { + return DocUtils.AssignDocField(doc, field, (opts) => Docs.Create.ImageGrouperDocument(opts), { + dontRegisterView: true, backgroundColor: "dimgray", ignoreClick: true, title: "Image Grouper", isSystem: true, childDragAction: dropActionType.embed, + _lockedPosition: true, _type_collection: CollectionViewType.Schema }); + } + /// Initializes the panel of draggable tools that is opened from the left sidebar. static setupToolsBtnPanel(doc: Doc, field:string) { const allTools = DocListCast(DocCast(doc[field])?.data); diff --git a/src/client/views/Main.tsx b/src/client/views/Main.tsx index 43b9a6b39..8242e7c27 100644 --- a/src/client/views/Main.tsx +++ b/src/client/views/Main.tsx @@ -60,6 +60,7 @@ import { SummaryView } from './nodes/formattedText/SummaryView'; import { ImportElementBox } from './nodes/importBox/ImportElementBox'; import { PresBox, PresElementBox } from './nodes/trails'; import { SearchBox } from './search/SearchBox'; +import { ImageLabelBox } from './collections/collectionFreeForm/ImageLabelBox'; dotenv.config(); @@ -131,6 +132,7 @@ FieldLoader.ServerLoadStatus = { requested: 0, retrieved: 0, message: 'cache' }; PresBox, PresElementBox, SearchBox, + ImageLabelBox, //Here! FunctionPlotBox, InkingStroke, LinkBox, diff --git a/src/client/views/MainView.tsx b/src/client/views/MainView.tsx index 31d88fb87..5b4c2b5ba 100644 --- a/src/client/views/MainView.tsx +++ b/src/client/views/MainView.tsx @@ -76,6 +76,7 @@ import { PresBox } from './nodes/trails'; import { AnchorMenu } from './pdf/AnchorMenu'; import { GPTPopup } from './pdf/GPTPopup/GPTPopup'; import { TopBar } from './topbar/TopBar'; +import { ImageLabelBox } from './collections/collectionFreeForm/ImageLabelBox'; const { LEFT_MENU_WIDTH, TOPBAR_HEIGHT } = require('./global/globalCssVariables.module.scss'); // prettier-ignore const _global = (window /* browser */ || global) /* node */ as any; diff --git a/src/client/views/collections/collectionFreeForm/ImageLabelBox.scss b/src/client/views/collections/collectionFreeForm/ImageLabelBox.scss new file mode 100644 index 000000000..d0c12814c --- /dev/null +++ b/src/client/views/collections/collectionFreeForm/ImageLabelBox.scss @@ -0,0 +1,26 @@ +.image-label-list { + display: flex; + flex-direction: column; + align-items: center; // Centers the content vertically in the flex container + width: 100%; + + > div { + display: flex; + justify-content: space-between; // Puts the content and delete button on opposite ends + align-items: center; + width: 100%; + margin-top: 8px; // Adds space between label rows + background-color: black; + + p { + text-align: center; // Centers the text of the paragraph + font-size: large; + vertical-align: middle; + } + + .IconButton { + // Styling for the delete button + margin-left: auto; // Pushes the button to the far right + } + } +} diff --git a/src/client/views/collections/collectionFreeForm/ImageLabelBox.tsx b/src/client/views/collections/collectionFreeForm/ImageLabelBox.tsx new file mode 100644 index 000000000..1c0035f0d --- /dev/null +++ b/src/client/views/collections/collectionFreeForm/ImageLabelBox.tsx @@ -0,0 +1,150 @@ +import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; +import { Colors, IconButton } from 'browndash-components'; +import { action, makeObservable, observable } from 'mobx'; +import { observer } from 'mobx-react'; +import React from 'react'; +import { Doc } from '../../../../fields/Doc'; +import { Docs } from '../../../documents/Documents'; +import { DocumentType } from '../../../documents/DocumentTypes'; +import { ViewBoxBaseComponent } from '../../DocComponent'; +import { FieldView, FieldViewProps } from '../../nodes/FieldView'; +import { MarqueeOptionsMenu } from './MarqueeOptionsMenu'; +import './ImageLabelBox.scss'; +import { MainView } from '../../MainView'; +import { MarqueeView } from './MarqueeView'; + +@observer +export class ImageLabelBox extends ViewBoxBaseComponent() { + public static LayoutString(fieldKey: string) { + return FieldView.LayoutString(ImageLabelBox, fieldKey); + } + + public static Instance: ImageLabelBox | null = null; + private _inputRef = React.createRef(); + @observable _loading: boolean = true; + @observable _currentLabel: string = ''; + @observable _labelGroups: string[] = []; + + constructor(props: any) { + super(props); + makeObservable(this); + ImageLabelBox.Instance = this; + + console.log('Image Box Has Been Initialized'); + } + + /** + * This method is called when the SearchBox component is first mounted. When the user opens + * the search panel, the search input box is automatically selected. This allows the user to + * type in the search input box immediately, without needing clicking on it first. + */ + componentDidMount() { + // if (this._inputRef.current) { + // this._inputRef.current.focus(); + // } + } + + @action + addLabel = (label: string) => { + label = label.toUpperCase().trim(); + if (label.length > 0) { + if (!this._labelGroups.includes(label)) { + this._labelGroups = [...this._labelGroups, label]; + } + } + }; + + @action + removeLabel = (label: string) => { + const labelUp = label.toUpperCase(); + this._labelGroups = this._labelGroups.filter(group => group !== labelUp); + }; + + @action + groupImages = () => { + MarqueeOptionsMenu.Instance.groupImages(); + this._labelGroups = []; + MainView.Instance.closeFlyout(); + }; + + @action + startLoading = () => { + this._loading = true; + }; + + @action + endLoading = () => { + this._loading = false; + }; + + render() { + if (this._loading) { + return
Loading...
; + } + + return ( +
+
+ { + // e.key === 'Enter' ? this.submitSearch() : null; + // e.stopPropagation(); + // }} + type="text" + placeholder="Input a group to put images into..." + aria-label="label-input" + id="new-label" + className="searchBox-input" + style={{ width: '100%', borderRadius: '5px' }} + ref={this._inputRef} + /> + { + const input = document.getElementById('new-label') as HTMLInputElement; + const newLabel = input.value; + this.addLabel(newLabel); + this._currentLabel = ''; + input.value = ''; + }} + icon={} + color={MarqueeOptionsMenu.Instance.userColor} + style={{ width: '19px' }} + /> + {this._labelGroups.length > 0 ? ( + } color={MarqueeOptionsMenu.Instance.userColor} style={{ width: '19px' }} /> + ) : ( +
+ )} +
+
+
+ {this._labelGroups.map(group => { + return ( +
+

{group}

+ { + this.removeLabel(group); + }} + icon={'x'} + color={MarqueeOptionsMenu.Instance.userColor} + style={{ width: '8px' }} + /> +
+ ); + })} +
+
+
+ ); + } +} + +Docs.Prototypes.TemplateMap.set(DocumentType.IMAGEGROUPER, { + layout: { view: ImageLabelBox, dataField: 'data' }, + options: { acl: '', _width: 400 }, +}); diff --git a/src/client/views/collections/collectionFreeForm/ImageLabelHandler.tsx b/src/client/views/collections/collectionFreeForm/ImageLabelHandler.tsx index 7f27c6b5c..73befb205 100644 --- a/src/client/views/collections/collectionFreeForm/ImageLabelHandler.tsx +++ b/src/client/views/collections/collectionFreeForm/ImageLabelHandler.tsx @@ -77,7 +77,7 @@ export class ImageLabelHandler extends ObservableReactComponent<{}> { }}>
} color={MarqueeOptionsMenu.Instance.userColor} style={{ width: '19px' }} /> - + { diff --git a/src/client/views/collections/collectionFreeForm/MarqueeView.tsx b/src/client/views/collections/collectionFreeForm/MarqueeView.tsx index dc15c83c5..bb5a2a66e 100644 --- a/src/client/views/collections/collectionFreeForm/MarqueeView.tsx +++ b/src/client/views/collections/collectionFreeForm/MarqueeView.tsx @@ -5,7 +5,7 @@ import { observer } from 'mobx-react'; import * as React from 'react'; import { ClientUtils, lightOrDark, returnFalse } from '../../../../ClientUtils'; import { intersectRect, numberRange } from '../../../../Utils'; -import { Doc, NumListCast, Opt } from '../../../../fields/Doc'; +import { Doc, DocListCast, NumListCast, Opt } from '../../../../fields/Doc'; import { AclAdmin, AclAugment, AclEdit, DocData } from '../../../../fields/DocSymbols'; import { Id } from '../../../../fields/FieldSymbols'; import { InkData, InkField, InkTool } from '../../../../fields/InkField'; @@ -36,6 +36,9 @@ import { CollectionFreeFormView } from './CollectionFreeFormView'; import { ImageLabelHandler } from './ImageLabelHandler'; import { MarqueeOptionsMenu } from './MarqueeOptionsMenu'; import './MarqueeView.scss'; +import { MainView } from '../../MainView'; +import { ImageLabelBox } from './ImageLabelBox'; +import { SearchBox } from '../../search/SearchBox'; interface MarqueeViewProps { getContainerTransform: () => Transform; @@ -53,6 +56,9 @@ interface MarqueeViewProps { slowLoadDocuments: (files: File[] | string, options: DocumentOptions, generatedDocuments: Doc[], text: string, completed: ((doc: Doc[]) => void) | undefined, addDocument: (doc: Doc | Doc[]) => boolean) => Promise; } +/** + * A component that deals with the marquee select in the freeform canvas. + */ @observer export class MarqueeView extends ObservableReactComponent { public static CurViewBounds(pinDoc: Doc, panelWidth: number, panelHeight: number) { @@ -60,9 +66,12 @@ export class MarqueeView extends ObservableReactComponent { - this._selectedDocs = this.marqueeSelect(false, DocumentType.IMG); + if (e) { + const groupButton = DocListCast(Doc.MyLeftSidebarMenu.data).find(d => d.target === Doc.MyImageGrouper); + if (groupButton) { + MainView.Instance.expandFlyout(groupButton); + while (!ImageLabelBox.Instance) { + await new Promise(resolve => setTimeout(resolve, 1000)).then(() => { + console.log('Waiting for Image Label Box'); + }); + } + ImageLabelBox.Instance.startLoading(); + } + } + + this._selectedDocs = this.marqueeSelect(false, DocumentType.IMG); // Get the selected documents from the marquee select. const imageInfos = this._selectedDocs.map(async doc => { - const [name, type] = ImageCast(doc[Doc.LayoutFieldKey(doc)]).url.href.split('.'); - return CollectionCardView.imageUrlToBase64(`${name}_o.${type}`).then(hrefBase64 => + if (!doc[DocData].data_labels) { + const [name, type] = ImageCast(doc[Doc.LayoutFieldKey(doc)]).url.href.split('.'); + return CollectionCardView.imageUrlToBase64(`${name}_o.${type}`).then(hrefBase64 => !hrefBase64 ? undefined : gptImageLabel(hrefBase64).then(labels => Promise.all(labels.split('\n').map(label => gptGetEmbedding(label))).then(embeddings => ({ doc, embeddings, labels }))) ); // prettier-ignore - }); + } + }); // Converts the images into a Base64 format, afterwhich the information is sent to GPT to label them. (await Promise.all(imageInfos)).forEach(imageInfo => { - if (imageInfo && Array.isArray(imageInfo.embeddings)) { + if (imageInfo && imageInfo.embeddings && Array.isArray(imageInfo.embeddings)) { imageInfo.doc[DocData].data_labels = imageInfo.labels; numberRange(3).forEach(n => { imageInfo.doc[`data_labels_embedding_${n + 1}`] = new List(imageInfo.embeddings[n]); }); } - }); + }); // Add the labels as fields to each image. - if (e) { - ImageLabelHandler.Instance.displayLabelHandler(e.pageX, e.pageY); - } + ImageLabelBox.Instance!.endLoading(); + console.log('Complete!'); }); /** @@ -464,7 +486,7 @@ export class MarqueeView extends ObservableReactComponent { - const labelGroups = ImageLabelHandler.Instance._labelGroups; + const labelGroups = ImageLabelBox.Instance!._labelGroups; const labelToEmbedding = new Map(); // Create embeddings for the labels. await Promise.all(labelGroups.map(async label => gptGetEmbedding(label).then(labelEmbedding => labelToEmbedding.set(label, labelEmbedding)))); @@ -478,14 +500,10 @@ export class MarqueeView extends ObservableReactComponent ({ label, similarityScore: bestEmbedScore(labelToEmbedding.get(label)) })) .reduce((prev, cur) => cur.similarityScore < 0.3 || cur.similarityScore <= prev.similarityScore ? prev: cur, { label: '', similarityScore: 0, }); // prettier-ignore - - numberRange(3).forEach(n => { - doc[`data_labels_embedding_${n + 1}`] = undefined; - }); - doc[DocData].data_label = mostSimilarLabelCollect; + doc[DocData].data_label = mostSimilarLabelCollect; // The label most similar to the image's contents. }); - this._props.Document._type_collection = CollectionViewType.Time; - this._props.Document.pivotField = 'data_label'; + this._props.Document._type_collection = CollectionViewType.Time; // Change the collection view to a Time view. + this._props.Document.pivotField = 'data_label'; // Sets the pivot to be the 'data_label'. }); @undoBatch diff --git a/src/client/views/linking/LinkPopup.tsx b/src/client/views/linking/LinkPopup.tsx index 76a8396ff..2405e375d 100644 --- a/src/client/views/linking/LinkPopup.tsx +++ b/src/client/views/linking/LinkPopup.tsx @@ -45,7 +45,6 @@ export class LinkPopup extends React.Component { {/* */} - Date: Mon, 10 Jun 2024 11:06:02 -0400 Subject: feat: ui changes --- package-lock.json | 6 +++ package.json | 1 + .../collectionFreeForm/ImageLabelBox.scss | 23 +++++++++ .../collectionFreeForm/ImageLabelBox.tsx | 60 ++++++++++++++++++---- .../collectionFreeForm/MarqueeOptionsMenu.tsx | 2 +- .../collections/collectionFreeForm/MarqueeView.tsx | 9 ++-- 6 files changed, 86 insertions(+), 15 deletions(-) (limited to 'src') diff --git a/package-lock.json b/package-lock.json index 9b3904ff7..6cd2ab605 100644 --- a/package-lock.json +++ b/package-lock.json @@ -136,6 +136,7 @@ "js-datepicker": "^5.18.2", "jsonschema": "^1.4.1", "jszip": "^3.10.1", + "ldrs": "^1.0.2", "lodash": "^4.17.21", "mapbox-gl": "^3.0.1", "markdown-it": "^14.1.0", @@ -23613,6 +23614,11 @@ "safe-buffer": "~5.1.0" } }, + "node_modules/ldrs": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/ldrs/-/ldrs-1.0.2.tgz", + "integrity": "sha512-sYJmivdkIiHrUEqTrEWccBoLdaENpzbzkABI5rk8rRxTXrg9i2xVuDvUUuhOhJY3RmQyaoxs046pM1DCRdcIpg==" + }, "node_modules/leac": { "version": "0.6.0", "resolved": "https://registry.npmjs.org/leac/-/leac-0.6.0.tgz", diff --git a/package.json b/package.json index ed05c4f45..bbd2a4c46 100644 --- a/package.json +++ b/package.json @@ -221,6 +221,7 @@ "js-datepicker": "^5.18.2", "jsonschema": "^1.4.1", "jszip": "^3.10.1", + "ldrs": "^1.0.2", "lodash": "^4.17.21", "mapbox-gl": "^3.0.1", "markdown-it": "^14.1.0", diff --git a/src/client/views/collections/collectionFreeForm/ImageLabelBox.scss b/src/client/views/collections/collectionFreeForm/ImageLabelBox.scss index d0c12814c..5f2ce4e14 100644 --- a/src/client/views/collections/collectionFreeForm/ImageLabelBox.scss +++ b/src/client/views/collections/collectionFreeForm/ImageLabelBox.scss @@ -1,3 +1,18 @@ +.image-box-container { + display: flex; + align-items: center; + justify-content: center; + width: 100%; + height: 100%; + font-size: 10px; + line-height: 1; + background: none; + z-index: 1000; + padding: 0px; + overflow: auto; + cursor: default; +} + .image-label-list { display: flex; flex-direction: column; @@ -16,6 +31,7 @@ text-align: center; // Centers the text of the paragraph font-size: large; vertical-align: middle; + margin-left: 10px; } .IconButton { @@ -24,3 +40,10 @@ } } } + +.image-information { + display: flex; + flex-direction: column; + //align-items: center; + width: 100%; +} diff --git a/src/client/views/collections/collectionFreeForm/ImageLabelBox.tsx b/src/client/views/collections/collectionFreeForm/ImageLabelBox.tsx index 1c0035f0d..83ae69cfb 100644 --- a/src/client/views/collections/collectionFreeForm/ImageLabelBox.tsx +++ b/src/client/views/collections/collectionFreeForm/ImageLabelBox.tsx @@ -12,6 +12,11 @@ import { MarqueeOptionsMenu } from './MarqueeOptionsMenu'; import './ImageLabelBox.scss'; import { MainView } from '../../MainView'; import { MarqueeView } from './MarqueeView'; +import 'ldrs/ring'; +import { ring } from 'ldrs'; +import { SnappingManager } from '../../../util/SnappingManager'; +import { ImageCast } from '../../../../fields/Types'; +import { DocData } from '../../../../fields/DocSymbols'; @observer export class ImageLabelBox extends ViewBoxBaseComponent() { @@ -24,12 +29,14 @@ export class ImageLabelBox extends ViewBoxBaseComponent() { @observable _loading: boolean = true; @observable _currentLabel: string = ''; @observable _labelGroups: string[] = []; + @observable _selectedImages: Doc[] = []; + @observable _displayImageInformation: boolean = false; constructor(props: any) { super(props); makeObservable(this); ImageLabelBox.Instance = this; - + ring.register(); console.log('Image Box Has Been Initialized'); } @@ -77,14 +84,32 @@ export class ImageLabelBox extends ViewBoxBaseComponent() { this._loading = false; }; + @action + setSelectedImages = (selected: Doc[]) => { + this._selectedImages = selected; + }; + render() { if (this._loading) { - return
Loading...
; + return ( +
+ +
+ ); } return ( -
-
+
+
+ { + this._displayImageInformation = !this._displayImageInformation; + }} + icon={this._displayImageInformation ? : } + color={MarqueeOptionsMenu.Instance.userColor} + style={{ width: '19px' }} + /> () { // e.stopPropagation(); // }} type="text" - placeholder="Input a group to put images into..." + placeholder="Input labels for image groupings..." aria-label="label-input" id="new-label" className="searchBox-input" @@ -101,7 +126,7 @@ export class ImageLabelBox extends ViewBoxBaseComponent() { ref={this._inputRef} /> { const input = document.getElementById('new-label') as HTMLInputElement; const newLabel = input.value; @@ -113,11 +138,7 @@ export class ImageLabelBox extends ViewBoxBaseComponent() { color={MarqueeOptionsMenu.Instance.userColor} style={{ width: '19px' }} /> - {this._labelGroups.length > 0 ? ( - } color={MarqueeOptionsMenu.Instance.userColor} style={{ width: '19px' }} /> - ) : ( -
- )} + {this._labelGroups.length > 0 ? } color={Colors.MEDIUM_BLUE} style={{ width: '19px' }} /> :
}
@@ -139,6 +160,23 @@ export class ImageLabelBox extends ViewBoxBaseComponent() { })}
+ {this._displayImageInformation ? ( +
+ {this._selectedImages.map(doc => { + const [name, type] = ImageCast(doc[Doc.LayoutFieldKey(doc)]).url.href.split('.'); + return ( +
+ + {(doc[DocData].data_labels as string).split('\n').map(label => { + return
{label}
; + })} +
+ ); + })} +
+ ) : ( +
+ )}
); } diff --git a/src/client/views/collections/collectionFreeForm/MarqueeOptionsMenu.tsx b/src/client/views/collections/collectionFreeForm/MarqueeOptionsMenu.tsx index f02cd9d45..b94a22d04 100644 --- a/src/client/views/collections/collectionFreeForm/MarqueeOptionsMenu.tsx +++ b/src/client/views/collections/collectionFreeForm/MarqueeOptionsMenu.tsx @@ -39,7 +39,7 @@ export class MarqueeOptionsMenu extends AntimodeMenu { } color={this.userColor} /> } color={this.userColor} /> } color={this.userColor} /> - } color={this.userColor} /> + } color={this.userColor} /> ); return this.getElement(buttons); diff --git a/src/client/views/collections/collectionFreeForm/MarqueeView.tsx b/src/client/views/collections/collectionFreeForm/MarqueeView.tsx index bb5a2a66e..f98883bff 100644 --- a/src/client/views/collections/collectionFreeForm/MarqueeView.tsx +++ b/src/client/views/collections/collectionFreeForm/MarqueeView.tsx @@ -456,6 +456,7 @@ export class MarqueeView extends ObservableReactComponent { if (!doc[DocData].data_labels) { @@ -478,7 +479,7 @@ export class MarqueeView extends ObservableReactComponent Date: Wed, 12 Jun 2024 12:32:19 -0400 Subject: feat: updated and fixed ui --- src/client/util/CurrentUserUtils.ts | 2 +- src/client/views/MainView.tsx | 2 +- .../collectionFreeForm/ImageLabelBox.scss | 40 +++++- .../collectionFreeForm/ImageLabelBox.tsx | 159 +++++++++++++++------ .../collections/collectionFreeForm/MarqueeView.tsx | 46 +----- 5 files changed, 164 insertions(+), 85 deletions(-) (limited to 'src') diff --git a/src/client/util/CurrentUserUtils.ts b/src/client/util/CurrentUserUtils.ts index 486b6815a..cb3d9df62 100644 --- a/src/client/util/CurrentUserUtils.ts +++ b/src/client/util/CurrentUserUtils.ts @@ -458,7 +458,7 @@ pie title Minerals in my tap water { title: "Shared", toolTip: "Shared Docs", target: Doc.MySharedDocs, ignoreClick: true, icon: "users", funcs: {badgeValue: badgeValue}}, { title: "Trails", toolTip: "Trails ⌘R", target: Doc.UserDoc(), ignoreClick: true, icon: "pres-trail", funcs: {target: getActiveDashTrails}}, { title: "User Doc", toolTip: "User Doc", target: this.setupUserDocView(doc, "myUserDocView"), ignoreClick: true, icon: "address-card",funcs: {hidden: "IsNoviceMode()"} }, - { title: "Image Grouper", toolTip: "Image Grouper", target: this.setupImageGrouper(doc, "myImageGrouper"), ignoreClick: true, icon: "folder-open", hidden: true } + { title: "Image Grouper", toolTip: "Image Grouper", target: this.setupImageGrouper(doc, "myImageGrouper"), ignoreClick: true, icon: "folder-open", hidden: false } ].map(tuple => ({...tuple, scripts:{onClick: 'selectMainMenu(this)'}})); } diff --git a/src/client/views/MainView.tsx b/src/client/views/MainView.tsx index 5b4c2b5ba..716edc22d 100644 --- a/src/client/views/MainView.tsx +++ b/src/client/views/MainView.tsx @@ -76,7 +76,7 @@ import { PresBox } from './nodes/trails'; import { AnchorMenu } from './pdf/AnchorMenu'; import { GPTPopup } from './pdf/GPTPopup/GPTPopup'; import { TopBar } from './topbar/TopBar'; -import { ImageLabelBox } from './collections/collectionFreeForm/ImageLabelBox'; +import { ImageLabelBox, ImageLabelBoxData } from './collections/collectionFreeForm/ImageLabelBox'; const { LEFT_MENU_WIDTH, TOPBAR_HEIGHT } = require('./global/globalCssVariables.module.scss'); // prettier-ignore const _global = (window /* browser */ || global) /* node */ as any; diff --git a/src/client/views/collections/collectionFreeForm/ImageLabelBox.scss b/src/client/views/collections/collectionFreeForm/ImageLabelBox.scss index 5f2ce4e14..819c72760 100644 --- a/src/client/views/collections/collectionFreeForm/ImageLabelBox.scss +++ b/src/client/views/collections/collectionFreeForm/ImageLabelBox.scss @@ -41,9 +41,45 @@ } } -.image-information { +.image-information-list { display: flex; flex-direction: column; - //align-items: center; + align-items: center; + width: 100%; + margin-top: 10px; +} + +.image-information { + border: 1px solid; width: 100%; + display: inline-flex; + flex-direction: column; + justify-content: center; + align-items: center; + overflow: hidden; + padding: 2px; + overflow-x: auto; + overflow-y: auto; + + img { + max-width: 200px; + max-height: 200px; + width: auto; + height: auto; + } +} + +.image-information-labels { + display: flex; + flex-direction: column; + justify-content: center; + align-items: center; + + .image-label { + margin-top: 5px; + margin-bottom: 5px; + padding: 3px; + border-radius: 2px; + border: solid 1px; + } } diff --git a/src/client/views/collections/collectionFreeForm/ImageLabelBox.tsx b/src/client/views/collections/collectionFreeForm/ImageLabelBox.tsx index 83ae69cfb..f5530ccc4 100644 --- a/src/client/views/collections/collectionFreeForm/ImageLabelBox.tsx +++ b/src/client/views/collections/collectionFreeForm/ImageLabelBox.tsx @@ -1,6 +1,6 @@ import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; import { Colors, IconButton } from 'browndash-components'; -import { action, makeObservable, observable } from 'mobx'; +import { action, computed, makeObservable, observable, reaction } from 'mobx'; import { observer } from 'mobx-react'; import React from 'react'; import { Doc } from '../../../../fields/Doc'; @@ -11,12 +11,51 @@ import { FieldView, FieldViewProps } from '../../nodes/FieldView'; import { MarqueeOptionsMenu } from './MarqueeOptionsMenu'; import './ImageLabelBox.scss'; import { MainView } from '../../MainView'; -import { MarqueeView } from './MarqueeView'; import 'ldrs/ring'; import { ring } from 'ldrs'; import { SnappingManager } from '../../../util/SnappingManager'; import { ImageCast } from '../../../../fields/Types'; import { DocData } from '../../../../fields/DocSymbols'; +import { SettingsManager } from '../../../util/SettingsManager'; +import { CollectionCardView } from '../CollectionCardDeckView'; +import { gptGetEmbedding, gptImageLabel } from '../../../apis/gpt/GPT'; +import { numberRange, Utils } from '../../../../Utils'; +import { List } from '../../../../fields/List'; + +export class ImageLabelBoxData { + static _instance: ImageLabelBoxData; + @observable _docs: Doc[] = []; + @observable _labelGroups: string[] = []; + + constructor() { + makeObservable(this); + ImageLabelBoxData._instance = this; + } + public static get Instance() { + return ImageLabelBoxData._instance ?? new ImageLabelBoxData(); + } + + @action + public setData = (docs: Doc[]) => { + this._docs = docs; + }; + + @action + addLabel = (label: string) => { + label = label.toUpperCase().trim(); + if (label.length > 0) { + if (!this._labelGroups.includes(label)) { + this._labelGroups = [...this._labelGroups, label]; + } + } + }; + + @action + removeLabel = (label: string) => { + const labelUp = label.toUpperCase(); + this._labelGroups = this._labelGroups.filter(group => group !== labelUp); + }; +} @observer export class ImageLabelBox extends ViewBoxBaseComponent() { @@ -24,59 +63,53 @@ export class ImageLabelBox extends ViewBoxBaseComponent() { return FieldView.LayoutString(ImageLabelBox, fieldKey); } - public static Instance: ImageLabelBox | null = null; + public static Instance: ImageLabelBox; private _inputRef = React.createRef(); - @observable _loading: boolean = true; - @observable _currentLabel: string = ''; - @observable _labelGroups: string[] = []; - @observable _selectedImages: Doc[] = []; + @observable _loading: boolean = false; + private _currentLabel: string = ''; + + @computed get _labelGroups() { + return ImageLabelBoxData.Instance._labelGroups; + } + + @computed get _selectedImages() { + // return DocListCast(this.dataDoc.data); + return ImageLabelBoxData.Instance._docs; + } @observable _displayImageInformation: boolean = false; constructor(props: any) { super(props); makeObservable(this); - ImageLabelBox.Instance = this; ring.register(); + ImageLabelBox.Instance = this; console.log('Image Box Has Been Initialized'); } + // ImageLabelBox.Instance.setData() /** * This method is called when the SearchBox component is first mounted. When the user opens * the search panel, the search input box is automatically selected. This allows the user to * type in the search input box immediately, without needing clicking on it first. */ componentDidMount() { - // if (this._inputRef.current) { - // this._inputRef.current.focus(); - // } + this.classifyImagesInBox(); + reaction( + () => this._selectedImages, + () => this.classifyImagesInBox() + ); } - @action - addLabel = (label: string) => { - label = label.toUpperCase().trim(); - if (label.length > 0) { - if (!this._labelGroups.includes(label)) { - this._labelGroups = [...this._labelGroups, label]; - } - } - }; - - @action - removeLabel = (label: string) => { - const labelUp = label.toUpperCase(); - this._labelGroups = this._labelGroups.filter(group => group !== labelUp); - }; - @action groupImages = () => { MarqueeOptionsMenu.Instance.groupImages(); - this._labelGroups = []; MainView.Instance.closeFlyout(); }; @action startLoading = () => { this._loading = true; + console.log('Start loading has been called!'); }; @action @@ -85,8 +118,38 @@ export class ImageLabelBox extends ViewBoxBaseComponent() { }; @action - setSelectedImages = (selected: Doc[]) => { - this._selectedImages = selected; + toggleDisplayInformation = () => { + this._displayImageInformation = !this._displayImageInformation; + }; + + onInputChange = action((e: React.ChangeEvent) => { + this._currentLabel = e.target.value; + }); + + classifyImagesInBox = async () => { + this.startLoading(); + + const imageInfos = this._selectedImages.map(async doc => { + if (!doc[DocData].data_labels) { + const [name, type] = ImageCast(doc[Doc.LayoutFieldKey(doc)]).url.href.split('.'); + return CollectionCardView.imageUrlToBase64(`${name}_o.${type}`).then(hrefBase64 => + !hrefBase64 ? undefined : + gptImageLabel(hrefBase64).then(labels => + Promise.all(labels.split('\n').map(label => gptGetEmbedding(label))).then(embeddings => + ({ doc, embeddings, labels }))) ); // prettier-ignore + } + }); // Converts the images into a Base64 format, afterwhich the information is sent to GPT to label them. + + (await Promise.all(imageInfos)).forEach(imageInfo => { + if (imageInfo && imageInfo.embeddings && Array.isArray(imageInfo.embeddings)) { + imageInfo.doc[DocData].data_labels = imageInfo.labels; + numberRange(3).forEach(n => { + imageInfo.doc[`data_labels_embedding_${n + 1}`] = new List(imageInfo.embeddings[n]); + }); + } + }); // Add the labels as fields to each image. + + this.endLoading(); }; render() { @@ -98,14 +161,20 @@ export class ImageLabelBox extends ViewBoxBaseComponent() { ); } + if (this._selectedImages.length === 0) { + return ( +
+

In order to classify and sort images, marquee select the desired images and press the 'Classify and Sort Images' button. Then, add the desired groups for the images to be put in.

+
+ ); + } + return (
{ - this._displayImageInformation = !this._displayImageInformation; - }} + onPointerDown={this.toggleDisplayInformation} icon={this._displayImageInformation ? : } color={MarqueeOptionsMenu.Instance.userColor} style={{ width: '19px' }} @@ -113,6 +182,7 @@ export class ImageLabelBox extends ViewBoxBaseComponent() { { // e.key === 'Enter' ? this.submitSearch() : null; // e.stopPropagation(); @@ -129,8 +199,7 @@ export class ImageLabelBox extends ViewBoxBaseComponent() { tooltip={'Add a label'} onPointerDown={() => { const input = document.getElementById('new-label') as HTMLInputElement; - const newLabel = input.value; - this.addLabel(newLabel); + ImageLabelBoxData.Instance.addLabel(this._currentLabel); this._currentLabel = ''; input.value = ''; }} @@ -144,12 +213,12 @@ export class ImageLabelBox extends ViewBoxBaseComponent() {
{this._labelGroups.map(group => { return ( -
+

{group}

{ - this.removeLabel(group); + ImageLabelBoxData.Instance.removeLabel(group); }} icon={'x'} color={MarqueeOptionsMenu.Instance.userColor} @@ -161,15 +230,21 @@ export class ImageLabelBox extends ViewBoxBaseComponent() {
{this._displayImageInformation ? ( -
+
{this._selectedImages.map(doc => { const [name, type] = ImageCast(doc[Doc.LayoutFieldKey(doc)]).url.href.split('.'); return ( -
+
- {(doc[DocData].data_labels as string).split('\n').map(label => { - return
{label}
; - })} +
+ {(doc[DocData].data_labels as string).split('\n').map(label => { + return ( +
+ {label} +
+ ); + })} +
); })} diff --git a/src/client/views/collections/collectionFreeForm/MarqueeView.tsx b/src/client/views/collections/collectionFreeForm/MarqueeView.tsx index f98883bff..81f2a94c1 100644 --- a/src/client/views/collections/collectionFreeForm/MarqueeView.tsx +++ b/src/client/views/collections/collectionFreeForm/MarqueeView.tsx @@ -37,7 +37,7 @@ import { ImageLabelHandler } from './ImageLabelHandler'; import { MarqueeOptionsMenu } from './MarqueeOptionsMenu'; import './MarqueeView.scss'; import { MainView } from '../../MainView'; -import { ImageLabelBox } from './ImageLabelBox'; +import { ImageLabelBox, ImageLabelBoxData } from './ImageLabelBox'; import { SearchBox } from '../../search/SearchBox'; interface MarqueeViewProps { @@ -442,44 +442,12 @@ export class MarqueeView extends ObservableReactComponent { - if (e) { - const groupButton = DocListCast(Doc.MyLeftSidebarMenu.data).find(d => d.target === Doc.MyImageGrouper); - if (groupButton) { - MainView.Instance.expandFlyout(groupButton); - while (!ImageLabelBox.Instance) { - await new Promise(resolve => setTimeout(resolve, 1000)).then(() => { - console.log('Waiting for Image Label Box'); - }); - } - ImageLabelBox.Instance.startLoading(); - } + const groupButton = DocListCast(Doc.MyLeftSidebarMenu.data).find(d => d.target === Doc.MyImageGrouper); + if (groupButton) { + this._selectedDocs = this.marqueeSelect(false, DocumentType.IMG); + ImageLabelBoxData.Instance.setData(this._selectedDocs); + MainView.Instance.expandFlyout(groupButton); } - - this._selectedDocs = this.marqueeSelect(false, DocumentType.IMG); // Get the selected documents from the marquee select. - ImageLabelBox.Instance!.setSelectedImages(this._selectedDocs); - - const imageInfos = this._selectedDocs.map(async doc => { - if (!doc[DocData].data_labels) { - const [name, type] = ImageCast(doc[Doc.LayoutFieldKey(doc)]).url.href.split('.'); - return CollectionCardView.imageUrlToBase64(`${name}_o.${type}`).then(hrefBase64 => - !hrefBase64 ? undefined : - gptImageLabel(hrefBase64).then(labels => - Promise.all(labels.split('\n').map(label => gptGetEmbedding(label))).then(embeddings => - ({ doc, embeddings, labels }))) ); // prettier-ignore - } - }); // Converts the images into a Base64 format, afterwhich the information is sent to GPT to label them. - - (await Promise.all(imageInfos)).forEach(imageInfo => { - if (imageInfo && imageInfo.embeddings && Array.isArray(imageInfo.embeddings)) { - imageInfo.doc[DocData].data_labels = imageInfo.labels; - numberRange(3).forEach(n => { - imageInfo.doc[`data_labels_embedding_${n + 1}`] = new List(imageInfo.embeddings[n]); - }); - } - }); // Add the labels as fields to each image. - - ImageLabelBox.Instance!.endLoading(); - console.log(this._selectedDocs); }); /** @@ -496,7 +464,7 @@ export class MarqueeView extends ObservableReactComponent { const embedLists = numberRange(3).map(n => Array.from(NumListCast(doc[`data_labels_embedding_${n + 1}`]))); - const bestEmbedScore = (embedding: Opt) => Math.max(...embedLists.map(l => (embedding && similarity(Array.from(embedding), l)) || 0)); + const bestEmbedScore = (embedding: Opt) => Math.max(...embedLists.map((l, index) => (embedding && (1 - index * 0.1) * similarity(Array.from(embedding), l)!) || 0)); const {label: mostSimilarLabelCollect} = labelGroups.map(label => ({ label, similarityScore: bestEmbedScore(labelToEmbedding.get(label)) })) .reduce((prev, cur) => cur.similarityScore < 0.3 || cur.similarityScore <= prev.similarityScore ? prev: cur, -- cgit v1.2.3-70-g09d2 From 8aee62b8623e23f6478960291857ee47f50f9aaf Mon Sep 17 00:00:00 2001 From: IEatChili Date: Thu, 13 Jun 2024 16:28:24 -0400 Subject: feat: more ui updates --- .../collectionFreeForm/ImageLabelBox.tsx | 81 +++++++++++++++++++--- .../collections/collectionFreeForm/MarqueeView.tsx | 22 +----- src/client/views/nodes/ImageBox.tsx | 4 ++ 3 files changed, 76 insertions(+), 31 deletions(-) (limited to 'src') diff --git a/src/client/views/collections/collectionFreeForm/ImageLabelBox.tsx b/src/client/views/collections/collectionFreeForm/ImageLabelBox.tsx index f5530ccc4..571a4504f 100644 --- a/src/client/views/collections/collectionFreeForm/ImageLabelBox.tsx +++ b/src/client/views/collections/collectionFreeForm/ImageLabelBox.tsx @@ -3,9 +3,9 @@ import { Colors, IconButton } from 'browndash-components'; import { action, computed, makeObservable, observable, reaction } from 'mobx'; import { observer } from 'mobx-react'; import React from 'react'; -import { Doc } from '../../../../fields/Doc'; +import { Doc, NumListCast, Opt } from '../../../../fields/Doc'; import { Docs } from '../../../documents/Documents'; -import { DocumentType } from '../../../documents/DocumentTypes'; +import { CollectionViewType, DocumentType } from '../../../documents/DocumentTypes'; import { ViewBoxBaseComponent } from '../../DocComponent'; import { FieldView, FieldViewProps } from '../../nodes/FieldView'; import { MarqueeOptionsMenu } from './MarqueeOptionsMenu'; @@ -21,6 +21,9 @@ import { CollectionCardView } from '../CollectionCardDeckView'; import { gptGetEmbedding, gptImageLabel } from '../../../apis/gpt/GPT'; import { numberRange, Utils } from '../../../../Utils'; import { List } from '../../../../fields/List'; +import { DragManager } from '../../../util/DragManager'; +import { OpenWhere } from '../../nodes/OpenWhere'; +import similarity from 'compute-cosine-similarity'; export class ImageLabelBoxData { static _instance: ImageLabelBoxData; @@ -63,11 +66,26 @@ export class ImageLabelBox extends ViewBoxBaseComponent() { return FieldView.LayoutString(ImageLabelBox, fieldKey); } + private _dropDisposer?: DragManager.DragDropDisposer; public static Instance: ImageLabelBox; private _inputRef = React.createRef(); @observable _loading: boolean = false; private _currentLabel: string = ''; + protected createDropTarget = (ele: HTMLDivElement) => { + this._dropDisposer?.(); + ele && (this._dropDisposer = DragManager.MakeDropTarget(ele, this.onInternalDrop.bind(this), this.layoutDoc)); + }; + + protected onInternalDrop(e: Event, de: DragManager.DropEvent): boolean { + const { docDragData } = de.complete; + if (docDragData) { + ImageLabelBoxData.Instance.setData(ImageLabelBoxData.Instance._docs.concat(docDragData.droppedDocuments)); + return false; + } + return false; + } + @computed get _labelGroups() { return ImageLabelBoxData.Instance._labelGroups; } @@ -102,7 +120,7 @@ export class ImageLabelBox extends ViewBoxBaseComponent() { @action groupImages = () => { - MarqueeOptionsMenu.Instance.groupImages(); + this.groupImagesInBox(); MainView.Instance.closeFlyout(); }; @@ -122,6 +140,14 @@ export class ImageLabelBox extends ViewBoxBaseComponent() { this._displayImageInformation = !this._displayImageInformation; }; + @action + submitLabel = () => { + const input = document.getElementById('new-label') as HTMLInputElement; + ImageLabelBoxData.Instance.addLabel(this._currentLabel); + this._currentLabel = ''; + input.value = ''; + }; + onInputChange = action((e: React.ChangeEvent) => { this._currentLabel = e.target.value; }); @@ -143,6 +169,13 @@ export class ImageLabelBox extends ViewBoxBaseComponent() { (await Promise.all(imageInfos)).forEach(imageInfo => { if (imageInfo && imageInfo.embeddings && Array.isArray(imageInfo.embeddings)) { imageInfo.doc[DocData].data_labels = imageInfo.labels; + + const labels = imageInfo.labels.split('\n'); + labels.forEach(label => { + label = label.replace(/^\d+\.\s*/, '').trim(); + imageInfo.doc[DocData][`${label}`] = true; + }); + numberRange(3).forEach(n => { imageInfo.doc[`data_labels_embedding_${n + 1}`] = new List(imageInfo.embeddings[n]); }); @@ -152,6 +185,32 @@ export class ImageLabelBox extends ViewBoxBaseComponent() { this.endLoading(); }; + /** + * Groups images to most similar labels. + */ + groupImagesInBox = action(async () => { + console.log('Calling!'); + const labelToEmbedding = new Map(); + // Create embeddings for the labels. + await Promise.all(this._labelGroups.map(async label => gptGetEmbedding(label).then(labelEmbedding => labelToEmbedding.set(label, labelEmbedding)))); + + // For each image, loop through the labels, and calculate similarity. Associate it with the + // most similar one. + this._selectedImages.forEach(doc => { + const embedLists = numberRange(3).map(n => Array.from(NumListCast(doc[`data_labels_embedding_${n + 1}`]))); + const bestEmbedScore = (embedding: Opt) => Math.max(...embedLists.map((l, index) => (embedding && (1 - index * 0.1) * similarity(Array.from(embedding), l)!) || 0)); + const {label: mostSimilarLabelCollect} = + this._labelGroups.map(label => ({ label, similarityScore: bestEmbedScore(labelToEmbedding.get(label)) })) + .reduce((prev, cur) => cur.similarityScore < 0.3 || cur.similarityScore <= prev.similarityScore ? prev: cur, + { label: '', similarityScore: 0, }); // prettier-ignore + doc[DocData].data_label = mostSimilarLabelCollect; // The label most similar to the image's contents. + }); + + if (this._selectedImages) { + MarqueeOptionsMenu.Instance.groupImages(); + } + }); + render() { if (this._loading) { return ( @@ -163,14 +222,14 @@ export class ImageLabelBox extends ViewBoxBaseComponent() { if (this._selectedImages.length === 0) { return ( -
+
this.createDropTarget(ele)}>

In order to classify and sort images, marquee select the desired images and press the 'Classify and Sort Images' button. Then, add the desired groups for the images to be put in.

); } return ( -
+
this.createDropTarget(ele)}>
() { defaultValue="" autoComplete="off" onChange={this.onInputChange} - // onKeyDown={e => { - // e.key === 'Enter' ? this.submitSearch() : null; - // e.stopPropagation(); - // }} + onKeyDown={e => { + e.key === 'Enter' ? this.submitLabel() : null; + e.stopPropagation(); + }} type="text" - placeholder="Input labels for image groupings..." + placeholder="Input groups for images to be put into..." aria-label="label-input" id="new-label" className="searchBox-input" @@ -234,7 +293,7 @@ export class ImageLabelBox extends ViewBoxBaseComponent() { {this._selectedImages.map(doc => { const [name, type] = ImageCast(doc[Doc.LayoutFieldKey(doc)]).url.href.split('.'); return ( -
+
this._props.addDocTab(doc, OpenWhere.addRightKeyvalue)}>
{(doc[DocData].data_labels as string).split('\n').map(label => { diff --git a/src/client/views/collections/collectionFreeForm/MarqueeView.tsx b/src/client/views/collections/collectionFreeForm/MarqueeView.tsx index 81f2a94c1..f03a9d62d 100644 --- a/src/client/views/collections/collectionFreeForm/MarqueeView.tsx +++ b/src/client/views/collections/collectionFreeForm/MarqueeView.tsx @@ -455,26 +455,8 @@ export class MarqueeView extends ObservableReactComponent { - const labelGroups = ImageLabelBox.Instance!._labelGroups; - const labelToEmbedding = new Map(); - // Create embeddings for the labels. - await Promise.all(labelGroups.map(async label => gptGetEmbedding(label).then(labelEmbedding => labelToEmbedding.set(label, labelEmbedding)))); - - // For each image, loop through the labels, and calculate similarity. Associate it with the - // most similar one. - this._selectedDocs.forEach(doc => { - const embedLists = numberRange(3).map(n => Array.from(NumListCast(doc[`data_labels_embedding_${n + 1}`]))); - const bestEmbedScore = (embedding: Opt) => Math.max(...embedLists.map((l, index) => (embedding && (1 - index * 0.1) * similarity(Array.from(embedding), l)!) || 0)); - const {label: mostSimilarLabelCollect} = - labelGroups.map(label => ({ label, similarityScore: bestEmbedScore(labelToEmbedding.get(label)) })) - .reduce((prev, cur) => cur.similarityScore < 0.3 || cur.similarityScore <= prev.similarityScore ? prev: cur, - { label: '', similarityScore: 0, }); // prettier-ignore - doc[DocData].data_label = mostSimilarLabelCollect; // The label most similar to the image's contents. - }); - if (this._selectedDocs) { - this._props.Document._type_collection = CollectionViewType.Time; // Change the collection view to a Time view. - this._props.Document.pivotField = 'data_label'; // Sets the pivot to be the 'data_label'. - } + this._props.Document._type_collection = CollectionViewType.Time; // Change the collection view to a Time view. + this._props.Document.pivotField = 'data_label'; // Sets the pivot to be the 'data_label'. }); @undoBatch diff --git a/src/client/views/nodes/ImageBox.tsx b/src/client/views/nodes/ImageBox.tsx index e4b3a1b9b..3da878a4f 100644 --- a/src/client/views/nodes/ImageBox.tsx +++ b/src/client/views/nodes/ImageBox.tsx @@ -173,6 +173,10 @@ export class ImageBox extends ViewBoxAnnotatableComponent() { Doc.SetNativeHeight(this.dataDoc, Doc.NativeHeight(targetDoc), this.fieldKey); } } + const layoutDoc = de.complete.docDragData?.draggedDocuments[0]; + const targetField = Doc.LayoutFieldKey(layoutDoc); + const targetDoc = layoutDoc[DocData]; + console.log(targetDoc[targetField]); added === false && e.preventDefault(); added !== undefined && e.stopPropagation(); return added; -- cgit v1.2.3-70-g09d2 From 376ff1626b24cbac12b27ad072690424549f05c7 Mon Sep 17 00:00:00 2001 From: IEatChili Date: Tue, 18 Jun 2024 14:33:47 -0400 Subject: feat: added view of labels on docs in freeform --- src/client/views/KeywordBox.tsx | 69 +++++++++++++++++++ src/client/views/StyleProvider.scss | 13 ++++ src/client/views/StyleProvider.tsx | 9 +++ .../collectionFreeForm/ImageLabelBox.tsx | 79 ++++++++++++++++------ src/client/views/nodes/DocumentView.tsx | 2 +- src/client/views/nodes/ImageBox.tsx | 9 +++ 6 files changed, 161 insertions(+), 20 deletions(-) create mode 100644 src/client/views/KeywordBox.tsx (limited to 'src') diff --git a/src/client/views/KeywordBox.tsx b/src/client/views/KeywordBox.tsx new file mode 100644 index 000000000..3faddeb64 --- /dev/null +++ b/src/client/views/KeywordBox.tsx @@ -0,0 +1,69 @@ +import { action, makeObservable } from 'mobx'; +import { observer } from 'mobx-react'; +import React from 'react'; +import { Doc } from '../../fields/Doc'; +import { DocData } from '../../fields/DocSymbols'; +import { List } from '../../fields/List'; +import { ObservableReactComponent } from './ObservableReactComponent'; + +interface KeywordBoxProps { + _doc: Doc; + _isEditing: boolean; +} + +@observer +export class KeywordBox extends ObservableReactComponent { + constructor(props: any) { + super(props); + makeObservable(this); + } + + @action + setToEditing = () => { + this._props._isEditing = true; + }; + + @action + setToView = () => { + this._props._isEditing = false; + }; + + submitLabel = () => {}; + + onInputChange = () => {}; + + render() { + const keywordsList = this._props._doc![DocData].data_labels; + return ( +
+ {(keywordsList as List).map(label => { + return ( +
+ {label} +
+ ); + })} + {this._props._isEditing ? ( +
+ { + e.key === 'Enter' ? this.submitLabel() : null; + e.stopPropagation(); + }} + type="text" + placeholder="Input keywords for document..." + aria-label="keyword-input" + className="keyword-input" + style={{ width: '100%', borderRadius: '5px' }} + /> +
+ ) : ( +
+ )} +
+ ); + } +} diff --git a/src/client/views/StyleProvider.scss b/src/client/views/StyleProvider.scss index ce00f6101..1e2af9a3a 100644 --- a/src/client/views/StyleProvider.scss +++ b/src/client/views/StyleProvider.scss @@ -53,3 +53,16 @@ .styleProvider-treeView-icon { opacity: 0; } + +.keywords-container { + display: flex; + flex-wrap: wrap; +} + +.keyword { + padding: 5px 10px; + background-color: lightblue; + border: 1px solid black; + border-radius: 5px; + white-space: nowrap; +} diff --git a/src/client/views/StyleProvider.tsx b/src/client/views/StyleProvider.tsx index b7f8a3170..fb509516a 100644 --- a/src/client/views/StyleProvider.tsx +++ b/src/client/views/StyleProvider.tsx @@ -12,7 +12,9 @@ import { BsArrowDown, BsArrowDownUp, BsArrowUp } from 'react-icons/bs'; import { FaFilter } from 'react-icons/fa'; import { ClientUtils, DashColor, lightOrDark } from '../../ClientUtils'; import { Doc, Opt, StrListCast } from '../../fields/Doc'; +import { DocData } from '../../fields/DocSymbols'; import { Id } from '../../fields/FieldSymbols'; +import { List } from '../../fields/List'; import { ScriptField } from '../../fields/ScriptField'; import { BoolCast, Cast, DocCast, ImageCast, NumCast, ScriptCast, StrCast } from '../../fields/Types'; import { AudioAnnoState } from '../../server/SharedMediaTypes'; @@ -23,6 +25,7 @@ import { SnappingManager } from '../util/SnappingManager'; import { undoBatch, UndoManager } from '../util/UndoManager'; import { TreeSort } from './collections/TreeSort'; import { Colors } from './global/globalEnums'; +import { KeywordBox } from './KeywordBox'; import { DocumentView, DocumentViewProps } from './nodes/DocumentView'; import { FieldViewProps } from './nodes/FieldView'; import { StyleProp } from './StyleProp'; @@ -367,12 +370,18 @@ export function DefaultStyleProvider(doc: Opt, props: Opt ); }; + const keywords = () => { + if (doc && doc![DocData].data_labels && doc![DocData].showLabels) { + return () + } + } return ( <> {paint()} {lock()} {filter()} {audio()} + {keywords()} ); } diff --git a/src/client/views/collections/collectionFreeForm/ImageLabelBox.tsx b/src/client/views/collections/collectionFreeForm/ImageLabelBox.tsx index 571a4504f..cfb81e1a0 100644 --- a/src/client/views/collections/collectionFreeForm/ImageLabelBox.tsx +++ b/src/client/views/collections/collectionFreeForm/ImageLabelBox.tsx @@ -5,7 +5,7 @@ import { observer } from 'mobx-react'; import React from 'react'; import { Doc, NumListCast, Opt } from '../../../../fields/Doc'; import { Docs } from '../../../documents/Documents'; -import { CollectionViewType, DocumentType } from '../../../documents/DocumentTypes'; +import { DocumentType } from '../../../documents/DocumentTypes'; import { ViewBoxBaseComponent } from '../../DocComponent'; import { FieldView, FieldViewProps } from '../../nodes/FieldView'; import { MarqueeOptionsMenu } from './MarqueeOptionsMenu'; @@ -24,6 +24,9 @@ import { List } from '../../../../fields/List'; import { DragManager } from '../../../util/DragManager'; import { OpenWhere } from '../../nodes/OpenWhere'; import similarity from 'compute-cosine-similarity'; +import { DocumentView } from '../../nodes/DocumentView'; + +export class ImageInformationItem {} export class ImageLabelBoxData { static _instance: ImageLabelBoxData; @@ -101,7 +104,6 @@ export class ImageLabelBox extends ViewBoxBaseComponent() { makeObservable(this); ring.register(); ImageLabelBox.Instance = this; - console.log('Image Box Has Been Initialized'); } // ImageLabelBox.Instance.setData() @@ -127,7 +129,6 @@ export class ImageLabelBox extends ViewBoxBaseComponent() { @action startLoading = () => { this._loading = true; - console.log('Start loading has been called!'); }; @action @@ -138,6 +139,11 @@ export class ImageLabelBox extends ViewBoxBaseComponent() { @action toggleDisplayInformation = () => { this._displayImageInformation = !this._displayImageInformation; + if (this._displayImageInformation) { + this._selectedImages.forEach(doc => (doc[DocData].showLabels = true)); + } else { + this._selectedImages.forEach(doc => (doc[DocData].showLabels = false)); + } }; @action @@ -155,33 +161,58 @@ export class ImageLabelBox extends ViewBoxBaseComponent() { classifyImagesInBox = async () => { this.startLoading(); + // const imageInfos = this._selectedImages.map(async doc => { + // if (!doc[DocData].data_labels) { + // const [name, type] = ImageCast(doc[Doc.LayoutFieldKey(doc)]).url.href.split('.'); + // return CollectionCardView.imageUrlToBase64(`${name}_o.${type}`).then(hrefBase64 => + // !hrefBase64 ? undefined : + // gptImageLabel(hrefBase64).then(labels => + // Promise.all(labels.split('\n').map(label => gptGetEmbedding(label))).then(embeddings => + // ({ doc, embeddings, labels }))) ); // prettier-ignore + // } + // }); // Converts the images into a Base64 format, afterwhich the information is sent to GPT to label them. + const imageInfos = this._selectedImages.map(async doc => { if (!doc[DocData].data_labels) { const [name, type] = ImageCast(doc[Doc.LayoutFieldKey(doc)]).url.href.split('.'); return CollectionCardView.imageUrlToBase64(`${name}_o.${type}`).then(hrefBase64 => !hrefBase64 ? undefined : gptImageLabel(hrefBase64).then(labels => - Promise.all(labels.split('\n').map(label => gptGetEmbedding(label))).then(embeddings => - ({ doc, embeddings, labels }))) ); // prettier-ignore + + ({ doc, labels }))) ; // prettier-ignore } - }); // Converts the images into a Base64 format, afterwhich the information is sent to GPT to label them. + }); (await Promise.all(imageInfos)).forEach(imageInfo => { - if (imageInfo && imageInfo.embeddings && Array.isArray(imageInfo.embeddings)) { - imageInfo.doc[DocData].data_labels = imageInfo.labels; + if (imageInfo) { + imageInfo.doc[DocData].data_labels = new List(); const labels = imageInfo.labels.split('\n'); labels.forEach(label => { - label = label.replace(/^\d+\.\s*/, '').trim(); + label = label.replace(/^\d+\.\s*|-|\*/, '').trim(); imageInfo.doc[DocData][`${label}`] = true; - }); - - numberRange(3).forEach(n => { - imageInfo.doc[`data_labels_embedding_${n + 1}`] = new List(imageInfo.embeddings[n]); + (imageInfo.doc[DocData].data_labels as List).push(label); }); } }); // Add the labels as fields to each image. + // (await Promise.all(imageInfos)).forEach(imageInfo => { + // if (imageInfo && imageInfo.embeddings && Array.isArray(imageInfo.embeddings)) { + // imageInfo.doc[DocData].data_labels = new List(); + + // const labels = imageInfo.labels.split('\n'); + // labels.forEach(label => { + // label = label.replace(/^\d+\.\s*|-|\*/, '').trim(); + // imageInfo.doc[DocData][`${label}`] = true; + // (imageInfo.doc[DocData].data_labels as List).push(label); + // }); + + // numberRange(5).forEach(n => { + // imageInfo.doc[`data_labels_embedding_${n + 1}`] = new List(imageInfo.embeddings[n]); + // }); + // } + // }); // Add the labels as fields to each image. + this.endLoading(); }; @@ -189,7 +220,13 @@ export class ImageLabelBox extends ViewBoxBaseComponent() { * Groups images to most similar labels. */ groupImagesInBox = action(async () => { - console.log('Calling!'); + this._selectedImages.forEach(doc => { + (doc[DocData].data_labels as List).forEach(async (label, index) => { + const embedding = await gptGetEmbedding(label); + doc[`data_labels_embedding_${index + 1}`] = new List(embedding); + }); + }); + const labelToEmbedding = new Map(); // Create embeddings for the labels. await Promise.all(this._labelGroups.map(async label => gptGetEmbedding(label).then(labelEmbedding => labelToEmbedding.set(label, labelEmbedding)))); @@ -197,7 +234,7 @@ export class ImageLabelBox extends ViewBoxBaseComponent() { // For each image, loop through the labels, and calculate similarity. Associate it with the // most similar one. this._selectedImages.forEach(doc => { - const embedLists = numberRange(3).map(n => Array.from(NumListCast(doc[`data_labels_embedding_${n + 1}`]))); + const embedLists = numberRange(5).map(n => Array.from(NumListCast(doc[`data_labels_embedding_${n + 1}`]))); const bestEmbedScore = (embedding: Opt) => Math.max(...embedLists.map((l, index) => (embedding && (1 - index * 0.1) * similarity(Array.from(embedding), l)!) || 0)); const {label: mostSimilarLabelCollect} = this._labelGroups.map(label => ({ label, similarityScore: bestEmbedScore(labelToEmbedding.get(label)) })) @@ -293,10 +330,14 @@ export class ImageLabelBox extends ViewBoxBaseComponent() { {this._selectedImages.map(doc => { const [name, type] = ImageCast(doc[Doc.LayoutFieldKey(doc)]).url.href.split('.'); return ( -
this._props.addDocTab(doc, OpenWhere.addRightKeyvalue)}> - -
- {(doc[DocData].data_labels as string).split('\n').map(label => { +
+ { + await DocumentView.showDocument(doc, { willZoomCentered: true }); + }}> +
this._props.addDocTab(doc, OpenWhere.addRightKeyvalue)}> + {(doc[DocData].data_labels as List).map(label => { return (
{label} diff --git a/src/client/views/nodes/DocumentView.tsx b/src/client/views/nodes/DocumentView.tsx index 7a1f94948..9ff96c692 100644 --- a/src/client/views/nodes/DocumentView.tsx +++ b/src/client/views/nodes/DocumentView.tsx @@ -573,7 +573,7 @@ export class DocumentViewInternal extends DocComponent DocumentView.SetLightboxDoc(this.Document), icon: 'external-link-alt' }); } - appearanceItems.push({ description: 'Pin', event: () => this._props.pinToPres(this.Document, {}), icon: 'eye' }); + appearanceItems.push({ description: 'Pin', event: () => this._props.pinToPres(this.Document, {}), icon: 'map-pin' }); if (this.Document._layout_isFlashcard) { appearanceItems.push({ description: 'Create ChatCard', event: () => this.askGPT(), icon: 'id-card' }); } diff --git a/src/client/views/nodes/ImageBox.tsx b/src/client/views/nodes/ImageBox.tsx index 3da878a4f..fb90f907f 100644 --- a/src/client/views/nodes/ImageBox.tsx +++ b/src/client/views/nodes/ImageBox.tsx @@ -1,5 +1,6 @@ import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; import { Tooltip } from '@mui/material'; +import zIndex from '@mui/material/styles/zIndex'; import { Colors } from 'browndash-components'; import { action, computed, IReactionDisposer, makeObservable, observable, ObservableMap, reaction } from 'mobx'; import { observer } from 'mobx-react'; @@ -10,6 +11,7 @@ import { Doc, DocListCast, Opt } from '../../../fields/Doc'; import { DocData } from '../../../fields/DocSymbols'; import { Id } from '../../../fields/FieldSymbols'; import { InkTool } from '../../../fields/InkField'; +import { List } from '../../../fields/List'; import { ObjectField } from '../../../fields/ObjectField'; import { Cast, ImageCast, NumCast, StrCast } from '../../../fields/Types'; import { ImageField } from '../../../fields/URLField'; @@ -281,6 +283,13 @@ export class ImageBox extends ViewBoxAnnotatableComponent() { }), icon: 'pencil-alt', }); + funcs.push({ + description: 'Toggle Keywords', + event: () => { + this.Document[DocData].showLabels = !this.Document[DocData].showLabels; + }, + icon: 'eye', + }); ContextMenu.Instance?.addItem({ description: 'Options...', subitems: funcs, icon: 'asterisk' }); } }; -- cgit v1.2.3-70-g09d2 From 3190f1eb07a47a5e1ccdd20e346b47094118292d Mon Sep 17 00:00:00 2001 From: IEatChili Date: Wed, 26 Jun 2024 13:58:20 -0400 Subject: feat: worked more on keyword input for docs --- src/client/views/DocumentButtonBar.tsx | 19 +++ src/client/views/DocumentDecorations.tsx | 1 + src/client/views/KeywordBox.tsx | 168 +++++++++++++++++---- src/client/views/StyleProvider.scss | 31 +++- src/client/views/StyleProvider.tsx | 6 +- .../collectionFreeForm/ImageLabelBox.tsx | 53 ++----- .../collections/collectionFreeForm/MarqueeView.tsx | 28 +++- src/client/views/nodes/ImageBox.tsx | 7 - 8 files changed, 233 insertions(+), 80 deletions(-) (limited to 'src') diff --git a/src/client/views/DocumentButtonBar.tsx b/src/client/views/DocumentButtonBar.tsx index 487868169..a75c7098c 100644 --- a/src/client/views/DocumentButtonBar.tsx +++ b/src/client/views/DocumentButtonBar.tsx @@ -31,6 +31,7 @@ import { DocumentLinksButton } from './nodes/DocumentLinksButton'; import { DocumentView } from './nodes/DocumentView'; import { OpenWhere } from './nodes/OpenWhere'; import { DashFieldView } from './nodes/formattedText/DashFieldView'; +import { DocData } from '../../fields/DocSymbols'; @observer export class DocumentButtonBar extends ObservableReactComponent<{ views: () => (DocumentView | undefined)[]; stack?: any }> { @@ -282,6 +283,23 @@ export class DocumentButtonBar extends ObservableReactComponent<{ views: () => ( ); } + @computed + get keywordButton() { + const targetDoc = this.view0?.Document; + return !targetDoc ? null : ( + Open keyword menu
}> +
{ + targetDoc[DocData].showLabels = !targetDoc[DocData].showLabels; + }}> + +
+ + ); + } + @observable _isRecording = false; _stopFunc: () => void = emptyFunction; @computed @@ -452,6 +470,7 @@ export class DocumentButtonBar extends ObservableReactComponent<{ views: () => (
{this.pinButton}
{this.recordButton}
{this.calendarButton}
+
{this.keywordButton}
{!Doc.UserDoc().documentLinksButton_fullMenu ? null :
{this.shareButton}
}
{this.menuButton}
diff --git a/src/client/views/DocumentDecorations.tsx b/src/client/views/DocumentDecorations.tsx index 93c3e3338..20bf8fd9f 100644 --- a/src/client/views/DocumentDecorations.tsx +++ b/src/client/views/DocumentDecorations.tsx @@ -88,6 +88,7 @@ export class DocumentDecorations extends ObservableReactComponent center.x+x || this.Bounds.r < center.x+x || this.Bounds.y > center.y+y || this.Bounds.b < center.y+y ))); + })); // prettier-ignore } diff --git a/src/client/views/KeywordBox.tsx b/src/client/views/KeywordBox.tsx index 3faddeb64..8c69f446d 100644 --- a/src/client/views/KeywordBox.tsx +++ b/src/client/views/KeywordBox.tsx @@ -1,64 +1,170 @@ -import { action, makeObservable } from 'mobx'; +import { Colors, IconButton } from 'browndash-components'; +import { action, computed, makeObservable, observable, reaction } from 'mobx'; import { observer } from 'mobx-react'; import React from 'react'; import { Doc } from '../../fields/Doc'; import { DocData } from '../../fields/DocSymbols'; import { List } from '../../fields/List'; +import { DragManager, SetupDrag } from '../util/DragManager'; +import { SnappingManager } from '../util/SnappingManager'; +import { DocumentView } from './nodes/DocumentView'; import { ObservableReactComponent } from './ObservableReactComponent'; +interface KeywordItemProps { + doc: Doc; + label: string; + setToEditing: () => void; + isEditing: boolean; +} + +@observer +export class KeywordItem extends ObservableReactComponent { + constructor(props: any) { + super(props); + makeObservable(this); + this.ref = React.createRef(); + } + + private _dropDisposer?: DragManager.DragDropDisposer; + private ref: React.RefObject; + + protected createDropTarget = (ele: HTMLDivElement) => { + this._dropDisposer?.(); + SetupDrag(this.ref, () => undefined); + //ele && (this._dropDisposer = DragManager. (ele, this.onInternalDrop.bind(this), this.layoutDoc)); + //ele && (this._dropDisposer = DragManager.MakeDropTarget(ele, this.onInternalDrop.bind(this), this.layoutDoc)); + }; + + @action + removeLabel = () => { + if (this._props.doc[DocData].data_labels) { + this._props.doc[DocData].data_labels = (this._props.doc[DocData].data_labels as List).filter(label => label !== this._props.label) as List; + } + }; + + render() { + return ( +
{}} ref={this.ref}> + {this._props.label} + {this.props.isEditing && } +
+ ); + } +} + interface KeywordBoxProps { - _doc: Doc; - _isEditing: boolean; + doc: Doc; + isEditing: boolean; } @observer export class KeywordBox extends ObservableReactComponent { + @observable _currentInput: string = ''; + //private disposer: () => void; + constructor(props: any) { super(props); makeObservable(this); } + // componentDidMount(): void { + // reaction( + // () => ({ + // isDragging: SnappingManager.IsDragging, + // selectedDoc: DocumentView.SelectedDocs().lastElement(), + // isEditing: this._props.isEditing, + // }), + // ({ isDragging, selectedDoc, isEditing }) => { + // if (isDragging || selectedDoc !== this._props.doc || !isEditing) { + // this.setToView(); + // } + // } + // ); + // } + + // componentWillUnmount() { + // this.disposer(); + // } + @action setToEditing = () => { - this._props._isEditing = true; + this._props.isEditing = true; }; @action setToView = () => { - this._props._isEditing = false; + this._props.isEditing = false; }; - submitLabel = () => {}; + submitLabel = () => { + if (this._currentInput.trim()) { + if (!this._props.doc[DocData].data_labels) { + this._props.doc[DocData].data_labels = new List(); + } - onInputChange = () => {}; + (this._props.doc![DocData].data_labels! as List).push(this._currentInput.trim()); + this._currentInput = ''; // Clear the input box + } + }; + + @action + onInputChange = (e: React.ChangeEvent) => { + this._currentInput = e.target.value; + }; render() { - const keywordsList = this._props._doc![DocData].data_labels; + const keywordsList = this._props.doc[DocData].data_labels ? this._props.doc[DocData].data_labels : new List(); + const seldoc = DocumentView.SelectedDocs().lastElement(); + if (SnappingManager.IsDragging || !(seldoc === this._props.doc) || !this._props.isEditing) { + setTimeout( + action(() => { + if ((keywordsList as List).length === 0) { + this._props.doc[DocData].showLabels = false; + } + this.setToView(); + }) + ); + } + return ( -
- {(keywordsList as List).map(label => { - return ( -
- {label} +
+
+ {(keywordsList as List).map(label => { + return ; + })} +
+ {this._props.isEditing ? ( +
+
+ { + e.key === 'Enter' ? this.submitLabel() : null; + e.stopPropagation(); + }} + type="text" + placeholder="Input keywords for document..." + aria-label="keyword-input" + className="keyword-input" + style={{ width: '100%', borderRadius: '5px' }} + /> +
+
+ { + if ((keywordsList as List).length === 0) { + this._props.doc[DocData].showLabels = false; + } else { + this.setToView(); + } + }} + icon={'x'} + style={{ width: '4px' }} + />
- ); - })} - {this._props._isEditing ? ( -
- { - e.key === 'Enter' ? this.submitLabel() : null; - e.stopPropagation(); - }} - type="text" - placeholder="Input keywords for document..." - aria-label="keyword-input" - className="keyword-input" - style={{ width: '100%', borderRadius: '5px' }} - />
) : (
diff --git a/src/client/views/StyleProvider.scss b/src/client/views/StyleProvider.scss index 1e2af9a3a..7cc06f922 100644 --- a/src/client/views/StyleProvider.scss +++ b/src/client/views/StyleProvider.scss @@ -57,12 +57,41 @@ .keywords-container { display: flex; flex-wrap: wrap; + flex-direction: column; + padding-bottom: 4px; + border: 1px solid; + border-radius: 4px; +} + +.keywords-list { + display: flex; + flex-wrap: wrap; } .keyword { - padding: 5px 10px; + padding: 5px 5px; background-color: lightblue; border: 1px solid black; border-radius: 5px; white-space: nowrap; + display: flex; + align-items: center; +} + +.keyword-editing-box { + margin-top: 8px; +} + +.keyword-input-box { + // display: flex; + // align-items: center; + // align-content: center; + margin: auto; + align-self: center; + width: 90%; +} + +.keyword-buttons { + margin-left: auto; + width: 10%; } diff --git a/src/client/views/StyleProvider.tsx b/src/client/views/StyleProvider.tsx index fb509516a..f4d73cd1d 100644 --- a/src/client/views/StyleProvider.tsx +++ b/src/client/views/StyleProvider.tsx @@ -371,8 +371,10 @@ export function DefaultStyleProvider(doc: Opt, props: Opt { - if (doc && doc![DocData].data_labels && doc![DocData].showLabels) { - return () + if (doc && doc![DocData].showLabels && (!doc[DocData].data_labels || (doc[DocData].data_labels as List).length === 0)){ + return () + } else if (doc && doc![DocData].data_labels && doc![DocData].showLabels) { + return () } } return ( diff --git a/src/client/views/collections/collectionFreeForm/ImageLabelBox.tsx b/src/client/views/collections/collectionFreeForm/ImageLabelBox.tsx index cfb81e1a0..fec4d3e12 100644 --- a/src/client/views/collections/collectionFreeForm/ImageLabelBox.tsx +++ b/src/client/views/collections/collectionFreeForm/ImageLabelBox.tsx @@ -123,7 +123,6 @@ export class ImageLabelBox extends ViewBoxBaseComponent() { @action groupImages = () => { this.groupImagesInBox(); - MainView.Instance.closeFlyout(); }; @action @@ -161,16 +160,7 @@ export class ImageLabelBox extends ViewBoxBaseComponent() { classifyImagesInBox = async () => { this.startLoading(); - // const imageInfos = this._selectedImages.map(async doc => { - // if (!doc[DocData].data_labels) { - // const [name, type] = ImageCast(doc[Doc.LayoutFieldKey(doc)]).url.href.split('.'); - // return CollectionCardView.imageUrlToBase64(`${name}_o.${type}`).then(hrefBase64 => - // !hrefBase64 ? undefined : - // gptImageLabel(hrefBase64).then(labels => - // Promise.all(labels.split('\n').map(label => gptGetEmbedding(label))).then(embeddings => - // ({ doc, embeddings, labels }))) ); // prettier-ignore - // } - // }); // Converts the images into a Base64 format, afterwhich the information is sent to GPT to label them. + // Converts the images into a Base64 format, afterwhich the information is sent to GPT to label them. const imageInfos = this._selectedImages.map(async doc => { if (!doc[DocData].data_labels) { @@ -178,8 +168,7 @@ export class ImageLabelBox extends ViewBoxBaseComponent() { return CollectionCardView.imageUrlToBase64(`${name}_o.${type}`).then(hrefBase64 => !hrefBase64 ? undefined : gptImageLabel(hrefBase64).then(labels => - - ({ doc, labels }))) ; // prettier-ignore + ({ doc, labels }))) ; // prettier-ignore } }); @@ -194,24 +183,7 @@ export class ImageLabelBox extends ViewBoxBaseComponent() { (imageInfo.doc[DocData].data_labels as List).push(label); }); } - }); // Add the labels as fields to each image. - - // (await Promise.all(imageInfos)).forEach(imageInfo => { - // if (imageInfo && imageInfo.embeddings && Array.isArray(imageInfo.embeddings)) { - // imageInfo.doc[DocData].data_labels = new List(); - - // const labels = imageInfo.labels.split('\n'); - // labels.forEach(label => { - // label = label.replace(/^\d+\.\s*|-|\*/, '').trim(); - // imageInfo.doc[DocData][`${label}`] = true; - // (imageInfo.doc[DocData].data_labels as List).push(label); - // }); - - // numberRange(5).forEach(n => { - // imageInfo.doc[`data_labels_embedding_${n + 1}`] = new List(imageInfo.embeddings[n]); - // }); - // } - // }); // Add the labels as fields to each image. + }); this.endLoading(); }; @@ -220,12 +192,15 @@ export class ImageLabelBox extends ViewBoxBaseComponent() { * Groups images to most similar labels. */ groupImagesInBox = action(async () => { - this._selectedImages.forEach(doc => { - (doc[DocData].data_labels as List).forEach(async (label, index) => { + this.startLoading(); + + for (const doc of this._selectedImages) { + for (let index = 0; index < (doc[DocData].data_labels as List).length; index++) { + const label = (doc[DocData].data_labels as List)[index]; const embedding = await gptGetEmbedding(label); doc[`data_labels_embedding_${index + 1}`] = new List(embedding); - }); - }); + } + } const labelToEmbedding = new Map(); // Create embeddings for the labels. @@ -234,8 +209,8 @@ export class ImageLabelBox extends ViewBoxBaseComponent() { // For each image, loop through the labels, and calculate similarity. Associate it with the // most similar one. this._selectedImages.forEach(doc => { - const embedLists = numberRange(5).map(n => Array.from(NumListCast(doc[`data_labels_embedding_${n + 1}`]))); - const bestEmbedScore = (embedding: Opt) => Math.max(...embedLists.map((l, index) => (embedding && (1 - index * 0.1) * similarity(Array.from(embedding), l)!) || 0)); + const embedLists = numberRange((doc[DocData].data_labels as List).length).map(n => Array.from(NumListCast(doc[`data_labels_embedding_${n + 1}`]))); + const bestEmbedScore = (embedding: Opt) => Math.max(...embedLists.map((l, index) => (embedding && similarity(Array.from(embedding), l)!) || 0)); const {label: mostSimilarLabelCollect} = this._labelGroups.map(label => ({ label, similarityScore: bestEmbedScore(labelToEmbedding.get(label)) })) .reduce((prev, cur) => cur.similarityScore < 0.3 || cur.similarityScore <= prev.similarityScore ? prev: cur, @@ -243,9 +218,13 @@ export class ImageLabelBox extends ViewBoxBaseComponent() { doc[DocData].data_label = mostSimilarLabelCollect; // The label most similar to the image's contents. }); + this.endLoading(); + if (this._selectedImages) { MarqueeOptionsMenu.Instance.groupImages(); } + + MainView.Instance.closeFlyout(); }); render() { diff --git a/src/client/views/collections/collectionFreeForm/MarqueeView.tsx b/src/client/views/collections/collectionFreeForm/MarqueeView.tsx index f03a9d62d..197681f62 100644 --- a/src/client/views/collections/collectionFreeForm/MarqueeView.tsx +++ b/src/client/views/collections/collectionFreeForm/MarqueeView.tsx @@ -455,8 +455,32 @@ export class MarqueeView extends ObservableReactComponent { - this._props.Document._type_collection = CollectionViewType.Time; // Change the collection view to a Time view. - this._props.Document.pivotField = 'data_label'; // Sets the pivot to be the 'data_label'. + const labelGroups: string[] = ImageLabelBoxData.Instance._labelGroups; + const labelToCollection: Map = new Map(); + const selectedImages = ImageLabelBoxData.Instance._docs; + + // Create new collections associated with each label and get the embeddings for the labels. + let x_offset = 0; + for (const label of labelGroups) { + const newCollection = this.getCollection([], undefined, false); + newCollection._freeform_panX = this.Bounds.left + this.Bounds.width / 2; + newCollection._freeform_panY = this.Bounds.top + this.Bounds.height / 2; + console.log(newCollection._x); + labelToCollection.set(label, newCollection); + this._props.addDocument?.(newCollection); + //newCollection._x = (newCollection._x as number) + x_offset; + //x_offset += newCollection._width as number; + } + + for (const doc of selectedImages) { + if (doc[DocData].data_label) { + Doc.AddDocToList(labelToCollection.get(doc[DocData].data_label as string)!, undefined, doc); + this._props.removeDocument?.(doc); + } + } + + //this._props.Document._type_collection = CollectionViewType.Time; // Change the collection view to a Time view. + //this._props.Document.pivotField = 'data_label'; // Sets the pivot to be the 'data_label'. }); @undoBatch diff --git a/src/client/views/nodes/ImageBox.tsx b/src/client/views/nodes/ImageBox.tsx index fb90f907f..1c90fae9e 100644 --- a/src/client/views/nodes/ImageBox.tsx +++ b/src/client/views/nodes/ImageBox.tsx @@ -283,13 +283,6 @@ export class ImageBox extends ViewBoxAnnotatableComponent() { }), icon: 'pencil-alt', }); - funcs.push({ - description: 'Toggle Keywords', - event: () => { - this.Document[DocData].showLabels = !this.Document[DocData].showLabels; - }, - icon: 'eye', - }); ContextMenu.Instance?.addItem({ description: 'Options...', subitems: funcs, icon: 'asterisk' }); } }; -- cgit v1.2.3-70-g09d2 From 7e13e1df797f1d3358f553802527bf42c5574e81 Mon Sep 17 00:00:00 2001 From: IEatChili Date: Thu, 27 Jun 2024 13:55:55 -0400 Subject: feat: added grid of collections when sorting --- src/client/views/KeywordBox.tsx | 2 ++ .../collections/collectionFreeForm/MarqueeView.tsx | 21 ++++++++++++++++++--- 2 files changed, 20 insertions(+), 3 deletions(-) (limited to 'src') diff --git a/src/client/views/KeywordBox.tsx b/src/client/views/KeywordBox.tsx index 8c69f446d..d94f011f4 100644 --- a/src/client/views/KeywordBox.tsx +++ b/src/client/views/KeywordBox.tsx @@ -39,6 +39,7 @@ export class KeywordItem extends ObservableReactComponent { removeLabel = () => { if (this._props.doc[DocData].data_labels) { this._props.doc[DocData].data_labels = (this._props.doc[DocData].data_labels as List).filter(label => label !== this._props.label) as List; + this._props.doc![DocData][`${this._props.label}`] = false; } }; @@ -103,6 +104,7 @@ export class KeywordBox extends ObservableReactComponent { } (this._props.doc![DocData].data_labels! as List).push(this._currentInput.trim()); + this._props.doc![DocData][`${this._currentInput}`] = true; this._currentInput = ''; // Clear the input box } }; diff --git a/src/client/views/collections/collectionFreeForm/MarqueeView.tsx b/src/client/views/collections/collectionFreeForm/MarqueeView.tsx index 197681f62..07e3acb1d 100644 --- a/src/client/views/collections/collectionFreeForm/MarqueeView.tsx +++ b/src/client/views/collections/collectionFreeForm/MarqueeView.tsx @@ -461,17 +461,32 @@ export class MarqueeView extends ObservableReactComponent