diff options
Diffstat (limited to 'src/client/views/collections')
9 files changed, 777 insertions, 139 deletions
diff --git a/src/client/views/collections/CollectionSubView.tsx b/src/client/views/collections/CollectionSubView.tsx index 5782d407e..a6768ab35 100644 --- a/src/client/views/collections/CollectionSubView.tsx +++ b/src/client/views/collections/CollectionSubView.tsx @@ -1,4 +1,4 @@ -import { action, computed, makeObservable, observable } from 'mobx'; +import { action, computed, IReactionDisposer, makeObservable, observable, reaction } from 'mobx'; import * as React from 'react'; import * as rp from 'request-promise'; import { ClientUtils, returnFalse } from '../../../ClientUtils'; @@ -9,7 +9,7 @@ import { Id } from '../../../fields/FieldSymbols'; import { List } from '../../../fields/List'; import { listSpec } from '../../../fields/Schema'; import { ScriptField } from '../../../fields/ScriptField'; -import { BoolCast, Cast, ScriptCast, StrCast } from '../../../fields/Types'; +import { BoolCast, Cast, DocCast, ScriptCast, StrCast } from '../../../fields/Types'; import { WebField } from '../../../fields/URLField'; import { GetEffectiveAcl, TraceMobx } from '../../../fields/util'; import { GestureUtils } from '../../../pen-gestures/GestureUtils'; diff --git a/src/client/views/collections/CollectionView.tsx b/src/client/views/collections/CollectionView.tsx index c9ab5f661..ab93abab6 100644 --- a/src/client/views/collections/CollectionView.tsx +++ b/src/client/views/collections/CollectionView.tsx @@ -17,6 +17,7 @@ import { ViewBoxAnnotatableComponent } from '../DocComponent'; import { FieldView } from '../nodes/FieldView'; import { OpenWhere } from '../nodes/OpenWhere'; import { CollectionCalendarView } from './CollectionCalendarView'; +import { CollectionCardView } from './CollectionCardDeckView'; import { CollectionCarousel3DView } from './CollectionCarousel3DView'; import { CollectionCarouselView } from './CollectionCarouselView'; import { CollectionDockingView } from './CollectionDockingView'; @@ -33,7 +34,6 @@ import { CollectionLinearView } from './collectionLinear'; import { CollectionMulticolumnView } from './collectionMulticolumn/CollectionMulticolumnView'; import { CollectionMultirowView } from './collectionMulticolumn/CollectionMultirowView'; import { CollectionSchemaView } from './collectionSchema/CollectionSchemaView'; -import { CollectionCardView } from './CollectionCardDeckView'; @observer export class CollectionView extends ViewBoxAnnotatableComponent<CollectionViewProps>() { diff --git a/src/client/views/collections/collectionFreeForm/FaceCollectionBox.scss b/src/client/views/collections/collectionFreeForm/FaceCollectionBox.scss new file mode 100644 index 000000000..480d109c8 --- /dev/null +++ b/src/client/views/collections/collectionFreeForm/FaceCollectionBox.scss @@ -0,0 +1,74 @@ +.face-document-item { + background: #555555; + margin-top: 10px; + margin-bottom: 10px; + padding: 10px; + border-radius: 10px; + position: relative; + + h1 { + color: white; + font-size: 24px; + text-align: center; + } + + .face-collection-buttons { + position: absolute; + top: 10px; + right: 10px; + } + + .face-document-image-container { + display: flex; + justify-content: center; + flex-wrap: wrap; + + .image-wrapper { + position: relative; + width: 70px; + height: 70px; + margin: 10px; + display: flex; + align-items: center; // Center vertically + justify-content: center; // Center horizontally + + img { + width: 100%; + height: 100%; + object-fit: cover; // This ensures the image covers the container without stretching + border-radius: 5px; + border: 2px solid white; + transition: border-color 0.4s; + + &:hover { + border-color: orange; // Change this to your desired hover border color + } + } + + .remove-item { + position: absolute; + bottom: -5; + right: -5; + background-color: rgba(0, 0, 0, 0.5); // Optional: to add a background behind the icon for better visibility + border-radius: 30%; + width: 10px; // Adjust size as needed + height: 10px; // Adjust size as needed + display: flex; + align-items: center; + justify-content: center; + } + } + + // img { + // max-width: 60px; + // margin: 10px; + // border-radius: 5px; + // border: 2px solid white; + // transition: 0.4s; + + // &:hover { + // border-color: orange; + // } + // } + } +} diff --git a/src/client/views/collections/collectionFreeForm/FaceCollectionBox.tsx b/src/client/views/collections/collectionFreeForm/FaceCollectionBox.tsx new file mode 100644 index 000000000..d5a2809dc --- /dev/null +++ b/src/client/views/collections/collectionFreeForm/FaceCollectionBox.tsx @@ -0,0 +1,200 @@ +import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; +import { IconButton, Size } from 'browndash-components'; +import * as faceapi from 'face-api.js'; +import { FaceMatcher } from 'face-api.js'; +import 'ldrs/ring'; +import { action, computed, makeObservable, observable } from 'mobx'; +import { observer } from 'mobx-react'; +import React from 'react'; +import { Utils } from '../../../../Utils'; +import { Doc, DocListCast } from '../../../../fields/Doc'; +import { DocData } from '../../../../fields/DocSymbols'; +import { Id } from '../../../../fields/FieldSymbols'; +import { List } from '../../../../fields/List'; +import { listSpec } from '../../../../fields/Schema'; +import { Cast, ImageCast, StrCast } from '../../../../fields/Types'; +import { DocumentType } from '../../../documents/DocumentTypes'; +import { Docs } from '../../../documents/Documents'; +import { DragManager } from '../../../util/DragManager'; +import { SnappingManager } from '../../../util/SnappingManager'; +import { undoable } from '../../../util/UndoManager'; +import { ViewBoxBaseComponent } from '../../DocComponent'; +import { ObservableReactComponent } from '../../ObservableReactComponent'; +import { DocumentView } from '../../nodes/DocumentView'; +import { FieldView, FieldViewProps } from '../../nodes/FieldView'; +import './FaceCollectionBox.scss'; +import { MarqueeOptionsMenu } from './MarqueeOptionsMenu'; +import { FaceRecognitionHandler } from '../../search/FaceRecognitionHandler'; + +interface FaceDocumentProps { + faceDoc: Doc; +} + +/** + * A componenent to visually represent a Face Document. + */ +@observer +export class FaceDocumentItem extends ObservableReactComponent<FaceDocumentProps> { + private _dropDisposer?: DragManager.DragDropDisposer; + + constructor(props: FaceDocumentProps) { + super(props); + makeObservable(this); + } + + @observable _displayImages: boolean = true; + + protected createDropTarget = (ele: HTMLDivElement) => { + this._dropDisposer?.(); + ele && (this._dropDisposer = DragManager.MakeDropTarget(ele, this.onInternalDrop.bind(this), this._props.faceDoc)); + }; + + protected onInternalDrop(e: Event, de: DragManager.DropEvent): boolean { + const { docDragData } = de.complete; + if (docDragData) { + const filteredDocs = docDragData.droppedDocuments.filter(doc => doc.type === DocumentType.IMG); + filteredDocs.forEach(doc => { + // If the current Face Document has no items, and the doc has more than one face descriptor, don't let the user add the document first. + if ((this._props.faceDoc[DocData].face_descriptors as List<List<number>>).length === 0 && (doc[DocData][FaceRecognitionHandler.FacesField(doc)] as List<List<number>>).length > 1) { + alert('Cannot add a document with multiple faces as the first item!'); + } else { + // Loop through the documents' face descriptors. + // Choose the face with the smallest distance to add. + const float32Array = (this._props.faceDoc[DocData].face_descriptors as List<List<number>>).map(faceDescriptor => new Float32Array(Array.from(faceDescriptor))); + const labeledFaceDescriptor = new faceapi.LabeledFaceDescriptors(StrCast(this._props.faceDoc[DocData].face_label), float32Array); + const faceDescriptors: faceapi.LabeledFaceDescriptors[] = [labeledFaceDescriptor]; + + const faceMatcher = new FaceMatcher(faceDescriptors, 1); + let cur_lowest_distance = 1; + let cur_matching_face = new List<number>(); + + (doc[DocData][FaceRecognitionHandler.FacesField(doc)] as List<List<number>>).forEach(face => { + // If the face has the current lowest distance, mark it as such + // Once that lowest distance is found, add the face descriptor to the faceDoc, and add the associated doc + const convered_32_array: Float32Array = new Float32Array(Array.from(face)); + const match = faceMatcher.matchDescriptor(convered_32_array); + + if (match.distance < cur_lowest_distance) { + cur_lowest_distance = match.distance; + cur_matching_face = face; + } + }); + + const faceFieldKey = FaceRecognitionHandler.FaceField(doc, this._props.faceDoc); + if (doc[DocData][faceFieldKey]) { + Cast(doc[DocData][faceFieldKey], listSpec('number'), null).push(cur_matching_face as unknown as number); // items are lists of numbers, not numbers, but type system can't handle that + } else { + doc[DocData][faceFieldKey] = new List<List<number>>([cur_matching_face]); + } + + Doc.AddDocToList(this._props.faceDoc[DocData], 'face_docList', doc); + Cast(this._props.faceDoc[DocData].face_descriptors, listSpec('number'), null).push(cur_matching_face as unknown as number); // items are lists of numbers, not numbers, but type system can't handle that + } + }); + return false; + } + return false; + } + + /** + * Toggles whether a Face Document displays its associated docs. + */ + @action + onDisplayClick() { + this._displayImages = !this._displayImages; + } + + /** + * Deletes a Face Document. + */ + deleteFaceDocument = undoable(() => { + if (Doc.ActiveDashboard) { + Doc.RemoveDocFromList(Doc.ActiveDashboard[DocData], 'faceDocuments', this._props.faceDoc); + } + }, 'remove face'); + + /** + * Deletes a document from a Face Document's associated docs list. + * @param doc + */ + @action + deleteAssociatedDoc = (doc: Doc) => { + this._props.faceDoc[DocData].face_descriptors = new List<List<number>>( + (this._props.faceDoc[DocData].face_descriptors as List<List<number>>).filter(fd => !(doc[DocData][FaceRecognitionHandler.FaceField(doc, this._props.faceDoc)] as List<List<number>>).includes(fd)) + ); + doc[DocData][FaceRecognitionHandler.FaceField(doc, this._props.faceDoc)] = new List<List<number>>(); + Doc.RemoveDocFromList(this._props.faceDoc[DocData], 'face_docList', doc); + }; + + render() { + return ( + <div className="face-document-item" ref={ele => this.createDropTarget(ele!)}> + <div className="face-collection-buttons"> + <IconButton tooltip="Delete Face From Collection" onPointerDown={this.deleteFaceDocument} icon={'x'} style={{ width: '4px' }} size={Size.XSMALL} /> + </div> + <div className="face-document-top"> + <h1>{StrCast(this._props.faceDoc[DocData].face_label)}</h1> + </div> + <IconButton + tooltip="See image information" + onPointerDown={() => this.onDisplayClick()} + icon={<FontAwesomeIcon icon={this._displayImages ? 'caret-up' : 'caret-down'} />} + color={MarqueeOptionsMenu.Instance.userColor} + style={{ width: '19px' }} + /> + {this._displayImages ? ( + <div className="face-document-image-container"> + {DocListCast(this._props.faceDoc[DocData].face_docList).map(doc => { + const [name, type] = ImageCast(doc[Doc.LayoutFieldKey(doc)]).url.href.split('.'); + return ( + <div className="image-wrapper" key={Utils.GenerateGuid()}> + <img onClick={() => DocumentView.showDocument(doc, { willZoomCentered: true })} style={{ maxWidth: '60px', margin: '10px' }} src={`${name}_o.${type}`} /> + <div className="remove-item"> + <IconButton tooltip={'Remove Doc From Face Collection'} onPointerDown={() => this.deleteAssociatedDoc(doc)} icon={'x'} style={{ width: '4px' }} size={Size.XSMALL} /> + </div> + </div> + ); + })} + </div> + ) : null} + </div> + ); + } +} + +@observer +export class FaceCollectionBox extends ViewBoxBaseComponent<FieldViewProps>() { + public static LayoutString(fieldKey: string) { + return FieldView.LayoutString(FaceCollectionBox, fieldKey); + } + + public static Instance: FaceCollectionBox; + + @computed get currentDocs() { + if (Doc.ActiveDashboard) { + return DocListCast(Doc.ActiveDashboard[DocData].faceDocuments); + } + return []; + } + + constructor(props: FieldViewProps) { + super(props); + makeObservable(this); + FaceCollectionBox.Instance = this; + } + + render() { + return ( + <div className="searchBox-container" style={{ pointerEvents: 'all', color: SnappingManager.userColor, background: SnappingManager.userBackgroundColor }}> + {this.currentDocs.map(doc => ( + <FaceDocumentItem key={doc[Id]} faceDoc={doc} /> + ))} + </div> + ); + } +} + +Docs.Prototypes.TemplateMap.set(DocumentType.FACECOLLECTION, { + layout: { view: FaceCollectionBox, dataField: 'data' }, + options: { acl: '', _width: 400 }, +}); diff --git a/src/client/views/collections/collectionFreeForm/ImageLabelBox.scss b/src/client/views/collections/collectionFreeForm/ImageLabelBox.scss new file mode 100644 index 000000000..819c72760 --- /dev/null +++ b/src/client/views/collections/collectionFreeForm/ImageLabelBox.scss @@ -0,0 +1,85 @@ +.image-box-container { + display: flex; + align-items: center; + justify-content: center; + width: 100%; + height: 100%; + font-size: 10px; + line-height: 1; + background: none; + z-index: 1000; + padding: 0px; + overflow: auto; + cursor: default; +} + +.image-label-list { + display: flex; + flex-direction: column; + align-items: center; // Centers the content vertically in the flex container + width: 100%; + + > div { + display: flex; + justify-content: space-between; // Puts the content and delete button on opposite ends + align-items: center; + width: 100%; + margin-top: 8px; // Adds space between label rows + background-color: black; + + p { + text-align: center; // Centers the text of the paragraph + font-size: large; + vertical-align: middle; + margin-left: 10px; + } + + .IconButton { + // Styling for the delete button + margin-left: auto; // Pushes the button to the far right + } + } +} + +.image-information-list { + display: flex; + flex-direction: column; + align-items: center; + width: 100%; + margin-top: 10px; +} + +.image-information { + border: 1px solid; + width: 100%; + display: inline-flex; + flex-direction: column; + justify-content: center; + align-items: center; + overflow: hidden; + padding: 2px; + overflow-x: auto; + overflow-y: auto; + + img { + max-width: 200px; + max-height: 200px; + width: auto; + height: auto; + } +} + +.image-information-labels { + display: flex; + flex-direction: column; + justify-content: center; + align-items: center; + + .image-label { + margin-top: 5px; + margin-bottom: 5px; + padding: 3px; + border-radius: 2px; + border: solid 1px; + } +} diff --git a/src/client/views/collections/collectionFreeForm/ImageLabelBox.tsx b/src/client/views/collections/collectionFreeForm/ImageLabelBox.tsx new file mode 100644 index 000000000..421b5d0a6 --- /dev/null +++ b/src/client/views/collections/collectionFreeForm/ImageLabelBox.tsx @@ -0,0 +1,343 @@ +import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; +import { Colors, IconButton } from 'browndash-components'; +import { action, computed, makeObservable, observable, reaction } from 'mobx'; +import { observer } from 'mobx-react'; +import React from 'react'; +import { Doc, NumListCast, Opt } from '../../../../fields/Doc'; +import { Docs } from '../../../documents/Documents'; +import { DocumentType } from '../../../documents/DocumentTypes'; +import { ViewBoxBaseComponent } from '../../DocComponent'; +import { FieldView, FieldViewProps } from '../../nodes/FieldView'; +import { MarqueeOptionsMenu } from './MarqueeOptionsMenu'; +import './ImageLabelBox.scss'; +import { MainView } from '../../MainView'; +import 'ldrs/ring'; +import { ring } from 'ldrs'; +import { SnappingManager } from '../../../util/SnappingManager'; +import { ImageCast } from '../../../../fields/Types'; +import { DocData } from '../../../../fields/DocSymbols'; +import { SettingsManager } from '../../../util/SettingsManager'; +import { CollectionCardView } from '../CollectionCardDeckView'; +import { gptGetEmbedding, gptImageLabel } from '../../../apis/gpt/GPT'; +import { numberRange, Utils } from '../../../../Utils'; +import { List } from '../../../../fields/List'; +import { DragManager } from '../../../util/DragManager'; +import { OpenWhere } from '../../nodes/OpenWhere'; +import similarity from 'compute-cosine-similarity'; +import { DocumentView } from '../../nodes/DocumentView'; + +export class ImageInformationItem {} + +export class ImageLabelBoxData { + static _instance: ImageLabelBoxData; + @observable _docs: Doc[] = []; + @observable _labelGroups: string[] = []; + + constructor() { + makeObservable(this); + ImageLabelBoxData._instance = this; + } + public static get Instance() { + return ImageLabelBoxData._instance ?? new ImageLabelBoxData(); + } + + @action + public setData = (docs: Doc[]) => { + this._docs = docs; + }; + + @action + addLabel = (label: string) => { + label = label.toUpperCase().trim(); + if (label.length > 0) { + if (!this._labelGroups.includes(label)) { + this._labelGroups = [...this._labelGroups, label]; + } + } + }; + + @action + removeLabel = (label: string) => { + const labelUp = label.toUpperCase(); + this._labelGroups = this._labelGroups.filter(group => group !== labelUp); + }; +} + +@observer +export class ImageLabelBox extends ViewBoxBaseComponent<FieldViewProps>() { + public static LayoutString(fieldKey: string) { + return FieldView.LayoutString(ImageLabelBox, fieldKey); + } + + private _dropDisposer?: DragManager.DragDropDisposer; + public static Instance: ImageLabelBox; + private _inputRef = React.createRef<HTMLInputElement>(); + @observable _loading: boolean = false; + private _currentLabel: string = ''; + + protected createDropTarget = (ele: HTMLDivElement) => { + this._dropDisposer?.(); + ele && (this._dropDisposer = DragManager.MakeDropTarget(ele, this.onInternalDrop.bind(this), this.layoutDoc)); + }; + + protected onInternalDrop(e: Event, de: DragManager.DropEvent): boolean { + const { docDragData } = de.complete; + if (docDragData) { + ImageLabelBoxData.Instance.setData(ImageLabelBoxData.Instance._docs.concat(docDragData.droppedDocuments)); + return false; + } + return false; + } + + @computed get _labelGroups() { + return ImageLabelBoxData.Instance._labelGroups; + } + + @computed get _selectedImages() { + // return DocListCast(this.dataDoc.data); + return ImageLabelBoxData.Instance._docs; + } + @observable _displayImageInformation: boolean = false; + + constructor(props: any) { + super(props); + makeObservable(this); + ring.register(); + ImageLabelBox.Instance = this; + } + + // ImageLabelBox.Instance.setData() + /** + * This method is called when the SearchBox component is first mounted. When the user opens + * the search panel, the search input box is automatically selected. This allows the user to + * type in the search input box immediately, without needing clicking on it first. + */ + componentDidMount() { + this.classifyImagesInBox(); + reaction( + () => this._selectedImages, + () => this.classifyImagesInBox() + ); + } + + @action + groupImages = () => { + this.groupImagesInBox(); + }; + + @action + startLoading = () => { + this._loading = true; + }; + + @action + endLoading = () => { + this._loading = false; + }; + + @action + toggleDisplayInformation = () => { + this._displayImageInformation = !this._displayImageInformation; + if (this._displayImageInformation) { + this._selectedImages.forEach(doc => (doc[DocData].showLabels = true)); + } else { + this._selectedImages.forEach(doc => (doc[DocData].showLabels = false)); + } + }; + + @action + submitLabel = () => { + const input = document.getElementById('new-label') as HTMLInputElement; + ImageLabelBoxData.Instance.addLabel(this._currentLabel); + this._currentLabel = ''; + input.value = ''; + }; + + onInputChange = action((e: React.ChangeEvent<HTMLInputElement>) => { + this._currentLabel = e.target.value; + }); + + classifyImagesInBox = async () => { + this.startLoading(); + + // Converts the images into a Base64 format, afterwhich the information is sent to GPT to label them. + + const imageInfos = this._selectedImages.map(async doc => { + if (!doc[DocData].data_labels) { + const [name, type] = ImageCast(doc[Doc.LayoutFieldKey(doc)]).url.href.split('.'); + return CollectionCardView.imageUrlToBase64(`${name}_o.${type}`).then(hrefBase64 => + !hrefBase64 ? undefined : + gptImageLabel(hrefBase64).then(labels => + ({ doc, labels }))) ; // prettier-ignore + } + }); + + (await Promise.all(imageInfos)).forEach(imageInfo => { + if (imageInfo) { + imageInfo.doc[DocData].data_labels = new List<string>(); + + const labels = imageInfo.labels.split('\n'); + labels.forEach(label => { + label = label.replace(/^\d+\.\s*|-|\*/, '').trim(); + console.log(label); + imageInfo.doc[DocData][`${label}`] = true; + (imageInfo.doc[DocData].data_labels as List<string>).push(label); + }); + } + }); + + this.endLoading(); + }; + + /** + * Groups images to most similar labels. + */ + groupImagesInBox = action(async () => { + this.startLoading(); + + for (const doc of this._selectedImages) { + for (let index = 0; index < (doc[DocData].data_labels as List<string>).length; index++) { + const label = (doc[DocData].data_labels as List<string>)[index]; + const embedding = await gptGetEmbedding(label); + doc[DocData][`data_labels_embedding_${index + 1}`] = new List<number>(embedding); + } + } + + const labelToEmbedding = new Map<string, number[]>(); + // Create embeddings for the labels. + await Promise.all(this._labelGroups.map(async label => gptGetEmbedding(label).then(labelEmbedding => labelToEmbedding.set(label, labelEmbedding)))); + + // For each image, loop through the labels, and calculate similarity. Associate it with the + // most similar one. + this._selectedImages.forEach(doc => { + const embedLists = numberRange((doc[DocData].data_labels as List<string>).length).map(n => Array.from(NumListCast(doc[DocData][`data_labels_embedding_${n + 1}`]))); + const bestEmbedScore = (embedding: Opt<number[]>) => Math.max(...embedLists.map((l, index) => (embedding && similarity(Array.from(embedding), l)!) || 0)); + const {label: mostSimilarLabelCollect} = + this._labelGroups.map(label => ({ label, similarityScore: bestEmbedScore(labelToEmbedding.get(label)) })) + .reduce((prev, cur) => cur.similarityScore < 0.3 || cur.similarityScore <= prev.similarityScore ? prev: cur, + { label: '', similarityScore: 0, }); // prettier-ignore + doc[DocData].data_label = mostSimilarLabelCollect; // The label most similar to the image's contents. + }); + + this.endLoading(); + + if (this._selectedImages) { + MarqueeOptionsMenu.Instance.groupImages(); + } + + MainView.Instance.closeFlyout(); + }); + + render() { + if (this._loading) { + return ( + <div className="image-box-container" style={{ pointerEvents: 'all', color: SnappingManager.userColor, background: SnappingManager.userBackgroundColor }}> + <l-ring size="60" color="white" /> + </div> + ); + } + + if (this._selectedImages.length === 0) { + return ( + <div className="searchBox-container" style={{ pointerEvents: 'all', color: SnappingManager.userColor, background: SnappingManager.userBackgroundColor }} ref={ele => this.createDropTarget(ele!)}> + <p style={{ fontSize: 'large' }}>In order to classify and sort images, marquee select the desired images and press the 'Classify and Sort Images' button. Then, add the desired groups for the images to be put in.</p> + </div> + ); + } + + return ( + <div className="searchBox-container" style={{ pointerEvents: 'all', color: SnappingManager.userColor, background: SnappingManager.userBackgroundColor }} ref={ele => this.createDropTarget(ele!)}> + <div className="searchBox-bar" style={{ pointerEvents: 'all', color: SnappingManager.userColor, background: SnappingManager.userBackgroundColor }}> + <IconButton + tooltip={'See image information'} + onPointerDown={this.toggleDisplayInformation} + icon={this._displayImageInformation ? <FontAwesomeIcon icon="caret-up" /> : <FontAwesomeIcon icon="caret-down" />} + color={MarqueeOptionsMenu.Instance.userColor} + style={{ width: '19px' }} + /> + <input + defaultValue="" + autoComplete="off" + onChange={this.onInputChange} + onKeyDown={e => { + e.key === 'Enter' ? this.submitLabel() : null; + e.stopPropagation(); + }} + type="text" + placeholder="Input groups for images to be put into..." + aria-label="label-input" + id="new-label" + className="searchBox-input" + style={{ width: '100%', borderRadius: '5px' }} + ref={this._inputRef} + /> + <IconButton + tooltip={'Add a label'} + onPointerDown={() => { + const input = document.getElementById('new-label') as HTMLInputElement; + ImageLabelBoxData.Instance.addLabel(this._currentLabel); + this._currentLabel = ''; + input.value = ''; + }} + icon={<FontAwesomeIcon icon="plus" />} + color={MarqueeOptionsMenu.Instance.userColor} + style={{ width: '19px' }} + /> + {this._labelGroups.length > 0 ? <IconButton tooltip={'Group Images'} onPointerDown={this.groupImages} icon={<FontAwesomeIcon icon="object-group" />} color={Colors.MEDIUM_BLUE} style={{ width: '19px' }} /> : <div></div>} + </div> + <div> + <div className="image-label-list"> + {this._labelGroups.map(group => { + return ( + <div key={Utils.GenerateGuid()}> + <p style={{ color: MarqueeOptionsMenu.Instance.userColor }}>{group}</p> + <IconButton + tooltip={'Remove Label'} + onPointerDown={() => { + ImageLabelBoxData.Instance.removeLabel(group); + }} + icon={'x'} + color={MarqueeOptionsMenu.Instance.userColor} + style={{ width: '8px' }} + /> + </div> + ); + })} + </div> + </div> + {this._displayImageInformation ? ( + <div className="image-information-list"> + {this._selectedImages.map(doc => { + const [name, type] = ImageCast(doc[Doc.LayoutFieldKey(doc)]).url.href.split('.'); + return ( + <div className="image-information" style={{ borderColor: SettingsManager.userColor }} key={Utils.GenerateGuid()}> + <img + src={`${name}_o.${type}`} + onClick={async () => { + await DocumentView.showDocument(doc, { willZoomCentered: true }); + }}></img> + <div className="image-information-labels" onClick={() => this._props.addDocTab(doc, OpenWhere.addRightKeyvalue)}> + {(doc[DocData].data_labels as List<string>).map(label => { + return ( + <div key={Utils.GenerateGuid()} className="image-label" style={{ backgroundColor: SettingsManager.userVariantColor, borderColor: SettingsManager.userColor }}> + {label} + </div> + ); + })} + </div> + </div> + ); + })} + </div> + ) : ( + <div></div> + )} + </div> + ); + } +} + +Docs.Prototypes.TemplateMap.set(DocumentType.IMAGEGROUPER, { + layout: { view: ImageLabelBox, dataField: 'data' }, + options: { acl: '', _width: 400 }, +}); diff --git a/src/client/views/collections/collectionFreeForm/ImageLabelHandler.tsx b/src/client/views/collections/collectionFreeForm/ImageLabelHandler.tsx index 7f27c6b5c..73befb205 100644 --- a/src/client/views/collections/collectionFreeForm/ImageLabelHandler.tsx +++ b/src/client/views/collections/collectionFreeForm/ImageLabelHandler.tsx @@ -77,7 +77,7 @@ export class ImageLabelHandler extends ObservableReactComponent<{}> { }}> <div> <IconButton tooltip={'Cancel'} onPointerDown={this.hideLabelhandler} icon={<FontAwesomeIcon icon="eye-slash" />} color={MarqueeOptionsMenu.Instance.userColor} style={{ width: '19px' }} /> - <input aria-label="label-input" id="new-label" type="text" style={{ color: 'black' }} /> + <input aria-label="label-input" id="new-label" type="text" placeholder="Input a classification" style={{ color: 'black' }} /> <IconButton tooltip={'Add Label'} onPointerDown={() => { diff --git a/src/client/views/collections/collectionFreeForm/MarqueeOptionsMenu.tsx b/src/client/views/collections/collectionFreeForm/MarqueeOptionsMenu.tsx index f02cd9d45..44c916ab9 100644 --- a/src/client/views/collections/collectionFreeForm/MarqueeOptionsMenu.tsx +++ b/src/client/views/collections/collectionFreeForm/MarqueeOptionsMenu.tsx @@ -18,10 +18,10 @@ export class MarqueeOptionsMenu extends AntimodeMenu<AntimodeMenuProps> { public showMarquee: () => void = unimplementedFunction; public hideMarquee: () => void = unimplementedFunction; public pinWithView: (e: KeyboardEvent | React.PointerEvent | undefined) => void = unimplementedFunction; - public classifyImages: (e: React.MouseEvent | undefined) => void = unimplementedFunction; + public classifyImages: () => void = unimplementedFunction; public groupImages: () => void = unimplementedFunction; public isShown = () => this._opacity > 0; - constructor(props: any) { + constructor(props: AntimodeMenuProps) { super(props); makeObservable(this); MarqueeOptionsMenu.Instance = this; @@ -39,7 +39,7 @@ export class MarqueeOptionsMenu extends AntimodeMenu<AntimodeMenuProps> { <IconButton tooltip="Summarize Documents" onPointerDown={this.summarize} icon={<FontAwesomeIcon icon="compress-arrows-alt" />} color={this.userColor} /> <IconButton tooltip="Delete Documents" onPointerDown={this.delete} icon={<FontAwesomeIcon icon="trash-alt" />} color={this.userColor} /> <IconButton tooltip="Pin selected region" onPointerDown={this.pinWithView} icon={<FontAwesomeIcon icon="map-pin" />} color={this.userColor} /> - <IconButton tooltip="Classify Images" onPointerDown={this.classifyImages} icon={<FontAwesomeIcon icon="object-group" />} color={this.userColor} /> + <IconButton tooltip="Classify and Sort Images" onPointerDown={this.classifyImages} icon={<FontAwesomeIcon icon="object-group" />} color={this.userColor} /> </> ); return this.getElement(buttons); diff --git a/src/client/views/collections/collectionFreeForm/MarqueeView.tsx b/src/client/views/collections/collectionFreeForm/MarqueeView.tsx index dc15c83c5..6cc75aa4b 100644 --- a/src/client/views/collections/collectionFreeForm/MarqueeView.tsx +++ b/src/client/views/collections/collectionFreeForm/MarqueeView.tsx @@ -1,28 +1,24 @@ -/* eslint-disable jsx-a11y/no-static-element-interactions */ -import similarity from 'compute-cosine-similarity'; import { action, computed, makeObservable, observable } from 'mobx'; import { observer } from 'mobx-react'; import * as React from 'react'; import { ClientUtils, lightOrDark, returnFalse } from '../../../../ClientUtils'; -import { intersectRect, numberRange } from '../../../../Utils'; -import { Doc, NumListCast, Opt } from '../../../../fields/Doc'; +import { intersectRect } from '../../../../Utils'; +import { Doc, DocListCast, Opt } from '../../../../fields/Doc'; import { AclAdmin, AclAugment, AclEdit, DocData } from '../../../../fields/DocSymbols'; import { Id } from '../../../../fields/FieldSymbols'; -import { InkData, InkField, InkTool } from '../../../../fields/InkField'; +import { InkTool } from '../../../../fields/InkField'; import { List } from '../../../../fields/List'; -import { RichTextField } from '../../../../fields/RichTextField'; -import { Cast, FieldValue, ImageCast, NumCast, StrCast } from '../../../../fields/Types'; +import { Cast, NumCast, StrCast } from '../../../../fields/Types'; import { ImageField } from '../../../../fields/URLField'; import { GetEffectiveAcl } from '../../../../fields/util'; -import { gptGetEmbedding, gptImageLabel } from '../../../apis/gpt/GPT'; -import { CognitiveServices } from '../../../cognitive_services/CognitiveServices'; import { DocUtils } from '../../../documents/DocUtils'; -import { CollectionViewType, DocumentType } from '../../../documents/DocumentTypes'; +import { DocumentType } from '../../../documents/DocumentTypes'; import { Docs, DocumentOptions } from '../../../documents/Documents'; import { SnappingManager, freeformScrollMode } from '../../../util/SnappingManager'; import { Transform } from '../../../util/Transform'; import { UndoManager, undoBatch } from '../../../util/UndoManager'; import { ContextMenu } from '../../ContextMenu'; +import { MainView } from '../../MainView'; import { ObservableReactComponent } from '../../ObservableReactComponent'; import { MarqueeViewBounds } from '../../PinFuncs'; import { PreviewCursor } from '../../PreviewCursor'; @@ -30,10 +26,8 @@ import { DocumentView } from '../../nodes/DocumentView'; import { OpenWhere } from '../../nodes/OpenWhere'; import { pasteImageBitmap } from '../../nodes/WebBoxRenderer'; import { FormattedTextBox } from '../../nodes/formattedText/FormattedTextBox'; -import { CollectionCardView } from '../CollectionCardDeckView'; import { SubCollectionViewProps } from '../CollectionSubView'; -import { CollectionFreeFormView } from './CollectionFreeFormView'; -import { ImageLabelHandler } from './ImageLabelHandler'; +import { ImageLabelBoxData } from './ImageLabelBox'; import { MarqueeOptionsMenu } from './MarqueeOptionsMenu'; import './MarqueeView.scss'; @@ -53,6 +47,9 @@ interface MarqueeViewProps { slowLoadDocuments: (files: File[] | string, options: DocumentOptions, generatedDocuments: Doc[], text: string, completed: ((doc: Doc[]) => void) | undefined, addDocument: (doc: Doc | Doc[]) => boolean) => Promise<void>; } +/** + * A component that deals with the marquee select in the freeform canvas. + */ @observer export class MarqueeView extends ObservableReactComponent<SubCollectionViewProps & MarqueeViewProps> { public static CurViewBounds(pinDoc: Doc, panelWidth: number, panelHeight: number) { @@ -60,9 +57,12 @@ export class MarqueeView extends ObservableReactComponent<SubCollectionViewProps return { left: NumCast(pinDoc._freeform_panX) - panelWidth / 2 / ps, top: NumCast(pinDoc._freeform_panY) - panelHeight / 2 / ps, width: panelWidth / ps, height: panelHeight / ps }; } - constructor(props: any) { + static Instance: MarqueeView; + + constructor(props: SubCollectionViewProps & MarqueeViewProps) { super(props); makeObservable(this); + MarqueeView.Instance = this; } private _commandExecuted = false; @@ -156,6 +156,7 @@ export class MarqueeView extends ObservableReactComponent<SubCollectionViewProps } else if (e.key === 'b' && e.ctrlKey) { document.body.focus(); // so that we can access the clipboard without an error setTimeout(() => + // eslint-disable-next-line @typescript-eslint/no-explicit-any pasteImageBitmap((data: any, error: any) => { error && console.log(error); data && @@ -430,32 +431,14 @@ export class MarqueeView extends ObservableReactComponent<SubCollectionViewProps /** * Classifies images and assigns the labels as document fields. - * TODO: Turn into lists of labels instead of individual fields. */ @undoBatch - classifyImages = action(async (e: React.MouseEvent | undefined) => { - this._selectedDocs = this.marqueeSelect(false, DocumentType.IMG); - - const imageInfos = this._selectedDocs.map(async doc => { - const [name, type] = ImageCast(doc[Doc.LayoutFieldKey(doc)]).url.href.split('.'); - return CollectionCardView.imageUrlToBase64(`${name}_o.${type}`).then(hrefBase64 => - !hrefBase64 ? undefined : - gptImageLabel(hrefBase64).then(labels => - Promise.all(labels.split('\n').map(label => gptGetEmbedding(label))).then(embeddings => - ({ doc, embeddings, labels }))) ); // prettier-ignore - }); - - (await Promise.all(imageInfos)).forEach(imageInfo => { - if (imageInfo && Array.isArray(imageInfo.embeddings)) { - imageInfo.doc[DocData].data_labels = imageInfo.labels; - numberRange(3).forEach(n => { - imageInfo.doc[`data_labels_embedding_${n + 1}`] = new List<number>(imageInfo.embeddings[n]); - }); - } - }); - - if (e) { - ImageLabelHandler.Instance.displayLabelHandler(e.pageX, e.pageY); + classifyImages = action(async () => { + const groupButton = DocListCast(Doc.MyLeftSidebarMenu.data).find(d => d.target === Doc.MyImageGrouper); + if (groupButton) { + this._selectedDocs = this.marqueeSelect(false, DocumentType.IMG); + ImageLabelBoxData.Instance.setData(this._selectedDocs); + MainView.Instance.expandFlyout(groupButton); } }); @@ -464,93 +447,44 @@ export class MarqueeView extends ObservableReactComponent<SubCollectionViewProps */ @undoBatch groupImages = action(async () => { - const labelGroups = ImageLabelHandler.Instance._labelGroups; - const labelToEmbedding = new Map<string, number[]>(); - // Create embeddings for the labels. - await Promise.all(labelGroups.map(async label => gptGetEmbedding(label).then(labelEmbedding => labelToEmbedding.set(label, labelEmbedding)))); - - // For each image, loop through the labels, and calculate similarity. Associate it with the - // most similar one. - this._selectedDocs.forEach(doc => { - const embedLists = numberRange(3).map(n => Array.from(NumListCast(doc[`data_labels_embedding_${n + 1}`]))); - const bestEmbedScore = (embedding: Opt<number[]>) => Math.max(...embedLists.map(l => (embedding && similarity(Array.from(embedding), l)) || 0)); - const {label: mostSimilarLabelCollect} = - labelGroups.map(label => ({ label, similarityScore: bestEmbedScore(labelToEmbedding.get(label)) })) - .reduce((prev, cur) => cur.similarityScore < 0.3 || cur.similarityScore <= prev.similarityScore ? prev: cur, - { label: '', similarityScore: 0, }); // prettier-ignore - - numberRange(3).forEach(n => { - doc[`data_labels_embedding_${n + 1}`] = undefined; - }); - doc[DocData].data_label = mostSimilarLabelCollect; - }); - this._props.Document._type_collection = CollectionViewType.Time; - this._props.Document.pivotField = 'data_label'; - }); + const labelGroups: string[] = ImageLabelBoxData.Instance._labelGroups; + const labelToCollection: Map<string, Doc> = new Map(); + const selectedImages = ImageLabelBoxData.Instance._docs; + + // Create new collections associated with each label and get the embeddings for the labels. + let x_offset = 0; + let y_offset = 0; + let row_count = 0; + for (const label of labelGroups) { + const newCollection = this.getCollection([], undefined, false); + newCollection._width = 900; + newCollection._height = 900; + newCollection._x = this.Bounds.left; + newCollection._y = this.Bounds.top; + newCollection._freeform_panX = this.Bounds.left + this.Bounds.width / 2; + newCollection._freeform_panY = this.Bounds.top + this.Bounds.height / 2; + newCollection._x = (newCollection._x as number) + x_offset; + newCollection._y = (newCollection._y as number) + y_offset; + x_offset += (newCollection._width as number) + 40; + row_count += 1; + if (row_count == 3) { + y_offset += (newCollection._height as number) + 40; + x_offset = 0; + row_count = 0; + } + labelToCollection.set(label, newCollection); + this._props.addDocument?.(newCollection); + } - @undoBatch - syntaxHighlight = action((e: KeyboardEvent | React.PointerEvent | undefined) => { - const selected = this.marqueeSelect(false); - if (e instanceof KeyboardEvent ? e.key === 'i' : true) { - const inks = selected.filter(s => s.type === DocumentType.INK); - const setDocs = selected.filter(s => s.type === DocumentType.RTF && s.color); - const sets = setDocs.map(sd => Cast(sd.data, RichTextField)?.Text as string); - const colors = setDocs.map(sd => FieldValue(sd.color) as string); - const wordToColor = new Map<string, string>(); - sets.forEach((st: string, i: number) => st.split(',').forEach(word => wordToColor.set(word, colors[i]))); - const strokes: InkData[] = []; - inks.filter(i => Cast(i.data, InkField)).forEach(i => { - const d = Cast(i.data, InkField, null); - const left = Math.min(...(d?.inkData.map(pd => pd.X) ?? [0])); - const top = Math.min(...(d?.inkData.map(pd => pd.Y) ?? [0])); - strokes.push(d.inkData.map(pd => ({ X: pd.X + NumCast(i.x) - left, Y: pd.Y + NumCast(i.y) - top }))); - }); - CognitiveServices.Inking.Appliers.InterpretStrokes(strokes).then(results => { - // const wordResults = results.filter((r: any) => r.category === "inkWord"); - // for (const word of wordResults) { - // const indices: number[] = word.strokeIds; - // indices.forEach(i => { - // if (wordToColor.has(word.recognizedText.toLowerCase())) { - // inks[i].color = wordToColor.get(word.recognizedText.toLowerCase()); - // } - // else { - // for (const alt of word.alternates) { - // if (wordToColor.has(alt.recognizedString.toLowerCase())) { - // inks[i].color = wordToColor.get(alt.recognizedString.toLowerCase()); - // break; - // } - // } - // } - // }) - // } - // const wordResults = results.filter((r: any) => r.category === "inkWord"); - // for (const word of wordResults) { - // const indices: number[] = word.strokeIds; - // indices.forEach(i => { - // const otherInks: Doc[] = []; - // indices.forEach(i2 => i2 !== i && otherInks.push(inks[i2])); - // inks[i].relatedInks = new List<Doc>(otherInks); - // const uniqueColors: string[] = []; - // Array.from(wordToColor.values()).forEach(c => uniqueColors.indexOf(c) === -1 && uniqueColors.push(c)); - // inks[i].alternativeColors = new List<string>(uniqueColors); - // if (wordToColor.has(word.recognizedText.toLowerCase())) { - // inks[i].color = wordToColor.get(word.recognizedText.toLowerCase()); - // } - // else if (word.alternates) { - // for (const alt of word.alternates) { - // if (wordToColor.has(alt.recognizedString.toLowerCase())) { - // inks[i].color = wordToColor.get(alt.recognizedString.toLowerCase()); - // break; - // } - // } - // } - // }); - // } - const lines = results.filter((r: any) => r.category === 'line'); - const text = lines.map((l: any) => l.recognizedText).join('\r\n'); - this._props.addDocument?.(Docs.Create.TextDocument(text, { _width: this.Bounds.width, _height: this.Bounds.height, x: this.Bounds.left + this.Bounds.width, y: this.Bounds.top, title: text })); - }); + for (const doc of selectedImages) { + if (doc[DocData].data_label) { + Doc.AddDocToList(labelToCollection.get(doc[DocData].data_label as string)!, undefined, doc); + this._props.removeDocument?.(doc); + } } + + //this._props.Document._type_collection = CollectionViewType.Time; // Change the collection view to a Time view. + //this._props.Document.pivotField = 'data_label'; // Sets the pivot to be the 'data_label'. }); @undoBatch @@ -582,13 +516,14 @@ export class MarqueeView extends ObservableReactComponent<SubCollectionViewProps @action marqueeCommand = (e: KeyboardEvent) => { - if (this._commandExecuted || (e as any).propagationIsStopped) { + const ee = e as unknown as KeyboardEvent & { propagationIsStopped?: boolean }; + if (this._commandExecuted || ee.propagationIsStopped) { return; } if (e.key === 'Backspace' || e.key === 'Delete' || e.key === 'd' || e.key === 'h') { this._commandExecuted = true; e.stopPropagation(); - (e as any).propagationIsStopped = true; + ee.propagationIsStopped = true; this.delete(e, e.key === 'h'); e.stopPropagation(); } @@ -596,7 +531,7 @@ export class MarqueeView extends ObservableReactComponent<SubCollectionViewProps this._commandExecuted = true; e.stopPropagation(); e.preventDefault(); - (e as any).propagationIsStopped = true; + ee.propagationIsStopped = true; if (e.key === 'g') this.collection(e, true); if (e.key === 'c' || e.key === 't') this.collection(e); if (e.key === 's' || e.key === 'S') this.summary(); @@ -697,8 +632,8 @@ export class MarqueeView extends ObservableReactComponent<SubCollectionViewProps transform: `translate(${p[0]}px, ${p[1]}px)`, width: Math.abs(v[0]), height: Math.abs(v[1]), - color: lightOrDark(this._props.Document?.backgroundColor ?? 'white'), - borderColor: lightOrDark(this._props.Document?.backgroundColor ?? 'white'), + color: lightOrDark((this._props.Document?.backgroundColor as string) ?? 'white'), + borderColor: lightOrDark((this._props.Document?.backgroundColor as string) ?? 'white'), zIndex: 2000, }}> {' '} @@ -707,7 +642,7 @@ export class MarqueeView extends ObservableReactComponent<SubCollectionViewProps <polyline // points={this._lassoPts.reduce((s, pt) => s + pt[0] + ',' + pt[1] + ' ', '')} fill="none" - stroke={lightOrDark(this._props.Document?.backgroundColor ?? 'white')} + stroke={lightOrDark((this._props.Document?.backgroundColor as string) ?? 'white')} strokeWidth="1" strokeDasharray="3" /> @@ -727,8 +662,9 @@ export class MarqueeView extends ObservableReactComponent<SubCollectionViewProps */ @action onDragMovePause = (e: CustomEvent<React.DragEvent>) => { - if ((e as any).handlePan || this._props.isAnnotationOverlay) return; - (e as any).handlePan = true; + const ee = e as CustomEvent<React.DragEvent> & { handlePan?: boolean }; + if (ee.handlePan || this._props.isAnnotationOverlay) return; + ee.handlePan = true; const bounds = this.MarqueeRef?.getBoundingClientRect(); if (!this._props.Document._freeform_noAutoPan && !this._props.renderDepth && bounds) { @@ -746,10 +682,10 @@ export class MarqueeView extends ObservableReactComponent<SubCollectionViewProps }; render() { return ( - // eslint-disable-next-line jsx-a11y/click-events-have-key-events <div className="marqueeView" ref={r => { + // eslint-disable-next-line @typescript-eslint/no-explicit-any r?.addEventListener('dashDragMovePause', this.onDragMovePause as any); this.MarqueeRef = r; }} |