diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/client/apis/gpt/GPT.ts | 2 | ||||
| -rw-r--r-- | src/client/documents/DocumentTypes.ts | 1 | ||||
| -rw-r--r-- | src/client/documents/Documents.ts | 4 | ||||
| -rw-r--r-- | src/client/util/CurrentUserUtils.ts | 13 | ||||
| -rw-r--r-- | src/client/views/KeywordBox.tsx | 7 | ||||
| -rw-r--r-- | src/client/views/Main.tsx | 2 | ||||
| -rw-r--r-- | src/client/views/collections/collectionFreeForm/FaceCollectionBox.scss | 74 | ||||
| -rw-r--r-- | src/client/views/collections/collectionFreeForm/FaceCollectionBox.tsx | 217 | ||||
| -rw-r--r-- | src/client/views/collections/collectionFreeForm/ImageLabelBox.tsx | 5 | ||||
| -rw-r--r-- | src/client/views/search/FaceRecognitionHandler.tsx | 74 | ||||
| -rw-r--r-- | src/fields/Doc.ts | 1 |
11 files changed, 375 insertions, 25 deletions
diff --git a/src/client/apis/gpt/GPT.ts b/src/client/apis/gpt/GPT.ts index 05007960d..8dd3fd6e2 100644 --- a/src/client/apis/gpt/GPT.ts +++ b/src/client/apis/gpt/GPT.ts @@ -128,7 +128,7 @@ const gptImageLabel = async (src: string): Promise<string> => { { role: 'user', content: [ - { type: 'text', text: 'Give three to five labels to describe this image.' }, + { type: 'text', text: 'Give three labels to describe this image.' }, { type: 'image_url', image_url: { diff --git a/src/client/documents/DocumentTypes.ts b/src/client/documents/DocumentTypes.ts index a9ea889b3..b66a29ac2 100644 --- a/src/client/documents/DocumentTypes.ts +++ b/src/client/documents/DocumentTypes.ts @@ -17,6 +17,7 @@ export enum DocumentType { FONTICON = 'fonticonbox', SEARCH = 'search', // search query IMAGEGROUPER = 'imagegrouper', + FACECOLLECTION = 'facecollection', LABEL = 'label', // simple text label BUTTON = 'button', // onClick button WEBCAM = 'webcam', // webcam diff --git a/src/client/documents/Documents.ts b/src/client/documents/Documents.ts index 3737aa0b5..ecea74fab 100644 --- a/src/client/documents/Documents.ts +++ b/src/client/documents/Documents.ts @@ -795,6 +795,10 @@ export namespace Docs { return InstanceFromProto(Prototypes.get(DocumentType.IMAGEGROUPER), undefined, options); } + export function FaceCollectionDocument(options: DocumentOptions = {}) { + return InstanceFromProto(Prototypes.get(DocumentType.FACECOLLECTION), undefined, options); + } + export function LoadingDocument(file: File | string, options: DocumentOptions) { return InstanceFromProto(Prototypes.get(DocumentType.LOADING), undefined, { _height: 150, _width: 200, title: typeof file === 'string' ? file : file.name, ...options }, undefined, ''); } diff --git a/src/client/util/CurrentUserUtils.ts b/src/client/util/CurrentUserUtils.ts index cb3d9df62..db0de83b9 100644 --- a/src/client/util/CurrentUserUtils.ts +++ b/src/client/util/CurrentUserUtils.ts @@ -458,7 +458,8 @@ pie title Minerals in my tap water { title: "Shared", toolTip: "Shared Docs", target: Doc.MySharedDocs, ignoreClick: true, icon: "users", funcs: {badgeValue: badgeValue}}, { title: "Trails", toolTip: "Trails ⌘R", target: Doc.UserDoc(), ignoreClick: true, icon: "pres-trail", funcs: {target: getActiveDashTrails}}, { title: "User Doc", toolTip: "User Doc", target: this.setupUserDocView(doc, "myUserDocView"), ignoreClick: true, icon: "address-card",funcs: {hidden: "IsNoviceMode()"} }, - { title: "Image Grouper", toolTip: "Image Grouper", target: this.setupImageGrouper(doc, "myImageGrouper"), ignoreClick: true, icon: "folder-open", hidden: false } + { title: "Image Grouper", toolTip: "Image Grouper", target: this.setupImageGrouper(doc, "myImageGrouper"), ignoreClick: true, icon: "folder-open", hidden: false }, + { title: "Face Collection", toolTip: "Face Collection", target: this.setupFaceCollection(doc, "myFaceCollection"), ignoreClick: true, icon: "face-smile", hidden: false }, ].map(tuple => ({...tuple, scripts:{onClick: 'selectMainMenu(this)'}})); } @@ -500,6 +501,12 @@ pie title Minerals in my tap water _lockedPosition: true, _type_collection: CollectionViewType.Schema }); } + static setupFaceCollection(doc: Doc, field: string) { + return DocUtils.AssignDocField(doc, field, (opts) => Docs.Create.FaceCollectionDocument(opts), { + dontRegisterView: true, backgroundColor: "dimgray", ignoreClick: true, title: "Face Collection", isSystem: true, childDragAction: dropActionType.embed, + _lockedPosition: true, _type_collection: CollectionViewType.Schema }); + } + /// Initializes the panel of draggable tools that is opened from the left sidebar. static setupToolsBtnPanel(doc: Doc, field:string) { const allTools = DocListCast(DocCast(doc[field])?.data); @@ -702,8 +709,8 @@ pie title Minerals in my tap water { title: "Fit All", icon: "object-group", toolTip: "Fit Docs to View (double click to make sticky)",btnType: ButtonType.ToggleButton, ignoreClick:true, expertMode: false, toolType:"viewAll", funcs: {}, scripts: { onClick: '{ return showFreeform(this.toolType, _readOnly_);}', onDoubleClick: '{ return showFreeform(this.toolType, _readOnly_, true);}'}}, // Only when floating document is selected in freeform { title: "Clusters", icon: "braille", toolTip: "Show Doc Clusters", btnType: ButtonType.ToggleButton, ignoreClick: true, expertMode: false, toolType:"clusters", funcs: {}, scripts: { onClick: '{ return showFreeform(this.toolType, _readOnly_);}'}}, // Only when floating document is selected in freeform { title: "Cards", icon: "brain", toolTip: "Flashcards", btnType: ButtonType.ToggleButton, ignoreClick: true, expertMode: false, toolType:"flashcards", funcs: {}, scripts: { onClick: '{ return showFreeform(this.toolType, _readOnly_);}'}}, // Only when floating document is selected in freeform - { title: "Arrange", icon:"arrow-down-short-wide",toolTip:"Toggle Auto Arrange", btnType: ButtonType.ToggleButton, ignoreClick: true, expertMode: false, toolType:"arrange", funcs: {hidden: 'IsNoviceMode()'}, scripts: { onClick: '{ return showFreeform(this.toolType, _readOnly_);}'}}, // Only when floating document is selected in freeform - + { title: "Arrange", icon:"arrow-down-short-wide",toolTip:"Auto Arrange", btnType: ButtonType.ToggleButton, ignoreClick: true, expertMode: false, toolType:"arrange", funcs: {hidden: 'IsNoviceMode()'}, scripts: { onClick: '{ return showFreeform(this.toolType, _readOnly_);}'}}, // Only when floating document is selected in freeform + ] } static textTools():Button[] { diff --git a/src/client/views/KeywordBox.tsx b/src/client/views/KeywordBox.tsx index 68584a7fa..fc9c38a11 100644 --- a/src/client/views/KeywordBox.tsx +++ b/src/client/views/KeywordBox.tsx @@ -7,7 +7,7 @@ import { Doc, DocListCast } from '../../fields/Doc'; import { DocData } from '../../fields/DocSymbols'; import { List } from '../../fields/List'; import { NumCast, StrCast } from '../../fields/Types'; -import { emptyFunction } from '../../Utils'; +import { emptyFunction, Utils } from '../../Utils'; import { DocumentType } from '../documents/DocumentTypes'; import { DragManager } from '../util/DragManager'; import { SnappingManager } from '../util/SnappingManager'; @@ -124,7 +124,7 @@ export class KeywordItem extends ObservableReactComponent<KeywordItemProps> { render() { return ( - <div className="keyword" onClick={this._props.setToEditing} onPointerDown={this.handleDragStart} ref={this.ref}> + <div className="keyword" onClick={this._props.setToEditing} onPointerDown={this.handleDragStart} ref={this.ref} key={Utils.GenerateGuid()}> {this._props.keyword} {this.props.isEditing && <IconButton tooltip={'Remove label'} onPointerDown={this.removeLabel} icon={'X'} style={{ width: '8px', height: '8px', marginLeft: '10px' }} />} </div> @@ -297,7 +297,7 @@ export class KeywordBox extends ObservableReactComponent<KeywordBoxProps> { <div className="keywords-content"> <div className="keywords-list"> {(keywordsList as List<string>).map(keyword => { - return <KeywordItem doc={this._props.doc} keyword={keyword} keywordDoc={this.getKeywordCollection(keyword)} setToEditing={this.setToEditing} isEditing={this._props.isEditing}></KeywordItem>; + return <KeywordItem key={Utils.GenerateGuid()} doc={this._props.doc} keyword={keyword} keywordDoc={this.getKeywordCollection(keyword)} setToEditing={this.setToEditing} isEditing={this._props.isEditing}></KeywordItem>; })} </div> {this._props.isEditing ? ( @@ -331,6 +331,7 @@ export class KeywordBox extends ObservableReactComponent<KeywordBoxProps> { onClick={() => { this.submitLabel(keyword); }} + key={Utils.GenerateGuid()} /> ); })} diff --git a/src/client/views/Main.tsx b/src/client/views/Main.tsx index ada934aea..85c2b3d47 100644 --- a/src/client/views/Main.tsx +++ b/src/client/views/Main.tsx @@ -62,6 +62,7 @@ import { PresBox, PresElementBox } from './nodes/trails'; import { SearchBox } from './search/SearchBox'; import { ImageLabelBox } from './collections/collectionFreeForm/ImageLabelBox'; import { FaceRecognitionHandler } from './search/FaceRecognitionHandler'; +import { FaceCollectionBox } from './collections/collectionFreeForm/FaceCollectionBox'; dotenv.config(); @@ -135,6 +136,7 @@ FieldLoader.ServerLoadStatus = { requested: 0, retrieved: 0, message: 'cache' }; PresElementBox, SearchBox, ImageLabelBox, //Here! + FaceCollectionBox, FunctionPlotBox, InkingStroke, LinkBox, diff --git a/src/client/views/collections/collectionFreeForm/FaceCollectionBox.scss b/src/client/views/collections/collectionFreeForm/FaceCollectionBox.scss new file mode 100644 index 000000000..480d109c8 --- /dev/null +++ b/src/client/views/collections/collectionFreeForm/FaceCollectionBox.scss @@ -0,0 +1,74 @@ +.face-document-item { + background: #555555; + margin-top: 10px; + margin-bottom: 10px; + padding: 10px; + border-radius: 10px; + position: relative; + + h1 { + color: white; + font-size: 24px; + text-align: center; + } + + .face-collection-buttons { + position: absolute; + top: 10px; + right: 10px; + } + + .face-document-image-container { + display: flex; + justify-content: center; + flex-wrap: wrap; + + .image-wrapper { + position: relative; + width: 70px; + height: 70px; + margin: 10px; + display: flex; + align-items: center; // Center vertically + justify-content: center; // Center horizontally + + img { + width: 100%; + height: 100%; + object-fit: cover; // This ensures the image covers the container without stretching + border-radius: 5px; + border: 2px solid white; + transition: border-color 0.4s; + + &:hover { + border-color: orange; // Change this to your desired hover border color + } + } + + .remove-item { + position: absolute; + bottom: -5; + right: -5; + background-color: rgba(0, 0, 0, 0.5); // Optional: to add a background behind the icon for better visibility + border-radius: 30%; + width: 10px; // Adjust size as needed + height: 10px; // Adjust size as needed + display: flex; + align-items: center; + justify-content: center; + } + } + + // img { + // max-width: 60px; + // margin: 10px; + // border-radius: 5px; + // border: 2px solid white; + // transition: 0.4s; + + // &:hover { + // border-color: orange; + // } + // } + } +} diff --git a/src/client/views/collections/collectionFreeForm/FaceCollectionBox.tsx b/src/client/views/collections/collectionFreeForm/FaceCollectionBox.tsx new file mode 100644 index 000000000..1d3f88df1 --- /dev/null +++ b/src/client/views/collections/collectionFreeForm/FaceCollectionBox.tsx @@ -0,0 +1,217 @@ +import { observer } from 'mobx-react'; +import React from 'react'; +import { Docs } from '../../../documents/Documents'; +import { DocumentType } from '../../../documents/DocumentTypes'; +import { ViewBoxBaseComponent } from '../../DocComponent'; +import { FieldView, FieldViewProps } from '../../nodes/FieldView'; +import 'ldrs/ring'; +import { SnappingManager } from '../../../util/SnappingManager'; +import { action, computed, makeObservable, observable, reaction } from 'mobx'; +import { Doc, DocListCast, NumListCast } from '../../../../fields/Doc'; +import { DocData } from '../../../../fields/DocSymbols'; +import { ImageCast, StrCast } from '../../../../fields/Types'; +import { ObservableReactComponent } from '../../ObservableReactComponent'; +import './FaceCollectionBox.scss'; +import { IconButton, Size } from 'browndash-components'; +import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; +import { MarqueeOptionsMenu } from './MarqueeOptionsMenu'; +import { List } from '../../../../fields/List'; +import { DocumentView } from '../../nodes/DocumentView'; +import { Utils } from '../../../../Utils'; +import { DragManager } from '../../../util/DragManager'; +import * as faceapi from 'face-api.js'; +import { FaceMatcher } from 'face-api.js'; + +interface FaceDocumentProps { + faceDoc: Doc; +} + +/** + * A componenent to visually represent a Face Document. + */ +@observer +export class FaceDocumentItem extends ObservableReactComponent<FaceDocumentProps> { + private ref: React.RefObject<HTMLDivElement>; + @observable _displayImages: boolean = true; + private _dropDisposer?: DragManager.DragDropDisposer; + private _inputRef = React.createRef<HTMLInputElement>(); + + constructor(props: any) { + super(props); + makeObservable(this); + this.ref = React.createRef(); + } + + protected createDropTarget = (ele: HTMLDivElement) => { + this._dropDisposer?.(); + ele && (this._dropDisposer = DragManager.MakeDropTarget(ele, this.onInternalDrop.bind(this), this._props.faceDoc)); + }; + + protected onInternalDrop(e: Event, de: DragManager.DropEvent): boolean { + const { docDragData } = de.complete; + if (docDragData) { + const filteredDocs = docDragData.droppedDocuments.filter(doc => doc.type === DocumentType.IMG); + filteredDocs.forEach(doc => { + // If the current Face Document has no items, and the doc has more than one face descriptor, don't let the user add the document first. + if ((this._props.faceDoc[DocData].faceDescriptors as List<List<number>>).length === 0 && (doc[DocData].faces as List<List<number>>).length > 1) { + alert('Cannot add a document with multiple faces as the first item!'); + } else { + // Loop through the documents' face descriptors. + // Choose the face with the smallest distance to add. + const float32Array = (this._props.faceDoc[DocData].faceDescriptors as List<List<number>>).map(faceDescriptor => new Float32Array(Array.from(faceDescriptor))); + const labeledFaceDescriptor = new faceapi.LabeledFaceDescriptors(StrCast(this._props.faceDoc[DocData].label), float32Array); + const faceDescriptors: faceapi.LabeledFaceDescriptors[] = [labeledFaceDescriptor]; + + const faceMatcher = new FaceMatcher(faceDescriptors, 1); + let cur_lowest_distance = 1; + let cur_matching_face = new List<number>(); + + (doc[DocData].faces as List<List<number>>).forEach(face => { + // If the face has the current lowest distance, mark it as such + // Once that lowest distance is found, add the face descriptor to the faceDoc, and add the associated doc + const convered_32_array: Float32Array = new Float32Array(Array.from(face)); + const match = faceMatcher.matchDescriptor(convered_32_array); + + if (match.distance < cur_lowest_distance) { + cur_lowest_distance = match.distance; + cur_matching_face = face; + } + }); + + if (doc[DocData][`FACE DESCRIPTOR - ${this._props.faceDoc[DocData].label}`]) { + doc[DocData][`FACE DESCRIPTOR - ${this._props.faceDoc[DocData].label}`] = new List<List<number>>([...(doc[DocData][`FACE DESCRIPTOR - ${this._props.faceDoc[DocData].label}`] as List<List<number>>), cur_matching_face]); + } else { + doc[DocData][`FACE DESCRIPTOR - ${this._props.faceDoc[DocData].label}`] = new List<List<number>>([cur_matching_face]); + } + + this._props.faceDoc[DocData].associatedDocs = new List<Doc>([...DocListCast(this._props.faceDoc[DocData].associatedDocs), doc]); + this._props.faceDoc[DocData].faceDescriptors = new List<List<number>>([...(this._props.faceDoc[DocData].faceDescriptors as List<List<number>>), cur_matching_face]); + + //const match = faceMatcher.findBestMatch(cur_descriptor); + } + }); + return false; + } + return false; + } + + /** + * Toggles whether a Face Document displays its associated docs. + */ + @action + onDisplayClick() { + this._displayImages = !this._displayImages; + } + + /** + * Deletes a Face Document. + */ + @action + deleteFaceDocument = () => { + if (Doc.ActiveDashboard) { + Doc.ActiveDashboard[DocData].faceDocuments = new List<Doc>(DocListCast(Doc.ActiveDashboard[DocData].faceDocuments).filter(doc => doc !== this._props.faceDoc)); + } + }; + + /** + * Deletes a document from a Face Document's associated docs list. + * @param doc + */ + @action + deleteAssociatedDoc = (doc: Doc) => { + this._props.faceDoc[DocData].faceDescriptors = new List<List<number>>( + (this._props.faceDoc[DocData].faceDescriptors as List<List<number>>).filter(fd => !(doc[DocData][`FACE DESCRIPTOR - ${this._props.faceDoc[DocData].label}`] as List<List<number>>).includes(fd)) + ); + doc[DocData][`FACE DESCRIPTOR - ${this._props.faceDoc[DocData].label}`] = new List<List<number>>(); + this._props.faceDoc[DocData].associatedDocs = new List<Doc>(DocListCast(this._props.faceDoc[DocData].associatedDocs).filter(associatedDoc => associatedDoc !== doc)); + }; + + render() { + return ( + <div className="face-document-item" ref={ele => this.createDropTarget(ele!)}> + <div className="face-collection-buttons"> + <IconButton tooltip={'Delete Face From Collection'} onPointerDown={this.deleteFaceDocument} icon={'x'} style={{ width: '4px' }} size={Size.XSMALL} /> + </div> + <div className="face-document-top"> + <h1>{StrCast(this._props.faceDoc[DocData].label)}</h1> + </div> + <IconButton + tooltip={'See image information'} + onPointerDown={() => this.onDisplayClick()} + icon={this._displayImages ? <FontAwesomeIcon icon="caret-up" /> : <FontAwesomeIcon icon="caret-down" />} + color={MarqueeOptionsMenu.Instance.userColor} + style={{ width: '19px' }} + /> + {this._displayImages ? ( + <div className="face-document-image-container"> + {DocListCast(this._props.faceDoc[DocData].associatedDocs).map(doc => { + const [name, type] = ImageCast(doc[Doc.LayoutFieldKey(doc)]).url.href.split('.'); + return ( + <div className="image-wrapper" key={Utils.GenerateGuid()}> + <img + onClick={async () => { + await DocumentView.showDocument(doc, { willZoomCentered: true }); + }} + style={{ maxWidth: '60px', margin: '10px' }} + src={`${name}_o.${type}`} + /> + <div className="remove-item"> + <IconButton + tooltip={'Remove Doc From Face Collection'} + onPointerDown={() => { + this.deleteAssociatedDoc(doc); + }} + icon={'x'} + style={{ width: '4px' }} + size={Size.XSMALL} + /> + </div> + </div> + ); + })} + </div> + ) : ( + <div></div> + )} + </div> + ); + } +} + +@observer +export class FaceCollectionBox extends ViewBoxBaseComponent<FieldViewProps>() { + public static LayoutString(fieldKey: string) { + return FieldView.LayoutString(FaceCollectionBox, fieldKey); + } + + public static Instance: FaceCollectionBox; + + @computed get currentDocs() { + if (Doc.ActiveDashboard) { + return DocListCast(Doc.ActiveDashboard[DocData].faceDocuments); + } else { + return []; + } + } + + constructor(props: any) { + super(props); + makeObservable(this); + FaceCollectionBox.Instance = this; + } + + render() { + return ( + <div className="searchBox-container" style={{ pointerEvents: 'all', color: SnappingManager.userColor, background: SnappingManager.userBackgroundColor }}> + {this.currentDocs.map(doc => { + return <FaceDocumentItem faceDoc={doc} />; + })} + </div> + ); + } +} + +Docs.Prototypes.TemplateMap.set(DocumentType.FACECOLLECTION, { + layout: { view: FaceCollectionBox, dataField: 'data' }, + options: { acl: '', _width: 400 }, +}); diff --git a/src/client/views/collections/collectionFreeForm/ImageLabelBox.tsx b/src/client/views/collections/collectionFreeForm/ImageLabelBox.tsx index af01d6cbc..421b5d0a6 100644 --- a/src/client/views/collections/collectionFreeForm/ImageLabelBox.tsx +++ b/src/client/views/collections/collectionFreeForm/ImageLabelBox.tsx @@ -179,6 +179,7 @@ export class ImageLabelBox extends ViewBoxBaseComponent<FieldViewProps>() { const labels = imageInfo.labels.split('\n'); labels.forEach(label => { label = label.replace(/^\d+\.\s*|-|\*/, '').trim(); + console.log(label); imageInfo.doc[DocData][`${label}`] = true; (imageInfo.doc[DocData].data_labels as List<string>).push(label); }); @@ -198,7 +199,7 @@ export class ImageLabelBox extends ViewBoxBaseComponent<FieldViewProps>() { for (let index = 0; index < (doc[DocData].data_labels as List<string>).length; index++) { const label = (doc[DocData].data_labels as List<string>)[index]; const embedding = await gptGetEmbedding(label); - doc[`data_labels_embedding_${index + 1}`] = new List<number>(embedding); + doc[DocData][`data_labels_embedding_${index + 1}`] = new List<number>(embedding); } } @@ -209,7 +210,7 @@ export class ImageLabelBox extends ViewBoxBaseComponent<FieldViewProps>() { // For each image, loop through the labels, and calculate similarity. Associate it with the // most similar one. this._selectedImages.forEach(doc => { - const embedLists = numberRange((doc[DocData].data_labels as List<string>).length).map(n => Array.from(NumListCast(doc[`data_labels_embedding_${n + 1}`]))); + const embedLists = numberRange((doc[DocData].data_labels as List<string>).length).map(n => Array.from(NumListCast(doc[DocData][`data_labels_embedding_${n + 1}`]))); const bestEmbedScore = (embedding: Opt<number[]>) => Math.max(...embedLists.map((l, index) => (embedding && similarity(Array.from(embedding), l)!) || 0)); const {label: mostSimilarLabelCollect} = this._labelGroups.map(label => ({ label, similarityScore: bestEmbedScore(labelToEmbedding.get(label)) })) diff --git a/src/client/views/search/FaceRecognitionHandler.tsx b/src/client/views/search/FaceRecognitionHandler.tsx index fcd38c42f..ef4622ea2 100644 --- a/src/client/views/search/FaceRecognitionHandler.tsx +++ b/src/client/views/search/FaceRecognitionHandler.tsx @@ -1,14 +1,13 @@ import * as faceapi from 'face-api.js'; -import { FaceMatcher, TinyFaceDetectorOptions } from 'face-api.js'; -import { Doc, DocListCast, NumListCast } from '../../../fields/Doc'; +import { FaceMatcher } from 'face-api.js'; +import { Doc, DocListCast } from '../../../fields/Doc'; import { DocData } from '../../../fields/DocSymbols'; import { List } from '../../../fields/List'; -import { ObjectField } from '../../../fields/ObjectField'; -import { ImageCast, StrCast } from '../../../fields/Types'; -import { DocUtils } from '../../documents/DocUtils'; -import { Deserializable } from '../../util/SerializationHelper'; -import { DocumentView } from '../nodes/DocumentView'; +import { ImageCast, NumCast, StrCast } from '../../../fields/Types'; +/** + * A class that handles face recognition. + */ export class FaceRecognitionHandler { static _instance: FaceRecognitionHandler; private loadedModels: boolean = false; @@ -18,8 +17,12 @@ export class FaceRecognitionHandler { constructor() { FaceRecognitionHandler._instance = this; this.loadModels(); + this.examinedDocs = new Set(DocListCast(Doc.UserDoc()[DocData].examinedFaceDocs, [])); } + /** + * Loads the face detection models. + */ async loadModels() { const MODEL_URL = `/models`; await faceapi.loadFaceDetectionModel(MODEL_URL); @@ -32,19 +35,31 @@ export class FaceRecognitionHandler { return FaceRecognitionHandler._instance ?? new FaceRecognitionHandler(); } + /** + * When a document is added, look for matching face documents. + * @param doc The document being analyzed. + */ public async findMatches(doc: Doc) { if (this.loadedModels) { + // If the Dashboard doesn't have a list of face documents yet, initialize the list. if (!Doc.ActiveDashboard![DocData].faceDocuments) { Doc.ActiveDashboard![DocData].faceDocuments = new List<Doc>(); } + // If the doc is currently already been examined, or it is being processed, stop examining the document. if (this.examinedDocs.has(doc) || this.processingDocs.has(doc)) { return; } + // Mark the document as being processed. this.processingDocs.add(doc); try { + if (!Doc.UserDoc()[DocData].faceDocNum) { + Doc.UserDoc()[DocData].faceDocNum = 0; + } + + // Get the image the document contains and analyze for faces. const [name, type] = ImageCast(doc[Doc.LayoutFieldKey(doc)]).url.href.split('.'); const imageURL = `${name}_o.${type}`; @@ -52,30 +67,51 @@ export class FaceRecognitionHandler { const fullFaceDescriptions = await faceapi.detectAllFaces(img).withFaceLandmarks().withFaceDescriptors(); + doc[DocData].faces = new List<List<number>>(); + + // For each face detected, find a match. for (const fd of fullFaceDescriptions) { - const match = this.findMatch(fd.descriptor); + let match = this.findMatch(fd.descriptor); + let converted_list = new List<number>(); + if (match) { + // If a matching Face Document has been found, add the document to the Face Document's associated docs and append the face + // descriptor to the Face Document's descriptor list. const converted_array = Array.from(fd.descriptor); - const converted_list = new List<number>(converted_array); + converted_list = new List<number>(converted_array); match[DocData].associatedDocs = new List<Doc>([...DocListCast(match[DocData].associatedDocs), doc]); match[DocData].faceDescriptors = new List<List<number>>([...(match[DocData].faceDescriptors as List<List<number>>), converted_list]); } else { + // If a matching Face Document has not been found, create a new Face Document. const newFaceDocument = new Doc(); const converted_array = Array.from(fd.descriptor); - const converted_list = new List<number>(converted_array); + converted_list = new List<number>(converted_array); newFaceDocument[DocData].faceDescriptors = new List<List<number>>(); (newFaceDocument[DocData].faceDescriptors as List<List<number>>).push(converted_list); - newFaceDocument[DocData].label = `Person ${DocListCast(Doc.ActiveDashboard![DocData].faceDocuments).length + 1}`; + Doc.UserDoc()[DocData].faceDocNum = NumCast(Doc.UserDoc()[DocData].faceDocNum) + 1; + newFaceDocument[DocData].label = `Face ${Doc.UserDoc()[DocData].faceDocNum}`; newFaceDocument[DocData].associatedDocs = new List<Doc>([doc]); Doc.ActiveDashboard![DocData].faceDocuments = new List<Doc>([...DocListCast(Doc.ActiveDashboard![DocData].faceDocuments), newFaceDocument]); + match = newFaceDocument; } + + // Assign a field in the document of the matching Face Document. + if (doc[DocData][`FACE DESCRIPTOR - ${match[DocData].label}`]) { + doc[DocData][`FACE DESCRIPTOR - ${match[DocData].label}`] = new List<List<number>>([...(doc[DocData][`FACE DESCRIPTOR - ${match[DocData].label}`] as List<List<number>>), converted_list]); + } else { + doc[DocData][`FACE DESCRIPTOR - ${match[DocData].label}`] = new List<List<number>>([converted_list]); + } + + doc[DocData].faces = new List<List<number>>([...(doc[DocData].faces as List<List<number>>), converted_list]); } + // Updates the examined docs field. this.examinedDocs.add(doc); - console.log(this.examinedDocs); - - DocListCast(Doc.ActiveDashboard![DocData].faceDocuments).forEach(doc => console.log(DocListCast(doc[DocData].associatedDocs))); + if (!Doc.UserDoc()[DocData].examinedFaceDocs) { + Doc.UserDoc()[DocData].examinedFaceDocs = new List<Doc>(); + } + Doc.UserDoc()[DocData].examinedFaceDocs = new List<Doc>([...DocListCast(Doc.UserDoc()[DocData].examinedFaceDocs), doc]); } catch (error) { console.error('Error processing document:', error); } finally { @@ -84,6 +120,11 @@ export class FaceRecognitionHandler { } } + /** + * Finds a matching Face Document given a descriptor + * @param cur_descriptor The current descriptor whose match is being searched for. + * @returns The most similar Face Document. + */ private findMatch(cur_descriptor: Float32Array) { if (DocListCast(Doc.ActiveDashboard![DocData].faceDocuments).length < 1) { return null; @@ -95,19 +136,20 @@ export class FaceRecognitionHandler { }); const faceMatcher = new FaceMatcher(faceDescriptors, 0.6); const match = faceMatcher.findBestMatch(cur_descriptor); - if (match.label == 'unknown') { return null; } else { for (const doc of DocListCast(Doc.ActiveDashboard![DocData].faceDocuments)) { if (doc[DocData].label === match.label) { - console.log(match.label); return doc; } } } } + /** + * Loads an image + */ private loadImage = (src: string): Promise<HTMLImageElement> => { return new Promise((resolve, reject) => { const img = new Image(); diff --git a/src/fields/Doc.ts b/src/fields/Doc.ts index 4abb23404..ebd30ceba 100644 --- a/src/fields/Doc.ts +++ b/src/fields/Doc.ts @@ -265,6 +265,7 @@ export class Doc extends RefField { public static get MyDockedBtns() { return DocCast(Doc.UserDoc().myDockedBtns); } // prettier-ignore public static get MySearcher() { return DocCast(Doc.UserDoc().mySearcher); } // prettier-ignore public static get MyImageGrouper() { return DocCast(Doc.UserDoc().myImageGrouper); } //prettier-ignore + public static get MyFaceCollection() { return DocCast(Doc.UserDoc().myFaceCollection); } //prettier-ignore public static get MyHeaderBar() { return DocCast(Doc.UserDoc().myHeaderBar); } // prettier-ignore public static get MyLeftSidebarMenu() { return DocCast(Doc.UserDoc().myLeftSidebarMenu); } // prettier-ignore public static get MyLeftSidebarPanel() { return DocCast(Doc.UserDoc().myLeftSidebarPanel); } // prettier-ignore |