diff options
author | bobzel <zzzman@gmail.com> | 2024-08-22 11:00:14 -0400 |
---|---|---|
committer | bobzel <zzzman@gmail.com> | 2024-08-22 11:00:14 -0400 |
commit | 95071fa118ee36d7365250f10756dce335dc76d9 (patch) | |
tree | d1af95ed4161414f9d7ef7ef3e647dcb8130bab9 | |
parent | b6275b9d630131e50727f76af44936acc0cb7d7d (diff) |
more cleanup for face collections
-rw-r--r-- | src/client/views/Main.tsx | 2 | ||||
-rw-r--r-- | src/client/views/collections/collectionFreeForm/FaceCollectionBox.tsx | 62 | ||||
-rw-r--r-- | src/client/views/search/FaceRecognitionHandler.tsx | 151 |
3 files changed, 121 insertions, 94 deletions
diff --git a/src/client/views/Main.tsx b/src/client/views/Main.tsx index 5a408f593..023324881 100644 --- a/src/client/views/Main.tsx +++ b/src/client/views/Main.tsx @@ -138,7 +138,7 @@ FieldLoader.ServerLoadStatus = { requested: 0, retrieved: 0, message: 'cache' }; PresBox, PresElementBox, SearchBox, - ImageLabelBox, //Here! + ImageLabelBox, FaceCollectionBox, FunctionPlotBox, InkingStroke, diff --git a/src/client/views/collections/collectionFreeForm/FaceCollectionBox.tsx b/src/client/views/collections/collectionFreeForm/FaceCollectionBox.tsx index dd8dea41e..6005da6dd 100644 --- a/src/client/views/collections/collectionFreeForm/FaceCollectionBox.tsx +++ b/src/client/views/collections/collectionFreeForm/FaceCollectionBox.tsx @@ -26,18 +26,27 @@ import { FaceRecognitionHandler } from '../../search/FaceRecognitionHandler'; import './FaceCollectionBox.scss'; import { MarqueeOptionsMenu } from './MarqueeOptionsMenu'; -interface FaceDocumentProps { +/** + * This code is used to render the sidebar collection of unique recognized faces, where each + * unique face in turn displays the set of images that correspond to the face. + */ + +interface UniqueFaceProps { faceDoc: Doc; } /** - * A componenent to visually represent a Face Document. + * React component for rendering a unique face and its collection of image Docs. + * + * This both displays a collection of images corresponding tp a unique face, and + * allows for editing the face collection by removing an image, or drag-and-dropping + * an image that was not recognized. */ @observer -export class FaceDocumentItem extends ObservableReactComponent<FaceDocumentProps> { +export class UniqueFaceView extends ObservableReactComponent<UniqueFaceProps> { private _dropDisposer?: DragManager.DragDropDisposer; - constructor(props: FaceDocumentProps) { + constructor(props: UniqueFaceProps) { super(props); makeObservable(this); } @@ -54,12 +63,12 @@ export class FaceDocumentItem extends ObservableReactComponent<FaceDocumentProps ?.filter(doc => doc.type === DocumentType.IMG) .forEach(imgDoc => { // If the current Face Document has no faces, and the doc has more than one face descriptor, don't let the user add the document first. Or should we just use the first face ? - if (FaceRecognitionHandler.FaceDocDescriptors(this._props.faceDoc).length === 0 && FaceRecognitionHandler.ImageDocFaceDescriptors(imgDoc).length > 1) { + if (FaceRecognitionHandler.UniqueFaceDescriptors(this._props.faceDoc).length === 0 && FaceRecognitionHandler.ImageDocFaceDescriptors(imgDoc).length > 1) { alert('Cannot add a document with multiple faces as the first item!'); } else { // Loop through the documents' face descriptors and choose the face in the iage with the smallest distance (most similar to the face colleciton) - const faceDescriptorsAsFloat32Array = FaceRecognitionHandler.FaceDocDescriptors(this._props.faceDoc).map(fd => new Float32Array(Array.from(fd))); - const labeledFaceDescriptor = new faceapi.LabeledFaceDescriptors(FaceRecognitionHandler.FaceDocLabel(this._props.faceDoc), faceDescriptorsAsFloat32Array); + const faceDescriptorsAsFloat32Array = FaceRecognitionHandler.UniqueFaceDescriptors(this._props.faceDoc).map(fd => new Float32Array(Array.from(fd))); + const labeledFaceDescriptor = new faceapi.LabeledFaceDescriptors(FaceRecognitionHandler.UniqueFaceLabel(this._props.faceDoc), faceDescriptorsAsFloat32Array); const faceMatcher = new FaceMatcher([labeledFaceDescriptor], 1); const { face_match } = FaceRecognitionHandler.ImageDocFaceDescriptors(imgDoc).reduce( (prev, face) => { @@ -71,8 +80,8 @@ export class FaceDocumentItem extends ObservableReactComponent<FaceDocumentProps // assign the face in the image that's closest to the face collection to be the face that's assigned to the collection if (face_match) { - FaceRecognitionHandler.ImageDocAddFace(imgDoc, face_match, this._props.faceDoc); - FaceRecognitionHandler.FaceDocAddImageDocFace(imgDoc, face_match, this._props.faceDoc); + FaceRecognitionHandler.ImageDocAssociateUniqueFace(imgDoc, face_match, this._props.faceDoc); + FaceRecognitionHandler.UniqueFaceAddFaceImage(imgDoc, face_match, this._props.faceDoc); } } }); @@ -88,28 +97,29 @@ export class FaceDocumentItem extends ObservableReactComponent<FaceDocumentProps } /** - * Deletes a Face Document. + * Removes a unique face Doc from the colelction of unique faces. */ - deleteFaceDocument = undoable(() => { - FaceRecognitionHandler.DeleteFaceDoc(this._props.faceDoc); + deleteUniqueFace = undoable(() => { + FaceRecognitionHandler.DeleteUniqueFace(this._props.faceDoc); }, 'delete face'); /** - * Deletes a document from a Face Document's associated docs list. - * @param doc + * Removes a face image Doc from a unique face's list of images. + * @param imgDoc - image Doc to remove */ - deleteAssociatedDoc = undoable((imgDoc: Doc) => { - FaceRecognitionHandler.FaceDocRemoveImageDocFace(imgDoc, this._props.faceDoc); + removeFaceImageFromUniqueFace = undoable((imgDoc: Doc) => { + FaceRecognitionHandler.ImageDocDeassociateUniqueFace(imgDoc, this._props.faceDoc); + FaceRecognitionHandler.UniqueFaceRemoveFaceImage(imgDoc, this._props.faceDoc); }, 'remove doc from face'); render() { return ( <div className="face-document-item" ref={ele => this.createDropTarget(ele!)}> <div className="face-collection-buttons"> - <IconButton tooltip="Delete Face From Collection" onPointerDown={this.deleteFaceDocument} icon={'x'} style={{ width: '4px' }} size={Size.XSMALL} /> + <IconButton tooltip="Delete Face From Collection" onPointerDown={this.deleteUniqueFace} icon={'x'} style={{ width: '4px' }} size={Size.XSMALL} /> </div> <div className="face-document-top"> - <h1>{FaceRecognitionHandler.FaceDocLabel(this._props.faceDoc)}</h1> + <h1>{FaceRecognitionHandler.UniqueFaceLabel(this._props.faceDoc)}</h1> </div> <IconButton tooltip="See image information" @@ -120,7 +130,7 @@ export class FaceDocumentItem extends ObservableReactComponent<FaceDocumentProps /> {this._displayImages ? ( <div className="face-document-image-container"> - {FaceRecognitionHandler.FaceDocFaces(this._props.faceDoc).map(doc => { + {FaceRecognitionHandler.UniqueFaceImages(this._props.faceDoc).map(doc => { const [name, type] = ImageCast(doc[Doc.LayoutFieldKey(doc)]).url.href.split('.'); return ( <div @@ -140,7 +150,7 @@ export class FaceDocumentItem extends ObservableReactComponent<FaceDocumentProps }> <img onClick={() => DocumentView.showDocument(doc, { willZoomCentered: true })} style={{ maxWidth: '60px', margin: '10px' }} src={`${name}_o.${type}`} /> <div className="remove-item"> - <IconButton tooltip={'Remove Doc From Face Collection'} onPointerDown={() => this.deleteAssociatedDoc(doc)} icon={'x'} style={{ width: '4px' }} size={Size.XSMALL} /> + <IconButton tooltip={'Remove Doc From Face Collection'} onPointerDown={() => this.removeFaceImageFromUniqueFace(doc)} icon={'x'} style={{ width: '4px' }} size={Size.XSMALL} /> </div> </div> ); @@ -152,6 +162,14 @@ export class FaceDocumentItem extends ObservableReactComponent<FaceDocumentProps } } +/** + * This renders the sidebar collection of the unique faces that have been recognized. + * + * Since the collection of recognized faces is stored on the active dashboard, this class + * does not itself store any Docs, but accesses the uniqueFaces field of the current + * dashboard. Each Face collection Doc is rendered using a FaceCollectionDocView which + * is not a CollectionView or even a DocumentView (but probably should be). + */ @observer export class FaceCollectionBox extends ViewBoxBaseComponent<FieldViewProps>() { public static LayoutString(fieldKey: string) { @@ -166,8 +184,8 @@ export class FaceCollectionBox extends ViewBoxBaseComponent<FieldViewProps>() { render() { return ( <div className="searchBox-container" style={{ pointerEvents: 'all', color: SnappingManager.userColor, background: SnappingManager.userBackgroundColor }}> - {FaceRecognitionHandler.FaceDocuments().map(doc => ( - <FaceDocumentItem key={doc[Id]} faceDoc={doc} /> + {FaceRecognitionHandler.UniqueFaces().map(doc => ( + <UniqueFaceView key={doc[Id]} faceDoc={doc} /> ))} </div> ); diff --git a/src/client/views/search/FaceRecognitionHandler.tsx b/src/client/views/search/FaceRecognitionHandler.tsx index c446df6ff..c239c775c 100644 --- a/src/client/views/search/FaceRecognitionHandler.tsx +++ b/src/client/views/search/FaceRecognitionHandler.tsx @@ -1,6 +1,5 @@ import * as faceapi from 'face-api.js'; import { FaceMatcher } from 'face-api.js'; -import { computed } from 'mobx'; import { Doc, DocListCast } from '../../../fields/Doc'; import { DocData } from '../../../fields/DocSymbols'; import { List } from '../../../fields/List'; @@ -12,28 +11,27 @@ import { DocumentManager } from '../../util/DocumentManager'; /** * A singleton class that handles face recognition and manages face Doc collections for each face found. * Displaying an image doc anywhere will trigger this class to test if the image contains any faces. - * If it does, each recognized face will be compared to a global set of faces (each is a face collection Doc - * that have already been found. If the face matches a face collection Doc, then it will be added to that + * If it does, each recognized face will be compared to a stored, global set of faces (each face is represented + * as a face collection Doc). If the face matches a face collection Doc, then it will be added to that * collection along with the numerical representation of the face, its face descriptor. * * Image Doc's that are added to one or more face collection Docs will be given these metadata fields: - * <image data field>_Face<N> - a nunerical representation of the Nth face found in the image - * <image data field>_Faces - a list of all the numerical face representations found in the image (why is this needed?) + * <image data field>_Face<N> - a numerical representation of the Nth face found in the image + * <image data field>_Faces - a list of all the numerical face representations found in the image. (TODO: this is inelegant as it duplicates each Face<N>) * - * Face collection Doc's are created for each person identified and are stored in the Dashboard's faceDocument's list + * unique face Doc's are created for each person identified and are stored in the Dashboard's uniqueFaces field * - * Each Face collection Doc represents all the images found for that person. It has these fields: - * face_label - a string label for the person that was recognized (currently it's just a 'face#') + * Each unique face Doc represents a unique face and collects all matching face images for that person. It has these fields: + * face_label - a string label for the person that was recognized (TODO: currently it's just a 'face#') * face_descriptors - a list of all the face descriptors for different images of the person * face_docList - a list of all image Docs that contain a face for the person */ export class FaceRecognitionHandler { static _instance: FaceRecognitionHandler; private _loadedModels: boolean = false; - private _processingDocs: Set<Doc> = new Set(); private _pendingLoadDocs: Doc[] = []; - private static imgDocFaceField = (imgDoc: Doc, faceDoc: Doc) => `${Doc.LayoutFieldKey(imgDoc)}_${FaceRecognitionHandler.FaceDocLabel(faceDoc)}`; + private static imgDocFaceField = (imgDoc: Doc, faceDoc: Doc) => `${Doc.LayoutFieldKey(imgDoc)}_${FaceRecognitionHandler.UniqueFaceLabel(faceDoc)}`; /** * initializes an image with an empty list of face descriptors * @param imgDoc image to initialize @@ -49,67 +47,82 @@ export class FaceRecognitionHandler { public static ImageDocFaceDescriptors = (imgDoc: Doc) => imgDoc[DocData][`${Doc.LayoutFieldKey(imgDoc)}_Faces`] as List<List<number>>; /** - * Adds metadata to an image Doc describing a face found in the image + * Adds metadata to an image Doc associating it to a unique face that corresponds to a face found in the image * @param imgDoc image Doc containing faces - * @param faceDescriptor descriptor for the face found - * @param faceDoc face collection Doc containing the same face + * @param faceDescriptor descriptor for the face + * @param faceDoc unique face */ - public static ImageDocAddFace = (imgDoc: Doc, faceDescriptor: List<number>, faceDoc: Doc) => { + public static ImageDocAssociateUniqueFace = (imgDoc: Doc, faceDescriptor: List<number>, faceDoc: Doc) => { const faceFieldKey = FaceRecognitionHandler.imgDocFaceField(imgDoc, faceDoc); if (imgDoc[DocData][faceFieldKey]) { Cast(imgDoc[DocData][faceFieldKey], listSpec('number'), null).push(faceDescriptor as unknown as number); // items are lists of numbers, not numbers, but type system can't handle that } else { imgDoc[DocData][faceFieldKey] = new List<List<number>>([faceDescriptor]); } + Cast(imgDoc[DocData][`${Doc.LayoutFieldKey(imgDoc)}_Faces`], listSpec('number'), null).push(faceDescriptor as unknown as number); + }; + + /** + * Removes metadata from an image Doc to deassociate it from a unique face + * @param imgDoc image Doc containing faces + * @param faceDoc unique face + */ + public static ImageDocDeassociateUniqueFace = (imgDoc: Doc, faceDoc: Doc) => { + // fill in.. }; /** * returns a list of all face collection Docs on the current dashboard * @returns face collection Doc list */ - public static FaceDocuments = () => DocListCast(Doc.ActiveDashboard?.[DocData].faceDocuments); + public static UniqueFaces = () => DocListCast(Doc.ActiveDashboard?.[DocData].uniqueFaces); - public static DeleteFaceDoc = (faceDoc: Doc) => Doc.ActiveDashboard && Doc.RemoveDocFromList(Doc.ActiveDashboard[DocData], 'faceDocuments', faceDoc); + /** + * Removes a unique face from the set of recognized unique faces + * @param faceDoc unique face Doc + * @returns + */ + public static DeleteUniqueFace = (faceDoc: Doc) => Doc.ActiveDashboard && Doc.RemoveDocFromList(Doc.ActiveDashboard[DocData], 'uniqueFaces', faceDoc); /** * returns the labels associated with a face collection Doc - * @param faceDoc the face collection Doc + * @param faceDoc unique face Doc * @returns label string */ - public static FaceDocLabel = (faceDoc: Doc) => StrCast(faceDoc[DocData].face_label); + public static UniqueFaceLabel = (faceDoc: Doc) => StrCast(faceDoc[DocData].face_label); /** - * Returns all the face descriptors associated with a face collection Doc - * @param faceDoc a face collection Doc + * Returns all the face descriptors associated with a unique face Doc + * @param faceDoc unique face Doc * @returns face descriptors */ - public static FaceDocDescriptors = (faceDoc: Doc) => faceDoc[DocData].face_descriptors as List<List<number>>; + public static UniqueFaceDescriptors = (faceDoc: Doc) => faceDoc[DocData].face_descriptors as List<List<number>>; /** - * Returns a list of all face image Docs associated with the face collection - * @param faceDoc a face collection Doc + * Returns a list of all face image Docs associated with a unique face Doc + * @param faceDoc unique face Doc * @returns image Docs */ - public static FaceDocFaces = (faceDoc: Doc) => DocListCast(faceDoc[DocData].face_docList); + public static UniqueFaceImages = (faceDoc: Doc) => DocListCast(faceDoc[DocData].face_images); /** - * Adds a face image to the list of faces in a face collection Doc, and updates the face collection's list of image descriptors + * Adds a face image to a unique face Doc, and updates the unique face's set of face image descriptors * @param img - image with faces to add to a face collection Doc * @param faceDescriptor - the face descriptor for the face in the image to add - * @param faceDoc - the face collection Doc + * @param faceDoc - unique face Doc */ - public static FaceDocAddImageDocFace = (img: Doc, faceDescriptor: List<number>, faceDoc: Doc) => { - Doc.AddDocToList(faceDoc, 'face_docList', img); + public static UniqueFaceAddFaceImage = (img: Doc, faceDescriptor: List<number>, faceDoc: Doc) => { + Doc.AddDocToList(faceDoc, 'face_images', img); Cast(faceDoc.face_descriptors, listSpec('number'), null).push(faceDescriptor as unknown as number); // items are lists of numbers, not numbers, but type system can't handle that }; /** - * Removes a face from a face Doc collection, and updates the face collection's list of image descriptors - * @param imgDoc - image with faces to remove from the face Doc collectoin - * @param faceDoc - the face Doc collection + * Removes a face from a unique Face Doc, and updates the unique face's set of face image descriptors + * @param imgDoc - image with faces to remove + * @param faceDoc - unique face Doc */ - public static FaceDocRemoveImageDocFace = (imgDoc: Doc, faceDoc: Doc) => { - Doc.RemoveDocFromList(faceDoc[DocData], 'face_docList', imgDoc); - faceDoc[DocData].face_descriptors = new List<List<number>>(FaceRecognitionHandler.FaceDocDescriptors(faceDoc).filter(fd => !(imgDoc[DocData][FaceRecognitionHandler.imgDocFaceField(imgDoc, faceDoc)] as List<List<number>>).includes(fd))); + public static UniqueFaceRemoveFaceImage = (imgDoc: Doc, faceDoc: Doc) => { + Doc.RemoveDocFromList(faceDoc[DocData], 'face_images', imgDoc); + faceDoc[DocData].face_descriptors = new List<List<number>>(FaceRecognitionHandler.UniqueFaceDescriptors(faceDoc).filter(fd => !(imgDoc[DocData][FaceRecognitionHandler.imgDocFaceField(imgDoc, faceDoc)] as List<List<number>>).includes(fd))); }; constructor() { @@ -118,10 +131,6 @@ export class FaceRecognitionHandler { DocumentManager.Instance.AddAnyViewRenderedCB(dv => FaceRecognitionHandler.Instance.classifyFacesInImage(dv.Document)); } - @computed get examinedFaceDocs() { - return DocListCast(Doc.UserDoc().examinedFaceDocs); - } - /** * Loads the face detection models. */ @@ -138,7 +147,27 @@ export class FaceRecognitionHandler { } /** - * When a document is added, look for matching face documents. + * Creates a new, empty unique face Doc + * @returns a unique face Doc + */ + createUniqueFaceDoc = (dashboard: Doc) => { + const faceDocNum = NumCast(dashboard.uniqueFaces_count) + 1; + dashboard.uniqueFaces_count = faceDocNum; // TODO: improve to a better name + + const uniqueFaceDoc = new Doc(); + uniqueFaceDoc.title = `Face ${faceDocNum}`; + uniqueFaceDoc.face = ''; // just to make prettyprinting look better + uniqueFaceDoc.face_label = `Face${faceDocNum}`; + uniqueFaceDoc.face_images = new List<Doc>(); + uniqueFaceDoc.face_descriptors = new List<List<number>>(); + + Doc.ActiveDashboard && Doc.AddDocToList(Doc.ActiveDashboard[DocData], 'uniqueFaces', uniqueFaceDoc); + return uniqueFaceDoc; + }; + + /** + * When a document is added, this finds faces in the images and tries to + * match them to existing unique faces, otherwise new unique face(s) are created. * @param imgDoc The document being analyzed. */ public classifyFacesInImage = async (imgDoc: Doc) => { @@ -154,12 +183,11 @@ export class FaceRecognitionHandler { const imgUrl = ImageCast(imgDoc[Doc.LayoutFieldKey(imgDoc)]); // If the doc isn't an image or currently already been examined or is being processed, stop examining the document. - if (!imgUrl || this.examinedFaceDocs.includes(imgDoc) || this._processingDocs.has(imgDoc)) { + if (!imgUrl || DocListCast(Doc.MyFaceCollection.examinedFaceDocs).includes(imgDoc)) { return; } + Doc.AddDocToList(Doc.MyFaceCollection, 'examinedFaceDocs', imgDoc); - // Mark the document as being processed. - this._processingDocs.add(imgDoc); FaceRecognitionHandler.initImageDocFaceDescriptors(imgDoc); // Get the image the document contains and analyze for faces. @@ -170,31 +198,12 @@ export class FaceRecognitionHandler { // For each face detected, find a match. for (const fd of imgDocFaceDescriptions) { - let faceDocMatch = this.findMatchingFaceDoc(fd.descriptor); const faceDescriptor = new List<number>(Array.from(fd.descriptor)); - - if (faceDocMatch) { - FaceRecognitionHandler.FaceDocAddImageDocFace(imgDoc, faceDescriptor, faceDocMatch); - } else { - // If a matching Face Document has not been found, create a new Face Document. - Doc.UserDoc().faceDocNum = NumCast(Doc.UserDoc().faceDocNum) + 1; - - const newFaceDocument = new Doc(); - newFaceDocument.title = `Face ${Doc.UserDoc().faceDocNum}`; - newFaceDocument.face = ''; // just to make prettyprinting look better - newFaceDocument.face_label = `Face${Doc.UserDoc().faceDocNum}`; - newFaceDocument.face_docList = new List<Doc>([imgDoc]); - newFaceDocument.face_descriptors = new List<List<number>>([faceDescriptor]); - - Doc.AddDocToList(Doc.ActiveDashboard[DocData], 'faceDocuments', newFaceDocument); - faceDocMatch = newFaceDocument; - } - - // Assign a field in the document of the matching Face Document. - FaceRecognitionHandler.ImageDocAddFace(imgDoc, faceDescriptor, faceDocMatch); - Doc.AddDocToList(Doc.UserDoc(), 'examinedFaceDocs', imgDoc); + const matchedUniqueFace = this.findMatchingFaceDoc(fd.descriptor) ?? this.createUniqueFaceDoc(Doc.ActiveDashboard); + // Add image to unique face's image collection, and assign image metadata referencing unique face + FaceRecognitionHandler.UniqueFaceAddFaceImage(imgDoc, faceDescriptor, matchedUniqueFace); + FaceRecognitionHandler.ImageDocAssociateUniqueFace(imgDoc, faceDescriptor, matchedUniqueFace); } - this._processingDocs.delete(imgDoc); }; /** @@ -203,19 +212,19 @@ export class FaceRecognitionHandler { * @returns face Doc */ private findMatchingFaceDoc = (faceDescriptor: Float32Array) => { - if (!Doc.ActiveDashboard || FaceRecognitionHandler.FaceDocuments().length < 1) { + if (!Doc.ActiveDashboard || FaceRecognitionHandler.UniqueFaces().length < 1) { return undefined; } - const faceDescriptors = FaceRecognitionHandler.FaceDocuments().map(faceDoc => { - const float32Array = FaceRecognitionHandler.FaceDocDescriptors(faceDoc).map(fd => new Float32Array(Array.from(fd))); - return new faceapi.LabeledFaceDescriptors(FaceRecognitionHandler.FaceDocLabel(faceDoc), float32Array); + const faceDescriptors = FaceRecognitionHandler.UniqueFaces().map(faceDoc => { + const float32Array = FaceRecognitionHandler.UniqueFaceDescriptors(faceDoc).map(fd => new Float32Array(Array.from(fd))); + return new faceapi.LabeledFaceDescriptors(FaceRecognitionHandler.UniqueFaceLabel(faceDoc), float32Array); }); const faceMatcher = new FaceMatcher(faceDescriptors, 0.6); const match = faceMatcher.findBestMatch(faceDescriptor); if (match.label !== 'unknown') { - for (const faceDoc of FaceRecognitionHandler.FaceDocuments()) { - if (FaceRecognitionHandler.FaceDocLabel(faceDoc) === match.label) { + for (const faceDoc of FaceRecognitionHandler.UniqueFaces()) { + if (FaceRecognitionHandler.UniqueFaceLabel(faceDoc) === match.label) { return faceDoc; } } |