diff options
Diffstat (limited to 'src/client/views/search')
| -rw-r--r-- | src/client/views/search/FaceRecognitionHandler.tsx | 151 |
1 files changed, 80 insertions, 71 deletions
diff --git a/src/client/views/search/FaceRecognitionHandler.tsx b/src/client/views/search/FaceRecognitionHandler.tsx index c446df6ff..c239c775c 100644 --- a/src/client/views/search/FaceRecognitionHandler.tsx +++ b/src/client/views/search/FaceRecognitionHandler.tsx @@ -1,6 +1,5 @@ import * as faceapi from 'face-api.js'; import { FaceMatcher } from 'face-api.js'; -import { computed } from 'mobx'; import { Doc, DocListCast } from '../../../fields/Doc'; import { DocData } from '../../../fields/DocSymbols'; import { List } from '../../../fields/List'; @@ -12,28 +11,27 @@ import { DocumentManager } from '../../util/DocumentManager'; /** * A singleton class that handles face recognition and manages face Doc collections for each face found. * Displaying an image doc anywhere will trigger this class to test if the image contains any faces. - * If it does, each recognized face will be compared to a global set of faces (each is a face collection Doc - * that have already been found. If the face matches a face collection Doc, then it will be added to that + * If it does, each recognized face will be compared to a stored, global set of faces (each face is represented + * as a face collection Doc). If the face matches a face collection Doc, then it will be added to that * collection along with the numerical representation of the face, its face descriptor. * * Image Doc's that are added to one or more face collection Docs will be given these metadata fields: - * <image data field>_Face<N> - a nunerical representation of the Nth face found in the image - * <image data field>_Faces - a list of all the numerical face representations found in the image (why is this needed?) + * <image data field>_Face<N> - a numerical representation of the Nth face found in the image + * <image data field>_Faces - a list of all the numerical face representations found in the image. (TODO: this is inelegant as it duplicates each Face<N>) * - * Face collection Doc's are created for each person identified and are stored in the Dashboard's faceDocument's list + * unique face Doc's are created for each person identified and are stored in the Dashboard's uniqueFaces field * - * Each Face collection Doc represents all the images found for that person. It has these fields: - * face_label - a string label for the person that was recognized (currently it's just a 'face#') + * Each unique face Doc represents a unique face and collects all matching face images for that person. It has these fields: + * face_label - a string label for the person that was recognized (TODO: currently it's just a 'face#') * face_descriptors - a list of all the face descriptors for different images of the person * face_docList - a list of all image Docs that contain a face for the person */ export class FaceRecognitionHandler { static _instance: FaceRecognitionHandler; private _loadedModels: boolean = false; - private _processingDocs: Set<Doc> = new Set(); private _pendingLoadDocs: Doc[] = []; - private static imgDocFaceField = (imgDoc: Doc, faceDoc: Doc) => `${Doc.LayoutFieldKey(imgDoc)}_${FaceRecognitionHandler.FaceDocLabel(faceDoc)}`; + private static imgDocFaceField = (imgDoc: Doc, faceDoc: Doc) => `${Doc.LayoutFieldKey(imgDoc)}_${FaceRecognitionHandler.UniqueFaceLabel(faceDoc)}`; /** * initializes an image with an empty list of face descriptors * @param imgDoc image to initialize @@ -49,67 +47,82 @@ export class FaceRecognitionHandler { public static ImageDocFaceDescriptors = (imgDoc: Doc) => imgDoc[DocData][`${Doc.LayoutFieldKey(imgDoc)}_Faces`] as List<List<number>>; /** - * Adds metadata to an image Doc describing a face found in the image + * Adds metadata to an image Doc associating it to a unique face that corresponds to a face found in the image * @param imgDoc image Doc containing faces - * @param faceDescriptor descriptor for the face found - * @param faceDoc face collection Doc containing the same face + * @param faceDescriptor descriptor for the face + * @param faceDoc unique face */ - public static ImageDocAddFace = (imgDoc: Doc, faceDescriptor: List<number>, faceDoc: Doc) => { + public static ImageDocAssociateUniqueFace = (imgDoc: Doc, faceDescriptor: List<number>, faceDoc: Doc) => { const faceFieldKey = FaceRecognitionHandler.imgDocFaceField(imgDoc, faceDoc); if (imgDoc[DocData][faceFieldKey]) { Cast(imgDoc[DocData][faceFieldKey], listSpec('number'), null).push(faceDescriptor as unknown as number); // items are lists of numbers, not numbers, but type system can't handle that } else { imgDoc[DocData][faceFieldKey] = new List<List<number>>([faceDescriptor]); } + Cast(imgDoc[DocData][`${Doc.LayoutFieldKey(imgDoc)}_Faces`], listSpec('number'), null).push(faceDescriptor as unknown as number); + }; + + /** + * Removes metadata from an image Doc to deassociate it from a unique face + * @param imgDoc image Doc containing faces + * @param faceDoc unique face + */ + public static ImageDocDeassociateUniqueFace = (imgDoc: Doc, faceDoc: Doc) => { + // fill in.. }; /** * returns a list of all face collection Docs on the current dashboard * @returns face collection Doc list */ - public static FaceDocuments = () => DocListCast(Doc.ActiveDashboard?.[DocData].faceDocuments); + public static UniqueFaces = () => DocListCast(Doc.ActiveDashboard?.[DocData].uniqueFaces); - public static DeleteFaceDoc = (faceDoc: Doc) => Doc.ActiveDashboard && Doc.RemoveDocFromList(Doc.ActiveDashboard[DocData], 'faceDocuments', faceDoc); + /** + * Removes a unique face from the set of recognized unique faces + * @param faceDoc unique face Doc + * @returns + */ + public static DeleteUniqueFace = (faceDoc: Doc) => Doc.ActiveDashboard && Doc.RemoveDocFromList(Doc.ActiveDashboard[DocData], 'uniqueFaces', faceDoc); /** * returns the labels associated with a face collection Doc - * @param faceDoc the face collection Doc + * @param faceDoc unique face Doc * @returns label string */ - public static FaceDocLabel = (faceDoc: Doc) => StrCast(faceDoc[DocData].face_label); + public static UniqueFaceLabel = (faceDoc: Doc) => StrCast(faceDoc[DocData].face_label); /** - * Returns all the face descriptors associated with a face collection Doc - * @param faceDoc a face collection Doc + * Returns all the face descriptors associated with a unique face Doc + * @param faceDoc unique face Doc * @returns face descriptors */ - public static FaceDocDescriptors = (faceDoc: Doc) => faceDoc[DocData].face_descriptors as List<List<number>>; + public static UniqueFaceDescriptors = (faceDoc: Doc) => faceDoc[DocData].face_descriptors as List<List<number>>; /** - * Returns a list of all face image Docs associated with the face collection - * @param faceDoc a face collection Doc + * Returns a list of all face image Docs associated with a unique face Doc + * @param faceDoc unique face Doc * @returns image Docs */ - public static FaceDocFaces = (faceDoc: Doc) => DocListCast(faceDoc[DocData].face_docList); + public static UniqueFaceImages = (faceDoc: Doc) => DocListCast(faceDoc[DocData].face_images); /** - * Adds a face image to the list of faces in a face collection Doc, and updates the face collection's list of image descriptors + * Adds a face image to a unique face Doc, and updates the unique face's set of face image descriptors * @param img - image with faces to add to a face collection Doc * @param faceDescriptor - the face descriptor for the face in the image to add - * @param faceDoc - the face collection Doc + * @param faceDoc - unique face Doc */ - public static FaceDocAddImageDocFace = (img: Doc, faceDescriptor: List<number>, faceDoc: Doc) => { - Doc.AddDocToList(faceDoc, 'face_docList', img); + public static UniqueFaceAddFaceImage = (img: Doc, faceDescriptor: List<number>, faceDoc: Doc) => { + Doc.AddDocToList(faceDoc, 'face_images', img); Cast(faceDoc.face_descriptors, listSpec('number'), null).push(faceDescriptor as unknown as number); // items are lists of numbers, not numbers, but type system can't handle that }; /** - * Removes a face from a face Doc collection, and updates the face collection's list of image descriptors - * @param imgDoc - image with faces to remove from the face Doc collectoin - * @param faceDoc - the face Doc collection + * Removes a face from a unique Face Doc, and updates the unique face's set of face image descriptors + * @param imgDoc - image with faces to remove + * @param faceDoc - unique face Doc */ - public static FaceDocRemoveImageDocFace = (imgDoc: Doc, faceDoc: Doc) => { - Doc.RemoveDocFromList(faceDoc[DocData], 'face_docList', imgDoc); - faceDoc[DocData].face_descriptors = new List<List<number>>(FaceRecognitionHandler.FaceDocDescriptors(faceDoc).filter(fd => !(imgDoc[DocData][FaceRecognitionHandler.imgDocFaceField(imgDoc, faceDoc)] as List<List<number>>).includes(fd))); + public static UniqueFaceRemoveFaceImage = (imgDoc: Doc, faceDoc: Doc) => { + Doc.RemoveDocFromList(faceDoc[DocData], 'face_images', imgDoc); + faceDoc[DocData].face_descriptors = new List<List<number>>(FaceRecognitionHandler.UniqueFaceDescriptors(faceDoc).filter(fd => !(imgDoc[DocData][FaceRecognitionHandler.imgDocFaceField(imgDoc, faceDoc)] as List<List<number>>).includes(fd))); }; constructor() { @@ -118,10 +131,6 @@ export class FaceRecognitionHandler { DocumentManager.Instance.AddAnyViewRenderedCB(dv => FaceRecognitionHandler.Instance.classifyFacesInImage(dv.Document)); } - @computed get examinedFaceDocs() { - return DocListCast(Doc.UserDoc().examinedFaceDocs); - } - /** * Loads the face detection models. */ @@ -138,7 +147,27 @@ export class FaceRecognitionHandler { } /** - * When a document is added, look for matching face documents. + * Creates a new, empty unique face Doc + * @returns a unique face Doc + */ + createUniqueFaceDoc = (dashboard: Doc) => { + const faceDocNum = NumCast(dashboard.uniqueFaces_count) + 1; + dashboard.uniqueFaces_count = faceDocNum; // TODO: improve to a better name + + const uniqueFaceDoc = new Doc(); + uniqueFaceDoc.title = `Face ${faceDocNum}`; + uniqueFaceDoc.face = ''; // just to make prettyprinting look better + uniqueFaceDoc.face_label = `Face${faceDocNum}`; + uniqueFaceDoc.face_images = new List<Doc>(); + uniqueFaceDoc.face_descriptors = new List<List<number>>(); + + Doc.ActiveDashboard && Doc.AddDocToList(Doc.ActiveDashboard[DocData], 'uniqueFaces', uniqueFaceDoc); + return uniqueFaceDoc; + }; + + /** + * When a document is added, this finds faces in the images and tries to + * match them to existing unique faces, otherwise new unique face(s) are created. * @param imgDoc The document being analyzed. */ public classifyFacesInImage = async (imgDoc: Doc) => { @@ -154,12 +183,11 @@ export class FaceRecognitionHandler { const imgUrl = ImageCast(imgDoc[Doc.LayoutFieldKey(imgDoc)]); // If the doc isn't an image or currently already been examined or is being processed, stop examining the document. - if (!imgUrl || this.examinedFaceDocs.includes(imgDoc) || this._processingDocs.has(imgDoc)) { + if (!imgUrl || DocListCast(Doc.MyFaceCollection.examinedFaceDocs).includes(imgDoc)) { return; } + Doc.AddDocToList(Doc.MyFaceCollection, 'examinedFaceDocs', imgDoc); - // Mark the document as being processed. - this._processingDocs.add(imgDoc); FaceRecognitionHandler.initImageDocFaceDescriptors(imgDoc); // Get the image the document contains and analyze for faces. @@ -170,31 +198,12 @@ export class FaceRecognitionHandler { // For each face detected, find a match. for (const fd of imgDocFaceDescriptions) { - let faceDocMatch = this.findMatchingFaceDoc(fd.descriptor); const faceDescriptor = new List<number>(Array.from(fd.descriptor)); - - if (faceDocMatch) { - FaceRecognitionHandler.FaceDocAddImageDocFace(imgDoc, faceDescriptor, faceDocMatch); - } else { - // If a matching Face Document has not been found, create a new Face Document. - Doc.UserDoc().faceDocNum = NumCast(Doc.UserDoc().faceDocNum) + 1; - - const newFaceDocument = new Doc(); - newFaceDocument.title = `Face ${Doc.UserDoc().faceDocNum}`; - newFaceDocument.face = ''; // just to make prettyprinting look better - newFaceDocument.face_label = `Face${Doc.UserDoc().faceDocNum}`; - newFaceDocument.face_docList = new List<Doc>([imgDoc]); - newFaceDocument.face_descriptors = new List<List<number>>([faceDescriptor]); - - Doc.AddDocToList(Doc.ActiveDashboard[DocData], 'faceDocuments', newFaceDocument); - faceDocMatch = newFaceDocument; - } - - // Assign a field in the document of the matching Face Document. - FaceRecognitionHandler.ImageDocAddFace(imgDoc, faceDescriptor, faceDocMatch); - Doc.AddDocToList(Doc.UserDoc(), 'examinedFaceDocs', imgDoc); + const matchedUniqueFace = this.findMatchingFaceDoc(fd.descriptor) ?? this.createUniqueFaceDoc(Doc.ActiveDashboard); + // Add image to unique face's image collection, and assign image metadata referencing unique face + FaceRecognitionHandler.UniqueFaceAddFaceImage(imgDoc, faceDescriptor, matchedUniqueFace); + FaceRecognitionHandler.ImageDocAssociateUniqueFace(imgDoc, faceDescriptor, matchedUniqueFace); } - this._processingDocs.delete(imgDoc); }; /** @@ -203,19 +212,19 @@ export class FaceRecognitionHandler { * @returns face Doc */ private findMatchingFaceDoc = (faceDescriptor: Float32Array) => { - if (!Doc.ActiveDashboard || FaceRecognitionHandler.FaceDocuments().length < 1) { + if (!Doc.ActiveDashboard || FaceRecognitionHandler.UniqueFaces().length < 1) { return undefined; } - const faceDescriptors = FaceRecognitionHandler.FaceDocuments().map(faceDoc => { - const float32Array = FaceRecognitionHandler.FaceDocDescriptors(faceDoc).map(fd => new Float32Array(Array.from(fd))); - return new faceapi.LabeledFaceDescriptors(FaceRecognitionHandler.FaceDocLabel(faceDoc), float32Array); + const faceDescriptors = FaceRecognitionHandler.UniqueFaces().map(faceDoc => { + const float32Array = FaceRecognitionHandler.UniqueFaceDescriptors(faceDoc).map(fd => new Float32Array(Array.from(fd))); + return new faceapi.LabeledFaceDescriptors(FaceRecognitionHandler.UniqueFaceLabel(faceDoc), float32Array); }); const faceMatcher = new FaceMatcher(faceDescriptors, 0.6); const match = faceMatcher.findBestMatch(faceDescriptor); if (match.label !== 'unknown') { - for (const faceDoc of FaceRecognitionHandler.FaceDocuments()) { - if (FaceRecognitionHandler.FaceDocLabel(faceDoc) === match.label) { + for (const faceDoc of FaceRecognitionHandler.UniqueFaces()) { + if (FaceRecognitionHandler.UniqueFaceLabel(faceDoc) === match.label) { return faceDoc; } } |