From 95071fa118ee36d7365250f10756dce335dc76d9 Mon Sep 17 00:00:00 2001
From: bobzel <zzzman@gmail.com>
Date: Thu, 22 Aug 2024 11:00:14 -0400
Subject: more cleanup for face collections

---
 src/client/views/search/FaceRecognitionHandler.tsx | 151 +++++++++++----------
 1 file changed, 80 insertions(+), 71 deletions(-)

(limited to 'src/client/views/search/FaceRecognitionHandler.tsx')

diff --git a/src/client/views/search/FaceRecognitionHandler.tsx b/src/client/views/search/FaceRecognitionHandler.tsx
index c446df6ff..c239c775c 100644
--- a/src/client/views/search/FaceRecognitionHandler.tsx
+++ b/src/client/views/search/FaceRecognitionHandler.tsx
@@ -1,6 +1,5 @@
 import * as faceapi from 'face-api.js';
 import { FaceMatcher } from 'face-api.js';
-import { computed } from 'mobx';
 import { Doc, DocListCast } from '../../../fields/Doc';
 import { DocData } from '../../../fields/DocSymbols';
 import { List } from '../../../fields/List';
@@ -12,28 +11,27 @@ import { DocumentManager } from '../../util/DocumentManager';
 /**
  * A singleton class that handles face recognition and manages face Doc collections for each face found.
  * Displaying an image doc anywhere will trigger this class to test if the image contains any faces.
- * If it does, each recognized face will be compared to a global set of faces (each is a face collection Doc
- * that have already been found. If the face matches a face collection Doc, then it will be added to that
+ * If it does, each recognized face will be compared to a stored, global set of faces (each face is represented
+ * as a face collection Doc). If the face matches a face collection Doc, then it will be added to that
  * collection along with the numerical representation of the face, its face descriptor.
  *
  * Image Doc's that are added to one or more face collection Docs will be given these metadata fields:
- *     <image data field>_Face<N> - a nunerical representation of the Nth face found in the image
- *     <image data field>_Faces  - a list of all the numerical face representations found in the image (why is this needed?)
+ *     <image data field>_Face<N> - a numerical representation of the Nth face found in the image
+ *     <image data field>_Faces  - a list of all the numerical face representations found in the image.   (TODO: this is inelegant as it duplicates each Face<N>)
  *
- * Face collection Doc's are created for each person identified and are stored in the Dashboard's faceDocument's list
+ * unique face Doc's are created for each person identified and are stored in the Dashboard's uniqueFaces field
  *
- * Each Face collection Doc represents all the images found for that person.  It has these fields:
- *     face_label - a string label for the person that was recognized (currently it's just a 'face#')
+ * Each unique face Doc represents a unique face and collects all matching face images for that person.  It has these fields:
+ *     face_label - a string label for the person that was recognized                                     (TODO: currently it's just a 'face#')
  *     face_descriptors - a list of all the face descriptors for different images of the person
  *     face_docList - a list of all image Docs that contain a face for the person
  */
 export class FaceRecognitionHandler {
     static _instance: FaceRecognitionHandler;
     private _loadedModels: boolean = false;
-    private _processingDocs: Set<Doc> = new Set();
     private _pendingLoadDocs: Doc[] = [];
 
-    private static imgDocFaceField = (imgDoc: Doc, faceDoc: Doc) => `${Doc.LayoutFieldKey(imgDoc)}_${FaceRecognitionHandler.FaceDocLabel(faceDoc)}`;
+    private static imgDocFaceField = (imgDoc: Doc, faceDoc: Doc) => `${Doc.LayoutFieldKey(imgDoc)}_${FaceRecognitionHandler.UniqueFaceLabel(faceDoc)}`;
     /**
      * initializes an image with an empty list of face descriptors
      * @param imgDoc image to initialize
@@ -49,67 +47,82 @@ export class FaceRecognitionHandler {
     public static ImageDocFaceDescriptors = (imgDoc: Doc) => imgDoc[DocData][`${Doc.LayoutFieldKey(imgDoc)}_Faces`] as List<List<number>>;
 
     /**
-     * Adds metadata to an image Doc describing a face found in the image
+     * Adds metadata to an image Doc associating it to a unique face that corresponds to a face found in the image
      * @param imgDoc image Doc containing faces
-     * @param faceDescriptor descriptor for the face found
-     * @param faceDoc face collection Doc containing the same face
+     * @param faceDescriptor descriptor for the face
+     * @param faceDoc unique face
      */
-    public static ImageDocAddFace = (imgDoc: Doc, faceDescriptor: List<number>, faceDoc: Doc) => {
+    public static ImageDocAssociateUniqueFace = (imgDoc: Doc, faceDescriptor: List<number>, faceDoc: Doc) => {
         const faceFieldKey = FaceRecognitionHandler.imgDocFaceField(imgDoc, faceDoc);
         if (imgDoc[DocData][faceFieldKey]) {
             Cast(imgDoc[DocData][faceFieldKey], listSpec('number'), null).push(faceDescriptor as unknown as number); // items are lists of numbers, not numbers, but type system can't handle that
         } else {
             imgDoc[DocData][faceFieldKey] = new List<List<number>>([faceDescriptor]);
         }
+        Cast(imgDoc[DocData][`${Doc.LayoutFieldKey(imgDoc)}_Faces`], listSpec('number'), null).push(faceDescriptor as unknown as number);
+    };
+
+    /**
+     * Removes metadata from an image Doc to deassociate it from a unique face
+     * @param imgDoc image Doc containing faces
+     * @param faceDoc unique face
+     */
+    public static ImageDocDeassociateUniqueFace = (imgDoc: Doc, faceDoc: Doc) => {
+        // fill in..
     };
 
     /**
      * returns a list of all face collection Docs on the current dashboard
      * @returns face collection Doc list
      */
-    public static FaceDocuments = () => DocListCast(Doc.ActiveDashboard?.[DocData].faceDocuments);
+    public static UniqueFaces = () => DocListCast(Doc.ActiveDashboard?.[DocData].uniqueFaces);
 
-    public static DeleteFaceDoc = (faceDoc: Doc) => Doc.ActiveDashboard && Doc.RemoveDocFromList(Doc.ActiveDashboard[DocData], 'faceDocuments', faceDoc);
+    /**
+     * Removes a unique face from the set of recognized unique faces
+     * @param faceDoc unique face Doc
+     * @returns
+     */
+    public static DeleteUniqueFace = (faceDoc: Doc) => Doc.ActiveDashboard && Doc.RemoveDocFromList(Doc.ActiveDashboard[DocData], 'uniqueFaces', faceDoc);
 
     /**
      * returns the labels associated with a face collection Doc
-     * @param faceDoc the face collection Doc
+     * @param faceDoc unique face Doc
      * @returns label string
      */
-    public static FaceDocLabel = (faceDoc: Doc) => StrCast(faceDoc[DocData].face_label);
+    public static UniqueFaceLabel = (faceDoc: Doc) => StrCast(faceDoc[DocData].face_label);
     /**
-     * Returns all the face descriptors associated with a face collection Doc
-     * @param faceDoc a face collection Doc
+     * Returns all the face descriptors associated with a unique face Doc
+     * @param faceDoc unique face Doc
      * @returns face descriptors
      */
-    public static FaceDocDescriptors = (faceDoc: Doc) => faceDoc[DocData].face_descriptors as List<List<number>>;
+    public static UniqueFaceDescriptors = (faceDoc: Doc) => faceDoc[DocData].face_descriptors as List<List<number>>;
 
     /**
-     * Returns a list of all face image Docs associated with the face collection
-     * @param faceDoc a face collection Doc
+     * Returns a list of all face image Docs associated with a unique face Doc
+     * @param faceDoc unique face Doc
      * @returns image Docs
      */
-    public static FaceDocFaces = (faceDoc: Doc) => DocListCast(faceDoc[DocData].face_docList);
+    public static UniqueFaceImages = (faceDoc: Doc) => DocListCast(faceDoc[DocData].face_images);
 
     /**
-     * Adds a face image to the list of faces in a face collection Doc, and updates the face collection's list of image descriptors
+     * Adds a face image to a unique face Doc, and updates the unique face's set of face image descriptors
      * @param img - image with faces to add to a face collection Doc
      * @param faceDescriptor - the face descriptor for the face in the image to add
-     * @param faceDoc - the face collection Doc
+     * @param faceDoc - unique face Doc
      */
-    public static FaceDocAddImageDocFace = (img: Doc, faceDescriptor: List<number>, faceDoc: Doc) => {
-        Doc.AddDocToList(faceDoc, 'face_docList', img);
+    public static UniqueFaceAddFaceImage = (img: Doc, faceDescriptor: List<number>, faceDoc: Doc) => {
+        Doc.AddDocToList(faceDoc, 'face_images', img);
         Cast(faceDoc.face_descriptors, listSpec('number'), null).push(faceDescriptor as unknown as number); // items are lists of numbers, not numbers, but type system can't handle that
     };
 
     /**
-     * Removes a face from a face Doc collection, and updates the face collection's list of image descriptors
-     * @param imgDoc - image with faces to remove from the face Doc collectoin
-     * @param faceDoc - the face Doc collection
+     * Removes a face from a unique Face Doc, and updates the unique face's set of face image descriptors
+     * @param imgDoc - image with faces to remove
+     * @param faceDoc - unique face Doc
      */
-    public static FaceDocRemoveImageDocFace = (imgDoc: Doc, faceDoc: Doc) => {
-        Doc.RemoveDocFromList(faceDoc[DocData], 'face_docList', imgDoc);
-        faceDoc[DocData].face_descriptors = new List<List<number>>(FaceRecognitionHandler.FaceDocDescriptors(faceDoc).filter(fd => !(imgDoc[DocData][FaceRecognitionHandler.imgDocFaceField(imgDoc, faceDoc)] as List<List<number>>).includes(fd)));
+    public static UniqueFaceRemoveFaceImage = (imgDoc: Doc, faceDoc: Doc) => {
+        Doc.RemoveDocFromList(faceDoc[DocData], 'face_images', imgDoc);
+        faceDoc[DocData].face_descriptors = new List<List<number>>(FaceRecognitionHandler.UniqueFaceDescriptors(faceDoc).filter(fd => !(imgDoc[DocData][FaceRecognitionHandler.imgDocFaceField(imgDoc, faceDoc)] as List<List<number>>).includes(fd)));
     };
 
     constructor() {
@@ -118,10 +131,6 @@ export class FaceRecognitionHandler {
         DocumentManager.Instance.AddAnyViewRenderedCB(dv => FaceRecognitionHandler.Instance.classifyFacesInImage(dv.Document));
     }
 
-    @computed get examinedFaceDocs() {
-        return DocListCast(Doc.UserDoc().examinedFaceDocs);
-    }
-
     /**
      * Loads the face detection models.
      */
@@ -138,7 +147,27 @@ export class FaceRecognitionHandler {
     }
 
     /**
-     * When a document is added, look for matching face documents.
+     * Creates a new, empty unique face Doc
+     * @returns a unique face Doc
+     */
+    createUniqueFaceDoc = (dashboard: Doc) => {
+        const faceDocNum = NumCast(dashboard.uniqueFaces_count) + 1;
+        dashboard.uniqueFaces_count = faceDocNum; // TODO: improve to a better name
+
+        const uniqueFaceDoc = new Doc();
+        uniqueFaceDoc.title = `Face ${faceDocNum}`;
+        uniqueFaceDoc.face = ''; // just to make prettyprinting look better
+        uniqueFaceDoc.face_label = `Face${faceDocNum}`;
+        uniqueFaceDoc.face_images = new List<Doc>();
+        uniqueFaceDoc.face_descriptors = new List<List<number>>();
+
+        Doc.ActiveDashboard && Doc.AddDocToList(Doc.ActiveDashboard[DocData], 'uniqueFaces', uniqueFaceDoc);
+        return uniqueFaceDoc;
+    };
+
+    /**
+     * When a document is added, this finds faces in the images and tries to
+     * match them to existing unique faces, otherwise new unique face(s) are created.
      * @param imgDoc The document being analyzed.
      */
     public classifyFacesInImage = async (imgDoc: Doc) => {
@@ -154,12 +183,11 @@ export class FaceRecognitionHandler {
 
         const imgUrl = ImageCast(imgDoc[Doc.LayoutFieldKey(imgDoc)]);
         // If the doc isn't an image or currently already been examined or is being processed, stop examining the document.
-        if (!imgUrl || this.examinedFaceDocs.includes(imgDoc) || this._processingDocs.has(imgDoc)) {
+        if (!imgUrl || DocListCast(Doc.MyFaceCollection.examinedFaceDocs).includes(imgDoc)) {
             return;
         }
+        Doc.AddDocToList(Doc.MyFaceCollection, 'examinedFaceDocs', imgDoc);
 
-        // Mark the document as being processed.
-        this._processingDocs.add(imgDoc);
         FaceRecognitionHandler.initImageDocFaceDescriptors(imgDoc);
 
         // Get the image the document contains and analyze for faces.
@@ -170,31 +198,12 @@ export class FaceRecognitionHandler {
 
         // For each face detected, find a match.
         for (const fd of imgDocFaceDescriptions) {
-            let faceDocMatch = this.findMatchingFaceDoc(fd.descriptor);
             const faceDescriptor = new List<number>(Array.from(fd.descriptor));
-
-            if (faceDocMatch) {
-                FaceRecognitionHandler.FaceDocAddImageDocFace(imgDoc, faceDescriptor, faceDocMatch);
-            } else {
-                // If a matching Face Document has not been found, create a new Face Document.
-                Doc.UserDoc().faceDocNum = NumCast(Doc.UserDoc().faceDocNum) + 1;
-
-                const newFaceDocument = new Doc();
-                newFaceDocument.title = `Face ${Doc.UserDoc().faceDocNum}`;
-                newFaceDocument.face = ''; // just to make prettyprinting look better
-                newFaceDocument.face_label = `Face${Doc.UserDoc().faceDocNum}`;
-                newFaceDocument.face_docList = new List<Doc>([imgDoc]);
-                newFaceDocument.face_descriptors = new List<List<number>>([faceDescriptor]);
-
-                Doc.AddDocToList(Doc.ActiveDashboard[DocData], 'faceDocuments', newFaceDocument);
-                faceDocMatch = newFaceDocument;
-            }
-
-            // Assign a field in the document of the matching Face Document.
-            FaceRecognitionHandler.ImageDocAddFace(imgDoc, faceDescriptor, faceDocMatch);
-            Doc.AddDocToList(Doc.UserDoc(), 'examinedFaceDocs', imgDoc);
+            const matchedUniqueFace = this.findMatchingFaceDoc(fd.descriptor) ?? this.createUniqueFaceDoc(Doc.ActiveDashboard);
+            // Add image to unique face's image collection, and assign image metadata referencing unique face
+            FaceRecognitionHandler.UniqueFaceAddFaceImage(imgDoc, faceDescriptor, matchedUniqueFace);
+            FaceRecognitionHandler.ImageDocAssociateUniqueFace(imgDoc, faceDescriptor, matchedUniqueFace);
         }
-        this._processingDocs.delete(imgDoc);
     };
 
     /**
@@ -203,19 +212,19 @@ export class FaceRecognitionHandler {
      * @returns face Doc
      */
     private findMatchingFaceDoc = (faceDescriptor: Float32Array) => {
-        if (!Doc.ActiveDashboard || FaceRecognitionHandler.FaceDocuments().length < 1) {
+        if (!Doc.ActiveDashboard || FaceRecognitionHandler.UniqueFaces().length < 1) {
             return undefined;
         }
 
-        const faceDescriptors = FaceRecognitionHandler.FaceDocuments().map(faceDoc => {
-            const float32Array = FaceRecognitionHandler.FaceDocDescriptors(faceDoc).map(fd => new Float32Array(Array.from(fd)));
-            return new faceapi.LabeledFaceDescriptors(FaceRecognitionHandler.FaceDocLabel(faceDoc), float32Array);
+        const faceDescriptors = FaceRecognitionHandler.UniqueFaces().map(faceDoc => {
+            const float32Array = FaceRecognitionHandler.UniqueFaceDescriptors(faceDoc).map(fd => new Float32Array(Array.from(fd)));
+            return new faceapi.LabeledFaceDescriptors(FaceRecognitionHandler.UniqueFaceLabel(faceDoc), float32Array);
         });
         const faceMatcher = new FaceMatcher(faceDescriptors, 0.6);
         const match = faceMatcher.findBestMatch(faceDescriptor);
         if (match.label !== 'unknown') {
-            for (const faceDoc of FaceRecognitionHandler.FaceDocuments()) {
-                if (FaceRecognitionHandler.FaceDocLabel(faceDoc) === match.label) {
+            for (const faceDoc of FaceRecognitionHandler.UniqueFaces()) {
+                if (FaceRecognitionHandler.UniqueFaceLabel(faceDoc) === match.label) {
                     return faceDoc;
                 }
             }
-- 
cgit v1.2.3-70-g09d2

