fixed up a bunch of things in face recognition

1 files changed, 76 insertions, 83 deletions

diff --git a/src/client/views/search/FaceRecognitionHandler.tsx b/src/client/views/search/FaceRecognitionHandler.tsx
index ef4622ea2..29ca6e797 100644
--- a/src/client/views/search/FaceRecognitionHandler.tsx
+++ b/src/client/views/search/FaceRecognitionHandler.tsx
@@ -3,7 +3,11 @@ import { FaceMatcher } from 'face-api.js';
 import { Doc, DocListCast } from '../../../fields/Doc';
 import { DocData } from '../../../fields/DocSymbols';
 import { List } from '../../../fields/List';
-import { ImageCast, NumCast, StrCast } from '../../../fields/Types';
+import { Cast, ImageCast, NumCast, StrCast } from '../../../fields/Types';
+import { DocumentManager } from '../../util/DocumentManager';
+import { computed } from 'mobx';
+import { DocumentType } from '../../documents/DocumentTypes';
+import { listSpec } from '../../../fields/Schema';
 
 /**
  * A class that handles face recognition.
@@ -11,13 +15,16 @@ import { ImageCast, NumCast, StrCast } from '../../../fields/Types';
 export class FaceRecognitionHandler {
     static _instance: FaceRecognitionHandler;
     private loadedModels: boolean = false;
-    examinedDocs: Set<Doc> = new Set();
+    @computed get examinedFaceDocs() {
+        return DocListCast(Doc.UserDoc().examinedFaceDocs);
+    }
     processingDocs: Set<Doc> = new Set();
+    pendingLoadDocs: Doc[] = [];
 
     constructor() {
         FaceRecognitionHandler._instance = this;
-        this.loadModels();
-        this.examinedDocs = new Set(DocListCast(Doc.UserDoc()[DocData].examinedFaceDocs, []));
+        this.loadModels().then(() => this.pendingLoadDocs.forEach(this.findMatches));
+        DocumentManager.Instance.AddAnyViewRenderedCB(dv => FaceRecognitionHandler.Instance.findMatches(dv.Document));
     }
 
     /**
@@ -39,86 +46,73 @@ export class FaceRecognitionHandler {
      * When a document is added, look for matching face documents.
      * @param doc The document being analyzed.
      */
-    public async findMatches(doc: Doc) {
-        if (this.loadedModels) {
-            // If the Dashboard doesn't have a list of face documents yet, initialize the list.
-            if (!Doc.ActiveDashboard![DocData].faceDocuments) {
-                Doc.ActiveDashboard![DocData].faceDocuments = new List<Doc>();
-            }
+    public findMatches = async (doc: Doc) => {
+        if (!this.loadedModels || !Doc.ActiveDashboard) {
+            this.pendingLoadDocs.push(doc);
+            return;
+        }
 
-            // If the doc is currently already been examined, or it is being processed, stop examining the document.
-            if (this.examinedDocs.has(doc) || this.processingDocs.has(doc)) {
-                return;
-            }
+        if (doc.type === DocumentType.LOADING && !doc.loadingError) {
+            setTimeout(() => this.findMatches(doc), 1000);
+            return;
+        }
 
-            // Mark the document as being processed.
-            this.processingDocs.add(doc);
+        const imgUrl = ImageCast(doc[Doc.LayoutFieldKey(doc)]);
+        // If the doc isn't an image or currently already been examined or is being processed, stop examining the document.
+        if (!imgUrl || this.examinedFaceDocs.includes(doc) || this.processingDocs.has(doc)) {
+            return;
+        }
 
-            try {
-                if (!Doc.UserDoc()[DocData].faceDocNum) {
-                    Doc.UserDoc()[DocData].faceDocNum = 0;
-                }
+        // Mark the document as being processed.
+        this.processingDocs.add(doc);
 
-                // Get the image the document contains and analyze for faces.
-                const [name, type] = ImageCast(doc[Doc.LayoutFieldKey(doc)]).url.href.split('.');
-                const imageURL = `${name}_o.${type}`;
-
-                const img = await this.loadImage(imageURL);
-
-                const fullFaceDescriptions = await faceapi.detectAllFaces(img).withFaceLandmarks().withFaceDescriptors();
-
-                doc[DocData].faces = new List<List<number>>();
-
-                // For each face detected, find a match.
-                for (const fd of fullFaceDescriptions) {
-                    let match = this.findMatch(fd.descriptor);
-                    let converted_list = new List<number>();
-
-                    if (match) {
-                        // If a matching Face Document has been found, add the document to the Face Document's associated docs and append the face
-                        // descriptor to the Face Document's descriptor list.
-                        const converted_array = Array.from(fd.descriptor);
-                        converted_list = new List<number>(converted_array);
-                        match[DocData].associatedDocs = new List<Doc>([...DocListCast(match[DocData].associatedDocs), doc]);
-                        match[DocData].faceDescriptors = new List<List<number>>([...(match[DocData].faceDescriptors as List<List<number>>), converted_list]);
-                    } else {
-                        // If a matching Face Document has not been found, create a new Face Document.
-                        const newFaceDocument = new Doc();
-                        const converted_array = Array.from(fd.descriptor);
-                        converted_list = new List<number>(converted_array);
-                        newFaceDocument[DocData].faceDescriptors = new List<List<number>>();
-                        (newFaceDocument[DocData].faceDescriptors as List<List<number>>).push(converted_list);
-                        Doc.UserDoc()[DocData].faceDocNum = NumCast(Doc.UserDoc()[DocData].faceDocNum) + 1;
-                        newFaceDocument[DocData].label = `Face ${Doc.UserDoc()[DocData].faceDocNum}`;
-                        newFaceDocument[DocData].associatedDocs = new List<Doc>([doc]);
-
-                        Doc.ActiveDashboard![DocData].faceDocuments = new List<Doc>([...DocListCast(Doc.ActiveDashboard![DocData].faceDocuments), newFaceDocument]);
-                        match = newFaceDocument;
-                    }
-
-                    // Assign a field in the document of the matching Face Document.
-                    if (doc[DocData][`FACE DESCRIPTOR - ${match[DocData].label}`]) {
-                        doc[DocData][`FACE DESCRIPTOR - ${match[DocData].label}`] = new List<List<number>>([...(doc[DocData][`FACE DESCRIPTOR - ${match[DocData].label}`] as List<List<number>>), converted_list]);
-                    } else {
-                        doc[DocData][`FACE DESCRIPTOR - ${match[DocData].label}`] = new List<List<number>>([converted_list]);
-                    }
-
-                    doc[DocData].faces = new List<List<number>>([...(doc[DocData].faces as List<List<number>>), converted_list]);
-                }
+        // Get the image the document contains and analyze for faces.
+        const [name, type] = imgUrl.url.href.split('.');
+        const imageURL = `${name}_o.${type}`;
 
-                // Updates the examined docs field.
-                this.examinedDocs.add(doc);
-                if (!Doc.UserDoc()[DocData].examinedFaceDocs) {
-                    Doc.UserDoc()[DocData].examinedFaceDocs = new List<Doc>();
-                }
-                Doc.UserDoc()[DocData].examinedFaceDocs = new List<Doc>([...DocListCast(Doc.UserDoc()[DocData].examinedFaceDocs), doc]);
-            } catch (error) {
-                console.error('Error processing document:', error);
-            } finally {
-                this.processingDocs.delete(doc);
+        const img = await this.loadImage(imageURL);
+
+        const fullFaceDescriptions = await faceapi.detectAllFaces(img).withFaceLandmarks().withFaceDescriptors();
+
+        doc[DocData].faces = new List<List<number>>();
+
+        // For each face detected, find a match.
+        for (const fd of fullFaceDescriptions) {
+            let match = this.findMatch(fd.descriptor);
+            const converted_list = new List<number>(Array.from(fd.descriptor));
+
+            if (match) {
+                // If a matching Face Document has been found, add the document to the Face Document's associated docs and append the face
+                // descriptor to the Face Document's descriptor list.
+                Doc.AddDocToList(match, 'associatedDocs', doc);
+                Cast(match.faceDescriptors, listSpec('number'), null).push(converted_list as unknown as number); // items are lists of numbers, not numbers, but type system can't handle that
+            } else {
+                // If a matching Face Document has not been found, create a new Face Document.
+                Doc.UserDoc().faceDocNum = NumCast(Doc.UserDoc().faceDocNum) + 1;
+
+                const newFaceDocument = new Doc();
+                newFaceDocument.label = `Face ${Doc.UserDoc().faceDocNum}`;
+                newFaceDocument.associatedDocs = new List<Doc>([doc]);
+                newFaceDocument.faceDescriptors = new List<List<number>>([converted_list]);
+
+                Doc.AddDocToList(Doc.ActiveDashboard[DocData], 'faceDocuments', newFaceDocument);
+                match = newFaceDocument;
+            }
+
+            // Assign a field in the document of the matching Face Document.
+            const faceDescripField = `FACE DESCRIPTOR - ${match[DocData].label}`;
+            if (doc[DocData][faceDescripField]) {
+                Cast(doc[DocData][faceDescripField], listSpec('number'), null).push(converted_list as unknown as number); // items are lists of numbers, not numbers, but type system can't handle that
+            } else {
+                doc[DocData][faceDescripField] = new List<List<number>>([converted_list]);
             }
+
+            Cast(doc[DocData].faces, listSpec('number'), null).push(converted_list as unknown as number); // items are lists of numbers, not numbers, but type system can't handle that
+
+            Doc.AddDocToList(Doc.UserDoc(), 'examinedFaceDocs', doc);
         }
-    }
+        this.processingDocs.delete(doc);
+    };
 
     /**
      * Finds a matching Face Document given a descriptor
@@ -126,25 +120,24 @@ export class FaceRecognitionHandler {
      * @returns The most similar Face Document.
      */
     private findMatch(cur_descriptor: Float32Array) {
-        if (DocListCast(Doc.ActiveDashboard![DocData].faceDocuments).length < 1) {
+        if (!Doc.ActiveDashboard || DocListCast(Doc.ActiveDashboard[DocData].faceDocuments).length < 1) {
             return null;
         }
 
-        const faceDescriptors: faceapi.LabeledFaceDescriptors[] = DocListCast(Doc.ActiveDashboard![DocData].faceDocuments).map(faceDocument => {
+        const faceDescriptors: faceapi.LabeledFaceDescriptors[] = DocListCast(Doc.ActiveDashboard[DocData].faceDocuments).map(faceDocument => {
             const float32Array = (faceDocument[DocData].faceDescriptors as List<List<number>>).map(faceDescriptor => new Float32Array(Array.from(faceDescriptor)));
             return new faceapi.LabeledFaceDescriptors(StrCast(faceDocument[DocData].label), float32Array);
         });
         const faceMatcher = new FaceMatcher(faceDescriptors, 0.6);
         const match = faceMatcher.findBestMatch(cur_descriptor);
-        if (match.label == 'unknown') {
-            return null;
-        } else {
-            for (const doc of DocListCast(Doc.ActiveDashboard![DocData].faceDocuments)) {
+        if (match.label !== 'unknown') {
+            for (const doc of DocListCast(Doc.ActiveDashboard[DocData].faceDocuments)) {
                 if (doc[DocData].label === match.label) {
                     return doc;
                 }
             }
         }
+        return null;
     }
 
     /**