feat: added face recogntion box

3 files changed, 294 insertions, 2 deletions

diff --git a/src/client/views/collections/collectionFreeForm/FaceCollectionBox.scss b/src/client/views/collections/collectionFreeForm/FaceCollectionBox.scss
new file mode 100644
index 000000000..480d109c8
--- /dev/null
+++ b/src/client/views/collections/collectionFreeForm/FaceCollectionBox.scss
@@ -0,0 +1,74 @@
+.face-document-item {
+    background: #555555;
+    margin-top: 10px;
+    margin-bottom: 10px;
+    padding: 10px;
+    border-radius: 10px;
+    position: relative;
+
+    h1 {
+        color: white;
+        font-size: 24px;
+        text-align: center;
+    }
+
+    .face-collection-buttons {
+        position: absolute;
+        top: 10px;
+        right: 10px;
+    }
+
+    .face-document-image-container {
+        display: flex;
+        justify-content: center;
+        flex-wrap: wrap;
+
+        .image-wrapper {
+            position: relative;
+            width: 70px;
+            height: 70px;
+            margin: 10px;
+            display: flex;
+            align-items: center; // Center vertically
+            justify-content: center; // Center horizontally
+
+            img {
+                width: 100%;
+                height: 100%;
+                object-fit: cover; // This ensures the image covers the container without stretching
+                border-radius: 5px;
+                border: 2px solid white;
+                transition: border-color 0.4s;
+
+                &:hover {
+                    border-color: orange; // Change this to your desired hover border color
+                }
+            }
+
+            .remove-item {
+                position: absolute;
+                bottom: -5;
+                right: -5;
+                background-color: rgba(0, 0, 0, 0.5); // Optional: to add a background behind the icon for better visibility
+                border-radius: 30%;
+                width: 10px; // Adjust size as needed
+                height: 10px; // Adjust size as needed
+                display: flex;
+                align-items: center;
+                justify-content: center;
+            }
+        }
+
+        // img {
+        //     max-width: 60px;
+        //     margin: 10px;
+        //     border-radius: 5px;
+        //     border: 2px solid white;
+        //     transition: 0.4s;
+
+        //     &:hover {
+        //         border-color: orange;
+        //     }
+        // }
+    }
+}
diff --git a/src/client/views/collections/collectionFreeForm/FaceCollectionBox.tsx b/src/client/views/collections/collectionFreeForm/FaceCollectionBox.tsx
new file mode 100644
index 000000000..1d3f88df1
--- /dev/null
+++ b/src/client/views/collections/collectionFreeForm/FaceCollectionBox.tsx
@@ -0,0 +1,217 @@
+import { observer } from 'mobx-react';
+import React from 'react';
+import { Docs } from '../../../documents/Documents';
+import { DocumentType } from '../../../documents/DocumentTypes';
+import { ViewBoxBaseComponent } from '../../DocComponent';
+import { FieldView, FieldViewProps } from '../../nodes/FieldView';
+import 'ldrs/ring';
+import { SnappingManager } from '../../../util/SnappingManager';
+import { action, computed, makeObservable, observable, reaction } from 'mobx';
+import { Doc, DocListCast, NumListCast } from '../../../../fields/Doc';
+import { DocData } from '../../../../fields/DocSymbols';
+import { ImageCast, StrCast } from '../../../../fields/Types';
+import { ObservableReactComponent } from '../../ObservableReactComponent';
+import './FaceCollectionBox.scss';
+import { IconButton, Size } from 'browndash-components';
+import { FontAwesomeIcon } from '@fortawesome/react-fontawesome';
+import { MarqueeOptionsMenu } from './MarqueeOptionsMenu';
+import { List } from '../../../../fields/List';
+import { DocumentView } from '../../nodes/DocumentView';
+import { Utils } from '../../../../Utils';
+import { DragManager } from '../../../util/DragManager';
+import * as faceapi from 'face-api.js';
+import { FaceMatcher } from 'face-api.js';
+
+interface FaceDocumentProps {
+    faceDoc: Doc;
+}
+
+/**
+ * A componenent to visually represent a Face Document.
+ */
+@observer
+export class FaceDocumentItem extends ObservableReactComponent<FaceDocumentProps> {
+    private ref: React.RefObject<HTMLDivElement>;
+    @observable _displayImages: boolean = true;
+    private _dropDisposer?: DragManager.DragDropDisposer;
+    private _inputRef = React.createRef<HTMLInputElement>();
+
+    constructor(props: any) {
+        super(props);
+        makeObservable(this);
+        this.ref = React.createRef();
+    }
+
+    protected createDropTarget = (ele: HTMLDivElement) => {
+        this._dropDisposer?.();
+        ele && (this._dropDisposer = DragManager.MakeDropTarget(ele, this.onInternalDrop.bind(this), this._props.faceDoc));
+    };
+
+    protected onInternalDrop(e: Event, de: DragManager.DropEvent): boolean {
+        const { docDragData } = de.complete;
+        if (docDragData) {
+            const filteredDocs = docDragData.droppedDocuments.filter(doc => doc.type === DocumentType.IMG);
+            filteredDocs.forEach(doc => {
+                // If the current Face Document has no items, and the doc has more than one face descriptor, don't let the user add the document first.
+                if ((this._props.faceDoc[DocData].faceDescriptors as List<List<number>>).length === 0 && (doc[DocData].faces as List<List<number>>).length > 1) {
+                    alert('Cannot add a document with multiple faces as the first item!');
+                } else {
+                    // Loop through the documents' face descriptors.
+                    // Choose the face with the smallest distance to add.
+                    const float32Array = (this._props.faceDoc[DocData].faceDescriptors as List<List<number>>).map(faceDescriptor => new Float32Array(Array.from(faceDescriptor)));
+                    const labeledFaceDescriptor = new faceapi.LabeledFaceDescriptors(StrCast(this._props.faceDoc[DocData].label), float32Array);
+                    const faceDescriptors: faceapi.LabeledFaceDescriptors[] = [labeledFaceDescriptor];
+
+                    const faceMatcher = new FaceMatcher(faceDescriptors, 1);
+                    let cur_lowest_distance = 1;
+                    let cur_matching_face = new List<number>();
+
+                    (doc[DocData].faces as List<List<number>>).forEach(face => {
+                        // If the face has the current lowest distance, mark it as such
+                        // Once that lowest distance is found, add the face descriptor to the faceDoc, and add the associated doc
+                        const convered_32_array: Float32Array = new Float32Array(Array.from(face));
+                        const match = faceMatcher.matchDescriptor(convered_32_array);
+
+                        if (match.distance < cur_lowest_distance) {
+                            cur_lowest_distance = match.distance;
+                            cur_matching_face = face;
+                        }
+                    });
+
+                    if (doc[DocData][`FACE DESCRIPTOR - ${this._props.faceDoc[DocData].label}`]) {
+                        doc[DocData][`FACE DESCRIPTOR - ${this._props.faceDoc[DocData].label}`] = new List<List<number>>([...(doc[DocData][`FACE DESCRIPTOR - ${this._props.faceDoc[DocData].label}`] as List<List<number>>), cur_matching_face]);
+                    } else {
+                        doc[DocData][`FACE DESCRIPTOR - ${this._props.faceDoc[DocData].label}`] = new List<List<number>>([cur_matching_face]);
+                    }
+
+                    this._props.faceDoc[DocData].associatedDocs = new List<Doc>([...DocListCast(this._props.faceDoc[DocData].associatedDocs), doc]);
+                    this._props.faceDoc[DocData].faceDescriptors = new List<List<number>>([...(this._props.faceDoc[DocData].faceDescriptors as List<List<number>>), cur_matching_face]);
+
+                    //const match = faceMatcher.findBestMatch(cur_descriptor);
+                }
+            });
+            return false;
+        }
+        return false;
+    }
+
+    /**
+     * Toggles whether a Face Document displays its associated docs.
+     */
+    @action
+    onDisplayClick() {
+        this._displayImages = !this._displayImages;
+    }
+
+    /**
+     * Deletes a Face Document.
+     */
+    @action
+    deleteFaceDocument = () => {
+        if (Doc.ActiveDashboard) {
+            Doc.ActiveDashboard[DocData].faceDocuments = new List<Doc>(DocListCast(Doc.ActiveDashboard[DocData].faceDocuments).filter(doc => doc !== this._props.faceDoc));
+        }
+    };
+
+    /**
+     * Deletes a document from a Face Document's associated docs list.
+     * @param doc
+     */
+    @action
+    deleteAssociatedDoc = (doc: Doc) => {
+        this._props.faceDoc[DocData].faceDescriptors = new List<List<number>>(
+            (this._props.faceDoc[DocData].faceDescriptors as List<List<number>>).filter(fd => !(doc[DocData][`FACE DESCRIPTOR - ${this._props.faceDoc[DocData].label}`] as List<List<number>>).includes(fd))
+        );
+        doc[DocData][`FACE DESCRIPTOR - ${this._props.faceDoc[DocData].label}`] = new List<List<number>>();
+        this._props.faceDoc[DocData].associatedDocs = new List<Doc>(DocListCast(this._props.faceDoc[DocData].associatedDocs).filter(associatedDoc => associatedDoc !== doc));
+    };
+
+    render() {
+        return (
+            <div className="face-document-item" ref={ele => this.createDropTarget(ele!)}>
+                <div className="face-collection-buttons">
+                    <IconButton tooltip={'Delete Face From Collection'} onPointerDown={this.deleteFaceDocument} icon={'x'} style={{ width: '4px' }} size={Size.XSMALL} />
+                </div>
+                <div className="face-document-top">
+                    <h1>{StrCast(this._props.faceDoc[DocData].label)}</h1>
+                </div>
+                <IconButton
+                    tooltip={'See image information'}
+                    onPointerDown={() => this.onDisplayClick()}
+                    icon={this._displayImages ? <FontAwesomeIcon icon="caret-up" /> : <FontAwesomeIcon icon="caret-down" />}
+                    color={MarqueeOptionsMenu.Instance.userColor}
+                    style={{ width: '19px' }}
+                />
+                {this._displayImages ? (
+                    <div className="face-document-image-container">
+                        {DocListCast(this._props.faceDoc[DocData].associatedDocs).map(doc => {
+                            const [name, type] = ImageCast(doc[Doc.LayoutFieldKey(doc)]).url.href.split('.');
+                            return (
+                                <div className="image-wrapper" key={Utils.GenerateGuid()}>
+                                    <img
+                                        onClick={async () => {
+                                            await DocumentView.showDocument(doc, { willZoomCentered: true });
+                                        }}
+                                        style={{ maxWidth: '60px', margin: '10px' }}
+                                        src={`${name}_o.${type}`}
+                                    />
+                                    <div className="remove-item">
+                                        <IconButton
+                                            tooltip={'Remove Doc From Face Collection'}
+                                            onPointerDown={() => {
+                                                this.deleteAssociatedDoc(doc);
+                                            }}
+                                            icon={'x'}
+                                            style={{ width: '4px' }}
+                                            size={Size.XSMALL}
+                                        />
+                                    </div>
+                                </div>
+                            );
+                        })}
+                    </div>
+                ) : (
+                    <div></div>
+                )}
+            </div>
+        );
+    }
+}
+
+@observer
+export class FaceCollectionBox extends ViewBoxBaseComponent<FieldViewProps>() {
+    public static LayoutString(fieldKey: string) {
+        return FieldView.LayoutString(FaceCollectionBox, fieldKey);
+    }
+
+    public static Instance: FaceCollectionBox;
+
+    @computed get currentDocs() {
+        if (Doc.ActiveDashboard) {
+            return DocListCast(Doc.ActiveDashboard[DocData].faceDocuments);
+        } else {
+            return [];
+        }
+    }
+
+    constructor(props: any) {
+        super(props);
+        makeObservable(this);
+        FaceCollectionBox.Instance = this;
+    }
+
+    render() {
+        return (
+            <div className="searchBox-container" style={{ pointerEvents: 'all', color: SnappingManager.userColor, background: SnappingManager.userBackgroundColor }}>
+                {this.currentDocs.map(doc => {
+                    return <FaceDocumentItem faceDoc={doc} />;
+                })}
+            </div>
+        );
+    }
+}
+
+Docs.Prototypes.TemplateMap.set(DocumentType.FACECOLLECTION, {
+    layout: { view: FaceCollectionBox, dataField: 'data' },
+    options: { acl: '', _width: 400 },
+});
diff --git a/src/client/views/collections/collectionFreeForm/ImageLabelBox.tsx b/src/client/views/collections/collectionFreeForm/ImageLabelBox.tsx
index af01d6cbc..421b5d0a6 100644
--- a/src/client/views/collections/collectionFreeForm/ImageLabelBox.tsx
+++ b/src/client/views/collections/collectionFreeForm/ImageLabelBox.tsx
@@ -179,6 +179,7 @@ export class ImageLabelBox extends ViewBoxBaseComponent<FieldViewProps>() {
                 const labels = imageInfo.labels.split('\n');
                 labels.forEach(label => {
                     label = label.replace(/^\d+\.\s*|-|\*/, '').trim();
+                    console.log(label);
                     imageInfo.doc[DocData][`${label}`] = true;
                     (imageInfo.doc[DocData].data_labels as List<string>).push(label);
                 });
@@ -198,7 +199,7 @@ export class ImageLabelBox extends ViewBoxBaseComponent<FieldViewProps>() {
             for (let index = 0; index < (doc[DocData].data_labels as List<string>).length; index++) {
                 const label = (doc[DocData].data_labels as List<string>)[index];
                 const embedding = await gptGetEmbedding(label);
-                doc[`data_labels_embedding_${index + 1}`] = new List<number>(embedding);
+                doc[DocData][`data_labels_embedding_${index + 1}`] = new List<number>(embedding);
             }
         }
 
@@ -209,7 +210,7 @@ export class ImageLabelBox extends ViewBoxBaseComponent<FieldViewProps>() {
         // For each image, loop through the labels, and calculate similarity. Associate it with the
         // most similar one.
         this._selectedImages.forEach(doc => {
-            const embedLists = numberRange((doc[DocData].data_labels as List<string>).length).map(n => Array.from(NumListCast(doc[`data_labels_embedding_${n + 1}`])));
+            const embedLists = numberRange((doc[DocData].data_labels as List<string>).length).map(n => Array.from(NumListCast(doc[DocData][`data_labels_embedding_${n + 1}`])));
             const bestEmbedScore = (embedding: Opt<number[]>) => Math.max(...embedLists.map((l, index) => (embedding && similarity(Array.from(embedding), l)!) || 0));
             const {label: mostSimilarLabelCollect} = 
                 this._labelGroups.map(label => ({ label, similarityScore: bestEmbedScore(labelToEmbedding.get(label)) }))