feat: added face recogntion box

1 files changed, 217 insertions, 0 deletions

diff --git a/src/client/views/collections/collectionFreeForm/FaceCollectionBox.tsx b/src/client/views/collections/collectionFreeForm/FaceCollectionBox.tsx
new file mode 100644
index 000000000..1d3f88df1
--- /dev/null
+++ b/src/client/views/collections/collectionFreeForm/FaceCollectionBox.tsx
@@ -0,0 +1,217 @@
+import { observer } from 'mobx-react';
+import React from 'react';
+import { Docs } from '../../../documents/Documents';
+import { DocumentType } from '../../../documents/DocumentTypes';
+import { ViewBoxBaseComponent } from '../../DocComponent';
+import { FieldView, FieldViewProps } from '../../nodes/FieldView';
+import 'ldrs/ring';
+import { SnappingManager } from '../../../util/SnappingManager';
+import { action, computed, makeObservable, observable, reaction } from 'mobx';
+import { Doc, DocListCast, NumListCast } from '../../../../fields/Doc';
+import { DocData } from '../../../../fields/DocSymbols';
+import { ImageCast, StrCast } from '../../../../fields/Types';
+import { ObservableReactComponent } from '../../ObservableReactComponent';
+import './FaceCollectionBox.scss';
+import { IconButton, Size } from 'browndash-components';
+import { FontAwesomeIcon } from '@fortawesome/react-fontawesome';
+import { MarqueeOptionsMenu } from './MarqueeOptionsMenu';
+import { List } from '../../../../fields/List';
+import { DocumentView } from '../../nodes/DocumentView';
+import { Utils } from '../../../../Utils';
+import { DragManager } from '../../../util/DragManager';
+import * as faceapi from 'face-api.js';
+import { FaceMatcher } from 'face-api.js';
+
+interface FaceDocumentProps {
+    faceDoc: Doc;
+}
+
+/**
+ * A componenent to visually represent a Face Document.
+ */
+@observer
+export class FaceDocumentItem extends ObservableReactComponent<FaceDocumentProps> {
+    private ref: React.RefObject<HTMLDivElement>;
+    @observable _displayImages: boolean = true;
+    private _dropDisposer?: DragManager.DragDropDisposer;
+    private _inputRef = React.createRef<HTMLInputElement>();
+
+    constructor(props: any) {
+        super(props);
+        makeObservable(this);
+        this.ref = React.createRef();
+    }
+
+    protected createDropTarget = (ele: HTMLDivElement) => {
+        this._dropDisposer?.();
+        ele && (this._dropDisposer = DragManager.MakeDropTarget(ele, this.onInternalDrop.bind(this), this._props.faceDoc));
+    };
+
+    protected onInternalDrop(e: Event, de: DragManager.DropEvent): boolean {
+        const { docDragData } = de.complete;
+        if (docDragData) {
+            const filteredDocs = docDragData.droppedDocuments.filter(doc => doc.type === DocumentType.IMG);
+            filteredDocs.forEach(doc => {
+                // If the current Face Document has no items, and the doc has more than one face descriptor, don't let the user add the document first.
+                if ((this._props.faceDoc[DocData].faceDescriptors as List<List<number>>).length === 0 && (doc[DocData].faces as List<List<number>>).length > 1) {
+                    alert('Cannot add a document with multiple faces as the first item!');
+                } else {
+                    // Loop through the documents' face descriptors.
+                    // Choose the face with the smallest distance to add.
+                    const float32Array = (this._props.faceDoc[DocData].faceDescriptors as List<List<number>>).map(faceDescriptor => new Float32Array(Array.from(faceDescriptor)));
+                    const labeledFaceDescriptor = new faceapi.LabeledFaceDescriptors(StrCast(this._props.faceDoc[DocData].label), float32Array);
+                    const faceDescriptors: faceapi.LabeledFaceDescriptors[] = [labeledFaceDescriptor];
+
+                    const faceMatcher = new FaceMatcher(faceDescriptors, 1);
+                    let cur_lowest_distance = 1;
+                    let cur_matching_face = new List<number>();
+
+                    (doc[DocData].faces as List<List<number>>).forEach(face => {
+                        // If the face has the current lowest distance, mark it as such
+                        // Once that lowest distance is found, add the face descriptor to the faceDoc, and add the associated doc
+                        const convered_32_array: Float32Array = new Float32Array(Array.from(face));
+                        const match = faceMatcher.matchDescriptor(convered_32_array);
+
+                        if (match.distance < cur_lowest_distance) {
+                            cur_lowest_distance = match.distance;
+                            cur_matching_face = face;
+                        }
+                    });
+
+                    if (doc[DocData][`FACE DESCRIPTOR - ${this._props.faceDoc[DocData].label}`]) {
+                        doc[DocData][`FACE DESCRIPTOR - ${this._props.faceDoc[DocData].label}`] = new List<List<number>>([...(doc[DocData][`FACE DESCRIPTOR - ${this._props.faceDoc[DocData].label}`] as List<List<number>>), cur_matching_face]);
+                    } else {
+                        doc[DocData][`FACE DESCRIPTOR - ${this._props.faceDoc[DocData].label}`] = new List<List<number>>([cur_matching_face]);
+                    }
+
+                    this._props.faceDoc[DocData].associatedDocs = new List<Doc>([...DocListCast(this._props.faceDoc[DocData].associatedDocs), doc]);
+                    this._props.faceDoc[DocData].faceDescriptors = new List<List<number>>([...(this._props.faceDoc[DocData].faceDescriptors as List<List<number>>), cur_matching_face]);
+
+                    //const match = faceMatcher.findBestMatch(cur_descriptor);
+                }
+            });
+            return false;
+        }
+        return false;
+    }
+
+    /**
+     * Toggles whether a Face Document displays its associated docs.
+     */
+    @action
+    onDisplayClick() {
+        this._displayImages = !this._displayImages;
+    }
+
+    /**
+     * Deletes a Face Document.
+     */
+    @action
+    deleteFaceDocument = () => {
+        if (Doc.ActiveDashboard) {
+            Doc.ActiveDashboard[DocData].faceDocuments = new List<Doc>(DocListCast(Doc.ActiveDashboard[DocData].faceDocuments).filter(doc => doc !== this._props.faceDoc));
+        }
+    };
+
+    /**
+     * Deletes a document from a Face Document's associated docs list.
+     * @param doc
+     */
+    @action
+    deleteAssociatedDoc = (doc: Doc) => {
+        this._props.faceDoc[DocData].faceDescriptors = new List<List<number>>(
+            (this._props.faceDoc[DocData].faceDescriptors as List<List<number>>).filter(fd => !(doc[DocData][`FACE DESCRIPTOR - ${this._props.faceDoc[DocData].label}`] as List<List<number>>).includes(fd))
+        );
+        doc[DocData][`FACE DESCRIPTOR - ${this._props.faceDoc[DocData].label}`] = new List<List<number>>();
+        this._props.faceDoc[DocData].associatedDocs = new List<Doc>(DocListCast(this._props.faceDoc[DocData].associatedDocs).filter(associatedDoc => associatedDoc !== doc));
+    };
+
+    render() {
+        return (
+            <div className="face-document-item" ref={ele => this.createDropTarget(ele!)}>
+                <div className="face-collection-buttons">
+                    <IconButton tooltip={'Delete Face From Collection'} onPointerDown={this.deleteFaceDocument} icon={'x'} style={{ width: '4px' }} size={Size.XSMALL} />
+                </div>
+                <div className="face-document-top">
+                    <h1>{StrCast(this._props.faceDoc[DocData].label)}</h1>
+                </div>
+                <IconButton
+                    tooltip={'See image information'}
+                    onPointerDown={() => this.onDisplayClick()}
+                    icon={this._displayImages ? <FontAwesomeIcon icon="caret-up" /> : <FontAwesomeIcon icon="caret-down" />}
+                    color={MarqueeOptionsMenu.Instance.userColor}
+                    style={{ width: '19px' }}
+                />
+                {this._displayImages ? (
+                    <div className="face-document-image-container">
+                        {DocListCast(this._props.faceDoc[DocData].associatedDocs).map(doc => {
+                            const [name, type] = ImageCast(doc[Doc.LayoutFieldKey(doc)]).url.href.split('.');
+                            return (
+                                <div className="image-wrapper" key={Utils.GenerateGuid()}>
+                                    <img
+                                        onClick={async () => {
+                                            await DocumentView.showDocument(doc, { willZoomCentered: true });
+                                        }}
+                                        style={{ maxWidth: '60px', margin: '10px' }}
+                                        src={`${name}_o.${type}`}
+                                    />
+                                    <div className="remove-item">
+                                        <IconButton
+                                            tooltip={'Remove Doc From Face Collection'}
+                                            onPointerDown={() => {
+                                                this.deleteAssociatedDoc(doc);
+                                            }}
+                                            icon={'x'}
+                                            style={{ width: '4px' }}
+                                            size={Size.XSMALL}
+                                        />
+                                    </div>
+                                </div>
+                            );
+                        })}
+                    </div>
+                ) : (
+                    <div></div>
+                )}
+            </div>
+        );
+    }
+}
+
+@observer
+export class FaceCollectionBox extends ViewBoxBaseComponent<FieldViewProps>() {
+    public static LayoutString(fieldKey: string) {
+        return FieldView.LayoutString(FaceCollectionBox, fieldKey);
+    }
+
+    public static Instance: FaceCollectionBox;
+
+    @computed get currentDocs() {
+        if (Doc.ActiveDashboard) {
+            return DocListCast(Doc.ActiveDashboard[DocData].faceDocuments);
+        } else {
+            return [];
+        }
+    }
+
+    constructor(props: any) {
+        super(props);
+        makeObservable(this);
+        FaceCollectionBox.Instance = this;
+    }
+
+    render() {
+        return (
+            <div className="searchBox-container" style={{ pointerEvents: 'all', color: SnappingManager.userColor, background: SnappingManager.userBackgroundColor }}>
+                {this.currentDocs.map(doc => {
+                    return <FaceDocumentItem faceDoc={doc} />;
+                })}
+            </div>
+        );
+    }
+}
+
+Docs.Prototypes.TemplateMap.set(DocumentType.FACECOLLECTION, {
+    layout: { view: FaceCollectionBox, dataField: 'data' },
+    options: { acl: '', _width: 400 },
+});