aboutsummaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
authorIEatChili <nanunguyen99@gmail.com>2024-08-15 14:13:02 -0400
committerIEatChili <nanunguyen99@gmail.com>2024-08-15 14:13:02 -0400
commit0e975569e5686138e52bdc554b3f0391f42aeead (patch)
treebab5aff6665cdd07a37948d943d687c6d5158b2d /src
parent9e03f9333641c818ed9c711282f27f7213cbe3c1 (diff)
feat: added face recogntion box
Diffstat (limited to 'src')
-rw-r--r--src/client/apis/gpt/GPT.ts2
-rw-r--r--src/client/documents/DocumentTypes.ts1
-rw-r--r--src/client/documents/Documents.ts4
-rw-r--r--src/client/util/CurrentUserUtils.ts13
-rw-r--r--src/client/views/KeywordBox.tsx7
-rw-r--r--src/client/views/Main.tsx2
-rw-r--r--src/client/views/collections/collectionFreeForm/FaceCollectionBox.scss74
-rw-r--r--src/client/views/collections/collectionFreeForm/FaceCollectionBox.tsx217
-rw-r--r--src/client/views/collections/collectionFreeForm/ImageLabelBox.tsx5
-rw-r--r--src/client/views/search/FaceRecognitionHandler.tsx74
-rw-r--r--src/fields/Doc.ts1
11 files changed, 375 insertions, 25 deletions
diff --git a/src/client/apis/gpt/GPT.ts b/src/client/apis/gpt/GPT.ts
index 05007960d..8dd3fd6e2 100644
--- a/src/client/apis/gpt/GPT.ts
+++ b/src/client/apis/gpt/GPT.ts
@@ -128,7 +128,7 @@ const gptImageLabel = async (src: string): Promise<string> => {
{
role: 'user',
content: [
- { type: 'text', text: 'Give three to five labels to describe this image.' },
+ { type: 'text', text: 'Give three labels to describe this image.' },
{
type: 'image_url',
image_url: {
diff --git a/src/client/documents/DocumentTypes.ts b/src/client/documents/DocumentTypes.ts
index a9ea889b3..b66a29ac2 100644
--- a/src/client/documents/DocumentTypes.ts
+++ b/src/client/documents/DocumentTypes.ts
@@ -17,6 +17,7 @@ export enum DocumentType {
FONTICON = 'fonticonbox',
SEARCH = 'search', // search query
IMAGEGROUPER = 'imagegrouper',
+ FACECOLLECTION = 'facecollection',
LABEL = 'label', // simple text label
BUTTON = 'button', // onClick button
WEBCAM = 'webcam', // webcam
diff --git a/src/client/documents/Documents.ts b/src/client/documents/Documents.ts
index 3737aa0b5..ecea74fab 100644
--- a/src/client/documents/Documents.ts
+++ b/src/client/documents/Documents.ts
@@ -795,6 +795,10 @@ export namespace Docs {
return InstanceFromProto(Prototypes.get(DocumentType.IMAGEGROUPER), undefined, options);
}
+ export function FaceCollectionDocument(options: DocumentOptions = {}) {
+ return InstanceFromProto(Prototypes.get(DocumentType.FACECOLLECTION), undefined, options);
+ }
+
export function LoadingDocument(file: File | string, options: DocumentOptions) {
return InstanceFromProto(Prototypes.get(DocumentType.LOADING), undefined, { _height: 150, _width: 200, title: typeof file === 'string' ? file : file.name, ...options }, undefined, '');
}
diff --git a/src/client/util/CurrentUserUtils.ts b/src/client/util/CurrentUserUtils.ts
index cb3d9df62..db0de83b9 100644
--- a/src/client/util/CurrentUserUtils.ts
+++ b/src/client/util/CurrentUserUtils.ts
@@ -458,7 +458,8 @@ pie title Minerals in my tap water
{ title: "Shared", toolTip: "Shared Docs", target: Doc.MySharedDocs, ignoreClick: true, icon: "users", funcs: {badgeValue: badgeValue}},
{ title: "Trails", toolTip: "Trails ⌘R", target: Doc.UserDoc(), ignoreClick: true, icon: "pres-trail", funcs: {target: getActiveDashTrails}},
{ title: "User Doc", toolTip: "User Doc", target: this.setupUserDocView(doc, "myUserDocView"), ignoreClick: true, icon: "address-card",funcs: {hidden: "IsNoviceMode()"} },
- { title: "Image Grouper", toolTip: "Image Grouper", target: this.setupImageGrouper(doc, "myImageGrouper"), ignoreClick: true, icon: "folder-open", hidden: false }
+ { title: "Image Grouper", toolTip: "Image Grouper", target: this.setupImageGrouper(doc, "myImageGrouper"), ignoreClick: true, icon: "folder-open", hidden: false },
+ { title: "Face Collection", toolTip: "Face Collection", target: this.setupFaceCollection(doc, "myFaceCollection"), ignoreClick: true, icon: "face-smile", hidden: false },
].map(tuple => ({...tuple, scripts:{onClick: 'selectMainMenu(this)'}}));
}
@@ -500,6 +501,12 @@ pie title Minerals in my tap water
_lockedPosition: true, _type_collection: CollectionViewType.Schema });
}
+ static setupFaceCollection(doc: Doc, field: string) {
+ return DocUtils.AssignDocField(doc, field, (opts) => Docs.Create.FaceCollectionDocument(opts), {
+ dontRegisterView: true, backgroundColor: "dimgray", ignoreClick: true, title: "Face Collection", isSystem: true, childDragAction: dropActionType.embed,
+ _lockedPosition: true, _type_collection: CollectionViewType.Schema });
+ }
+
/// Initializes the panel of draggable tools that is opened from the left sidebar.
static setupToolsBtnPanel(doc: Doc, field:string) {
const allTools = DocListCast(DocCast(doc[field])?.data);
@@ -702,8 +709,8 @@ pie title Minerals in my tap water
{ title: "Fit All", icon: "object-group", toolTip: "Fit Docs to View (double click to make sticky)",btnType: ButtonType.ToggleButton, ignoreClick:true, expertMode: false, toolType:"viewAll", funcs: {}, scripts: { onClick: '{ return showFreeform(this.toolType, _readOnly_);}', onDoubleClick: '{ return showFreeform(this.toolType, _readOnly_, true);}'}}, // Only when floating document is selected in freeform
{ title: "Clusters", icon: "braille", toolTip: "Show Doc Clusters", btnType: ButtonType.ToggleButton, ignoreClick: true, expertMode: false, toolType:"clusters", funcs: {}, scripts: { onClick: '{ return showFreeform(this.toolType, _readOnly_);}'}}, // Only when floating document is selected in freeform
{ title: "Cards", icon: "brain", toolTip: "Flashcards", btnType: ButtonType.ToggleButton, ignoreClick: true, expertMode: false, toolType:"flashcards", funcs: {}, scripts: { onClick: '{ return showFreeform(this.toolType, _readOnly_);}'}}, // Only when floating document is selected in freeform
- { title: "Arrange", icon:"arrow-down-short-wide",toolTip:"Toggle Auto Arrange", btnType: ButtonType.ToggleButton, ignoreClick: true, expertMode: false, toolType:"arrange", funcs: {hidden: 'IsNoviceMode()'}, scripts: { onClick: '{ return showFreeform(this.toolType, _readOnly_);}'}}, // Only when floating document is selected in freeform
-
+ { title: "Arrange", icon:"arrow-down-short-wide",toolTip:"Auto Arrange", btnType: ButtonType.ToggleButton, ignoreClick: true, expertMode: false, toolType:"arrange", funcs: {hidden: 'IsNoviceMode()'}, scripts: { onClick: '{ return showFreeform(this.toolType, _readOnly_);}'}}, // Only when floating document is selected in freeform
+
]
}
static textTools():Button[] {
diff --git a/src/client/views/KeywordBox.tsx b/src/client/views/KeywordBox.tsx
index 68584a7fa..fc9c38a11 100644
--- a/src/client/views/KeywordBox.tsx
+++ b/src/client/views/KeywordBox.tsx
@@ -7,7 +7,7 @@ import { Doc, DocListCast } from '../../fields/Doc';
import { DocData } from '../../fields/DocSymbols';
import { List } from '../../fields/List';
import { NumCast, StrCast } from '../../fields/Types';
-import { emptyFunction } from '../../Utils';
+import { emptyFunction, Utils } from '../../Utils';
import { DocumentType } from '../documents/DocumentTypes';
import { DragManager } from '../util/DragManager';
import { SnappingManager } from '../util/SnappingManager';
@@ -124,7 +124,7 @@ export class KeywordItem extends ObservableReactComponent<KeywordItemProps> {
render() {
return (
- <div className="keyword" onClick={this._props.setToEditing} onPointerDown={this.handleDragStart} ref={this.ref}>
+ <div className="keyword" onClick={this._props.setToEditing} onPointerDown={this.handleDragStart} ref={this.ref} key={Utils.GenerateGuid()}>
{this._props.keyword}
{this.props.isEditing && <IconButton tooltip={'Remove label'} onPointerDown={this.removeLabel} icon={'X'} style={{ width: '8px', height: '8px', marginLeft: '10px' }} />}
</div>
@@ -297,7 +297,7 @@ export class KeywordBox extends ObservableReactComponent<KeywordBoxProps> {
<div className="keywords-content">
<div className="keywords-list">
{(keywordsList as List<string>).map(keyword => {
- return <KeywordItem doc={this._props.doc} keyword={keyword} keywordDoc={this.getKeywordCollection(keyword)} setToEditing={this.setToEditing} isEditing={this._props.isEditing}></KeywordItem>;
+ return <KeywordItem key={Utils.GenerateGuid()} doc={this._props.doc} keyword={keyword} keywordDoc={this.getKeywordCollection(keyword)} setToEditing={this.setToEditing} isEditing={this._props.isEditing}></KeywordItem>;
})}
</div>
{this._props.isEditing ? (
@@ -331,6 +331,7 @@ export class KeywordBox extends ObservableReactComponent<KeywordBoxProps> {
onClick={() => {
this.submitLabel(keyword);
}}
+ key={Utils.GenerateGuid()}
/>
);
})}
diff --git a/src/client/views/Main.tsx b/src/client/views/Main.tsx
index ada934aea..85c2b3d47 100644
--- a/src/client/views/Main.tsx
+++ b/src/client/views/Main.tsx
@@ -62,6 +62,7 @@ import { PresBox, PresElementBox } from './nodes/trails';
import { SearchBox } from './search/SearchBox';
import { ImageLabelBox } from './collections/collectionFreeForm/ImageLabelBox';
import { FaceRecognitionHandler } from './search/FaceRecognitionHandler';
+import { FaceCollectionBox } from './collections/collectionFreeForm/FaceCollectionBox';
dotenv.config();
@@ -135,6 +136,7 @@ FieldLoader.ServerLoadStatus = { requested: 0, retrieved: 0, message: 'cache' };
PresElementBox,
SearchBox,
ImageLabelBox, //Here!
+ FaceCollectionBox,
FunctionPlotBox,
InkingStroke,
LinkBox,
diff --git a/src/client/views/collections/collectionFreeForm/FaceCollectionBox.scss b/src/client/views/collections/collectionFreeForm/FaceCollectionBox.scss
new file mode 100644
index 000000000..480d109c8
--- /dev/null
+++ b/src/client/views/collections/collectionFreeForm/FaceCollectionBox.scss
@@ -0,0 +1,74 @@
+.face-document-item {
+ background: #555555;
+ margin-top: 10px;
+ margin-bottom: 10px;
+ padding: 10px;
+ border-radius: 10px;
+ position: relative;
+
+ h1 {
+ color: white;
+ font-size: 24px;
+ text-align: center;
+ }
+
+ .face-collection-buttons {
+ position: absolute;
+ top: 10px;
+ right: 10px;
+ }
+
+ .face-document-image-container {
+ display: flex;
+ justify-content: center;
+ flex-wrap: wrap;
+
+ .image-wrapper {
+ position: relative;
+ width: 70px;
+ height: 70px;
+ margin: 10px;
+ display: flex;
+ align-items: center; // Center vertically
+ justify-content: center; // Center horizontally
+
+ img {
+ width: 100%;
+ height: 100%;
+ object-fit: cover; // This ensures the image covers the container without stretching
+ border-radius: 5px;
+ border: 2px solid white;
+ transition: border-color 0.4s;
+
+ &:hover {
+ border-color: orange; // Change this to your desired hover border color
+ }
+ }
+
+ .remove-item {
+ position: absolute;
+ bottom: -5;
+ right: -5;
+ background-color: rgba(0, 0, 0, 0.5); // Optional: to add a background behind the icon for better visibility
+ border-radius: 30%;
+ width: 10px; // Adjust size as needed
+ height: 10px; // Adjust size as needed
+ display: flex;
+ align-items: center;
+ justify-content: center;
+ }
+ }
+
+ // img {
+ // max-width: 60px;
+ // margin: 10px;
+ // border-radius: 5px;
+ // border: 2px solid white;
+ // transition: 0.4s;
+
+ // &:hover {
+ // border-color: orange;
+ // }
+ // }
+ }
+}
diff --git a/src/client/views/collections/collectionFreeForm/FaceCollectionBox.tsx b/src/client/views/collections/collectionFreeForm/FaceCollectionBox.tsx
new file mode 100644
index 000000000..1d3f88df1
--- /dev/null
+++ b/src/client/views/collections/collectionFreeForm/FaceCollectionBox.tsx
@@ -0,0 +1,217 @@
+import { observer } from 'mobx-react';
+import React from 'react';
+import { Docs } from '../../../documents/Documents';
+import { DocumentType } from '../../../documents/DocumentTypes';
+import { ViewBoxBaseComponent } from '../../DocComponent';
+import { FieldView, FieldViewProps } from '../../nodes/FieldView';
+import 'ldrs/ring';
+import { SnappingManager } from '../../../util/SnappingManager';
+import { action, computed, makeObservable, observable, reaction } from 'mobx';
+import { Doc, DocListCast, NumListCast } from '../../../../fields/Doc';
+import { DocData } from '../../../../fields/DocSymbols';
+import { ImageCast, StrCast } from '../../../../fields/Types';
+import { ObservableReactComponent } from '../../ObservableReactComponent';
+import './FaceCollectionBox.scss';
+import { IconButton, Size } from 'browndash-components';
+import { FontAwesomeIcon } from '@fortawesome/react-fontawesome';
+import { MarqueeOptionsMenu } from './MarqueeOptionsMenu';
+import { List } from '../../../../fields/List';
+import { DocumentView } from '../../nodes/DocumentView';
+import { Utils } from '../../../../Utils';
+import { DragManager } from '../../../util/DragManager';
+import * as faceapi from 'face-api.js';
+import { FaceMatcher } from 'face-api.js';
+
+interface FaceDocumentProps {
+ faceDoc: Doc;
+}
+
+/**
+ * A componenent to visually represent a Face Document.
+ */
+@observer
+export class FaceDocumentItem extends ObservableReactComponent<FaceDocumentProps> {
+ private ref: React.RefObject<HTMLDivElement>;
+ @observable _displayImages: boolean = true;
+ private _dropDisposer?: DragManager.DragDropDisposer;
+ private _inputRef = React.createRef<HTMLInputElement>();
+
+ constructor(props: any) {
+ super(props);
+ makeObservable(this);
+ this.ref = React.createRef();
+ }
+
+ protected createDropTarget = (ele: HTMLDivElement) => {
+ this._dropDisposer?.();
+ ele && (this._dropDisposer = DragManager.MakeDropTarget(ele, this.onInternalDrop.bind(this), this._props.faceDoc));
+ };
+
+ protected onInternalDrop(e: Event, de: DragManager.DropEvent): boolean {
+ const { docDragData } = de.complete;
+ if (docDragData) {
+ const filteredDocs = docDragData.droppedDocuments.filter(doc => doc.type === DocumentType.IMG);
+ filteredDocs.forEach(doc => {
+ // If the current Face Document has no items, and the doc has more than one face descriptor, don't let the user add the document first.
+ if ((this._props.faceDoc[DocData].faceDescriptors as List<List<number>>).length === 0 && (doc[DocData].faces as List<List<number>>).length > 1) {
+ alert('Cannot add a document with multiple faces as the first item!');
+ } else {
+ // Loop through the documents' face descriptors.
+ // Choose the face with the smallest distance to add.
+ const float32Array = (this._props.faceDoc[DocData].faceDescriptors as List<List<number>>).map(faceDescriptor => new Float32Array(Array.from(faceDescriptor)));
+ const labeledFaceDescriptor = new faceapi.LabeledFaceDescriptors(StrCast(this._props.faceDoc[DocData].label), float32Array);
+ const faceDescriptors: faceapi.LabeledFaceDescriptors[] = [labeledFaceDescriptor];
+
+ const faceMatcher = new FaceMatcher(faceDescriptors, 1);
+ let cur_lowest_distance = 1;
+ let cur_matching_face = new List<number>();
+
+ (doc[DocData].faces as List<List<number>>).forEach(face => {
+ // If the face has the current lowest distance, mark it as such
+ // Once that lowest distance is found, add the face descriptor to the faceDoc, and add the associated doc
+ const convered_32_array: Float32Array = new Float32Array(Array.from(face));
+ const match = faceMatcher.matchDescriptor(convered_32_array);
+
+ if (match.distance < cur_lowest_distance) {
+ cur_lowest_distance = match.distance;
+ cur_matching_face = face;
+ }
+ });
+
+ if (doc[DocData][`FACE DESCRIPTOR - ${this._props.faceDoc[DocData].label}`]) {
+ doc[DocData][`FACE DESCRIPTOR - ${this._props.faceDoc[DocData].label}`] = new List<List<number>>([...(doc[DocData][`FACE DESCRIPTOR - ${this._props.faceDoc[DocData].label}`] as List<List<number>>), cur_matching_face]);
+ } else {
+ doc[DocData][`FACE DESCRIPTOR - ${this._props.faceDoc[DocData].label}`] = new List<List<number>>([cur_matching_face]);
+ }
+
+ this._props.faceDoc[DocData].associatedDocs = new List<Doc>([...DocListCast(this._props.faceDoc[DocData].associatedDocs), doc]);
+ this._props.faceDoc[DocData].faceDescriptors = new List<List<number>>([...(this._props.faceDoc[DocData].faceDescriptors as List<List<number>>), cur_matching_face]);
+
+ //const match = faceMatcher.findBestMatch(cur_descriptor);
+ }
+ });
+ return false;
+ }
+ return false;
+ }
+
+ /**
+ * Toggles whether a Face Document displays its associated docs.
+ */
+ @action
+ onDisplayClick() {
+ this._displayImages = !this._displayImages;
+ }
+
+ /**
+ * Deletes a Face Document.
+ */
+ @action
+ deleteFaceDocument = () => {
+ if (Doc.ActiveDashboard) {
+ Doc.ActiveDashboard[DocData].faceDocuments = new List<Doc>(DocListCast(Doc.ActiveDashboard[DocData].faceDocuments).filter(doc => doc !== this._props.faceDoc));
+ }
+ };
+
+ /**
+ * Deletes a document from a Face Document's associated docs list.
+ * @param doc
+ */
+ @action
+ deleteAssociatedDoc = (doc: Doc) => {
+ this._props.faceDoc[DocData].faceDescriptors = new List<List<number>>(
+ (this._props.faceDoc[DocData].faceDescriptors as List<List<number>>).filter(fd => !(doc[DocData][`FACE DESCRIPTOR - ${this._props.faceDoc[DocData].label}`] as List<List<number>>).includes(fd))
+ );
+ doc[DocData][`FACE DESCRIPTOR - ${this._props.faceDoc[DocData].label}`] = new List<List<number>>();
+ this._props.faceDoc[DocData].associatedDocs = new List<Doc>(DocListCast(this._props.faceDoc[DocData].associatedDocs).filter(associatedDoc => associatedDoc !== doc));
+ };
+
+ render() {
+ return (
+ <div className="face-document-item" ref={ele => this.createDropTarget(ele!)}>
+ <div className="face-collection-buttons">
+ <IconButton tooltip={'Delete Face From Collection'} onPointerDown={this.deleteFaceDocument} icon={'x'} style={{ width: '4px' }} size={Size.XSMALL} />
+ </div>
+ <div className="face-document-top">
+ <h1>{StrCast(this._props.faceDoc[DocData].label)}</h1>
+ </div>
+ <IconButton
+ tooltip={'See image information'}
+ onPointerDown={() => this.onDisplayClick()}
+ icon={this._displayImages ? <FontAwesomeIcon icon="caret-up" /> : <FontAwesomeIcon icon="caret-down" />}
+ color={MarqueeOptionsMenu.Instance.userColor}
+ style={{ width: '19px' }}
+ />
+ {this._displayImages ? (
+ <div className="face-document-image-container">
+ {DocListCast(this._props.faceDoc[DocData].associatedDocs).map(doc => {
+ const [name, type] = ImageCast(doc[Doc.LayoutFieldKey(doc)]).url.href.split('.');
+ return (
+ <div className="image-wrapper" key={Utils.GenerateGuid()}>
+ <img
+ onClick={async () => {
+ await DocumentView.showDocument(doc, { willZoomCentered: true });
+ }}
+ style={{ maxWidth: '60px', margin: '10px' }}
+ src={`${name}_o.${type}`}
+ />
+ <div className="remove-item">
+ <IconButton
+ tooltip={'Remove Doc From Face Collection'}
+ onPointerDown={() => {
+ this.deleteAssociatedDoc(doc);
+ }}
+ icon={'x'}
+ style={{ width: '4px' }}
+ size={Size.XSMALL}
+ />
+ </div>
+ </div>
+ );
+ })}
+ </div>
+ ) : (
+ <div></div>
+ )}
+ </div>
+ );
+ }
+}
+
+@observer
+export class FaceCollectionBox extends ViewBoxBaseComponent<FieldViewProps>() {
+ public static LayoutString(fieldKey: string) {
+ return FieldView.LayoutString(FaceCollectionBox, fieldKey);
+ }
+
+ public static Instance: FaceCollectionBox;
+
+ @computed get currentDocs() {
+ if (Doc.ActiveDashboard) {
+ return DocListCast(Doc.ActiveDashboard[DocData].faceDocuments);
+ } else {
+ return [];
+ }
+ }
+
+ constructor(props: any) {
+ super(props);
+ makeObservable(this);
+ FaceCollectionBox.Instance = this;
+ }
+
+ render() {
+ return (
+ <div className="searchBox-container" style={{ pointerEvents: 'all', color: SnappingManager.userColor, background: SnappingManager.userBackgroundColor }}>
+ {this.currentDocs.map(doc => {
+ return <FaceDocumentItem faceDoc={doc} />;
+ })}
+ </div>
+ );
+ }
+}
+
+Docs.Prototypes.TemplateMap.set(DocumentType.FACECOLLECTION, {
+ layout: { view: FaceCollectionBox, dataField: 'data' },
+ options: { acl: '', _width: 400 },
+});
diff --git a/src/client/views/collections/collectionFreeForm/ImageLabelBox.tsx b/src/client/views/collections/collectionFreeForm/ImageLabelBox.tsx
index af01d6cbc..421b5d0a6 100644
--- a/src/client/views/collections/collectionFreeForm/ImageLabelBox.tsx
+++ b/src/client/views/collections/collectionFreeForm/ImageLabelBox.tsx
@@ -179,6 +179,7 @@ export class ImageLabelBox extends ViewBoxBaseComponent<FieldViewProps>() {
const labels = imageInfo.labels.split('\n');
labels.forEach(label => {
label = label.replace(/^\d+\.\s*|-|\*/, '').trim();
+ console.log(label);
imageInfo.doc[DocData][`${label}`] = true;
(imageInfo.doc[DocData].data_labels as List<string>).push(label);
});
@@ -198,7 +199,7 @@ export class ImageLabelBox extends ViewBoxBaseComponent<FieldViewProps>() {
for (let index = 0; index < (doc[DocData].data_labels as List<string>).length; index++) {
const label = (doc[DocData].data_labels as List<string>)[index];
const embedding = await gptGetEmbedding(label);
- doc[`data_labels_embedding_${index + 1}`] = new List<number>(embedding);
+ doc[DocData][`data_labels_embedding_${index + 1}`] = new List<number>(embedding);
}
}
@@ -209,7 +210,7 @@ export class ImageLabelBox extends ViewBoxBaseComponent<FieldViewProps>() {
// For each image, loop through the labels, and calculate similarity. Associate it with the
// most similar one.
this._selectedImages.forEach(doc => {
- const embedLists = numberRange((doc[DocData].data_labels as List<string>).length).map(n => Array.from(NumListCast(doc[`data_labels_embedding_${n + 1}`])));
+ const embedLists = numberRange((doc[DocData].data_labels as List<string>).length).map(n => Array.from(NumListCast(doc[DocData][`data_labels_embedding_${n + 1}`])));
const bestEmbedScore = (embedding: Opt<number[]>) => Math.max(...embedLists.map((l, index) => (embedding && similarity(Array.from(embedding), l)!) || 0));
const {label: mostSimilarLabelCollect} =
this._labelGroups.map(label => ({ label, similarityScore: bestEmbedScore(labelToEmbedding.get(label)) }))
diff --git a/src/client/views/search/FaceRecognitionHandler.tsx b/src/client/views/search/FaceRecognitionHandler.tsx
index fcd38c42f..ef4622ea2 100644
--- a/src/client/views/search/FaceRecognitionHandler.tsx
+++ b/src/client/views/search/FaceRecognitionHandler.tsx
@@ -1,14 +1,13 @@
import * as faceapi from 'face-api.js';
-import { FaceMatcher, TinyFaceDetectorOptions } from 'face-api.js';
-import { Doc, DocListCast, NumListCast } from '../../../fields/Doc';
+import { FaceMatcher } from 'face-api.js';
+import { Doc, DocListCast } from '../../../fields/Doc';
import { DocData } from '../../../fields/DocSymbols';
import { List } from '../../../fields/List';
-import { ObjectField } from '../../../fields/ObjectField';
-import { ImageCast, StrCast } from '../../../fields/Types';
-import { DocUtils } from '../../documents/DocUtils';
-import { Deserializable } from '../../util/SerializationHelper';
-import { DocumentView } from '../nodes/DocumentView';
+import { ImageCast, NumCast, StrCast } from '../../../fields/Types';
+/**
+ * A class that handles face recognition.
+ */
export class FaceRecognitionHandler {
static _instance: FaceRecognitionHandler;
private loadedModels: boolean = false;
@@ -18,8 +17,12 @@ export class FaceRecognitionHandler {
constructor() {
FaceRecognitionHandler._instance = this;
this.loadModels();
+ this.examinedDocs = new Set(DocListCast(Doc.UserDoc()[DocData].examinedFaceDocs, []));
}
+ /**
+ * Loads the face detection models.
+ */
async loadModels() {
const MODEL_URL = `/models`;
await faceapi.loadFaceDetectionModel(MODEL_URL);
@@ -32,19 +35,31 @@ export class FaceRecognitionHandler {
return FaceRecognitionHandler._instance ?? new FaceRecognitionHandler();
}
+ /**
+ * When a document is added, look for matching face documents.
+ * @param doc The document being analyzed.
+ */
public async findMatches(doc: Doc) {
if (this.loadedModels) {
+ // If the Dashboard doesn't have a list of face documents yet, initialize the list.
if (!Doc.ActiveDashboard![DocData].faceDocuments) {
Doc.ActiveDashboard![DocData].faceDocuments = new List<Doc>();
}
+ // If the doc is currently already been examined, or it is being processed, stop examining the document.
if (this.examinedDocs.has(doc) || this.processingDocs.has(doc)) {
return;
}
+ // Mark the document as being processed.
this.processingDocs.add(doc);
try {
+ if (!Doc.UserDoc()[DocData].faceDocNum) {
+ Doc.UserDoc()[DocData].faceDocNum = 0;
+ }
+
+ // Get the image the document contains and analyze for faces.
const [name, type] = ImageCast(doc[Doc.LayoutFieldKey(doc)]).url.href.split('.');
const imageURL = `${name}_o.${type}`;
@@ -52,30 +67,51 @@ export class FaceRecognitionHandler {
const fullFaceDescriptions = await faceapi.detectAllFaces(img).withFaceLandmarks().withFaceDescriptors();
+ doc[DocData].faces = new List<List<number>>();
+
+ // For each face detected, find a match.
for (const fd of fullFaceDescriptions) {
- const match = this.findMatch(fd.descriptor);
+ let match = this.findMatch(fd.descriptor);
+ let converted_list = new List<number>();
+
if (match) {
+ // If a matching Face Document has been found, add the document to the Face Document's associated docs and append the face
+ // descriptor to the Face Document's descriptor list.
const converted_array = Array.from(fd.descriptor);
- const converted_list = new List<number>(converted_array);
+ converted_list = new List<number>(converted_array);
match[DocData].associatedDocs = new List<Doc>([...DocListCast(match[DocData].associatedDocs), doc]);
match[DocData].faceDescriptors = new List<List<number>>([...(match[DocData].faceDescriptors as List<List<number>>), converted_list]);
} else {
+ // If a matching Face Document has not been found, create a new Face Document.
const newFaceDocument = new Doc();
const converted_array = Array.from(fd.descriptor);
- const converted_list = new List<number>(converted_array);
+ converted_list = new List<number>(converted_array);
newFaceDocument[DocData].faceDescriptors = new List<List<number>>();
(newFaceDocument[DocData].faceDescriptors as List<List<number>>).push(converted_list);
- newFaceDocument[DocData].label = `Person ${DocListCast(Doc.ActiveDashboard![DocData].faceDocuments).length + 1}`;
+ Doc.UserDoc()[DocData].faceDocNum = NumCast(Doc.UserDoc()[DocData].faceDocNum) + 1;
+ newFaceDocument[DocData].label = `Face ${Doc.UserDoc()[DocData].faceDocNum}`;
newFaceDocument[DocData].associatedDocs = new List<Doc>([doc]);
Doc.ActiveDashboard![DocData].faceDocuments = new List<Doc>([...DocListCast(Doc.ActiveDashboard![DocData].faceDocuments), newFaceDocument]);
+ match = newFaceDocument;
}
+
+ // Assign a field in the document of the matching Face Document.
+ if (doc[DocData][`FACE DESCRIPTOR - ${match[DocData].label}`]) {
+ doc[DocData][`FACE DESCRIPTOR - ${match[DocData].label}`] = new List<List<number>>([...(doc[DocData][`FACE DESCRIPTOR - ${match[DocData].label}`] as List<List<number>>), converted_list]);
+ } else {
+ doc[DocData][`FACE DESCRIPTOR - ${match[DocData].label}`] = new List<List<number>>([converted_list]);
+ }
+
+ doc[DocData].faces = new List<List<number>>([...(doc[DocData].faces as List<List<number>>), converted_list]);
}
+ // Updates the examined docs field.
this.examinedDocs.add(doc);
- console.log(this.examinedDocs);
-
- DocListCast(Doc.ActiveDashboard![DocData].faceDocuments).forEach(doc => console.log(DocListCast(doc[DocData].associatedDocs)));
+ if (!Doc.UserDoc()[DocData].examinedFaceDocs) {
+ Doc.UserDoc()[DocData].examinedFaceDocs = new List<Doc>();
+ }
+ Doc.UserDoc()[DocData].examinedFaceDocs = new List<Doc>([...DocListCast(Doc.UserDoc()[DocData].examinedFaceDocs), doc]);
} catch (error) {
console.error('Error processing document:', error);
} finally {
@@ -84,6 +120,11 @@ export class FaceRecognitionHandler {
}
}
+ /**
+ * Finds a matching Face Document given a descriptor
+ * @param cur_descriptor The current descriptor whose match is being searched for.
+ * @returns The most similar Face Document.
+ */
private findMatch(cur_descriptor: Float32Array) {
if (DocListCast(Doc.ActiveDashboard![DocData].faceDocuments).length < 1) {
return null;
@@ -95,19 +136,20 @@ export class FaceRecognitionHandler {
});
const faceMatcher = new FaceMatcher(faceDescriptors, 0.6);
const match = faceMatcher.findBestMatch(cur_descriptor);
-
if (match.label == 'unknown') {
return null;
} else {
for (const doc of DocListCast(Doc.ActiveDashboard![DocData].faceDocuments)) {
if (doc[DocData].label === match.label) {
- console.log(match.label);
return doc;
}
}
}
}
+ /**
+ * Loads an image
+ */
private loadImage = (src: string): Promise<HTMLImageElement> => {
return new Promise((resolve, reject) => {
const img = new Image();
diff --git a/src/fields/Doc.ts b/src/fields/Doc.ts
index 4abb23404..ebd30ceba 100644
--- a/src/fields/Doc.ts
+++ b/src/fields/Doc.ts
@@ -265,6 +265,7 @@ export class Doc extends RefField {
public static get MyDockedBtns() { return DocCast(Doc.UserDoc().myDockedBtns); } // prettier-ignore
public static get MySearcher() { return DocCast(Doc.UserDoc().mySearcher); } // prettier-ignore
public static get MyImageGrouper() { return DocCast(Doc.UserDoc().myImageGrouper); } //prettier-ignore
+ public static get MyFaceCollection() { return DocCast(Doc.UserDoc().myFaceCollection); } //prettier-ignore
public static get MyHeaderBar() { return DocCast(Doc.UserDoc().myHeaderBar); } // prettier-ignore
public static get MyLeftSidebarMenu() { return DocCast(Doc.UserDoc().myLeftSidebarMenu); } // prettier-ignore
public static get MyLeftSidebarPanel() { return DocCast(Doc.UserDoc().myLeftSidebarPanel); } // prettier-ignore