diff options
| -rw-r--r-- | src/client/Network.ts | 11 | ||||
| -rw-r--r-- | src/client/documents/DocUtils.ts | 2 | ||||
| -rw-r--r-- | src/client/views/collections/collectionFreeForm/FaceCollectionBox.tsx | 88 | ||||
| -rw-r--r-- | src/client/views/collections/collectionFreeForm/MarqueeView.tsx | 97 | ||||
| -rw-r--r-- | src/client/views/nodes/FaceRectangles.tsx | 3 | ||||
| -rw-r--r-- | src/client/views/search/FaceRecognitionHandler.tsx | 64 |
6 files changed, 87 insertions, 178 deletions
diff --git a/src/client/Network.ts b/src/client/Network.ts index 8876d8190..17f8a6534 100644 --- a/src/client/Network.ts +++ b/src/client/Network.ts @@ -1,3 +1,4 @@ +import formidable from 'formidable'; import * as requestPromise from 'request-promise'; import { ClientUtils } from '../ClientUtils'; import { Utils } from '../Utils'; @@ -49,15 +50,9 @@ export namespace Networking { if (!fileguidpairs.length) { return []; } - const maxFileSize = 5000000; + const maxFileSize = 6000000; if (fileguidpairs.some(f => f.file.size > maxFileSize)) { - return new Promise<Upload.FileResponse<T>[]>(res => { - res([{ - source: { size: 0, filepath: '', mimetype: '', originalFilename: '', newFilename: '',hashAlgorithm: false, - toJSON: () => ({ size: 0, filepath: '', mimetype: '', originalFilename: '', newFilename: '',name: '', length: 0, mtime: new Date(), type: '' }) }, - result: { name: '', message: `max file size (${maxFileSize / 1000000}MB) exceeded` } - }]) // prettier-ignore - }); + return new Promise<Upload.FileResponse<T>[]>(res => res([{ source: { newFilename: '', mimetype: '' } as formidable.File, result: new Error(`max file size (${maxFileSize / 1000000}MB) exceeded`) }])); } formData.set('fileguids', fileguidpairs.map(pair => pair.guid).join(';')); formData.set('filesize', fileguidpairs.reduce((sum, pair) => sum + pair.file.size, 0).toString()); diff --git a/src/client/documents/DocUtils.ts b/src/client/documents/DocUtils.ts index a503d732b..35d835f1f 100644 --- a/src/client/documents/DocUtils.ts +++ b/src/client/documents/DocUtils.ts @@ -739,7 +739,7 @@ export namespace DocUtils { const { source: { newFilename, mimetype }, result, - } = upfiles.lastElement() ?? { source: { newFilename: '', mimetype: '' }, result: { message: 'upload failed' } }; + } = upfiles.lastElement() ?? { source: { newFilename: '', mimetype: '' }, result: new Error('upload failed') }; if (result instanceof Error) { if (overwriteDoc) { overwriteDoc.loadingError = result.message; diff --git a/src/client/views/collections/collectionFreeForm/FaceCollectionBox.tsx b/src/client/views/collections/collectionFreeForm/FaceCollectionBox.tsx index 50b91e8fe..d5a2809dc 100644 --- a/src/client/views/collections/collectionFreeForm/FaceCollectionBox.tsx +++ b/src/client/views/collections/collectionFreeForm/FaceCollectionBox.tsx @@ -11,17 +11,20 @@ import { Doc, DocListCast } from '../../../../fields/Doc'; import { DocData } from '../../../../fields/DocSymbols'; import { Id } from '../../../../fields/FieldSymbols'; import { List } from '../../../../fields/List'; -import { ImageCast, StrCast } from '../../../../fields/Types'; +import { listSpec } from '../../../../fields/Schema'; +import { Cast, ImageCast, StrCast } from '../../../../fields/Types'; import { DocumentType } from '../../../documents/DocumentTypes'; import { Docs } from '../../../documents/Documents'; import { DragManager } from '../../../util/DragManager'; import { SnappingManager } from '../../../util/SnappingManager'; +import { undoable } from '../../../util/UndoManager'; import { ViewBoxBaseComponent } from '../../DocComponent'; import { ObservableReactComponent } from '../../ObservableReactComponent'; import { DocumentView } from '../../nodes/DocumentView'; import { FieldView, FieldViewProps } from '../../nodes/FieldView'; import './FaceCollectionBox.scss'; import { MarqueeOptionsMenu } from './MarqueeOptionsMenu'; +import { FaceRecognitionHandler } from '../../search/FaceRecognitionHandler'; interface FaceDocumentProps { faceDoc: Doc; @@ -32,17 +35,15 @@ interface FaceDocumentProps { */ @observer export class FaceDocumentItem extends ObservableReactComponent<FaceDocumentProps> { - private ref: React.RefObject<HTMLDivElement>; - @observable _displayImages: boolean = true; private _dropDisposer?: DragManager.DragDropDisposer; - private _inputRef = React.createRef<HTMLInputElement>(); constructor(props: FaceDocumentProps) { super(props); makeObservable(this); - this.ref = React.createRef(); } + @observable _displayImages: boolean = true; + protected createDropTarget = (ele: HTMLDivElement) => { this._dropDisposer?.(); ele && (this._dropDisposer = DragManager.MakeDropTarget(ele, this.onInternalDrop.bind(this), this._props.faceDoc)); @@ -54,20 +55,20 @@ export class FaceDocumentItem extends ObservableReactComponent<FaceDocumentProps const filteredDocs = docDragData.droppedDocuments.filter(doc => doc.type === DocumentType.IMG); filteredDocs.forEach(doc => { // If the current Face Document has no items, and the doc has more than one face descriptor, don't let the user add the document first. - if ((this._props.faceDoc[DocData].faceDescriptors as List<List<number>>).length === 0 && (doc[DocData].faces as List<List<number>>).length > 1) { + if ((this._props.faceDoc[DocData].face_descriptors as List<List<number>>).length === 0 && (doc[DocData][FaceRecognitionHandler.FacesField(doc)] as List<List<number>>).length > 1) { alert('Cannot add a document with multiple faces as the first item!'); } else { // Loop through the documents' face descriptors. // Choose the face with the smallest distance to add. - const float32Array = (this._props.faceDoc[DocData].faceDescriptors as List<List<number>>).map(faceDescriptor => new Float32Array(Array.from(faceDescriptor))); - const labeledFaceDescriptor = new faceapi.LabeledFaceDescriptors(StrCast(this._props.faceDoc[DocData].label), float32Array); + const float32Array = (this._props.faceDoc[DocData].face_descriptors as List<List<number>>).map(faceDescriptor => new Float32Array(Array.from(faceDescriptor))); + const labeledFaceDescriptor = new faceapi.LabeledFaceDescriptors(StrCast(this._props.faceDoc[DocData].face_label), float32Array); const faceDescriptors: faceapi.LabeledFaceDescriptors[] = [labeledFaceDescriptor]; const faceMatcher = new FaceMatcher(faceDescriptors, 1); let cur_lowest_distance = 1; let cur_matching_face = new List<number>(); - (doc[DocData].faces as List<List<number>>).forEach(face => { + (doc[DocData][FaceRecognitionHandler.FacesField(doc)] as List<List<number>>).forEach(face => { // If the face has the current lowest distance, mark it as such // Once that lowest distance is found, add the face descriptor to the faceDoc, and add the associated doc const convered_32_array: Float32Array = new Float32Array(Array.from(face)); @@ -79,16 +80,15 @@ export class FaceDocumentItem extends ObservableReactComponent<FaceDocumentProps } }); - if (doc[DocData][`FACE DESCRIPTOR - ${this._props.faceDoc[DocData].label}`]) { - doc[DocData][`FACE DESCRIPTOR - ${this._props.faceDoc[DocData].label}`] = new List<List<number>>([...(doc[DocData][`FACE DESCRIPTOR - ${this._props.faceDoc[DocData].label}`] as List<List<number>>), cur_matching_face]); + const faceFieldKey = FaceRecognitionHandler.FaceField(doc, this._props.faceDoc); + if (doc[DocData][faceFieldKey]) { + Cast(doc[DocData][faceFieldKey], listSpec('number'), null).push(cur_matching_face as unknown as number); // items are lists of numbers, not numbers, but type system can't handle that } else { - doc[DocData][`FACE DESCRIPTOR - ${this._props.faceDoc[DocData].label}`] = new List<List<number>>([cur_matching_face]); + doc[DocData][faceFieldKey] = new List<List<number>>([cur_matching_face]); } - this._props.faceDoc[DocData].associatedDocs = new List<Doc>([...DocListCast(this._props.faceDoc[DocData].associatedDocs), doc]); - this._props.faceDoc[DocData].faceDescriptors = new List<List<number>>([...(this._props.faceDoc[DocData].faceDescriptors as List<List<number>>), cur_matching_face]); - - //const match = faceMatcher.findBestMatch(cur_descriptor); + Doc.AddDocToList(this._props.faceDoc[DocData], 'face_docList', doc); + Cast(this._props.faceDoc[DocData].face_descriptors, listSpec('number'), null).push(cur_matching_face as unknown as number); // items are lists of numbers, not numbers, but type system can't handle that } }); return false; @@ -107,12 +107,11 @@ export class FaceDocumentItem extends ObservableReactComponent<FaceDocumentProps /** * Deletes a Face Document. */ - @action - deleteFaceDocument = () => { + deleteFaceDocument = undoable(() => { if (Doc.ActiveDashboard) { - Doc.ActiveDashboard[DocData].faceDocuments = new List<Doc>(DocListCast(Doc.ActiveDashboard[DocData].faceDocuments).filter(doc => doc !== this._props.faceDoc)); + Doc.RemoveDocFromList(Doc.ActiveDashboard[DocData], 'faceDocuments', this._props.faceDoc); } - }; + }, 'remove face'); /** * Deletes a document from a Face Document's associated docs list. @@ -120,60 +119,44 @@ export class FaceDocumentItem extends ObservableReactComponent<FaceDocumentProps */ @action deleteAssociatedDoc = (doc: Doc) => { - this._props.faceDoc[DocData].faceDescriptors = new List<List<number>>( - (this._props.faceDoc[DocData].faceDescriptors as List<List<number>>).filter(fd => !(doc[DocData][`FACE DESCRIPTOR - ${this._props.faceDoc[DocData].label}`] as List<List<number>>).includes(fd)) + this._props.faceDoc[DocData].face_descriptors = new List<List<number>>( + (this._props.faceDoc[DocData].face_descriptors as List<List<number>>).filter(fd => !(doc[DocData][FaceRecognitionHandler.FaceField(doc, this._props.faceDoc)] as List<List<number>>).includes(fd)) ); - doc[DocData][`FACE DESCRIPTOR - ${this._props.faceDoc[DocData].label}`] = new List<List<number>>(); - this._props.faceDoc[DocData].associatedDocs = new List<Doc>(DocListCast(this._props.faceDoc[DocData].associatedDocs).filter(associatedDoc => associatedDoc !== doc)); + doc[DocData][FaceRecognitionHandler.FaceField(doc, this._props.faceDoc)] = new List<List<number>>(); + Doc.RemoveDocFromList(this._props.faceDoc[DocData], 'face_docList', doc); }; render() { return ( <div className="face-document-item" ref={ele => this.createDropTarget(ele!)}> <div className="face-collection-buttons"> - <IconButton tooltip={'Delete Face From Collection'} onPointerDown={this.deleteFaceDocument} icon={'x'} style={{ width: '4px' }} size={Size.XSMALL} /> + <IconButton tooltip="Delete Face From Collection" onPointerDown={this.deleteFaceDocument} icon={'x'} style={{ width: '4px' }} size={Size.XSMALL} /> </div> <div className="face-document-top"> - <h1>{StrCast(this._props.faceDoc[DocData].label)}</h1> + <h1>{StrCast(this._props.faceDoc[DocData].face_label)}</h1> </div> <IconButton - tooltip={'See image information'} + tooltip="See image information" onPointerDown={() => this.onDisplayClick()} - icon={this._displayImages ? <FontAwesomeIcon icon="caret-up" /> : <FontAwesomeIcon icon="caret-down" />} + icon={<FontAwesomeIcon icon={this._displayImages ? 'caret-up' : 'caret-down'} />} color={MarqueeOptionsMenu.Instance.userColor} style={{ width: '19px' }} /> {this._displayImages ? ( <div className="face-document-image-container"> - {DocListCast(this._props.faceDoc[DocData].associatedDocs).map(doc => { + {DocListCast(this._props.faceDoc[DocData].face_docList).map(doc => { const [name, type] = ImageCast(doc[Doc.LayoutFieldKey(doc)]).url.href.split('.'); return ( <div className="image-wrapper" key={Utils.GenerateGuid()}> - <img - onClick={async () => { - await DocumentView.showDocument(doc, { willZoomCentered: true }); - }} - style={{ maxWidth: '60px', margin: '10px' }} - src={`${name}_o.${type}`} - /> + <img onClick={() => DocumentView.showDocument(doc, { willZoomCentered: true })} style={{ maxWidth: '60px', margin: '10px' }} src={`${name}_o.${type}`} /> <div className="remove-item"> - <IconButton - tooltip={'Remove Doc From Face Collection'} - onPointerDown={() => { - this.deleteAssociatedDoc(doc); - }} - icon={'x'} - style={{ width: '4px' }} - size={Size.XSMALL} - /> + <IconButton tooltip={'Remove Doc From Face Collection'} onPointerDown={() => this.deleteAssociatedDoc(doc)} icon={'x'} style={{ width: '4px' }} size={Size.XSMALL} /> </div> </div> ); })} </div> - ) : ( - <div></div> - )} + ) : null} </div> ); } @@ -190,9 +173,8 @@ export class FaceCollectionBox extends ViewBoxBaseComponent<FieldViewProps>() { @computed get currentDocs() { if (Doc.ActiveDashboard) { return DocListCast(Doc.ActiveDashboard[DocData].faceDocuments); - } else { - return []; } + return []; } constructor(props: FieldViewProps) { @@ -204,9 +186,9 @@ export class FaceCollectionBox extends ViewBoxBaseComponent<FieldViewProps>() { render() { return ( <div className="searchBox-container" style={{ pointerEvents: 'all', color: SnappingManager.userColor, background: SnappingManager.userBackgroundColor }}> - {this.currentDocs.map(doc => { - return <FaceDocumentItem key={doc[Id]} faceDoc={doc} />; - })} + {this.currentDocs.map(doc => ( + <FaceDocumentItem key={doc[Id]} faceDoc={doc} /> + ))} </div> ); } diff --git a/src/client/views/collections/collectionFreeForm/MarqueeView.tsx b/src/client/views/collections/collectionFreeForm/MarqueeView.tsx index d7a41df64..6fee076ee 100644 --- a/src/client/views/collections/collectionFreeForm/MarqueeView.tsx +++ b/src/client/views/collections/collectionFreeForm/MarqueeView.tsx @@ -1,28 +1,24 @@ -/* eslint-disable jsx-a11y/no-static-element-interactions */ -import similarity from 'compute-cosine-similarity'; import { action, computed, makeObservable, observable } from 'mobx'; import { observer } from 'mobx-react'; import * as React from 'react'; import { ClientUtils, lightOrDark, returnFalse } from '../../../../ClientUtils'; -import { intersectRect, numberRange } from '../../../../Utils'; -import { Doc, DocListCast, NumListCast, Opt } from '../../../../fields/Doc'; +import { intersectRect } from '../../../../Utils'; +import { Doc, DocListCast, Opt } from '../../../../fields/Doc'; import { AclAdmin, AclAugment, AclEdit, DocData } from '../../../../fields/DocSymbols'; import { Id } from '../../../../fields/FieldSymbols'; -import { InkData, InkField, InkTool } from '../../../../fields/InkField'; +import { InkTool } from '../../../../fields/InkField'; import { List } from '../../../../fields/List'; -import { RichTextField } from '../../../../fields/RichTextField'; -import { Cast, FieldValue, ImageCast, NumCast, StrCast } from '../../../../fields/Types'; +import { Cast, NumCast, StrCast } from '../../../../fields/Types'; import { ImageField } from '../../../../fields/URLField'; import { GetEffectiveAcl } from '../../../../fields/util'; -import { gptGetEmbedding, gptImageLabel } from '../../../apis/gpt/GPT'; -import { CognitiveServices } from '../../../cognitive_services/CognitiveServices'; import { DocUtils } from '../../../documents/DocUtils'; -import { CollectionViewType, DocumentType } from '../../../documents/DocumentTypes'; +import { DocumentType } from '../../../documents/DocumentTypes'; import { Docs, DocumentOptions } from '../../../documents/Documents'; import { SnappingManager, freeformScrollMode } from '../../../util/SnappingManager'; import { Transform } from '../../../util/Transform'; import { UndoManager, undoBatch } from '../../../util/UndoManager'; import { ContextMenu } from '../../ContextMenu'; +import { MainView } from '../../MainView'; import { ObservableReactComponent } from '../../ObservableReactComponent'; import { MarqueeViewBounds } from '../../PinFuncs'; import { PreviewCursor } from '../../PreviewCursor'; @@ -30,15 +26,10 @@ import { DocumentView } from '../../nodes/DocumentView'; import { OpenWhere } from '../../nodes/OpenWhere'; import { pasteImageBitmap } from '../../nodes/WebBoxRenderer'; import { FormattedTextBox } from '../../nodes/formattedText/FormattedTextBox'; -import { CollectionCardView } from '../CollectionCardDeckView'; import { SubCollectionViewProps } from '../CollectionSubView'; -import { CollectionFreeFormView } from './CollectionFreeFormView'; -import { ImageLabelHandler } from './ImageLabelHandler'; +import { ImageLabelBoxData } from './ImageLabelBox'; import { MarqueeOptionsMenu } from './MarqueeOptionsMenu'; import './MarqueeView.scss'; -import { MainView } from '../../MainView'; -import { ImageLabelBox, ImageLabelBoxData } from './ImageLabelBox'; -import { SearchBox } from '../../search/SearchBox'; interface MarqueeViewProps { getContainerTransform: () => Transform; @@ -68,7 +59,7 @@ export class MarqueeView extends ObservableReactComponent<SubCollectionViewProps static Instance: MarqueeView; - constructor(props: any) { + constructor(props: SubCollectionViewProps & MarqueeViewProps) { super(props); makeObservable(this); MarqueeView.Instance = this; @@ -496,71 +487,6 @@ export class MarqueeView extends ObservableReactComponent<SubCollectionViewProps }); @undoBatch - syntaxHighlight = action((e: KeyboardEvent | React.PointerEvent | undefined) => { - const selected = this.marqueeSelect(false); - if (e instanceof KeyboardEvent ? e.key === 'i' : true) { - const inks = selected.filter(s => s.type === DocumentType.INK); - const setDocs = selected.filter(s => s.type === DocumentType.RTF && s.color); - const sets = setDocs.map(sd => Cast(sd.data, RichTextField)?.Text as string); - const colors = setDocs.map(sd => FieldValue(sd.color) as string); - const wordToColor = new Map<string, string>(); - sets.forEach((st: string, i: number) => st.split(',').forEach(word => wordToColor.set(word, colors[i]))); - const strokes: InkData[] = []; - inks.filter(i => Cast(i.data, InkField)).forEach(i => { - const d = Cast(i.data, InkField, null); - const left = Math.min(...(d?.inkData.map(pd => pd.X) ?? [0])); - const top = Math.min(...(d?.inkData.map(pd => pd.Y) ?? [0])); - strokes.push(d.inkData.map(pd => ({ X: pd.X + NumCast(i.x) - left, Y: pd.Y + NumCast(i.y) - top }))); - }); - CognitiveServices.Inking.Appliers.InterpretStrokes(strokes).then(results => { - // const wordResults = results.filter((r: any) => r.category === "inkWord"); - // for (const word of wordResults) { - // const indices: number[] = word.strokeIds; - // indices.forEach(i => { - // if (wordToColor.has(word.recognizedText.toLowerCase())) { - // inks[i].color = wordToColor.get(word.recognizedText.toLowerCase()); - // } - // else { - // for (const alt of word.alternates) { - // if (wordToColor.has(alt.recognizedString.toLowerCase())) { - // inks[i].color = wordToColor.get(alt.recognizedString.toLowerCase()); - // break; - // } - // } - // } - // }) - // } - // const wordResults = results.filter((r: any) => r.category === "inkWord"); - // for (const word of wordResults) { - // const indices: number[] = word.strokeIds; - // indices.forEach(i => { - // const otherInks: Doc[] = []; - // indices.forEach(i2 => i2 !== i && otherInks.push(inks[i2])); - // inks[i].relatedInks = new List<Doc>(otherInks); - // const uniqueColors: string[] = []; - // Array.from(wordToColor.values()).forEach(c => uniqueColors.indexOf(c) === -1 && uniqueColors.push(c)); - // inks[i].alternativeColors = new List<string>(uniqueColors); - // if (wordToColor.has(word.recognizedText.toLowerCase())) { - // inks[i].color = wordToColor.get(word.recognizedText.toLowerCase()); - // } - // else if (word.alternates) { - // for (const alt of word.alternates) { - // if (wordToColor.has(alt.recognizedString.toLowerCase())) { - // inks[i].color = wordToColor.get(alt.recognizedString.toLowerCase()); - // break; - // } - // } - // } - // }); - // } - const lines = results.filter((r: any) => r.category === 'line'); - const text = lines.map((l: any) => l.recognizedText).join('\r\n'); - this._props.addDocument?.(Docs.Create.TextDocument(text, { _width: this.Bounds.width, _height: this.Bounds.height, x: this.Bounds.left + this.Bounds.width, y: this.Bounds.top, title: text })); - }); - } - }); - - @undoBatch summary = action(() => { const selected = this.marqueeSelect(false).map(d => { this._props.removeDocument?.(d); @@ -704,8 +630,8 @@ export class MarqueeView extends ObservableReactComponent<SubCollectionViewProps transform: `translate(${p[0]}px, ${p[1]}px)`, width: Math.abs(v[0]), height: Math.abs(v[1]), - color: lightOrDark(this._props.Document?.backgroundColor ?? 'white'), - borderColor: lightOrDark(this._props.Document?.backgroundColor ?? 'white'), + color: lightOrDark((this._props.Document?.backgroundColor as string) ?? 'white'), + borderColor: lightOrDark((this._props.Document?.backgroundColor as string) ?? 'white'), zIndex: 2000, }}> {' '} @@ -714,7 +640,7 @@ export class MarqueeView extends ObservableReactComponent<SubCollectionViewProps <polyline // points={this._lassoPts.reduce((s, pt) => s + pt[0] + ',' + pt[1] + ' ', '')} fill="none" - stroke={lightOrDark(this._props.Document?.backgroundColor ?? 'white')} + stroke={lightOrDark((this._props.Document?.backgroundColor as string) ?? 'white')} strokeWidth="1" strokeDasharray="3" /> @@ -753,7 +679,6 @@ export class MarqueeView extends ObservableReactComponent<SubCollectionViewProps }; render() { return ( - // eslint-disable-next-line jsx-a11y/click-events-have-key-events <div className="marqueeView" ref={r => { diff --git a/src/client/views/nodes/FaceRectangles.tsx b/src/client/views/nodes/FaceRectangles.tsx index ade4225d9..19aa90a8b 100644 --- a/src/client/views/nodes/FaceRectangles.tsx +++ b/src/client/views/nodes/FaceRectangles.tsx @@ -4,6 +4,7 @@ import { Doc, DocListCast } from '../../../fields/Doc'; import { Id } from '../../../fields/FieldSymbols'; import { Cast, NumCast } from '../../../fields/Types'; import FaceRectangle from './FaceRectangle'; +import { FaceRecognitionHandler } from '../search/FaceRecognitionHandler'; interface FaceRectanglesProps { document: Doc; @@ -19,7 +20,7 @@ export interface RectangleTemplate { @observer export class FaceRectangles extends React.Component<FaceRectanglesProps> { render() { - const faces = DocListCast(this.props.document.faces); + const faces = DocListCast(this.props.document[FaceRecognitionHandler.FacesField(this.props.document)]); const templates: RectangleTemplate[] = faces.map(faceDoc => { const rectangle = Cast(faceDoc.faceRectangle, Doc) as Doc; const style = { diff --git a/src/client/views/search/FaceRecognitionHandler.tsx b/src/client/views/search/FaceRecognitionHandler.tsx index 29ca6e797..dc271fe73 100644 --- a/src/client/views/search/FaceRecognitionHandler.tsx +++ b/src/client/views/search/FaceRecognitionHandler.tsx @@ -1,42 +1,46 @@ import * as faceapi from 'face-api.js'; import { FaceMatcher } from 'face-api.js'; +import { computed } from 'mobx'; import { Doc, DocListCast } from '../../../fields/Doc'; import { DocData } from '../../../fields/DocSymbols'; import { List } from '../../../fields/List'; +import { listSpec } from '../../../fields/Schema'; import { Cast, ImageCast, NumCast, StrCast } from '../../../fields/Types'; -import { DocumentManager } from '../../util/DocumentManager'; -import { computed } from 'mobx'; import { DocumentType } from '../../documents/DocumentTypes'; -import { listSpec } from '../../../fields/Schema'; +import { DocumentManager } from '../../util/DocumentManager'; /** * A class that handles face recognition. */ export class FaceRecognitionHandler { static _instance: FaceRecognitionHandler; - private loadedModels: boolean = false; - @computed get examinedFaceDocs() { - return DocListCast(Doc.UserDoc().examinedFaceDocs); - } - processingDocs: Set<Doc> = new Set(); - pendingLoadDocs: Doc[] = []; + private _loadedModels: boolean = false; + private _processingDocs: Set<Doc> = new Set(); + private _pendingLoadDocs: Doc[] = []; + + public static FaceField = (target: Doc, doc: Doc) => `${Doc.LayoutFieldKey(target)}_${doc.face_label}`; + public static FacesField = (target: Doc) => `${Doc.LayoutFieldKey(target)}_Faces`; constructor() { FaceRecognitionHandler._instance = this; - this.loadModels().then(() => this.pendingLoadDocs.forEach(this.findMatches)); + this.loadModels().then(() => this._pendingLoadDocs.forEach(this.findMatches)); DocumentManager.Instance.AddAnyViewRenderedCB(dv => FaceRecognitionHandler.Instance.findMatches(dv.Document)); } + @computed get examinedFaceDocs() { + return DocListCast(Doc.UserDoc().examinedFaceDocs); + } + /** * Loads the face detection models. */ - async loadModels() { + loadModels = async () => { const MODEL_URL = `/models`; await faceapi.loadFaceDetectionModel(MODEL_URL); await faceapi.loadFaceLandmarkModel(MODEL_URL); await faceapi.loadFaceRecognitionModel(MODEL_URL); - this.loadedModels = true; - } + this._loadedModels = true; + }; public static get Instance() { return FaceRecognitionHandler._instance ?? new FaceRecognitionHandler(); @@ -47,8 +51,8 @@ export class FaceRecognitionHandler { * @param doc The document being analyzed. */ public findMatches = async (doc: Doc) => { - if (!this.loadedModels || !Doc.ActiveDashboard) { - this.pendingLoadDocs.push(doc); + if (!this._loadedModels || !Doc.ActiveDashboard) { + this._pendingLoadDocs.push(doc); return; } @@ -59,12 +63,12 @@ export class FaceRecognitionHandler { const imgUrl = ImageCast(doc[Doc.LayoutFieldKey(doc)]); // If the doc isn't an image or currently already been examined or is being processed, stop examining the document. - if (!imgUrl || this.examinedFaceDocs.includes(doc) || this.processingDocs.has(doc)) { + if (!imgUrl || this.examinedFaceDocs.includes(doc) || this._processingDocs.has(doc)) { return; } // Mark the document as being processed. - this.processingDocs.add(doc); + this._processingDocs.add(doc); // Get the image the document contains and analyze for faces. const [name, type] = imgUrl.url.href.split('.'); @@ -74,7 +78,7 @@ export class FaceRecognitionHandler { const fullFaceDescriptions = await faceapi.detectAllFaces(img).withFaceLandmarks().withFaceDescriptors(); - doc[DocData].faces = new List<List<number>>(); + doc[DocData][FaceRecognitionHandler.FacesField(doc)] = new List<List<number>>(); // For each face detected, find a match. for (const fd of fullFaceDescriptions) { @@ -84,34 +88,36 @@ export class FaceRecognitionHandler { if (match) { // If a matching Face Document has been found, add the document to the Face Document's associated docs and append the face // descriptor to the Face Document's descriptor list. - Doc.AddDocToList(match, 'associatedDocs', doc); - Cast(match.faceDescriptors, listSpec('number'), null).push(converted_list as unknown as number); // items are lists of numbers, not numbers, but type system can't handle that + Doc.AddDocToList(match, 'face_docList', doc); + Cast(match.face_descriptors, listSpec('number'), null).push(converted_list as unknown as number); // items are lists of numbers, not numbers, but type system can't handle that } else { // If a matching Face Document has not been found, create a new Face Document. Doc.UserDoc().faceDocNum = NumCast(Doc.UserDoc().faceDocNum) + 1; const newFaceDocument = new Doc(); - newFaceDocument.label = `Face ${Doc.UserDoc().faceDocNum}`; - newFaceDocument.associatedDocs = new List<Doc>([doc]); - newFaceDocument.faceDescriptors = new List<List<number>>([converted_list]); + newFaceDocument.title = `Face ${Doc.UserDoc().faceDocNum}`; + newFaceDocument.face = ''; // just to make prettyprinting look better + newFaceDocument.face_label = `Face${Doc.UserDoc().faceDocNum}`; + newFaceDocument.face_docList = new List<Doc>([doc]); + newFaceDocument.face_descriptors = new List<List<number>>([converted_list]); Doc.AddDocToList(Doc.ActiveDashboard[DocData], 'faceDocuments', newFaceDocument); match = newFaceDocument; } // Assign a field in the document of the matching Face Document. - const faceDescripField = `FACE DESCRIPTOR - ${match[DocData].label}`; + const faceDescripField = FaceRecognitionHandler.FaceField(doc, match); if (doc[DocData][faceDescripField]) { Cast(doc[DocData][faceDescripField], listSpec('number'), null).push(converted_list as unknown as number); // items are lists of numbers, not numbers, but type system can't handle that } else { doc[DocData][faceDescripField] = new List<List<number>>([converted_list]); } - Cast(doc[DocData].faces, listSpec('number'), null).push(converted_list as unknown as number); // items are lists of numbers, not numbers, but type system can't handle that + Cast(doc[DocData][FaceRecognitionHandler.FacesField(doc)], listSpec('number'), null).push(converted_list as unknown as number); // items are lists of numbers, not numbers, but type system can't handle that Doc.AddDocToList(Doc.UserDoc(), 'examinedFaceDocs', doc); } - this.processingDocs.delete(doc); + this._processingDocs.delete(doc); }; /** @@ -125,14 +131,14 @@ export class FaceRecognitionHandler { } const faceDescriptors: faceapi.LabeledFaceDescriptors[] = DocListCast(Doc.ActiveDashboard[DocData].faceDocuments).map(faceDocument => { - const float32Array = (faceDocument[DocData].faceDescriptors as List<List<number>>).map(faceDescriptor => new Float32Array(Array.from(faceDescriptor))); - return new faceapi.LabeledFaceDescriptors(StrCast(faceDocument[DocData].label), float32Array); + const float32Array = (faceDocument[DocData].face_descriptors as List<List<number>>).map(faceDescriptor => new Float32Array(Array.from(faceDescriptor))); + return new faceapi.LabeledFaceDescriptors(StrCast(faceDocument[DocData].face_label), float32Array); }); const faceMatcher = new FaceMatcher(faceDescriptors, 0.6); const match = faceMatcher.findBestMatch(cur_descriptor); if (match.label !== 'unknown') { for (const doc of DocListCast(Doc.ActiveDashboard[DocData].faceDocuments)) { - if (doc[DocData].label === match.label) { + if (doc[DocData].face_label === match.label) { return doc; } } |