From cdb21e036ff65d63991a53798133407be1d5755f Mon Sep 17 00:00:00 2001 From: bobzel Date: Tue, 20 Aug 2024 21:30:32 -0400 Subject: fixed error handling of images too big to load. cleaned up facecollectionbox. changed metadata field naming to match conventions. --- .../collectionFreeForm/FaceCollectionBox.tsx | 88 ++++++++------------ .../collections/collectionFreeForm/MarqueeView.tsx | 97 +++------------------- 2 files changed, 46 insertions(+), 139 deletions(-) (limited to 'src/client/views/collections') diff --git a/src/client/views/collections/collectionFreeForm/FaceCollectionBox.tsx b/src/client/views/collections/collectionFreeForm/FaceCollectionBox.tsx index 50b91e8fe..d5a2809dc 100644 --- a/src/client/views/collections/collectionFreeForm/FaceCollectionBox.tsx +++ b/src/client/views/collections/collectionFreeForm/FaceCollectionBox.tsx @@ -11,17 +11,20 @@ import { Doc, DocListCast } from '../../../../fields/Doc'; import { DocData } from '../../../../fields/DocSymbols'; import { Id } from '../../../../fields/FieldSymbols'; import { List } from '../../../../fields/List'; -import { ImageCast, StrCast } from '../../../../fields/Types'; +import { listSpec } from '../../../../fields/Schema'; +import { Cast, ImageCast, StrCast } from '../../../../fields/Types'; import { DocumentType } from '../../../documents/DocumentTypes'; import { Docs } from '../../../documents/Documents'; import { DragManager } from '../../../util/DragManager'; import { SnappingManager } from '../../../util/SnappingManager'; +import { undoable } from '../../../util/UndoManager'; import { ViewBoxBaseComponent } from '../../DocComponent'; import { ObservableReactComponent } from '../../ObservableReactComponent'; import { DocumentView } from '../../nodes/DocumentView'; import { FieldView, FieldViewProps } from '../../nodes/FieldView'; import './FaceCollectionBox.scss'; import { MarqueeOptionsMenu } from './MarqueeOptionsMenu'; +import { FaceRecognitionHandler } from '../../search/FaceRecognitionHandler'; interface FaceDocumentProps { faceDoc: Doc; @@ -32,17 +35,15 @@ interface FaceDocumentProps { */ @observer export class FaceDocumentItem extends ObservableReactComponent { - private ref: React.RefObject; - @observable _displayImages: boolean = true; private _dropDisposer?: DragManager.DragDropDisposer; - private _inputRef = React.createRef(); constructor(props: FaceDocumentProps) { super(props); makeObservable(this); - this.ref = React.createRef(); } + @observable _displayImages: boolean = true; + protected createDropTarget = (ele: HTMLDivElement) => { this._dropDisposer?.(); ele && (this._dropDisposer = DragManager.MakeDropTarget(ele, this.onInternalDrop.bind(this), this._props.faceDoc)); @@ -54,20 +55,20 @@ export class FaceDocumentItem extends ObservableReactComponent doc.type === DocumentType.IMG); filteredDocs.forEach(doc => { // If the current Face Document has no items, and the doc has more than one face descriptor, don't let the user add the document first. - if ((this._props.faceDoc[DocData].faceDescriptors as List>).length === 0 && (doc[DocData].faces as List>).length > 1) { + if ((this._props.faceDoc[DocData].face_descriptors as List>).length === 0 && (doc[DocData][FaceRecognitionHandler.FacesField(doc)] as List>).length > 1) { alert('Cannot add a document with multiple faces as the first item!'); } else { // Loop through the documents' face descriptors. // Choose the face with the smallest distance to add. - const float32Array = (this._props.faceDoc[DocData].faceDescriptors as List>).map(faceDescriptor => new Float32Array(Array.from(faceDescriptor))); - const labeledFaceDescriptor = new faceapi.LabeledFaceDescriptors(StrCast(this._props.faceDoc[DocData].label), float32Array); + const float32Array = (this._props.faceDoc[DocData].face_descriptors as List>).map(faceDescriptor => new Float32Array(Array.from(faceDescriptor))); + const labeledFaceDescriptor = new faceapi.LabeledFaceDescriptors(StrCast(this._props.faceDoc[DocData].face_label), float32Array); const faceDescriptors: faceapi.LabeledFaceDescriptors[] = [labeledFaceDescriptor]; const faceMatcher = new FaceMatcher(faceDescriptors, 1); let cur_lowest_distance = 1; let cur_matching_face = new List(); - (doc[DocData].faces as List>).forEach(face => { + (doc[DocData][FaceRecognitionHandler.FacesField(doc)] as List>).forEach(face => { // If the face has the current lowest distance, mark it as such // Once that lowest distance is found, add the face descriptor to the faceDoc, and add the associated doc const convered_32_array: Float32Array = new Float32Array(Array.from(face)); @@ -79,16 +80,15 @@ export class FaceDocumentItem extends ObservableReactComponent>([...(doc[DocData][`FACE DESCRIPTOR - ${this._props.faceDoc[DocData].label}`] as List>), cur_matching_face]); + const faceFieldKey = FaceRecognitionHandler.FaceField(doc, this._props.faceDoc); + if (doc[DocData][faceFieldKey]) { + Cast(doc[DocData][faceFieldKey], listSpec('number'), null).push(cur_matching_face as unknown as number); // items are lists of numbers, not numbers, but type system can't handle that } else { - doc[DocData][`FACE DESCRIPTOR - ${this._props.faceDoc[DocData].label}`] = new List>([cur_matching_face]); + doc[DocData][faceFieldKey] = new List>([cur_matching_face]); } - this._props.faceDoc[DocData].associatedDocs = new List([...DocListCast(this._props.faceDoc[DocData].associatedDocs), doc]); - this._props.faceDoc[DocData].faceDescriptors = new List>([...(this._props.faceDoc[DocData].faceDescriptors as List>), cur_matching_face]); - - //const match = faceMatcher.findBestMatch(cur_descriptor); + Doc.AddDocToList(this._props.faceDoc[DocData], 'face_docList', doc); + Cast(this._props.faceDoc[DocData].face_descriptors, listSpec('number'), null).push(cur_matching_face as unknown as number); // items are lists of numbers, not numbers, but type system can't handle that } }); return false; @@ -107,12 +107,11 @@ export class FaceDocumentItem extends ObservableReactComponent { + deleteFaceDocument = undoable(() => { if (Doc.ActiveDashboard) { - Doc.ActiveDashboard[DocData].faceDocuments = new List(DocListCast(Doc.ActiveDashboard[DocData].faceDocuments).filter(doc => doc !== this._props.faceDoc)); + Doc.RemoveDocFromList(Doc.ActiveDashboard[DocData], 'faceDocuments', this._props.faceDoc); } - }; + }, 'remove face'); /** * Deletes a document from a Face Document's associated docs list. @@ -120,60 +119,44 @@ export class FaceDocumentItem extends ObservableReactComponent { - this._props.faceDoc[DocData].faceDescriptors = new List>( - (this._props.faceDoc[DocData].faceDescriptors as List>).filter(fd => !(doc[DocData][`FACE DESCRIPTOR - ${this._props.faceDoc[DocData].label}`] as List>).includes(fd)) + this._props.faceDoc[DocData].face_descriptors = new List>( + (this._props.faceDoc[DocData].face_descriptors as List>).filter(fd => !(doc[DocData][FaceRecognitionHandler.FaceField(doc, this._props.faceDoc)] as List>).includes(fd)) ); - doc[DocData][`FACE DESCRIPTOR - ${this._props.faceDoc[DocData].label}`] = new List>(); - this._props.faceDoc[DocData].associatedDocs = new List(DocListCast(this._props.faceDoc[DocData].associatedDocs).filter(associatedDoc => associatedDoc !== doc)); + doc[DocData][FaceRecognitionHandler.FaceField(doc, this._props.faceDoc)] = new List>(); + Doc.RemoveDocFromList(this._props.faceDoc[DocData], 'face_docList', doc); }; render() { return (
this.createDropTarget(ele!)}>
- +
-

{StrCast(this._props.faceDoc[DocData].label)}

+

{StrCast(this._props.faceDoc[DocData].face_label)}

this.onDisplayClick()} - icon={this._displayImages ? : } + icon={} color={MarqueeOptionsMenu.Instance.userColor} style={{ width: '19px' }} /> {this._displayImages ? (
- {DocListCast(this._props.faceDoc[DocData].associatedDocs).map(doc => { + {DocListCast(this._props.faceDoc[DocData].face_docList).map(doc => { const [name, type] = ImageCast(doc[Doc.LayoutFieldKey(doc)]).url.href.split('.'); return (
- { - await DocumentView.showDocument(doc, { willZoomCentered: true }); - }} - style={{ maxWidth: '60px', margin: '10px' }} - src={`${name}_o.${type}`} - /> + DocumentView.showDocument(doc, { willZoomCentered: true })} style={{ maxWidth: '60px', margin: '10px' }} src={`${name}_o.${type}`} />
- { - this.deleteAssociatedDoc(doc); - }} - icon={'x'} - style={{ width: '4px' }} - size={Size.XSMALL} - /> + this.deleteAssociatedDoc(doc)} icon={'x'} style={{ width: '4px' }} size={Size.XSMALL} />
); })}
- ) : ( -
- )} + ) : null}
); } @@ -190,9 +173,8 @@ export class FaceCollectionBox extends ViewBoxBaseComponent() { @computed get currentDocs() { if (Doc.ActiveDashboard) { return DocListCast(Doc.ActiveDashboard[DocData].faceDocuments); - } else { - return []; } + return []; } constructor(props: FieldViewProps) { @@ -204,9 +186,9 @@ export class FaceCollectionBox extends ViewBoxBaseComponent() { render() { return (
- {this.currentDocs.map(doc => { - return ; - })} + {this.currentDocs.map(doc => ( + + ))}
); } diff --git a/src/client/views/collections/collectionFreeForm/MarqueeView.tsx b/src/client/views/collections/collectionFreeForm/MarqueeView.tsx index d7a41df64..6fee076ee 100644 --- a/src/client/views/collections/collectionFreeForm/MarqueeView.tsx +++ b/src/client/views/collections/collectionFreeForm/MarqueeView.tsx @@ -1,28 +1,24 @@ -/* eslint-disable jsx-a11y/no-static-element-interactions */ -import similarity from 'compute-cosine-similarity'; import { action, computed, makeObservable, observable } from 'mobx'; import { observer } from 'mobx-react'; import * as React from 'react'; import { ClientUtils, lightOrDark, returnFalse } from '../../../../ClientUtils'; -import { intersectRect, numberRange } from '../../../../Utils'; -import { Doc, DocListCast, NumListCast, Opt } from '../../../../fields/Doc'; +import { intersectRect } from '../../../../Utils'; +import { Doc, DocListCast, Opt } from '../../../../fields/Doc'; import { AclAdmin, AclAugment, AclEdit, DocData } from '../../../../fields/DocSymbols'; import { Id } from '../../../../fields/FieldSymbols'; -import { InkData, InkField, InkTool } from '../../../../fields/InkField'; +import { InkTool } from '../../../../fields/InkField'; import { List } from '../../../../fields/List'; -import { RichTextField } from '../../../../fields/RichTextField'; -import { Cast, FieldValue, ImageCast, NumCast, StrCast } from '../../../../fields/Types'; +import { Cast, NumCast, StrCast } from '../../../../fields/Types'; import { ImageField } from '../../../../fields/URLField'; import { GetEffectiveAcl } from '../../../../fields/util'; -import { gptGetEmbedding, gptImageLabel } from '../../../apis/gpt/GPT'; -import { CognitiveServices } from '../../../cognitive_services/CognitiveServices'; import { DocUtils } from '../../../documents/DocUtils'; -import { CollectionViewType, DocumentType } from '../../../documents/DocumentTypes'; +import { DocumentType } from '../../../documents/DocumentTypes'; import { Docs, DocumentOptions } from '../../../documents/Documents'; import { SnappingManager, freeformScrollMode } from '../../../util/SnappingManager'; import { Transform } from '../../../util/Transform'; import { UndoManager, undoBatch } from '../../../util/UndoManager'; import { ContextMenu } from '../../ContextMenu'; +import { MainView } from '../../MainView'; import { ObservableReactComponent } from '../../ObservableReactComponent'; import { MarqueeViewBounds } from '../../PinFuncs'; import { PreviewCursor } from '../../PreviewCursor'; @@ -30,15 +26,10 @@ import { DocumentView } from '../../nodes/DocumentView'; import { OpenWhere } from '../../nodes/OpenWhere'; import { pasteImageBitmap } from '../../nodes/WebBoxRenderer'; import { FormattedTextBox } from '../../nodes/formattedText/FormattedTextBox'; -import { CollectionCardView } from '../CollectionCardDeckView'; import { SubCollectionViewProps } from '../CollectionSubView'; -import { CollectionFreeFormView } from './CollectionFreeFormView'; -import { ImageLabelHandler } from './ImageLabelHandler'; +import { ImageLabelBoxData } from './ImageLabelBox'; import { MarqueeOptionsMenu } from './MarqueeOptionsMenu'; import './MarqueeView.scss'; -import { MainView } from '../../MainView'; -import { ImageLabelBox, ImageLabelBoxData } from './ImageLabelBox'; -import { SearchBox } from '../../search/SearchBox'; interface MarqueeViewProps { getContainerTransform: () => Transform; @@ -68,7 +59,7 @@ export class MarqueeView extends ObservableReactComponent { - const selected = this.marqueeSelect(false); - if (e instanceof KeyboardEvent ? e.key === 'i' : true) { - const inks = selected.filter(s => s.type === DocumentType.INK); - const setDocs = selected.filter(s => s.type === DocumentType.RTF && s.color); - const sets = setDocs.map(sd => Cast(sd.data, RichTextField)?.Text as string); - const colors = setDocs.map(sd => FieldValue(sd.color) as string); - const wordToColor = new Map(); - sets.forEach((st: string, i: number) => st.split(',').forEach(word => wordToColor.set(word, colors[i]))); - const strokes: InkData[] = []; - inks.filter(i => Cast(i.data, InkField)).forEach(i => { - const d = Cast(i.data, InkField, null); - const left = Math.min(...(d?.inkData.map(pd => pd.X) ?? [0])); - const top = Math.min(...(d?.inkData.map(pd => pd.Y) ?? [0])); - strokes.push(d.inkData.map(pd => ({ X: pd.X + NumCast(i.x) - left, Y: pd.Y + NumCast(i.y) - top }))); - }); - CognitiveServices.Inking.Appliers.InterpretStrokes(strokes).then(results => { - // const wordResults = results.filter((r: any) => r.category === "inkWord"); - // for (const word of wordResults) { - // const indices: number[] = word.strokeIds; - // indices.forEach(i => { - // if (wordToColor.has(word.recognizedText.toLowerCase())) { - // inks[i].color = wordToColor.get(word.recognizedText.toLowerCase()); - // } - // else { - // for (const alt of word.alternates) { - // if (wordToColor.has(alt.recognizedString.toLowerCase())) { - // inks[i].color = wordToColor.get(alt.recognizedString.toLowerCase()); - // break; - // } - // } - // } - // }) - // } - // const wordResults = results.filter((r: any) => r.category === "inkWord"); - // for (const word of wordResults) { - // const indices: number[] = word.strokeIds; - // indices.forEach(i => { - // const otherInks: Doc[] = []; - // indices.forEach(i2 => i2 !== i && otherInks.push(inks[i2])); - // inks[i].relatedInks = new List(otherInks); - // const uniqueColors: string[] = []; - // Array.from(wordToColor.values()).forEach(c => uniqueColors.indexOf(c) === -1 && uniqueColors.push(c)); - // inks[i].alternativeColors = new List(uniqueColors); - // if (wordToColor.has(word.recognizedText.toLowerCase())) { - // inks[i].color = wordToColor.get(word.recognizedText.toLowerCase()); - // } - // else if (word.alternates) { - // for (const alt of word.alternates) { - // if (wordToColor.has(alt.recognizedString.toLowerCase())) { - // inks[i].color = wordToColor.get(alt.recognizedString.toLowerCase()); - // break; - // } - // } - // } - // }); - // } - const lines = results.filter((r: any) => r.category === 'line'); - const text = lines.map((l: any) => l.recognizedText).join('\r\n'); - this._props.addDocument?.(Docs.Create.TextDocument(text, { _width: this.Bounds.width, _height: this.Bounds.height, x: this.Bounds.left + this.Bounds.width, y: this.Bounds.top, title: text })); - }); - } - }); - @undoBatch summary = action(() => { const selected = this.marqueeSelect(false).map(d => { @@ -704,8 +630,8 @@ export class MarqueeView extends ObservableReactComponent {' '} @@ -714,7 +640,7 @@ export class MarqueeView extends ObservableReactComponent s + pt[0] + ',' + pt[1] + ' ', '')} fill="none" - stroke={lightOrDark(this._props.Document?.backgroundColor ?? 'white')} + stroke={lightOrDark((this._props.Document?.backgroundColor as string) ?? 'white')} strokeWidth="1" strokeDasharray="3" /> @@ -753,7 +679,6 @@ export class MarqueeView extends ObservableReactComponent { -- cgit v1.2.3-70-g09d2