fixed error handling of images too big to load. cleaned up facecollectionbox. changed metadata field naming to match conventions.

2 files changed, 46 insertions, 139 deletions

diff --git a/src/client/views/collections/collectionFreeForm/FaceCollectionBox.tsx b/src/client/views/collections/collectionFreeForm/FaceCollectionBox.tsx
index 50b91e8fe..d5a2809dc 100644
--- a/src/client/views/collections/collectionFreeForm/FaceCollectionBox.tsx
+++ b/src/client/views/collections/collectionFreeForm/FaceCollectionBox.tsx
@@ -11,17 +11,20 @@ import { Doc, DocListCast } from '../../../../fields/Doc';
 import { DocData } from '../../../../fields/DocSymbols';
 import { Id } from '../../../../fields/FieldSymbols';
 import { List } from '../../../../fields/List';
-import { ImageCast, StrCast } from '../../../../fields/Types';
+import { listSpec } from '../../../../fields/Schema';
+import { Cast, ImageCast, StrCast } from '../../../../fields/Types';
 import { DocumentType } from '../../../documents/DocumentTypes';
 import { Docs } from '../../../documents/Documents';
 import { DragManager } from '../../../util/DragManager';
 import { SnappingManager } from '../../../util/SnappingManager';
+import { undoable } from '../../../util/UndoManager';
 import { ViewBoxBaseComponent } from '../../DocComponent';
 import { ObservableReactComponent } from '../../ObservableReactComponent';
 import { DocumentView } from '../../nodes/DocumentView';
 import { FieldView, FieldViewProps } from '../../nodes/FieldView';
 import './FaceCollectionBox.scss';
 import { MarqueeOptionsMenu } from './MarqueeOptionsMenu';
+import { FaceRecognitionHandler } from '../../search/FaceRecognitionHandler';
 
 interface FaceDocumentProps {
     faceDoc: Doc;
@@ -32,17 +35,15 @@ interface FaceDocumentProps {
  */
 @observer
 export class FaceDocumentItem extends ObservableReactComponent<FaceDocumentProps> {
-    private ref: React.RefObject<HTMLDivElement>;
-    @observable _displayImages: boolean = true;
     private _dropDisposer?: DragManager.DragDropDisposer;
-    private _inputRef = React.createRef<HTMLInputElement>();
 
     constructor(props: FaceDocumentProps) {
         super(props);
         makeObservable(this);
-        this.ref = React.createRef();
     }
 
+    @observable _displayImages: boolean = true;
+
     protected createDropTarget = (ele: HTMLDivElement) => {
         this._dropDisposer?.();
         ele && (this._dropDisposer = DragManager.MakeDropTarget(ele, this.onInternalDrop.bind(this), this._props.faceDoc));
@@ -54,20 +55,20 @@ export class FaceDocumentItem extends ObservableReactComponent<FaceDocumentProps
             const filteredDocs = docDragData.droppedDocuments.filter(doc => doc.type === DocumentType.IMG);
             filteredDocs.forEach(doc => {
                 // If the current Face Document has no items, and the doc has more than one face descriptor, don't let the user add the document first.
-                if ((this._props.faceDoc[DocData].faceDescriptors as List<List<number>>).length === 0 && (doc[DocData].faces as List<List<number>>).length > 1) {
+                if ((this._props.faceDoc[DocData].face_descriptors as List<List<number>>).length === 0 && (doc[DocData][FaceRecognitionHandler.FacesField(doc)] as List<List<number>>).length > 1) {
                     alert('Cannot add a document with multiple faces as the first item!');
                 } else {
                     // Loop through the documents' face descriptors.
                     // Choose the face with the smallest distance to add.
-                    const float32Array = (this._props.faceDoc[DocData].faceDescriptors as List<List<number>>).map(faceDescriptor => new Float32Array(Array.from(faceDescriptor)));
-                    const labeledFaceDescriptor = new faceapi.LabeledFaceDescriptors(StrCast(this._props.faceDoc[DocData].label), float32Array);
+                    const float32Array = (this._props.faceDoc[DocData].face_descriptors as List<List<number>>).map(faceDescriptor => new Float32Array(Array.from(faceDescriptor)));
+                    const labeledFaceDescriptor = new faceapi.LabeledFaceDescriptors(StrCast(this._props.faceDoc[DocData].face_label), float32Array);
                     const faceDescriptors: faceapi.LabeledFaceDescriptors[] = [labeledFaceDescriptor];
 
                     const faceMatcher = new FaceMatcher(faceDescriptors, 1);
                     let cur_lowest_distance = 1;
                     let cur_matching_face = new List<number>();
 
-                    (doc[DocData].faces as List<List<number>>).forEach(face => {
+                    (doc[DocData][FaceRecognitionHandler.FacesField(doc)] as List<List<number>>).forEach(face => {
                         // If the face has the current lowest distance, mark it as such
                         // Once that lowest distance is found, add the face descriptor to the faceDoc, and add the associated doc
                         const convered_32_array: Float32Array = new Float32Array(Array.from(face));
@@ -79,16 +80,15 @@ export class FaceDocumentItem extends ObservableReactComponent<FaceDocumentProps
                         }
                     });
 
-                    if (doc[DocData][`FACE DESCRIPTOR - ${this._props.faceDoc[DocData].label}`]) {
-                        doc[DocData][`FACE DESCRIPTOR - ${this._props.faceDoc[DocData].label}`] = new List<List<number>>([...(doc[DocData][`FACE DESCRIPTOR - ${this._props.faceDoc[DocData].label}`] as List<List<number>>), cur_matching_face]);
+                    const faceFieldKey = FaceRecognitionHandler.FaceField(doc, this._props.faceDoc);
+                    if (doc[DocData][faceFieldKey]) {
+                        Cast(doc[DocData][faceFieldKey], listSpec('number'), null).push(cur_matching_face as unknown as number); // items are lists of numbers, not numbers, but type system can't handle that
                     } else {
-                        doc[DocData][`FACE DESCRIPTOR - ${this._props.faceDoc[DocData].label}`] = new List<List<number>>([cur_matching_face]);
+                        doc[DocData][faceFieldKey] = new List<List<number>>([cur_matching_face]);
                     }
 
-                    this._props.faceDoc[DocData].associatedDocs = new List<Doc>([...DocListCast(this._props.faceDoc[DocData].associatedDocs), doc]);
-                    this._props.faceDoc[DocData].faceDescriptors = new List<List<number>>([...(this._props.faceDoc[DocData].faceDescriptors as List<List<number>>), cur_matching_face]);
-
-                    //const match = faceMatcher.findBestMatch(cur_descriptor);
+                    Doc.AddDocToList(this._props.faceDoc[DocData], 'face_docList', doc);
+                    Cast(this._props.faceDoc[DocData].face_descriptors, listSpec('number'), null).push(cur_matching_face as unknown as number); // items are lists of numbers, not numbers, but type system can't handle that
                 }
             });
             return false;
@@ -107,12 +107,11 @@ export class FaceDocumentItem extends ObservableReactComponent<FaceDocumentProps
     /**
      * Deletes a Face Document.
      */
-    @action
-    deleteFaceDocument = () => {
+    deleteFaceDocument = undoable(() => {
         if (Doc.ActiveDashboard) {
-            Doc.ActiveDashboard[DocData].faceDocuments = new List<Doc>(DocListCast(Doc.ActiveDashboard[DocData].faceDocuments).filter(doc => doc !== this._props.faceDoc));
+            Doc.RemoveDocFromList(Doc.ActiveDashboard[DocData], 'faceDocuments', this._props.faceDoc);
         }
-    };
+    }, 'remove face');
 
     /**
      * Deletes a document from a Face Document's associated docs list.
@@ -120,60 +119,44 @@ export class FaceDocumentItem extends ObservableReactComponent<FaceDocumentProps
      */
     @action
     deleteAssociatedDoc = (doc: Doc) => {
-        this._props.faceDoc[DocData].faceDescriptors = new List<List<number>>(
-            (this._props.faceDoc[DocData].faceDescriptors as List<List<number>>).filter(fd => !(doc[DocData][`FACE DESCRIPTOR - ${this._props.faceDoc[DocData].label}`] as List<List<number>>).includes(fd))
+        this._props.faceDoc[DocData].face_descriptors = new List<List<number>>(
+            (this._props.faceDoc[DocData].face_descriptors as List<List<number>>).filter(fd => !(doc[DocData][FaceRecognitionHandler.FaceField(doc, this._props.faceDoc)] as List<List<number>>).includes(fd))
         );
-        doc[DocData][`FACE DESCRIPTOR - ${this._props.faceDoc[DocData].label}`] = new List<List<number>>();
-        this._props.faceDoc[DocData].associatedDocs = new List<Doc>(DocListCast(this._props.faceDoc[DocData].associatedDocs).filter(associatedDoc => associatedDoc !== doc));
+        doc[DocData][FaceRecognitionHandler.FaceField(doc, this._props.faceDoc)] = new List<List<number>>();
+        Doc.RemoveDocFromList(this._props.faceDoc[DocData], 'face_docList', doc);
     };
 
     render() {
         return (
             <div className="face-document-item" ref={ele => this.createDropTarget(ele!)}>
                 <div className="face-collection-buttons">
-                    <IconButton tooltip={'Delete Face From Collection'} onPointerDown={this.deleteFaceDocument} icon={'x'} style={{ width: '4px' }} size={Size.XSMALL} />
+                    <IconButton tooltip="Delete Face From Collection" onPointerDown={this.deleteFaceDocument} icon={'x'} style={{ width: '4px' }} size={Size.XSMALL} />
                 </div>
                 <div className="face-document-top">
-                    <h1>{StrCast(this._props.faceDoc[DocData].label)}</h1>
+                    <h1>{StrCast(this._props.faceDoc[DocData].face_label)}</h1>
                 </div>
                 <IconButton
-                    tooltip={'See image information'}
+                    tooltip="See image information"
                     onPointerDown={() => this.onDisplayClick()}
-                    icon={this._displayImages ? <FontAwesomeIcon icon="caret-up" /> : <FontAwesomeIcon icon="caret-down" />}
+                    icon={<FontAwesomeIcon icon={this._displayImages ? 'caret-up' : 'caret-down'} />}
                     color={MarqueeOptionsMenu.Instance.userColor}
                     style={{ width: '19px' }}
                 />
                 {this._displayImages ? (
                     <div className="face-document-image-container">
-                        {DocListCast(this._props.faceDoc[DocData].associatedDocs).map(doc => {
+                        {DocListCast(this._props.faceDoc[DocData].face_docList).map(doc => {
                             const [name, type] = ImageCast(doc[Doc.LayoutFieldKey(doc)]).url.href.split('.');
                             return (
                                 <div className="image-wrapper" key={Utils.GenerateGuid()}>
-                                    <img
-                                        onClick={async () => {
-                                            await DocumentView.showDocument(doc, { willZoomCentered: true });
-                                        }}
-                                        style={{ maxWidth: '60px', margin: '10px' }}
-                                        src={`${name}_o.${type}`}
-                                    />
+                                    <img onClick={() => DocumentView.showDocument(doc, { willZoomCentered: true })} style={{ maxWidth: '60px', margin: '10px' }} src={`${name}_o.${type}`} />
                                     <div className="remove-item">
-                                        <IconButton
-                                            tooltip={'Remove Doc From Face Collection'}
-                                            onPointerDown={() => {
-                                                this.deleteAssociatedDoc(doc);
-                                            }}
-                                            icon={'x'}
-                                            style={{ width: '4px' }}
-                                            size={Size.XSMALL}
-                                        />
+                                        <IconButton tooltip={'Remove Doc From Face Collection'} onPointerDown={() => this.deleteAssociatedDoc(doc)} icon={'x'} style={{ width: '4px' }} size={Size.XSMALL} />
                                     </div>
                                 </div>
                             );
                         })}
                     </div>
-                ) : (
-                    <div></div>
-                )}
+                ) : null}
             </div>
         );
     }
@@ -190,9 +173,8 @@ export class FaceCollectionBox extends ViewBoxBaseComponent<FieldViewProps>() {
     @computed get currentDocs() {
         if (Doc.ActiveDashboard) {
             return DocListCast(Doc.ActiveDashboard[DocData].faceDocuments);
-        } else {
-            return [];
         }
+        return [];
     }
 
     constructor(props: FieldViewProps) {
@@ -204,9 +186,9 @@ export class FaceCollectionBox extends ViewBoxBaseComponent<FieldViewProps>() {
     render() {
         return (
             <div className="searchBox-container" style={{ pointerEvents: 'all', color: SnappingManager.userColor, background: SnappingManager.userBackgroundColor }}>
-                {this.currentDocs.map(doc => {
-                    return <FaceDocumentItem key={doc[Id]} faceDoc={doc} />;
-                })}
+                {this.currentDocs.map(doc => (
+                    <FaceDocumentItem key={doc[Id]} faceDoc={doc} />
+                ))}
             </div>
         );
     }
diff --git a/src/client/views/collections/collectionFreeForm/MarqueeView.tsx b/src/client/views/collections/collectionFreeForm/MarqueeView.tsx
index d7a41df64..6fee076ee 100644
--- a/src/client/views/collections/collectionFreeForm/MarqueeView.tsx
+++ b/src/client/views/collections/collectionFreeForm/MarqueeView.tsx
@@ -1,28 +1,24 @@
-/* eslint-disable jsx-a11y/no-static-element-interactions */
-import similarity from 'compute-cosine-similarity';
 import { action, computed, makeObservable, observable } from 'mobx';
 import { observer } from 'mobx-react';
 import * as React from 'react';
 import { ClientUtils, lightOrDark, returnFalse } from '../../../../ClientUtils';
-import { intersectRect, numberRange } from '../../../../Utils';
-import { Doc, DocListCast, NumListCast, Opt } from '../../../../fields/Doc';
+import { intersectRect } from '../../../../Utils';
+import { Doc, DocListCast, Opt } from '../../../../fields/Doc';
 import { AclAdmin, AclAugment, AclEdit, DocData } from '../../../../fields/DocSymbols';
 import { Id } from '../../../../fields/FieldSymbols';
-import { InkData, InkField, InkTool } from '../../../../fields/InkField';
+import { InkTool } from '../../../../fields/InkField';
 import { List } from '../../../../fields/List';
-import { RichTextField } from '../../../../fields/RichTextField';
-import { Cast, FieldValue, ImageCast, NumCast, StrCast } from '../../../../fields/Types';
+import { Cast, NumCast, StrCast } from '../../../../fields/Types';
 import { ImageField } from '../../../../fields/URLField';
 import { GetEffectiveAcl } from '../../../../fields/util';
-import { gptGetEmbedding, gptImageLabel } from '../../../apis/gpt/GPT';
-import { CognitiveServices } from '../../../cognitive_services/CognitiveServices';
 import { DocUtils } from '../../../documents/DocUtils';
-import { CollectionViewType, DocumentType } from '../../../documents/DocumentTypes';
+import { DocumentType } from '../../../documents/DocumentTypes';
 import { Docs, DocumentOptions } from '../../../documents/Documents';
 import { SnappingManager, freeformScrollMode } from '../../../util/SnappingManager';
 import { Transform } from '../../../util/Transform';
 import { UndoManager, undoBatch } from '../../../util/UndoManager';
 import { ContextMenu } from '../../ContextMenu';
+import { MainView } from '../../MainView';
 import { ObservableReactComponent } from '../../ObservableReactComponent';
 import { MarqueeViewBounds } from '../../PinFuncs';
 import { PreviewCursor } from '../../PreviewCursor';
@@ -30,15 +26,10 @@ import { DocumentView } from '../../nodes/DocumentView';
 import { OpenWhere } from '../../nodes/OpenWhere';
 import { pasteImageBitmap } from '../../nodes/WebBoxRenderer';
 import { FormattedTextBox } from '../../nodes/formattedText/FormattedTextBox';
-import { CollectionCardView } from '../CollectionCardDeckView';
 import { SubCollectionViewProps } from '../CollectionSubView';
-import { CollectionFreeFormView } from './CollectionFreeFormView';
-import { ImageLabelHandler } from './ImageLabelHandler';
+import { ImageLabelBoxData } from './ImageLabelBox';
 import { MarqueeOptionsMenu } from './MarqueeOptionsMenu';
 import './MarqueeView.scss';
-import { MainView } from '../../MainView';
-import { ImageLabelBox, ImageLabelBoxData } from './ImageLabelBox';
-import { SearchBox } from '../../search/SearchBox';
 
 interface MarqueeViewProps {
     getContainerTransform: () => Transform;
@@ -68,7 +59,7 @@ export class MarqueeView extends ObservableReactComponent<SubCollectionViewProps
 
     static Instance: MarqueeView;
 
-    constructor(props: any) {
+    constructor(props: SubCollectionViewProps & MarqueeViewProps) {
         super(props);
         makeObservable(this);
         MarqueeView.Instance = this;
@@ -496,71 +487,6 @@ export class MarqueeView extends ObservableReactComponent<SubCollectionViewProps
     });
 
     @undoBatch
-    syntaxHighlight = action((e: KeyboardEvent | React.PointerEvent | undefined) => {
-        const selected = this.marqueeSelect(false);
-        if (e instanceof KeyboardEvent ? e.key === 'i' : true) {
-            const inks = selected.filter(s => s.type === DocumentType.INK);
-            const setDocs = selected.filter(s => s.type === DocumentType.RTF && s.color);
-            const sets = setDocs.map(sd => Cast(sd.data, RichTextField)?.Text as string);
-            const colors = setDocs.map(sd => FieldValue(sd.color) as string);
-            const wordToColor = new Map<string, string>();
-            sets.forEach((st: string, i: number) => st.split(',').forEach(word => wordToColor.set(word, colors[i])));
-            const strokes: InkData[] = [];
-            inks.filter(i => Cast(i.data, InkField)).forEach(i => {
-                const d = Cast(i.data, InkField, null);
-                const left = Math.min(...(d?.inkData.map(pd => pd.X) ?? [0]));
-                const top = Math.min(...(d?.inkData.map(pd => pd.Y) ?? [0]));
-                strokes.push(d.inkData.map(pd => ({ X: pd.X + NumCast(i.x) - left, Y: pd.Y + NumCast(i.y) - top })));
-            });
-            CognitiveServices.Inking.Appliers.InterpretStrokes(strokes).then(results => {
-                // const wordResults = results.filter((r: any) => r.category === "inkWord");
-                // for (const word of wordResults) {
-                //     const indices: number[] = word.strokeIds;
-                //     indices.forEach(i => {
-                //         if (wordToColor.has(word.recognizedText.toLowerCase())) {
-                //             inks[i].color = wordToColor.get(word.recognizedText.toLowerCase());
-                //         }
-                //         else {
-                //             for (const alt of word.alternates) {
-                //                 if (wordToColor.has(alt.recognizedString.toLowerCase())) {
-                //                     inks[i].color = wordToColor.get(alt.recognizedString.toLowerCase());
-                //                     break;
-                //                 }
-                //             }
-                //         }
-                //     })
-                // }
-                // const wordResults = results.filter((r: any) => r.category === "inkWord");
-                // for (const word of wordResults) {
-                //     const indices: number[] = word.strokeIds;
-                //     indices.forEach(i => {
-                //         const otherInks: Doc[] = [];
-                //         indices.forEach(i2 => i2 !== i && otherInks.push(inks[i2]));
-                //         inks[i].relatedInks = new List<Doc>(otherInks);
-                //         const uniqueColors: string[] = [];
-                //         Array.from(wordToColor.values()).forEach(c => uniqueColors.indexOf(c) === -1 && uniqueColors.push(c));
-                //         inks[i].alternativeColors = new List<string>(uniqueColors);
-                //         if (wordToColor.has(word.recognizedText.toLowerCase())) {
-                //             inks[i].color = wordToColor.get(word.recognizedText.toLowerCase());
-                //         }
-                //         else if (word.alternates) {
-                //             for (const alt of word.alternates) {
-                //                 if (wordToColor.has(alt.recognizedString.toLowerCase())) {
-                //                     inks[i].color = wordToColor.get(alt.recognizedString.toLowerCase());
-                //                     break;
-                //                 }
-                //             }
-                //         }
-                //     });
-                // }
-                const lines = results.filter((r: any) => r.category === 'line');
-                const text = lines.map((l: any) => l.recognizedText).join('\r\n');
-                this._props.addDocument?.(Docs.Create.TextDocument(text, { _width: this.Bounds.width, _height: this.Bounds.height, x: this.Bounds.left + this.Bounds.width, y: this.Bounds.top, title: text }));
-            });
-        }
-    });
-
-    @undoBatch
     summary = action(() => {
         const selected = this.marqueeSelect(false).map(d => {
             this._props.removeDocument?.(d);
@@ -704,8 +630,8 @@ export class MarqueeView extends ObservableReactComponent<SubCollectionViewProps
                     transform: `translate(${p[0]}px, ${p[1]}px)`,
                     width: Math.abs(v[0]),
                     height: Math.abs(v[1]),
-                    color: lightOrDark(this._props.Document?.backgroundColor ?? 'white'),
-                    borderColor: lightOrDark(this._props.Document?.backgroundColor ?? 'white'),
+                    color: lightOrDark((this._props.Document?.backgroundColor as string) ?? 'white'),
+                    borderColor: lightOrDark((this._props.Document?.backgroundColor as string) ?? 'white'),
                     zIndex: 2000,
                 }}>
                 {' '}
@@ -714,7 +640,7 @@ export class MarqueeView extends ObservableReactComponent<SubCollectionViewProps
                         <polyline //
                             points={this._lassoPts.reduce((s, pt) => s + pt[0] + ',' + pt[1] + ' ', '')}
                             fill="none"
-                            stroke={lightOrDark(this._props.Document?.backgroundColor ?? 'white')}
+                            stroke={lightOrDark((this._props.Document?.backgroundColor as string) ?? 'white')}
                             strokeWidth="1"
                             strokeDasharray="3"
                         />
@@ -753,7 +679,6 @@ export class MarqueeView extends ObservableReactComponent<SubCollectionViewProps
     };
     render() {
         return (
-            // eslint-disable-next-line jsx-a11y/click-events-have-key-events
             <div
                 className="marqueeView"
                 ref={r => {