fixed pivotView to provide a white background to docs with no background so that images will show up, and so text doesn't blend in to column. changed image labeling to show a pivot view and assigne metadata instead of creating collections.

6 files changed, 83 insertions, 111 deletions

diff --git a/src/Utils.ts b/src/Utils.ts
index 44f1d55c2..23ae38bdb 100644
--- a/src/Utils.ts
+++ b/src/Utils.ts
@@ -211,6 +211,11 @@ export type Without<T, K extends keyof T> = Pick<T, Exclude<keyof T, K>>;
 
 export type Predicate<K, V> = (entry: [K, V]) => boolean;
 
+/**
+ * creates a list of numbers ordered from 0 to 'num'
+ * @param num range of numbers
+ * @returns list of values from 0 to num -1
+ */
 export function numberRange(num: number) {
     return num > 0 && num < 1000 ? Array.from(Array(num)).map((v, i) => i) : [];
 }
diff --git a/src/client/apis/gpt/GPT.ts b/src/client/apis/gpt/GPT.ts
index 455352068..7af71a6a2 100644
--- a/src/client/apis/gpt/GPT.ts
+++ b/src/client/apis/gpt/GPT.ts
@@ -119,7 +119,7 @@ const gptGetEmbedding = async (src: string): Promise<number[]> => {
         });
 
         // Assume the embeddingResponse structure is correct; adjust based on actual API response
-        const embedding = embeddingResponse.data[0].embedding;
+        const { embedding } = embeddingResponse.data[0];
         return embedding;
     } catch (err) {
         console.log(err);
@@ -155,9 +155,8 @@ const gptImageLabel = async (src: string): Promise<string> => {
         });
         if (response.choices[0].message.content) {
             return response.choices[0].message.content;
-        } else {
-            return 'Missing labels';
         }
+        return 'Missing labels';
     } catch (err) {
         console.log(err);
         return 'Error connecting with API';
diff --git a/src/client/views/collections/CollectionCardDeckView.tsx b/src/client/views/collections/CollectionCardDeckView.tsx
index 6036a2ead..de46180e6 100644
--- a/src/client/views/collections/CollectionCardDeckView.tsx
+++ b/src/client/views/collections/CollectionCardDeckView.tsx
@@ -161,7 +161,7 @@ export class CollectionCardView extends CollectionSubView() {
     panelHeight = (layout: Doc) => () => (this.panelWidth() * NumCast(layout._height)) / NumCast(layout._width);
     onChildDoubleClick = () => ScriptCast(this.layoutDoc.onChildDoubleClick);
     isContentActive = () => this._props.isSelected() || this._props.isContentActive() || this._props.isAnyChildContentActive();
-    isChildContentActive = () => (this.isContentActive() ? true : false);
+    isChildContentActive = () => !!this.isContentActive();
 
     /**
      * Returns the degree to rotate a card dependind on the amount of cards in their row and their index in said row
@@ -173,10 +173,10 @@ export class CollectionCardView extends CollectionSubView() {
         const possRotate = -30 + index * (30 / ((amCards - (amCards % 2)) / 2));
         const stepMag = Math.abs(-30 + (amCards / 2 - 1) * (30 / ((amCards - (amCards % 2)) / 2)));
 
-        if (amCards % 2 == 0 && possRotate == 0) {
+        if (amCards % 2 === 0 && possRotate === 0) {
             return possRotate + Math.abs(-30 + (index - 1) * (30 / (amCards / 2)));
         }
-        if (amCards % 2 == 0 && index > (amCards + 1) / 2) {
+        if (amCards % 2 === 0 && index > (amCards + 1) / 2) {
             return possRotate + stepMag;
         }
 
@@ -194,10 +194,10 @@ export class CollectionCardView extends CollectionSubView() {
         if (realIndex > this._maxRowCount - 1) {
             rowOffset = 400 * ((realIndex - (realIndex % this._maxRowCount)) / this._maxRowCount);
         }
-        if (evenOdd == 1 || index < apex - 1) {
+        if (evenOdd === 1 || index < apex - 1) {
             return Math.abs(stepMag * (apex - index)) - rowOffset;
         }
-        if (index == apex || index == apex - 1) {
+        if (index === apex || index === apex - 1) {
             return 0 - rowOffset;
         }
 
@@ -259,27 +259,26 @@ export class CollectionCardView extends CollectionSubView() {
         return docs;
     };
 
-    displayDoc = (doc: Doc, screenToLocalTransform: () => Transform) => {
-        return (
-            <DocumentView
-                {...this._props}
-                ref={action((r: DocumentView) => r?.ContentDiv && this._docRefs.set(doc, r))}
-                Document={doc}
-                NativeWidth={returnZero}
-                NativeHeight={returnZero}
-                fitWidth={returnFalse}
-                onDoubleClickScript={this.onChildDoubleClick}
-                renderDepth={this._props.renderDepth + 1}
-                LayoutTemplate={this._props.childLayoutTemplate}
-                LayoutTemplateString={this._props.childLayoutString}
-                ScreenToLocalTransform={screenToLocalTransform} //makes sure the box wrapper thing is in the right spot
-                isContentActive={this.isChildContentActive}
-                isDocumentActive={this._props.childDocumentsActive?.() || this.Document._childDocumentsActive ? this._props.isDocumentActive : this.isContentActive}
-                PanelWidth={this.panelWidth}
-                PanelHeight={this.panelHeight(doc)}
-            />
-        );
-    };
+    displayDoc = (doc: Doc, screenToLocalTransform: () => Transform) => (
+        <DocumentView
+            // eslint-disable-next-line react/jsx-props-no-spreading
+            {...this._props}
+            ref={action((r: DocumentView) => r?.ContentDiv && this._docRefs.set(doc, r))}
+            Document={doc}
+            NativeWidth={returnZero}
+            NativeHeight={returnZero}
+            fitWidth={returnFalse}
+            onDoubleClickScript={this.onChildDoubleClick}
+            renderDepth={this._props.renderDepth + 1}
+            LayoutTemplate={this._props.childLayoutTemplate}
+            LayoutTemplateString={this._props.childLayoutString}
+            ScreenToLocalTransform={screenToLocalTransform} // makes sure the box wrapper thing is in the right spot
+            isContentActive={this.isChildContentActive}
+            isDocumentActive={this._props.childDocumentsActive?.() || this.Document._childDocumentsActive ? this._props.isDocumentActive : this.isContentActive}
+            PanelWidth={this.panelWidth}
+            PanelHeight={this.panelHeight(doc)}
+        />
+    );
 
     /**
      * Determines how many cards are in the row of a card at a specific index
@@ -296,7 +295,7 @@ export class CollectionCardView extends CollectionSubView() {
         if (index < totalCards - (totalCards % 10)) {
             return this._maxRowCount;
         }
-        //(3)
+        // (3)
         return totalCards % 10;
     };
     /**
@@ -335,7 +334,9 @@ export class CollectionCardView extends CollectionSubView() {
      * @param buttonID
      * @param doc
      */
-    toggleButton = undoable((buttonID: number, doc: Doc) => this.cardSort_customField && (doc[this.cardSort_customField] = buttonID), 'toggle custom button');
+    toggleButton = undoable((buttonID: number, doc: Doc) => {
+        this.cardSort_customField && (doc[this.cardSort_customField] = buttonID);
+    }, 'toggle custom button');
 
     /**
      * A list of the text content of all the child docs. RTF documents will have just their text and pdf documents will have the first 50 words.
@@ -346,8 +347,7 @@ export class CollectionCardView extends CollectionSubView() {
     childPairStringList = () => {
         const docToText = (doc: Doc) => {
             switch (doc.type) {
-                case DocumentType.PDF:  const words = StrCast(doc.text).split(/\s+/);
-                                        return words.slice(0, 50).join(' '); // first 50 words of pdf text
+                case DocumentType.PDF:  return StrCast(doc.text).split(/\s+/).slice(0, 50).join(' '); // first 50 words of pdf text
                 case DocumentType.IMG:  return this.getImageDesc(doc);
                 case DocumentType.RTF:  return StrCast(RTFCast(doc.text).Text);
                 default:                return StrCast(doc.title);
@@ -368,7 +368,7 @@ export class CollectionCardView extends CollectionSubView() {
      */
     getImageDesc = async (image: Doc) => {
         if (StrCast(image.description)) return StrCast(image.description); // Return existing description
-        const href = (image.data as URLField).url.href;
+        const { href } = (image.data as URLField).url;
         const hrefParts = href.split('.');
         const hrefComplete = `${hrefParts[0]}_o.${hrefParts[1]}`;
         try {
@@ -422,12 +422,13 @@ export class CollectionCardView extends CollectionSubView() {
      */
     renderButtons = (doc: Doc, cardSort: cardSortings) => {
         if (cardSort !== cardSortings.Custom) return '';
-        const amButtons = Math.max(4, this.childDocs?.reduce((set, doc) => this.cardSort_customField && set.add(NumCast(doc[this.cardSort_customField])), new Set<number>()).size ?? 0);
+        const amButtons = Math.max(4, this.childDocs?.reduce((set, d) => this.cardSort_customField && set.add(NumCast(d[this.cardSort_customField])), new Set<number>()).size ?? 0);
         const activeButtonIndex = CollectionCardView.getButtonGroup(this.cardSort_customField, doc);
         const totalWidth = amButtons * 35 + amButtons * 2 * 5 + 6;
         return (
             <div className="card-button-container" style={{ width: `${totalWidth}px` }}>
                 {numberRange(amButtons).map(i => (
+                    // eslint-disable-next-line jsx-a11y/control-has-associated-label
                     <button
                         key={i}
                         type="button"
diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormLayoutEngines.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormLayoutEngines.tsx
index a4496a417..de51cc73c 100644
--- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormLayoutEngines.tsx
+++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormLayoutEngines.tsx
@@ -245,7 +245,7 @@ export function computePivotLayout(poolData: Map<string, PoolData>, pivotDoc: Do
                 y: -y + (pivotAxisWidth - hgt) / 2,
                 width: wid,
                 height: hgt,
-                backgroundColor: StrCast(layoutDoc.backgroundColor),
+                backgroundColor: StrCast(layoutDoc.backgroundColor, 'white'),
                 pair: { layout: doc },
                 replica: val.replicas[i],
             });
diff --git a/src/client/views/collections/collectionFreeForm/ImageLabelHandler.tsx b/src/client/views/collections/collectionFreeForm/ImageLabelHandler.tsx
index 46bc3d946..7f27c6b5c 100644
--- a/src/client/views/collections/collectionFreeForm/ImageLabelHandler.tsx
+++ b/src/client/views/collections/collectionFreeForm/ImageLabelHandler.tsx
@@ -52,8 +52,8 @@ export class ImageLabelHandler extends ObservableReactComponent<{}> {
 
     @action
     removeLabel = (label: string) => {
-        label = label.toUpperCase();
-        this._labelGroups = this._labelGroups.filter(group => group !== label);
+        const labelUp = label.toUpperCase();
+        this._labelGroups = this._labelGroups.filter(group => group !== labelUp);
     };
 
     @action
diff --git a/src/client/views/collections/collectionFreeForm/MarqueeView.tsx b/src/client/views/collections/collectionFreeForm/MarqueeView.tsx
index ff8c2d318..dc15c83c5 100644
--- a/src/client/views/collections/collectionFreeForm/MarqueeView.tsx
+++ b/src/client/views/collections/collectionFreeForm/MarqueeView.tsx
@@ -1,22 +1,23 @@
 /* eslint-disable jsx-a11y/no-static-element-interactions */
+import similarity from 'compute-cosine-similarity';
 import { action, computed, makeObservable, observable } from 'mobx';
 import { observer } from 'mobx-react';
 import * as React from 'react';
 import { ClientUtils, lightOrDark, returnFalse } from '../../../../ClientUtils';
-import { intersectRect } from '../../../../Utils';
+import { intersectRect, numberRange } from '../../../../Utils';
 import { Doc, NumListCast, Opt } from '../../../../fields/Doc';
 import { AclAdmin, AclAugment, AclEdit, DocData } from '../../../../fields/DocSymbols';
 import { Id } from '../../../../fields/FieldSymbols';
 import { InkData, InkField, InkTool } from '../../../../fields/InkField';
 import { List } from '../../../../fields/List';
 import { RichTextField } from '../../../../fields/RichTextField';
-import { Cast, FieldValue, NumCast, StrCast } from '../../../../fields/Types';
-import { ImageField, URLField } from '../../../../fields/URLField';
+import { Cast, FieldValue, ImageCast, NumCast, StrCast } from '../../../../fields/Types';
+import { ImageField } from '../../../../fields/URLField';
 import { GetEffectiveAcl } from '../../../../fields/util';
 import { gptGetEmbedding, gptImageLabel } from '../../../apis/gpt/GPT';
 import { CognitiveServices } from '../../../cognitive_services/CognitiveServices';
 import { DocUtils } from '../../../documents/DocUtils';
-import { DocumentType } from '../../../documents/DocumentTypes';
+import { CollectionViewType, DocumentType } from '../../../documents/DocumentTypes';
 import { Docs, DocumentOptions } from '../../../documents/Documents';
 import { SnappingManager, freeformScrollMode } from '../../../util/SnappingManager';
 import { Transform } from '../../../util/Transform';
@@ -31,9 +32,11 @@ import { pasteImageBitmap } from '../../nodes/WebBoxRenderer';
 import { FormattedTextBox } from '../../nodes/formattedText/FormattedTextBox';
 import { CollectionCardView } from '../CollectionCardDeckView';
 import { SubCollectionViewProps } from '../CollectionSubView';
+import { CollectionFreeFormView } from './CollectionFreeFormView';
 import { ImageLabelHandler } from './ImageLabelHandler';
 import { MarqueeOptionsMenu } from './MarqueeOptionsMenu';
 import './MarqueeView.scss';
+
 interface MarqueeViewProps {
     getContainerTransform: () => Transform;
     getTransform: () => Transform;
@@ -431,38 +434,25 @@ export class MarqueeView extends ObservableReactComponent<SubCollectionViewProps
      */
     @undoBatch
     classifyImages = action(async (e: React.MouseEvent | undefined) => {
-        const selected = this.marqueeSelect(false, DocumentType.IMG);
-        this._selectedDocs = selected;
-
-        const imagePromises = selected.map(doc => {
-            const href = (doc['data'] as URLField).url.href;
-            const hrefParts = href.split('.');
-            const hrefComplete = `${hrefParts[0]}_o.${hrefParts[1]}`;
-            return CollectionCardView.imageUrlToBase64(hrefComplete).then(hrefBase64 =>
-                !hrefBase64
-                    ? undefined
-                    : gptImageLabel(hrefBase64).then(response => {
-                          const labels = response.split('\n');
-                          doc.image_labels = new List<string>(Array.from(labels!));
-                          return Promise.all(labels!.map(label => gptGetEmbedding(label))).then(embeddings => {
-                              return { doc, embeddings };
-                          });
-                      })
-            );
+        this._selectedDocs = this.marqueeSelect(false, DocumentType.IMG);
+
+        const imageInfos = this._selectedDocs.map(async doc => {
+            const [name, type] = ImageCast(doc[Doc.LayoutFieldKey(doc)]).url.href.split('.');
+            return CollectionCardView.imageUrlToBase64(`${name}_o.${type}`).then(hrefBase64 =>
+                !hrefBase64 ? undefined : 
+                    gptImageLabel(hrefBase64).then(labels => 
+                        Promise.all(labels.split('\n').map(label => gptGetEmbedding(label))).then(embeddings => 
+                            ({ doc, embeddings, labels }))) ); // prettier-ignore
         });
 
-        const docsAndEmbeddings = await Promise.all(imagePromises);
-        docsAndEmbeddings
-            .filter(d => d)
-            .map(d => d!)
-            .forEach(docAndEmbedding => {
-                if (Array.isArray(docAndEmbedding.embeddings)) {
-                    let doc = docAndEmbedding.doc;
-                    for (let i = 0; i < 3; i++) {
-                        doc[`label_embedding_${i + 1}`] = new List<number>(docAndEmbedding.embeddings[i]);
-                    }
-                }
-            });
+        (await Promise.all(imageInfos)).forEach(imageInfo => {
+            if (imageInfo && Array.isArray(imageInfo.embeddings)) {
+                imageInfo.doc[DocData].data_labels = imageInfo.labels;
+                numberRange(3).forEach(n => {
+                    imageInfo.doc[`data_labels_embedding_${n + 1}`] = new List<number>(imageInfo.embeddings[n]);
+                });
+            }
+        });
 
         if (e) {
             ImageLabelHandler.Instance.displayLabelHandler(e.pageX, e.pageY);
@@ -474,51 +464,28 @@ export class MarqueeView extends ObservableReactComponent<SubCollectionViewProps
      */
     @undoBatch
     groupImages = action(async () => {
-        const labelGroups: string[] = ImageLabelHandler.Instance._labelGroups;
-        const labelToCollection: Map<string, Doc> = new Map();
-        const labelToEmbedding: Map<string, number[]> = new Map();
-        var similarity = require('compute-cosine-similarity');
-
-        // Create new collections associated with each label and get the embeddings for the labels.
-        for (const label of labelGroups) {
-            const newCollection = this.getCollection([], undefined, false);
-            newCollection._freeform_panX = this.Bounds.left + this.Bounds.width / 2;
-            newCollection._freeform_panY = this.Bounds.top + this.Bounds.height / 2;
-            labelToCollection.set(label, newCollection);
-            this._props.addDocument?.(newCollection);
-            const labelEmbedding = await gptGetEmbedding(label);
-            if (Array.isArray(labelEmbedding)) {
-                labelToEmbedding.set(label, labelEmbedding);
-            }
-        }
+        const labelGroups = ImageLabelHandler.Instance._labelGroups;
+        const labelToEmbedding = new Map<string, number[]>();
+        // Create embeddings for the labels.
+        await Promise.all(labelGroups.map(async label => gptGetEmbedding(label).then(labelEmbedding => labelToEmbedding.set(label, labelEmbedding))));
 
         // For each image, loop through the labels, and calculate similarity. Associate it with the
         // most similar one.
         this._selectedDocs.forEach(doc => {
-            let mostSimilarLabel: string | undefined;
-            let maxSimilarity: number = 0;
-            const embeddingAsList1 = NumListCast(doc.label_embedding_1);
-            const embeddingAsList2 = NumListCast(doc.label_embedding_2);
-            const embeddingAsList3 = NumListCast(doc.label_embedding_3);
-
-            labelGroups.forEach(label => {
-                let curSimilarity1 = similarity(labelToEmbedding.get(label)!, Array.from(embeddingAsList1));
-                let curSimilarity2 = similarity(labelToEmbedding.get(label)!, Array.from(embeddingAsList2));
-                let curSimilarity3 = similarity(labelToEmbedding.get(label)!, Array.from(embeddingAsList3));
-                let maxCurSimilarity = Math.max(curSimilarity1, curSimilarity2, curSimilarity3);
-                if (maxCurSimilarity >= 0.3 && maxCurSimilarity > maxSimilarity) {
-                    mostSimilarLabel = label;
-                    maxSimilarity = maxCurSimilarity;
-                }
-
-                console.log('Doc with labels ' + doc.image_labels + 'has similarity score ' + maxCurSimilarity + ' to ' + mostSimilarLabel);
+            const embedLists = numberRange(3).map(n => Array.from(NumListCast(doc[`data_labels_embedding_${n + 1}`])));
+            const bestEmbedScore = (embedding: Opt<number[]>) => Math.max(...embedLists.map(l => (embedding && similarity(Array.from(embedding), l)) || 0));
+            const {label: mostSimilarLabelCollect} = 
+                labelGroups.map(label => ({ label, similarityScore: bestEmbedScore(labelToEmbedding.get(label)) }))
+                           .reduce((prev, cur) => cur.similarityScore < 0.3 || cur.similarityScore <= prev.similarityScore ? prev: cur,
+                                   { label: '', similarityScore: 0, }); // prettier-ignore
+
+            numberRange(3).forEach(n => {
+                doc[`data_labels_embedding_${n + 1}`] = undefined;
             });
-
-            if (mostSimilarLabel) {
-                Doc.AddDocToList(labelToCollection.get(mostSimilarLabel)!, undefined, doc);
-                this._props.removeDocument?.(doc);
-            }
+            doc[DocData].data_label = mostSimilarLabelCollect;
         });
+        this._props.Document._type_collection = CollectionViewType.Time;
+        this._props.Document.pivotField = 'data_label';
     });
 
     @undoBatch