fixed pivotView to provide a white background to docs with no background so that images will show up, and so text doesn't blend in to column. changed image labeling to show a pivot view and assigne metadata instead of creating collections.

3 files changed, 43 insertions, 76 deletions

diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormLayoutEngines.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormLayoutEngines.tsx
index a4496a417..de51cc73c 100644
--- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormLayoutEngines.tsx
+++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormLayoutEngines.tsx
@@ -245,7 +245,7 @@ export function computePivotLayout(poolData: Map<string, PoolData>, pivotDoc: Do
                 y: -y + (pivotAxisWidth - hgt) / 2,
                 width: wid,
                 height: hgt,
-                backgroundColor: StrCast(layoutDoc.backgroundColor),
+                backgroundColor: StrCast(layoutDoc.backgroundColor, 'white'),
                 pair: { layout: doc },
                 replica: val.replicas[i],
             });
diff --git a/src/client/views/collections/collectionFreeForm/ImageLabelHandler.tsx b/src/client/views/collections/collectionFreeForm/ImageLabelHandler.tsx
index 46bc3d946..7f27c6b5c 100644
--- a/src/client/views/collections/collectionFreeForm/ImageLabelHandler.tsx
+++ b/src/client/views/collections/collectionFreeForm/ImageLabelHandler.tsx
@@ -52,8 +52,8 @@ export class ImageLabelHandler extends ObservableReactComponent<{}> {
 
     @action
     removeLabel = (label: string) => {
-        label = label.toUpperCase();
-        this._labelGroups = this._labelGroups.filter(group => group !== label);
+        const labelUp = label.toUpperCase();
+        this._labelGroups = this._labelGroups.filter(group => group !== labelUp);
     };
 
     @action
diff --git a/src/client/views/collections/collectionFreeForm/MarqueeView.tsx b/src/client/views/collections/collectionFreeForm/MarqueeView.tsx
index ff8c2d318..dc15c83c5 100644
--- a/src/client/views/collections/collectionFreeForm/MarqueeView.tsx
+++ b/src/client/views/collections/collectionFreeForm/MarqueeView.tsx
@@ -1,22 +1,23 @@
 /* eslint-disable jsx-a11y/no-static-element-interactions */
+import similarity from 'compute-cosine-similarity';
 import { action, computed, makeObservable, observable } from 'mobx';
 import { observer } from 'mobx-react';
 import * as React from 'react';
 import { ClientUtils, lightOrDark, returnFalse } from '../../../../ClientUtils';
-import { intersectRect } from '../../../../Utils';
+import { intersectRect, numberRange } from '../../../../Utils';
 import { Doc, NumListCast, Opt } from '../../../../fields/Doc';
 import { AclAdmin, AclAugment, AclEdit, DocData } from '../../../../fields/DocSymbols';
 import { Id } from '../../../../fields/FieldSymbols';
 import { InkData, InkField, InkTool } from '../../../../fields/InkField';
 import { List } from '../../../../fields/List';
 import { RichTextField } from '../../../../fields/RichTextField';
-import { Cast, FieldValue, NumCast, StrCast } from '../../../../fields/Types';
-import { ImageField, URLField } from '../../../../fields/URLField';
+import { Cast, FieldValue, ImageCast, NumCast, StrCast } from '../../../../fields/Types';
+import { ImageField } from '../../../../fields/URLField';
 import { GetEffectiveAcl } from '../../../../fields/util';
 import { gptGetEmbedding, gptImageLabel } from '../../../apis/gpt/GPT';
 import { CognitiveServices } from '../../../cognitive_services/CognitiveServices';
 import { DocUtils } from '../../../documents/DocUtils';
-import { DocumentType } from '../../../documents/DocumentTypes';
+import { CollectionViewType, DocumentType } from '../../../documents/DocumentTypes';
 import { Docs, DocumentOptions } from '../../../documents/Documents';
 import { SnappingManager, freeformScrollMode } from '../../../util/SnappingManager';
 import { Transform } from '../../../util/Transform';
@@ -31,9 +32,11 @@ import { pasteImageBitmap } from '../../nodes/WebBoxRenderer';
 import { FormattedTextBox } from '../../nodes/formattedText/FormattedTextBox';
 import { CollectionCardView } from '../CollectionCardDeckView';
 import { SubCollectionViewProps } from '../CollectionSubView';
+import { CollectionFreeFormView } from './CollectionFreeFormView';
 import { ImageLabelHandler } from './ImageLabelHandler';
 import { MarqueeOptionsMenu } from './MarqueeOptionsMenu';
 import './MarqueeView.scss';
+
 interface MarqueeViewProps {
     getContainerTransform: () => Transform;
     getTransform: () => Transform;
@@ -431,38 +434,25 @@ export class MarqueeView extends ObservableReactComponent<SubCollectionViewProps
      */
     @undoBatch
     classifyImages = action(async (e: React.MouseEvent | undefined) => {
-        const selected = this.marqueeSelect(false, DocumentType.IMG);
-        this._selectedDocs = selected;
-
-        const imagePromises = selected.map(doc => {
-            const href = (doc['data'] as URLField).url.href;
-            const hrefParts = href.split('.');
-            const hrefComplete = `${hrefParts[0]}_o.${hrefParts[1]}`;
-            return CollectionCardView.imageUrlToBase64(hrefComplete).then(hrefBase64 =>
-                !hrefBase64
-                    ? undefined
-                    : gptImageLabel(hrefBase64).then(response => {
-                          const labels = response.split('\n');
-                          doc.image_labels = new List<string>(Array.from(labels!));
-                          return Promise.all(labels!.map(label => gptGetEmbedding(label))).then(embeddings => {
-                              return { doc, embeddings };
-                          });
-                      })
-            );
+        this._selectedDocs = this.marqueeSelect(false, DocumentType.IMG);
+
+        const imageInfos = this._selectedDocs.map(async doc => {
+            const [name, type] = ImageCast(doc[Doc.LayoutFieldKey(doc)]).url.href.split('.');
+            return CollectionCardView.imageUrlToBase64(`${name}_o.${type}`).then(hrefBase64 =>
+                !hrefBase64 ? undefined : 
+                    gptImageLabel(hrefBase64).then(labels => 
+                        Promise.all(labels.split('\n').map(label => gptGetEmbedding(label))).then(embeddings => 
+                            ({ doc, embeddings, labels }))) ); // prettier-ignore
         });
 
-        const docsAndEmbeddings = await Promise.all(imagePromises);
-        docsAndEmbeddings
-            .filter(d => d)
-            .map(d => d!)
-            .forEach(docAndEmbedding => {
-                if (Array.isArray(docAndEmbedding.embeddings)) {
-                    let doc = docAndEmbedding.doc;
-                    for (let i = 0; i < 3; i++) {
-                        doc[`label_embedding_${i + 1}`] = new List<number>(docAndEmbedding.embeddings[i]);
-                    }
-                }
-            });
+        (await Promise.all(imageInfos)).forEach(imageInfo => {
+            if (imageInfo && Array.isArray(imageInfo.embeddings)) {
+                imageInfo.doc[DocData].data_labels = imageInfo.labels;
+                numberRange(3).forEach(n => {
+                    imageInfo.doc[`data_labels_embedding_${n + 1}`] = new List<number>(imageInfo.embeddings[n]);
+                });
+            }
+        });
 
         if (e) {
             ImageLabelHandler.Instance.displayLabelHandler(e.pageX, e.pageY);
@@ -474,51 +464,28 @@ export class MarqueeView extends ObservableReactComponent<SubCollectionViewProps
      */
     @undoBatch
     groupImages = action(async () => {
-        const labelGroups: string[] = ImageLabelHandler.Instance._labelGroups;
-        const labelToCollection: Map<string, Doc> = new Map();
-        const labelToEmbedding: Map<string, number[]> = new Map();
-        var similarity = require('compute-cosine-similarity');
-
-        // Create new collections associated with each label and get the embeddings for the labels.
-        for (const label of labelGroups) {
-            const newCollection = this.getCollection([], undefined, false);
-            newCollection._freeform_panX = this.Bounds.left + this.Bounds.width / 2;
-            newCollection._freeform_panY = this.Bounds.top + this.Bounds.height / 2;
-            labelToCollection.set(label, newCollection);
-            this._props.addDocument?.(newCollection);
-            const labelEmbedding = await gptGetEmbedding(label);
-            if (Array.isArray(labelEmbedding)) {
-                labelToEmbedding.set(label, labelEmbedding);
-            }
-        }
+        const labelGroups = ImageLabelHandler.Instance._labelGroups;
+        const labelToEmbedding = new Map<string, number[]>();
+        // Create embeddings for the labels.
+        await Promise.all(labelGroups.map(async label => gptGetEmbedding(label).then(labelEmbedding => labelToEmbedding.set(label, labelEmbedding))));
 
         // For each image, loop through the labels, and calculate similarity. Associate it with the
         // most similar one.
         this._selectedDocs.forEach(doc => {
-            let mostSimilarLabel: string | undefined;
-            let maxSimilarity: number = 0;
-            const embeddingAsList1 = NumListCast(doc.label_embedding_1);
-            const embeddingAsList2 = NumListCast(doc.label_embedding_2);
-            const embeddingAsList3 = NumListCast(doc.label_embedding_3);
-
-            labelGroups.forEach(label => {
-                let curSimilarity1 = similarity(labelToEmbedding.get(label)!, Array.from(embeddingAsList1));
-                let curSimilarity2 = similarity(labelToEmbedding.get(label)!, Array.from(embeddingAsList2));
-                let curSimilarity3 = similarity(labelToEmbedding.get(label)!, Array.from(embeddingAsList3));
-                let maxCurSimilarity = Math.max(curSimilarity1, curSimilarity2, curSimilarity3);
-                if (maxCurSimilarity >= 0.3 && maxCurSimilarity > maxSimilarity) {
-                    mostSimilarLabel = label;
-                    maxSimilarity = maxCurSimilarity;
-                }
-
-                console.log('Doc with labels ' + doc.image_labels + 'has similarity score ' + maxCurSimilarity + ' to ' + mostSimilarLabel);
+            const embedLists = numberRange(3).map(n => Array.from(NumListCast(doc[`data_labels_embedding_${n + 1}`])));
+            const bestEmbedScore = (embedding: Opt<number[]>) => Math.max(...embedLists.map(l => (embedding && similarity(Array.from(embedding), l)) || 0));
+            const {label: mostSimilarLabelCollect} = 
+                labelGroups.map(label => ({ label, similarityScore: bestEmbedScore(labelToEmbedding.get(label)) }))
+                           .reduce((prev, cur) => cur.similarityScore < 0.3 || cur.similarityScore <= prev.similarityScore ? prev: cur,
+                                   { label: '', similarityScore: 0, }); // prettier-ignore
+
+            numberRange(3).forEach(n => {
+                doc[`data_labels_embedding_${n + 1}`] = undefined;
             });
-
-            if (mostSimilarLabel) {
-                Doc.AddDocToList(labelToCollection.get(mostSimilarLabel)!, undefined, doc);
-                this._props.removeDocument?.(doc);
-            }
+            doc[DocData].data_label = mostSimilarLabelCollect;
         });
+        this._props.Document._type_collection = CollectionViewType.Time;
+        this._props.Document.pivotField = 'data_label';
     });
 
     @undoBatch