From 376ff1626b24cbac12b27ad072690424549f05c7 Mon Sep 17 00:00:00 2001 From: IEatChili Date: Tue, 18 Jun 2024 14:33:47 -0400 Subject: feat: added view of labels on docs in freeform --- .../collectionFreeForm/ImageLabelBox.tsx | 79 ++++++++++++++++------ 1 file changed, 60 insertions(+), 19 deletions(-) (limited to 'src/client/views/collections/collectionFreeForm') diff --git a/src/client/views/collections/collectionFreeForm/ImageLabelBox.tsx b/src/client/views/collections/collectionFreeForm/ImageLabelBox.tsx index 571a4504f..cfb81e1a0 100644 --- a/src/client/views/collections/collectionFreeForm/ImageLabelBox.tsx +++ b/src/client/views/collections/collectionFreeForm/ImageLabelBox.tsx @@ -5,7 +5,7 @@ import { observer } from 'mobx-react'; import React from 'react'; import { Doc, NumListCast, Opt } from '../../../../fields/Doc'; import { Docs } from '../../../documents/Documents'; -import { CollectionViewType, DocumentType } from '../../../documents/DocumentTypes'; +import { DocumentType } from '../../../documents/DocumentTypes'; import { ViewBoxBaseComponent } from '../../DocComponent'; import { FieldView, FieldViewProps } from '../../nodes/FieldView'; import { MarqueeOptionsMenu } from './MarqueeOptionsMenu'; @@ -24,6 +24,9 @@ import { List } from '../../../../fields/List'; import { DragManager } from '../../../util/DragManager'; import { OpenWhere } from '../../nodes/OpenWhere'; import similarity from 'compute-cosine-similarity'; +import { DocumentView } from '../../nodes/DocumentView'; + +export class ImageInformationItem {} export class ImageLabelBoxData { static _instance: ImageLabelBoxData; @@ -101,7 +104,6 @@ export class ImageLabelBox extends ViewBoxBaseComponent() { makeObservable(this); ring.register(); ImageLabelBox.Instance = this; - console.log('Image Box Has Been Initialized'); } // ImageLabelBox.Instance.setData() @@ -127,7 +129,6 @@ export class ImageLabelBox extends ViewBoxBaseComponent() { @action startLoading = () => { this._loading = true; - console.log('Start loading has been called!'); }; @action @@ -138,6 +139,11 @@ export class ImageLabelBox extends ViewBoxBaseComponent() { @action toggleDisplayInformation = () => { this._displayImageInformation = !this._displayImageInformation; + if (this._displayImageInformation) { + this._selectedImages.forEach(doc => (doc[DocData].showLabels = true)); + } else { + this._selectedImages.forEach(doc => (doc[DocData].showLabels = false)); + } }; @action @@ -155,33 +161,58 @@ export class ImageLabelBox extends ViewBoxBaseComponent() { classifyImagesInBox = async () => { this.startLoading(); + // const imageInfos = this._selectedImages.map(async doc => { + // if (!doc[DocData].data_labels) { + // const [name, type] = ImageCast(doc[Doc.LayoutFieldKey(doc)]).url.href.split('.'); + // return CollectionCardView.imageUrlToBase64(`${name}_o.${type}`).then(hrefBase64 => + // !hrefBase64 ? undefined : + // gptImageLabel(hrefBase64).then(labels => + // Promise.all(labels.split('\n').map(label => gptGetEmbedding(label))).then(embeddings => + // ({ doc, embeddings, labels }))) ); // prettier-ignore + // } + // }); // Converts the images into a Base64 format, afterwhich the information is sent to GPT to label them. + const imageInfos = this._selectedImages.map(async doc => { if (!doc[DocData].data_labels) { const [name, type] = ImageCast(doc[Doc.LayoutFieldKey(doc)]).url.href.split('.'); return CollectionCardView.imageUrlToBase64(`${name}_o.${type}`).then(hrefBase64 => !hrefBase64 ? undefined : gptImageLabel(hrefBase64).then(labels => - Promise.all(labels.split('\n').map(label => gptGetEmbedding(label))).then(embeddings => - ({ doc, embeddings, labels }))) ); // prettier-ignore + + ({ doc, labels }))) ; // prettier-ignore } - }); // Converts the images into a Base64 format, afterwhich the information is sent to GPT to label them. + }); (await Promise.all(imageInfos)).forEach(imageInfo => { - if (imageInfo && imageInfo.embeddings && Array.isArray(imageInfo.embeddings)) { - imageInfo.doc[DocData].data_labels = imageInfo.labels; + if (imageInfo) { + imageInfo.doc[DocData].data_labels = new List(); const labels = imageInfo.labels.split('\n'); labels.forEach(label => { - label = label.replace(/^\d+\.\s*/, '').trim(); + label = label.replace(/^\d+\.\s*|-|\*/, '').trim(); imageInfo.doc[DocData][`${label}`] = true; - }); - - numberRange(3).forEach(n => { - imageInfo.doc[`data_labels_embedding_${n + 1}`] = new List(imageInfo.embeddings[n]); + (imageInfo.doc[DocData].data_labels as List).push(label); }); } }); // Add the labels as fields to each image. + // (await Promise.all(imageInfos)).forEach(imageInfo => { + // if (imageInfo && imageInfo.embeddings && Array.isArray(imageInfo.embeddings)) { + // imageInfo.doc[DocData].data_labels = new List(); + + // const labels = imageInfo.labels.split('\n'); + // labels.forEach(label => { + // label = label.replace(/^\d+\.\s*|-|\*/, '').trim(); + // imageInfo.doc[DocData][`${label}`] = true; + // (imageInfo.doc[DocData].data_labels as List).push(label); + // }); + + // numberRange(5).forEach(n => { + // imageInfo.doc[`data_labels_embedding_${n + 1}`] = new List(imageInfo.embeddings[n]); + // }); + // } + // }); // Add the labels as fields to each image. + this.endLoading(); }; @@ -189,7 +220,13 @@ export class ImageLabelBox extends ViewBoxBaseComponent() { * Groups images to most similar labels. */ groupImagesInBox = action(async () => { - console.log('Calling!'); + this._selectedImages.forEach(doc => { + (doc[DocData].data_labels as List).forEach(async (label, index) => { + const embedding = await gptGetEmbedding(label); + doc[`data_labels_embedding_${index + 1}`] = new List(embedding); + }); + }); + const labelToEmbedding = new Map(); // Create embeddings for the labels. await Promise.all(this._labelGroups.map(async label => gptGetEmbedding(label).then(labelEmbedding => labelToEmbedding.set(label, labelEmbedding)))); @@ -197,7 +234,7 @@ export class ImageLabelBox extends ViewBoxBaseComponent() { // For each image, loop through the labels, and calculate similarity. Associate it with the // most similar one. this._selectedImages.forEach(doc => { - const embedLists = numberRange(3).map(n => Array.from(NumListCast(doc[`data_labels_embedding_${n + 1}`]))); + const embedLists = numberRange(5).map(n => Array.from(NumListCast(doc[`data_labels_embedding_${n + 1}`]))); const bestEmbedScore = (embedding: Opt) => Math.max(...embedLists.map((l, index) => (embedding && (1 - index * 0.1) * similarity(Array.from(embedding), l)!) || 0)); const {label: mostSimilarLabelCollect} = this._labelGroups.map(label => ({ label, similarityScore: bestEmbedScore(labelToEmbedding.get(label)) })) @@ -293,10 +330,14 @@ export class ImageLabelBox extends ViewBoxBaseComponent() { {this._selectedImages.map(doc => { const [name, type] = ImageCast(doc[Doc.LayoutFieldKey(doc)]).url.href.split('.'); return ( -
this._props.addDocTab(doc, OpenWhere.addRightKeyvalue)}> - -
- {(doc[DocData].data_labels as string).split('\n').map(label => { +
+ { + await DocumentView.showDocument(doc, { willZoomCentered: true }); + }}> +
this._props.addDocTab(doc, OpenWhere.addRightKeyvalue)}> + {(doc[DocData].data_labels as List).map(label => { return (
{label} -- cgit v1.2.3-70-g09d2