diff options
Diffstat (limited to 'src')
6 files changed, 83 insertions, 111 deletions
diff --git a/src/Utils.ts b/src/Utils.ts index 44f1d55c2..23ae38bdb 100644 --- a/src/Utils.ts +++ b/src/Utils.ts @@ -211,6 +211,11 @@ export type Without<T, K extends keyof T> = Pick<T, Exclude<keyof T, K>>; export type Predicate<K, V> = (entry: [K, V]) => boolean; +/** + * creates a list of numbers ordered from 0 to 'num' + * @param num range of numbers + * @returns list of values from 0 to num -1 + */ export function numberRange(num: number) { return num > 0 && num < 1000 ? Array.from(Array(num)).map((v, i) => i) : []; } diff --git a/src/client/apis/gpt/GPT.ts b/src/client/apis/gpt/GPT.ts index 455352068..7af71a6a2 100644 --- a/src/client/apis/gpt/GPT.ts +++ b/src/client/apis/gpt/GPT.ts @@ -119,7 +119,7 @@ const gptGetEmbedding = async (src: string): Promise<number[]> => { }); // Assume the embeddingResponse structure is correct; adjust based on actual API response - const embedding = embeddingResponse.data[0].embedding; + const { embedding } = embeddingResponse.data[0]; return embedding; } catch (err) { console.log(err); @@ -155,9 +155,8 @@ const gptImageLabel = async (src: string): Promise<string> => { }); if (response.choices[0].message.content) { return response.choices[0].message.content; - } else { - return 'Missing labels'; } + return 'Missing labels'; } catch (err) { console.log(err); return 'Error connecting with API'; diff --git a/src/client/views/collections/CollectionCardDeckView.tsx b/src/client/views/collections/CollectionCardDeckView.tsx index 6036a2ead..de46180e6 100644 --- a/src/client/views/collections/CollectionCardDeckView.tsx +++ b/src/client/views/collections/CollectionCardDeckView.tsx @@ -161,7 +161,7 @@ export class CollectionCardView extends CollectionSubView() { panelHeight = (layout: Doc) => () => (this.panelWidth() * NumCast(layout._height)) / NumCast(layout._width); onChildDoubleClick = () => ScriptCast(this.layoutDoc.onChildDoubleClick); isContentActive = () => this._props.isSelected() || this._props.isContentActive() || this._props.isAnyChildContentActive(); - isChildContentActive = () => (this.isContentActive() ? true : false); + isChildContentActive = () => !!this.isContentActive(); /** * Returns the degree to rotate a card dependind on the amount of cards in their row and their index in said row @@ -173,10 +173,10 @@ export class CollectionCardView extends CollectionSubView() { const possRotate = -30 + index * (30 / ((amCards - (amCards % 2)) / 2)); const stepMag = Math.abs(-30 + (amCards / 2 - 1) * (30 / ((amCards - (amCards % 2)) / 2))); - if (amCards % 2 == 0 && possRotate == 0) { + if (amCards % 2 === 0 && possRotate === 0) { return possRotate + Math.abs(-30 + (index - 1) * (30 / (amCards / 2))); } - if (amCards % 2 == 0 && index > (amCards + 1) / 2) { + if (amCards % 2 === 0 && index > (amCards + 1) / 2) { return possRotate + stepMag; } @@ -194,10 +194,10 @@ export class CollectionCardView extends CollectionSubView() { if (realIndex > this._maxRowCount - 1) { rowOffset = 400 * ((realIndex - (realIndex % this._maxRowCount)) / this._maxRowCount); } - if (evenOdd == 1 || index < apex - 1) { + if (evenOdd === 1 || index < apex - 1) { return Math.abs(stepMag * (apex - index)) - rowOffset; } - if (index == apex || index == apex - 1) { + if (index === apex || index === apex - 1) { return 0 - rowOffset; } @@ -259,27 +259,26 @@ export class CollectionCardView extends CollectionSubView() { return docs; }; - displayDoc = (doc: Doc, screenToLocalTransform: () => Transform) => { - return ( - <DocumentView - {...this._props} - ref={action((r: DocumentView) => r?.ContentDiv && this._docRefs.set(doc, r))} - Document={doc} - NativeWidth={returnZero} - NativeHeight={returnZero} - fitWidth={returnFalse} - onDoubleClickScript={this.onChildDoubleClick} - renderDepth={this._props.renderDepth + 1} - LayoutTemplate={this._props.childLayoutTemplate} - LayoutTemplateString={this._props.childLayoutString} - ScreenToLocalTransform={screenToLocalTransform} //makes sure the box wrapper thing is in the right spot - isContentActive={this.isChildContentActive} - isDocumentActive={this._props.childDocumentsActive?.() || this.Document._childDocumentsActive ? this._props.isDocumentActive : this.isContentActive} - PanelWidth={this.panelWidth} - PanelHeight={this.panelHeight(doc)} - /> - ); - }; + displayDoc = (doc: Doc, screenToLocalTransform: () => Transform) => ( + <DocumentView + // eslint-disable-next-line react/jsx-props-no-spreading + {...this._props} + ref={action((r: DocumentView) => r?.ContentDiv && this._docRefs.set(doc, r))} + Document={doc} + NativeWidth={returnZero} + NativeHeight={returnZero} + fitWidth={returnFalse} + onDoubleClickScript={this.onChildDoubleClick} + renderDepth={this._props.renderDepth + 1} + LayoutTemplate={this._props.childLayoutTemplate} + LayoutTemplateString={this._props.childLayoutString} + ScreenToLocalTransform={screenToLocalTransform} // makes sure the box wrapper thing is in the right spot + isContentActive={this.isChildContentActive} + isDocumentActive={this._props.childDocumentsActive?.() || this.Document._childDocumentsActive ? this._props.isDocumentActive : this.isContentActive} + PanelWidth={this.panelWidth} + PanelHeight={this.panelHeight(doc)} + /> + ); /** * Determines how many cards are in the row of a card at a specific index @@ -296,7 +295,7 @@ export class CollectionCardView extends CollectionSubView() { if (index < totalCards - (totalCards % 10)) { return this._maxRowCount; } - //(3) + // (3) return totalCards % 10; }; /** @@ -335,7 +334,9 @@ export class CollectionCardView extends CollectionSubView() { * @param buttonID * @param doc */ - toggleButton = undoable((buttonID: number, doc: Doc) => this.cardSort_customField && (doc[this.cardSort_customField] = buttonID), 'toggle custom button'); + toggleButton = undoable((buttonID: number, doc: Doc) => { + this.cardSort_customField && (doc[this.cardSort_customField] = buttonID); + }, 'toggle custom button'); /** * A list of the text content of all the child docs. RTF documents will have just their text and pdf documents will have the first 50 words. @@ -346,8 +347,7 @@ export class CollectionCardView extends CollectionSubView() { childPairStringList = () => { const docToText = (doc: Doc) => { switch (doc.type) { - case DocumentType.PDF: const words = StrCast(doc.text).split(/\s+/); - return words.slice(0, 50).join(' '); // first 50 words of pdf text + case DocumentType.PDF: return StrCast(doc.text).split(/\s+/).slice(0, 50).join(' '); // first 50 words of pdf text case DocumentType.IMG: return this.getImageDesc(doc); case DocumentType.RTF: return StrCast(RTFCast(doc.text).Text); default: return StrCast(doc.title); @@ -368,7 +368,7 @@ export class CollectionCardView extends CollectionSubView() { */ getImageDesc = async (image: Doc) => { if (StrCast(image.description)) return StrCast(image.description); // Return existing description - const href = (image.data as URLField).url.href; + const { href } = (image.data as URLField).url; const hrefParts = href.split('.'); const hrefComplete = `${hrefParts[0]}_o.${hrefParts[1]}`; try { @@ -422,12 +422,13 @@ export class CollectionCardView extends CollectionSubView() { */ renderButtons = (doc: Doc, cardSort: cardSortings) => { if (cardSort !== cardSortings.Custom) return ''; - const amButtons = Math.max(4, this.childDocs?.reduce((set, doc) => this.cardSort_customField && set.add(NumCast(doc[this.cardSort_customField])), new Set<number>()).size ?? 0); + const amButtons = Math.max(4, this.childDocs?.reduce((set, d) => this.cardSort_customField && set.add(NumCast(d[this.cardSort_customField])), new Set<number>()).size ?? 0); const activeButtonIndex = CollectionCardView.getButtonGroup(this.cardSort_customField, doc); const totalWidth = amButtons * 35 + amButtons * 2 * 5 + 6; return ( <div className="card-button-container" style={{ width: `${totalWidth}px` }}> {numberRange(amButtons).map(i => ( + // eslint-disable-next-line jsx-a11y/control-has-associated-label <button key={i} type="button" diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormLayoutEngines.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormLayoutEngines.tsx index a4496a417..de51cc73c 100644 --- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormLayoutEngines.tsx +++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormLayoutEngines.tsx @@ -245,7 +245,7 @@ export function computePivotLayout(poolData: Map<string, PoolData>, pivotDoc: Do y: -y + (pivotAxisWidth - hgt) / 2, width: wid, height: hgt, - backgroundColor: StrCast(layoutDoc.backgroundColor), + backgroundColor: StrCast(layoutDoc.backgroundColor, 'white'), pair: { layout: doc }, replica: val.replicas[i], }); diff --git a/src/client/views/collections/collectionFreeForm/ImageLabelHandler.tsx b/src/client/views/collections/collectionFreeForm/ImageLabelHandler.tsx index 46bc3d946..7f27c6b5c 100644 --- a/src/client/views/collections/collectionFreeForm/ImageLabelHandler.tsx +++ b/src/client/views/collections/collectionFreeForm/ImageLabelHandler.tsx @@ -52,8 +52,8 @@ export class ImageLabelHandler extends ObservableReactComponent<{}> { @action removeLabel = (label: string) => { - label = label.toUpperCase(); - this._labelGroups = this._labelGroups.filter(group => group !== label); + const labelUp = label.toUpperCase(); + this._labelGroups = this._labelGroups.filter(group => group !== labelUp); }; @action diff --git a/src/client/views/collections/collectionFreeForm/MarqueeView.tsx b/src/client/views/collections/collectionFreeForm/MarqueeView.tsx index ff8c2d318..dc15c83c5 100644 --- a/src/client/views/collections/collectionFreeForm/MarqueeView.tsx +++ b/src/client/views/collections/collectionFreeForm/MarqueeView.tsx @@ -1,22 +1,23 @@ /* eslint-disable jsx-a11y/no-static-element-interactions */ +import similarity from 'compute-cosine-similarity'; import { action, computed, makeObservable, observable } from 'mobx'; import { observer } from 'mobx-react'; import * as React from 'react'; import { ClientUtils, lightOrDark, returnFalse } from '../../../../ClientUtils'; -import { intersectRect } from '../../../../Utils'; +import { intersectRect, numberRange } from '../../../../Utils'; import { Doc, NumListCast, Opt } from '../../../../fields/Doc'; import { AclAdmin, AclAugment, AclEdit, DocData } from '../../../../fields/DocSymbols'; import { Id } from '../../../../fields/FieldSymbols'; import { InkData, InkField, InkTool } from '../../../../fields/InkField'; import { List } from '../../../../fields/List'; import { RichTextField } from '../../../../fields/RichTextField'; -import { Cast, FieldValue, NumCast, StrCast } from '../../../../fields/Types'; -import { ImageField, URLField } from '../../../../fields/URLField'; +import { Cast, FieldValue, ImageCast, NumCast, StrCast } from '../../../../fields/Types'; +import { ImageField } from '../../../../fields/URLField'; import { GetEffectiveAcl } from '../../../../fields/util'; import { gptGetEmbedding, gptImageLabel } from '../../../apis/gpt/GPT'; import { CognitiveServices } from '../../../cognitive_services/CognitiveServices'; import { DocUtils } from '../../../documents/DocUtils'; -import { DocumentType } from '../../../documents/DocumentTypes'; +import { CollectionViewType, DocumentType } from '../../../documents/DocumentTypes'; import { Docs, DocumentOptions } from '../../../documents/Documents'; import { SnappingManager, freeformScrollMode } from '../../../util/SnappingManager'; import { Transform } from '../../../util/Transform'; @@ -31,9 +32,11 @@ import { pasteImageBitmap } from '../../nodes/WebBoxRenderer'; import { FormattedTextBox } from '../../nodes/formattedText/FormattedTextBox'; import { CollectionCardView } from '../CollectionCardDeckView'; import { SubCollectionViewProps } from '../CollectionSubView'; +import { CollectionFreeFormView } from './CollectionFreeFormView'; import { ImageLabelHandler } from './ImageLabelHandler'; import { MarqueeOptionsMenu } from './MarqueeOptionsMenu'; import './MarqueeView.scss'; + interface MarqueeViewProps { getContainerTransform: () => Transform; getTransform: () => Transform; @@ -431,38 +434,25 @@ export class MarqueeView extends ObservableReactComponent<SubCollectionViewProps */ @undoBatch classifyImages = action(async (e: React.MouseEvent | undefined) => { - const selected = this.marqueeSelect(false, DocumentType.IMG); - this._selectedDocs = selected; - - const imagePromises = selected.map(doc => { - const href = (doc['data'] as URLField).url.href; - const hrefParts = href.split('.'); - const hrefComplete = `${hrefParts[0]}_o.${hrefParts[1]}`; - return CollectionCardView.imageUrlToBase64(hrefComplete).then(hrefBase64 => - !hrefBase64 - ? undefined - : gptImageLabel(hrefBase64).then(response => { - const labels = response.split('\n'); - doc.image_labels = new List<string>(Array.from(labels!)); - return Promise.all(labels!.map(label => gptGetEmbedding(label))).then(embeddings => { - return { doc, embeddings }; - }); - }) - ); + this._selectedDocs = this.marqueeSelect(false, DocumentType.IMG); + + const imageInfos = this._selectedDocs.map(async doc => { + const [name, type] = ImageCast(doc[Doc.LayoutFieldKey(doc)]).url.href.split('.'); + return CollectionCardView.imageUrlToBase64(`${name}_o.${type}`).then(hrefBase64 => + !hrefBase64 ? undefined : + gptImageLabel(hrefBase64).then(labels => + Promise.all(labels.split('\n').map(label => gptGetEmbedding(label))).then(embeddings => + ({ doc, embeddings, labels }))) ); // prettier-ignore }); - const docsAndEmbeddings = await Promise.all(imagePromises); - docsAndEmbeddings - .filter(d => d) - .map(d => d!) - .forEach(docAndEmbedding => { - if (Array.isArray(docAndEmbedding.embeddings)) { - let doc = docAndEmbedding.doc; - for (let i = 0; i < 3; i++) { - doc[`label_embedding_${i + 1}`] = new List<number>(docAndEmbedding.embeddings[i]); - } - } - }); + (await Promise.all(imageInfos)).forEach(imageInfo => { + if (imageInfo && Array.isArray(imageInfo.embeddings)) { + imageInfo.doc[DocData].data_labels = imageInfo.labels; + numberRange(3).forEach(n => { + imageInfo.doc[`data_labels_embedding_${n + 1}`] = new List<number>(imageInfo.embeddings[n]); + }); + } + }); if (e) { ImageLabelHandler.Instance.displayLabelHandler(e.pageX, e.pageY); @@ -474,51 +464,28 @@ export class MarqueeView extends ObservableReactComponent<SubCollectionViewProps */ @undoBatch groupImages = action(async () => { - const labelGroups: string[] = ImageLabelHandler.Instance._labelGroups; - const labelToCollection: Map<string, Doc> = new Map(); - const labelToEmbedding: Map<string, number[]> = new Map(); - var similarity = require('compute-cosine-similarity'); - - // Create new collections associated with each label and get the embeddings for the labels. - for (const label of labelGroups) { - const newCollection = this.getCollection([], undefined, false); - newCollection._freeform_panX = this.Bounds.left + this.Bounds.width / 2; - newCollection._freeform_panY = this.Bounds.top + this.Bounds.height / 2; - labelToCollection.set(label, newCollection); - this._props.addDocument?.(newCollection); - const labelEmbedding = await gptGetEmbedding(label); - if (Array.isArray(labelEmbedding)) { - labelToEmbedding.set(label, labelEmbedding); - } - } + const labelGroups = ImageLabelHandler.Instance._labelGroups; + const labelToEmbedding = new Map<string, number[]>(); + // Create embeddings for the labels. + await Promise.all(labelGroups.map(async label => gptGetEmbedding(label).then(labelEmbedding => labelToEmbedding.set(label, labelEmbedding)))); // For each image, loop through the labels, and calculate similarity. Associate it with the // most similar one. this._selectedDocs.forEach(doc => { - let mostSimilarLabel: string | undefined; - let maxSimilarity: number = 0; - const embeddingAsList1 = NumListCast(doc.label_embedding_1); - const embeddingAsList2 = NumListCast(doc.label_embedding_2); - const embeddingAsList3 = NumListCast(doc.label_embedding_3); - - labelGroups.forEach(label => { - let curSimilarity1 = similarity(labelToEmbedding.get(label)!, Array.from(embeddingAsList1)); - let curSimilarity2 = similarity(labelToEmbedding.get(label)!, Array.from(embeddingAsList2)); - let curSimilarity3 = similarity(labelToEmbedding.get(label)!, Array.from(embeddingAsList3)); - let maxCurSimilarity = Math.max(curSimilarity1, curSimilarity2, curSimilarity3); - if (maxCurSimilarity >= 0.3 && maxCurSimilarity > maxSimilarity) { - mostSimilarLabel = label; - maxSimilarity = maxCurSimilarity; - } - - console.log('Doc with labels ' + doc.image_labels + 'has similarity score ' + maxCurSimilarity + ' to ' + mostSimilarLabel); + const embedLists = numberRange(3).map(n => Array.from(NumListCast(doc[`data_labels_embedding_${n + 1}`]))); + const bestEmbedScore = (embedding: Opt<number[]>) => Math.max(...embedLists.map(l => (embedding && similarity(Array.from(embedding), l)) || 0)); + const {label: mostSimilarLabelCollect} = + labelGroups.map(label => ({ label, similarityScore: bestEmbedScore(labelToEmbedding.get(label)) })) + .reduce((prev, cur) => cur.similarityScore < 0.3 || cur.similarityScore <= prev.similarityScore ? prev: cur, + { label: '', similarityScore: 0, }); // prettier-ignore + + numberRange(3).forEach(n => { + doc[`data_labels_embedding_${n + 1}`] = undefined; }); - - if (mostSimilarLabel) { - Doc.AddDocToList(labelToCollection.get(mostSimilarLabel)!, undefined, doc); - this._props.removeDocument?.(doc); - } + doc[DocData].data_label = mostSimilarLabelCollect; }); + this._props.Document._type_collection = CollectionViewType.Time; + this._props.Document.pivotField = 'data_label'; }); @undoBatch |