diff options
Diffstat (limited to 'src/client')
29 files changed, 1454 insertions, 240 deletions
diff --git a/src/client/Network.ts b/src/client/Network.ts index 6c60c4151..204fcf0ac 100644 --- a/src/client/Network.ts +++ b/src/client/Network.ts @@ -1,3 +1,4 @@ +import formidable from 'formidable'; import * as requestPromise from 'request-promise'; import { ClientUtils } from '../ClientUtils'; import { Utils } from '../Utils'; @@ -49,15 +50,9 @@ export namespace Networking { if (!fileguidpairs.length) { return []; } - const maxFileSize = 5000000; + const maxFileSize = 6000000; if (fileguidpairs.some(f => f.file.size > maxFileSize)) { - return new Promise<Upload.FileResponse<T>[]>(res => { - res([{ - source: { size: 0, filepath: '', mimetype: '', originalFilename: '', newFilename: '',hashAlgorithm: false, - toJSON: () => ({ size: 0, filepath: '', mimetype: '', originalFilename: '', newFilename: '',name: '', length: 0, mtime: new Date(), type: '' }) }, - result: { name: '', message: `max file size (${maxFileSize / 1000000}MB) exceeded` } - }]) // prettier-ignore - }); + return new Promise<Upload.FileResponse<T>[]>(res => res([{ source: { newFilename: '', mimetype: '' } as formidable.File, result: new Error(`max file size (${maxFileSize / 1000000}MB) exceeded`) }])); } formData.set('fileguids', fileguidpairs.map(pair => pair.guid).join(';')); formData.set('filesize', fileguidpairs.reduce((sum, pair) => sum + pair.file.size, 0).toString()); diff --git a/src/client/apis/gpt/GPT.ts b/src/client/apis/gpt/GPT.ts index 05007960d..8dd3fd6e2 100644 --- a/src/client/apis/gpt/GPT.ts +++ b/src/client/apis/gpt/GPT.ts @@ -128,7 +128,7 @@ const gptImageLabel = async (src: string): Promise<string> => { { role: 'user', content: [ - { type: 'text', text: 'Give three to five labels to describe this image.' }, + { type: 'text', text: 'Give three labels to describe this image.' }, { type: 'image_url', image_url: { diff --git a/src/client/documents/DocUtils.ts b/src/client/documents/DocUtils.ts index 5ee7d0f9c..30b71a09b 100644 --- a/src/client/documents/DocUtils.ts +++ b/src/client/documents/DocUtils.ts @@ -35,7 +35,7 @@ import { TaskCompletionBox } from '../views/nodes/TaskCompletedBox'; import { DocumentType } from './DocumentTypes'; import { Docs, DocumentOptions } from './Documents'; -// eslint-disable-next-line @typescript-eslint/no-var-requires +// eslint-disable-next-line @typescript-eslint/no-var-requires, @typescript-eslint/no-require-imports const { DFLT_IMAGE_NATIVE_DIM } = require('../views/global/globalCssVariables.module.scss'); // prettier-ignore const defaultNativeImageDim = Number(DFLT_IMAGE_NATIVE_DIM.replace('px', '')); @@ -738,7 +738,7 @@ export namespace DocUtils { const { source: { newFilename, mimetype }, result, - } = upfiles.lastElement() ?? { source: { newFilename: '', mimetype: '' }, result: { message: 'upload failed' } }; + } = upfiles.lastElement() ?? { source: { newFilename: '', mimetype: '' }, result: new Error('upload failed') }; if (result instanceof Error) { if (overwriteDoc) { overwriteDoc.loadingError = result.message; diff --git a/src/client/documents/DocumentTypes.ts b/src/client/documents/DocumentTypes.ts index 53edb2e31..49df943d8 100644 --- a/src/client/documents/DocumentTypes.ts +++ b/src/client/documents/DocumentTypes.ts @@ -16,6 +16,8 @@ export enum DocumentType { SCREENSHOT = 'screenshot', FONTICON = 'fonticonbox', SEARCH = 'search', // search query + IMAGEGROUPER = 'imagegrouper', + FACECOLLECTION = 'facecollection', LABEL = 'label', // simple text label BUTTON = 'button', // onClick button WEBCAM = 'webcam', // webcam diff --git a/src/client/documents/Documents.ts b/src/client/documents/Documents.ts index 41c6ce39b..6b5469cca 100644 --- a/src/client/documents/Documents.ts +++ b/src/client/documents/Documents.ts @@ -37,12 +37,13 @@ export enum FInfoFieldType { date = 'date', list = 'list', rtf = 'rich text', + map = 'map', } export class FInfo { description: string = ''; readOnly: boolean = false; fieldType?: FInfoFieldType; - values?: FieldType[]; + values?: FieldType[] | Map<any, any>; filterable?: boolean = true; // can be used as a Filter in FilterPanel // format?: string; // format to display values (e.g, decimal places, $, etc) @@ -143,6 +144,10 @@ class ListInfo extends FInfo { fieldType? = FInfoFieldType.list; values?: List<FieldType>[] = []; } +class MapInfo extends FInfo { + fieldType? = FInfoFieldType.map; + values?: Map<any, any> = new Map(); +} type BOOLt = BoolInfo | boolean; type NUMt = NumInfo | number; type STRt = StrInfo | string; @@ -155,6 +160,7 @@ type COLLt = CTypeInfo | CollectionViewType; type DROPt = DAInfo | dropActionType; type DATEt = DateInfo | number; type DTYPEt = DTypeInfo | string; +type MAPt = MapInfo | Map<any, any>; export class DocumentOptions { // coordinate and dimensions depending on view x?: NUMt = new NumInfo('horizontal coordinate in freeform view', false); @@ -483,6 +489,7 @@ export class DocumentOptions { cardSort?: STRt = new StrInfo('way cards are sorted in deck view'); cardSort_customField?: STRt = new StrInfo('field key used for sorting cards'); cardSort_visibleSortGroups?: List<number>; // which sorting values are being filtered (shown) + keywords?: MAPt = new MapInfo('keywords', true); } export const DocOptions = new DocumentOptions(); @@ -786,6 +793,14 @@ export namespace Docs { return InstanceFromProto(Prototypes.get(DocumentType.SEARCH), new List<Doc>([]), options); } + export function ImageGrouperDocument(options: DocumentOptions = {}) { + return InstanceFromProto(Prototypes.get(DocumentType.IMAGEGROUPER), undefined, options); + } + + export function FaceCollectionDocument(options: DocumentOptions = {}) { + return InstanceFromProto(Prototypes.get(DocumentType.FACECOLLECTION), undefined, options); + } + export function LoadingDocument(file: File | string, options: DocumentOptions) { return InstanceFromProto(Prototypes.get(DocumentType.LOADING), undefined, { _height: 150, _width: 200, title: typeof file === 'string' ? file : file.name, ...options }, undefined, ''); } diff --git a/src/client/util/CurrentUserUtils.ts b/src/client/util/CurrentUserUtils.ts index 311b86fa4..2962682c2 100644 --- a/src/client/util/CurrentUserUtils.ts +++ b/src/client/util/CurrentUserUtils.ts @@ -1,3 +1,4 @@ + import { reaction, runInAction } from "mobx"; import * as rp from 'request-promise'; import { ClientUtils, OmitKeys } from "../../ClientUtils"; @@ -456,6 +457,8 @@ pie title Minerals in my tap water { title: "Shared", toolTip: "Shared Docs", target: Doc.MySharedDocs, ignoreClick: true, icon: "users", funcs: {badgeValue: badgeValue}}, { title: "Trails", toolTip: "Trails ⌘R", target: Doc.UserDoc(), ignoreClick: true, icon: "pres-trail", funcs: {target: getActiveDashTrails}}, { title: "User Doc", toolTip: "User Doc", target: this.setupUserDocView(doc, "myUserDocView"), ignoreClick: true, icon: "address-card",funcs: {hidden: "IsNoviceMode()"} }, + { title: "Image Grouper", toolTip: "Image Grouper", target: this.setupImageGrouper(doc, "myImageGrouper"), ignoreClick: true, icon: "folder-open", hidden: false }, + { title: "Face Collection", toolTip: "Face Collection", target: this.setupFaceCollection(doc, "myFaceCollection"), ignoreClick: true, icon: "face-smile", hidden: false }, ].map(tuple => ({...tuple, scripts:{onClick: 'selectMainMenu(this)'}})); } @@ -491,6 +494,18 @@ pie title Minerals in my tap water _lockedPosition: true, _type_collection: CollectionViewType.Schema }); } + static setupImageGrouper(doc: Doc, field: string) { + return DocUtils.AssignDocField(doc, field, (opts) => Docs.Create.ImageGrouperDocument(opts), { + dontRegisterView: true, backgroundColor: "dimgray", ignoreClick: true, title: "Image Grouper", isSystem: true, childDragAction: dropActionType.embed, + _lockedPosition: true, _type_collection: CollectionViewType.Schema }); + } + + static setupFaceCollection(doc: Doc, field: string) { + return DocUtils.AssignDocField(doc, field, (opts) => Docs.Create.FaceCollectionDocument(opts), { + dontRegisterView: true, backgroundColor: "dimgray", ignoreClick: true, title: "Face Collection", isSystem: true, childDragAction: dropActionType.embed, + _lockedPosition: true, _type_collection: CollectionViewType.Schema }); + } + /// Initializes the panel of draggable tools that is opened from the left sidebar. static setupToolsBtnPanel(doc: Doc, field:string) { const allTools = DocListCast(DocCast(doc[field])?.data); @@ -693,8 +708,8 @@ pie title Minerals in my tap water { title: "Fit All", icon: "object-group", toolTip: "Fit Docs to View (double click to make sticky)",btnType: ButtonType.ToggleButton, ignoreClick:true, expertMode: false, toolType:"viewAll", funcs: {}, scripts: { onClick: '{ return showFreeform(this.toolType, _readOnly_);}', onDoubleClick: '{ return showFreeform(this.toolType, _readOnly_, true);}'}}, // Only when floating document is selected in freeform { title: "Clusters", icon: "braille", toolTip: "Show Doc Clusters", btnType: ButtonType.ToggleButton, ignoreClick: true, expertMode: false, toolType:"clusters", funcs: {}, scripts: { onClick: '{ return showFreeform(this.toolType, _readOnly_);}'}}, // Only when floating document is selected in freeform { title: "Cards", icon: "brain", toolTip: "Flashcards", btnType: ButtonType.ToggleButton, ignoreClick: true, expertMode: false, toolType:"flashcards", funcs: {}, scripts: { onClick: '{ return showFreeform(this.toolType, _readOnly_);}'}}, // Only when floating document is selected in freeform - { title: "Arrange", icon:"arrow-down-short-wide",toolTip:"Toggle Auto Arrange", btnType: ButtonType.ToggleButton, ignoreClick: true, expertMode: false, toolType:"arrange", funcs: {hidden: 'IsNoviceMode()'}, scripts: { onClick: '{ return showFreeform(this.toolType, _readOnly_);}'}}, // Only when floating document is selected in freeform - + { title: "Arrange", icon:"arrow-down-short-wide",toolTip:"Auto Arrange", btnType: ButtonType.ToggleButton, ignoreClick: true, expertMode: false, toolType:"arrange", funcs: {hidden: 'IsNoviceMode()'}, scripts: { onClick: '{ return showFreeform(this.toolType, _readOnly_);}'}}, // Only when floating document is selected in freeform + ] } static textTools():Button[] { diff --git a/src/client/util/DocumentManager.ts b/src/client/util/DocumentManager.ts index b12bf4390..a10237fcc 100644 --- a/src/client/util/DocumentManager.ts +++ b/src/client/util/DocumentManager.ts @@ -61,6 +61,10 @@ export class DocumentManager { }); } + private _anyViewRenderedCbs: ((dv: DocumentView) => unknown)[] = []; + public AddAnyViewRenderedCB = (func: (dv: DocumentView) => unknown) => { + this._anyViewRenderedCbs.push(func); + }; private _viewRenderedCbs: { doc: Doc; func: (dv: DocumentView) => unknown }[] = []; public AddViewRenderedCb = (doc: Opt<Doc>, func: (dv: DocumentView) => unknown) => { if (doc) { @@ -77,13 +81,14 @@ export class DocumentManager { return false; }; callAddViewFuncs = (view: DocumentView) => { - const callFuncs = this._viewRenderedCbs.filter(vc => vc.doc === view.Document); + const docCallFuncs = this._viewRenderedCbs.filter(vc => vc.doc === view.Document); + const callFuncs = docCallFuncs.map(vc => vc.func).concat(this._anyViewRenderedCbs); if (callFuncs.length) { - this._viewRenderedCbs = this._viewRenderedCbs.filter(vc => !callFuncs.includes(vc)); + this._viewRenderedCbs = this._viewRenderedCbs.filter(vc => !docCallFuncs.includes(vc)); const intTimer = setInterval( () => { if (!view.ComponentView?.incrementalRendering?.()) { - callFuncs.forEach(cf => cf.func(view)); + callFuncs.forEach(cf => cf(view)); clearInterval(intTimer); } }, diff --git a/src/client/views/DocumentButtonBar.tsx b/src/client/views/DocumentButtonBar.tsx index b778a4fb9..d42a18e4e 100644 --- a/src/client/views/DocumentButtonBar.tsx +++ b/src/client/views/DocumentButtonBar.tsx @@ -28,6 +28,7 @@ import { DocumentLinksButton } from './nodes/DocumentLinksButton'; import { DocumentView } from './nodes/DocumentView'; import { OpenWhere } from './nodes/OpenWhere'; import { DashFieldView } from './nodes/formattedText/DashFieldView'; +import { DocData } from '../../fields/DocSymbols'; @observer export class DocumentButtonBar extends ObservableReactComponent<{ views: () => (DocumentView | undefined)[]; stack?: unknown }> { @@ -279,6 +280,23 @@ export class DocumentButtonBar extends ObservableReactComponent<{ views: () => ( ); } + @computed + get keywordButton() { + const targetDoc = this.view0?.Document; + return !targetDoc ? null : ( + <Tooltip title={<div className="dash-keyword-button">Open keyword menu</div>}> + <div + className="documentButtonBar-icon" + style={{ color: 'white' }} + onClick={() => { + targetDoc[DocData].showLabels = !targetDoc[DocData].showLabels; + }}> + <FontAwesomeIcon className="documentdecorations-icon" icon="tag" /> + </div> + </Tooltip> + ); + } + @observable _isRecording = false; _stopFunc: () => void = emptyFunction; @computed @@ -449,6 +467,7 @@ export class DocumentButtonBar extends ObservableReactComponent<{ views: () => ( <div className="documentButtonBar-button">{this.pinButton}</div> <div className="documentButtonBar-button">{this.recordButton}</div> <div className="documentButtonBar-button">{this.calendarButton}</div> + <div className="documentButtonBar-button">{this.keywordButton}</div> {!Doc.UserDoc().documentLinksButton_fullMenu ? null : <div className="documentButtonBar-button">{this.shareButton}</div>} <div className="documentButtonBar-button">{this.menuButton}</div> </div> diff --git a/src/client/views/DocumentDecorations.scss b/src/client/views/DocumentDecorations.scss index 239c0a977..67e1054c3 100644 --- a/src/client/views/DocumentDecorations.scss +++ b/src/client/views/DocumentDecorations.scss @@ -512,7 +512,7 @@ $resizeHandler: 8px; justify-content: center; align-items: center; gap: 5px; - top: 4px; + //top: 4px; background: $light-gray; opacity: 0.2; pointer-events: all; diff --git a/src/client/views/DocumentDecorations.tsx b/src/client/views/DocumentDecorations.tsx index 68970223a..ce1138b7a 100644 --- a/src/client/views/DocumentDecorations.tsx +++ b/src/client/views/DocumentDecorations.tsx @@ -89,6 +89,7 @@ export class DocumentDecorations extends ObservableReactComponent<DocumentDecora (this._showNothing = !inputting && !DocumentButtonBar.Instance?._tooltipOpen && !(this.Bounds.x !== Number.MAX_VALUE && // (this.Bounds.x > center.x+x || this.Bounds.r < center.x+x || this.Bounds.y > center.y+y || this.Bounds.b < center.y+y ))); + })); // prettier-ignore } @@ -642,6 +643,7 @@ export class DocumentDecorations extends ObservableReactComponent<DocumentDecora render() { const { b, r, x, y } = this.Bounds; const seldocview = DocumentView.Selected().lastElement(); + const doc = DocumentView.SelectedDocs().lastElement(); if (SnappingManager.IsDragging || r - x < 1 || x === Number.MAX_VALUE || !seldocview || this._hidden || isNaN(r) || isNaN(b) || isNaN(x) || isNaN(y)) { setTimeout( action(() => { @@ -833,6 +835,7 @@ export class DocumentDecorations extends ObservableReactComponent<DocumentDecora <div className="link-button-container" style={{ + top: `${doc[DocData].showLabels ? 4 + (doc[DocData].keywordHeight as number) : 4}px`, transform: `translate(${-this._resizeBorderWidth / 2 + 10}px, ${this._resizeBorderWidth + bounds.b - bounds.y + this._titleHeight}px) `, }}> <DocumentButtonBar views={() => DocumentView.Selected()} /> diff --git a/src/client/views/KeywordBox.tsx b/src/client/views/KeywordBox.tsx new file mode 100644 index 000000000..fc9c38a11 --- /dev/null +++ b/src/client/views/KeywordBox.tsx @@ -0,0 +1,364 @@ +import { Button, Colors, IconButton } from 'browndash-components'; +import { action, computed, makeObservable, observable, reaction } from 'mobx'; +import { observer } from 'mobx-react'; +import React from 'react'; +import { returnFalse, setupMoveUpEvents } from '../../ClientUtils'; +import { Doc, DocListCast } from '../../fields/Doc'; +import { DocData } from '../../fields/DocSymbols'; +import { List } from '../../fields/List'; +import { NumCast, StrCast } from '../../fields/Types'; +import { emptyFunction, Utils } from '../../Utils'; +import { DocumentType } from '../documents/DocumentTypes'; +import { DragManager } from '../util/DragManager'; +import { SnappingManager } from '../util/SnappingManager'; +import { DocumentView } from './nodes/DocumentView'; +import { ObservableReactComponent } from './ObservableReactComponent'; + +interface KeywordItemProps { + doc: Doc; + keyword: string; + keywordDoc: Doc; + setToEditing: () => void; + isEditing: boolean; +} + +/** + * A component that handles individual keywords. + */ +@observer +export class KeywordItem extends ObservableReactComponent<KeywordItemProps> { + constructor(props: any) { + super(props); + makeObservable(this); + this.ref = React.createRef(); + } + + private ref: React.RefObject<HTMLDivElement>; + + /** + * Gets the documents that a keyword is associated with. + * @returns An array of documents that contain the keyword. + */ + getKeywordCollectionDocs = () => { + for (const doc of DocListCast(Doc.ActiveDashboard?.myKeywordCollections)) { + if (doc.title === this._props.keyword) { + return doc[DocData].docs; + } + } + return null; + }; + + /** + * Creates a smart collection. + * @returns + */ + createCollection = () => { + // Get the documents that contain the keyword. + const selected = DocListCast(this.getKeywordCollectionDocs()!); + const newEmbeddings = selected.map(doc => Doc.MakeEmbedding(doc)); + + // Create a new collection and set up configurations. + const newCollection = ((doc: Doc) => { + const docData = doc[DocData]; + docData.data = new List<Doc>(newEmbeddings); + docData.title = this._props.keyword; + doc._freeform_panX = doc._freeform_panY = 0; + return doc; + })(Doc.MakeCopy(Doc.UserDoc().emptyCollection as Doc, true)); + newEmbeddings.forEach(embed => (embed.embedContainer = newCollection)); + newCollection._width = 900; + newCollection._height = 900; + newCollection.layout_fitWidth = true; + + // Add the collection to the keyword document's list of associated smart collections. + this._props.keywordDoc.collections = new List<Doc>([...DocListCast(this._props.keywordDoc.collections), newCollection]); + newCollection[DocData].data_labels = new List<string>([this._props.keyword]); + newCollection[DocData][`${this._props.keyword}`] = true; + newCollection[DocData].showLabels = true; + return newCollection; + }; + + @action + handleDragStart = (e: React.PointerEvent) => { + if (this._props.isEditing) { + const clone = this.ref.current?.cloneNode(true) as HTMLElement; + if (!clone) return; + + setupMoveUpEvents( + this, + e, + () => { + const dragData = new DragManager.DocumentDragData([this.createCollection()]); + DragManager.StartDocumentDrag([this.ref.current!], dragData, e.clientX, e.clientY, {}); + return true; + }, + returnFalse, + emptyFunction + ); + e.preventDefault(); + } + }; + + @action + removeLabel = () => { + if (this._props.doc[DocData].data_labels) { + if (this._props.doc.type === DocumentType.COL) { + const filtered_collections = new List<Doc>(DocListCast(this._props.keywordDoc.collections).filter(doc => doc !== this._props.doc)); + this._props.keywordDoc.collections = filtered_collections; + + for (const cur_doc of DocListCast(this.getKeywordCollectionDocs()!, [])) { + this._props.doc[DocData].data = new List<Doc>(DocListCast(this._props.doc[DocData].data).filter(doc => !Doc.AreProtosEqual(cur_doc, doc))); + } + } else { + const filtered_docs = new List<Doc>(DocListCast(this.getKeywordCollectionDocs()!).filter(doc => doc !== this._props.doc)); + this._props.keywordDoc[DocData].docs = filtered_docs; + + for (const collection of DocListCast(this._props.keywordDoc.collections)) { + collection[DocData].data = new List<Doc>(DocListCast(collection[DocData].data).filter(doc => !Doc.AreProtosEqual(this._props.doc, doc))); + } + } + } + this._props.doc[DocData].data_labels = (this._props.doc[DocData].data_labels as List<string>).filter(label => label !== this._props.keyword) as List<string>; + this._props.doc![DocData][`${this._props.keyword}`] = false; + }; + + render() { + return ( + <div className="keyword" onClick={this._props.setToEditing} onPointerDown={this.handleDragStart} ref={this.ref} key={Utils.GenerateGuid()}> + {this._props.keyword} + {this.props.isEditing && <IconButton tooltip={'Remove label'} onPointerDown={this.removeLabel} icon={'X'} style={{ width: '8px', height: '8px', marginLeft: '10px' }} />} + </div> + ); + } +} + +interface KeywordBoxProps { + doc: Doc; + isEditing: boolean; +} + +/** + * A component that handles the keyword display for documents. + */ +@observer +export class KeywordBox extends ObservableReactComponent<KeywordBoxProps> { + @observable _currentInput: string = ''; + private height: number = 0; + private ref: React.RefObject<HTMLDivElement>; + + @computed + get currentScale() { + return NumCast((this._props.doc.embedContainer as Doc)?._freeform_scale, 1); + } + + @computed + get cur_height() { + return this.ref.current?.offsetHeight ? this.ref.current?.offsetHeight : 0; + } + + constructor(props: any) { + super(props); + makeObservable(this); + this.ref = React.createRef(); + + reaction( + () => this.cur_height, + () => { + this._props.doc[DocData].keywordHeight = this.height; + } + ); + } + + componentDidMount(): void { + this.height = this.ref.current?.offsetHeight ? this.ref.current?.offsetHeight : 0; + this._props.doc[DocData].keywordHeight = this.height; + } + + componentDidUpdate(prevProps: Readonly<KeywordBoxProps>): void { + this.height = this.ref.current?.offsetHeight ? this.ref.current?.offsetHeight : 0; + this._props.doc[DocData].keywordHeight = this.height; + } + + @action + setToEditing = () => { + this._props.isEditing = true; + }; + + @action + setToView = () => { + this._props.isEditing = false; + }; + + /** + * Gets the document associated with a keyword. + * @param keyword The keyword being searched for + * @returns A Doc containing keyword information + */ + getKeywordCollection = (keyword: string) => { + // Look for the keyword document. + for (const doc of DocListCast(Doc.ActiveDashboard!.myKeywordCollections)) { + if (doc.title === keyword) { + return doc; + } + } + + // If not contained, create a new document and add it to the active Dashboard's keyword list. + const keywordCollection = new Doc(); + keywordCollection.title = keyword; + keywordCollection[DocData].docs = new List<Doc>(); + keywordCollection.collections = new List<Doc>(); + if (Doc.ActiveDashboard) { + Doc.ActiveDashboard.myKeywordCollections = new List<Doc>([...DocListCast(Doc.ActiveDashboard.myKeywordCollections), keywordCollection]); + } + + return keywordCollection; + }; + + /** + * Adds the keyword to the document. + * @param keyword + */ + submitLabel = (keyword: string) => { + // If the active Dashboard does not have a keyword collection, create it. + if (Doc.ActiveDashboard && !Doc.ActiveDashboard.myKeywordCollections) { + Doc.ActiveDashboard.myKeywordCollections = new List<Doc>(); + } + + const submittedLabel = keyword.trim(); + if (submittedLabel && !this._props.doc![DocData][`${submittedLabel}`]) { + // If the keyword collection is not in active Dashboard, add it as a new doc, with the keyword as its title. + const keywordCollection = this.getKeywordCollection(submittedLabel); + + // If the document has no keywords field, create the field. + if (!this._props.doc[DocData].data_labels) { + this._props.doc[DocData].data_labels = new List<string>(); + } + + // If the document is of type COLLECTION, make it a smart collection, otherwise, add the keyword to the document. + if (this._props.doc.type === DocumentType.COL) { + keywordCollection.collections = new List<Doc>([...DocListCast(keywordCollection.collections), this._props.doc]); + + // Iterate through the keyword document's collections and add a copy of the document to each collection + for (const doc of DocListCast(keywordCollection[DocData].docs)) { + const newEmbedding = Doc.MakeEmbedding(doc); + this._props.doc[DocData].data = new List<Doc>([...DocListCast(this._props.doc[DocData].data), newEmbedding]); + newEmbedding.embedContainer = this._props.doc; + } + } else { + // Add this document to the keyword's collection of associated documents. + keywordCollection[DocData].docs = new List<Doc>([...DocListCast(keywordCollection[DocData].docs), this._props.doc]); + + // Iterate through the keyword document's collections and add a copy of the document to each collection + for (const collection of DocListCast(keywordCollection.collections)) { + const newEmbedding = Doc.MakeEmbedding(this._props.doc); + collection[DocData].data = new List<Doc>([...DocListCast(collection.data), newEmbedding]); + newEmbedding.embedContainer = collection; + } + } + + // Push the keyword to the document's keyword list field. + (this._props.doc![DocData].data_labels! as List<string>).push(submittedLabel); + this._props.doc![DocData][`${submittedLabel}`] = true; + this._currentInput = ''; // Clear the input box + } + }; + + @action + onInputChange = (e: React.ChangeEvent<HTMLInputElement>) => { + this._currentInput = e.target.value; + }; + + render() { + const keywordsList = this._props.doc[DocData].data_labels ? this._props.doc[DocData].data_labels : new List<string>(); + const seldoc = DocumentView.SelectedDocs().lastElement(); + if (SnappingManager.IsDragging || !(seldoc === this._props.doc) || !this._props.isEditing) { + setTimeout( + action(() => { + if ((keywordsList as List<string>).length === 0) { + this._props.doc[DocData].showLabels = false; + } + this.setToView(); + }) + ); + } + + return ( + <div + className="keywords-container" + ref={this.ref} + style={{ + transformOrigin: 'top left', + overflow: 'hidden', + transform: `scale(${1 / this.currentScale})`, + backgroundColor: this._props.isEditing ? Colors.LIGHT_GRAY : Colors.TRANSPARENT, + borderColor: this._props.isEditing ? Colors.BLACK : Colors.TRANSPARENT, + maxWidth: `400px`, + }}> + <div className="keywords-content"> + <div className="keywords-list"> + {(keywordsList as List<string>).map(keyword => { + return <KeywordItem key={Utils.GenerateGuid()} doc={this._props.doc} keyword={keyword} keywordDoc={this.getKeywordCollection(keyword)} setToEditing={this.setToEditing} isEditing={this._props.isEditing}></KeywordItem>; + })} + </div> + {this._props.isEditing ? ( + <div className="keyword-editing-box"> + <div className="keyword-input-box"> + <input + value={this._currentInput} + autoComplete="off" + onChange={this.onInputChange} + onKeyDown={e => { + e.key === 'Enter' ? this.submitLabel(this._currentInput) : null; + e.stopPropagation(); + }} + type="text" + placeholder="Input keywords for document..." + aria-label="keyword-input" + className="keyword-input" + style={{ width: '100%', borderRadius: '5px' }} + /> + </div> + {Doc.ActiveDashboard?.myKeywordCollections ? ( + <div className="keyword-suggestions-box"> + {DocListCast(Doc.ActiveDashboard?.myKeywordCollections).map(doc => { + const keyword = StrCast(doc.title); + return ( + <Button + style={{ margin: '2px 2px', border: '1px solid black', backgroundColor: 'lightblue', color: 'black' }} + text={keyword} + color={SnappingManager.userVariantColor} + tooltip={'Add existing keyword'} + onClick={() => { + this.submitLabel(keyword); + }} + key={Utils.GenerateGuid()} + /> + ); + })} + </div> + ) : ( + <div></div> + )} + <div className="keyword-buttons"> + <IconButton + tooltip={'Close Menu'} + onPointerDown={() => { + if ((keywordsList as List<string>).length === 0) { + this._props.doc[DocData].showLabels = false; + } else { + this.setToView(); + } + }} + icon={'x'} + style={{ width: '4px' }} + /> + </div> + </div> + ) : ( + <div></div> + )} + </div> + </div> + ); + } +} diff --git a/src/client/views/Main.tsx b/src/client/views/Main.tsx index 044162e4e..5a408f593 100644 --- a/src/client/views/Main.tsx +++ b/src/client/views/Main.tsx @@ -61,6 +61,9 @@ import { SummaryView } from './nodes/formattedText/SummaryView'; import { ImportElementBox } from './nodes/importBox/ImportElementBox'; import { PresBox, PresElementBox } from './nodes/trails'; import { SearchBox } from './search/SearchBox'; +import { ImageLabelBox } from './collections/collectionFreeForm/ImageLabelBox'; +import { FaceRecognitionHandler } from './search/FaceRecognitionHandler'; +import { FaceCollectionBox } from './collections/collectionFreeForm/FaceCollectionBox'; import { Node } from 'prosemirror-model'; import { EditorView } from 'prosemirror-view'; @@ -98,6 +101,7 @@ FieldLoader.ServerLoadStatus = { requested: 0, retrieved: 0, message: 'cache' }; new BranchingTrailManager({}); new PingManager(); new KeyManager(); + new FaceRecognitionHandler(); // initialize plugins and classes that require plugins CollectionDockingView.Init(TabDocView); @@ -134,6 +138,8 @@ FieldLoader.ServerLoadStatus = { requested: 0, retrieved: 0, message: 'cache' }; PresBox, PresElementBox, SearchBox, + ImageLabelBox, //Here! + FaceCollectionBox, FunctionPlotBox, InkingStroke, LinkBox, diff --git a/src/client/views/StyleProvider.scss b/src/client/views/StyleProvider.scss index ce00f6101..1d41697f5 100644 --- a/src/client/views/StyleProvider.scss +++ b/src/client/views/StyleProvider.scss @@ -53,3 +53,64 @@ .styleProvider-treeView-icon { opacity: 0; } + +.keywords-container { + display: flex; + flex-wrap: wrap; + flex-direction: column; + border: 1px solid; + border-radius: 4px; +} + +.keywords-list { + display: flex; + flex-wrap: wrap; +} + +.keyword { + padding: 5px 5px; + background-color: lightblue; + border: 1px solid black; + border-radius: 5px; + white-space: nowrap; + display: flex; + align-items: center; +} + +.keyword-suggestions-box { + display: flex; + flex-wrap: wrap; + margin: auto; + align-self: center; + width: 90%; + border: 1px solid black; + border-radius: 2px; + margin-top: 8px; +} + +.keyword-suggestion { + cursor: pointer; + padding: 1px 1px; + margin: 2px 2px; + background-color: lightblue; + border: 1px solid black; + border-radius: 5px; + white-space: nowrap; + display: flex; + align-items: center; +} + +.keyword-editing-box { + margin-top: 8px; +} + +.keyword-input-box { + margin: auto; + align-self: center; + width: 90%; +} + +.keyword-buttons { + margin-left: auto; + width: 10%; +} diff --git a/src/client/views/StyleProvider.tsx b/src/client/views/StyleProvider.tsx index 374f8ca3a..374399445 100644 --- a/src/client/views/StyleProvider.tsx +++ b/src/client/views/StyleProvider.tsx @@ -9,7 +9,9 @@ import { BsArrowDown, BsArrowDownUp, BsArrowUp } from 'react-icons/bs'; import { FaFilter } from 'react-icons/fa'; import { ClientUtils, DashColor, lightOrDark } from '../../ClientUtils'; import { Doc, Opt, StrListCast } from '../../fields/Doc'; +import { DocData } from '../../fields/DocSymbols'; import { Id } from '../../fields/FieldSymbols'; +import { List } from '../../fields/List'; import { ScriptField } from '../../fields/ScriptField'; import { BoolCast, Cast, DocCast, ImageCast, NumCast, ScriptCast, StrCast } from '../../fields/Types'; import { AudioAnnoState } from '../../server/SharedMediaTypes'; @@ -19,6 +21,7 @@ import { SnappingManager } from '../util/SnappingManager'; import { undoable, UndoManager } from '../util/UndoManager'; import { TreeSort } from './collections/TreeSort'; import { Colors } from './global/globalEnums'; +import { KeywordBox } from './KeywordBox'; import { DocumentView, DocumentViewProps } from './nodes/DocumentView'; import { FieldViewProps } from './nodes/FieldView'; import { StyleProp } from './StyleProp'; @@ -361,12 +364,20 @@ export function DefaultStyleProvider(doc: Opt<Doc>, props: Opt<FieldViewProps & </Tooltip> ); }; + const keywords = () => { + if (doc && doc![DocData].showLabels && (!doc[DocData].data_labels || (doc[DocData].data_labels as List<string>).length === 0)){ + return (<KeywordBox isEditing={true} doc={doc}></KeywordBox>) + } else if (doc && doc![DocData].data_labels && doc![DocData].showLabels) { + return (<KeywordBox isEditing={false} doc={doc}></KeywordBox>) + } + } return ( <> {paint()} {lock()} {filter()} {audio()} + {keywords()} </> ); } diff --git a/src/client/views/collections/CollectionSubView.tsx b/src/client/views/collections/CollectionSubView.tsx index 5782d407e..a6768ab35 100644 --- a/src/client/views/collections/CollectionSubView.tsx +++ b/src/client/views/collections/CollectionSubView.tsx @@ -1,4 +1,4 @@ -import { action, computed, makeObservable, observable } from 'mobx'; +import { action, computed, IReactionDisposer, makeObservable, observable, reaction } from 'mobx'; import * as React from 'react'; import * as rp from 'request-promise'; import { ClientUtils, returnFalse } from '../../../ClientUtils'; @@ -9,7 +9,7 @@ import { Id } from '../../../fields/FieldSymbols'; import { List } from '../../../fields/List'; import { listSpec } from '../../../fields/Schema'; import { ScriptField } from '../../../fields/ScriptField'; -import { BoolCast, Cast, ScriptCast, StrCast } from '../../../fields/Types'; +import { BoolCast, Cast, DocCast, ScriptCast, StrCast } from '../../../fields/Types'; import { WebField } from '../../../fields/URLField'; import { GetEffectiveAcl, TraceMobx } from '../../../fields/util'; import { GestureUtils } from '../../../pen-gestures/GestureUtils'; diff --git a/src/client/views/collections/CollectionView.tsx b/src/client/views/collections/CollectionView.tsx index c9ab5f661..ab93abab6 100644 --- a/src/client/views/collections/CollectionView.tsx +++ b/src/client/views/collections/CollectionView.tsx @@ -17,6 +17,7 @@ import { ViewBoxAnnotatableComponent } from '../DocComponent'; import { FieldView } from '../nodes/FieldView'; import { OpenWhere } from '../nodes/OpenWhere'; import { CollectionCalendarView } from './CollectionCalendarView'; +import { CollectionCardView } from './CollectionCardDeckView'; import { CollectionCarousel3DView } from './CollectionCarousel3DView'; import { CollectionCarouselView } from './CollectionCarouselView'; import { CollectionDockingView } from './CollectionDockingView'; @@ -33,7 +34,6 @@ import { CollectionLinearView } from './collectionLinear'; import { CollectionMulticolumnView } from './collectionMulticolumn/CollectionMulticolumnView'; import { CollectionMultirowView } from './collectionMulticolumn/CollectionMultirowView'; import { CollectionSchemaView } from './collectionSchema/CollectionSchemaView'; -import { CollectionCardView } from './CollectionCardDeckView'; @observer export class CollectionView extends ViewBoxAnnotatableComponent<CollectionViewProps>() { diff --git a/src/client/views/collections/collectionFreeForm/FaceCollectionBox.scss b/src/client/views/collections/collectionFreeForm/FaceCollectionBox.scss new file mode 100644 index 000000000..480d109c8 --- /dev/null +++ b/src/client/views/collections/collectionFreeForm/FaceCollectionBox.scss @@ -0,0 +1,74 @@ +.face-document-item { + background: #555555; + margin-top: 10px; + margin-bottom: 10px; + padding: 10px; + border-radius: 10px; + position: relative; + + h1 { + color: white; + font-size: 24px; + text-align: center; + } + + .face-collection-buttons { + position: absolute; + top: 10px; + right: 10px; + } + + .face-document-image-container { + display: flex; + justify-content: center; + flex-wrap: wrap; + + .image-wrapper { + position: relative; + width: 70px; + height: 70px; + margin: 10px; + display: flex; + align-items: center; // Center vertically + justify-content: center; // Center horizontally + + img { + width: 100%; + height: 100%; + object-fit: cover; // This ensures the image covers the container without stretching + border-radius: 5px; + border: 2px solid white; + transition: border-color 0.4s; + + &:hover { + border-color: orange; // Change this to your desired hover border color + } + } + + .remove-item { + position: absolute; + bottom: -5; + right: -5; + background-color: rgba(0, 0, 0, 0.5); // Optional: to add a background behind the icon for better visibility + border-radius: 30%; + width: 10px; // Adjust size as needed + height: 10px; // Adjust size as needed + display: flex; + align-items: center; + justify-content: center; + } + } + + // img { + // max-width: 60px; + // margin: 10px; + // border-radius: 5px; + // border: 2px solid white; + // transition: 0.4s; + + // &:hover { + // border-color: orange; + // } + // } + } +} diff --git a/src/client/views/collections/collectionFreeForm/FaceCollectionBox.tsx b/src/client/views/collections/collectionFreeForm/FaceCollectionBox.tsx new file mode 100644 index 000000000..d5a2809dc --- /dev/null +++ b/src/client/views/collections/collectionFreeForm/FaceCollectionBox.tsx @@ -0,0 +1,200 @@ +import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; +import { IconButton, Size } from 'browndash-components'; +import * as faceapi from 'face-api.js'; +import { FaceMatcher } from 'face-api.js'; +import 'ldrs/ring'; +import { action, computed, makeObservable, observable } from 'mobx'; +import { observer } from 'mobx-react'; +import React from 'react'; +import { Utils } from '../../../../Utils'; +import { Doc, DocListCast } from '../../../../fields/Doc'; +import { DocData } from '../../../../fields/DocSymbols'; +import { Id } from '../../../../fields/FieldSymbols'; +import { List } from '../../../../fields/List'; +import { listSpec } from '../../../../fields/Schema'; +import { Cast, ImageCast, StrCast } from '../../../../fields/Types'; +import { DocumentType } from '../../../documents/DocumentTypes'; +import { Docs } from '../../../documents/Documents'; +import { DragManager } from '../../../util/DragManager'; +import { SnappingManager } from '../../../util/SnappingManager'; +import { undoable } from '../../../util/UndoManager'; +import { ViewBoxBaseComponent } from '../../DocComponent'; +import { ObservableReactComponent } from '../../ObservableReactComponent'; +import { DocumentView } from '../../nodes/DocumentView'; +import { FieldView, FieldViewProps } from '../../nodes/FieldView'; +import './FaceCollectionBox.scss'; +import { MarqueeOptionsMenu } from './MarqueeOptionsMenu'; +import { FaceRecognitionHandler } from '../../search/FaceRecognitionHandler'; + +interface FaceDocumentProps { + faceDoc: Doc; +} + +/** + * A componenent to visually represent a Face Document. + */ +@observer +export class FaceDocumentItem extends ObservableReactComponent<FaceDocumentProps> { + private _dropDisposer?: DragManager.DragDropDisposer; + + constructor(props: FaceDocumentProps) { + super(props); + makeObservable(this); + } + + @observable _displayImages: boolean = true; + + protected createDropTarget = (ele: HTMLDivElement) => { + this._dropDisposer?.(); + ele && (this._dropDisposer = DragManager.MakeDropTarget(ele, this.onInternalDrop.bind(this), this._props.faceDoc)); + }; + + protected onInternalDrop(e: Event, de: DragManager.DropEvent): boolean { + const { docDragData } = de.complete; + if (docDragData) { + const filteredDocs = docDragData.droppedDocuments.filter(doc => doc.type === DocumentType.IMG); + filteredDocs.forEach(doc => { + // If the current Face Document has no items, and the doc has more than one face descriptor, don't let the user add the document first. + if ((this._props.faceDoc[DocData].face_descriptors as List<List<number>>).length === 0 && (doc[DocData][FaceRecognitionHandler.FacesField(doc)] as List<List<number>>).length > 1) { + alert('Cannot add a document with multiple faces as the first item!'); + } else { + // Loop through the documents' face descriptors. + // Choose the face with the smallest distance to add. + const float32Array = (this._props.faceDoc[DocData].face_descriptors as List<List<number>>).map(faceDescriptor => new Float32Array(Array.from(faceDescriptor))); + const labeledFaceDescriptor = new faceapi.LabeledFaceDescriptors(StrCast(this._props.faceDoc[DocData].face_label), float32Array); + const faceDescriptors: faceapi.LabeledFaceDescriptors[] = [labeledFaceDescriptor]; + + const faceMatcher = new FaceMatcher(faceDescriptors, 1); + let cur_lowest_distance = 1; + let cur_matching_face = new List<number>(); + + (doc[DocData][FaceRecognitionHandler.FacesField(doc)] as List<List<number>>).forEach(face => { + // If the face has the current lowest distance, mark it as such + // Once that lowest distance is found, add the face descriptor to the faceDoc, and add the associated doc + const convered_32_array: Float32Array = new Float32Array(Array.from(face)); + const match = faceMatcher.matchDescriptor(convered_32_array); + + if (match.distance < cur_lowest_distance) { + cur_lowest_distance = match.distance; + cur_matching_face = face; + } + }); + + const faceFieldKey = FaceRecognitionHandler.FaceField(doc, this._props.faceDoc); + if (doc[DocData][faceFieldKey]) { + Cast(doc[DocData][faceFieldKey], listSpec('number'), null).push(cur_matching_face as unknown as number); // items are lists of numbers, not numbers, but type system can't handle that + } else { + doc[DocData][faceFieldKey] = new List<List<number>>([cur_matching_face]); + } + + Doc.AddDocToList(this._props.faceDoc[DocData], 'face_docList', doc); + Cast(this._props.faceDoc[DocData].face_descriptors, listSpec('number'), null).push(cur_matching_face as unknown as number); // items are lists of numbers, not numbers, but type system can't handle that + } + }); + return false; + } + return false; + } + + /** + * Toggles whether a Face Document displays its associated docs. + */ + @action + onDisplayClick() { + this._displayImages = !this._displayImages; + } + + /** + * Deletes a Face Document. + */ + deleteFaceDocument = undoable(() => { + if (Doc.ActiveDashboard) { + Doc.RemoveDocFromList(Doc.ActiveDashboard[DocData], 'faceDocuments', this._props.faceDoc); + } + }, 'remove face'); + + /** + * Deletes a document from a Face Document's associated docs list. + * @param doc + */ + @action + deleteAssociatedDoc = (doc: Doc) => { + this._props.faceDoc[DocData].face_descriptors = new List<List<number>>( + (this._props.faceDoc[DocData].face_descriptors as List<List<number>>).filter(fd => !(doc[DocData][FaceRecognitionHandler.FaceField(doc, this._props.faceDoc)] as List<List<number>>).includes(fd)) + ); + doc[DocData][FaceRecognitionHandler.FaceField(doc, this._props.faceDoc)] = new List<List<number>>(); + Doc.RemoveDocFromList(this._props.faceDoc[DocData], 'face_docList', doc); + }; + + render() { + return ( + <div className="face-document-item" ref={ele => this.createDropTarget(ele!)}> + <div className="face-collection-buttons"> + <IconButton tooltip="Delete Face From Collection" onPointerDown={this.deleteFaceDocument} icon={'x'} style={{ width: '4px' }} size={Size.XSMALL} /> + </div> + <div className="face-document-top"> + <h1>{StrCast(this._props.faceDoc[DocData].face_label)}</h1> + </div> + <IconButton + tooltip="See image information" + onPointerDown={() => this.onDisplayClick()} + icon={<FontAwesomeIcon icon={this._displayImages ? 'caret-up' : 'caret-down'} />} + color={MarqueeOptionsMenu.Instance.userColor} + style={{ width: '19px' }} + /> + {this._displayImages ? ( + <div className="face-document-image-container"> + {DocListCast(this._props.faceDoc[DocData].face_docList).map(doc => { + const [name, type] = ImageCast(doc[Doc.LayoutFieldKey(doc)]).url.href.split('.'); + return ( + <div className="image-wrapper" key={Utils.GenerateGuid()}> + <img onClick={() => DocumentView.showDocument(doc, { willZoomCentered: true })} style={{ maxWidth: '60px', margin: '10px' }} src={`${name}_o.${type}`} /> + <div className="remove-item"> + <IconButton tooltip={'Remove Doc From Face Collection'} onPointerDown={() => this.deleteAssociatedDoc(doc)} icon={'x'} style={{ width: '4px' }} size={Size.XSMALL} /> + </div> + </div> + ); + })} + </div> + ) : null} + </div> + ); + } +} + +@observer +export class FaceCollectionBox extends ViewBoxBaseComponent<FieldViewProps>() { + public static LayoutString(fieldKey: string) { + return FieldView.LayoutString(FaceCollectionBox, fieldKey); + } + + public static Instance: FaceCollectionBox; + + @computed get currentDocs() { + if (Doc.ActiveDashboard) { + return DocListCast(Doc.ActiveDashboard[DocData].faceDocuments); + } + return []; + } + + constructor(props: FieldViewProps) { + super(props); + makeObservable(this); + FaceCollectionBox.Instance = this; + } + + render() { + return ( + <div className="searchBox-container" style={{ pointerEvents: 'all', color: SnappingManager.userColor, background: SnappingManager.userBackgroundColor }}> + {this.currentDocs.map(doc => ( + <FaceDocumentItem key={doc[Id]} faceDoc={doc} /> + ))} + </div> + ); + } +} + +Docs.Prototypes.TemplateMap.set(DocumentType.FACECOLLECTION, { + layout: { view: FaceCollectionBox, dataField: 'data' }, + options: { acl: '', _width: 400 }, +}); diff --git a/src/client/views/collections/collectionFreeForm/ImageLabelBox.scss b/src/client/views/collections/collectionFreeForm/ImageLabelBox.scss new file mode 100644 index 000000000..819c72760 --- /dev/null +++ b/src/client/views/collections/collectionFreeForm/ImageLabelBox.scss @@ -0,0 +1,85 @@ +.image-box-container { + display: flex; + align-items: center; + justify-content: center; + width: 100%; + height: 100%; + font-size: 10px; + line-height: 1; + background: none; + z-index: 1000; + padding: 0px; + overflow: auto; + cursor: default; +} + +.image-label-list { + display: flex; + flex-direction: column; + align-items: center; // Centers the content vertically in the flex container + width: 100%; + + > div { + display: flex; + justify-content: space-between; // Puts the content and delete button on opposite ends + align-items: center; + width: 100%; + margin-top: 8px; // Adds space between label rows + background-color: black; + + p { + text-align: center; // Centers the text of the paragraph + font-size: large; + vertical-align: middle; + margin-left: 10px; + } + + .IconButton { + // Styling for the delete button + margin-left: auto; // Pushes the button to the far right + } + } +} + +.image-information-list { + display: flex; + flex-direction: column; + align-items: center; + width: 100%; + margin-top: 10px; +} + +.image-information { + border: 1px solid; + width: 100%; + display: inline-flex; + flex-direction: column; + justify-content: center; + align-items: center; + overflow: hidden; + padding: 2px; + overflow-x: auto; + overflow-y: auto; + + img { + max-width: 200px; + max-height: 200px; + width: auto; + height: auto; + } +} + +.image-information-labels { + display: flex; + flex-direction: column; + justify-content: center; + align-items: center; + + .image-label { + margin-top: 5px; + margin-bottom: 5px; + padding: 3px; + border-radius: 2px; + border: solid 1px; + } +} diff --git a/src/client/views/collections/collectionFreeForm/ImageLabelBox.tsx b/src/client/views/collections/collectionFreeForm/ImageLabelBox.tsx new file mode 100644 index 000000000..421b5d0a6 --- /dev/null +++ b/src/client/views/collections/collectionFreeForm/ImageLabelBox.tsx @@ -0,0 +1,343 @@ +import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; +import { Colors, IconButton } from 'browndash-components'; +import { action, computed, makeObservable, observable, reaction } from 'mobx'; +import { observer } from 'mobx-react'; +import React from 'react'; +import { Doc, NumListCast, Opt } from '../../../../fields/Doc'; +import { Docs } from '../../../documents/Documents'; +import { DocumentType } from '../../../documents/DocumentTypes'; +import { ViewBoxBaseComponent } from '../../DocComponent'; +import { FieldView, FieldViewProps } from '../../nodes/FieldView'; +import { MarqueeOptionsMenu } from './MarqueeOptionsMenu'; +import './ImageLabelBox.scss'; +import { MainView } from '../../MainView'; +import 'ldrs/ring'; +import { ring } from 'ldrs'; +import { SnappingManager } from '../../../util/SnappingManager'; +import { ImageCast } from '../../../../fields/Types'; +import { DocData } from '../../../../fields/DocSymbols'; +import { SettingsManager } from '../../../util/SettingsManager'; +import { CollectionCardView } from '../CollectionCardDeckView'; +import { gptGetEmbedding, gptImageLabel } from '../../../apis/gpt/GPT'; +import { numberRange, Utils } from '../../../../Utils'; +import { List } from '../../../../fields/List'; +import { DragManager } from '../../../util/DragManager'; +import { OpenWhere } from '../../nodes/OpenWhere'; +import similarity from 'compute-cosine-similarity'; +import { DocumentView } from '../../nodes/DocumentView'; + +export class ImageInformationItem {} + +export class ImageLabelBoxData { + static _instance: ImageLabelBoxData; + @observable _docs: Doc[] = []; + @observable _labelGroups: string[] = []; + + constructor() { + makeObservable(this); + ImageLabelBoxData._instance = this; + } + public static get Instance() { + return ImageLabelBoxData._instance ?? new ImageLabelBoxData(); + } + + @action + public setData = (docs: Doc[]) => { + this._docs = docs; + }; + + @action + addLabel = (label: string) => { + label = label.toUpperCase().trim(); + if (label.length > 0) { + if (!this._labelGroups.includes(label)) { + this._labelGroups = [...this._labelGroups, label]; + } + } + }; + + @action + removeLabel = (label: string) => { + const labelUp = label.toUpperCase(); + this._labelGroups = this._labelGroups.filter(group => group !== labelUp); + }; +} + +@observer +export class ImageLabelBox extends ViewBoxBaseComponent<FieldViewProps>() { + public static LayoutString(fieldKey: string) { + return FieldView.LayoutString(ImageLabelBox, fieldKey); + } + + private _dropDisposer?: DragManager.DragDropDisposer; + public static Instance: ImageLabelBox; + private _inputRef = React.createRef<HTMLInputElement>(); + @observable _loading: boolean = false; + private _currentLabel: string = ''; + + protected createDropTarget = (ele: HTMLDivElement) => { + this._dropDisposer?.(); + ele && (this._dropDisposer = DragManager.MakeDropTarget(ele, this.onInternalDrop.bind(this), this.layoutDoc)); + }; + + protected onInternalDrop(e: Event, de: DragManager.DropEvent): boolean { + const { docDragData } = de.complete; + if (docDragData) { + ImageLabelBoxData.Instance.setData(ImageLabelBoxData.Instance._docs.concat(docDragData.droppedDocuments)); + return false; + } + return false; + } + + @computed get _labelGroups() { + return ImageLabelBoxData.Instance._labelGroups; + } + + @computed get _selectedImages() { + // return DocListCast(this.dataDoc.data); + return ImageLabelBoxData.Instance._docs; + } + @observable _displayImageInformation: boolean = false; + + constructor(props: any) { + super(props); + makeObservable(this); + ring.register(); + ImageLabelBox.Instance = this; + } + + // ImageLabelBox.Instance.setData() + /** + * This method is called when the SearchBox component is first mounted. When the user opens + * the search panel, the search input box is automatically selected. This allows the user to + * type in the search input box immediately, without needing clicking on it first. + */ + componentDidMount() { + this.classifyImagesInBox(); + reaction( + () => this._selectedImages, + () => this.classifyImagesInBox() + ); + } + + @action + groupImages = () => { + this.groupImagesInBox(); + }; + + @action + startLoading = () => { + this._loading = true; + }; + + @action + endLoading = () => { + this._loading = false; + }; + + @action + toggleDisplayInformation = () => { + this._displayImageInformation = !this._displayImageInformation; + if (this._displayImageInformation) { + this._selectedImages.forEach(doc => (doc[DocData].showLabels = true)); + } else { + this._selectedImages.forEach(doc => (doc[DocData].showLabels = false)); + } + }; + + @action + submitLabel = () => { + const input = document.getElementById('new-label') as HTMLInputElement; + ImageLabelBoxData.Instance.addLabel(this._currentLabel); + this._currentLabel = ''; + input.value = ''; + }; + + onInputChange = action((e: React.ChangeEvent<HTMLInputElement>) => { + this._currentLabel = e.target.value; + }); + + classifyImagesInBox = async () => { + this.startLoading(); + + // Converts the images into a Base64 format, afterwhich the information is sent to GPT to label them. + + const imageInfos = this._selectedImages.map(async doc => { + if (!doc[DocData].data_labels) { + const [name, type] = ImageCast(doc[Doc.LayoutFieldKey(doc)]).url.href.split('.'); + return CollectionCardView.imageUrlToBase64(`${name}_o.${type}`).then(hrefBase64 => + !hrefBase64 ? undefined : + gptImageLabel(hrefBase64).then(labels => + ({ doc, labels }))) ; // prettier-ignore + } + }); + + (await Promise.all(imageInfos)).forEach(imageInfo => { + if (imageInfo) { + imageInfo.doc[DocData].data_labels = new List<string>(); + + const labels = imageInfo.labels.split('\n'); + labels.forEach(label => { + label = label.replace(/^\d+\.\s*|-|\*/, '').trim(); + console.log(label); + imageInfo.doc[DocData][`${label}`] = true; + (imageInfo.doc[DocData].data_labels as List<string>).push(label); + }); + } + }); + + this.endLoading(); + }; + + /** + * Groups images to most similar labels. + */ + groupImagesInBox = action(async () => { + this.startLoading(); + + for (const doc of this._selectedImages) { + for (let index = 0; index < (doc[DocData].data_labels as List<string>).length; index++) { + const label = (doc[DocData].data_labels as List<string>)[index]; + const embedding = await gptGetEmbedding(label); + doc[DocData][`data_labels_embedding_${index + 1}`] = new List<number>(embedding); + } + } + + const labelToEmbedding = new Map<string, number[]>(); + // Create embeddings for the labels. + await Promise.all(this._labelGroups.map(async label => gptGetEmbedding(label).then(labelEmbedding => labelToEmbedding.set(label, labelEmbedding)))); + + // For each image, loop through the labels, and calculate similarity. Associate it with the + // most similar one. + this._selectedImages.forEach(doc => { + const embedLists = numberRange((doc[DocData].data_labels as List<string>).length).map(n => Array.from(NumListCast(doc[DocData][`data_labels_embedding_${n + 1}`]))); + const bestEmbedScore = (embedding: Opt<number[]>) => Math.max(...embedLists.map((l, index) => (embedding && similarity(Array.from(embedding), l)!) || 0)); + const {label: mostSimilarLabelCollect} = + this._labelGroups.map(label => ({ label, similarityScore: bestEmbedScore(labelToEmbedding.get(label)) })) + .reduce((prev, cur) => cur.similarityScore < 0.3 || cur.similarityScore <= prev.similarityScore ? prev: cur, + { label: '', similarityScore: 0, }); // prettier-ignore + doc[DocData].data_label = mostSimilarLabelCollect; // The label most similar to the image's contents. + }); + + this.endLoading(); + + if (this._selectedImages) { + MarqueeOptionsMenu.Instance.groupImages(); + } + + MainView.Instance.closeFlyout(); + }); + + render() { + if (this._loading) { + return ( + <div className="image-box-container" style={{ pointerEvents: 'all', color: SnappingManager.userColor, background: SnappingManager.userBackgroundColor }}> + <l-ring size="60" color="white" /> + </div> + ); + } + + if (this._selectedImages.length === 0) { + return ( + <div className="searchBox-container" style={{ pointerEvents: 'all', color: SnappingManager.userColor, background: SnappingManager.userBackgroundColor }} ref={ele => this.createDropTarget(ele!)}> + <p style={{ fontSize: 'large' }}>In order to classify and sort images, marquee select the desired images and press the 'Classify and Sort Images' button. Then, add the desired groups for the images to be put in.</p> + </div> + ); + } + + return ( + <div className="searchBox-container" style={{ pointerEvents: 'all', color: SnappingManager.userColor, background: SnappingManager.userBackgroundColor }} ref={ele => this.createDropTarget(ele!)}> + <div className="searchBox-bar" style={{ pointerEvents: 'all', color: SnappingManager.userColor, background: SnappingManager.userBackgroundColor }}> + <IconButton + tooltip={'See image information'} + onPointerDown={this.toggleDisplayInformation} + icon={this._displayImageInformation ? <FontAwesomeIcon icon="caret-up" /> : <FontAwesomeIcon icon="caret-down" />} + color={MarqueeOptionsMenu.Instance.userColor} + style={{ width: '19px' }} + /> + <input + defaultValue="" + autoComplete="off" + onChange={this.onInputChange} + onKeyDown={e => { + e.key === 'Enter' ? this.submitLabel() : null; + e.stopPropagation(); + }} + type="text" + placeholder="Input groups for images to be put into..." + aria-label="label-input" + id="new-label" + className="searchBox-input" + style={{ width: '100%', borderRadius: '5px' }} + ref={this._inputRef} + /> + <IconButton + tooltip={'Add a label'} + onPointerDown={() => { + const input = document.getElementById('new-label') as HTMLInputElement; + ImageLabelBoxData.Instance.addLabel(this._currentLabel); + this._currentLabel = ''; + input.value = ''; + }} + icon={<FontAwesomeIcon icon="plus" />} + color={MarqueeOptionsMenu.Instance.userColor} + style={{ width: '19px' }} + /> + {this._labelGroups.length > 0 ? <IconButton tooltip={'Group Images'} onPointerDown={this.groupImages} icon={<FontAwesomeIcon icon="object-group" />} color={Colors.MEDIUM_BLUE} style={{ width: '19px' }} /> : <div></div>} + </div> + <div> + <div className="image-label-list"> + {this._labelGroups.map(group => { + return ( + <div key={Utils.GenerateGuid()}> + <p style={{ color: MarqueeOptionsMenu.Instance.userColor }}>{group}</p> + <IconButton + tooltip={'Remove Label'} + onPointerDown={() => { + ImageLabelBoxData.Instance.removeLabel(group); + }} + icon={'x'} + color={MarqueeOptionsMenu.Instance.userColor} + style={{ width: '8px' }} + /> + </div> + ); + })} + </div> + </div> + {this._displayImageInformation ? ( + <div className="image-information-list"> + {this._selectedImages.map(doc => { + const [name, type] = ImageCast(doc[Doc.LayoutFieldKey(doc)]).url.href.split('.'); + return ( + <div className="image-information" style={{ borderColor: SettingsManager.userColor }} key={Utils.GenerateGuid()}> + <img + src={`${name}_o.${type}`} + onClick={async () => { + await DocumentView.showDocument(doc, { willZoomCentered: true }); + }}></img> + <div className="image-information-labels" onClick={() => this._props.addDocTab(doc, OpenWhere.addRightKeyvalue)}> + {(doc[DocData].data_labels as List<string>).map(label => { + return ( + <div key={Utils.GenerateGuid()} className="image-label" style={{ backgroundColor: SettingsManager.userVariantColor, borderColor: SettingsManager.userColor }}> + {label} + </div> + ); + })} + </div> + </div> + ); + })} + </div> + ) : ( + <div></div> + )} + </div> + ); + } +} + +Docs.Prototypes.TemplateMap.set(DocumentType.IMAGEGROUPER, { + layout: { view: ImageLabelBox, dataField: 'data' }, + options: { acl: '', _width: 400 }, +}); diff --git a/src/client/views/collections/collectionFreeForm/ImageLabelHandler.tsx b/src/client/views/collections/collectionFreeForm/ImageLabelHandler.tsx index 7f27c6b5c..73befb205 100644 --- a/src/client/views/collections/collectionFreeForm/ImageLabelHandler.tsx +++ b/src/client/views/collections/collectionFreeForm/ImageLabelHandler.tsx @@ -77,7 +77,7 @@ export class ImageLabelHandler extends ObservableReactComponent<{}> { }}> <div> <IconButton tooltip={'Cancel'} onPointerDown={this.hideLabelhandler} icon={<FontAwesomeIcon icon="eye-slash" />} color={MarqueeOptionsMenu.Instance.userColor} style={{ width: '19px' }} /> - <input aria-label="label-input" id="new-label" type="text" style={{ color: 'black' }} /> + <input aria-label="label-input" id="new-label" type="text" placeholder="Input a classification" style={{ color: 'black' }} /> <IconButton tooltip={'Add Label'} onPointerDown={() => { diff --git a/src/client/views/collections/collectionFreeForm/MarqueeOptionsMenu.tsx b/src/client/views/collections/collectionFreeForm/MarqueeOptionsMenu.tsx index f02cd9d45..44c916ab9 100644 --- a/src/client/views/collections/collectionFreeForm/MarqueeOptionsMenu.tsx +++ b/src/client/views/collections/collectionFreeForm/MarqueeOptionsMenu.tsx @@ -18,10 +18,10 @@ export class MarqueeOptionsMenu extends AntimodeMenu<AntimodeMenuProps> { public showMarquee: () => void = unimplementedFunction; public hideMarquee: () => void = unimplementedFunction; public pinWithView: (e: KeyboardEvent | React.PointerEvent | undefined) => void = unimplementedFunction; - public classifyImages: (e: React.MouseEvent | undefined) => void = unimplementedFunction; + public classifyImages: () => void = unimplementedFunction; public groupImages: () => void = unimplementedFunction; public isShown = () => this._opacity > 0; - constructor(props: any) { + constructor(props: AntimodeMenuProps) { super(props); makeObservable(this); MarqueeOptionsMenu.Instance = this; @@ -39,7 +39,7 @@ export class MarqueeOptionsMenu extends AntimodeMenu<AntimodeMenuProps> { <IconButton tooltip="Summarize Documents" onPointerDown={this.summarize} icon={<FontAwesomeIcon icon="compress-arrows-alt" />} color={this.userColor} /> <IconButton tooltip="Delete Documents" onPointerDown={this.delete} icon={<FontAwesomeIcon icon="trash-alt" />} color={this.userColor} /> <IconButton tooltip="Pin selected region" onPointerDown={this.pinWithView} icon={<FontAwesomeIcon icon="map-pin" />} color={this.userColor} /> - <IconButton tooltip="Classify Images" onPointerDown={this.classifyImages} icon={<FontAwesomeIcon icon="object-group" />} color={this.userColor} /> + <IconButton tooltip="Classify and Sort Images" onPointerDown={this.classifyImages} icon={<FontAwesomeIcon icon="object-group" />} color={this.userColor} /> </> ); return this.getElement(buttons); diff --git a/src/client/views/collections/collectionFreeForm/MarqueeView.tsx b/src/client/views/collections/collectionFreeForm/MarqueeView.tsx index dc15c83c5..6cc75aa4b 100644 --- a/src/client/views/collections/collectionFreeForm/MarqueeView.tsx +++ b/src/client/views/collections/collectionFreeForm/MarqueeView.tsx @@ -1,28 +1,24 @@ -/* eslint-disable jsx-a11y/no-static-element-interactions */ -import similarity from 'compute-cosine-similarity'; import { action, computed, makeObservable, observable } from 'mobx'; import { observer } from 'mobx-react'; import * as React from 'react'; import { ClientUtils, lightOrDark, returnFalse } from '../../../../ClientUtils'; -import { intersectRect, numberRange } from '../../../../Utils'; -import { Doc, NumListCast, Opt } from '../../../../fields/Doc'; +import { intersectRect } from '../../../../Utils'; +import { Doc, DocListCast, Opt } from '../../../../fields/Doc'; import { AclAdmin, AclAugment, AclEdit, DocData } from '../../../../fields/DocSymbols'; import { Id } from '../../../../fields/FieldSymbols'; -import { InkData, InkField, InkTool } from '../../../../fields/InkField'; +import { InkTool } from '../../../../fields/InkField'; import { List } from '../../../../fields/List'; -import { RichTextField } from '../../../../fields/RichTextField'; -import { Cast, FieldValue, ImageCast, NumCast, StrCast } from '../../../../fields/Types'; +import { Cast, NumCast, StrCast } from '../../../../fields/Types'; import { ImageField } from '../../../../fields/URLField'; import { GetEffectiveAcl } from '../../../../fields/util'; -import { gptGetEmbedding, gptImageLabel } from '../../../apis/gpt/GPT'; -import { CognitiveServices } from '../../../cognitive_services/CognitiveServices'; import { DocUtils } from '../../../documents/DocUtils'; -import { CollectionViewType, DocumentType } from '../../../documents/DocumentTypes'; +import { DocumentType } from '../../../documents/DocumentTypes'; import { Docs, DocumentOptions } from '../../../documents/Documents'; import { SnappingManager, freeformScrollMode } from '../../../util/SnappingManager'; import { Transform } from '../../../util/Transform'; import { UndoManager, undoBatch } from '../../../util/UndoManager'; import { ContextMenu } from '../../ContextMenu'; +import { MainView } from '../../MainView'; import { ObservableReactComponent } from '../../ObservableReactComponent'; import { MarqueeViewBounds } from '../../PinFuncs'; import { PreviewCursor } from '../../PreviewCursor'; @@ -30,10 +26,8 @@ import { DocumentView } from '../../nodes/DocumentView'; import { OpenWhere } from '../../nodes/OpenWhere'; import { pasteImageBitmap } from '../../nodes/WebBoxRenderer'; import { FormattedTextBox } from '../../nodes/formattedText/FormattedTextBox'; -import { CollectionCardView } from '../CollectionCardDeckView'; import { SubCollectionViewProps } from '../CollectionSubView'; -import { CollectionFreeFormView } from './CollectionFreeFormView'; -import { ImageLabelHandler } from './ImageLabelHandler'; +import { ImageLabelBoxData } from './ImageLabelBox'; import { MarqueeOptionsMenu } from './MarqueeOptionsMenu'; import './MarqueeView.scss'; @@ -53,6 +47,9 @@ interface MarqueeViewProps { slowLoadDocuments: (files: File[] | string, options: DocumentOptions, generatedDocuments: Doc[], text: string, completed: ((doc: Doc[]) => void) | undefined, addDocument: (doc: Doc | Doc[]) => boolean) => Promise<void>; } +/** + * A component that deals with the marquee select in the freeform canvas. + */ @observer export class MarqueeView extends ObservableReactComponent<SubCollectionViewProps & MarqueeViewProps> { public static CurViewBounds(pinDoc: Doc, panelWidth: number, panelHeight: number) { @@ -60,9 +57,12 @@ export class MarqueeView extends ObservableReactComponent<SubCollectionViewProps return { left: NumCast(pinDoc._freeform_panX) - panelWidth / 2 / ps, top: NumCast(pinDoc._freeform_panY) - panelHeight / 2 / ps, width: panelWidth / ps, height: panelHeight / ps }; } - constructor(props: any) { + static Instance: MarqueeView; + + constructor(props: SubCollectionViewProps & MarqueeViewProps) { super(props); makeObservable(this); + MarqueeView.Instance = this; } private _commandExecuted = false; @@ -156,6 +156,7 @@ export class MarqueeView extends ObservableReactComponent<SubCollectionViewProps } else if (e.key === 'b' && e.ctrlKey) { document.body.focus(); // so that we can access the clipboard without an error setTimeout(() => + // eslint-disable-next-line @typescript-eslint/no-explicit-any pasteImageBitmap((data: any, error: any) => { error && console.log(error); data && @@ -430,32 +431,14 @@ export class MarqueeView extends ObservableReactComponent<SubCollectionViewProps /** * Classifies images and assigns the labels as document fields. - * TODO: Turn into lists of labels instead of individual fields. */ @undoBatch - classifyImages = action(async (e: React.MouseEvent | undefined) => { - this._selectedDocs = this.marqueeSelect(false, DocumentType.IMG); - - const imageInfos = this._selectedDocs.map(async doc => { - const [name, type] = ImageCast(doc[Doc.LayoutFieldKey(doc)]).url.href.split('.'); - return CollectionCardView.imageUrlToBase64(`${name}_o.${type}`).then(hrefBase64 => - !hrefBase64 ? undefined : - gptImageLabel(hrefBase64).then(labels => - Promise.all(labels.split('\n').map(label => gptGetEmbedding(label))).then(embeddings => - ({ doc, embeddings, labels }))) ); // prettier-ignore - }); - - (await Promise.all(imageInfos)).forEach(imageInfo => { - if (imageInfo && Array.isArray(imageInfo.embeddings)) { - imageInfo.doc[DocData].data_labels = imageInfo.labels; - numberRange(3).forEach(n => { - imageInfo.doc[`data_labels_embedding_${n + 1}`] = new List<number>(imageInfo.embeddings[n]); - }); - } - }); - - if (e) { - ImageLabelHandler.Instance.displayLabelHandler(e.pageX, e.pageY); + classifyImages = action(async () => { + const groupButton = DocListCast(Doc.MyLeftSidebarMenu.data).find(d => d.target === Doc.MyImageGrouper); + if (groupButton) { + this._selectedDocs = this.marqueeSelect(false, DocumentType.IMG); + ImageLabelBoxData.Instance.setData(this._selectedDocs); + MainView.Instance.expandFlyout(groupButton); } }); @@ -464,93 +447,44 @@ export class MarqueeView extends ObservableReactComponent<SubCollectionViewProps */ @undoBatch groupImages = action(async () => { - const labelGroups = ImageLabelHandler.Instance._labelGroups; - const labelToEmbedding = new Map<string, number[]>(); - // Create embeddings for the labels. - await Promise.all(labelGroups.map(async label => gptGetEmbedding(label).then(labelEmbedding => labelToEmbedding.set(label, labelEmbedding)))); - - // For each image, loop through the labels, and calculate similarity. Associate it with the - // most similar one. - this._selectedDocs.forEach(doc => { - const embedLists = numberRange(3).map(n => Array.from(NumListCast(doc[`data_labels_embedding_${n + 1}`]))); - const bestEmbedScore = (embedding: Opt<number[]>) => Math.max(...embedLists.map(l => (embedding && similarity(Array.from(embedding), l)) || 0)); - const {label: mostSimilarLabelCollect} = - labelGroups.map(label => ({ label, similarityScore: bestEmbedScore(labelToEmbedding.get(label)) })) - .reduce((prev, cur) => cur.similarityScore < 0.3 || cur.similarityScore <= prev.similarityScore ? prev: cur, - { label: '', similarityScore: 0, }); // prettier-ignore - - numberRange(3).forEach(n => { - doc[`data_labels_embedding_${n + 1}`] = undefined; - }); - doc[DocData].data_label = mostSimilarLabelCollect; - }); - this._props.Document._type_collection = CollectionViewType.Time; - this._props.Document.pivotField = 'data_label'; - }); + const labelGroups: string[] = ImageLabelBoxData.Instance._labelGroups; + const labelToCollection: Map<string, Doc> = new Map(); + const selectedImages = ImageLabelBoxData.Instance._docs; + + // Create new collections associated with each label and get the embeddings for the labels. + let x_offset = 0; + let y_offset = 0; + let row_count = 0; + for (const label of labelGroups) { + const newCollection = this.getCollection([], undefined, false); + newCollection._width = 900; + newCollection._height = 900; + newCollection._x = this.Bounds.left; + newCollection._y = this.Bounds.top; + newCollection._freeform_panX = this.Bounds.left + this.Bounds.width / 2; + newCollection._freeform_panY = this.Bounds.top + this.Bounds.height / 2; + newCollection._x = (newCollection._x as number) + x_offset; + newCollection._y = (newCollection._y as number) + y_offset; + x_offset += (newCollection._width as number) + 40; + row_count += 1; + if (row_count == 3) { + y_offset += (newCollection._height as number) + 40; + x_offset = 0; + row_count = 0; + } + labelToCollection.set(label, newCollection); + this._props.addDocument?.(newCollection); + } - @undoBatch - syntaxHighlight = action((e: KeyboardEvent | React.PointerEvent | undefined) => { - const selected = this.marqueeSelect(false); - if (e instanceof KeyboardEvent ? e.key === 'i' : true) { - const inks = selected.filter(s => s.type === DocumentType.INK); - const setDocs = selected.filter(s => s.type === DocumentType.RTF && s.color); - const sets = setDocs.map(sd => Cast(sd.data, RichTextField)?.Text as string); - const colors = setDocs.map(sd => FieldValue(sd.color) as string); - const wordToColor = new Map<string, string>(); - sets.forEach((st: string, i: number) => st.split(',').forEach(word => wordToColor.set(word, colors[i]))); - const strokes: InkData[] = []; - inks.filter(i => Cast(i.data, InkField)).forEach(i => { - const d = Cast(i.data, InkField, null); - const left = Math.min(...(d?.inkData.map(pd => pd.X) ?? [0])); - const top = Math.min(...(d?.inkData.map(pd => pd.Y) ?? [0])); - strokes.push(d.inkData.map(pd => ({ X: pd.X + NumCast(i.x) - left, Y: pd.Y + NumCast(i.y) - top }))); - }); - CognitiveServices.Inking.Appliers.InterpretStrokes(strokes).then(results => { - // const wordResults = results.filter((r: any) => r.category === "inkWord"); - // for (const word of wordResults) { - // const indices: number[] = word.strokeIds; - // indices.forEach(i => { - // if (wordToColor.has(word.recognizedText.toLowerCase())) { - // inks[i].color = wordToColor.get(word.recognizedText.toLowerCase()); - // } - // else { - // for (const alt of word.alternates) { - // if (wordToColor.has(alt.recognizedString.toLowerCase())) { - // inks[i].color = wordToColor.get(alt.recognizedString.toLowerCase()); - // break; - // } - // } - // } - // }) - // } - // const wordResults = results.filter((r: any) => r.category === "inkWord"); - // for (const word of wordResults) { - // const indices: number[] = word.strokeIds; - // indices.forEach(i => { - // const otherInks: Doc[] = []; - // indices.forEach(i2 => i2 !== i && otherInks.push(inks[i2])); - // inks[i].relatedInks = new List<Doc>(otherInks); - // const uniqueColors: string[] = []; - // Array.from(wordToColor.values()).forEach(c => uniqueColors.indexOf(c) === -1 && uniqueColors.push(c)); - // inks[i].alternativeColors = new List<string>(uniqueColors); - // if (wordToColor.has(word.recognizedText.toLowerCase())) { - // inks[i].color = wordToColor.get(word.recognizedText.toLowerCase()); - // } - // else if (word.alternates) { - // for (const alt of word.alternates) { - // if (wordToColor.has(alt.recognizedString.toLowerCase())) { - // inks[i].color = wordToColor.get(alt.recognizedString.toLowerCase()); - // break; - // } - // } - // } - // }); - // } - const lines = results.filter((r: any) => r.category === 'line'); - const text = lines.map((l: any) => l.recognizedText).join('\r\n'); - this._props.addDocument?.(Docs.Create.TextDocument(text, { _width: this.Bounds.width, _height: this.Bounds.height, x: this.Bounds.left + this.Bounds.width, y: this.Bounds.top, title: text })); - }); + for (const doc of selectedImages) { + if (doc[DocData].data_label) { + Doc.AddDocToList(labelToCollection.get(doc[DocData].data_label as string)!, undefined, doc); + this._props.removeDocument?.(doc); + } } + + //this._props.Document._type_collection = CollectionViewType.Time; // Change the collection view to a Time view. + //this._props.Document.pivotField = 'data_label'; // Sets the pivot to be the 'data_label'. }); @undoBatch @@ -582,13 +516,14 @@ export class MarqueeView extends ObservableReactComponent<SubCollectionViewProps @action marqueeCommand = (e: KeyboardEvent) => { - if (this._commandExecuted || (e as any).propagationIsStopped) { + const ee = e as unknown as KeyboardEvent & { propagationIsStopped?: boolean }; + if (this._commandExecuted || ee.propagationIsStopped) { return; } if (e.key === 'Backspace' || e.key === 'Delete' || e.key === 'd' || e.key === 'h') { this._commandExecuted = true; e.stopPropagation(); - (e as any).propagationIsStopped = true; + ee.propagationIsStopped = true; this.delete(e, e.key === 'h'); e.stopPropagation(); } @@ -596,7 +531,7 @@ export class MarqueeView extends ObservableReactComponent<SubCollectionViewProps this._commandExecuted = true; e.stopPropagation(); e.preventDefault(); - (e as any).propagationIsStopped = true; + ee.propagationIsStopped = true; if (e.key === 'g') this.collection(e, true); if (e.key === 'c' || e.key === 't') this.collection(e); if (e.key === 's' || e.key === 'S') this.summary(); @@ -697,8 +632,8 @@ export class MarqueeView extends ObservableReactComponent<SubCollectionViewProps transform: `translate(${p[0]}px, ${p[1]}px)`, width: Math.abs(v[0]), height: Math.abs(v[1]), - color: lightOrDark(this._props.Document?.backgroundColor ?? 'white'), - borderColor: lightOrDark(this._props.Document?.backgroundColor ?? 'white'), + color: lightOrDark((this._props.Document?.backgroundColor as string) ?? 'white'), + borderColor: lightOrDark((this._props.Document?.backgroundColor as string) ?? 'white'), zIndex: 2000, }}> {' '} @@ -707,7 +642,7 @@ export class MarqueeView extends ObservableReactComponent<SubCollectionViewProps <polyline // points={this._lassoPts.reduce((s, pt) => s + pt[0] + ',' + pt[1] + ' ', '')} fill="none" - stroke={lightOrDark(this._props.Document?.backgroundColor ?? 'white')} + stroke={lightOrDark((this._props.Document?.backgroundColor as string) ?? 'white')} strokeWidth="1" strokeDasharray="3" /> @@ -727,8 +662,9 @@ export class MarqueeView extends ObservableReactComponent<SubCollectionViewProps */ @action onDragMovePause = (e: CustomEvent<React.DragEvent>) => { - if ((e as any).handlePan || this._props.isAnnotationOverlay) return; - (e as any).handlePan = true; + const ee = e as CustomEvent<React.DragEvent> & { handlePan?: boolean }; + if (ee.handlePan || this._props.isAnnotationOverlay) return; + ee.handlePan = true; const bounds = this.MarqueeRef?.getBoundingClientRect(); if (!this._props.Document._freeform_noAutoPan && !this._props.renderDepth && bounds) { @@ -746,10 +682,10 @@ export class MarqueeView extends ObservableReactComponent<SubCollectionViewProps }; render() { return ( - // eslint-disable-next-line jsx-a11y/click-events-have-key-events <div className="marqueeView" ref={r => { + // eslint-disable-next-line @typescript-eslint/no-explicit-any r?.addEventListener('dashDragMovePause', this.onDragMovePause as any); this.MarqueeRef = r; }} diff --git a/src/client/views/linking/LinkPopup.tsx b/src/client/views/linking/LinkPopup.tsx index 9ce5f8fc9..b654f9bd0 100644 --- a/src/client/views/linking/LinkPopup.tsx +++ b/src/client/views/linking/LinkPopup.tsx @@ -45,7 +45,6 @@ export class LinkPopup extends React.Component<LinkPopupProps> { {/* <i></i> <input defaultValue={""} autoComplete="off" type="text" placeholder="Search for Document..." id="search-input" className="linkPopup-searchBox searchBox-input" /> */} - <SearchBox Document={Doc.MySearcher} docViewPath={returnEmptyDocViewList} diff --git a/src/client/views/nodes/DocumentView.tsx b/src/client/views/nodes/DocumentView.tsx index 5ae232aa9..c807d99ac 100644 --- a/src/client/views/nodes/DocumentView.tsx +++ b/src/client/views/nodes/DocumentView.tsx @@ -565,7 +565,7 @@ export class DocumentViewInternal extends DocComponent<FieldViewProps & Document if (this._props.renderDepth === 0) { appearanceItems.splice(0, 0, { description: 'Open in Lightbox', event: () => DocumentView.SetLightboxDoc(this.Document), icon: 'external-link-alt' }); } - appearanceItems.push({ description: 'Pin', event: () => this._props.pinToPres(this.Document, {}), icon: 'eye' }); + appearanceItems.push({ description: 'Pin', event: () => this._props.pinToPres(this.Document, {}), icon: 'map-pin' }); if (this.Document._layout_isFlashcard) { appearanceItems.push({ description: 'Create ChatCard', event: () => this.askGPT(), icon: 'id-card' }); } diff --git a/src/client/views/nodes/FaceRectangle.tsx b/src/client/views/nodes/FaceRectangle.tsx deleted file mode 100644 index 2b66b83fe..000000000 --- a/src/client/views/nodes/FaceRectangle.tsx +++ /dev/null @@ -1,34 +0,0 @@ -import { observable, runInAction } from 'mobx'; -import { observer } from 'mobx-react'; -import * as React from 'react'; -import { RectangleTemplate } from './FaceRectangles'; - -@observer -export default class FaceRectangle extends React.Component<{ rectangle: RectangleTemplate }> { - @observable private opacity = 0; - - componentDidMount() { - setTimeout( - () => - runInAction(() => { - this.opacity = 1; - }), - 500 - ); - } - - render() { - const { rectangle } = this.props; - return ( - <div - style={{ - ...rectangle.style, - opacity: this.opacity, - transition: '1s ease opacity', - position: 'absolute', - borderRadius: 5, - }} - /> - ); - } -} diff --git a/src/client/views/nodes/FaceRectangles.tsx b/src/client/views/nodes/FaceRectangles.tsx deleted file mode 100644 index ade4225d9..000000000 --- a/src/client/views/nodes/FaceRectangles.tsx +++ /dev/null @@ -1,46 +0,0 @@ -import { observer } from 'mobx-react'; -import * as React from 'react'; -import { Doc, DocListCast } from '../../../fields/Doc'; -import { Id } from '../../../fields/FieldSymbols'; -import { Cast, NumCast } from '../../../fields/Types'; -import FaceRectangle from './FaceRectangle'; - -interface FaceRectanglesProps { - document: Doc; - color: string; - backgroundColor: string; -} - -export interface RectangleTemplate { - id: string; - style: Partial<React.CSSProperties>; -} - -@observer -export class FaceRectangles extends React.Component<FaceRectanglesProps> { - render() { - const faces = DocListCast(this.props.document.faces); - const templates: RectangleTemplate[] = faces.map(faceDoc => { - const rectangle = Cast(faceDoc.faceRectangle, Doc) as Doc; - const style = { - top: NumCast(rectangle.top), - left: NumCast(rectangle.left), - width: NumCast(rectangle.width), - height: NumCast(rectangle.height), - backgroundColor: `${this.props.backgroundColor}33`, - border: `solid 2px ${this.props.color}`, - } as React.CSSProperties; - return { - id: rectangle[Id], - style: style, - }; - }); - return ( - <div> - {templates.map(rectangle => ( - <FaceRectangle key={rectangle.id} rectangle={rectangle} /> - ))} - </div> - ); - } -} diff --git a/src/client/views/nodes/ImageBox.tsx b/src/client/views/nodes/ImageBox.tsx index 51dd494da..d0a7fc6ac 100644 --- a/src/client/views/nodes/ImageBox.tsx +++ b/src/client/views/nodes/ImageBox.tsx @@ -20,6 +20,7 @@ import { DocumentType } from '../../documents/DocumentTypes'; import { DocUtils } from '../../documents/DocUtils'; import { Networking } from '../../Network'; import { DragManager } from '../../util/DragManager'; +import { SnappingManager } from '../../util/SnappingManager'; import { undoBatch } from '../../util/UndoManager'; import { CollectionFreeFormView } from '../collections/collectionFreeForm/CollectionFreeFormView'; import { ContextMenu } from '../ContextMenu'; @@ -35,7 +36,6 @@ import { FieldView, FieldViewProps } from './FieldView'; import { FocusViewOptions } from './FocusViewOptions'; import './ImageBox.scss'; import { OpenWhere } from './OpenWhere'; -import { SnappingManager } from '../../util/SnappingManager'; export class ImageEditorData { // eslint-disable-next-line no-use-before-define diff --git a/src/client/views/search/FaceRecognitionHandler.tsx b/src/client/views/search/FaceRecognitionHandler.tsx new file mode 100644 index 000000000..dc271fe73 --- /dev/null +++ b/src/client/views/search/FaceRecognitionHandler.tsx @@ -0,0 +1,161 @@ +import * as faceapi from 'face-api.js'; +import { FaceMatcher } from 'face-api.js'; +import { computed } from 'mobx'; +import { Doc, DocListCast } from '../../../fields/Doc'; +import { DocData } from '../../../fields/DocSymbols'; +import { List } from '../../../fields/List'; +import { listSpec } from '../../../fields/Schema'; +import { Cast, ImageCast, NumCast, StrCast } from '../../../fields/Types'; +import { DocumentType } from '../../documents/DocumentTypes'; +import { DocumentManager } from '../../util/DocumentManager'; + +/** + * A class that handles face recognition. + */ +export class FaceRecognitionHandler { + static _instance: FaceRecognitionHandler; + private _loadedModels: boolean = false; + private _processingDocs: Set<Doc> = new Set(); + private _pendingLoadDocs: Doc[] = []; + + public static FaceField = (target: Doc, doc: Doc) => `${Doc.LayoutFieldKey(target)}_${doc.face_label}`; + public static FacesField = (target: Doc) => `${Doc.LayoutFieldKey(target)}_Faces`; + + constructor() { + FaceRecognitionHandler._instance = this; + this.loadModels().then(() => this._pendingLoadDocs.forEach(this.findMatches)); + DocumentManager.Instance.AddAnyViewRenderedCB(dv => FaceRecognitionHandler.Instance.findMatches(dv.Document)); + } + + @computed get examinedFaceDocs() { + return DocListCast(Doc.UserDoc().examinedFaceDocs); + } + + /** + * Loads the face detection models. + */ + loadModels = async () => { + const MODEL_URL = `/models`; + await faceapi.loadFaceDetectionModel(MODEL_URL); + await faceapi.loadFaceLandmarkModel(MODEL_URL); + await faceapi.loadFaceRecognitionModel(MODEL_URL); + this._loadedModels = true; + }; + + public static get Instance() { + return FaceRecognitionHandler._instance ?? new FaceRecognitionHandler(); + } + + /** + * When a document is added, look for matching face documents. + * @param doc The document being analyzed. + */ + public findMatches = async (doc: Doc) => { + if (!this._loadedModels || !Doc.ActiveDashboard) { + this._pendingLoadDocs.push(doc); + return; + } + + if (doc.type === DocumentType.LOADING && !doc.loadingError) { + setTimeout(() => this.findMatches(doc), 1000); + return; + } + + const imgUrl = ImageCast(doc[Doc.LayoutFieldKey(doc)]); + // If the doc isn't an image or currently already been examined or is being processed, stop examining the document. + if (!imgUrl || this.examinedFaceDocs.includes(doc) || this._processingDocs.has(doc)) { + return; + } + + // Mark the document as being processed. + this._processingDocs.add(doc); + + // Get the image the document contains and analyze for faces. + const [name, type] = imgUrl.url.href.split('.'); + const imageURL = `${name}_o.${type}`; + + const img = await this.loadImage(imageURL); + + const fullFaceDescriptions = await faceapi.detectAllFaces(img).withFaceLandmarks().withFaceDescriptors(); + + doc[DocData][FaceRecognitionHandler.FacesField(doc)] = new List<List<number>>(); + + // For each face detected, find a match. + for (const fd of fullFaceDescriptions) { + let match = this.findMatch(fd.descriptor); + const converted_list = new List<number>(Array.from(fd.descriptor)); + + if (match) { + // If a matching Face Document has been found, add the document to the Face Document's associated docs and append the face + // descriptor to the Face Document's descriptor list. + Doc.AddDocToList(match, 'face_docList', doc); + Cast(match.face_descriptors, listSpec('number'), null).push(converted_list as unknown as number); // items are lists of numbers, not numbers, but type system can't handle that + } else { + // If a matching Face Document has not been found, create a new Face Document. + Doc.UserDoc().faceDocNum = NumCast(Doc.UserDoc().faceDocNum) + 1; + + const newFaceDocument = new Doc(); + newFaceDocument.title = `Face ${Doc.UserDoc().faceDocNum}`; + newFaceDocument.face = ''; // just to make prettyprinting look better + newFaceDocument.face_label = `Face${Doc.UserDoc().faceDocNum}`; + newFaceDocument.face_docList = new List<Doc>([doc]); + newFaceDocument.face_descriptors = new List<List<number>>([converted_list]); + + Doc.AddDocToList(Doc.ActiveDashboard[DocData], 'faceDocuments', newFaceDocument); + match = newFaceDocument; + } + + // Assign a field in the document of the matching Face Document. + const faceDescripField = FaceRecognitionHandler.FaceField(doc, match); + if (doc[DocData][faceDescripField]) { + Cast(doc[DocData][faceDescripField], listSpec('number'), null).push(converted_list as unknown as number); // items are lists of numbers, not numbers, but type system can't handle that + } else { + doc[DocData][faceDescripField] = new List<List<number>>([converted_list]); + } + + Cast(doc[DocData][FaceRecognitionHandler.FacesField(doc)], listSpec('number'), null).push(converted_list as unknown as number); // items are lists of numbers, not numbers, but type system can't handle that + + Doc.AddDocToList(Doc.UserDoc(), 'examinedFaceDocs', doc); + } + this._processingDocs.delete(doc); + }; + + /** + * Finds a matching Face Document given a descriptor + * @param cur_descriptor The current descriptor whose match is being searched for. + * @returns The most similar Face Document. + */ + private findMatch(cur_descriptor: Float32Array) { + if (!Doc.ActiveDashboard || DocListCast(Doc.ActiveDashboard[DocData].faceDocuments).length < 1) { + return null; + } + + const faceDescriptors: faceapi.LabeledFaceDescriptors[] = DocListCast(Doc.ActiveDashboard[DocData].faceDocuments).map(faceDocument => { + const float32Array = (faceDocument[DocData].face_descriptors as List<List<number>>).map(faceDescriptor => new Float32Array(Array.from(faceDescriptor))); + return new faceapi.LabeledFaceDescriptors(StrCast(faceDocument[DocData].face_label), float32Array); + }); + const faceMatcher = new FaceMatcher(faceDescriptors, 0.6); + const match = faceMatcher.findBestMatch(cur_descriptor); + if (match.label !== 'unknown') { + for (const doc of DocListCast(Doc.ActiveDashboard[DocData].faceDocuments)) { + if (doc[DocData].face_label === match.label) { + return doc; + } + } + } + return null; + } + + /** + * Loads an image + */ + private loadImage = (src: string): Promise<HTMLImageElement> => { + return new Promise((resolve, reject) => { + const img = new Image(); + img.crossOrigin = 'anonymous'; + img.onload = () => resolve(img); + img.onerror = err => reject(err); + img.src = src; + }); + }; +} |