diff options
Diffstat (limited to 'src')
22 files changed, 1135 insertions, 286 deletions
diff --git a/src/client/apis/gpt/GPT.ts b/src/client/apis/gpt/GPT.ts index 372e2a4e5..7878e9bfe 100644 --- a/src/client/apis/gpt/GPT.ts +++ b/src/client/apis/gpt/GPT.ts @@ -11,6 +11,10 @@ export enum GPTDocCommand { export const DescriptionSeperator = '======'; export const DocSeperator = '------'; +export enum TextClassifications { + Title = 'word', //a few words + Caption = 'sentence', //few sentences + LengthyDescription = 'paragraphs' } enum GPTCallType { SUMMARY = 'summary', @@ -36,6 +40,9 @@ enum GPTCallType { SUBSETDOCS = 'subset_docs', // select a subset of documents based on their descriptions DOCINFO = 'doc_info', // provide information about a document SORTDOCS = 'sort_docs', + CLASSIFYTEXTMINIMAL = 'classify_text_minimal', // classify text into one of the three categories: title, caption, lengthy description + CLASSIFYTEXTFULL = 'classify_text_full', //tags pdf content + GENERATESCRAPBOOK = 'generate_scrapbook' } type GPTCallOpts = { @@ -48,6 +55,23 @@ type GPTCallOpts = { const callTypeMap: { [type in GPTCallType]: GPTCallOpts } = { // newest model: gpt-4 summary: { model: 'gpt-4-turbo', maxTokens: 256, temp: 0.5, prompt: 'Summarize the text given in simpler terms.' }, + + + sort_docs: { + model: 'gpt-4o', + maxTokens: 2048, + temp: 0.25, + prompt: + `The user is going to give you a list of descriptions. + Each one is separated by '${DescriptionSeperator}' on either side. + Descriptions will vary in length, so make sure to only separate when you see '${DescriptionSeperator}'. + Sort them by the user's specifications. + Make sure each description is only in the list once. Each item should be separated by '${DescriptionSeperator}'. + Immediately afterward, surrounded by '${DocSeperator}' on BOTH SIDES, provide some insight into your reasoning for the way you sorted (and mention nothing about the formatting details given in this description). + It is VERY important that you format it exactly as described, ensuring the proper number of '${DescriptionSeperator[0]}' and '${DocSeperator[0]}' (${DescriptionSeperator.length} of each) and NO commas`, + }, + + edit: { model: 'gpt-4-turbo', maxTokens: 256, temp: 0.5, prompt: 'Reword the text.' }, stack: { model: 'gpt-4o', @@ -69,17 +93,23 @@ const callTypeMap: { [type in GPTCallType]: GPTCallOpts } = { temp: 0.5, prompt: "You are a helpful resarch assistant. Analyze the user's data to find meaningful patterns and/or correlation. Please only return a JSON with a correlation column 1 propert, a correlation column 2 property, and an analysis property. ", }, - sort_docs: { + //new + classify_text_minimal: { model: 'gpt-4o', maxTokens: 2048, temp: 0.25, - prompt: `The user is going to give you a list of descriptions. - Each one is separated by '${DescriptionSeperator}' on either side. - Descriptions will vary in length, so make sure to only separate when you see '${DescriptionSeperator}'. - Sort them by the user's specifications. - Make sure each description is only in the list once. Each item should be separated by '${DescriptionSeperator}'. - Immediately afterward, surrounded by '${DocSeperator}' on BOTH SIDES, provide some insight into your reasoning for the way you sorted (and mention nothing about the formatting details given in this description). - It is VERY important that you format it exactly as described, ensuring the proper number of '${DescriptionSeperator[0]}' and '${DocSeperator[0]}' (${DescriptionSeperator.length} of each) and NO commas`, + prompt: `Based on the content of the the text, classify it into the + most appropriate category: '${TextClassifications.Title}' if it is a few words, '${TextClassifications.Caption}' if it is a couple sentences or less, or '${TextClassifications.LengthyDescription}' if it is a lengthy description. Output exclusively the classification in your response. + ` + }, + classify_text_full: { + model: 'gpt-4o', + maxTokens: 2048, + temp: 0.25, + prompt: `Based on the content of the the text, classify it into the + most appropriate category: '${TextClassifications.Title}', '${TextClassifications.Caption}', or '${TextClassifications.LengthyDescription}'. + Then provide five more descriptive tags (single words) separated by spaces. + Finally, include a more detailed summary phrase tag using underscores, for a total of seven tags.` }, describe: { model: 'gpt-4-vision-preview', maxTokens: 2048, temp: 0, prompt: 'Describe these images in 3-5 words' }, flashcard: { @@ -107,6 +137,7 @@ const callTypeMap: { [type in GPTCallType]: GPTCallOpts } = { temp: 0.5, prompt: 'You will be given a list of field descriptions for one or more templates in the format {field #0: “description”}{field #1: “description”}{...}, and a list of column descriptions in the format {“title”: “description”}{...}. Your job is to match columns with fields based on their descriptions. Your output should be in the following JSON format: {“template_title”:{“#”: “title”, “#”: “title”, “#”: “title” …}, “template_title”:{“#”: “title”, “#”: “title”, “#”: “title” …}} where “template_title” is the templates title as specified in the description provided, # represents the field # and “title” the title of the column assigned to it. A filled out example might look like {“fivefield2”:{“0”:”Name”, “1”:”Image”, “2”:”Caption”, “3”:”Position”, “4”:”Stats”}, “fivefield3”:{0:”Image”, 1:”Name”, 2:”Caption”, 3:”Stats”, 4:”Position”}. Include one object for each template. IT IS VERY IMPORTANT THAT YOU ONLY INCLUDE TEXT IN THE FORMAT ABOVE, WITH NO ADDITIONS WHATSOEVER. Do not include extraneous ‘#’ characters, ‘column’ for columns, or ‘template’ for templates: ONLY THE TITLES AND NUMBERS. There should never be one column assigned to more than one field (ie. if the “name” column is assigned to field 1, it can’t be assigned to any other fields) . Do this for each template whose fields are described. The descriptions are as follows:', }, + vizsum: { model: 'gpt-4.1', maxTokens: 512, @@ -138,6 +169,32 @@ const callTypeMap: { [type in GPTCallType]: GPTCallOpts } = { temp: 0.5, prompt: 'You will be coloring drawings. You will be given what the drawing is, then a list of descriptions for parts of the drawing. Based on each description, respond with the stroke and fill color that it should be. Follow the rules: 1. Avoid using black for stroke color 2. Make the stroke color 1-3 shades darker than the fill color 3. Use the same colors when possible. Format as {#abcdef #abcdef}, making sure theres a color for each description, and do not include any additional text.', }, + + generate_scrapbook: { + model: 'gpt-4o', + maxTokens: 2048, + temp: 0.5, + prompt: `Generate an aesthetically pleasing scrapbook layout preset based on these items. + Return your response as JSON in the format: + [{ + "type": rich text or image or pdf or video or collection + "tag": a singular tag summarizing the document + "acceptTags": [a list of all relevant tags that this document accepts, like ['PERSON', 'LANDSCAPE']] + "x": number, + "y": number, + "width": number, **note: if it is in an image, please respect existing aspect ratio if it is provided + "height": number + }, ...] + If there are mutliple documents and you wish to nest documents into a collection for aesthetic purposes, you may include + "children": [ + { type: + tag: + x: , y: , width: , height: + } + ] ` + + }, + command_type: { model: 'gpt-4-turbo', maxTokens: 1024, diff --git a/src/client/views/DocumentDecorations.tsx b/src/client/views/DocumentDecorations.tsx index 69c2467a3..f36312056 100644 --- a/src/client/views/DocumentDecorations.tsx +++ b/src/client/views/DocumentDecorations.tsx @@ -36,6 +36,7 @@ import { ImageBox } from './nodes/ImageBox'; import { OpenWhere, OpenWhereMod } from './nodes/OpenWhere'; import { FormattedTextBox } from './nodes/formattedText/FormattedTextBox'; import { TagsView } from './TagsView'; +import { ScrapbookBox } from './nodes/scrapbook/ScrapbookBox'; interface DocumentDecorationsProps { PanelWidth: number; @@ -430,7 +431,7 @@ export class DocumentDecorations extends ObservableReactComponent<DocumentDecora onPointerDown = (e: React.PointerEvent): void => { SnappingManager.SetIsResizing(DocumentView.Selected().lastElement()?.Document[Id]); // turns off pointer events on things like youtube videos and web pages so that dragging doesn't get "stuck" when cursor moves over them DocumentView.Selected() - .filter(dv => e.shiftKey && dv.ComponentView instanceof ImageBox) + .filter(dv => e.shiftKey && dv.ComponentView instanceof ImageBox || dv.ComponentView instanceof ScrapbookBox) .forEach(dv => { dv.Document[dv.ComponentView!.fieldKey + '_outpaintOriginalWidth'] = NumCast(dv.Document._width); dv.Document[dv.ComponentView!.fieldKey + '_outpaintOriginalHeight'] = NumCast(dv.Document._height); @@ -486,7 +487,7 @@ export class DocumentDecorations extends ObservableReactComponent<DocumentDecora this._interactionLock = true; this._snapPt = thisPt; - const outpainted = e.shiftKey ? DocumentView.Selected().filter(dv => dv.ComponentView instanceof ImageBox) : []; + const outpainted = e.shiftKey ? DocumentView.Selected().filter(dv => dv.ComponentView instanceof ImageBox || dv.ComponentView instanceof ScrapbookBox) : []; const notOutpainted = e.shiftKey ? DocumentView.Selected().filter(dv => !outpainted.includes(dv)) : DocumentView.Selected(); // Special handling for shift-drag resize (outpainting of Images by resizing without scaling content - fill in with firefly GAI) @@ -750,7 +751,7 @@ export class DocumentDecorations extends ObservableReactComponent<DocumentDecora const rotation = DocumentView.Selected().length === 1 ? seldocview.screenToContentsTransform().inverse().RotateDeg : 0; // Radius constants - const useRounding = seldocview.ComponentView instanceof ImageBox || seldocview.ComponentView instanceof FormattedTextBox || seldocview.ComponentView instanceof CollectionFreeFormView; + const useRounding = seldocview.ComponentView instanceof ImageBox || seldocview.ComponentView instanceof ScrapbookBox || seldocview.ComponentView instanceof FormattedTextBox || seldocview.ComponentView instanceof CollectionFreeFormView; const borderRadius = numberValue(Cast(seldocview.Document.layout_borderRounding, 'string', null)); const docMax = Math.min(NumCast(seldocview.Document._width) / 2, NumCast(seldocview.Document._height) / 2); const maxDist = Math.min((this.Bounds.r - this.Bounds.x) / 2, (this.Bounds.b - this.Bounds.y) / 2); diff --git a/src/client/views/ViewBoxInterface.ts b/src/client/views/ViewBoxInterface.ts index d8dab8e89..b532dfe35 100644 --- a/src/client/views/ViewBoxInterface.ts +++ b/src/client/views/ViewBoxInterface.ts @@ -24,6 +24,7 @@ export abstract class ViewBoxInterface<P> extends ObservableReactComponent<React promoteCollection?: () => void; // moves contents of collection to parent hasChildDocs?: () => Doc[]; docEditorView?: () => void; + autoTag?: () => void; // auto tag the document showSmartDraw?: (x: number, y: number, regenerate?: boolean) => void; updateIcon?: (usePanelDimensions?: boolean) => Promise<void>; // updates the icon representation of the document getAnchor?: (addAsAnnotation: boolean, pinData?: PinProps) => Doc; // returns an Anchor Doc that represents the current state of the doc's componentview (e.g., the current playhead location of a an audio/video box) diff --git a/src/client/views/collections/collectionFreeForm/ImageLabelBox.tsx b/src/client/views/collections/collectionFreeForm/ImageLabelBox.tsx index ff9fb14e7..038b1c6f9 100644 --- a/src/client/views/collections/collectionFreeForm/ImageLabelBox.tsx +++ b/src/client/views/collections/collectionFreeForm/ImageLabelBox.tsx @@ -158,17 +158,19 @@ export class ImageLabelBox extends ViewBoxBaseComponent<FieldViewProps>() { this._currentLabel = e.target.value; }); - classifyImagesInBox = async () => { + classifyImagesInBox = async (selectedImages? : Doc[], prompt? : string) => { this.startLoading(); + alert('Classifying images...'); + selectedImages ??= this._selectedImages; // Converts the images into a Base64 format, afterwhich the information is sent to GPT to label them. - const imageInfos = this._selectedImages.map(async doc => { + const imageInfos = selectedImages.map(async doc => { if (!doc.$tags_chat) { const url = ImageCastWithSuffix(doc[Doc.LayoutDataKey(doc)], '_o') ?? ''; return imageUrlToBase64(url).then(hrefBase64 => !hrefBase64 ? undefined : - gptImageLabel(hrefBase64,'Give three labels to describe this image.').then(labels => + gptImageLabel(hrefBase64, prompt ?? 'Give three labels to describe this image.').then(labels => ({ doc, labels }))) ; // prettier-ignore } }); diff --git a/src/client/views/collections/collectionFreeForm/MarqueeOptionsMenu.tsx b/src/client/views/collections/collectionFreeForm/MarqueeOptionsMenu.tsx index abd828945..2ec59e5d5 100644 --- a/src/client/views/collections/collectionFreeForm/MarqueeOptionsMenu.tsx +++ b/src/client/views/collections/collectionFreeForm/MarqueeOptionsMenu.tsx @@ -16,6 +16,7 @@ export class MarqueeOptionsMenu extends AntimodeMenu<AntimodeMenuProps> { public createCollection: (e: KeyboardEvent | React.PointerEvent | undefined, group?: boolean, selection?: Doc[]) => Doc | void = unimplementedFunction; public delete: (e: KeyboardEvent | React.PointerEvent | undefined) => void = unimplementedFunction; public summarize: (e: KeyboardEvent | React.PointerEvent | undefined) => void = unimplementedFunction; + public generateScrapbook: (e: KeyboardEvent | React.PointerEvent | undefined) => void = unimplementedFunction; public showMarquee: () => void = unimplementedFunction; public hideMarquee: () => void = unimplementedFunction; public pinWithView: (e: KeyboardEvent | React.PointerEvent | undefined) => void = unimplementedFunction; @@ -38,6 +39,7 @@ export class MarqueeOptionsMenu extends AntimodeMenu<AntimodeMenuProps> { <IconButton tooltip="Create a Collection" onPointerDown={this.createCollection} icon={<FontAwesomeIcon icon="object-group" />} color={this.userColor} /> <IconButton tooltip="Create a Grouping" onPointerDown={e => this.createCollection(e, true)} icon={<FontAwesomeIcon icon="layer-group" />} color={this.userColor} /> <IconButton tooltip="Summarize Documents" onPointerDown={this.summarize} icon={<FontAwesomeIcon icon="compress-arrows-alt" />} color={this.userColor} /> + <IconButton tooltip="Generate Scrapbook" onPointerDown={this.generateScrapbook} icon={<FontAwesomeIcon icon="palette" />} color={this.userColor} /> <IconButton tooltip="Delete Documents" onPointerDown={this.delete} icon={<FontAwesomeIcon icon="trash-alt" />} color={this.userColor} /> <IconButton tooltip="Pin selected region" onPointerDown={this.pinWithView} icon={<FontAwesomeIcon icon="map-pin" />} color={this.userColor} /> <IconButton tooltip="Classify and Sort Images" onPointerDown={this.classifyImages} icon={<FontAwesomeIcon icon="object-group" />} color={this.userColor} /> diff --git a/src/client/views/collections/collectionFreeForm/MarqueeView.scss b/src/client/views/collections/collectionFreeForm/MarqueeView.scss index 7c9d0f6e1..b514b0911 100644 --- a/src/client/views/collections/collectionFreeForm/MarqueeView.scss +++ b/src/client/views/collections/collectionFreeForm/MarqueeView.scss @@ -28,4 +28,4 @@ .marquee-legend::after { content: 'Press <space> for lasso'; } -} +}
\ No newline at end of file diff --git a/src/client/views/collections/collectionFreeForm/MarqueeView.tsx b/src/client/views/collections/collectionFreeForm/MarqueeView.tsx index c120cddf0..b2b904509 100644 --- a/src/client/views/collections/collectionFreeForm/MarqueeView.tsx +++ b/src/client/views/collections/collectionFreeForm/MarqueeView.tsx @@ -16,7 +16,7 @@ import { DocumentType } from '../../../documents/DocumentTypes'; import { Docs, DocumentOptions } from '../../../documents/Documents'; import { SnappingManager, freeformScrollMode } from '../../../util/SnappingManager'; import { Transform } from '../../../util/Transform'; -import { UndoManager, undoBatch } from '../../../util/UndoManager'; +import { UndoManager, undoBatch, undoable } from '../../../util/UndoManager'; import { ContextMenu } from '../../ContextMenu'; import { ObservableReactComponent } from '../../ObservableReactComponent'; import { MarqueeViewBounds } from '../../PinFuncs'; @@ -28,8 +28,13 @@ import { FormattedTextBox } from '../../nodes/formattedText/FormattedTextBox'; import { SubCollectionViewProps } from '../CollectionSubView'; import { ImageLabelBoxData } from './ImageLabelBox'; import { MarqueeOptionsMenu } from './MarqueeOptionsMenu'; +import { StrListCast } from '../../../../fields/Doc'; +import { requestAiGeneratedPreset, DocumentDescriptor } from '../../nodes/scrapbook/AIPresetGenerator'; +import { buildPlaceholdersFromConfigs, slotRealDocIntoPlaceholders } from '../../nodes/scrapbook/ScrapbookBox'; import './MarqueeView.scss'; +import { build } from 'xregexp'; + interface MarqueeViewProps { Doc: Doc; getContainerTransform: () => Transform; @@ -76,6 +81,14 @@ export class MarqueeView extends ObservableReactComponent<SubCollectionViewProps @observable _labelsVisibile: boolean = false; @observable _lassoPts: [number, number][] = []; @observable _lassoFreehand: boolean = false; + // ─── New Observables for “Pick 1 of N AI Scrapbook” ─── + @observable aiChoices: Doc[] = []; // temporary hidden Scrapbook docs + @observable pickerX = 0; // popup x coordinate + @observable pickerY = 0; // popup y coordinate + @observable pickerVisible = false; // show/hide ScrapbookPicker + + + @computed get Transform() { return this._props.getTransform(); @@ -276,6 +289,7 @@ export class MarqueeView extends ObservableReactComponent<SubCollectionViewProps MarqueeOptionsMenu.Instance.createCollection = this.collection; MarqueeOptionsMenu.Instance.delete = this.delete; MarqueeOptionsMenu.Instance.summarize = this.summary; + MarqueeOptionsMenu.Instance.generateScrapbook = this.generateScrapbook; MarqueeOptionsMenu.Instance.showMarquee = this.showMarquee; MarqueeOptionsMenu.Instance.hideMarquee = this.hideMarquee; MarqueeOptionsMenu.Instance.jumpTo(e.clientX, e.clientY); @@ -518,6 +532,102 @@ export class MarqueeView extends ObservableReactComponent<SubCollectionViewProps MarqueeOptionsMenu.Instance.fadeOut(true); }); + + getAiPresetsDescriptors(): DocumentDescriptor[] { + const selected = this.marqueeSelect(false); + return selected.map((doc) => ({ + type: typeof doc.$type === 'string' ? doc.$type : 'UNKNOWN', + tags: (() => { + const s = new Set<string>(); + StrListCast(doc.$tags_chat ?? new List<string>()).forEach((t) => s.add(t)); + return Array.from(s); + })(), + })); + } + + + generateScrapbook = action(async () => { + + const selectedDocs = this.marqueeSelect(false); + if (!selectedDocs.length) return; + + const descriptors = this.getAiPresetsDescriptors(); + if (descriptors.length === 0) { + alert('No documents selected to generate a scrapbook from!'); + return; + } + + const aiPreset = await requestAiGeneratedPreset(descriptors); + if (!aiPreset.length) { + alert("Failed to generate preset"); + return; + } + const scrapbookPlaceholders: Doc[] = buildPlaceholdersFromConfigs(aiPreset); + /* + const scrapbookPlaceholders: Doc[] = aiPreset.map(cfg => { + const placeholderDoc = Docs.Create.TextDocument(cfg.tag); + placeholderDoc.accepts_docType = cfg.type as DocumentType; + placeholderDoc.accepts_tagType = new List<string>(cfg.acceptTags ?? [cfg.tag]); + + const placeholder = new Doc(); + placeholder.proto = placeholderDoc; + placeholder.original = placeholderDoc; + placeholder.x = cfg.x; + placeholder.y = cfg.y; + if (cfg.width != null) placeholder._width = cfg.width; + if (cfg.height != null) placeholder._height = cfg.height; + + return placeholder; + });*/ + + const scrapbook = Docs.Create.ScrapbookDocument(scrapbookPlaceholders, { + backgroundColor: '#e2ad32', + x: this.Bounds.left, + y: this.Bounds.top, + _width: 500, + _height: 500, + title: 'AI-generated Scrapbook' + }); + + + + // 3) Now grab that new scrapbook’s flat placeholders + const flatPl = DocListCast(scrapbook[Doc.LayoutDataKey(scrapbook)]) as Doc[]; + const unwrap = (items: Doc[]): Doc[] => + items.flatMap(d => + d.$type === DocumentType.COL + ? unwrap(DocListCast(d[Doc.LayoutDataKey(d)])) + : [d] + ); + const allPlaceholders = unwrap(flatPl); + + // 4) Slot each selectedDocs[i] into the first matching placeholder + selectedDocs.forEach(realDoc => { + slotRealDocIntoPlaceholders(realDoc, allPlaceholders + ); + }); + + const selected = this.marqueeSelect(false).map(d => { + this._props.removeDocument?.(d); + d.x = NumCast(d.x) - this.Bounds.left; + d.y = NumCast(d.y) - this.Bounds.top; + return d; + }); + + this._props.addDocument?.(scrapbook); + selectedDocs.forEach(doc => this._props.removeDocument?.(doc)); + const portal = Docs.Create.FreeformDocument(selected, { title: 'summarized documents', x: this.Bounds.left + 200, y: this.Bounds.top, isGroup: true, backgroundColor: 'transparent' }); + DocUtils.MakeLink(scrapbook, portal, { link_relationship: 'summary of:summarized by' }); + + portal.hidden = true; + this._props.addDocument?.(portal); + MarqueeOptionsMenu.Instance.fadeOut(true); + this.hideMarquee(); + }); + + + + @action marqueeCommand = (e: KeyboardEvent) => { const ee = e as unknown as KeyboardEvent & { propagationIsStopped?: boolean }; @@ -539,6 +649,7 @@ export class MarqueeView extends ObservableReactComponent<SubCollectionViewProps if (e.key === 'g') this.collection(e, true); if (e.key === 'c' || e.key === 't') this.collection(e); if (e.key === 's' || e.key === 'S') this.summary(); + if (e.key === 'g' || e.key === 'G') this.generateScrapbook(); // ← scrapbook shortcut if (e.key === 'p') this.pileup(); this.cleanupInteractions(false); } @@ -683,6 +794,7 @@ export class MarqueeView extends ObservableReactComponent<SubCollectionViewProps }; render() { return ( + <> <div className="marqueeView" ref={r => { @@ -702,6 +814,8 @@ export class MarqueeView extends ObservableReactComponent<SubCollectionViewProps {this._visible ? this.marqueeDiv : null} {this.props.children} </div> + </> + ); } } diff --git a/src/client/views/nodes/ImageBox.tsx b/src/client/views/nodes/ImageBox.tsx index 8ed59c6e1..1e16bbfc9 100644 --- a/src/client/views/nodes/ImageBox.tsx +++ b/src/client/views/nodes/ImageBox.tsx @@ -8,7 +8,7 @@ import { extname } from 'path'; import * as React from 'react'; import { AiOutlineSend } from 'react-icons/ai'; import ReactLoading from 'react-loading'; -import { ClientUtils, DashColor, returnEmptyString, returnFalse, returnOne, returnZero, setupMoveUpEvents, UpdateIcon } from '../../../ClientUtils'; +import { ClientUtils, imageUrlToBase64, DashColor, returnEmptyString, returnFalse, returnOne, returnZero, setupMoveUpEvents, UpdateIcon } from '../../../ClientUtils'; import { Doc, DocListCast, Opt } from '../../../fields/Doc'; import { DocData } from '../../../fields/DocSymbols'; import { Id } from '../../../fields/FieldSymbols'; @@ -16,7 +16,7 @@ import { InkTool } from '../../../fields/InkField'; import { List } from '../../../fields/List'; import { ObjectField } from '../../../fields/ObjectField'; import { ComputedField } from '../../../fields/ScriptField'; -import { Cast, DocCast, ImageCast, NumCast, RTFCast, StrCast } from '../../../fields/Types'; +import { Cast, DocCast, ImageCast, NumCast, RTFCast, StrCast, ImageCastWithSuffix } from '../../../fields/Types'; import { ImageField } from '../../../fields/URLField'; import { TraceMobx } from '../../../fields/util'; import { emptyFunction } from '../../../Utils'; @@ -45,6 +45,7 @@ import { FieldView, FieldViewProps } from './FieldView'; import { FocusViewOptions } from './FocusViewOptions'; import './ImageBox.scss'; import { OpenWhere } from './OpenWhere'; +import { gptImageLabel } from '../../apis/gpt/GPT'; const DefaultPath = '/assets/unknown-file-icon-hi.png'; export class ImageEditorData { @@ -139,6 +140,59 @@ export class ImageBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { this._dropDisposer?.(); ele && (this._dropDisposer = DragManager.MakeDropTarget(ele, this.drop.bind(this), this.Document)); }; + + autoTag = async () => { + + try { + // 1) grab the full-size URL + const layoutKey = Doc.LayoutDataKey(this.Document); + const url = ImageCastWithSuffix(this.Document[layoutKey], '_o') ?? ''; + if (!url) throw new Error('No image URL found'); + + // 2) convert to base64 + const base64 = await imageUrlToBase64(url); + if (!base64) throw new Error('Failed to load image data'); + + // 3) ask GPT for labels one label: PERSON or LANDSCAPE + const raw = await gptImageLabel( + base64, + `Classify this image as PERSON or LANDSCAPE. You may only respond with one of these two options. Then + provide five additional descriptive tags to describe the image for a total of 6 words outputted, + delimited by spaces. For example: "LANDSCAPE BUNNY NATURE FOREST PEACEFUL OUTDOORS". Then add one final lengthier summary tag (separated by underscores) + that describes the image.` + ); + + const { nativeWidth, nativeHeight } = this.nativeSize; + const aspectRatio = nativeWidth && nativeHeight + ? (nativeWidth / nativeHeight).toFixed(2) + : '1.00'; + + // 4) normalize and prefix + const label = raw + .trim() + .toUpperCase() + + // 5) stash it on the Doc + // overwrite any old tags so re-runs still work + const tokens = label.split(/\s+/); + this.Document.$tags_chat = new List<string>(); + tokens.forEach(tok => { + (this.Document.$tags_chat as List<string>).push(tok)}); + (this.Document.$tags_chat as List<string>).push(`ASPECT_${aspectRatio}`); + + // 6) flip on “show tags” in the layout + // (same flag that ImageLabelBox.toggleDisplayInformation uses) + this.Document._layout_showTags = true; + + } catch (err) { + console.error('autoTag failed:', err); + } finally { + } + }; + + + + getAnchor = (addAsAnnotation: boolean, pinProps?: PinProps) => { const visibleAnchor = this._getAnchor?.(this._savedAnnotations, true); // use marquee anchor, otherwise, save zoom/pan as anchor const anchor = diff --git a/src/client/views/nodes/PDFBox.tsx b/src/client/views/nodes/PDFBox.tsx index 45fa5cc12..a0c7d8d22 100644 --- a/src/client/views/nodes/PDFBox.tsx +++ b/src/client/views/nodes/PDFBox.tsx @@ -33,6 +33,9 @@ import { ImageBox } from './ImageBox'; import { OpenWhere } from './OpenWhere'; import './PDFBox.scss'; import { CreateImage } from './WebBoxRenderer'; +import { gptAPICall } from '../../apis/gpt/GPT'; +import { List } from '../../../fields/List'; +import { GPTCallType } from '../../apis/gpt/GPT'; @observer export class PDFBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { @@ -78,6 +81,47 @@ export class PDFBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { } } + autoTag = async () => { + try { + if (!this._pdf) { + throw new Error('PDF not loaded'); + } + + // 1) Extract text from the first few pages (e.g., first 2 pages) + const maxPages = Math.min(2, this._pdf.numPages); + let textContent = ''; + for (let pageNum = 1; pageNum <= maxPages; pageNum++) { + const page = await this._pdf.getPage(pageNum); + const text = await page.getTextContent(); + const pageText = text.items.map(item => ('str' in item ? item.str : '')).join(' '); + textContent += ` ${pageText}`; + } + + if (!textContent.trim()) { + throw new Error('No text found in PDF'); + } + + // 2) Ask GPT to classify and provide descriptive tags + const raw = await gptAPICall( + `"${textContent.trim().slice(0, 2000)}"`, + GPTCallType.CLASSIFYTEXTFULL + ); + + // 3) Normalize and store the labels + const label = raw.trim().toUpperCase(); + + const tokens = label.split(/\s+/); + this.Document.$tags_chat = new List<string>(); + tokens.forEach(tok => (this.Document.$tags_chat as List<string>).push(tok)); + + // 4) Show tags in layout + this.Document._layout_showTags = true; + + } catch (err) { + console.error('PDF autoTag failed:', err); + } +}; + replaceCanvases = (oldDiv: HTMLElement, newDiv: HTMLElement) => { if (oldDiv.childNodes) { for (let i = 0; i < oldDiv.childNodes.length; i++) { diff --git a/src/client/views/nodes/VideoBox.tsx b/src/client/views/nodes/VideoBox.tsx index b3cb0e1db..4d85b4942 100644 --- a/src/client/views/nodes/VideoBox.tsx +++ b/src/client/views/nodes/VideoBox.tsx @@ -30,6 +30,7 @@ import { StyleProp } from '../StyleProp'; import { DocumentView } from './DocumentView'; import { FieldView, FieldViewProps } from './FieldView'; import { FocusViewOptions } from './FocusViewOptions'; +import { gptImageLabel } from '../../apis/gpt/GPT'; import './VideoBox.scss'; /** @@ -109,6 +110,57 @@ export class VideoBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { return this._videoRef; } + + autoTag = async () => { + try { + if (!this.player) throw new Error('Video element not available.'); + + // 1) Extract a frame at the video's midpoint + const videoDuration = this.player.duration; + const snapshotTime = videoDuration / 2; + + // Seek the video element to the midpoint + await new Promise<void>((resolve, reject) => { + const onSeeked = () => { + this.player!.removeEventListener('seeked', onSeeked); + resolve(); + }; + this.player!.addEventListener('seeked', onSeeked); + this.player!.currentTime = snapshotTime; + }); + + // 2) Draw the frame onto a canvas and get a base64 representation + const canvas = document.createElement('canvas'); + canvas.width = this.player.videoWidth; + canvas.height = this.player.videoHeight; + const ctx = canvas.getContext('2d'); + if (!ctx) throw new Error('Failed to create canvas context.'); + ctx.drawImage(this.player, 0, 0, canvas.width, canvas.height); + const base64Image = canvas.toDataURL('image/png'); + + // 3) Send the image data to GPT for classification and descriptive tags + const raw = await gptImageLabel( + base64Image, + `Classify this video frame as either a PERSON or LANDSCAPE. + Then provide five additional descriptive tags (single words) separated by spaces. + Finally, add one detailed summary phrase using underscores.` + ); + + // 4) Normalize and store labels in the Document's tags + const label = raw.trim().toUpperCase(); + const tokens = label.split(/\s+/); + this.Document.$tags_chat = new List<string>(); + tokens.forEach(tok => (this.Document.$tags_chat as List<string>).push(tok)); + const aspect = this.player!.videoWidth / (this.player!.videoHeight || 1); + (this.Document.$tags_chat as List<string>).push(`ASPECT_${aspect}`); + // 5) Turn on tag display in layout + this.Document._layout_showTags = true; + + } catch (err) { + console.error('Video autoTag failed:', err); + } +}; + componentDidMount() { this.unmounting = false; this._props.setContentViewBox?.(this); // this tells the DocumentView that this VideoBox is the "content" of the document. this allows the DocumentView to indirectly call getAnchor() on the VideoBox when making a link. diff --git a/src/client/views/nodes/formattedText/FormattedTextBox.tsx b/src/client/views/nodes/formattedText/FormattedTextBox.tsx index c8df6e50f..0c3179173 100644 --- a/src/client/views/nodes/formattedText/FormattedTextBox.tsx +++ b/src/client/views/nodes/formattedText/FormattedTextBox.tsx @@ -8,6 +8,7 @@ import { baseKeymap, selectAll, splitBlock } from 'prosemirror-commands'; import { history } from 'prosemirror-history'; import { inputRules } from 'prosemirror-inputrules'; import { keymap } from 'prosemirror-keymap'; +import { runInAction } from 'mobx'; import { Fragment, Mark, Node, Slice } from 'prosemirror-model'; import { EditorState, NodeSelection, Plugin, Selection, TextSelection, Transaction } from 'prosemirror-state'; import { EditorView, NodeViewConstructor } from 'prosemirror-view'; @@ -64,6 +65,7 @@ import { removeMarkWithAttrs } from './prosemirrorPatches'; import { RichTextMenu, RichTextMenuPlugin } from './RichTextMenu'; import { RichTextRules } from './RichTextRules'; import { schema } from './schema_rts'; +import { tickStep } from 'd3'; // import * as applyDevTools from 'prosemirror-dev-tools'; export interface FormattedTextBoxProps extends FieldViewProps { @@ -308,6 +310,31 @@ export class FormattedTextBox extends ViewBoxAnnotatableComponent<FormattedTextB } }; + autoTag = async () => { + + const layoutKey = Doc.LayoutDataKey(this.Document); + const rawText = RTFCast(this.Document[layoutKey])?.Text ?? StrCast(this.Document[layoutKey]); + + const callType = rawText.includes("[placeholder]") + ? GPTCallType.CLASSIFYTEXTMINIMAL + : GPTCallType.CLASSIFYTEXTFULL; + + gptAPICall(rawText, callType).then(desc => { + runInAction(() => { + // Clear existing tags + this.Document.$tags_chat = new List<string>(); + + // Split GPT response into tokens and push individually + const tokens = desc.trim().split(/\s+/); + tokens.forEach(tok => { + (this.Document.$tags_chat as List<string>).push(tok); + }); + + this.Document._layout_showTags = true; + }); + }); +}; + leafText = (node: Node) => { if (node.type === this.EditorView?.state.schema.nodes.dashField) { const refDoc = !node.attrs.docId ? this.rootDoc : (DocServer.GetCachedRefField(node.attrs.docId as string) as Doc); @@ -1271,6 +1298,14 @@ export class FormattedTextBox extends ViewBoxAnnotatableComponent<FormattedTextB { fireImmediately: true } ); + this._disposers.tagger = reaction( + () => ({ title: this.Document.title, sel: this._props.isSelected() }), + action(() => { + this.autoTag(); + }), + { fireImmediately: true } + ); + if (!this._props.dontRegisterView) { this._disposers.record = reaction( () => this.recordingDictation, diff --git a/src/client/views/nodes/scrapbook/AIPresetGenerator.ts b/src/client/views/nodes/scrapbook/AIPresetGenerator.ts new file mode 100644 index 000000000..1f159222b --- /dev/null +++ b/src/client/views/nodes/scrapbook/AIPresetGenerator.ts @@ -0,0 +1,31 @@ +import { ScrapbookItemConfig } from './ScrapbookPreset'; +import { GPTCallType, gptAPICall } from '../../../apis/gpt/GPT'; + +// Represents the descriptor for each document +export interface DocumentDescriptor { + type: string; + tags: string[]; +} + +// Main function to request AI-generated presets +export async function requestAiGeneratedPreset(descriptors: DocumentDescriptor[]): Promise<ScrapbookItemConfig[]> { + const prompt = createPrompt(descriptors); + let aiResponse = await gptAPICall(prompt, GPTCallType.GENERATESCRAPBOOK); + // Strip out ```json and ``` if the model wrapped its answer in fences + aiResponse = aiResponse + .trim() + .replace(/^```(?:json)?\s*/, "") // remove leading ``` or ```json + .replace(/\s*```$/, ""); // remove trailing ``` + const parsedPreset = JSON.parse(aiResponse) as ScrapbookItemConfig[]; + return parsedPreset; +} + +// Helper to generate prompt text for AI +function createPrompt(descriptors: DocumentDescriptor[]): string { + let prompt = ""; + descriptors.forEach((desc, index) => { + prompt += `${index + 1}. Type: ${desc.type}, Tags: ${desc.tags.join(', ')}\n`; + }); + + return prompt; +} diff --git a/src/client/views/nodes/scrapbook/EmbeddedDocView.tsx b/src/client/views/nodes/scrapbook/EmbeddedDocView.tsx deleted file mode 100644 index e99bf67c7..000000000 --- a/src/client/views/nodes/scrapbook/EmbeddedDocView.tsx +++ /dev/null @@ -1,52 +0,0 @@ -//IGNORE FOR NOW, CURRENTLY NOT USED IN SCRAPBOOK IMPLEMENTATION -import * as React from "react"; -import { observer } from "mobx-react"; -import { Doc } from "../../../../fields/Doc"; -import { DocumentView } from "../DocumentView"; -import { Transform } from "../../../util/Transform"; - -interface EmbeddedDocViewProps { - doc: Doc; - width?: number; - height?: number; - slotId?: string; -} - -@observer -export class EmbeddedDocView extends React.Component<EmbeddedDocViewProps> { - render() { - const { doc, width = 300, height = 200, slotId } = this.props; - - // Use either an existing embedding or create one - let docToDisplay = doc; - - // If we need an embedding, create or use one - if (!docToDisplay.isEmbedding) { - docToDisplay = Doc.BestEmbedding(doc) || Doc.MakeEmbedding(doc); - // Set the container to the slot's ID so we can track it - if (slotId) { - docToDisplay.embedContainer = `scrapbook-slot-${slotId}`; - } - } - - return ( - <DocumentView - Document={docToDisplay} - renderDepth={0} - // Required sizing functions - NativeWidth={() => width} - NativeHeight={() => height} - PanelWidth={() => width} - PanelHeight={() => height} - // Required state functions - isContentActive={() => true} - childFilters={() => []} - ScreenToLocalTransform={() => new Transform()} - // Display options - hideDeleteButton={true} - hideDecorations={true} - hideResizeHandles={true} - /> - ); - } -}
\ No newline at end of file diff --git a/src/client/views/nodes/scrapbook/ScrapbookBox.scss b/src/client/views/nodes/scrapbook/ScrapbookBox.scss new file mode 100644 index 000000000..8dc93df60 --- /dev/null +++ b/src/client/views/nodes/scrapbook/ScrapbookBox.scss @@ -0,0 +1,63 @@ + +.scrapbook-box { + /* Make sure the container fills its parent, and set a base background */ + position: relative; /* so that absolute children (loading overlay, etc.) are positioned relative to this */ + width: 100%; + height: 100%; + background: beige; + overflow: hidden; /* prevent scrollbars if children overflow */ +} + +/* Loading overlay that covers the entire scrapbook while AI-generation is in progress */ +.scrapbook-box-loading-overlay { + position: absolute; + top: 0; + left: 0; + width: 100%; + height: 100%; + display: flex; + justify-content: center; + align-items: center; + background: rgba(255, 255, 255, 0.8); + z-index: 10; /* sits above the ImageBox and other content */ +} + +/* The <select> dropdown for choosing presets */ +.scrapbook-box-preset-select { + position: absolute; + top: 8px; + left: 8px; + z-index: 20; + padding: 4px 8px; + font-size: 14px; + border: 1px solid #ccc; + border-radius: 4px; + background: white; +} + +/* Container for the “Regenerate Background” button */ +.scrapbook-box-ui { + position: absolute; + top: 8px; + right: 8px; + z-index: 20; +} + +/* The button itself */ +.scrapbook-box-ui-button { + display: flex; + align-items: center; + gap: 6px; + padding: 4px 8px; + font-size: 14px; + color: black; + background: white; + border: 1px solid #ccc; + border-radius: 4px; + cursor: pointer; + white-space: nowrap; +} + +.scrapbook-box-ui-button:hover { + background: #f5f5f5; +} diff --git a/src/client/views/nodes/scrapbook/ScrapbookBox.tsx b/src/client/views/nodes/scrapbook/ScrapbookBox.tsx index 6cfe9a62c..52e3c26dc 100644 --- a/src/client/views/nodes/scrapbook/ScrapbookBox.tsx +++ b/src/client/views/nodes/scrapbook/ScrapbookBox.tsx @@ -1,32 +1,178 @@ -import { action, makeObservable, observable } from 'mobx'; +import { action, makeObservable, observable, reaction, computed } from 'mobx'; import * as React from 'react'; -import { Doc, DocListCast } from '../../../../fields/Doc'; +import { Doc, DocListCast, StrListCast } from '../../../../fields/Doc'; import { List } from '../../../../fields/List'; import { emptyFunction } from '../../../../Utils'; import { Docs } from '../../../documents/Documents'; import { DocumentType } from '../../../documents/DocumentTypes'; import { CollectionView } from '../../collections/CollectionView'; import { ViewBoxAnnotatableComponent } from '../../DocComponent'; +import { AspectRatioLimits } from '../../smartdraw/FireflyConstants'; import { DocumentView } from '../DocumentView'; import { FieldView, FieldViewProps } from '../FieldView'; import { DragManager } from '../../../util/DragManager'; -import { RTFCast, StrCast, toList } from '../../../../fields/Types'; +import { toList } from '../../../../fields/Types'; import { undoable } from '../../../util/UndoManager'; -// Scrapbook view: a container that lays out its child items in a grid/template +import ReactLoading from 'react-loading'; +import { NumCast } from '../../../../fields/Types'; +import { ScrapbookItemConfig } from './ScrapbookPreset'; +import { ImageBox } from '../ImageBox'; +import { FireflyImageDimensions } from '../../smartdraw/FireflyConstants'; +import { SmartDrawHandler } from '../../smartdraw/SmartDrawHandler'; +import { ImageCast } from '../../../../fields/Types'; +import { SnappingManager } from '../../../util/SnappingManager'; +import { IReactionDisposer } from 'mobx'; +import { observer } from 'mobx-react'; +import { runInAction } from 'mobx'; +import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; +import { faRedoAlt } from '@fortawesome/free-solid-svg-icons'; +import { getPresetNames, createPreset } from './ScrapbookPresetRegistry'; +import './ScrapbookBox.scss'; +import { isDestArraysEqual } from 'pdfjs-dist/types/web/pdf_history'; + + +export function buildPlaceholdersFromConfigs(configs: ScrapbookItemConfig[]): Doc[] { + const placeholders: Doc[] = []; + + for (const cfg of configs) { + if (cfg.children && cfg.children.length) { + const childDocs = cfg.children.map(child => { + const doc = Docs.Create.TextDocument("[placeholder] " + child.tag); + doc.accepts_docType = child.type; + doc.accepts_tagType = new List<string>(child.acceptTags ?? [child.tag]); + + const ph = new Doc(); + ph.proto = doc; + ph.original = doc; + ph.x = child.x; + ph.y = child.y; + if (child.width != null) ph._width = child.width; + if (child.height != null) ph._height = child.height; + return ph; + }); + + const protoW = cfg.containerWidth ?? cfg.width; + const protoH = cfg.containerHeight ?? cfg.height; + // Create a stacking document with the child placeholders + const containerProto = Docs.Create.StackingDocument(childDocs, { + ...(protoW != null ? { _width: protoW } : {}), + ...(protoH != null ? { _height: protoH } : {}), + title: cfg.tag, + }); + + const ph = new Doc(); + ph.proto = containerProto; + ph.original = containerProto; + ph.x = cfg.x; + ph.y = cfg.y; + if (cfg.width != null) ph._width = cfg.width; + if (cfg.height != null) ph._height = cfg.height; + placeholders.push(ph); + } + + else { + const doc = Docs.Create.TextDocument("[placeholder] " + cfg.tag); + doc.accepts_docType = cfg.type; + doc.accepts_tagType = new List<string>(cfg.acceptTags ?? [cfg.tag]); + + const ph = new Doc(); + ph.proto = doc; + ph.original = doc; + ph.x = cfg.x; + ph.y = cfg.y; + if (cfg.width != null) ph._width = cfg.width; + if (cfg.height != null) ph._height = cfg.height; + placeholders.push(ph); + } + } + + return placeholders; +} +export function slotRealDocIntoPlaceholders( + realDoc: Doc, + placeholders: Doc[] +): boolean { + const realTags = new Set<string>( + StrListCast(realDoc.$tags_chat ?? new List<string>()) + .map(t => t.toLowerCase()) +); + + // Find placeholder with most matching tags + let bestMatch: Doc | null = null; + let maxMatches = 0; +/* + (d.accepts_docType === docs[0].$type || // match fields based on type, or by analyzing content .. simple example of matching text in placeholder to dropped doc's type + RTFCast(d[Doc.LayoutDataKey(d)])?.Text.includes(StrCast(docs[0].$type)))*/ + + placeholders.forEach(ph => { + if (ph.accepts_docType !== realDoc.$type) { + // Skip this placeholder entirely if types do not match. + return; + }; + const phTagTypes = StrListCast(ph.accepts_tagType ?? new List<string>()) + .map(t => t.toLowerCase()); + console.log({ realTags, phTagTypes }); + const matches = phTagTypes.filter(tag => realTags.has(tag)); + + if (matches.length > maxMatches) { + maxMatches = matches.length; + bestMatch = ph; + } + + }); + + if (bestMatch && maxMatches > 0) { + setTimeout( + undoable(() => { + bestMatch!.proto = realDoc; + }, 'Scrapbook add'), + 0 + ); + return true; + } + + return false; +} + +// Scrapbook view: a container that lays out its child items in a template +@observer export class ScrapbookBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { + @observable selectedPreset = getPresetNames()[0]; + @observable createdDate: string; + @observable loading = false; + @observable src = ''; + @observable imgDoc: Doc | undefined; + private _disposers: { [name: string]: IReactionDisposer } = {}; + private imageBoxRef = React.createRef<ImageBox>(); + constructor(props: FieldViewProps) { super(props); makeObservable(this); - this.createdDate = this.getFormattedDate(); + const existingItems = DocListCast(this.dataDoc[this.fieldKey] as List<Doc>); + if (!existingItems || existingItems.length === 0) { + // Only wire up reaction/setTitle if it's truly a brand-new, empty Scrapbook + reaction( + () => this.selectedPreset, + presetName => this.initScrapbook(presetName), + { fireImmediately: true } + ); + + this.createdDate = this.getFormattedDate(); + this.setTitle(); + } else { + // If items are already present, just preserve whatever was injected. + // We still want `createdDate` set so that the UI title bar can show it if needed. + this.createdDate = this.getFormattedDate(); + } + // ensure we always have a List<Doc> in dataDoc['items'] if (!this.dataDoc[this.fieldKey]) { this.dataDoc[this.fieldKey] = new List<Doc>(); } - this.createdDate = this.getFormattedDate(); - this.setTitle(); + } public static LayoutString(fieldStr: string) { @@ -41,14 +187,34 @@ export class ScrapbookBox extends ViewBoxAnnotatableComponent<FieldViewProps>() }); } + + @action + initScrapbook(name: string) { + const configs = createPreset(name); + // 1) ensure title is set + const title = `Scrapbook - ${this.createdDate}`; + if (this.dataDoc.title !== title) { + this.dataDoc.title = title; + } + + // 2) build placeholders from the preset + const placeholders = buildPlaceholdersFromConfigs(configs); + + // 3) commit them into the field + this.dataDoc[this.fieldKey] = new List<Doc>(placeholders); + } + + + @action setTitle() { const title = `Scrapbook - ${this.createdDate}`; if (this.dataDoc.title !== title) { this.dataDoc.title = title; - - const image = Docs.Create.TextDocument('image'); + if (!this.dataDoc[this.fieldKey]){ + const image = Docs.Create.TextDocument('[placeholder] person image'); image.accepts_docType = DocumentType.IMG; + image.accepts_tagType = 'PERSON' const placeholder = new Doc(); placeholder.proto = image; placeholder.original = image; @@ -56,26 +222,163 @@ export class ScrapbookBox extends ViewBoxAnnotatableComponent<FieldViewProps>() placeholder._height = 200; placeholder.x = 0; placeholder.y = -100; - //placeholder.overrideFields = new List<string>(['x', 'y']); // shouldn't need to do this for layout fields since the placeholder already overrides its protos - - const summary = Docs.Create.TextDocument('summary'); + + + const summary = Docs.Create.TextDocument('[placeholder] long summary'); summary.accepts_docType = DocumentType.RTF; - summary.accepts_textType = 'one line'; + summary.accepts_tagType = 'lengthy description'; const placeholder2 = new Doc(); placeholder2.proto = summary; placeholder2.original = summary; placeholder2.x = 0; placeholder2.y = 200; placeholder2._width = 250; - //placeholder2.overrideFields = new List<string>(['x', 'y', '_width']); // shouldn't need to do this for layout fields since the placeholder already overrides its protos - this.dataDoc[this.fieldKey] = new List<Doc>([placeholder, placeholder2]); + + + const sidebar = Docs.Create.TextDocument('[placeholder] brief sidebar'); + sidebar.accepts_docType = DocumentType.RTF; + sidebar.accepts_tagType = 'title'; + const placeholder3 = new Doc(); + placeholder3.proto = sidebar; + placeholder3.original = sidebar; + placeholder3.x = 280; + placeholder3.y = -50; + placeholder3._width = 50; + placeholder3._height = 200; + + + + const internalImg = Docs.Create.TextDocument('[placeholder] landscape internal'); + internalImg.accepts_docType = DocumentType.IMG; + internalImg.accepts_tagType = 'LANDSCAPE' + const placeholder5 = new Doc(); + placeholder5.proto = internalImg; + placeholder5.original = internalImg; + placeholder5._width = 50; + placeholder5._height = 100; + placeholder5.x = 0; + placeholder5.y = -100; + + const collection = Docs.Create.StackingDocument([placeholder5], { _width: 300, _height: 300, title: "internal coll" }); + //collection.accepts_docType = DocumentType.COL; don't mark this field + const placeholder4 = new Doc(); + placeholder4.proto = collection; + placeholder4.original = collection; + placeholder4.x = -200; + placeholder4.y = -100; + placeholder4._width = 100; + placeholder4._height = 200; + + const starter = Docs.Create.TextDocument('To create a scrapbook from existing documents, marquee select. For existing scrapbook arrangements, select a preset from the dropdown.'); + starter.accepts_docType = DocumentType.RTF; + starter.accepts_tagType = 'n/a' + const starterplaceholder = new Doc(); + starterplaceholder.proto = summary; + starterplaceholder.original = summary; + starterplaceholder.x = 0; + starterplaceholder.y = 0; + starterplaceholder._width = 250; + + + + + + /*note-to-self + would doing: + + const collection = Docs.Create.ScrapbookDocument([placeholder, placeholder2, placeholder3]); + create issues with references to the same object?*/ + + /*note-to-self + Should we consider that there are more collections than just COL type collections? + when spreading*/ + + /*note-to-self + difference between passing a new List<Doc> versus just the raw array? + */ + this.dataDoc[this.fieldKey] = new List<Doc>([starterplaceholder]); + } + + } } componentDidMount() { this.setTitle(); + this.generateAiImage(); + + this._disposers.propagateResize = reaction( + () => ({ w: this.layoutDoc._width, h: this.layoutDoc._height }), + (dims, prev) => { + // prev is undefined on the first run, so bail early + if (!prev || !SnappingManager.ShiftKey || !this.imgDoc) return; + + // either guard the ref… + const imageBox = this.imageBoxRef.current; + if (!imageBox) return; + + // …or just hard-code the fieldKey if you know it’s always `"data"` + const key = imageBox.props.fieldKey; + + runInAction(() => { + if(!this.imgDoc){ + return + } + // use prev.w/h (the *old* size) as your orig dims + this.imgDoc[key + '_outpaintOriginalWidth'] = prev.w; + this.imgDoc[key + '_outpaintOriginalHeight'] = prev.h; + ;(this.imageBoxRef.current as any).layoutDoc._width = dims.w + ;(this.imageBoxRef.current as any).layoutDoc._height = dims.h + + }); + } + ); } + + @action + async generateAiImage(prompt?: string) { + this.loading = true; + try { + // 1) Default to regenPrompt if none provided + if (!prompt) prompt = this.regenPrompt; + + // 2) Measure the scrapbook’s current size + const w = NumCast(this.layoutDoc._width, 1); + const h = NumCast(this.layoutDoc._height, 1); + const ratio = w / h; + + // 3) Pick the Firefly preset that best matches the aspect ratio + let preset = FireflyImageDimensions.Square; + if (ratio > AspectRatioLimits[FireflyImageDimensions.Widescreen]) { + preset = FireflyImageDimensions.Widescreen; + } else if (ratio > AspectRatioLimits[FireflyImageDimensions.Landscape]) { + preset = FireflyImageDimensions.Landscape; + } else if (ratio < AspectRatioLimits[FireflyImageDimensions.Portrait]) { + preset = FireflyImageDimensions.Portrait; + } + + // 4) Call exactly the same CreateWithFirefly that ImageBox uses + const doc = await SmartDrawHandler.CreateWithFirefly(prompt, preset); + + if (doc instanceof Doc) { + // 5) Hook it into your state + this.imgDoc = doc; + const imgField = ImageCast(doc.data); + this.src = imgField?.url.href ?? ''; + } else { + alert('Failed to generate document.'); + this.src = ''; + } + } catch (e) { + alert(`Generation error: ${e}`); + } finally { + runInAction(() => { + this.loading = false; + }); + } + } + childRejectDrop = (de: DragManager.DropEvent, subView?: DocumentView) => { return true; // disable dropping documents onto any child of the scrapbook. }; @@ -86,45 +389,122 @@ export class ScrapbookBox extends ViewBoxAnnotatableComponent<FieldViewProps>() }; filterAddDocument = (docIn: Doc | Doc[]) => { - const docs = toList(docIn); - if (docs?.length === 1) { - const placeholder = DocListCast(this.dataDoc[this.fieldKey]).find(d => - (d.accepts_docType === docs[0].$type || // match fields based on type, or by analyzing content .. simple example of matching text in placeholder to dropped doc's type - RTFCast(d[Doc.LayoutDataKey(d)])?.Text.includes(StrCast(docs[0].$type))) - ); // prettier-ignore - - if (placeholder) { - // ugh. we have to tell the underlying view not to add the Doc so that we can add it where we want it. - // However, returning 'false' triggers an undo. so this settimeout is needed to make the assignment happen after the undo. - setTimeout( - undoable(() => { - //StrListCast(placeholder.overrideFields).map(field => (docs[0][field] = placeholder[field])); // // shouldn't need to do this for layout fields since the placeholder already overrides its protos - placeholder.proto = docs[0]; - }, 'Scrapbook add') - ); - return false; - } - } - return false; - }; + const docs = toList(docIn); //The docs being added to the scrapbook - render() { - return ( - <div style={{ background: 'beige', width: '100%', height: '100%' }}> - <CollectionView - {...this._props} // - setContentViewBox={emptyFunction} - rejectDrop={this.rejectDrop} - childRejectDrop={this.childRejectDrop} - filterAddDocument={this.filterAddDocument} - /> - {/* <div style={{ border: '1px black', borderStyle: 'dotted', position: 'absolute', top: '50%', width: '100%', textAlign: 'center' }}>Drop an image here</div> */} - </div> - ); + // 1) Grab all template slots: + const slots = DocListCast(this.dataDoc[this.fieldKey]); + + // 2) recursive unwrap: + const unwrap = (items: Doc[]): Doc[] => + items.flatMap(d => + d.$type === DocumentType.COL + ? unwrap(DocListCast(d[Doc.LayoutDataKey(d)])) + : [d] + ); + + // 3) produce a flat list of every doc, unwrapping any number of nested COLs + const allDocs: Doc[] = unwrap(slots); + if (docs?.length === 1) { + return slotRealDocIntoPlaceholders( + docs[0], + allDocs, + ) + ? false + : false; } + + return false; +}; + + + @computed get regenPrompt() { + const slots = DocListCast(this.dataDoc[this.fieldKey]); + + const unwrap = (items: Doc[]): Doc[] => + items.flatMap(d => + d.$type === DocumentType.COL + ? unwrap(DocListCast(d[Doc.LayoutDataKey(d)])) + : [d] + ); + + const allDocs: Doc[] = unwrap(slots); + const internalTagsSet = new Set<string>(); + + allDocs.forEach(doc => { + const tags = StrListCast(doc.$tags_chat ?? new List<string>()); + tags.forEach(tag => + {if (!tag.startsWith("ASPECT_")) { + internalTagsSet.add(tag); + } + }); + }); + + const internalTags = Array.from(internalTagsSet).join(', '); + + + return internalTags + ? `Create a new scrapbook background featuring: ${internalTags}` + : 'A serene mountain landscape at sunrise, ultra-wide, pastel sky, abstract, scrapbook background'; + } + + render() { + return ( + <div className="scrapbook-box"> + {this.loading && ( + <div className="scrapbook-box-loading-overlay"> + <ReactLoading type="spin" width={50} height={50} /> + </div> + )} + + {this.src && this.imgDoc && ( + <ImageBox + ref={this.imageBoxRef} + {...this._props} + Document={this.imgDoc} + fieldKey="data" + /> + )} + + <select + className="scrapbook-box-preset-select" + value={this.selectedPreset} + onChange={e => (this.selectedPreset = e.currentTarget.value)} + > + {getPresetNames().map(name => ( + <option key={name} value={name}> + {name} + </option> + ))} + </select> + + {this._props.isContentActive() && ( + <div className="scrapbook-box-ui"> + <button + type="button" + title="Regenerate Background" + onClick={() => this.generateAiImage(this.regenPrompt)} + className="scrapbook-box-ui-button" + > + <FontAwesomeIcon icon={faRedoAlt} /> + <span>Regenerate Background</span> + </button> + </div> + )} + + <CollectionView + {...this._props} + setContentViewBox={emptyFunction} + rejectDrop={this.rejectDrop} + childRejectDrop={this.childRejectDrop} + filterAddDocument={this.filterAddDocument} + /> + </div> + ); + } } -// Register scrapbook + + Docs.Prototypes.TemplateMap.set(DocumentType.SCRAPBOOK, { layout: { view: ScrapbookBox, dataField: 'items' }, options: { diff --git a/src/client/views/nodes/scrapbook/ScrapbookContent.tsx b/src/client/views/nodes/scrapbook/ScrapbookContent.tsx deleted file mode 100644 index ad1d308e8..000000000 --- a/src/client/views/nodes/scrapbook/ScrapbookContent.tsx +++ /dev/null @@ -1,23 +0,0 @@ -import React from "react"; -import { observer } from "mobx-react-lite"; -// Import the Doc type from your actual module. -import { Doc } from "../../../../fields/Doc"; - -export interface ScrapbookContentProps { - doc: Doc; -} - -// A simple view that displays a document's title and content. -// Adjust how you extract the text if your Doc fields are objects. -export const ScrapbookContent: React.FC<ScrapbookContentProps> = observer(({ doc }) => { - // If doc.title or doc.content are not plain strings, convert them. - const titleText = doc.title ? doc.title.toString() : "Untitled"; - const contentText = doc.content ? doc.content.toString() : "No content available."; - - return ( - <div className="scrapbook-content"> - <h3>{titleText}</h3> - <p>{contentText}</p> - </div> - ); -}); diff --git a/src/client/views/nodes/scrapbook/ScrapbookPreset.tsx b/src/client/views/nodes/scrapbook/ScrapbookPreset.tsx new file mode 100644 index 000000000..706b9dafd --- /dev/null +++ b/src/client/views/nodes/scrapbook/ScrapbookPreset.tsx @@ -0,0 +1,176 @@ +import { DocumentType } from '../../../documents/DocumentTypes'; + +export enum ScrapbookPresetType { + Default = 'Default', + Classic = 'Classic', + None = 'Select Template', + Collage = 'Collage', + Spotlight = 'Spotlight', + Gallery = 'Gallery' +} + +export interface ScrapbookItemConfig { + type: DocumentType; + /** text shown in the placeholder bubble */ + tag: string; + /** what this slot actually accepts (defaults to `tag`) */ + acceptTags?: string[]; + + x: number; + y: number; + /** the frame this placeholder occupies */ + width?: number; + height?: number; + /** if this is a container with children, use these for the proto’s own size */ + containerWidth?: number; + containerHeight?: number; + children?: ScrapbookItemConfig[]; +} + +export class ScrapbookPreset { + static createPreset(presetType: ScrapbookPresetType): ScrapbookItemConfig[] { + switch (presetType) { + case ScrapbookPresetType.None: + return ScrapbookPreset.createNonePreset(); + case ScrapbookPresetType.Classic: + return ScrapbookPreset.createClassicPreset(); + case ScrapbookPresetType.Collage: + return ScrapbookPreset.createCollagePreset(); + case ScrapbookPresetType.Spotlight: + return ScrapbookPreset.createSpotlightPreset(); + case ScrapbookPresetType.Default: + return ScrapbookPreset.createDefaultPreset(); + case ScrapbookPresetType.Gallery: + return ScrapbookPreset.createGalleryPreset(); + default: + throw new Error(`Unknown preset type: ${presetType}`); + } + } + + private static createNonePreset(): ScrapbookItemConfig[] { + return [ + + { type: DocumentType.RTF, + tag: 'To create a scrapbook from existing documents, marquee select. For existing scrapbook arrangements, select a preset from the dropdown.', + acceptTags: ['n/a'], + x: 0, y: 0, width: 250, height: 200 + }, + + ]; + } + + private static createClassicPreset(): ScrapbookItemConfig[] { + return [ + { type: DocumentType.IMG, + tag: '[placeholder] LANDSCAPE', + acceptTags: ['LANDSCAPE'], + x: 0, y: -100, width: 250, height: 200 + }, + { type: DocumentType.RTF, + tag: '[placeholder] caption', + acceptTags: ['sentence'], + x: 0, y: 200, width: 250, height: 50 + }, + { type: DocumentType.RTF, + tag: '[placeholder] lengthy description', + acceptTags: ['paragraphs'], + x: 280, y: -50, width: 50, height: 200 + }, + { type: DocumentType.IMG, + tag: '[placeholder] PERSON', + acceptTags: ['PERSON'], + x: -200, y: -100, width: 100, height: 200 + }, + ]; + } + + private static createGalleryPreset(): ScrapbookItemConfig[] { + return [ + { type: DocumentType.IMG, tag: 'Gallery 1', acceptTags: ['LANDSCAPE'], x: -150, y: -150, width: 150, height: 150 }, + { type: DocumentType.IMG, tag: 'Gallery 2', acceptTags: ['LANDSCAPE'], x: 0, y: -150, width: 150, height: 150 }, + { type: DocumentType.IMG, tag: 'Gallery 3', acceptTags: ['LANDSCAPE'], x: 150, y: -150, width: 150, height: 150 }, + { type: DocumentType.IMG, tag: 'Gallery 4', acceptTags: ['LANDSCAPE'], x: -150, y: 0, width: 150, height: 150 }, + { type: DocumentType.IMG, tag: 'Gallery 5', acceptTags: ['LANDSCAPE'], x: 0, y: 0, width: 150, height: 150 }, + { type: DocumentType.IMG, tag: 'Gallery 6', acceptTags: ['LANDSCAPE'], x: 150, y: 0, width: 150, height: 150 }, + ]; + } + + + private static createDefaultPreset(): ScrapbookItemConfig[] { + return [ + { type: DocumentType.IMG, + tag: 'image', + acceptTags: ['LANDSCAPE'], + x: 0, y: -100, width: 250, height: 200 + }, + { type: DocumentType.RTF, + tag: 'summary', + acceptTags: ['sentence'], + x: 0, y: 200, width: 250 + }, + { type: DocumentType.RTF, + tag: 'sidebar', + acceptTags: ['paragraphs'], + x: 280, y: -50, width: 50, height: 200 + }, + { + type: DocumentType.COL, + tag: 'internal coll', + x: -200, y: -100, width: 100, height: 200, + containerWidth: 300, containerHeight: 300, + children: [ + { type: DocumentType.IMG, + tag: 'image internal', + acceptTags: ['PERSON'], + x: 0, y: 0, width: 50, height: 100 + } + ] + } + ]; + } + + private static createCollagePreset(): ScrapbookItemConfig[] { + return [ + { type: DocumentType.IMG, + tag: 'LANDSCAPE', + acceptTags: ['LANDSCAPE'], + x: -150, y: -150, width: 150, height: 150 + }, + { type: DocumentType.IMG, + tag: 'PERSON', + acceptTags: ['PERSON'], + x: 0, y: -150, width: 150, height: 150 + }, + { type: DocumentType.RTF, + tag: 'caption', + acceptTags: ['sentence'], + x: -150, y: 0, width: 300, height: 100 + }, + { type: DocumentType.RTF, + tag: 'lengthy description', + acceptTags: ['paragraphs'], + x: 0, y: 100, width: 300, height: 100 + } + ]; + } + + private static createSpotlightPreset(): ScrapbookItemConfig[] { + return [ + { type: DocumentType.RTF, + tag: 'title', + acceptTags: ['word'], + x: 0, y: -180, width: 300, height: 40 + }, + { type: DocumentType.IMG, + tag: 'LANDSCAPE', + acceptTags: ['LANDSCAPE'], + x: 0, y: 0, width: 300, height: 200 + }, + { type: DocumentType.RTF, + tag: 'caption', + acceptTags: ['sentence'], + x: 0, y: 230, width: 300, height: 50 + } + ]; + } +} diff --git a/src/client/views/nodes/scrapbook/ScrapbookPresetRegistry.ts b/src/client/views/nodes/scrapbook/ScrapbookPresetRegistry.ts new file mode 100644 index 000000000..c6d67ab73 --- /dev/null +++ b/src/client/views/nodes/scrapbook/ScrapbookPresetRegistry.ts @@ -0,0 +1,44 @@ +import { ScrapbookItemConfig } from './ScrapbookPreset'; +import { ScrapbookPresetType } from './ScrapbookPreset'; + +type PresetGenerator = () => ScrapbookItemConfig[]; + +// Internal map of preset name to generator +const presetRegistry = new Map<string, PresetGenerator>(); + + + + +/** + * Register a new scrapbook preset under the given name. + */ +export function registerPreset(name: string, gen: PresetGenerator) { + presetRegistry.set(name, gen); +} + +/** + * List all registered preset names. + */ +export function getPresetNames(): string[] { + return Array.from(presetRegistry.keys()); +} + +/** + * Create the config array for the named preset. + */ +export function createPreset(name: string): ScrapbookItemConfig[] { + const gen = presetRegistry.get(name); + if (!gen) throw new Error(`Unknown scrapbook preset: ${name}`); + return gen(); +} + +// ------------------------ +// Register built-in presets +import { ScrapbookPreset } from './ScrapbookPreset'; + +registerPreset(ScrapbookPresetType.None, () => ScrapbookPreset.createPreset(ScrapbookPresetType.None)); +registerPreset(ScrapbookPresetType.Classic, () => ScrapbookPreset.createPreset(ScrapbookPresetType.Classic)); +registerPreset(ScrapbookPresetType.Collage, () => ScrapbookPreset.createPreset(ScrapbookPresetType.Collage)); +registerPreset(ScrapbookPresetType.Spotlight, () => ScrapbookPreset.createPreset(ScrapbookPresetType.Spotlight)); +registerPreset(ScrapbookPresetType.Default, () => ScrapbookPreset.createPreset(ScrapbookPresetType.Default)); +registerPreset(ScrapbookPresetType.Gallery, () => ScrapbookPreset.createPreset(ScrapbookPresetType.Gallery)); diff --git a/src/client/views/nodes/scrapbook/ScrapbookSlot.scss b/src/client/views/nodes/scrapbook/ScrapbookSlot.scss deleted file mode 100644 index ae647ad36..000000000 --- a/src/client/views/nodes/scrapbook/ScrapbookSlot.scss +++ /dev/null @@ -1,85 +0,0 @@ -//IGNORE FOR NOW, CURRENTLY NOT USED IN SCRAPBOOK IMPLEMENTATION -.scrapbook-slot { - position: absolute; - background-color: rgba(245, 245, 245, 0.7); - border: 2px dashed #ccc; - border-radius: 5px; - box-sizing: border-box; - transition: all 0.2s ease; - overflow: hidden; - - &.scrapbook-slot-over { - border-color: #4a90e2; - background-color: rgba(74, 144, 226, 0.1); - } - - &.scrapbook-slot-filled { - border-style: solid; - border-color: rgba(0, 0, 0, 0.1); - background-color: transparent; - - &.scrapbook-slot-over { - border-color: #4a90e2; - background-color: rgba(74, 144, 226, 0.1); - } - } - - .scrapbook-slot-empty { - display: flex; - align-items: center; - justify-content: center; - width: 100%; - height: 100%; - } - - .scrapbook-slot-placeholder { - text-align: center; - color: #888; - } - - .scrapbook-slot-title { - font-weight: bold; - margin-bottom: 5px; - } - - .scrapbook-slot-instruction { - font-size: 0.9em; - font-style: italic; - } - - .scrapbook-slot-content { - width: 100%; - height: 100%; - position: relative; - } - - .scrapbook-slot-controls { - position: absolute; - top: 5px; - right: 5px; - z-index: 10; - opacity: 0; - transition: opacity 0.2s ease; - - .scrapbook-slot-remove-btn { - background-color: rgba(255, 255, 255, 0.8); - border: 1px solid #ccc; - border-radius: 50%; - width: 20px; - height: 20px; - display: flex; - align-items: center; - justify-content: center; - cursor: pointer; - font-size: 10px; - - &:hover { - background-color: rgba(255, 0, 0, 0.1); - } - } - } - - &:hover .scrapbook-slot-controls { - opacity: 1; - } -}
\ No newline at end of file diff --git a/src/client/views/nodes/scrapbook/ScrapbookSlot.tsx b/src/client/views/nodes/scrapbook/ScrapbookSlot.tsx deleted file mode 100644 index 2c8f93778..000000000 --- a/src/client/views/nodes/scrapbook/ScrapbookSlot.tsx +++ /dev/null @@ -1,28 +0,0 @@ - -//IGNORE FOR NOW, CURRENTLY NOT USED IN SCRAPBOOK IMPLEMENTATION -export interface SlotDefinition { - id: string; - x: number; y: number; - defaultWidth: number; - defaultHeight: number; - } - - export interface SlotContentMap { - slotId: string; - docId?: string; - } - - export interface ScrapbookConfig { - slots: SlotDefinition[]; - contents?: SlotContentMap[]; - } - - export const DEFAULT_SCRAPBOOK_CONFIG: ScrapbookConfig = { - slots: [ - { id: "slot1", x: 10, y: 10, defaultWidth: 180, defaultHeight: 120 }, - { id: "slot2", x: 200, y: 10, defaultWidth: 180, defaultHeight: 120 }, - // …etc - ], - contents: [] - }; -
\ No newline at end of file diff --git a/src/client/views/nodes/scrapbook/ScrapbookSlotTypes.ts b/src/client/views/nodes/scrapbook/ScrapbookSlotTypes.ts deleted file mode 100644 index 686917d9a..000000000 --- a/src/client/views/nodes/scrapbook/ScrapbookSlotTypes.ts +++ /dev/null @@ -1,25 +0,0 @@ -// ScrapbookSlotTypes.ts -export interface SlotDefinition { - id: string; - title: string; - x: number; - y: number; - defaultWidth: number; - defaultHeight: number; - } - - export interface ScrapbookConfig { - slots: SlotDefinition[]; - contents?: { slotId: string; docId: string }[]; - } - - // give it three slots by default: - export const DEFAULT_SCRAPBOOK_CONFIG: ScrapbookConfig = { - slots: [ - { id: "main", title: "Main Content", x: 20, y: 20, defaultWidth: 360, defaultHeight: 200 }, - { id: "notes", title: "Notes", x: 20, y: 240, defaultWidth: 360, defaultHeight: 160 }, - { id: "resources", title: "Resources", x: 400, y: 20, defaultWidth: 320, defaultHeight: 380 }, - ], - contents: [], - }; -
\ No newline at end of file diff --git a/src/client/views/search/FaceRecognitionHandler.tsx b/src/client/views/search/FaceRecognitionHandler.tsx index 3ad5bc844..256e68afd 100644 --- a/src/client/views/search/FaceRecognitionHandler.tsx +++ b/src/client/views/search/FaceRecognitionHandler.tsx @@ -9,6 +9,8 @@ import { ImageField } from '../../../fields/URLField'; import { DocumentType } from '../../documents/DocumentTypes'; import { Docs } from '../../documents/Documents'; import { DocumentManager } from '../../util/DocumentManager'; +import { reaction } from 'mobx'; +import { DocumentView } from '../nodes/DocumentView'; /** * A singleton class that handles face recognition and manages face Doc collections for each face found. @@ -33,7 +35,7 @@ export class FaceRecognitionHandler { // eslint-disable-next-line no-use-before-define static _instance: FaceRecognitionHandler; private _apiModelReady = false; - private _pendingAPIModelReadyDocs: Doc[] = []; + private _pendingAPIModelReadyDocs: DocumentView[] = []; public static get Instance() { return FaceRecognitionHandler._instance ?? new FaceRecognitionHandler(); @@ -126,7 +128,7 @@ export class FaceRecognitionHandler { constructor() { FaceRecognitionHandler._instance = this; this.loadAPIModels().then(() => this._pendingAPIModelReadyDocs.forEach(this.classifyFacesInImage)); - DocumentManager.Instance.AddAnyViewRenderedCB(dv => FaceRecognitionHandler.Instance.classifyFacesInImage(dv.Document)); + DocumentManager.Instance.AddAnyViewRenderedCB(dv => FaceRecognitionHandler.Instance.classifyFacesInImage(dv)); } /** @@ -199,14 +201,18 @@ export class FaceRecognitionHandler { * match them to existing unique faces, otherwise new unique face(s) are created. * @param imgDoc The document being analyzed. */ - private classifyFacesInImage = async (imgDoc: Doc) => { + private classifyFacesInImage = async (imgDocView: DocumentView) => { + const imgDoc = imgDocView.Document; if (!Doc.UserDoc().recognizeFaceImages) return; const activeDashboard = Doc.ActiveDashboard; if (!this._apiModelReady || !activeDashboard) { - this._pendingAPIModelReadyDocs.push(imgDoc); + this._pendingAPIModelReadyDocs.push(imgDocView); } else if (imgDoc.type === DocumentType.LOADING && !imgDoc.loadingError) { - setTimeout(() => this.classifyFacesInImage(imgDoc), 1000); + setTimeout(() => this.classifyFacesInImage(imgDocView), 1000); } else { + reaction(() => ({sel:imgDocView.isSelected()}), ({sel}) => !sel && + imgDocView.ComponentView?.autoTag?.(), {fireImmediately: true} + ) const imgUrl = ImageCast(imgDoc[Doc.LayoutDataKey(imgDoc)]); if (imgUrl && !DocListCast(Doc.MyFaceCollection?.examinedFaceDocs).includes(imgDoc[DocData])) { // only examine Docs that have an image and that haven't already been examined. |