diff options
Diffstat (limited to 'src/client/views')
21 files changed, 1070 insertions, 278 deletions
diff --git a/src/client/views/DocumentDecorations.tsx b/src/client/views/DocumentDecorations.tsx index 69c2467a3..f36312056 100644 --- a/src/client/views/DocumentDecorations.tsx +++ b/src/client/views/DocumentDecorations.tsx @@ -36,6 +36,7 @@ import { ImageBox } from './nodes/ImageBox'; import { OpenWhere, OpenWhereMod } from './nodes/OpenWhere'; import { FormattedTextBox } from './nodes/formattedText/FormattedTextBox'; import { TagsView } from './TagsView'; +import { ScrapbookBox } from './nodes/scrapbook/ScrapbookBox'; interface DocumentDecorationsProps { PanelWidth: number; @@ -430,7 +431,7 @@ export class DocumentDecorations extends ObservableReactComponent<DocumentDecora onPointerDown = (e: React.PointerEvent): void => { SnappingManager.SetIsResizing(DocumentView.Selected().lastElement()?.Document[Id]); // turns off pointer events on things like youtube videos and web pages so that dragging doesn't get "stuck" when cursor moves over them DocumentView.Selected() - .filter(dv => e.shiftKey && dv.ComponentView instanceof ImageBox) + .filter(dv => e.shiftKey && dv.ComponentView instanceof ImageBox || dv.ComponentView instanceof ScrapbookBox) .forEach(dv => { dv.Document[dv.ComponentView!.fieldKey + '_outpaintOriginalWidth'] = NumCast(dv.Document._width); dv.Document[dv.ComponentView!.fieldKey + '_outpaintOriginalHeight'] = NumCast(dv.Document._height); @@ -486,7 +487,7 @@ export class DocumentDecorations extends ObservableReactComponent<DocumentDecora this._interactionLock = true; this._snapPt = thisPt; - const outpainted = e.shiftKey ? DocumentView.Selected().filter(dv => dv.ComponentView instanceof ImageBox) : []; + const outpainted = e.shiftKey ? DocumentView.Selected().filter(dv => dv.ComponentView instanceof ImageBox || dv.ComponentView instanceof ScrapbookBox) : []; const notOutpainted = e.shiftKey ? DocumentView.Selected().filter(dv => !outpainted.includes(dv)) : DocumentView.Selected(); // Special handling for shift-drag resize (outpainting of Images by resizing without scaling content - fill in with firefly GAI) @@ -750,7 +751,7 @@ export class DocumentDecorations extends ObservableReactComponent<DocumentDecora const rotation = DocumentView.Selected().length === 1 ? seldocview.screenToContentsTransform().inverse().RotateDeg : 0; // Radius constants - const useRounding = seldocview.ComponentView instanceof ImageBox || seldocview.ComponentView instanceof FormattedTextBox || seldocview.ComponentView instanceof CollectionFreeFormView; + const useRounding = seldocview.ComponentView instanceof ImageBox || seldocview.ComponentView instanceof ScrapbookBox || seldocview.ComponentView instanceof FormattedTextBox || seldocview.ComponentView instanceof CollectionFreeFormView; const borderRadius = numberValue(Cast(seldocview.Document.layout_borderRounding, 'string', null)); const docMax = Math.min(NumCast(seldocview.Document._width) / 2, NumCast(seldocview.Document._height) / 2); const maxDist = Math.min((this.Bounds.r - this.Bounds.x) / 2, (this.Bounds.b - this.Bounds.y) / 2); diff --git a/src/client/views/ViewBoxInterface.ts b/src/client/views/ViewBoxInterface.ts index d8dab8e89..b532dfe35 100644 --- a/src/client/views/ViewBoxInterface.ts +++ b/src/client/views/ViewBoxInterface.ts @@ -24,6 +24,7 @@ export abstract class ViewBoxInterface<P> extends ObservableReactComponent<React promoteCollection?: () => void; // moves contents of collection to parent hasChildDocs?: () => Doc[]; docEditorView?: () => void; + autoTag?: () => void; // auto tag the document showSmartDraw?: (x: number, y: number, regenerate?: boolean) => void; updateIcon?: (usePanelDimensions?: boolean) => Promise<void>; // updates the icon representation of the document getAnchor?: (addAsAnnotation: boolean, pinData?: PinProps) => Doc; // returns an Anchor Doc that represents the current state of the doc's componentview (e.g., the current playhead location of a an audio/video box) diff --git a/src/client/views/collections/collectionFreeForm/ImageLabelBox.tsx b/src/client/views/collections/collectionFreeForm/ImageLabelBox.tsx index ff9fb14e7..038b1c6f9 100644 --- a/src/client/views/collections/collectionFreeForm/ImageLabelBox.tsx +++ b/src/client/views/collections/collectionFreeForm/ImageLabelBox.tsx @@ -158,17 +158,19 @@ export class ImageLabelBox extends ViewBoxBaseComponent<FieldViewProps>() { this._currentLabel = e.target.value; }); - classifyImagesInBox = async () => { + classifyImagesInBox = async (selectedImages? : Doc[], prompt? : string) => { this.startLoading(); + alert('Classifying images...'); + selectedImages ??= this._selectedImages; // Converts the images into a Base64 format, afterwhich the information is sent to GPT to label them. - const imageInfos = this._selectedImages.map(async doc => { + const imageInfos = selectedImages.map(async doc => { if (!doc.$tags_chat) { const url = ImageCastWithSuffix(doc[Doc.LayoutDataKey(doc)], '_o') ?? ''; return imageUrlToBase64(url).then(hrefBase64 => !hrefBase64 ? undefined : - gptImageLabel(hrefBase64,'Give three labels to describe this image.').then(labels => + gptImageLabel(hrefBase64, prompt ?? 'Give three labels to describe this image.').then(labels => ({ doc, labels }))) ; // prettier-ignore } }); diff --git a/src/client/views/collections/collectionFreeForm/MarqueeOptionsMenu.tsx b/src/client/views/collections/collectionFreeForm/MarqueeOptionsMenu.tsx index abd828945..2ec59e5d5 100644 --- a/src/client/views/collections/collectionFreeForm/MarqueeOptionsMenu.tsx +++ b/src/client/views/collections/collectionFreeForm/MarqueeOptionsMenu.tsx @@ -16,6 +16,7 @@ export class MarqueeOptionsMenu extends AntimodeMenu<AntimodeMenuProps> { public createCollection: (e: KeyboardEvent | React.PointerEvent | undefined, group?: boolean, selection?: Doc[]) => Doc | void = unimplementedFunction; public delete: (e: KeyboardEvent | React.PointerEvent | undefined) => void = unimplementedFunction; public summarize: (e: KeyboardEvent | React.PointerEvent | undefined) => void = unimplementedFunction; + public generateScrapbook: (e: KeyboardEvent | React.PointerEvent | undefined) => void = unimplementedFunction; public showMarquee: () => void = unimplementedFunction; public hideMarquee: () => void = unimplementedFunction; public pinWithView: (e: KeyboardEvent | React.PointerEvent | undefined) => void = unimplementedFunction; @@ -38,6 +39,7 @@ export class MarqueeOptionsMenu extends AntimodeMenu<AntimodeMenuProps> { <IconButton tooltip="Create a Collection" onPointerDown={this.createCollection} icon={<FontAwesomeIcon icon="object-group" />} color={this.userColor} /> <IconButton tooltip="Create a Grouping" onPointerDown={e => this.createCollection(e, true)} icon={<FontAwesomeIcon icon="layer-group" />} color={this.userColor} /> <IconButton tooltip="Summarize Documents" onPointerDown={this.summarize} icon={<FontAwesomeIcon icon="compress-arrows-alt" />} color={this.userColor} /> + <IconButton tooltip="Generate Scrapbook" onPointerDown={this.generateScrapbook} icon={<FontAwesomeIcon icon="palette" />} color={this.userColor} /> <IconButton tooltip="Delete Documents" onPointerDown={this.delete} icon={<FontAwesomeIcon icon="trash-alt" />} color={this.userColor} /> <IconButton tooltip="Pin selected region" onPointerDown={this.pinWithView} icon={<FontAwesomeIcon icon="map-pin" />} color={this.userColor} /> <IconButton tooltip="Classify and Sort Images" onPointerDown={this.classifyImages} icon={<FontAwesomeIcon icon="object-group" />} color={this.userColor} /> diff --git a/src/client/views/collections/collectionFreeForm/MarqueeView.scss b/src/client/views/collections/collectionFreeForm/MarqueeView.scss index 7c9d0f6e1..b514b0911 100644 --- a/src/client/views/collections/collectionFreeForm/MarqueeView.scss +++ b/src/client/views/collections/collectionFreeForm/MarqueeView.scss @@ -28,4 +28,4 @@ .marquee-legend::after { content: 'Press <space> for lasso'; } -} +}
\ No newline at end of file diff --git a/src/client/views/collections/collectionFreeForm/MarqueeView.tsx b/src/client/views/collections/collectionFreeForm/MarqueeView.tsx index c120cddf0..b2b904509 100644 --- a/src/client/views/collections/collectionFreeForm/MarqueeView.tsx +++ b/src/client/views/collections/collectionFreeForm/MarqueeView.tsx @@ -16,7 +16,7 @@ import { DocumentType } from '../../../documents/DocumentTypes'; import { Docs, DocumentOptions } from '../../../documents/Documents'; import { SnappingManager, freeformScrollMode } from '../../../util/SnappingManager'; import { Transform } from '../../../util/Transform'; -import { UndoManager, undoBatch } from '../../../util/UndoManager'; +import { UndoManager, undoBatch, undoable } from '../../../util/UndoManager'; import { ContextMenu } from '../../ContextMenu'; import { ObservableReactComponent } from '../../ObservableReactComponent'; import { MarqueeViewBounds } from '../../PinFuncs'; @@ -28,8 +28,13 @@ import { FormattedTextBox } from '../../nodes/formattedText/FormattedTextBox'; import { SubCollectionViewProps } from '../CollectionSubView'; import { ImageLabelBoxData } from './ImageLabelBox'; import { MarqueeOptionsMenu } from './MarqueeOptionsMenu'; +import { StrListCast } from '../../../../fields/Doc'; +import { requestAiGeneratedPreset, DocumentDescriptor } from '../../nodes/scrapbook/AIPresetGenerator'; +import { buildPlaceholdersFromConfigs, slotRealDocIntoPlaceholders } from '../../nodes/scrapbook/ScrapbookBox'; import './MarqueeView.scss'; +import { build } from 'xregexp'; + interface MarqueeViewProps { Doc: Doc; getContainerTransform: () => Transform; @@ -76,6 +81,14 @@ export class MarqueeView extends ObservableReactComponent<SubCollectionViewProps @observable _labelsVisibile: boolean = false; @observable _lassoPts: [number, number][] = []; @observable _lassoFreehand: boolean = false; + // ─── New Observables for “Pick 1 of N AI Scrapbook” ─── + @observable aiChoices: Doc[] = []; // temporary hidden Scrapbook docs + @observable pickerX = 0; // popup x coordinate + @observable pickerY = 0; // popup y coordinate + @observable pickerVisible = false; // show/hide ScrapbookPicker + + + @computed get Transform() { return this._props.getTransform(); @@ -276,6 +289,7 @@ export class MarqueeView extends ObservableReactComponent<SubCollectionViewProps MarqueeOptionsMenu.Instance.createCollection = this.collection; MarqueeOptionsMenu.Instance.delete = this.delete; MarqueeOptionsMenu.Instance.summarize = this.summary; + MarqueeOptionsMenu.Instance.generateScrapbook = this.generateScrapbook; MarqueeOptionsMenu.Instance.showMarquee = this.showMarquee; MarqueeOptionsMenu.Instance.hideMarquee = this.hideMarquee; MarqueeOptionsMenu.Instance.jumpTo(e.clientX, e.clientY); @@ -518,6 +532,102 @@ export class MarqueeView extends ObservableReactComponent<SubCollectionViewProps MarqueeOptionsMenu.Instance.fadeOut(true); }); + + getAiPresetsDescriptors(): DocumentDescriptor[] { + const selected = this.marqueeSelect(false); + return selected.map((doc) => ({ + type: typeof doc.$type === 'string' ? doc.$type : 'UNKNOWN', + tags: (() => { + const s = new Set<string>(); + StrListCast(doc.$tags_chat ?? new List<string>()).forEach((t) => s.add(t)); + return Array.from(s); + })(), + })); + } + + + generateScrapbook = action(async () => { + + const selectedDocs = this.marqueeSelect(false); + if (!selectedDocs.length) return; + + const descriptors = this.getAiPresetsDescriptors(); + if (descriptors.length === 0) { + alert('No documents selected to generate a scrapbook from!'); + return; + } + + const aiPreset = await requestAiGeneratedPreset(descriptors); + if (!aiPreset.length) { + alert("Failed to generate preset"); + return; + } + const scrapbookPlaceholders: Doc[] = buildPlaceholdersFromConfigs(aiPreset); + /* + const scrapbookPlaceholders: Doc[] = aiPreset.map(cfg => { + const placeholderDoc = Docs.Create.TextDocument(cfg.tag); + placeholderDoc.accepts_docType = cfg.type as DocumentType; + placeholderDoc.accepts_tagType = new List<string>(cfg.acceptTags ?? [cfg.tag]); + + const placeholder = new Doc(); + placeholder.proto = placeholderDoc; + placeholder.original = placeholderDoc; + placeholder.x = cfg.x; + placeholder.y = cfg.y; + if (cfg.width != null) placeholder._width = cfg.width; + if (cfg.height != null) placeholder._height = cfg.height; + + return placeholder; + });*/ + + const scrapbook = Docs.Create.ScrapbookDocument(scrapbookPlaceholders, { + backgroundColor: '#e2ad32', + x: this.Bounds.left, + y: this.Bounds.top, + _width: 500, + _height: 500, + title: 'AI-generated Scrapbook' + }); + + + + // 3) Now grab that new scrapbook’s flat placeholders + const flatPl = DocListCast(scrapbook[Doc.LayoutDataKey(scrapbook)]) as Doc[]; + const unwrap = (items: Doc[]): Doc[] => + items.flatMap(d => + d.$type === DocumentType.COL + ? unwrap(DocListCast(d[Doc.LayoutDataKey(d)])) + : [d] + ); + const allPlaceholders = unwrap(flatPl); + + // 4) Slot each selectedDocs[i] into the first matching placeholder + selectedDocs.forEach(realDoc => { + slotRealDocIntoPlaceholders(realDoc, allPlaceholders + ); + }); + + const selected = this.marqueeSelect(false).map(d => { + this._props.removeDocument?.(d); + d.x = NumCast(d.x) - this.Bounds.left; + d.y = NumCast(d.y) - this.Bounds.top; + return d; + }); + + this._props.addDocument?.(scrapbook); + selectedDocs.forEach(doc => this._props.removeDocument?.(doc)); + const portal = Docs.Create.FreeformDocument(selected, { title: 'summarized documents', x: this.Bounds.left + 200, y: this.Bounds.top, isGroup: true, backgroundColor: 'transparent' }); + DocUtils.MakeLink(scrapbook, portal, { link_relationship: 'summary of:summarized by' }); + + portal.hidden = true; + this._props.addDocument?.(portal); + MarqueeOptionsMenu.Instance.fadeOut(true); + this.hideMarquee(); + }); + + + + @action marqueeCommand = (e: KeyboardEvent) => { const ee = e as unknown as KeyboardEvent & { propagationIsStopped?: boolean }; @@ -539,6 +649,7 @@ export class MarqueeView extends ObservableReactComponent<SubCollectionViewProps if (e.key === 'g') this.collection(e, true); if (e.key === 'c' || e.key === 't') this.collection(e); if (e.key === 's' || e.key === 'S') this.summary(); + if (e.key === 'g' || e.key === 'G') this.generateScrapbook(); // ← scrapbook shortcut if (e.key === 'p') this.pileup(); this.cleanupInteractions(false); } @@ -683,6 +794,7 @@ export class MarqueeView extends ObservableReactComponent<SubCollectionViewProps }; render() { return ( + <> <div className="marqueeView" ref={r => { @@ -702,6 +814,8 @@ export class MarqueeView extends ObservableReactComponent<SubCollectionViewProps {this._visible ? this.marqueeDiv : null} {this.props.children} </div> + </> + ); } } diff --git a/src/client/views/nodes/ImageBox.tsx b/src/client/views/nodes/ImageBox.tsx index 8ed59c6e1..1e16bbfc9 100644 --- a/src/client/views/nodes/ImageBox.tsx +++ b/src/client/views/nodes/ImageBox.tsx @@ -8,7 +8,7 @@ import { extname } from 'path'; import * as React from 'react'; import { AiOutlineSend } from 'react-icons/ai'; import ReactLoading from 'react-loading'; -import { ClientUtils, DashColor, returnEmptyString, returnFalse, returnOne, returnZero, setupMoveUpEvents, UpdateIcon } from '../../../ClientUtils'; +import { ClientUtils, imageUrlToBase64, DashColor, returnEmptyString, returnFalse, returnOne, returnZero, setupMoveUpEvents, UpdateIcon } from '../../../ClientUtils'; import { Doc, DocListCast, Opt } from '../../../fields/Doc'; import { DocData } from '../../../fields/DocSymbols'; import { Id } from '../../../fields/FieldSymbols'; @@ -16,7 +16,7 @@ import { InkTool } from '../../../fields/InkField'; import { List } from '../../../fields/List'; import { ObjectField } from '../../../fields/ObjectField'; import { ComputedField } from '../../../fields/ScriptField'; -import { Cast, DocCast, ImageCast, NumCast, RTFCast, StrCast } from '../../../fields/Types'; +import { Cast, DocCast, ImageCast, NumCast, RTFCast, StrCast, ImageCastWithSuffix } from '../../../fields/Types'; import { ImageField } from '../../../fields/URLField'; import { TraceMobx } from '../../../fields/util'; import { emptyFunction } from '../../../Utils'; @@ -45,6 +45,7 @@ import { FieldView, FieldViewProps } from './FieldView'; import { FocusViewOptions } from './FocusViewOptions'; import './ImageBox.scss'; import { OpenWhere } from './OpenWhere'; +import { gptImageLabel } from '../../apis/gpt/GPT'; const DefaultPath = '/assets/unknown-file-icon-hi.png'; export class ImageEditorData { @@ -139,6 +140,59 @@ export class ImageBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { this._dropDisposer?.(); ele && (this._dropDisposer = DragManager.MakeDropTarget(ele, this.drop.bind(this), this.Document)); }; + + autoTag = async () => { + + try { + // 1) grab the full-size URL + const layoutKey = Doc.LayoutDataKey(this.Document); + const url = ImageCastWithSuffix(this.Document[layoutKey], '_o') ?? ''; + if (!url) throw new Error('No image URL found'); + + // 2) convert to base64 + const base64 = await imageUrlToBase64(url); + if (!base64) throw new Error('Failed to load image data'); + + // 3) ask GPT for labels one label: PERSON or LANDSCAPE + const raw = await gptImageLabel( + base64, + `Classify this image as PERSON or LANDSCAPE. You may only respond with one of these two options. Then + provide five additional descriptive tags to describe the image for a total of 6 words outputted, + delimited by spaces. For example: "LANDSCAPE BUNNY NATURE FOREST PEACEFUL OUTDOORS". Then add one final lengthier summary tag (separated by underscores) + that describes the image.` + ); + + const { nativeWidth, nativeHeight } = this.nativeSize; + const aspectRatio = nativeWidth && nativeHeight + ? (nativeWidth / nativeHeight).toFixed(2) + : '1.00'; + + // 4) normalize and prefix + const label = raw + .trim() + .toUpperCase() + + // 5) stash it on the Doc + // overwrite any old tags so re-runs still work + const tokens = label.split(/\s+/); + this.Document.$tags_chat = new List<string>(); + tokens.forEach(tok => { + (this.Document.$tags_chat as List<string>).push(tok)}); + (this.Document.$tags_chat as List<string>).push(`ASPECT_${aspectRatio}`); + + // 6) flip on “show tags” in the layout + // (same flag that ImageLabelBox.toggleDisplayInformation uses) + this.Document._layout_showTags = true; + + } catch (err) { + console.error('autoTag failed:', err); + } finally { + } + }; + + + + getAnchor = (addAsAnnotation: boolean, pinProps?: PinProps) => { const visibleAnchor = this._getAnchor?.(this._savedAnnotations, true); // use marquee anchor, otherwise, save zoom/pan as anchor const anchor = diff --git a/src/client/views/nodes/PDFBox.tsx b/src/client/views/nodes/PDFBox.tsx index 45fa5cc12..a0c7d8d22 100644 --- a/src/client/views/nodes/PDFBox.tsx +++ b/src/client/views/nodes/PDFBox.tsx @@ -33,6 +33,9 @@ import { ImageBox } from './ImageBox'; import { OpenWhere } from './OpenWhere'; import './PDFBox.scss'; import { CreateImage } from './WebBoxRenderer'; +import { gptAPICall } from '../../apis/gpt/GPT'; +import { List } from '../../../fields/List'; +import { GPTCallType } from '../../apis/gpt/GPT'; @observer export class PDFBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { @@ -78,6 +81,47 @@ export class PDFBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { } } + autoTag = async () => { + try { + if (!this._pdf) { + throw new Error('PDF not loaded'); + } + + // 1) Extract text from the first few pages (e.g., first 2 pages) + const maxPages = Math.min(2, this._pdf.numPages); + let textContent = ''; + for (let pageNum = 1; pageNum <= maxPages; pageNum++) { + const page = await this._pdf.getPage(pageNum); + const text = await page.getTextContent(); + const pageText = text.items.map(item => ('str' in item ? item.str : '')).join(' '); + textContent += ` ${pageText}`; + } + + if (!textContent.trim()) { + throw new Error('No text found in PDF'); + } + + // 2) Ask GPT to classify and provide descriptive tags + const raw = await gptAPICall( + `"${textContent.trim().slice(0, 2000)}"`, + GPTCallType.CLASSIFYTEXTFULL + ); + + // 3) Normalize and store the labels + const label = raw.trim().toUpperCase(); + + const tokens = label.split(/\s+/); + this.Document.$tags_chat = new List<string>(); + tokens.forEach(tok => (this.Document.$tags_chat as List<string>).push(tok)); + + // 4) Show tags in layout + this.Document._layout_showTags = true; + + } catch (err) { + console.error('PDF autoTag failed:', err); + } +}; + replaceCanvases = (oldDiv: HTMLElement, newDiv: HTMLElement) => { if (oldDiv.childNodes) { for (let i = 0; i < oldDiv.childNodes.length; i++) { diff --git a/src/client/views/nodes/VideoBox.tsx b/src/client/views/nodes/VideoBox.tsx index b3cb0e1db..4d85b4942 100644 --- a/src/client/views/nodes/VideoBox.tsx +++ b/src/client/views/nodes/VideoBox.tsx @@ -30,6 +30,7 @@ import { StyleProp } from '../StyleProp'; import { DocumentView } from './DocumentView'; import { FieldView, FieldViewProps } from './FieldView'; import { FocusViewOptions } from './FocusViewOptions'; +import { gptImageLabel } from '../../apis/gpt/GPT'; import './VideoBox.scss'; /** @@ -109,6 +110,57 @@ export class VideoBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { return this._videoRef; } + + autoTag = async () => { + try { + if (!this.player) throw new Error('Video element not available.'); + + // 1) Extract a frame at the video's midpoint + const videoDuration = this.player.duration; + const snapshotTime = videoDuration / 2; + + // Seek the video element to the midpoint + await new Promise<void>((resolve, reject) => { + const onSeeked = () => { + this.player!.removeEventListener('seeked', onSeeked); + resolve(); + }; + this.player!.addEventListener('seeked', onSeeked); + this.player!.currentTime = snapshotTime; + }); + + // 2) Draw the frame onto a canvas and get a base64 representation + const canvas = document.createElement('canvas'); + canvas.width = this.player.videoWidth; + canvas.height = this.player.videoHeight; + const ctx = canvas.getContext('2d'); + if (!ctx) throw new Error('Failed to create canvas context.'); + ctx.drawImage(this.player, 0, 0, canvas.width, canvas.height); + const base64Image = canvas.toDataURL('image/png'); + + // 3) Send the image data to GPT for classification and descriptive tags + const raw = await gptImageLabel( + base64Image, + `Classify this video frame as either a PERSON or LANDSCAPE. + Then provide five additional descriptive tags (single words) separated by spaces. + Finally, add one detailed summary phrase using underscores.` + ); + + // 4) Normalize and store labels in the Document's tags + const label = raw.trim().toUpperCase(); + const tokens = label.split(/\s+/); + this.Document.$tags_chat = new List<string>(); + tokens.forEach(tok => (this.Document.$tags_chat as List<string>).push(tok)); + const aspect = this.player!.videoWidth / (this.player!.videoHeight || 1); + (this.Document.$tags_chat as List<string>).push(`ASPECT_${aspect}`); + // 5) Turn on tag display in layout + this.Document._layout_showTags = true; + + } catch (err) { + console.error('Video autoTag failed:', err); + } +}; + componentDidMount() { this.unmounting = false; this._props.setContentViewBox?.(this); // this tells the DocumentView that this VideoBox is the "content" of the document. this allows the DocumentView to indirectly call getAnchor() on the VideoBox when making a link. diff --git a/src/client/views/nodes/formattedText/FormattedTextBox.tsx b/src/client/views/nodes/formattedText/FormattedTextBox.tsx index c8df6e50f..0c3179173 100644 --- a/src/client/views/nodes/formattedText/FormattedTextBox.tsx +++ b/src/client/views/nodes/formattedText/FormattedTextBox.tsx @@ -8,6 +8,7 @@ import { baseKeymap, selectAll, splitBlock } from 'prosemirror-commands'; import { history } from 'prosemirror-history'; import { inputRules } from 'prosemirror-inputrules'; import { keymap } from 'prosemirror-keymap'; +import { runInAction } from 'mobx'; import { Fragment, Mark, Node, Slice } from 'prosemirror-model'; import { EditorState, NodeSelection, Plugin, Selection, TextSelection, Transaction } from 'prosemirror-state'; import { EditorView, NodeViewConstructor } from 'prosemirror-view'; @@ -64,6 +65,7 @@ import { removeMarkWithAttrs } from './prosemirrorPatches'; import { RichTextMenu, RichTextMenuPlugin } from './RichTextMenu'; import { RichTextRules } from './RichTextRules'; import { schema } from './schema_rts'; +import { tickStep } from 'd3'; // import * as applyDevTools from 'prosemirror-dev-tools'; export interface FormattedTextBoxProps extends FieldViewProps { @@ -308,6 +310,31 @@ export class FormattedTextBox extends ViewBoxAnnotatableComponent<FormattedTextB } }; + autoTag = async () => { + + const layoutKey = Doc.LayoutDataKey(this.Document); + const rawText = RTFCast(this.Document[layoutKey])?.Text ?? StrCast(this.Document[layoutKey]); + + const callType = rawText.includes("[placeholder]") + ? GPTCallType.CLASSIFYTEXTMINIMAL + : GPTCallType.CLASSIFYTEXTFULL; + + gptAPICall(rawText, callType).then(desc => { + runInAction(() => { + // Clear existing tags + this.Document.$tags_chat = new List<string>(); + + // Split GPT response into tokens and push individually + const tokens = desc.trim().split(/\s+/); + tokens.forEach(tok => { + (this.Document.$tags_chat as List<string>).push(tok); + }); + + this.Document._layout_showTags = true; + }); + }); +}; + leafText = (node: Node) => { if (node.type === this.EditorView?.state.schema.nodes.dashField) { const refDoc = !node.attrs.docId ? this.rootDoc : (DocServer.GetCachedRefField(node.attrs.docId as string) as Doc); @@ -1271,6 +1298,14 @@ export class FormattedTextBox extends ViewBoxAnnotatableComponent<FormattedTextB { fireImmediately: true } ); + this._disposers.tagger = reaction( + () => ({ title: this.Document.title, sel: this._props.isSelected() }), + action(() => { + this.autoTag(); + }), + { fireImmediately: true } + ); + if (!this._props.dontRegisterView) { this._disposers.record = reaction( () => this.recordingDictation, diff --git a/src/client/views/nodes/scrapbook/AIPresetGenerator.ts b/src/client/views/nodes/scrapbook/AIPresetGenerator.ts new file mode 100644 index 000000000..1f159222b --- /dev/null +++ b/src/client/views/nodes/scrapbook/AIPresetGenerator.ts @@ -0,0 +1,31 @@ +import { ScrapbookItemConfig } from './ScrapbookPreset'; +import { GPTCallType, gptAPICall } from '../../../apis/gpt/GPT'; + +// Represents the descriptor for each document +export interface DocumentDescriptor { + type: string; + tags: string[]; +} + +// Main function to request AI-generated presets +export async function requestAiGeneratedPreset(descriptors: DocumentDescriptor[]): Promise<ScrapbookItemConfig[]> { + const prompt = createPrompt(descriptors); + let aiResponse = await gptAPICall(prompt, GPTCallType.GENERATESCRAPBOOK); + // Strip out ```json and ``` if the model wrapped its answer in fences + aiResponse = aiResponse + .trim() + .replace(/^```(?:json)?\s*/, "") // remove leading ``` or ```json + .replace(/\s*```$/, ""); // remove trailing ``` + const parsedPreset = JSON.parse(aiResponse) as ScrapbookItemConfig[]; + return parsedPreset; +} + +// Helper to generate prompt text for AI +function createPrompt(descriptors: DocumentDescriptor[]): string { + let prompt = ""; + descriptors.forEach((desc, index) => { + prompt += `${index + 1}. Type: ${desc.type}, Tags: ${desc.tags.join(', ')}\n`; + }); + + return prompt; +} diff --git a/src/client/views/nodes/scrapbook/EmbeddedDocView.tsx b/src/client/views/nodes/scrapbook/EmbeddedDocView.tsx deleted file mode 100644 index e99bf67c7..000000000 --- a/src/client/views/nodes/scrapbook/EmbeddedDocView.tsx +++ /dev/null @@ -1,52 +0,0 @@ -//IGNORE FOR NOW, CURRENTLY NOT USED IN SCRAPBOOK IMPLEMENTATION -import * as React from "react"; -import { observer } from "mobx-react"; -import { Doc } from "../../../../fields/Doc"; -import { DocumentView } from "../DocumentView"; -import { Transform } from "../../../util/Transform"; - -interface EmbeddedDocViewProps { - doc: Doc; - width?: number; - height?: number; - slotId?: string; -} - -@observer -export class EmbeddedDocView extends React.Component<EmbeddedDocViewProps> { - render() { - const { doc, width = 300, height = 200, slotId } = this.props; - - // Use either an existing embedding or create one - let docToDisplay = doc; - - // If we need an embedding, create or use one - if (!docToDisplay.isEmbedding) { - docToDisplay = Doc.BestEmbedding(doc) || Doc.MakeEmbedding(doc); - // Set the container to the slot's ID so we can track it - if (slotId) { - docToDisplay.embedContainer = `scrapbook-slot-${slotId}`; - } - } - - return ( - <DocumentView - Document={docToDisplay} - renderDepth={0} - // Required sizing functions - NativeWidth={() => width} - NativeHeight={() => height} - PanelWidth={() => width} - PanelHeight={() => height} - // Required state functions - isContentActive={() => true} - childFilters={() => []} - ScreenToLocalTransform={() => new Transform()} - // Display options - hideDeleteButton={true} - hideDecorations={true} - hideResizeHandles={true} - /> - ); - } -}
\ No newline at end of file diff --git a/src/client/views/nodes/scrapbook/ScrapbookBox.scss b/src/client/views/nodes/scrapbook/ScrapbookBox.scss new file mode 100644 index 000000000..8dc93df60 --- /dev/null +++ b/src/client/views/nodes/scrapbook/ScrapbookBox.scss @@ -0,0 +1,63 @@ + +.scrapbook-box { + /* Make sure the container fills its parent, and set a base background */ + position: relative; /* so that absolute children (loading overlay, etc.) are positioned relative to this */ + width: 100%; + height: 100%; + background: beige; + overflow: hidden; /* prevent scrollbars if children overflow */ +} + +/* Loading overlay that covers the entire scrapbook while AI-generation is in progress */ +.scrapbook-box-loading-overlay { + position: absolute; + top: 0; + left: 0; + width: 100%; + height: 100%; + display: flex; + justify-content: center; + align-items: center; + background: rgba(255, 255, 255, 0.8); + z-index: 10; /* sits above the ImageBox and other content */ +} + +/* The <select> dropdown for choosing presets */ +.scrapbook-box-preset-select { + position: absolute; + top: 8px; + left: 8px; + z-index: 20; + padding: 4px 8px; + font-size: 14px; + border: 1px solid #ccc; + border-radius: 4px; + background: white; +} + +/* Container for the “Regenerate Background” button */ +.scrapbook-box-ui { + position: absolute; + top: 8px; + right: 8px; + z-index: 20; +} + +/* The button itself */ +.scrapbook-box-ui-button { + display: flex; + align-items: center; + gap: 6px; + padding: 4px 8px; + font-size: 14px; + color: black; + background: white; + border: 1px solid #ccc; + border-radius: 4px; + cursor: pointer; + white-space: nowrap; +} + +.scrapbook-box-ui-button:hover { + background: #f5f5f5; +} diff --git a/src/client/views/nodes/scrapbook/ScrapbookBox.tsx b/src/client/views/nodes/scrapbook/ScrapbookBox.tsx index 6cfe9a62c..52e3c26dc 100644 --- a/src/client/views/nodes/scrapbook/ScrapbookBox.tsx +++ b/src/client/views/nodes/scrapbook/ScrapbookBox.tsx @@ -1,32 +1,178 @@ -import { action, makeObservable, observable } from 'mobx'; +import { action, makeObservable, observable, reaction, computed } from 'mobx'; import * as React from 'react'; -import { Doc, DocListCast } from '../../../../fields/Doc'; +import { Doc, DocListCast, StrListCast } from '../../../../fields/Doc'; import { List } from '../../../../fields/List'; import { emptyFunction } from '../../../../Utils'; import { Docs } from '../../../documents/Documents'; import { DocumentType } from '../../../documents/DocumentTypes'; import { CollectionView } from '../../collections/CollectionView'; import { ViewBoxAnnotatableComponent } from '../../DocComponent'; +import { AspectRatioLimits } from '../../smartdraw/FireflyConstants'; import { DocumentView } from '../DocumentView'; import { FieldView, FieldViewProps } from '../FieldView'; import { DragManager } from '../../../util/DragManager'; -import { RTFCast, StrCast, toList } from '../../../../fields/Types'; +import { toList } from '../../../../fields/Types'; import { undoable } from '../../../util/UndoManager'; -// Scrapbook view: a container that lays out its child items in a grid/template +import ReactLoading from 'react-loading'; +import { NumCast } from '../../../../fields/Types'; +import { ScrapbookItemConfig } from './ScrapbookPreset'; +import { ImageBox } from '../ImageBox'; +import { FireflyImageDimensions } from '../../smartdraw/FireflyConstants'; +import { SmartDrawHandler } from '../../smartdraw/SmartDrawHandler'; +import { ImageCast } from '../../../../fields/Types'; +import { SnappingManager } from '../../../util/SnappingManager'; +import { IReactionDisposer } from 'mobx'; +import { observer } from 'mobx-react'; +import { runInAction } from 'mobx'; +import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; +import { faRedoAlt } from '@fortawesome/free-solid-svg-icons'; +import { getPresetNames, createPreset } from './ScrapbookPresetRegistry'; +import './ScrapbookBox.scss'; +import { isDestArraysEqual } from 'pdfjs-dist/types/web/pdf_history'; + + +export function buildPlaceholdersFromConfigs(configs: ScrapbookItemConfig[]): Doc[] { + const placeholders: Doc[] = []; + + for (const cfg of configs) { + if (cfg.children && cfg.children.length) { + const childDocs = cfg.children.map(child => { + const doc = Docs.Create.TextDocument("[placeholder] " + child.tag); + doc.accepts_docType = child.type; + doc.accepts_tagType = new List<string>(child.acceptTags ?? [child.tag]); + + const ph = new Doc(); + ph.proto = doc; + ph.original = doc; + ph.x = child.x; + ph.y = child.y; + if (child.width != null) ph._width = child.width; + if (child.height != null) ph._height = child.height; + return ph; + }); + + const protoW = cfg.containerWidth ?? cfg.width; + const protoH = cfg.containerHeight ?? cfg.height; + // Create a stacking document with the child placeholders + const containerProto = Docs.Create.StackingDocument(childDocs, { + ...(protoW != null ? { _width: protoW } : {}), + ...(protoH != null ? { _height: protoH } : {}), + title: cfg.tag, + }); + + const ph = new Doc(); + ph.proto = containerProto; + ph.original = containerProto; + ph.x = cfg.x; + ph.y = cfg.y; + if (cfg.width != null) ph._width = cfg.width; + if (cfg.height != null) ph._height = cfg.height; + placeholders.push(ph); + } + + else { + const doc = Docs.Create.TextDocument("[placeholder] " + cfg.tag); + doc.accepts_docType = cfg.type; + doc.accepts_tagType = new List<string>(cfg.acceptTags ?? [cfg.tag]); + + const ph = new Doc(); + ph.proto = doc; + ph.original = doc; + ph.x = cfg.x; + ph.y = cfg.y; + if (cfg.width != null) ph._width = cfg.width; + if (cfg.height != null) ph._height = cfg.height; + placeholders.push(ph); + } + } + + return placeholders; +} +export function slotRealDocIntoPlaceholders( + realDoc: Doc, + placeholders: Doc[] +): boolean { + const realTags = new Set<string>( + StrListCast(realDoc.$tags_chat ?? new List<string>()) + .map(t => t.toLowerCase()) +); + + // Find placeholder with most matching tags + let bestMatch: Doc | null = null; + let maxMatches = 0; +/* + (d.accepts_docType === docs[0].$type || // match fields based on type, or by analyzing content .. simple example of matching text in placeholder to dropped doc's type + RTFCast(d[Doc.LayoutDataKey(d)])?.Text.includes(StrCast(docs[0].$type)))*/ + + placeholders.forEach(ph => { + if (ph.accepts_docType !== realDoc.$type) { + // Skip this placeholder entirely if types do not match. + return; + }; + const phTagTypes = StrListCast(ph.accepts_tagType ?? new List<string>()) + .map(t => t.toLowerCase()); + console.log({ realTags, phTagTypes }); + const matches = phTagTypes.filter(tag => realTags.has(tag)); + + if (matches.length > maxMatches) { + maxMatches = matches.length; + bestMatch = ph; + } + + }); + + if (bestMatch && maxMatches > 0) { + setTimeout( + undoable(() => { + bestMatch!.proto = realDoc; + }, 'Scrapbook add'), + 0 + ); + return true; + } + + return false; +} + +// Scrapbook view: a container that lays out its child items in a template +@observer export class ScrapbookBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { + @observable selectedPreset = getPresetNames()[0]; + @observable createdDate: string; + @observable loading = false; + @observable src = ''; + @observable imgDoc: Doc | undefined; + private _disposers: { [name: string]: IReactionDisposer } = {}; + private imageBoxRef = React.createRef<ImageBox>(); + constructor(props: FieldViewProps) { super(props); makeObservable(this); - this.createdDate = this.getFormattedDate(); + const existingItems = DocListCast(this.dataDoc[this.fieldKey] as List<Doc>); + if (!existingItems || existingItems.length === 0) { + // Only wire up reaction/setTitle if it's truly a brand-new, empty Scrapbook + reaction( + () => this.selectedPreset, + presetName => this.initScrapbook(presetName), + { fireImmediately: true } + ); + + this.createdDate = this.getFormattedDate(); + this.setTitle(); + } else { + // If items are already present, just preserve whatever was injected. + // We still want `createdDate` set so that the UI title bar can show it if needed. + this.createdDate = this.getFormattedDate(); + } + // ensure we always have a List<Doc> in dataDoc['items'] if (!this.dataDoc[this.fieldKey]) { this.dataDoc[this.fieldKey] = new List<Doc>(); } - this.createdDate = this.getFormattedDate(); - this.setTitle(); + } public static LayoutString(fieldStr: string) { @@ -41,14 +187,34 @@ export class ScrapbookBox extends ViewBoxAnnotatableComponent<FieldViewProps>() }); } + + @action + initScrapbook(name: string) { + const configs = createPreset(name); + // 1) ensure title is set + const title = `Scrapbook - ${this.createdDate}`; + if (this.dataDoc.title !== title) { + this.dataDoc.title = title; + } + + // 2) build placeholders from the preset + const placeholders = buildPlaceholdersFromConfigs(configs); + + // 3) commit them into the field + this.dataDoc[this.fieldKey] = new List<Doc>(placeholders); + } + + + @action setTitle() { const title = `Scrapbook - ${this.createdDate}`; if (this.dataDoc.title !== title) { this.dataDoc.title = title; - - const image = Docs.Create.TextDocument('image'); + if (!this.dataDoc[this.fieldKey]){ + const image = Docs.Create.TextDocument('[placeholder] person image'); image.accepts_docType = DocumentType.IMG; + image.accepts_tagType = 'PERSON' const placeholder = new Doc(); placeholder.proto = image; placeholder.original = image; @@ -56,26 +222,163 @@ export class ScrapbookBox extends ViewBoxAnnotatableComponent<FieldViewProps>() placeholder._height = 200; placeholder.x = 0; placeholder.y = -100; - //placeholder.overrideFields = new List<string>(['x', 'y']); // shouldn't need to do this for layout fields since the placeholder already overrides its protos - - const summary = Docs.Create.TextDocument('summary'); + + + const summary = Docs.Create.TextDocument('[placeholder] long summary'); summary.accepts_docType = DocumentType.RTF; - summary.accepts_textType = 'one line'; + summary.accepts_tagType = 'lengthy description'; const placeholder2 = new Doc(); placeholder2.proto = summary; placeholder2.original = summary; placeholder2.x = 0; placeholder2.y = 200; placeholder2._width = 250; - //placeholder2.overrideFields = new List<string>(['x', 'y', '_width']); // shouldn't need to do this for layout fields since the placeholder already overrides its protos - this.dataDoc[this.fieldKey] = new List<Doc>([placeholder, placeholder2]); + + + const sidebar = Docs.Create.TextDocument('[placeholder] brief sidebar'); + sidebar.accepts_docType = DocumentType.RTF; + sidebar.accepts_tagType = 'title'; + const placeholder3 = new Doc(); + placeholder3.proto = sidebar; + placeholder3.original = sidebar; + placeholder3.x = 280; + placeholder3.y = -50; + placeholder3._width = 50; + placeholder3._height = 200; + + + + const internalImg = Docs.Create.TextDocument('[placeholder] landscape internal'); + internalImg.accepts_docType = DocumentType.IMG; + internalImg.accepts_tagType = 'LANDSCAPE' + const placeholder5 = new Doc(); + placeholder5.proto = internalImg; + placeholder5.original = internalImg; + placeholder5._width = 50; + placeholder5._height = 100; + placeholder5.x = 0; + placeholder5.y = -100; + + const collection = Docs.Create.StackingDocument([placeholder5], { _width: 300, _height: 300, title: "internal coll" }); + //collection.accepts_docType = DocumentType.COL; don't mark this field + const placeholder4 = new Doc(); + placeholder4.proto = collection; + placeholder4.original = collection; + placeholder4.x = -200; + placeholder4.y = -100; + placeholder4._width = 100; + placeholder4._height = 200; + + const starter = Docs.Create.TextDocument('To create a scrapbook from existing documents, marquee select. For existing scrapbook arrangements, select a preset from the dropdown.'); + starter.accepts_docType = DocumentType.RTF; + starter.accepts_tagType = 'n/a' + const starterplaceholder = new Doc(); + starterplaceholder.proto = summary; + starterplaceholder.original = summary; + starterplaceholder.x = 0; + starterplaceholder.y = 0; + starterplaceholder._width = 250; + + + + + + /*note-to-self + would doing: + + const collection = Docs.Create.ScrapbookDocument([placeholder, placeholder2, placeholder3]); + create issues with references to the same object?*/ + + /*note-to-self + Should we consider that there are more collections than just COL type collections? + when spreading*/ + + /*note-to-self + difference between passing a new List<Doc> versus just the raw array? + */ + this.dataDoc[this.fieldKey] = new List<Doc>([starterplaceholder]); + } + + } } componentDidMount() { this.setTitle(); + this.generateAiImage(); + + this._disposers.propagateResize = reaction( + () => ({ w: this.layoutDoc._width, h: this.layoutDoc._height }), + (dims, prev) => { + // prev is undefined on the first run, so bail early + if (!prev || !SnappingManager.ShiftKey || !this.imgDoc) return; + + // either guard the ref… + const imageBox = this.imageBoxRef.current; + if (!imageBox) return; + + // …or just hard-code the fieldKey if you know it’s always `"data"` + const key = imageBox.props.fieldKey; + + runInAction(() => { + if(!this.imgDoc){ + return + } + // use prev.w/h (the *old* size) as your orig dims + this.imgDoc[key + '_outpaintOriginalWidth'] = prev.w; + this.imgDoc[key + '_outpaintOriginalHeight'] = prev.h; + ;(this.imageBoxRef.current as any).layoutDoc._width = dims.w + ;(this.imageBoxRef.current as any).layoutDoc._height = dims.h + + }); + } + ); } + + @action + async generateAiImage(prompt?: string) { + this.loading = true; + try { + // 1) Default to regenPrompt if none provided + if (!prompt) prompt = this.regenPrompt; + + // 2) Measure the scrapbook’s current size + const w = NumCast(this.layoutDoc._width, 1); + const h = NumCast(this.layoutDoc._height, 1); + const ratio = w / h; + + // 3) Pick the Firefly preset that best matches the aspect ratio + let preset = FireflyImageDimensions.Square; + if (ratio > AspectRatioLimits[FireflyImageDimensions.Widescreen]) { + preset = FireflyImageDimensions.Widescreen; + } else if (ratio > AspectRatioLimits[FireflyImageDimensions.Landscape]) { + preset = FireflyImageDimensions.Landscape; + } else if (ratio < AspectRatioLimits[FireflyImageDimensions.Portrait]) { + preset = FireflyImageDimensions.Portrait; + } + + // 4) Call exactly the same CreateWithFirefly that ImageBox uses + const doc = await SmartDrawHandler.CreateWithFirefly(prompt, preset); + + if (doc instanceof Doc) { + // 5) Hook it into your state + this.imgDoc = doc; + const imgField = ImageCast(doc.data); + this.src = imgField?.url.href ?? ''; + } else { + alert('Failed to generate document.'); + this.src = ''; + } + } catch (e) { + alert(`Generation error: ${e}`); + } finally { + runInAction(() => { + this.loading = false; + }); + } + } + childRejectDrop = (de: DragManager.DropEvent, subView?: DocumentView) => { return true; // disable dropping documents onto any child of the scrapbook. }; @@ -86,45 +389,122 @@ export class ScrapbookBox extends ViewBoxAnnotatableComponent<FieldViewProps>() }; filterAddDocument = (docIn: Doc | Doc[]) => { - const docs = toList(docIn); - if (docs?.length === 1) { - const placeholder = DocListCast(this.dataDoc[this.fieldKey]).find(d => - (d.accepts_docType === docs[0].$type || // match fields based on type, or by analyzing content .. simple example of matching text in placeholder to dropped doc's type - RTFCast(d[Doc.LayoutDataKey(d)])?.Text.includes(StrCast(docs[0].$type))) - ); // prettier-ignore - - if (placeholder) { - // ugh. we have to tell the underlying view not to add the Doc so that we can add it where we want it. - // However, returning 'false' triggers an undo. so this settimeout is needed to make the assignment happen after the undo. - setTimeout( - undoable(() => { - //StrListCast(placeholder.overrideFields).map(field => (docs[0][field] = placeholder[field])); // // shouldn't need to do this for layout fields since the placeholder already overrides its protos - placeholder.proto = docs[0]; - }, 'Scrapbook add') - ); - return false; - } - } - return false; - }; + const docs = toList(docIn); //The docs being added to the scrapbook - render() { - return ( - <div style={{ background: 'beige', width: '100%', height: '100%' }}> - <CollectionView - {...this._props} // - setContentViewBox={emptyFunction} - rejectDrop={this.rejectDrop} - childRejectDrop={this.childRejectDrop} - filterAddDocument={this.filterAddDocument} - /> - {/* <div style={{ border: '1px black', borderStyle: 'dotted', position: 'absolute', top: '50%', width: '100%', textAlign: 'center' }}>Drop an image here</div> */} - </div> - ); + // 1) Grab all template slots: + const slots = DocListCast(this.dataDoc[this.fieldKey]); + + // 2) recursive unwrap: + const unwrap = (items: Doc[]): Doc[] => + items.flatMap(d => + d.$type === DocumentType.COL + ? unwrap(DocListCast(d[Doc.LayoutDataKey(d)])) + : [d] + ); + + // 3) produce a flat list of every doc, unwrapping any number of nested COLs + const allDocs: Doc[] = unwrap(slots); + if (docs?.length === 1) { + return slotRealDocIntoPlaceholders( + docs[0], + allDocs, + ) + ? false + : false; } + + return false; +}; + + + @computed get regenPrompt() { + const slots = DocListCast(this.dataDoc[this.fieldKey]); + + const unwrap = (items: Doc[]): Doc[] => + items.flatMap(d => + d.$type === DocumentType.COL + ? unwrap(DocListCast(d[Doc.LayoutDataKey(d)])) + : [d] + ); + + const allDocs: Doc[] = unwrap(slots); + const internalTagsSet = new Set<string>(); + + allDocs.forEach(doc => { + const tags = StrListCast(doc.$tags_chat ?? new List<string>()); + tags.forEach(tag => + {if (!tag.startsWith("ASPECT_")) { + internalTagsSet.add(tag); + } + }); + }); + + const internalTags = Array.from(internalTagsSet).join(', '); + + + return internalTags + ? `Create a new scrapbook background featuring: ${internalTags}` + : 'A serene mountain landscape at sunrise, ultra-wide, pastel sky, abstract, scrapbook background'; + } + + render() { + return ( + <div className="scrapbook-box"> + {this.loading && ( + <div className="scrapbook-box-loading-overlay"> + <ReactLoading type="spin" width={50} height={50} /> + </div> + )} + + {this.src && this.imgDoc && ( + <ImageBox + ref={this.imageBoxRef} + {...this._props} + Document={this.imgDoc} + fieldKey="data" + /> + )} + + <select + className="scrapbook-box-preset-select" + value={this.selectedPreset} + onChange={e => (this.selectedPreset = e.currentTarget.value)} + > + {getPresetNames().map(name => ( + <option key={name} value={name}> + {name} + </option> + ))} + </select> + + {this._props.isContentActive() && ( + <div className="scrapbook-box-ui"> + <button + type="button" + title="Regenerate Background" + onClick={() => this.generateAiImage(this.regenPrompt)} + className="scrapbook-box-ui-button" + > + <FontAwesomeIcon icon={faRedoAlt} /> + <span>Regenerate Background</span> + </button> + </div> + )} + + <CollectionView + {...this._props} + setContentViewBox={emptyFunction} + rejectDrop={this.rejectDrop} + childRejectDrop={this.childRejectDrop} + filterAddDocument={this.filterAddDocument} + /> + </div> + ); + } } -// Register scrapbook + + Docs.Prototypes.TemplateMap.set(DocumentType.SCRAPBOOK, { layout: { view: ScrapbookBox, dataField: 'items' }, options: { diff --git a/src/client/views/nodes/scrapbook/ScrapbookContent.tsx b/src/client/views/nodes/scrapbook/ScrapbookContent.tsx deleted file mode 100644 index ad1d308e8..000000000 --- a/src/client/views/nodes/scrapbook/ScrapbookContent.tsx +++ /dev/null @@ -1,23 +0,0 @@ -import React from "react"; -import { observer } from "mobx-react-lite"; -// Import the Doc type from your actual module. -import { Doc } from "../../../../fields/Doc"; - -export interface ScrapbookContentProps { - doc: Doc; -} - -// A simple view that displays a document's title and content. -// Adjust how you extract the text if your Doc fields are objects. -export const ScrapbookContent: React.FC<ScrapbookContentProps> = observer(({ doc }) => { - // If doc.title or doc.content are not plain strings, convert them. - const titleText = doc.title ? doc.title.toString() : "Untitled"; - const contentText = doc.content ? doc.content.toString() : "No content available."; - - return ( - <div className="scrapbook-content"> - <h3>{titleText}</h3> - <p>{contentText}</p> - </div> - ); -}); diff --git a/src/client/views/nodes/scrapbook/ScrapbookPreset.tsx b/src/client/views/nodes/scrapbook/ScrapbookPreset.tsx new file mode 100644 index 000000000..706b9dafd --- /dev/null +++ b/src/client/views/nodes/scrapbook/ScrapbookPreset.tsx @@ -0,0 +1,176 @@ +import { DocumentType } from '../../../documents/DocumentTypes'; + +export enum ScrapbookPresetType { + Default = 'Default', + Classic = 'Classic', + None = 'Select Template', + Collage = 'Collage', + Spotlight = 'Spotlight', + Gallery = 'Gallery' +} + +export interface ScrapbookItemConfig { + type: DocumentType; + /** text shown in the placeholder bubble */ + tag: string; + /** what this slot actually accepts (defaults to `tag`) */ + acceptTags?: string[]; + + x: number; + y: number; + /** the frame this placeholder occupies */ + width?: number; + height?: number; + /** if this is a container with children, use these for the proto’s own size */ + containerWidth?: number; + containerHeight?: number; + children?: ScrapbookItemConfig[]; +} + +export class ScrapbookPreset { + static createPreset(presetType: ScrapbookPresetType): ScrapbookItemConfig[] { + switch (presetType) { + case ScrapbookPresetType.None: + return ScrapbookPreset.createNonePreset(); + case ScrapbookPresetType.Classic: + return ScrapbookPreset.createClassicPreset(); + case ScrapbookPresetType.Collage: + return ScrapbookPreset.createCollagePreset(); + case ScrapbookPresetType.Spotlight: + return ScrapbookPreset.createSpotlightPreset(); + case ScrapbookPresetType.Default: + return ScrapbookPreset.createDefaultPreset(); + case ScrapbookPresetType.Gallery: + return ScrapbookPreset.createGalleryPreset(); + default: + throw new Error(`Unknown preset type: ${presetType}`); + } + } + + private static createNonePreset(): ScrapbookItemConfig[] { + return [ + + { type: DocumentType.RTF, + tag: 'To create a scrapbook from existing documents, marquee select. For existing scrapbook arrangements, select a preset from the dropdown.', + acceptTags: ['n/a'], + x: 0, y: 0, width: 250, height: 200 + }, + + ]; + } + + private static createClassicPreset(): ScrapbookItemConfig[] { + return [ + { type: DocumentType.IMG, + tag: '[placeholder] LANDSCAPE', + acceptTags: ['LANDSCAPE'], + x: 0, y: -100, width: 250, height: 200 + }, + { type: DocumentType.RTF, + tag: '[placeholder] caption', + acceptTags: ['sentence'], + x: 0, y: 200, width: 250, height: 50 + }, + { type: DocumentType.RTF, + tag: '[placeholder] lengthy description', + acceptTags: ['paragraphs'], + x: 280, y: -50, width: 50, height: 200 + }, + { type: DocumentType.IMG, + tag: '[placeholder] PERSON', + acceptTags: ['PERSON'], + x: -200, y: -100, width: 100, height: 200 + }, + ]; + } + + private static createGalleryPreset(): ScrapbookItemConfig[] { + return [ + { type: DocumentType.IMG, tag: 'Gallery 1', acceptTags: ['LANDSCAPE'], x: -150, y: -150, width: 150, height: 150 }, + { type: DocumentType.IMG, tag: 'Gallery 2', acceptTags: ['LANDSCAPE'], x: 0, y: -150, width: 150, height: 150 }, + { type: DocumentType.IMG, tag: 'Gallery 3', acceptTags: ['LANDSCAPE'], x: 150, y: -150, width: 150, height: 150 }, + { type: DocumentType.IMG, tag: 'Gallery 4', acceptTags: ['LANDSCAPE'], x: -150, y: 0, width: 150, height: 150 }, + { type: DocumentType.IMG, tag: 'Gallery 5', acceptTags: ['LANDSCAPE'], x: 0, y: 0, width: 150, height: 150 }, + { type: DocumentType.IMG, tag: 'Gallery 6', acceptTags: ['LANDSCAPE'], x: 150, y: 0, width: 150, height: 150 }, + ]; + } + + + private static createDefaultPreset(): ScrapbookItemConfig[] { + return [ + { type: DocumentType.IMG, + tag: 'image', + acceptTags: ['LANDSCAPE'], + x: 0, y: -100, width: 250, height: 200 + }, + { type: DocumentType.RTF, + tag: 'summary', + acceptTags: ['sentence'], + x: 0, y: 200, width: 250 + }, + { type: DocumentType.RTF, + tag: 'sidebar', + acceptTags: ['paragraphs'], + x: 280, y: -50, width: 50, height: 200 + }, + { + type: DocumentType.COL, + tag: 'internal coll', + x: -200, y: -100, width: 100, height: 200, + containerWidth: 300, containerHeight: 300, + children: [ + { type: DocumentType.IMG, + tag: 'image internal', + acceptTags: ['PERSON'], + x: 0, y: 0, width: 50, height: 100 + } + ] + } + ]; + } + + private static createCollagePreset(): ScrapbookItemConfig[] { + return [ + { type: DocumentType.IMG, + tag: 'LANDSCAPE', + acceptTags: ['LANDSCAPE'], + x: -150, y: -150, width: 150, height: 150 + }, + { type: DocumentType.IMG, + tag: 'PERSON', + acceptTags: ['PERSON'], + x: 0, y: -150, width: 150, height: 150 + }, + { type: DocumentType.RTF, + tag: 'caption', + acceptTags: ['sentence'], + x: -150, y: 0, width: 300, height: 100 + }, + { type: DocumentType.RTF, + tag: 'lengthy description', + acceptTags: ['paragraphs'], + x: 0, y: 100, width: 300, height: 100 + } + ]; + } + + private static createSpotlightPreset(): ScrapbookItemConfig[] { + return [ + { type: DocumentType.RTF, + tag: 'title', + acceptTags: ['word'], + x: 0, y: -180, width: 300, height: 40 + }, + { type: DocumentType.IMG, + tag: 'LANDSCAPE', + acceptTags: ['LANDSCAPE'], + x: 0, y: 0, width: 300, height: 200 + }, + { type: DocumentType.RTF, + tag: 'caption', + acceptTags: ['sentence'], + x: 0, y: 230, width: 300, height: 50 + } + ]; + } +} diff --git a/src/client/views/nodes/scrapbook/ScrapbookPresetRegistry.ts b/src/client/views/nodes/scrapbook/ScrapbookPresetRegistry.ts new file mode 100644 index 000000000..c6d67ab73 --- /dev/null +++ b/src/client/views/nodes/scrapbook/ScrapbookPresetRegistry.ts @@ -0,0 +1,44 @@ +import { ScrapbookItemConfig } from './ScrapbookPreset'; +import { ScrapbookPresetType } from './ScrapbookPreset'; + +type PresetGenerator = () => ScrapbookItemConfig[]; + +// Internal map of preset name to generator +const presetRegistry = new Map<string, PresetGenerator>(); + + + + +/** + * Register a new scrapbook preset under the given name. + */ +export function registerPreset(name: string, gen: PresetGenerator) { + presetRegistry.set(name, gen); +} + +/** + * List all registered preset names. + */ +export function getPresetNames(): string[] { + return Array.from(presetRegistry.keys()); +} + +/** + * Create the config array for the named preset. + */ +export function createPreset(name: string): ScrapbookItemConfig[] { + const gen = presetRegistry.get(name); + if (!gen) throw new Error(`Unknown scrapbook preset: ${name}`); + return gen(); +} + +// ------------------------ +// Register built-in presets +import { ScrapbookPreset } from './ScrapbookPreset'; + +registerPreset(ScrapbookPresetType.None, () => ScrapbookPreset.createPreset(ScrapbookPresetType.None)); +registerPreset(ScrapbookPresetType.Classic, () => ScrapbookPreset.createPreset(ScrapbookPresetType.Classic)); +registerPreset(ScrapbookPresetType.Collage, () => ScrapbookPreset.createPreset(ScrapbookPresetType.Collage)); +registerPreset(ScrapbookPresetType.Spotlight, () => ScrapbookPreset.createPreset(ScrapbookPresetType.Spotlight)); +registerPreset(ScrapbookPresetType.Default, () => ScrapbookPreset.createPreset(ScrapbookPresetType.Default)); +registerPreset(ScrapbookPresetType.Gallery, () => ScrapbookPreset.createPreset(ScrapbookPresetType.Gallery)); diff --git a/src/client/views/nodes/scrapbook/ScrapbookSlot.scss b/src/client/views/nodes/scrapbook/ScrapbookSlot.scss deleted file mode 100644 index ae647ad36..000000000 --- a/src/client/views/nodes/scrapbook/ScrapbookSlot.scss +++ /dev/null @@ -1,85 +0,0 @@ -//IGNORE FOR NOW, CURRENTLY NOT USED IN SCRAPBOOK IMPLEMENTATION -.scrapbook-slot { - position: absolute; - background-color: rgba(245, 245, 245, 0.7); - border: 2px dashed #ccc; - border-radius: 5px; - box-sizing: border-box; - transition: all 0.2s ease; - overflow: hidden; - - &.scrapbook-slot-over { - border-color: #4a90e2; - background-color: rgba(74, 144, 226, 0.1); - } - - &.scrapbook-slot-filled { - border-style: solid; - border-color: rgba(0, 0, 0, 0.1); - background-color: transparent; - - &.scrapbook-slot-over { - border-color: #4a90e2; - background-color: rgba(74, 144, 226, 0.1); - } - } - - .scrapbook-slot-empty { - display: flex; - align-items: center; - justify-content: center; - width: 100%; - height: 100%; - } - - .scrapbook-slot-placeholder { - text-align: center; - color: #888; - } - - .scrapbook-slot-title { - font-weight: bold; - margin-bottom: 5px; - } - - .scrapbook-slot-instruction { - font-size: 0.9em; - font-style: italic; - } - - .scrapbook-slot-content { - width: 100%; - height: 100%; - position: relative; - } - - .scrapbook-slot-controls { - position: absolute; - top: 5px; - right: 5px; - z-index: 10; - opacity: 0; - transition: opacity 0.2s ease; - - .scrapbook-slot-remove-btn { - background-color: rgba(255, 255, 255, 0.8); - border: 1px solid #ccc; - border-radius: 50%; - width: 20px; - height: 20px; - display: flex; - align-items: center; - justify-content: center; - cursor: pointer; - font-size: 10px; - - &:hover { - background-color: rgba(255, 0, 0, 0.1); - } - } - } - - &:hover .scrapbook-slot-controls { - opacity: 1; - } -}
\ No newline at end of file diff --git a/src/client/views/nodes/scrapbook/ScrapbookSlot.tsx b/src/client/views/nodes/scrapbook/ScrapbookSlot.tsx deleted file mode 100644 index 2c8f93778..000000000 --- a/src/client/views/nodes/scrapbook/ScrapbookSlot.tsx +++ /dev/null @@ -1,28 +0,0 @@ - -//IGNORE FOR NOW, CURRENTLY NOT USED IN SCRAPBOOK IMPLEMENTATION -export interface SlotDefinition { - id: string; - x: number; y: number; - defaultWidth: number; - defaultHeight: number; - } - - export interface SlotContentMap { - slotId: string; - docId?: string; - } - - export interface ScrapbookConfig { - slots: SlotDefinition[]; - contents?: SlotContentMap[]; - } - - export const DEFAULT_SCRAPBOOK_CONFIG: ScrapbookConfig = { - slots: [ - { id: "slot1", x: 10, y: 10, defaultWidth: 180, defaultHeight: 120 }, - { id: "slot2", x: 200, y: 10, defaultWidth: 180, defaultHeight: 120 }, - // …etc - ], - contents: [] - }; -
\ No newline at end of file diff --git a/src/client/views/nodes/scrapbook/ScrapbookSlotTypes.ts b/src/client/views/nodes/scrapbook/ScrapbookSlotTypes.ts deleted file mode 100644 index 686917d9a..000000000 --- a/src/client/views/nodes/scrapbook/ScrapbookSlotTypes.ts +++ /dev/null @@ -1,25 +0,0 @@ -// ScrapbookSlotTypes.ts -export interface SlotDefinition { - id: string; - title: string; - x: number; - y: number; - defaultWidth: number; - defaultHeight: number; - } - - export interface ScrapbookConfig { - slots: SlotDefinition[]; - contents?: { slotId: string; docId: string }[]; - } - - // give it three slots by default: - export const DEFAULT_SCRAPBOOK_CONFIG: ScrapbookConfig = { - slots: [ - { id: "main", title: "Main Content", x: 20, y: 20, defaultWidth: 360, defaultHeight: 200 }, - { id: "notes", title: "Notes", x: 20, y: 240, defaultWidth: 360, defaultHeight: 160 }, - { id: "resources", title: "Resources", x: 400, y: 20, defaultWidth: 320, defaultHeight: 380 }, - ], - contents: [], - }; -
\ No newline at end of file diff --git a/src/client/views/search/FaceRecognitionHandler.tsx b/src/client/views/search/FaceRecognitionHandler.tsx index 3ad5bc844..256e68afd 100644 --- a/src/client/views/search/FaceRecognitionHandler.tsx +++ b/src/client/views/search/FaceRecognitionHandler.tsx @@ -9,6 +9,8 @@ import { ImageField } from '../../../fields/URLField'; import { DocumentType } from '../../documents/DocumentTypes'; import { Docs } from '../../documents/Documents'; import { DocumentManager } from '../../util/DocumentManager'; +import { reaction } from 'mobx'; +import { DocumentView } from '../nodes/DocumentView'; /** * A singleton class that handles face recognition and manages face Doc collections for each face found. @@ -33,7 +35,7 @@ export class FaceRecognitionHandler { // eslint-disable-next-line no-use-before-define static _instance: FaceRecognitionHandler; private _apiModelReady = false; - private _pendingAPIModelReadyDocs: Doc[] = []; + private _pendingAPIModelReadyDocs: DocumentView[] = []; public static get Instance() { return FaceRecognitionHandler._instance ?? new FaceRecognitionHandler(); @@ -126,7 +128,7 @@ export class FaceRecognitionHandler { constructor() { FaceRecognitionHandler._instance = this; this.loadAPIModels().then(() => this._pendingAPIModelReadyDocs.forEach(this.classifyFacesInImage)); - DocumentManager.Instance.AddAnyViewRenderedCB(dv => FaceRecognitionHandler.Instance.classifyFacesInImage(dv.Document)); + DocumentManager.Instance.AddAnyViewRenderedCB(dv => FaceRecognitionHandler.Instance.classifyFacesInImage(dv)); } /** @@ -199,14 +201,18 @@ export class FaceRecognitionHandler { * match them to existing unique faces, otherwise new unique face(s) are created. * @param imgDoc The document being analyzed. */ - private classifyFacesInImage = async (imgDoc: Doc) => { + private classifyFacesInImage = async (imgDocView: DocumentView) => { + const imgDoc = imgDocView.Document; if (!Doc.UserDoc().recognizeFaceImages) return; const activeDashboard = Doc.ActiveDashboard; if (!this._apiModelReady || !activeDashboard) { - this._pendingAPIModelReadyDocs.push(imgDoc); + this._pendingAPIModelReadyDocs.push(imgDocView); } else if (imgDoc.type === DocumentType.LOADING && !imgDoc.loadingError) { - setTimeout(() => this.classifyFacesInImage(imgDoc), 1000); + setTimeout(() => this.classifyFacesInImage(imgDocView), 1000); } else { + reaction(() => ({sel:imgDocView.isSelected()}), ({sel}) => !sel && + imgDocView.ComponentView?.autoTag?.(), {fireImmediately: true} + ) const imgUrl = ImageCast(imgDoc[Doc.LayoutDataKey(imgDoc)]); if (imgUrl && !DocListCast(Doc.MyFaceCollection?.examinedFaceDocs).includes(imgDoc[DocData])) { // only examine Docs that have an image and that haven't already been examined. |