diff options
| -rw-r--r-- | src/client/documents/Documents.ts | 3 | ||||
| -rw-r--r-- | src/client/views/DocumentDecorations.tsx | 158 | ||||
| -rw-r--r-- | src/client/views/nodes/ImageBox.tsx | 409 | ||||
| -rw-r--r-- | src/fields/DimensionField.ts | 29 | ||||
| -rw-r--r-- | src/fields/Doc.ts | 25 | ||||
| -rw-r--r-- | src/server/ApiManagers/FireflyManager.ts | 319 |
6 files changed, 829 insertions, 114 deletions
diff --git a/src/client/documents/Documents.ts b/src/client/documents/Documents.ts index 317bb7feb..a2b55943a 100644 --- a/src/client/documents/Documents.ts +++ b/src/client/documents/Documents.ts @@ -524,6 +524,9 @@ export class DocumentOptions { ai?: string; // to mark items as ai generated ai_firefly_seed?: number; ai_firefly_prompt?: string; + + _outpaintingMetadata?: STRt = new StrInfo('serialized JSON metadata needed for image outpainting', false); + } export const DocOptions = new DocumentOptions(); diff --git a/src/client/views/DocumentDecorations.tsx b/src/client/views/DocumentDecorations.tsx index 54e050f9f..d7ff0d06a 100644 --- a/src/client/views/DocumentDecorations.tsx +++ b/src/client/views/DocumentDecorations.tsx @@ -1,6 +1,7 @@ import { IconProp } from '@fortawesome/fontawesome-svg-core'; import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; import { Tooltip } from '@mui/material'; +import { DimensionField } from '../../fields/DimensionField'; import { IconButton } from '@dash/components'; import { action, computed, makeObservable, observable, runInAction } from 'mobx'; import { observer } from 'mobx-react'; @@ -473,6 +474,8 @@ export class DocumentDecorations extends ObservableReactComponent<DocumentDecora const tl = docView.screenToContentsTransform().inverse().transformPoint(0, 0); return project([e.clientX + this._offset.x, e.clientY + this._offset.y], tl, [tl[0] + fixedAspect, tl[1] + 1]); }; + + // Modify the onPointerMove method to handle shift+click during resize onPointerMove = (e: PointerEvent): boolean => { const first = DocumentView.Selected()[0]; const effectiveAcl = GetEffectiveAcl(first.Document); @@ -491,17 +494,135 @@ export class DocumentDecorations extends ObservableReactComponent<DocumentDecora !this._interactionLock && runInAction(async () => { // resize selected docs if we're not in the middle of a resize (ie, throttle input events to frame rate) this._interactionLock = true; this._snapPt = thisPt; - e.ctrlKey && (DocumentView.Selected().forEach(docView => !Doc.NativeHeight(docView.Document) && docView.toggleNativeDimensions())); - const hasFixedAspect = DocumentView.Selected().map(dv => dv.Document).some(this.hasFixedAspect); - const scaleAspect = {x:scale.x === 1 && hasFixedAspect ? scale.y : scale.x, y: scale.x !== 1 && hasFixedAspect ? scale.x : scale.y}; - DocumentView.Selected().forEach(docView => - this.resizeView(docView, refPt, scaleAspect, { dragHdl, ctrlKey:e.ctrlKey })); // prettier-ignore + + // Special handling for shift+click (outpainting mode) + if (e.shiftKey && DocumentView.Selected().some(dv => dv.ComponentView instanceof ImageBox)) { + DocumentView.Selected().forEach(docView => { + if (docView.ComponentView instanceof ImageBox) { + // Set flag for outpainting mode + docView.Document._outpaintingResize = true; + + // Adjust only the document dimensions without scaling internal content + this.resizeViewForOutpainting(docView, refPt, scale, { dragHdl, shiftKey: e.shiftKey }); + } + else { + // Handle regular resize for non-image components + e.ctrlKey && !Doc.NativeHeight(docView.Document) && docView.toggleNativeDimensions(); + const hasFixedAspect = this.hasFixedAspect(docView.Document); + const scaleAspect = {x:scale.x === 1 && hasFixedAspect ? scale.y : scale.x, y: scale.x !== 1 && hasFixedAspect ? scale.x : scale.y}; + this.resizeView(docView, refPt, scaleAspect, { dragHdl, ctrlKey: e.ctrlKey }); + } + }); + } else { + // Regular resize behavior (existing code) + e.ctrlKey && (DocumentView.Selected().forEach(docView => !Doc.NativeHeight(docView.Document) && docView.toggleNativeDimensions())); + const hasFixedAspect = DocumentView.Selected().map(dv => dv.Document).some(this.hasFixedAspect); + const scaleAspect = {x:scale.x === 1 && hasFixedAspect ? scale.y : scale.x, y: scale.x !== 1 && hasFixedAspect ? scale.x : scale.y}; + DocumentView.Selected().forEach(docView => + this.resizeView(docView, refPt, scaleAspect, { dragHdl, ctrlKey:e.ctrlKey })); + } + await new Promise<void>(res => { setTimeout(() => { res(this._interactionLock = undefined)})}); - }); // prettier-ignore + }); return false; }; + resizeViewForOutpainting = ( + docView: DocumentView, + refPt: number[], + scale: { x: number; y: number }, + opts: { dragHdl: string; shiftKey: boolean } + ) => { + const doc = docView.Document; + + if (doc.isGroup) { + DocListCast(doc.data) + .map(member => DocumentView.getDocumentView(member, docView)!) + .forEach(member => this.resizeViewForOutpainting(member, refPt, scale, opts)); + doc.xPadding = NumCast(doc.xPadding) * scale.x; + doc.yPadding = NumCast(doc.yPadding) * scale.y; + return; + } + + if (!doc._outpaintingOriginalWidth || !doc._outpaintingOriginalHeight) { + doc._outpaintingOriginalWidth = NumCast(doc._width); + doc._outpaintingOriginalHeight = NumCast(doc._height); + + // Initialize or update the _originalDims ObjectField correctly + doc._originalDims = new DimensionField(doc._outpaintingOriginalWidth, doc._outpaintingOriginalHeight); + } + + // Calculate new boundary dimensions + const originalWidth = NumCast(doc._width); + const originalHeight = NumCast(doc._height); + const newWidth = Math.max(NumCast(doc._width_min, 25), originalWidth * scale.x); + const newHeight = Math.max(NumCast(doc._height_min, 10), originalHeight * scale.y); + + // Apply new dimensions + doc._width = newWidth; + doc._height = newHeight; + + const refCent = docView.screenToViewTransform().transformPoint(refPt[0], refPt[1]); + const { deltaX, deltaY } = this.realignRefPt(doc, refCent, originalWidth, originalHeight); + doc.x = NumCast(doc.x) + deltaX; + doc.y = NumCast(doc.y) + deltaY; + + doc._layout_modificationDate = new DateField(); + + // Trigger outpainting + doc._needsOutpainting = true; + + // Store metadata needed for outpainting + doc._outpaintingMetadata = JSON.stringify({ + originalWidth: doc._outpaintingOriginalWidth, + originalHeight: doc._outpaintingOriginalHeight, + newWidth, + newHeight, + scaleX: scale.x, + scaleY: scale.y, + anchorHandle: opts.dragHdl + }); + }; + + @action + onPointerUp = (): void => { + SnappingManager.SetIsResizing(undefined); + SnappingManager.clearSnapLines(); + + // Check if any outpainting needs to be processed + DocumentView.Selected().forEach(view => { + if (view.Document._needsOutpainting && view.ComponentView instanceof ImageBox) { + // Trigger outpainting process in the ImageBox component + (view.ComponentView as ImageBox).processOutpainting(); + + // Clear flags + view.Document._needsOutpainting = false; + view.Document._outpaintingResize = false; + } + }); + + this._resizeHdlId = ''; + this._resizeUndo?.end(); + + // detect layout_autoHeight gesture and apply + DocumentView.Selected().forEach(view => { + NumCast(view.Document._height) < 20 && (view.layoutDoc._layout_autoHeight = true); + }); + // need to change points for resize, or else rotation/control points will fail. + this._inkDragDocs + .map(oldbds => ({ oldbds, inkPts: Cast(oldbds.doc.data, InkField)?.inkData || [] })) + .forEach(({ oldbds: { doc, x, y, width, height }, inkPts }) => { + doc[DocData].data = new InkField(inkPts.map( + (ipt) => ({// (new x — oldx) + newWidth * (oldxpoint /oldWidth) + X: NumCast(doc.x) - x + (NumCast(doc._width) * ipt.X) / width, + Y: NumCast(doc.y) - y + (NumCast(doc._height) * ipt.Y) / height, + }))); // prettier-ignore + Doc.SetNativeWidth(doc, undefined); + Doc.SetNativeHeight(doc, undefined); + }); + }; + // // determines how much to resize, and determines the resize reference point // @@ -606,31 +727,6 @@ export class DocumentDecorations extends ObservableReactComponent<DocumentDecora }; }; - @action - onPointerUp = (): void => { - SnappingManager.SetIsResizing(undefined); - SnappingManager.clearSnapLines(); - this._resizeHdlId = ''; - this._resizeUndo?.end(); - - // detect layout_autoHeight gesture and apply - DocumentView.Selected().forEach(view => { - NumCast(view.Document._height) < 20 && (view.layoutDoc._layout_autoHeight = true); - }); - // need to change points for resize, or else rotation/control points will fail. - this._inkDragDocs - .map(oldbds => ({ oldbds, inkPts: Cast(oldbds.doc.data, InkField)?.inkData || [] })) - .forEach(({ oldbds: { doc, x, y, width, height }, inkPts }) => { - doc[DocData].data = new InkField(inkPts.map( - (ipt) => ({// (new x — oldx) + newWidth * (oldxpoint /oldWidth) - X: NumCast(doc.x) - x + (NumCast(doc.width) * ipt.X) / width, - Y: NumCast(doc.y) - y + (NumCast(doc.height) * ipt.Y) / height, - }))); // prettier-ignore - Doc.SetNativeWidth(doc, undefined); - Doc.SetNativeHeight(doc, undefined); - }); - }; - @computed get selectionTitle(): string { if (DocumentView.Selected().length === 1) { const selected = DocumentView.Selected()[0]; diff --git a/src/client/views/nodes/ImageBox.tsx b/src/client/views/nodes/ImageBox.tsx index 5b06e9fc5..114d5226b 100644 --- a/src/client/views/nodes/ImageBox.tsx +++ b/src/client/views/nodes/ImageBox.tsx @@ -1,6 +1,7 @@ import { Button, Colors, Size, Type } from '@dash/components'; import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; import { Slider, Tooltip } from '@mui/material'; +import { DimensionField } from '../../../fields/DimensionField'; import axios from 'axios'; import { action, computed, IReactionDisposer, makeObservable, observable, ObservableMap, reaction } from 'mobx'; import { observer } from 'mobx-react'; @@ -46,6 +47,7 @@ import { FocusViewOptions } from './FocusViewOptions'; import './ImageBox.scss'; import { OpenWhere } from './OpenWhere'; import { RichTextField } from '../../../fields/RichTextField'; +import { List } from '../../../fields/List'; export class ImageEditorData { // eslint-disable-next-line no-use-before-define @@ -103,6 +105,10 @@ export class ImageBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { @observable private _regenerateLoading = false; @observable private _prevImgs: FireflyImageData[] = StrCast(this.Document.ai_firefly_history) ? JSON.parse(StrCast(this.Document.ai_firefly_history)) : []; + // Add these observable properties to the ImageBox class + @observable private _outpaintingInProgress = false; + @observable private _outpaintingPrompt = ''; + constructor(props: FieldViewProps) { super(props); makeObservable(this); @@ -135,6 +141,7 @@ export class ImageBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { }; componentDidMount() { + super.componentDidMount?.(); this._disposers.sizer = reaction( () => ({ forceFull: this._props.renderDepth < 1 || this.layoutDoc._showFullRes, @@ -148,9 +155,9 @@ export class ImageBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { ); const { layoutDoc } = this; this._disposers.path = reaction( - () => ({ nativeSize: this.nativeSize, width: NumCast(this.layoutDoc._width), height: this.layoutDoc._height }), - ({ nativeSize, width, height }) => { - if ((layoutDoc === this.layoutDoc && !this.layoutDoc._layout_nativeDimEditable) || !height) { + () => ({ nativeSize: this.nativeSize, width: NumCast(this.layoutDoc._width) }), + ({ nativeSize, width }) => { + if ((layoutDoc === this.layoutDoc && !this.layoutDoc._layout_nativeDimEditable) || !this.layoutDoc._height) { this.layoutDoc._height = (width * nativeSize.nativeHeight) / nativeSize.nativeWidth; } }, @@ -166,6 +173,14 @@ export class ImageBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { }, { fireImmediately: true } ); + this._disposers.outpainting = reaction( + () => this.Document?._needsOutpainting, + needsOutpainting => { + if (needsOutpainting && this.Document?._outpaintingResize) { + this.processOutpainting(); + } + } + ); } componentWillUnmount() { @@ -214,7 +229,7 @@ export class ImageBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { }, true); } else if (hitDropTarget(e.target as HTMLElement, this._regenerateIconRef.current)) { this._regenerateLoading = true; - const drag = de.complete.docDragData.draggedDocuments.lastElement(); + const drag = de.complete.docDragData?.draggedDocuments.lastElement(); const dragField = drag[Doc.LayoutFieldKey(drag)]; const oldPrompt = StrCast(this.Document.ai_firefly_prompt, StrCast(this.Document.title)); const newPrompt = (text: string) => (oldPrompt ? `${oldPrompt} ~~~ ${text}` : text); @@ -339,6 +354,202 @@ export class ImageBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { } }); + // Add this method to process outpainting when resize is complete + @action + processOutpainting = async () => { + const field = Cast(this.dataDoc[this.fieldKey], ImageField); + if (!field) return; + + const origWidth = NumCast(this.Document._outpaintingOriginalWidth); + const origHeight = NumCast(this.Document._outpaintingOriginalHeight); + + if (!origWidth || !origHeight) { + console.error("Original dimensions (_outpaintingOriginalWidth/_outpaintingOriginalHeight) not set. Ensure resizeViewForOutpainting was called first."); + return; + } + + //alert(`Original dimensions: ${origWidth} x ${origHeight}`); + + + // Set flag that outpainting is in progress + this._outpaintingInProgress = true; + + try { + // Get the current path to the image + const currentPath = this.choosePath(field.url); + + // Get original and new dimensions for calculating mask + const newWidth = NumCast(this.Document._width); + const newHeight = NumCast(this.Document._height); + + // Optional: Ask user for a prompt to guide the outpainting + let prompt = "Extend this image naturally with matching content"; + const customPrompt = await new Promise<string>((resolve) => { + const dialog = document.createElement('div'); + Object.assign(dialog.style, { + position: 'fixed', + top: '50%', + left: '50%', + transform: 'translate(-50%, -50%)', + background: 'white', + padding: '20px', + borderRadius: '8px', + boxShadow: '0 4px 12px rgba(0,0,0,0.2)', + zIndex: '10000' + }); + + const title = document.createElement('h3'); + title.style.marginTop = '0'; + title.textContent = 'Outpaint Image'; + + const description = document.createElement('p'); + description.textContent = 'Enter a prompt for extending the image:'; + + const input = document.createElement('input'); + input.id = 'outpaint-prompt'; + input.type = 'text'; + input.value = 'Extend this image naturally with matching content'; + Object.assign(input.style, { + width: '300px', + padding: '8px', + marginBottom: '10px' + }); + + const buttonContainer = document.createElement('div'); + Object.assign(buttonContainer.style, { + display: 'flex', + justifyContent: 'flex-end', + gap: '10px' + }); + + const cancelButton = document.createElement('button'); + cancelButton.textContent = 'Cancel'; + + const confirmButton = document.createElement('button'); + confirmButton.textContent = 'Generate'; + Object.assign(confirmButton.style, { + background: '#0078d4', + color: 'white', + border: 'none', + padding: '8px 16px' + }); + + buttonContainer.appendChild(cancelButton); + buttonContainer.appendChild(confirmButton); + + dialog.appendChild(title); + dialog.appendChild(description); + dialog.appendChild(input); + dialog.appendChild(buttonContainer); + + document.body.appendChild(dialog); + + cancelButton.onclick = () => { + document.body.removeChild(dialog); + resolve(""); + }; + + confirmButton.onclick = () => { + const promptValue = input.value; + document.body.removeChild(dialog); + resolve(promptValue); + }; + }); + + // If user cancelled, reset dimensions to original + if (!customPrompt) { + this.Document._width = origWidth; + this.Document._height = origHeight; + this._outpaintingInProgress = false; + return; + } + + // Show loading indicator + const loadingOverlay = document.createElement('div'); + loadingOverlay.style.position = 'absolute'; + loadingOverlay.style.top = '0'; + loadingOverlay.style.left = '0'; + loadingOverlay.style.width = '100%'; + loadingOverlay.style.height = '100%'; + loadingOverlay.style.background = 'rgba(0,0,0,0.5)'; + loadingOverlay.style.display = 'flex'; + loadingOverlay.style.justifyContent = 'center'; + loadingOverlay.style.alignItems = 'center'; + loadingOverlay.innerHTML = '<div style="color: white; font-size: 16px;">Generating outpainted image...</div>'; + this._mainCont?.appendChild(loadingOverlay); + + // Call the outpaint API + const response = await Networking.PostToServer('/outpaintImageFour', { + imageUrl: currentPath, + prompt: customPrompt, + originalDimensions: { + width: origWidth, + height: origHeight + }, + newDimensions: { + width: newWidth, + height: newHeight + } + }); + if (response && typeof response === 'object' && 'url' in response && typeof response.url === 'string') { + console.log("Response is valid and contains URL:", response.url); + + } else { + console.error("Unexpected API response:", response); + alert("Failed to receive a valid image URL from server."); + } + + + if (response && 'url' in response && typeof response.url === 'string') { + // Save the original image as an alternate + if (!this.dataDoc[this.fieldKey + '_alternates']) { + this.dataDoc[this.fieldKey + '_alternates'] = new List<Doc>(); + } + + // Create a copy of the current image as an alternate + const originalDoc = Docs.Create.ImageDocument(field.url.href, { + title: `Original: ${this.Document.title}`, + _nativeWidth: Doc.NativeWidth(this.dataDoc), + _nativeHeight: Doc.NativeHeight(this.dataDoc) + }); + + // Add to alternates + Doc.AddDocToList(this.dataDoc, this.fieldKey + '_alternates', originalDoc); + + // Update the image with the outpainted version + this.dataDoc[this.fieldKey] = new ImageField(response.url); + + // Update native dimensions + Doc.SetNativeWidth(this.dataDoc, newWidth); + Doc.SetNativeHeight(this.dataDoc, newHeight); + + // Add AI metadata + this.Document.ai = true; + this.Document.ai_outpainted = true; + this.Document.ai_outpaint_prompt = customPrompt; + } else { + // If failed, revert to original dimensions + this.Document._width = origWidth; + this.Document._height = origHeight; + alert("Failed to outpaint image. Please try again."); + } + + // Remove loading overlay + this._mainCont?.removeChild(loadingOverlay); + + } catch (error) { + console.error("Error during outpainting:", error); + // Revert to original dimensions on error + this.Document._width = origWidth + this.Document._height = origHeight + alert("An error occurred while outpainting. Please try again."); + } finally { + // Clear the outpainting flags + this._outpaintingInProgress = false; + delete this.Document._originalDims; + } + }; + specificContextMenu = (): void => { const field = Cast(this.dataDoc[this.fieldKey], ImageField); if (field) { @@ -396,6 +607,27 @@ export class ImageBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { event: action(undoable(async () => await StickerPalette.addToPalette(this.Document), 'save to palette')), icon: this.Document.savedAsSticker ? 'clipboard-check' : 'file-arrow-down', }); + // Add new outpainting option + funcs.push({ + description: 'Outpaint Image', + event: () => { + this.processOutpainting(); + }, + icon: 'brush' + }); + // Add outpainting history option if the image was outpainted + this.Document.ai_outpainted && + funcs.push({ + description: 'View Original Image', + event: action(() => { + const alternates = DocListCast(this.dataDoc[this.fieldKey + '_alternates']); + if (alternates && alternates.length) { + // Toggle to show the original image + this.layoutDoc[this.fieldKey + '_usePath'] = 'alternate'; + } + }), + icon: 'image', + }); ContextMenu.Instance?.addItem({ description: 'Options...', subitems: funcs, icon: 'asterisk' }); } }; @@ -434,7 +666,7 @@ export class ImageBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { const ext = extname(url.href); return url.href.replace(ext, (this._error ? '_o' : this._curSuffix) + ext); }; - getScrollHeight = () => (this._props.fitWidth?.(this.Document) !== false && NumCast(this.layoutDoc._freeform_scale, 1) === NumCast(this.dataDoc._freeform_scaleMin, 1) ? this.nativeSize.nativeHeight : undefined); + getScrollHeight = () => (this._props.fitWidth?.(this.Document) !== false && NumCast(this.layoutDoc._freeform_scale, 1) === NumCast(this.dataDoc.freeform_scaleMin, 1) ? this.nativeSize.nativeHeight : undefined); @computed get usingAlternate() { const usePath = StrCast(this.Document[this.fieldKey + '_usePath']); @@ -587,7 +819,7 @@ export class ImageBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { /> {fadepath === srcpath ? null : ( <div className={`imageBox-fadeBlocker${this.usingAlternate ? '-hover' : ''}`} style={{ transition: StrCast(this.layoutDoc.viewTransition, 'opacity 1000ms') }}> - <img alt="" className="imageBox-fadeaway" key="fadeaway" src={fadepath} style={{ transform, transformOrigin }} draggable={false} width={nativeWidth} /> + <img alt="" className="imageBox-fadeaway" key="fadeaway" src={fadepath} style={{ transform, transformOrigin }} draggable={false} width={20} /> </div> )} </div> @@ -746,82 +978,89 @@ export class ImageBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { const alts = DocListCast(this.dataDoc[this.fieldKey + '_alternates']); const doc = this.usingAlternate ? (alts.lastElement() ?? this.Document) : this.Document; return ( - <div - className="imageBox" - onContextMenu={this.specificContextMenu} - ref={this.createDropTarget} - onScroll={action(() => { - if (!this._forcedScroll) { - if (this.layoutDoc._layout_scrollTop || this._mainCont?.scrollTop) { - this._ignoreScroll = true; - this.layoutDoc._layout_scrollTop = this._mainCont?.scrollTop; - this._ignoreScroll = false; + <> + <div + className="imageBox" + onContextMenu={this.specificContextMenu} + ref={this.createDropTarget} + onScroll={action(() => { + if (!this._forcedScroll) { + if (this.layoutDoc._layout_scrollTop || this._mainCont?.scrollTop) { + this._ignoreScroll = true; + this.layoutDoc._layout_scrollTop = this._mainCont?.scrollTop; + this._ignoreScroll = false; + } } - } - })} - style={{ - width: this._props.PanelWidth() ? undefined : `100%`, - height: this._props.PanelHeight() ? undefined : `100%`, - pointerEvents: this.layoutDoc._lockedPosition ? 'none' : undefined, - borderRadius, - overflow: this.layoutDoc.layout_fitWidth || this._props.fitWidth?.(this.Document) ? 'auto' : 'hidden', - }}> - <CollectionFreeFormView - ref={this._ffref} - {...this._props} - Document={doc} - setContentViewBox={emptyFunction} - NativeWidth={returnZero} - NativeHeight={returnZero} - renderDepth={this._props.renderDepth + 1} - fieldKey={this.annotationKey} - styleProvider={this._props.styleProvider} - isAnnotationOverlay - annotationLayerHostsContent - PanelWidth={this._props.PanelWidth} - PanelHeight={this._props.PanelHeight} - ScreenToLocalTransform={this.screenToLocalTransform} - select={emptyFunction} - focus={this.focus} - getScrollHeight={this.getScrollHeight} - NativeDimScaling={returnOne} - isAnyChildContentActive={returnFalse} - whenChildContentsActiveChanged={this.whenChildContentsActiveChanged} - removeDocument={this.removeDocument} - moveDocument={this.moveDocument} - addDocument={this.addDocument}> - {this.content} - </CollectionFreeFormView> - {this.Loading ? ( - <div className="loading-spinner" style={{ position: 'absolute' }}> - <ReactLoading type="spin" height={50} width={50} color={'blue'} /> + })} + style={{ + width: this._props.PanelWidth() ? undefined : `100%`, + height: this._props.PanelHeight() ? undefined : `100%`, + pointerEvents: this.layoutDoc._lockedPosition ? 'none' : undefined, + borderRadius, + overflow: this.layoutDoc.layout_fitWidth || this._props.fitWidth?.(this.Document) ? 'auto' : 'hidden', + }}> + <CollectionFreeFormView + ref={this._ffref} + {...this._props} + Document={doc} + setContentViewBox={emptyFunction} + NativeWidth={returnZero} + NativeHeight={returnZero} + renderDepth={this._props.renderDepth + 1} + fieldKey={this.annotationKey} + styleProvider={this._props.styleProvider} + isAnnotationOverlay + annotationLayerHostsContent + PanelWidth={this._props.PanelWidth} + PanelHeight={this._props.PanelHeight} + ScreenToLocalTransform={this.screenToLocalTransform} + select={emptyFunction} + focus={this.focus} + getScrollHeight={this.getScrollHeight} + NativeDimScaling={returnOne} + isAnyChildContentActive={returnFalse} + whenChildContentsActiveChanged={this.whenChildContentsActiveChanged} + removeDocument={this.removeDocument} + moveDocument={this.moveDocument} + addDocument={this.addDocument}> + {this.content} + </CollectionFreeFormView> + {this.Loading ? ( + <div className="loading-spinner" style={{ position: 'absolute' }}> + <ReactLoading type="spin" height={50} width={50} color={'blue'} /> + </div> + ) : null} + {this.regenerateImageIcon} + {this.overlayImageIcon} + {this.annotationLayer} + {!this._mainCont || !this.DocumentView || !this._annotationLayer.current ? null : ( + <MarqueeAnnotator + Document={this.Document} + ref={this.marqueeref} + scrollTop={0} + annotationLayerScrollTop={0} + scaling={returnOne} + annotationLayerScaling={this._props.NativeDimScaling} + screenTransform={this.DocumentView().screenToViewTransform} + docView={this.DocumentView} + addDocument={this.addDocument} + finishMarquee={this.finishMarquee} + savedAnnotations={this.savedAnnotations} + selectionText={returnEmptyString} + annotationLayer={this._annotationLayer.current} + marqueeContainer={this._mainCont} + highlightDragSrcColor="" + anchorMenuCrop={this.crop} + // anchorMenuFlashcard={() => this.getImageDesc()} + /> + )} + </div> + {this._outpaintingInProgress && ( + <div className="imageBox-outpaintingSpinner"> + <ReactLoading type="spin" color="#666" height={60} width={60} /> </div> - ) : null} - {this.regenerateImageIcon} - {this.overlayImageIcon} - {this.annotationLayer} - {!this._mainCont || !this.DocumentView || !this._annotationLayer.current ? null : ( - <MarqueeAnnotator - Document={this.Document} - ref={this.marqueeref} - scrollTop={0} - annotationLayerScrollTop={0} - scaling={returnOne} - annotationLayerScaling={this._props.NativeDimScaling} - screenTransform={this.DocumentView().screenToViewTransform} - docView={this.DocumentView} - addDocument={this.addDocument} - finishMarquee={this.finishMarquee} - savedAnnotations={this.savedAnnotations} - selectionText={returnEmptyString} - annotationLayer={this._annotationLayer.current} - marqueeContainer={this._mainCont} - highlightDragSrcColor="" - anchorMenuCrop={this.crop} - // anchorMenuFlashcard={() => this.getImageDesc()} - /> )} - </div> + </> ); } @@ -834,10 +1073,14 @@ export class ImageBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { const file = input.files?.[0]; if (file) { const disposer = OverlayView.ShowSpinner(); - DocUtils.uploadFileToDoc(file, {}, this.Document).then(doc => { - disposer(); - doc && (doc.height = undefined); - }); + const [{ result }] = await Networking.UploadFilesToServer({ file }); + if (result instanceof Error) { + alert('Error uploading files - possibly due to unsupported file types'); + } else { + this.dataDoc[this.fieldKey] = new ImageField(result.accessPaths.agnostic.client); + !(result instanceof Error) && DocUtils.assignImageInfo(result, this.dataDoc); + } + disposer(); } else { console.log('No file selected'); } diff --git a/src/fields/DimensionField.ts b/src/fields/DimensionField.ts new file mode 100644 index 000000000..ffbb9a732 --- /dev/null +++ b/src/fields/DimensionField.ts @@ -0,0 +1,29 @@ +import { ObjectField } from "./ObjectField"; +import { Copy, ToJavascriptString, ToScriptString, ToString } from "./FieldSymbols"; + +export class DimensionField extends ObjectField { + width: number; + height: number; + + constructor(width: number, height: number) { + super(); + this.width = width; + this.height = height; + } + + [Copy](): DimensionField { + return new DimensionField(this.width, this.height); + } + + [ToJavascriptString](): string { + return `{ width: ${this.width}, height: ${this.height} }`; + } + + [ToScriptString](): string { + return `{ width: ${this.width}, height: ${this.height} }`; + } + + [ToString](): string { + return `${this.width} x ${this.height}`; + } +} diff --git a/src/fields/Doc.ts b/src/fields/Doc.ts index fc89dcbe7..dded8ce03 100644 --- a/src/fields/Doc.ts +++ b/src/fields/Doc.ts @@ -468,6 +468,10 @@ export class Doc extends RefField { } } export namespace Doc { + export let SelectOnLoad: Doc | undefined; + export function SetSelectOnLoad(doc: Doc | undefined) { + SelectOnLoad = doc; + } export let DocDragDataName: string = ''; export function SetDocDragDataName(name: string) { DocDragDataName = name; @@ -1177,6 +1181,27 @@ export namespace Doc { const dheight = NumCast((dataDoc || doc)[Doc.LayoutFieldKey(doc) + '_nativeHeight'], useHeight ? NumCast(doc._height) : 0); return NumCast(doc._nativeHeight, nheight || dheight); } + + + export function OutpaintingWidth(doc?: Doc, dataDoc?: Doc, useWidth?: boolean) { + return !doc ? 0 : NumCast(doc._outpaintingWidth, NumCast((dataDoc || doc)[Doc.LayoutFieldKey(doc) + '_outpaintingWidth'], useWidth ? NumCast(doc._width) : 0)); + } + + export function OutpaintingHeight(doc?: Doc, dataDoc?: Doc, useHeight?: boolean) { + if (!doc) return 0; + const oheight = (Doc.OutpaintingWidth(doc, dataDoc, useHeight) / NumCast(doc._width)) * NumCast(doc._height); + const dheight = NumCast((dataDoc || doc)[Doc.LayoutFieldKey(doc) + '_outpaintingHeight'], useHeight ? NumCast(doc._height) : 0); + return NumCast(doc._outpaintingHeight, oheight || dheight); + } + + export function SetOutpaintingWidth(doc: Doc, width: number | undefined, fieldKey?: string) { + doc[(fieldKey || Doc.LayoutFieldKey(doc)) + '_outpaintingWidth'] = width; + } + + export function SetOutpaintingHeight(doc: Doc, height: number | undefined, fieldKey?: string) { + doc[(fieldKey || Doc.LayoutFieldKey(doc)) + '_outpaintingHeight'] = height; + } + export function SetNativeWidth(doc: Doc, width: number | undefined, fieldKey?: string) { doc[(fieldKey || Doc.LayoutFieldKey(doc)) + '_nativeWidth'] = width; } diff --git a/src/server/ApiManagers/FireflyManager.ts b/src/server/ApiManagers/FireflyManager.ts index e75ede9df..1e0fdb595 100644 --- a/src/server/ApiManagers/FireflyManager.ts +++ b/src/server/ApiManagers/FireflyManager.ts @@ -8,6 +8,7 @@ import { DashUploadUtils } from '../DashUploadUtils'; import { _error, _invalid, _success, Method } from '../RouteManager'; import { Directory, filesDirectory } from '../SocketData'; import ApiManager, { Registration } from './ApiManager'; +import { Upload } from '../SharedMediaTypes'; export default class FireflyManager extends ApiManager { getBearerToken = () => @@ -328,6 +329,324 @@ export default class FireflyManager extends ApiManager { }); }), }); + + register({ + method: Method.POST, + subscription: '/outpaintImageFour', + secureHandler: ({ req, res }) => + + this.uploadImageToDropbox(req.body.imageUrl, req.user as DashUserModel) + .then(uploadUrl => { + if (uploadUrl instanceof Error) { + _invalid(res, uploadUrl.message); + return; + } + return this.getBearerToken() + .then(tokenResponse => tokenResponse?.json()) + .then((tokenData: { access_token: string }) => + fetch('https://firefly-api.adobe.io/v3/images/expand', { + method: 'POST', + headers: { + //'Content-Type': 'application/json', + 'Accept': 'application/json', + 'x-api-key': process.env._CLIENT_FIREFLY_CLIENT_ID ?? '', + 'Authorization': `Bearer ${tokenData.access_token}`, + }, + body: JSON.stringify({ + image: { + source: { url: uploadUrl }, + }, + size: { + width: req.body.newDimensions.width, + height: req.body.newDimensions.height, + }, + prompt: req.body.prompt ?? '', + numVariations: 1, + placement: { + inset: { + left: 0, + top: 0, + right: req.body.newDimensions.width - req.body.originalDimensions.width, + bottom: req.body.newDimensions.height - req.body.originalDimensions.height, + }, + alignment: { + horizontal: 'center', + vertical: 'center', + }, + }, + }), + }) + ) + .then(expandResp => expandResp?.json()) + .then(expandData => { + if (expandData.error_code || !expandData.outputs?.[0]?.image?.url) { + console.error('Firefly validation error:', expandData); + _error(res, expandData.message ?? 'Failed to generate image'); + } else { + DashUploadUtils.UploadImage(expandData.outputs[0].image.url) + .then((info: Upload.ImageInformation | Error) => { + if (info instanceof Error) { + _invalid(res, info.message); + } else { + _success(res, { url: info.accessPaths.agnostic.client }); + } + }) + .catch(uploadErr => { + console.error('DashUpload Error:', uploadErr); + _error(res, 'Failed to upload generated image.'); + }); + } + }) + .catch(err => { + console.error('Firefly request error:', err); + _error(res, 'Failed to expand image'); + }); + }), + }); + register({ + method: Method.POST, + subscription: '/outpaintImageThree', + secureHandler: ({ req, res }) => + new Promise<void>(resolver => { + this.uploadImageToDropbox(req.body.imageUrl, req.user as DashUserModel) + .then(dropboxImgUrl => { + if (dropboxImgUrl instanceof Error) { + _invalid(res, dropboxImgUrl.message); + throw new Error('Error uploading image to dropbox'); + } + return dropboxImgUrl; + }) + .then(dropboxImgUrl => + this.getBearerToken().then(tokenResponse => + tokenResponse?.json().then((tokenData: { access_token: string }) => + fetch('https://firefly-api.adobe.io/v3/images/expand', { + method: 'POST', + headers: [ + ['Content-Type', 'application/json'], + ['Accept', 'application/json'], + ['x-api-key', process.env._CLIENT_FIREFLY_CLIENT_ID ?? ''], + ['Authorization', `Bearer ${tokenData.access_token}`], + ], + body: JSON.stringify({ + image: { + source: { + url: dropboxImgUrl, + }, + }, + numVariations: 1, + prompt: req.body.prompt, + size: { + width: req.body.newDimensions.width, + height: req.body.newDimensions.height, + }, + placement: { + inset: { + left: 0, + top: 0, + right: req.body.newDimensions.width - req.body.originalDimensions.width, + bottom: req.body.newDimensions.height - req.body.originalDimensions.height, + }, + alignment: { + horizontal: 'center', + vertical: 'center', + }, + }, + }), + }) + .then(resp => resp.json()) + .then(expandData => { + if (expandData.error_code || !expandData.outputs?.[0]?.image?.url) { + _invalid(res, expandData.message ?? 'Failed to expand image'); + resolver(); + } else { + DashUploadUtils.UploadImage(expandData.outputs[0].image.url) + .then(info => { + if (info instanceof Error) { + _invalid(res, info.message); + } else { + _success(res, { url: info.accessPaths.agnostic.client }); + } + }) + .then(resolver) + .catch(uploadErr => { + console.error('DashUpload Error:', uploadErr); + _invalid(res, 'Failed to upload generated image.'); + resolver(); + }); + } + }) + .catch(err => { + console.error('Firefly API Error:', err); + _error(res, 'Failed to expand image'); + resolver(); + }) + ) + ) + ) + .catch(err => { + console.error('Dropbox Error:', err); + _error(res, 'Failed to upload image to Dropbox'); + resolver(); + }); + }), + }); + + + register({ + method: Method.POST, + subscription: '/outpaintImageTwo', + secureHandler: async ({ req, res }) => { + try { + const uploadUrl = await this.uploadImageToDropbox(req.body.imageUrl, req.user as DashUserModel); + + if (uploadUrl instanceof Error) { + _invalid(res, uploadUrl.message); + return; + } + + const tokenResponse = await this.getBearerToken(); + const tokenData = await tokenResponse?.json(); + + const body = JSON.stringify({ + image: { source: { url: uploadUrl } }, + prompt: req.body.prompt, + numVariations: 1, + size: { + width: req.body.newDimensions.width, + height: req.body.newDimensions.height, + }, + placement: { + inset: { + left: 0, + top: 0, + right: req.body.newDimensions.width - req.body.originalDimensions.width, + bottom: req.body.newDimensions.height - req.body.originalDimensions.height, + }, + alignment: { + horizontal: 'center', + vertical: 'center', + }, + }, + }); + + console.log('Sending outpainting request:', body); + + const expandResp = await fetch('https://firefly-api.adobe.io/v3/images/expand', { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + Accept: 'application/json', + 'x-api-key': process.env._CLIENT_FIREFLY_CLIENT_ID ?? '', + Authorization: `Bearer ${tokenData.access_token}`, + }, + body, + }); + + const expandData = await expandResp.json(); + console.log('Received expandData:', expandData); + + if (expandData.error_code || !expandData.outputs?.[0]?.image?.url) { + console.error('Expand API Error:', expandData); + _error(res, expandData.message ?? 'Failed to generate image'); + return; + } + + const uploadedInfo = await DashUploadUtils.UploadImage(expandData.outputs[0].image.url); + + if (uploadedInfo instanceof Error) { + console.error('Upload Error:', uploadedInfo.message); + _invalid(res, uploadedInfo.message); + return; + } + + console.log('Successfully uploaded image URL:', uploadedInfo.accessPaths.agnostic.client); + _success(res, { url: uploadedInfo.accessPaths.agnostic.client }); + } catch (err) { + console.error('Unexpected error during outpainting:', err); + _error(res, 'Unexpected error during outpainting'); + } + }, + }); + + register({ + + + method: Method.POST, + subscription: '/outpaintImage', + secureHandler: ({ req, res }) => + this.uploadImageToDropbox(req.body.imageUrl, req.user as DashUserModel).then(uploadUrl => + uploadUrl instanceof Error + ? _invalid(res, uploadUrl.message) + : this.getBearerToken() + .then(tokenResponse => tokenResponse?.json()) + .then((tokenData: { access_token: string }) => + fetch('https://firefly-api.adobe.io/v3/images/expand', { + method: 'POST', + headers: [ + ['Content-Type', 'application/json'], + ['Accept', 'application/json'], + ['x-api-key', process.env._CLIENT_FIREFLY_CLIENT_ID ?? ''], + ['Authorization', `Bearer ${tokenData.access_token}`], + ], + body: JSON.stringify({ + image: { source: { url: uploadUrl } }, + numVariations: 1, + size: { + width: req.body.newDimensions.width, + height: req.body.newDimensions.height, + }, + prompt: req.body.prompt, + placement: { + inset: { + left: 0, + top: 0, + right: 0, //req.body.newDimensions.width - req.body.originalDimensions.width, + bottom: 0 //req.body.newDimensions.height - req.body.originalDimensions.height, + }, + alignment: { + horizontal: 'center', + vertical: 'center', + }, + }, + }), + }) + ) + .then(expandResp => expandResp.json()) + .then(expandData => { + if (expandData.error_code || !expandData.outputs?.[0]?.image?.url) { + _error(res, expandData.message ?? "Failed to generate image"); + } else { + DashUploadUtils.UploadImage(expandData.outputs[0].image.url) + .then((info: Upload.ImageInformation | Error) => { + if (info instanceof Error) { + _invalid(res, info.message); + } else { + // THIS IS CRUCIAL: Respond exactly how your front end expects + console.log("Successfully uploaded image. URL:", info.accessPaths.agnostic.client); + _success(res, { url: info.accessPaths.agnostic.client }); + } + }) + .catch(uploadErr => { + + console.error('DashUpload Error:', uploadErr); + _error(res, 'Failed to upload generated image.'); + }); + } + }) + .catch(err => { + console.error(err); + _error(res, 'Failed to expand image'); + }) + ), + }); + + /* register({ + method: Method.POST + subscription: '/queryFireflyOutpaint', + secureHandler: ({req, res}) => + this.outpaintImage() + })*/ + register({ method: Method.POST, subscription: '/queryFireflyImage', |