Doing outpainting implementation

1 files changed, 326 insertions, 83 deletions

diff --git a/src/client/views/nodes/ImageBox.tsx b/src/client/views/nodes/ImageBox.tsx
index 5b06e9fc5..114d5226b 100644
--- a/src/client/views/nodes/ImageBox.tsx
+++ b/src/client/views/nodes/ImageBox.tsx
@@ -1,6 +1,7 @@
 import { Button, Colors, Size, Type } from '@dash/components';
 import { FontAwesomeIcon } from '@fortawesome/react-fontawesome';
 import { Slider, Tooltip } from '@mui/material';
+import { DimensionField } from '../../../fields/DimensionField';
 import axios from 'axios';
 import { action, computed, IReactionDisposer, makeObservable, observable, ObservableMap, reaction } from 'mobx';
 import { observer } from 'mobx-react';
@@ -46,6 +47,7 @@ import { FocusViewOptions } from './FocusViewOptions';
 import './ImageBox.scss';
 import { OpenWhere } from './OpenWhere';
 import { RichTextField } from '../../../fields/RichTextField';
+import { List } from '../../../fields/List';
 
 export class ImageEditorData {
     // eslint-disable-next-line no-use-before-define
@@ -103,6 +105,10 @@ export class ImageBox extends ViewBoxAnnotatableComponent<FieldViewProps>() {
     @observable private _regenerateLoading = false;
     @observable private _prevImgs: FireflyImageData[] = StrCast(this.Document.ai_firefly_history) ? JSON.parse(StrCast(this.Document.ai_firefly_history)) : [];
 
+    // Add these observable properties to the ImageBox class
+    @observable private _outpaintingInProgress = false;
+    @observable private _outpaintingPrompt = '';
+
     constructor(props: FieldViewProps) {
         super(props);
         makeObservable(this);
@@ -135,6 +141,7 @@ export class ImageBox extends ViewBoxAnnotatableComponent<FieldViewProps>() {
     };
 
     componentDidMount() {
+        super.componentDidMount?.();
         this._disposers.sizer = reaction(
             () => ({
                 forceFull: this._props.renderDepth < 1 || this.layoutDoc._showFullRes,
@@ -148,9 +155,9 @@ export class ImageBox extends ViewBoxAnnotatableComponent<FieldViewProps>() {
         );
         const { layoutDoc } = this;
         this._disposers.path = reaction(
-            () => ({ nativeSize: this.nativeSize, width: NumCast(this.layoutDoc._width), height: this.layoutDoc._height }),
-            ({ nativeSize, width, height }) => {
-                if ((layoutDoc === this.layoutDoc && !this.layoutDoc._layout_nativeDimEditable) || !height) {
+            () => ({ nativeSize: this.nativeSize, width: NumCast(this.layoutDoc._width) }),
+            ({ nativeSize, width }) => {
+                if ((layoutDoc === this.layoutDoc && !this.layoutDoc._layout_nativeDimEditable) || !this.layoutDoc._height) {
                     this.layoutDoc._height = (width * nativeSize.nativeHeight) / nativeSize.nativeWidth;
                 }
             },
@@ -166,6 +173,14 @@ export class ImageBox extends ViewBoxAnnotatableComponent<FieldViewProps>() {
             },
             { fireImmediately: true }
         );
+        this._disposers.outpainting = reaction(
+            () => this.Document?._needsOutpainting,
+            needsOutpainting => {
+                if (needsOutpainting && this.Document?._outpaintingResize) {
+                    this.processOutpainting();
+                }
+            }
+        );
     }
 
     componentWillUnmount() {
@@ -214,7 +229,7 @@ export class ImageBox extends ViewBoxAnnotatableComponent<FieldViewProps>() {
                     }, true);
                 } else if (hitDropTarget(e.target as HTMLElement, this._regenerateIconRef.current)) {
                     this._regenerateLoading = true;
-                    const drag = de.complete.docDragData.draggedDocuments.lastElement();
+                    const drag = de.complete.docDragData?.draggedDocuments.lastElement();
                     const dragField = drag[Doc.LayoutFieldKey(drag)];
                     const oldPrompt = StrCast(this.Document.ai_firefly_prompt, StrCast(this.Document.title));
                     const newPrompt = (text: string) => (oldPrompt ? `${oldPrompt} ~~~ ${text}` : text);
@@ -339,6 +354,202 @@ export class ImageBox extends ViewBoxAnnotatableComponent<FieldViewProps>() {
         }
     });
 
+    // Add this method to process outpainting when resize is complete
+    @action
+    processOutpainting = async () => {
+        const field = Cast(this.dataDoc[this.fieldKey], ImageField);
+        if (!field) return;
+    
+        const origWidth = NumCast(this.Document._outpaintingOriginalWidth);
+        const origHeight = NumCast(this.Document._outpaintingOriginalHeight);
+    
+        if (!origWidth || !origHeight) {
+            console.error("Original dimensions (_outpaintingOriginalWidth/_outpaintingOriginalHeight) not set. Ensure resizeViewForOutpainting was called first.");
+            return;
+        }
+    
+        //alert(`Original dimensions: ${origWidth} x ${origHeight}`);
+    
+        
+        // Set flag that outpainting is in progress
+        this._outpaintingInProgress = true;
+        
+        try {
+            // Get the current path to the image
+            const currentPath = this.choosePath(field.url);
+            
+            // Get original and new dimensions for calculating mask
+            const newWidth = NumCast(this.Document._width);
+            const newHeight = NumCast(this.Document._height);
+            
+            // Optional: Ask user for a prompt to guide the outpainting
+            let prompt = "Extend this image naturally with matching content";
+            const customPrompt = await new Promise<string>((resolve) => {
+                const dialog = document.createElement('div');
+                Object.assign(dialog.style, {
+                    position: 'fixed',
+                    top: '50%',
+                    left: '50%',
+                    transform: 'translate(-50%, -50%)',
+                    background: 'white',
+                    padding: '20px',
+                    borderRadius: '8px',
+                    boxShadow: '0 4px 12px rgba(0,0,0,0.2)',
+                    zIndex: '10000'
+                });
+            
+                const title = document.createElement('h3');
+                title.style.marginTop = '0';
+                title.textContent = 'Outpaint Image';
+            
+                const description = document.createElement('p');
+                description.textContent = 'Enter a prompt for extending the image:';
+            
+                const input = document.createElement('input');
+                input.id = 'outpaint-prompt';
+                input.type = 'text';
+                input.value = 'Extend this image naturally with matching content';
+                Object.assign(input.style, {
+                    width: '300px',
+                    padding: '8px',
+                    marginBottom: '10px'
+                });
+            
+                const buttonContainer = document.createElement('div');
+                Object.assign(buttonContainer.style, {
+                    display: 'flex',
+                    justifyContent: 'flex-end',
+                    gap: '10px'
+                });
+            
+                const cancelButton = document.createElement('button');
+                cancelButton.textContent = 'Cancel';
+            
+                const confirmButton = document.createElement('button');
+                confirmButton.textContent = 'Generate';
+                Object.assign(confirmButton.style, {
+                    background: '#0078d4',
+                    color: 'white',
+                    border: 'none',
+                    padding: '8px 16px'
+                });
+            
+                buttonContainer.appendChild(cancelButton);
+                buttonContainer.appendChild(confirmButton);
+            
+                dialog.appendChild(title);
+                dialog.appendChild(description);
+                dialog.appendChild(input);
+                dialog.appendChild(buttonContainer);
+            
+                document.body.appendChild(dialog);
+            
+                cancelButton.onclick = () => {
+                    document.body.removeChild(dialog);
+                    resolve("");
+                };
+            
+                confirmButton.onclick = () => {
+                    const promptValue = input.value;
+                    document.body.removeChild(dialog);
+                    resolve(promptValue);
+                };
+            });
+            
+            // If user cancelled, reset dimensions to original
+            if (!customPrompt) {
+                this.Document._width = origWidth;
+                this.Document._height = origHeight;
+                this._outpaintingInProgress = false;
+                return;
+            }
+            
+            // Show loading indicator
+            const loadingOverlay = document.createElement('div');
+            loadingOverlay.style.position = 'absolute';
+            loadingOverlay.style.top = '0';
+            loadingOverlay.style.left = '0';
+            loadingOverlay.style.width = '100%';
+            loadingOverlay.style.height = '100%';
+            loadingOverlay.style.background = 'rgba(0,0,0,0.5)';
+            loadingOverlay.style.display = 'flex';
+            loadingOverlay.style.justifyContent = 'center';
+            loadingOverlay.style.alignItems = 'center';
+            loadingOverlay.innerHTML = '<div style="color: white; font-size: 16px;">Generating outpainted image...</div>';
+            this._mainCont?.appendChild(loadingOverlay);
+            
+            // Call the outpaint API
+            const response = await Networking.PostToServer('/outpaintImageFour', {
+                imageUrl: currentPath,
+                prompt: customPrompt,
+                originalDimensions: {
+                    width: origWidth,
+                    height: origHeight
+                },
+                newDimensions: {
+                    width: newWidth,
+                    height: newHeight
+                }
+            });
+            if (response && typeof response === 'object' && 'url' in response && typeof response.url === 'string') {
+                console.log("Response is valid and contains URL:", response.url);
+           
+            } else {
+                console.error("Unexpected API response:", response);
+                alert("Failed to receive a valid image URL from server.");
+            }
+            
+            
+            if (response && 'url' in response && typeof response.url === 'string') {
+                // Save the original image as an alternate
+                if (!this.dataDoc[this.fieldKey + '_alternates']) {
+                    this.dataDoc[this.fieldKey + '_alternates'] = new List<Doc>();
+                }
+                
+                // Create a copy of the current image as an alternate
+                const originalDoc = Docs.Create.ImageDocument(field.url.href, {
+                    title: `Original: ${this.Document.title}`,
+                    _nativeWidth: Doc.NativeWidth(this.dataDoc),
+                    _nativeHeight: Doc.NativeHeight(this.dataDoc)
+                });
+                
+                // Add to alternates
+                Doc.AddDocToList(this.dataDoc, this.fieldKey + '_alternates', originalDoc);
+                
+                // Update the image with the outpainted version
+                this.dataDoc[this.fieldKey] = new ImageField(response.url);
+                
+                // Update native dimensions
+                Doc.SetNativeWidth(this.dataDoc, newWidth);
+                Doc.SetNativeHeight(this.dataDoc, newHeight);
+                
+                // Add AI metadata
+                this.Document.ai = true;
+                this.Document.ai_outpainted = true;
+                this.Document.ai_outpaint_prompt = customPrompt;
+            } else {
+                // If failed, revert to original dimensions
+                this.Document._width = origWidth;
+                this.Document._height = origHeight;
+                alert("Failed to outpaint image. Please try again.");
+            }
+            
+            // Remove loading overlay
+            this._mainCont?.removeChild(loadingOverlay);
+            
+        } catch (error) {
+            console.error("Error during outpainting:", error);
+            // Revert to original dimensions on error
+            this.Document._width = origWidth
+            this.Document._height = origHeight
+            alert("An error occurred while outpainting. Please try again.");
+        } finally {
+            // Clear the outpainting flags
+            this._outpaintingInProgress = false;
+            delete this.Document._originalDims;
+        }
+    };
+
     specificContextMenu = (): void => {
         const field = Cast(this.dataDoc[this.fieldKey], ImageField);
         if (field) {
@@ -396,6 +607,27 @@ export class ImageBox extends ViewBoxAnnotatableComponent<FieldViewProps>() {
                 event: action(undoable(async () => await StickerPalette.addToPalette(this.Document), 'save to palette')),
                 icon: this.Document.savedAsSticker ? 'clipboard-check' : 'file-arrow-down',
             });
+            // Add new outpainting option
+            funcs.push({ 
+                description: 'Outpaint Image', 
+                event: () => {
+                    this.processOutpainting();
+                }, 
+                icon: 'brush' 
+            });
+            // Add outpainting history option if the image was outpainted
+            this.Document.ai_outpainted &&
+                funcs.push({
+                    description: 'View Original Image',
+                    event: action(() => {
+                        const alternates = DocListCast(this.dataDoc[this.fieldKey + '_alternates']);
+                        if (alternates && alternates.length) {
+                            // Toggle to show the original image
+                            this.layoutDoc[this.fieldKey + '_usePath'] = 'alternate';
+                        }
+                    }),
+                    icon: 'image',
+                });
             ContextMenu.Instance?.addItem({ description: 'Options...', subitems: funcs, icon: 'asterisk' });
         }
     };
@@ -434,7 +666,7 @@ export class ImageBox extends ViewBoxAnnotatableComponent<FieldViewProps>() {
         const ext = extname(url.href);
         return url.href.replace(ext, (this._error ? '_o' : this._curSuffix) + ext);
     };
-    getScrollHeight = () => (this._props.fitWidth?.(this.Document) !== false && NumCast(this.layoutDoc._freeform_scale, 1) === NumCast(this.dataDoc._freeform_scaleMin, 1) ? this.nativeSize.nativeHeight : undefined);
+    getScrollHeight = () => (this._props.fitWidth?.(this.Document) !== false && NumCast(this.layoutDoc._freeform_scale, 1) === NumCast(this.dataDoc.freeform_scaleMin, 1) ? this.nativeSize.nativeHeight : undefined);
 
     @computed get usingAlternate() {
         const usePath = StrCast(this.Document[this.fieldKey + '_usePath']);
@@ -587,7 +819,7 @@ export class ImageBox extends ViewBoxAnnotatableComponent<FieldViewProps>() {
                     />
                     {fadepath === srcpath ? null : (
                         <div className={`imageBox-fadeBlocker${this.usingAlternate ? '-hover' : ''}`} style={{ transition: StrCast(this.layoutDoc.viewTransition, 'opacity 1000ms') }}>
-                            <img alt="" className="imageBox-fadeaway" key="fadeaway" src={fadepath} style={{ transform, transformOrigin }} draggable={false} width={nativeWidth} />
+                            <img alt="" className="imageBox-fadeaway" key="fadeaway" src={fadepath} style={{ transform, transformOrigin }} draggable={false} width={20} />
                         </div>
                     )}
                 </div>
@@ -746,82 +978,89 @@ export class ImageBox extends ViewBoxAnnotatableComponent<FieldViewProps>() {
         const alts = DocListCast(this.dataDoc[this.fieldKey + '_alternates']);
         const doc = this.usingAlternate ? (alts.lastElement() ?? this.Document) : this.Document;
         return (
-            <div
-                className="imageBox"
-                onContextMenu={this.specificContextMenu}
-                ref={this.createDropTarget}
-                onScroll={action(() => {
-                    if (!this._forcedScroll) {
-                        if (this.layoutDoc._layout_scrollTop || this._mainCont?.scrollTop) {
-                            this._ignoreScroll = true;
-                            this.layoutDoc._layout_scrollTop = this._mainCont?.scrollTop;
-                            this._ignoreScroll = false;
+            <>
+                <div
+                    className="imageBox"
+                    onContextMenu={this.specificContextMenu}
+                    ref={this.createDropTarget}
+                    onScroll={action(() => {
+                        if (!this._forcedScroll) {
+                            if (this.layoutDoc._layout_scrollTop || this._mainCont?.scrollTop) {
+                                this._ignoreScroll = true;
+                                this.layoutDoc._layout_scrollTop = this._mainCont?.scrollTop;
+                                this._ignoreScroll = false;
+                            }
                         }
-                    }
-                })}
-                style={{
-                    width: this._props.PanelWidth() ? undefined : `100%`,
-                    height: this._props.PanelHeight() ? undefined : `100%`,
-                    pointerEvents: this.layoutDoc._lockedPosition ? 'none' : undefined,
-                    borderRadius,
-                    overflow: this.layoutDoc.layout_fitWidth || this._props.fitWidth?.(this.Document) ? 'auto' : 'hidden',
-                }}>
-                <CollectionFreeFormView
-                    ref={this._ffref}
-                    {...this._props}
-                    Document={doc}
-                    setContentViewBox={emptyFunction}
-                    NativeWidth={returnZero}
-                    NativeHeight={returnZero}
-                    renderDepth={this._props.renderDepth + 1}
-                    fieldKey={this.annotationKey}
-                    styleProvider={this._props.styleProvider}
-                    isAnnotationOverlay
-                    annotationLayerHostsContent
-                    PanelWidth={this._props.PanelWidth}
-                    PanelHeight={this._props.PanelHeight}
-                    ScreenToLocalTransform={this.screenToLocalTransform}
-                    select={emptyFunction}
-                    focus={this.focus}
-                    getScrollHeight={this.getScrollHeight}
-                    NativeDimScaling={returnOne}
-                    isAnyChildContentActive={returnFalse}
-                    whenChildContentsActiveChanged={this.whenChildContentsActiveChanged}
-                    removeDocument={this.removeDocument}
-                    moveDocument={this.moveDocument}
-                    addDocument={this.addDocument}>
-                    {this.content}
-                </CollectionFreeFormView>
-                {this.Loading ? (
-                    <div className="loading-spinner" style={{ position: 'absolute' }}>
-                        <ReactLoading type="spin" height={50} width={50} color={'blue'} />
+                    })}
+                    style={{
+                        width: this._props.PanelWidth() ? undefined : `100%`,
+                        height: this._props.PanelHeight() ? undefined : `100%`,
+                        pointerEvents: this.layoutDoc._lockedPosition ? 'none' : undefined,
+                        borderRadius,
+                        overflow: this.layoutDoc.layout_fitWidth || this._props.fitWidth?.(this.Document) ? 'auto' : 'hidden',
+                    }}>
+                    <CollectionFreeFormView
+                        ref={this._ffref}
+                        {...this._props}
+                        Document={doc}
+                        setContentViewBox={emptyFunction}
+                        NativeWidth={returnZero}
+                        NativeHeight={returnZero}
+                        renderDepth={this._props.renderDepth + 1}
+                        fieldKey={this.annotationKey}
+                        styleProvider={this._props.styleProvider}
+                        isAnnotationOverlay
+                        annotationLayerHostsContent
+                        PanelWidth={this._props.PanelWidth}
+                        PanelHeight={this._props.PanelHeight}
+                        ScreenToLocalTransform={this.screenToLocalTransform}
+                        select={emptyFunction}
+                        focus={this.focus}
+                        getScrollHeight={this.getScrollHeight}
+                        NativeDimScaling={returnOne}
+                        isAnyChildContentActive={returnFalse}
+                        whenChildContentsActiveChanged={this.whenChildContentsActiveChanged}
+                        removeDocument={this.removeDocument}
+                        moveDocument={this.moveDocument}
+                        addDocument={this.addDocument}>
+                        {this.content}
+                    </CollectionFreeFormView>
+                    {this.Loading ? (
+                        <div className="loading-spinner" style={{ position: 'absolute' }}>
+                            <ReactLoading type="spin" height={50} width={50} color={'blue'} />
+                        </div>
+                    ) : null}
+                    {this.regenerateImageIcon}
+                    {this.overlayImageIcon}
+                    {this.annotationLayer}
+                    {!this._mainCont || !this.DocumentView || !this._annotationLayer.current ? null : (
+                        <MarqueeAnnotator
+                            Document={this.Document}
+                            ref={this.marqueeref}
+                            scrollTop={0}
+                            annotationLayerScrollTop={0}
+                            scaling={returnOne}
+                            annotationLayerScaling={this._props.NativeDimScaling}
+                            screenTransform={this.DocumentView().screenToViewTransform}
+                            docView={this.DocumentView}
+                            addDocument={this.addDocument}
+                            finishMarquee={this.finishMarquee}
+                            savedAnnotations={this.savedAnnotations}
+                            selectionText={returnEmptyString}
+                            annotationLayer={this._annotationLayer.current}
+                            marqueeContainer={this._mainCont}
+                            highlightDragSrcColor=""
+                            anchorMenuCrop={this.crop}
+                            // anchorMenuFlashcard={() => this.getImageDesc()}
+                        />
+                    )}
+                </div>
+                {this._outpaintingInProgress && (
+                    <div className="imageBox-outpaintingSpinner">
+                        <ReactLoading type="spin" color="#666" height={60} width={60} />
                     </div>
-                ) : null}
-                {this.regenerateImageIcon}
-                {this.overlayImageIcon}
-                {this.annotationLayer}
-                {!this._mainCont || !this.DocumentView || !this._annotationLayer.current ? null : (
-                    <MarqueeAnnotator
-                        Document={this.Document}
-                        ref={this.marqueeref}
-                        scrollTop={0}
-                        annotationLayerScrollTop={0}
-                        scaling={returnOne}
-                        annotationLayerScaling={this._props.NativeDimScaling}
-                        screenTransform={this.DocumentView().screenToViewTransform}
-                        docView={this.DocumentView}
-                        addDocument={this.addDocument}
-                        finishMarquee={this.finishMarquee}
-                        savedAnnotations={this.savedAnnotations}
-                        selectionText={returnEmptyString}
-                        annotationLayer={this._annotationLayer.current}
-                        marqueeContainer={this._mainCont}
-                        highlightDragSrcColor=""
-                        anchorMenuCrop={this.crop}
-                        // anchorMenuFlashcard={() => this.getImageDesc()}
-                    />
                 )}
-            </div>
+            </>
         );
     }
 
@@ -834,10 +1073,14 @@ export class ImageBox extends ViewBoxAnnotatableComponent<FieldViewProps>() {
             const file = input.files?.[0];
             if (file) {
                 const disposer = OverlayView.ShowSpinner();
-                DocUtils.uploadFileToDoc(file, {}, this.Document).then(doc => {
-                    disposer();
-                    doc && (doc.height = undefined);
-                });
+                const [{ result }] = await Networking.UploadFilesToServer({ file });
+                if (result instanceof Error) {
+                    alert('Error uploading files - possibly due to unsupported file types');
+                } else {
+                    this.dataDoc[this.fieldKey] = new ImageField(result.accessPaths.agnostic.client);
+                    !(result instanceof Error) && DocUtils.assignImageInfo(result, this.dataDoc);
+                }
+                disposer();
             } else {
                 console.log('No file selected');
             }