switched to use firefly for image fill

7 files changed, 134 insertions, 409 deletions

diff --git a/src/.DS_Store b/src/.DS_Store
index 1ef749033..a3c57385c 100644
--- a/src/.DS_Store
+++ b/src/.DS_Store
diff --git a/src/client/Network.ts b/src/client/Network.ts
index b11dcb379..850ab4f91 100644
--- a/src/client/Network.ts
+++ b/src/client/Network.ts
@@ -27,6 +27,19 @@ export namespace Networking {
             return response.text().then(text => ({ error: '' + response.status + ':' + response.statusText + '-' + text }));
         });
     }
+    export async function PostFormToServer(prompt: string, source: Blob, mask: Blob, width: number, height: number) {
+        const formData = new FormData();
+        formData.set('prompt', prompt);
+        formData.set('source', source);
+        formData.set('mask', mask);
+        formData.set('width', width.toString());
+        formData.set('height', height.toString());
+        const parameters = {
+            method: 'POST',
+            body: formData,
+        };
+        return fetch('/queryFireflyImageFillWithMask', parameters);
+    }
 
     /**
      * FileGuidPair attaches a guid to a file that is being uploaded,
diff --git a/src/client/views/MainView.tsx b/src/client/views/MainView.tsx
index a3b2741d1..1fee7cd0e 100644
--- a/src/client/views/MainView.tsx
+++ b/src/client/views/MainView.tsx
@@ -1149,7 +1149,7 @@ export class MainView extends ObservableReactComponent<object> {
                 {this.snapLines}
                 <LightboxView key="lightbox" PanelWidth={this._windowWidth} addSplit={CollectionDockingView.AddSplit} PanelHeight={this._windowHeight} maxBorder={this.lightboxMaxBorder} />
                 <SchemaCSVPopUp key="schemacsvpopup" />
-                <ImageEditorBox imageEditorOpen={ImageEditor.Open} imageEditorSource={ImageEditor.Source} imageRootDoc={ImageEditor.RootDoc} addDoc={ImageEditor.AddDoc} />
+                {ImageEditor.Open ? <ImageEditorBox imageEditorOpen={ImageEditor.Open} imageEditorSource={ImageEditor.Source} imageRootDoc={ImageEditor.RootDoc} addDoc={ImageEditor.AddDoc} /> : null}
             </div>
         );
     }
diff --git a/src/client/views/nodes/imageEditor/ImageEditor.tsx b/src/client/views/nodes/imageEditor/ImageEditor.tsx
index abe235ad5..b56490bc3 100644
--- a/src/client/views/nodes/imageEditor/ImageEditor.tsx
+++ b/src/client/views/nodes/imageEditor/ImageEditor.tsx
@@ -269,7 +269,7 @@ const ImageEditor = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addDoc
     };
 
     // Get AI Edit for Generative Fill
-    const getEdit = async () => {
+    const getEdit = async (useFirefly = true) => {
         const img = currImg.current;
         if (!img) return;
         const canvas = canvasRef.current;
@@ -285,40 +285,41 @@ const ImageEditor = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addDoc
             if (!canvasMask) return;
             const maskBlob = await ImageUtility.canvasToBlob(canvasMask);
             const imgBlob = await ImageUtility.canvasToBlob(canvasOriginalImg);
-            const res = await ImageUtility.getEdit(imgBlob, maskBlob, input || 'Fill in the image in the same style', 2);
-            if (res.status == 'error') {
-                alert(res.message);
-            }
-
-            // create first image
-            if (!newCollectionRef.current) {
-                createNewCollection();
+            let imgUrls: string[] = [];
+            const setupCollection = async () => {
+                // create first image
+                if (!newCollectionRef.current) {
+                    createNewCollection();
+                } else {
+                    childrenDocs.current = [];
+                }
+                if (!(originalImg.current && imageRootDoc)) return;
+                // add the doc to the main freeform
+                await createNewImgDoc(originalImg.current, true);
+                originalImg.current = currImg.current;
+                originalDoc.current = parentDoc.current;
+            };
+            if (useFirefly) {
+                const res = await Networking.PostFormToServer(input || 'Fill in the image in the same style', imgBlob, maskBlob, img.width, img.height);
+                if (!res.ok) throw new Error(await res.text());
+                const json = (await res.json()) as APISuccess;
+                imgUrls = json.urls ?? [];
             } else {
-                childrenDocs.current = [];
-            }
-            if (!(originalImg.current && imageRootDoc)) return;
-            // add the doc to the main freeform
-            await createNewImgDoc(originalImg.current, true);
-            originalImg.current = currImg.current;
-            originalDoc.current = parentDoc.current;
-            const { urls } = res as APISuccess;
-            if (res.status !== 'error') {
-                const imgUrls = await Promise.all(urls.map(url => ImageUtility.convertImgToCanvasUrl(url, canvasDims.width, canvasDims.height)));
-                const imgRes = await Promise.all(
-                    imgUrls.map(async url => {
-                        const saveRes = await onSave(url);
-                        return { url, saveRes };
-                    })
-                );
-                setEdits(imgRes);
-                const image = new Image();
-                image.src = imgUrls[0];
-                ImageUtility.drawImgToCanvas(image, canvasRef, canvasDims.width, canvasDims.height);
-                currImg.current = image;
-                parentDoc.current = imgRes[0].saveRes ?? null;
+                const res = await ImageUtility.getEdit(imgBlob, maskBlob, input || 'Fill in the image in the same style', 2);
+                if (res.status == 'error') throw new Error(res.message);
+                const json = res as APISuccess;
+                imgUrls = await Promise.all((json.urls ?? []).map(url => ImageUtility.convertImgToCanvasUrl(url, canvasDims.width, canvasDims.height)));
             }
-        } catch (err) {
-            console.log(err);
+
+            setupCollection();
+            const imgRes = await Promise.all(imgUrls.map(async url => ({ url, saveRes: await onSave(url) })));
+            setEdits(imgRes);
+            currImg.current = new Image();
+            currImg.current.src = imgUrls[0];
+            ImageUtility.drawImgToCanvas(currImg.current, canvasRef, canvasDims.width, canvasDims.height);
+            parentDoc.current = imgRes[0].saveRes ?? null;
+        } catch (err: unknown) {
+            alert('message' in (err as object) ? (err as { message: string }).message : err);
         }
         setLoading(false);
     };
@@ -561,7 +562,7 @@ const ImageEditor = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addDoc
     }
 
     // defines the tools and sets current tool
-    const genFillTool: ImageEditTool = { type: ImageToolType.GenerativeFill, btnText: 'GET EDITS', icon: 'fill', applyFunc: getEdit, sliderMin: 25, sliderMax: 500, sliderDefault: 150 };
+    const genFillTool: ImageEditTool = { type: ImageToolType.GenerativeFill, btnText: 'GET EDITS', icon: 'fill', applyFunc: () => getEdit(), sliderMin: 25, sliderMax: 500, sliderDefault: 150 };
     const cutTool: ImageEditTool = { type: ImageToolType.Cut, btnText: 'CUT IMAGE', icon: 'scissors', applyFunc: cutImage, sliderMin: 1, sliderMax: 50, sliderDefault: 5 };
     const imageEditTools: ImageEditTool[] = [genFillTool, cutTool];
     const [currToolType, setCurrToolType] = useState<ImageToolType>(ImageToolType.GenerativeFill);
diff --git a/src/client/views/nodes/imageEditor/imageToolUtils/BrushHandler.ts b/src/client/views/nodes/imageEditor/imageToolUtils/BrushHandler.ts
deleted file mode 100644
index 7139bebc3..000000000
--- a/src/client/views/nodes/imageEditor/imageToolUtils/BrushHandler.ts
+++ /dev/null
@@ -1,35 +0,0 @@
-import { GenerativeFillMathHelpers } from '../imageEditorUtils/GenerativeFillMathHelpers';
-import { eraserColor } from '../imageEditorUtils/imageEditorConstants';
-import { Point } from '../imageEditorUtils/imageEditorInterfaces';
-import { points } from '@turf/turf';
-
-export enum BrushType {
-    GEN_FILL,
-    CUT,
-}
-
-export class BrushHandler {
-    static brushCircleOverlay = (x: number, y: number, brushRadius: number, ctx: CanvasRenderingContext2D, fillColor: string /* , erase: boolean */) => {
-        ctx.globalCompositeOperation = 'destination-out';
-        ctx.fillStyle = fillColor;
-        ctx.shadowColor = eraserColor;
-        ctx.shadowBlur = 5;
-        ctx.beginPath();
-        ctx.arc(x, y, brushRadius, 0, 2 * Math.PI);
-        ctx.fill();
-        ctx.closePath();
-    };
-
-    static createBrushPathOverlay = (startPoint: Point, endPoint: Point, brushRadius: number, ctx: CanvasRenderingContext2D, fillColor: string, brushType: BrushType) => {
-        const dist = GenerativeFillMathHelpers.distanceBetween(startPoint, endPoint);
-        const pts: Point[] = [];
-        for (let i = 0; i < dist; i += 5) {
-            const s = i / dist;
-            const x = startPoint.x * (1 - s) + endPoint.x * s;
-            const y = startPoint.y * (1 - s) + endPoint.y * s;
-            pts.push({ x: startPoint.x, y: startPoint.y });
-            BrushHandler.brushCircleOverlay(x, y, brushRadius, ctx, fillColor);
-        }
-        return pts;
-    };
-}
diff --git a/src/client/views/nodes/imageEditor/imageToolUtils/ImageHandler.ts b/src/client/views/nodes/imageEditor/imageToolUtils/ImageHandler.ts
deleted file mode 100644
index b9723b5be..000000000
--- a/src/client/views/nodes/imageEditor/imageToolUtils/ImageHandler.ts
+++ /dev/null
@@ -1,312 +0,0 @@
-import { RefObject } from 'react';
-import { bgColor, canvasSize } from '../imageEditorUtils/imageEditorConstants';
-
-export interface APISuccess {
-    status: 'success';
-    urls: string[];
-}
-
-export interface APIError {
-    status: 'error';
-    message: string;
-}
-
-export class ImageUtility {
-    /**
-     *
-     * @param canvas Canvas to convert
-     * @returns Blob of canvas
-     */
-    static canvasToBlob = (canvas: HTMLCanvasElement): Promise<Blob> =>
-        new Promise(resolve => {
-            canvas.toBlob(blob => {
-                if (blob) {
-                    resolve(blob);
-                }
-            }, 'image/png');
-        });
-
-    // given a square api image, get the cropped img
-    static getCroppedImg = (img: HTMLImageElement, width: number, height: number): HTMLCanvasElement | undefined => {
-        // Create a new canvas element
-        const canvas = document.createElement('canvas');
-        canvas.width = width;
-        canvas.height = height;
-        const ctx = canvas.getContext('2d');
-        if (ctx) {
-            // Clear the canvas
-            ctx.clearRect(0, 0, canvas.width, canvas.height);
-            if (width < height) {
-                // horizontal padding, x offset
-                const xOffset = (canvasSize - width) / 2;
-                ctx.drawImage(img, xOffset, 0, canvas.width, canvas.height, 0, 0, canvas.width, canvas.height);
-            } else {
-                // vertical padding, y offset
-                const yOffset = (canvasSize - height) / 2;
-                ctx.drawImage(img, 0, yOffset, canvas.width, canvas.height, 0, 0, canvas.width, canvas.height);
-            }
-            return canvas;
-        }
-        return undefined;
-    };
-
-    // converts an image to a canvas data url
-    static convertImgToCanvasUrl = async (imageSrc: string, width: number, height: number): Promise<string> =>
-        new Promise<string>((resolve, reject) => {
-            const img = new Image();
-            img.onload = () => {
-                const canvas = this.getCroppedImg(img, width, height);
-                if (canvas) {
-                    const dataUrl = canvas.toDataURL();
-                    resolve(dataUrl);
-                }
-            };
-            img.onerror = error => {
-                reject(error);
-            };
-            img.src = imageSrc;
-        });
-
-    // calls the openai api to get image edits
-    static getEdit = async (imgBlob: Blob, maskBlob: Blob, prompt: string, n?: number): Promise<APISuccess | APIError> => {
-        const apiUrl = 'https://api.openai.com/v1/images/edits';
-        const fd = new FormData();
-        fd.append('image', imgBlob, 'image.png');
-        fd.append('mask', maskBlob, 'mask.png');
-        fd.append('prompt', prompt);
-        fd.append('size', '1024x1024');
-        fd.append('n', n ? JSON.stringify(n) : '1');
-        fd.append('response_format', 'b64_json');
-
-        try {
-            const res = await fetch(apiUrl, {
-                method: 'POST',
-                headers: {
-                    Authorization: `Bearer ${process.env.OPENAI_KEY}`,
-                },
-                body: fd,
-            });
-            const data = await res.json();
-            console.log(data.data);
-            return {
-                status: 'success',
-                urls: (data.data as { b64_json: string }[]).map(urlData => `data:image/png;base64,${urlData.b64_json}`),
-            };
-        } catch (err) {
-            console.log(err);
-            return { status: 'error', message: 'API error.' };
-        }
-    };
-
-    // mock api call
-    static mockGetEdit = async (mockSrc: string): Promise<APISuccess | APIError> => ({
-        status: 'success',
-        urls: [mockSrc, mockSrc, mockSrc],
-    });
-
-    // Gets the canvas rendering context of a canvas
-    static getCanvasContext = (canvasRef: RefObject<HTMLCanvasElement>): CanvasRenderingContext2D | null => {
-        if (!canvasRef.current) return null;
-        const ctx = canvasRef.current.getContext('2d');
-        if (!ctx) return null;
-        return ctx;
-    };
-
-    // Helper for downloading the canvas (for debugging)
-    static downloadCanvas = (canvas: HTMLCanvasElement) => {
-        const url = canvas.toDataURL();
-        const downloadLink = document.createElement('a');
-        downloadLink.href = url;
-        downloadLink.download = 'canvas';
-
-        downloadLink.click();
-        downloadLink.remove();
-    };
-
-    // Download the canvas (for debugging)
-    static downloadImageCanvas = (imgUrl: string) => {
-        const img = new Image();
-        img.src = imgUrl;
-        img.onload = () => {
-            const canvas = document.createElement('canvas');
-            canvas.width = canvasSize;
-            canvas.height = canvasSize;
-            const ctx = canvas.getContext('2d');
-            ctx?.drawImage(img, 0, 0, canvasSize, canvasSize);
-
-            this.downloadCanvas(canvas);
-        };
-    };
-
-    // Clears the canvas
-    static clearCanvas = (canvasRef: React.RefObject<HTMLCanvasElement>) => {
-        const ctx = this.getCanvasContext(canvasRef);
-        if (!ctx || !canvasRef.current) return;
-        ctx.clearRect(0, 0, canvasRef.current.width, canvasRef.current.height);
-    };
-
-    // Draws the image to the current canvas
-    static drawImgToCanvas = (img: HTMLImageElement, canvasRef: React.RefObject<HTMLCanvasElement>, width: number, height: number) => {
-        const drawImg = (htmlImg: HTMLImageElement) => {
-            const ctx = this.getCanvasContext(canvasRef);
-            if (!ctx) return;
-            ctx.globalCompositeOperation = 'source-over';
-            ctx.clearRect(0, 0, width, height);
-            ctx.drawImage(htmlImg, 0, 0, width, height);
-        };
-
-        if (img.complete) {
-            drawImg(img);
-        } else {
-            img.onload = () => {
-                drawImg(img);
-            };
-        }
-    };
-
-    // Gets the image mask for the openai endpoint
-    static getCanvasMask = (srcCanvas: HTMLCanvasElement, paddedCanvas: HTMLCanvasElement): HTMLCanvasElement | undefined => {
-        const canvas = document.createElement('canvas');
-        canvas.width = canvasSize;
-        canvas.height = canvasSize;
-        const ctx = canvas.getContext('2d');
-        if (!ctx) return undefined;
-        ctx?.clearRect(0, 0, canvasSize, canvasSize);
-        ctx.drawImage(paddedCanvas, 0, 0);
-
-        // extract and set padding data
-        if (srcCanvas.height > srcCanvas.width) {
-            // horizontal padding, x offset
-            const xOffset = (canvasSize - srcCanvas.width) / 2;
-            ctx?.clearRect(xOffset, 0, srcCanvas.width, srcCanvas.height);
-            ctx.drawImage(srcCanvas, xOffset, 0, srcCanvas.width, srcCanvas.height);
-        } else {
-            // vertical padding, y offset
-            const yOffset = (canvasSize - srcCanvas.height) / 2;
-            ctx?.clearRect(0, yOffset, srcCanvas.width, srcCanvas.height);
-            ctx.drawImage(srcCanvas, 0, yOffset, srcCanvas.width, srcCanvas.height);
-        }
-        return canvas;
-    };
-
-    // Fills in the blank areas of the image with an image reflection (to fill in a square-shaped canvas)
-    static drawHorizontalReflection = (ctx: CanvasRenderingContext2D, canvas: HTMLCanvasElement, xOffset: number) => {
-        const imageData = ctx.getImageData(0, 0, canvas.width, canvas.height);
-        const { data } = imageData;
-        for (let i = 0; i < canvas.height; i++) {
-            for (let j = 0; j < xOffset; j++) {
-                const targetIdx = 4 * (i * canvas.width + j);
-                const sourceI = i;
-                const sourceJ = xOffset + (xOffset - j);
-                const sourceIdx = 4 * (sourceI * canvas.width + sourceJ);
-                data[targetIdx] = data[sourceIdx];
-                data[targetIdx + 1] = data[sourceIdx + 1];
-                data[targetIdx + 2] = data[sourceIdx + 2];
-            }
-        }
-        for (let i = 0; i < canvas.height; i++) {
-            for (let j = canvas.width - 1; j >= canvas.width - 1 - xOffset; j--) {
-                const targetIdx = 4 * (i * canvas.width + j);
-                const sourceI = i;
-                const sourceJ = canvas.width - 1 - xOffset - (xOffset - (canvas.width - j));
-                const sourceIdx = 4 * (sourceI * canvas.width + sourceJ);
-                data[targetIdx] = data[sourceIdx];
-                data[targetIdx + 1] = data[sourceIdx + 1];
-                data[targetIdx + 2] = data[sourceIdx + 2];
-            }
-        }
-        ctx.putImageData(imageData, 0, 0);
-    };
-
-    // Fills in the blank areas of the image with an image reflection (to fill in a square-shaped canvas)
-    static drawVerticalReflection = (ctx: CanvasRenderingContext2D, canvas: HTMLCanvasElement, yOffset: number) => {
-        const imageData = ctx.getImageData(0, 0, canvas.width, canvas.height);
-        const { data } = imageData;
-        for (let j = 0; j < canvas.width; j++) {
-            for (let i = 0; i < yOffset; i++) {
-                const targetIdx = 4 * (i * canvas.width + j);
-                const sourceJ = j;
-                const sourceI = yOffset + (yOffset - i);
-                const sourceIdx = 4 * (sourceI * canvas.width + sourceJ);
-                data[targetIdx] = data[sourceIdx];
-                data[targetIdx + 1] = data[sourceIdx + 1];
-                data[targetIdx + 2] = data[sourceIdx + 2];
-            }
-        }
-        for (let j = 0; j < canvas.width; j++) {
-            for (let i = canvas.height - 1; i >= canvas.height - 1 - yOffset; i--) {
-                const targetIdx = 4 * (i * canvas.width + j);
-                const sourceJ = j;
-                const sourceI = canvas.height - 1 - yOffset - (yOffset - (canvas.height - i));
-                const sourceIdx = 4 * (sourceI * canvas.width + sourceJ);
-                data[targetIdx] = data[sourceIdx];
-                data[targetIdx + 1] = data[sourceIdx + 1];
-                data[targetIdx + 2] = data[sourceIdx + 2];
-            }
-        }
-        ctx.putImageData(imageData, 0, 0);
-    };
-
-    // Gets the unaltered (besides filling in padding) version of the image for the api call
-    static getCanvasImg = (img: HTMLImageElement): HTMLCanvasElement | undefined => {
-        const canvas = document.createElement('canvas');
-        canvas.width = canvasSize;
-        canvas.height = canvasSize;
-        const ctx = canvas.getContext('2d');
-        if (!ctx) return undefined;
-        // fix scaling
-        const scale = Math.min(canvasSize / img.width, canvasSize / img.height);
-        const width = Math.floor(img.width * scale);
-        const height = Math.floor(img.height * scale);
-        ctx?.clearRect(0, 0, canvasSize, canvasSize);
-        ctx.fillStyle = bgColor;
-        ctx.fillRect(0, 0, canvasSize, canvasSize);
-
-        // extract and set padding data
-        if (img.naturalHeight > img.naturalWidth) {
-            // horizontal padding, x offset
-            const xOffset = Math.floor((canvasSize - width) / 2);
-            ctx.drawImage(img, xOffset, 0, width, height);
-
-            // draw reflected image padding
-            this.drawHorizontalReflection(ctx, canvas, xOffset);
-        } else {
-            // vertical padding, y offset
-            const yOffset = Math.floor((canvasSize - height) / 2);
-            ctx.drawImage(img, 0, yOffset, width, height);
-
-            // draw reflected image padding
-            this.drawVerticalReflection(ctx, canvas, yOffset);
-        }
-        return canvas;
-    };
-
-    /**
-     * Converts a url to base64 (tainted canvas workaround)
-     */
-    static urlToBase64 = async (imageUrl: string): Promise<string | undefined> => {
-        try {
-            const res = await fetch(imageUrl);
-            const blob = await res.blob();
-
-            return new Promise<string>((resolve, reject) => {
-                const reader = new FileReader();
-                reader.onload = () => {
-                    const base64Data = reader.result?.toString().split(',')[1];
-                    if (base64Data) {
-                        resolve(base64Data);
-                    } else {
-                        reject(new Error('Failed to convert.'));
-                    }
-                };
-                reader.onerror = () => {
-                    reject(new Error('Error reading image data'));
-                };
-                reader.readAsDataURL(blob);
-            });
-        } catch (err) {
-            console.error(err);
-        }
-        return undefined;
-    };
-}
diff --git a/src/server/ApiManagers/FireflyManager.ts b/src/server/ApiManagers/FireflyManager.ts
index 6393a1f74..fbf7d7202 100644
--- a/src/server/ApiManagers/FireflyManager.ts
+++ b/src/server/ApiManagers/FireflyManager.ts
@@ -7,8 +7,9 @@ import { DashUserModel } from '../authentication/DashUserModel';
 import { DashUploadUtils } from '../DashUploadUtils';
 import { _error, _invalid, _success, Method } from '../RouteManager';
 import { Upload } from '../SharedMediaTypes';
-import { Directory, filesDirectory } from '../SocketData';
+import { Directory, filesDirectory, pathToDirectory } from '../SocketData';
 import ApiManager, { Registration } from './ApiManager';
+import * as formidable from 'formidable';
 
 export default class FireflyManager extends ApiManager {
     getBearerToken = () =>
@@ -23,6 +24,44 @@ export default class FireflyManager extends ApiManager {
             return undefined;
         });
 
+    generateImageFillWithMask = (prompt: string = 'a realistic illustration of a cat coding', uploadUrl: string, maskUrl: string | undefined, width: number = 2048, height: number = 2048, variations: number = 1) =>
+        this.getBearerToken().then(response =>
+            response?.json().then((data: { access_token: string }) =>
+                //prettier-ignore
+                fetch('https://firefly-api.adobe.io/v3/images/fill', {
+                        method: 'POST',
+                        headers: [
+                            ['Content-Type', 'application/json'],
+                            ['Accept', 'application/json'],
+                            ['x-api-key', process.env._CLIENT_FIREFLY_CLIENT_ID ?? ''],
+                            ['Authorization', `Bearer ${data.access_token}`],
+                        ],
+                        body: JSON.stringify({
+                            image: {
+                                source: { url: uploadUrl },
+                                mask: {
+                                    invert: true,
+                                    url: maskUrl
+                                },
+                            },
+                            size: {
+                                width: Math.round(width),
+                                height: Math.round(height),
+                            },
+                            prompt: prompt ?? '',
+                            numVariations: variations,
+                        }),
+                    })
+                    .then(response2 => response2.json().then(json => 
+                    {
+                        if (json.outputs?.length)
+                            return (json.outputs as {image: {url:string }}[]).map(output => output.image);
+                        throw new Error(JSON.stringify(json));
+                    })
+                )
+            )
+        );
+
     generateImageFromStructure = (
         prompt: string = 'a realistic illustration of a cat coding',
         width: number = 2048,
@@ -79,6 +118,30 @@ export default class FireflyManager extends ApiManager {
             )
         );
 
+    uploadToDropbox = (dropboxClient: Dropbox, user: DashUserModel | undefined, fileUrl: string, contents: NonSharedBuffer, recursed?: boolean): Promise<string | Error> =>
+        dropboxClient
+            .filesUpload({ path: `/Apps/browndash/${path.basename(fileUrl)}`, contents, mode: { '.tag': 'overwrite' } })
+            .then(response =>
+                dropboxClient
+                    .filesGetTemporaryLink({ path: response.result.path_display ?? '' })
+                    .then(link => link.result.link)
+                    .catch(linkErr => new Error('Failed to get temporary link: ' + linkErr.message))
+            )
+            .catch(uploadErr => {
+                if (user?.dropboxRefresh && !recursed) {
+                    console.log('Attempting to refresh Dropbox token for user:', user.email);
+                    return this.refreshDropboxToken(user)
+                        .then(token => {
+                            if (!token) return new Error('Failed to refresh Dropbox token.' + user.email);
+
+                            const dbxNew = new Dropbox({ accessToken: token });
+                            return this.uploadToDropbox(dbxNew, user, fileUrl, contents, true).catch(finalErr => new Error('Failed to refresh Dropbox token:' + finalErr.message));
+                        })
+                        .catch(refreshErr => new Error('Failed to refresh Dropbox token: ' + refreshErr.message));
+                }
+                return new Error('Dropbox error: ' + uploadErr.message);
+            });
+
     uploadImageToDropbox = (fileUrl: string, user: DashUserModel | undefined, dbx = new Dropbox({ accessToken: user?.dropboxToken || '' })) =>
         new Promise<string>((resolve, reject) => {
             fs.readFile(path.join(filesDirectory, `${Directory.images}/${path.basename(fileUrl)}`), undefined, (err, contents) => {
@@ -86,32 +149,7 @@ export default class FireflyManager extends ApiManager {
                     return reject(new Error('Error reading file:' + err.message));
                 }
 
-                const uploadToDropbox = (dropboxClient: Dropbox) =>
-                    dropboxClient
-                        .filesUpload({ path: `/Apps/browndash/${path.basename(fileUrl)}`, contents })
-                        .then(response =>
-                            dropboxClient
-                                .filesGetTemporaryLink({ path: response.result.path_display ?? '' })
-                                .then(link => resolve(link.result.link))
-                                .catch(linkErr => reject(new Error('Failed to get temporary link: ' + linkErr.message)))
-                        )
-                        .catch(uploadErr => {
-                            if (user?.dropboxRefresh) {
-                                console.log('Attempting to refresh Dropbox token for user:', user.email);
-                                this.refreshDropboxToken(user)
-                                    .then(token => {
-                                        if (!token) return reject(new Error('Failed to refresh Dropbox token.' + user.email));
-
-                                        const dbxNew = new Dropbox({ accessToken: token });
-                                        uploadToDropbox(dbxNew).catch(finalErr => reject(new Error('Failed to refresh Dropbox token:' + finalErr.message)));
-                                    })
-                                    .catch(refreshErr => reject(new Error('Failed to refresh Dropbox token: ' + refreshErr.message)));
-                            } else {
-                                reject(new Error('Dropbox error: ' + uploadErr.message));
-                            }
-                        });
-
-                uploadToDropbox(dbx);
+                this.uploadToDropbox(dbx, user, fileUrl, contents).then(value => (value instanceof Error ? reject(value) : resolve(value)));
             });
         });
 
@@ -318,6 +356,26 @@ export default class FireflyManager extends ApiManager {
                     })
             ), // prettier-ignore
         });
+        register({
+            method: Method.POST,
+            subscription: '/queryFireflyImageFillWithMask',
+            secureHandler: ({ req, res }) =>
+                new Promise<string>(resolve => {
+                    const user = req.user as DashUserModel;
+                    const accessToken = user?.dropboxToken || '';
+                    const dbx = new Dropbox({ accessToken });
+                    const form = new formidable.IncomingForm({ keepExtensions: true, uploadDir: pathToDirectory(Directory.parsed_files) });
+                    form.parse(req, async (err, fields, files) => {
+                        if (files.source && files.mask) {
+                            Promise.all([this.uploadToDropbox(dbx, user, 'source.png', fs.readFileSync(files.source[0].filepath)), 
+                                         this.uploadToDropbox(dbx, user, 'mask.png', fs.readFileSync(files.mask[0].filepath))])
+                            .then(stuff => 
+                                stuff.some(s => s instanceof Error) ? resolve("") : this.generateImageFillWithMask(fields["prompt"]?.[0], stuff[0] as string, stuff[1] as string, 2048, 2048, 1).then(url => resolve(url![0].url))
+                            ).catch(() => resolve("") ); // prettier-ignore
+                        }
+                    });
+                }).then(url => (url ? _success(res, { urls: [url] }) : _invalid(res, 'Failed to fill image'))),
+        });
 
         register({
             method: Method.POST,