baseline working version, different image sizes work

1 files changed, 53 insertions, 40 deletions

diff --git a/src/client/views/nodes/generativeFill/GenerativeFill.tsx b/src/client/views/nodes/generativeFill/GenerativeFill.tsx
index f136982bc..4d475149d 100644
--- a/src/client/views/nodes/generativeFill/GenerativeFill.tsx
+++ b/src/client/views/nodes/generativeFill/GenerativeFill.tsx
@@ -42,10 +42,9 @@ enum BrushStyle {
     MARQUEE,
 }
 
-interface ImageEdit {
-    imgElement: HTMLImageElement;
-    parent: ImageEdit;
-    children: ImageEdit[];
+interface ImageDimensions {
+    width: number;
+    height: number;
 }
 
 interface GenerativeFillProps {
@@ -59,7 +58,6 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD
     const canvasRef = useRef<HTMLCanvasElement>(null);
     const canvasBackgroundRef = useRef<HTMLCanvasElement>(null);
     const drawingAreaRef = useRef<HTMLDivElement>(null);
-    const fileRef = useRef<HTMLInputElement>(null);
     const [cursorData, setCursorData] = useState<CursorData>({
         x: 0,
         y: 0,
@@ -72,6 +70,10 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD
     const [input, setInput] = useState('');
     const [loading, setLoading] = useState(false);
     const [saveLoading, setSaveLoading] = useState(false);
+    const [canvasDims, setCanvasDims] = useState<ImageDimensions>({
+        width: canvasSize,
+        height: canvasSize,
+    });
     // the current image in the main canvas
     const currImg = useRef<HTMLImageElement | null>(null);
     // the unedited version of each generation (parent)
@@ -91,7 +93,6 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD
     const newCollectionRef = useRef<Doc | null>(null);
     const parentDoc = useRef<Doc | null>(null);
     const childrenDocs = useRef<Doc[]>([]);
-    const addToExistingCollection = useRef<boolean>(false);
 
     // Undo and Redo
     const handleUndo = () => {
@@ -100,12 +101,12 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD
 
         const target = undoStack.current[undoStack.current.length - 1];
         if (!target) {
-            ImageUtility.drawImgToCanvas(currImg.current, canvasRef);
+            ImageUtility.drawImgToCanvas(currImg.current, canvasRef, canvasDims.width, canvasDims.height);
         } else {
             redoStack.current = [...redoStack.current, canvasRef.current.toDataURL()];
             const img = new Image();
             img.src = target;
-            ImageUtility.drawImgToCanvas(img, canvasRef);
+            ImageUtility.drawImgToCanvas(img, canvasRef, canvasDims.width, canvasDims.height);
             undoStack.current = undoStack.current.slice(0, -1);
         }
     };
@@ -117,7 +118,7 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD
         } else {
             const img = new Image();
             img.src = target;
-            ImageUtility.drawImgToCanvas(img, canvasRef);
+            ImageUtility.drawImgToCanvas(img, canvasRef, canvasDims.width, canvasDims.height);
             redoStack.current = redoStack.current.slice(0, -1);
         }
     };
@@ -129,7 +130,7 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD
         ctx.clearRect(0, 0, canvasSize, canvasSize);
         undoStack.current = [];
         redoStack.current = [];
-        ImageUtility.drawImgToCanvas(currImg.current, canvasRef, true);
+        ImageUtility.drawImgToCanvas(currImg.current, canvasRef, canvasSize, canvasSize);
     };
 
     // initiate brushing
@@ -144,7 +145,6 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD
 
         setIsBrushing(true);
         const { x, y } = PointerHandler.getPointRelativeToElement(canvas, e, canvasScale);
-
         BrushHandler.brushCircleOverlay(x, y, cursorData.width / 2 / canvasScale, ctx, eraserColor, brushStyle === BrushStyle.SUBTRACT);
     };
 
@@ -181,19 +181,21 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD
 
     // first load
     useEffect(() => {
-        console.log('first load');
         if (!imageEditorSource || imageEditorSource === '') return;
         const img = new Image();
-        console.log('source', imageEditorSource);
         img.src = imageEditorSource;
-        console.log('drawing image');
-        ImageUtility.drawImgToCanvas(img, canvasRef);
         currImg.current = img;
         originalImg.current = img;
-        freeformPosition.current = [0, 0];
+        img.onload = () => {
+            const imgWidth = img.naturalWidth;
+            const imgHeight = img.naturalHeight;
+            const scale = Math.min(canvasSize / imgWidth, canvasSize / imgHeight);
+            const width = imgWidth * scale;
+            const height = imgHeight * scale;
+            setCanvasDims({ width, height });
+        };
 
         return () => {
-            console.log('cleanup');
             newCollectionRef.current = null;
             parentDoc.current = null;
             childrenDocs.current = [];
@@ -202,9 +204,15 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD
             freeformPosition.current = [0, 0];
             undoStack.current = [];
             redoStack.current = [];
+            ImageUtility.clearCanvas(canvasRef);
         };
     }, [canvasRef, imageEditorSource]);
 
+    useEffect(() => {
+        if (!currImg.current) return;
+        ImageUtility.drawImgToCanvas(currImg.current, canvasRef, canvasDims.width, canvasDims.height);
+    }, [canvasDims]);
+
     // handles brush sizing
     useEffect(() => {
         const handleKeyPress = (e: KeyboardEvent) => {
@@ -261,15 +269,18 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD
         setLoading(true);
         // need to adjust later
         try {
-            const maskBlob = await ImageUtility.canvasToBlob(canvas);
-            const imgBlob = await ImageUtility.canvasToBlob(ImageUtility.getCanvasImg(img));
+            const canvasOriginalImg = ImageUtility.getCanvasImg(img);
+            if (!canvasOriginalImg) return;
+            const canvasMask = ImageUtility.getCanvasMask(canvas);
+            if (!canvasMask) return;
+            // ImageUtility.downloadCanvas(canvasMask);
+            const maskBlob = await ImageUtility.canvasToBlob(canvasMask);
+            const imgBlob = await ImageUtility.canvasToBlob(canvasOriginalImg);
             const res = await ImageUtility.getEdit(imgBlob, maskBlob, input !== '' ? input + ' in the same style' : 'Fill in the image in the same style', 2);
             // const res = await ImageUtility.mockGetEdit(img.src);
 
             // create first image
             if (!newCollectionRef.current) {
-                if (addToExistingCollection.current) {
-                }
                 if (!(originalImg.current && imageRootDoc)) return;
                 console.log('creating first image');
                 // create new collection and add it to the view
@@ -281,25 +292,30 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD
                     title: 'Image edit collection',
                 });
                 DocUtils.MakeLink(imageRootDoc, newCollectionRef.current, { link_relationship: 'Image Edit Version History', link_displayLine: false });
+
+                // opening new tab
+                newCollectionRef.current.fitContentOnce = true;
+                CollectionDockingView.AddSplit(newCollectionRef.current, OpenWhereMod.right);
+
                 // add the doc to the main freeform
                 // addDoc?.(newCollectionRef.current);
                 await createNewImgDoc(originalImg.current, true);
             } else {
-                parentDoc.current = childrenDocs.current[childrenDocs.current.length - 1];
+                // parentDoc.current = childrenDocs.current[childrenDocs.current.length - 1];
                 childrenDocs.current = [];
             }
 
             originalImg.current = currImg.current;
 
             const { urls } = res as APISuccess;
+            const imgUrls = await Promise.all(urls.map(url => ImageUtility.convertImageToCanvasDataURL(url, canvasDims.width, canvasDims.height)));
             const image = new Image();
-            image.src = urls[0];
-            setEdits(urls);
-            ImageUtility.drawImgToCanvas(image, canvasRef);
+            image.src = imgUrls[0];
+
+            setEdits(imgUrls);
+            ImageUtility.drawImgToCanvas(image, canvasRef, canvasDims.width, canvasDims.height);
             currImg.current = image;
             onSave();
-            freeformPosition.current[0] += 1;
-            freeformPosition.current[1] = 0;
         } catch (err) {
             console.log(err);
         }
@@ -334,7 +350,6 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD
         if (firstDoc) {
             const x = 0;
             const initialY = 0;
-            console.log('first doc');
 
             const newImg = Docs.Create.ImageDocument(source, {
                 x: x,
@@ -387,10 +402,10 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD
     };
 
     const handleViewClose = () => {
-        if (newCollectionRef.current) {
-            newCollectionRef.current.fitContentOnce = true;
-            CollectionDockingView.AddSplit(newCollectionRef.current, OpenWhereMod.right);
-        }
+        // if (newCollectionRef.current) {
+        //     newCollectionRef.current.fitContentOnce = true;
+        //     CollectionDockingView.AddSplit(newCollectionRef.current, OpenWhereMod.right);
+        // }
         MainView.Instance.setImageEditorOpen(false);
         MainView.Instance.setImageEditorSource('');
         setEdits([]);
@@ -420,8 +435,8 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD
                 onPointerMove={updateCursorData}
                 onPointerDown={handlePointerDown}
                 onPointerUp={handlePointerUp}>
-                <canvas ref={canvasRef} width={canvasSize} height={canvasSize} style={{ transform: `scale(${canvasScale})` }} />
-                <canvas ref={canvasBackgroundRef} width={canvasSize} height={canvasSize} style={{ transform: `scale(${canvasScale})` }} />
+                <canvas ref={canvasRef} width={canvasDims.width} height={canvasSize} style={{ transform: `scale(${canvasScale})` }} />
+                <canvas ref={canvasBackgroundRef} width={canvasDims.height} height={canvasSize} style={{ transform: `scale(${canvasScale})` }} />
                 <div
                     className="pointer"
                     style={{
@@ -489,14 +504,13 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD
                     {edits.map((edit, i) => (
                         <img
                             key={i}
-                            width={100}
-                            height={100}
+                            width={75}
                             src={edit}
                             onClick={async () => {
                                 // if (savedSrcs.current.has(edit)) return;
                                 const img = new Image();
                                 img.src = edit;
-                                ImageUtility.drawImgToCanvas(img, canvasRef);
+                                ImageUtility.drawImgToCanvas(img, canvasRef, canvasDims.width, canvasDims.height);
                                 currImg.current = img;
                                 savedSrcs.current.add(edit);
                                 await onSave();
@@ -519,15 +533,14 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD
                                 Original
                             </label>
                             <img
-                                width={100}
-                                height={100}
+                                width={75}
                                 src={originalImg.current?.src}
                                 style={{ cursor: 'pointer' }}
                                 onClick={() => {
                                     if (!originalImg.current) return;
                                     const img = new Image();
                                     img.src = originalImg.current.src;
-                                    ImageUtility.drawImgToCanvas(img, canvasRef);
+                                    ImageUtility.drawImgToCanvas(img, canvasRef, canvasDims.width, canvasDims.height);
                                     currImg.current = img;
                                 }}
                             />