1 files changed, 584 insertions, 0 deletions
diff --git a/src/client/views/nodes/generativeFill/GenerativeFill.tsx b/src/client/views/nodes/generativeFill/GenerativeFill.tsx
new file mode 100644
index 000000000..9c03600cf
--- /dev/null
+++ b/src/client/views/nodes/generativeFill/GenerativeFill.tsx
@@ -0,0 +1,584 @@
+import './GenerativeFill.scss';
+import React = require('react');
+import { useEffect, useRef, useState } from 'react';
+import { APISuccess, ImageUtility } from './generativeFillUtils/ImageHandler';
+import { BrushHandler } from './generativeFillUtils/BrushHandler';
+import { IconButton } from 'browndash-components';
+import { Checkbox, FormControlLabel, Slider, TextField } from '@mui/material';
+import { CursorData, ImageDimensions, Point } from './generativeFillUtils/generativeFillInterfaces';
+import { activeColor, canvasSize, eraserColor, freeformRenderSize, newCollectionSize, offsetDistanceY, offsetX } from './generativeFillUtils/generativeFillConstants';
+import { PointerHandler } from './generativeFillUtils/PointerHandler';
+import { IoMdUndo, IoMdRedo } from 'react-icons/io';
+import { MainView } from '../../MainView';
+import { Doc, DocListCast } from '../../../../fields/Doc';
+import { Networking } from '../../../Network';
+import { Utils } from '../../../../Utils';
+import { DocUtils, Docs } from '../../../documents/Documents';
+import { NumCast } from '../../../../fields/Types';
+import { CollectionDockingView } from '../../collections/CollectionDockingView';
+import { OpenWhereMod } from '../DocumentView';
+import Buttons from './GenerativeFillButtons';
+import { List } from '../../../../fields/List';
+import { CgClose } from 'react-icons/cg';
+import { ImageBox } from '../ImageBox';
+
+enum BrushStyle {
+    ADD,
+    SUBTRACT,
+    MARQUEE,
+}
+
+interface GenerativeFillProps {
+    imageEditorOpen: boolean;
+    imageEditorSource: string;
+    imageRootDoc: Doc | undefined;
+    addDoc: ((doc: Doc | Doc[], annotationKey?: string) => boolean) | undefined;
+}
+
+// Added field on image doc: gen_fill_children: List of children Docs
+
+const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addDoc }: GenerativeFillProps) => {
+    const canvasRef = useRef<HTMLCanvasElement>(null);
+    const canvasBackgroundRef = useRef<HTMLCanvasElement>(null);
+    const drawingAreaRef = useRef<HTMLDivElement>(null);
+    const [cursorData, setCursorData] = useState<CursorData>({
+        x: 0,
+        y: 0,
+        width: 150,
+    });
+    const [isBrushing, setIsBrushing] = useState(false);
+    const [canvasScale, setCanvasScale] = useState(0.5);
+    // format: array of [image source, corresponding image Doc]
+    const [edits, setEdits] = useState<(string | Doc)[][]>([]);
+    const [edited, setEdited] = useState(false);
+    const [brushStyle, setBrushStyle] = useState<BrushStyle>(BrushStyle.ADD);
+    const [input, setInput] = useState('');
+    const [loading, setLoading] = useState(false);
+    const [canvasDims, setCanvasDims] = useState<ImageDimensions>({
+        width: canvasSize,
+        height: canvasSize,
+    });
+    // whether to create a new collection or not
+    const [isNewCollection, setIsNewCollection] = useState(true);
+    // the current image in the main canvas
+    const currImg = useRef<HTMLImageElement | null>(null);
+    // the unedited version of each generation (parent)
+    const originalImg = useRef<HTMLImageElement | null>(null);
+    const originalDoc = useRef<Doc | null>(null);
+    // stores history of data urls
+    const undoStack = useRef<string[]>([]);
+    // stores redo stack
+    const redoStack = useRef<string[]>([]);
+
+    // references to keep track of tree structure
+    const newCollectionRef = useRef<Doc | null>(null);
+    const parentDoc = useRef<Doc | null>(null);
+    const childrenDocs = useRef<Doc[]>([]);
+
+    // Undo and Redo
+    const handleUndo = () => {
+        const ctx = ImageUtility.getCanvasContext(canvasRef);
+        if (!ctx || !currImg.current || !canvasRef.current) return;
+
+        const target = undoStack.current[undoStack.current.length - 1];
+        if (!target) {
+            ImageUtility.drawImgToCanvas(currImg.current, canvasRef, canvasDims.width, canvasDims.height);
+        } else {
+            redoStack.current = [...redoStack.current, canvasRef.current.toDataURL()];
+            const img = new Image();
+            img.src = target;
+            ImageUtility.drawImgToCanvas(img, canvasRef, canvasDims.width, canvasDims.height);
+            undoStack.current = undoStack.current.slice(0, -1);
+        }
+    };
+
+    const handleRedo = () => {
+        const ctx = ImageUtility.getCanvasContext(canvasRef);
+        if (!ctx || !currImg.current || !canvasRef.current) return;
+
+        const target = redoStack.current[redoStack.current.length - 1];
+        if (!target) {
+        } else {
+            undoStack.current = [...undoStack.current, canvasRef.current?.toDataURL()];
+            const img = new Image();
+            img.src = target;
+            ImageUtility.drawImgToCanvas(img, canvasRef, canvasDims.width, canvasDims.height);
+            redoStack.current = redoStack.current.slice(0, -1);
+        }
+    };
+
+    // resets any erase strokes
+    const handleReset = () => {
+        if (!canvasRef.current || !currImg.current) return;
+        const ctx = ImageUtility.getCanvasContext(canvasRef);
+        if (!ctx) return;
+        ctx.clearRect(0, 0, canvasSize, canvasSize);
+        undoStack.current = [];
+        redoStack.current = [];
+        ImageUtility.drawImgToCanvas(currImg.current, canvasRef, canvasDims.width, canvasDims.height);
+    };
+
+    // initiate brushing
+    const handlePointerDown = (e: React.PointerEvent) => {
+        const canvas = canvasRef.current;
+        if (!canvas) return;
+        const ctx = ImageUtility.getCanvasContext(canvasRef);
+        if (!ctx) return;
+
+        undoStack.current = [...undoStack.current, canvasRef.current.toDataURL()];
+        redoStack.current = [];
+
+        setIsBrushing(true);
+        const { x, y } = PointerHandler.getPointRelativeToElement(canvas, e, canvasScale);
+        BrushHandler.brushCircleOverlay(x, y, cursorData.width / 2 / canvasScale, ctx, eraserColor, brushStyle === BrushStyle.SUBTRACT);
+    };
+
+    // stop brushing, push to undo stack
+    const handlePointerUp = (e: React.PointerEvent) => {
+        const ctx = ImageUtility.getCanvasContext(canvasBackgroundRef);
+        if (!ctx) return;
+        if (!isBrushing) return;
+        setIsBrushing(false);
+    };
+
+    // handles brushing on pointer movement
+    useEffect(() => {
+        if (!isBrushing) return;
+        const canvas = canvasRef.current;
+        if (!canvas) return;
+        const ctx = ImageUtility.getCanvasContext(canvasRef);
+        if (!ctx) return;
+
+        const handlePointerMove = (e: PointerEvent) => {
+            const currPoint = PointerHandler.getPointRelativeToElement(canvas, e, canvasScale);
+            const lastPoint: Point = {
+                x: currPoint.x - e.movementX / canvasScale,
+                y: currPoint.y - e.movementY / canvasScale,
+            };
+            BrushHandler.createBrushPathOverlay(lastPoint, currPoint, cursorData.width / 2 / canvasScale, ctx, eraserColor, brushStyle === BrushStyle.SUBTRACT);
+        };
+
+        drawingAreaRef.current?.addEventListener('pointermove', handlePointerMove);
+        return () => {
+            drawingAreaRef.current?.removeEventListener('pointermove', handlePointerMove);
+        };
+    }, [isBrushing]);
+
+    // first load
+    useEffect(() => {
+        if (!imageEditorSource || imageEditorSource === '') return;
+        const img = new Image();
+        img.src = imageEditorSource;
+        currImg.current = img;
+        originalImg.current = img;
+        img.onload = () => {
+            const imgWidth = img.naturalWidth;
+            const imgHeight = img.naturalHeight;
+            const scale = Math.min(canvasSize / imgWidth, canvasSize / imgHeight);
+            const width = imgWidth * scale;
+            const height = imgHeight * scale;
+            setCanvasDims({ width, height });
+        };
+
+        // cleanup
+        return () => {
+            setInput('');
+            setEdited(false);
+            newCollectionRef.current = null;
+            parentDoc.current = null;
+            childrenDocs.current = [];
+            currImg.current = null;
+            originalImg.current = null;
+            originalDoc.current = null;
+            undoStack.current = [];
+            redoStack.current = [];
+            ImageUtility.clearCanvas(canvasRef);
+        };
+    }, [canvasRef, imageEditorSource]);
+
+    // once the appropriate dimensions are set, draw the image to the canvas
+    useEffect(() => {
+        if (!currImg.current) return;
+        ImageUtility.drawImgToCanvas(currImg.current, canvasRef, canvasDims.width, canvasDims.height);
+    }, [canvasDims]);
+
+    // handles brush sizing
+    useEffect(() => {
+        const handleKeyPress = (e: KeyboardEvent) => {
+            if (e.key === 'ArrowUp') {
+                e.preventDefault();
+                e.stopPropagation();
+                setCursorData(data => ({ ...data, width: data.width + 5 }));
+            } else if (e.key === 'ArrowDown') {
+                e.preventDefault();
+                e.stopPropagation();
+                setCursorData(data => (data.width >= 20 ? { ...data, width: data.width - 5 } : data));
+            }
+        };
+        window.addEventListener('keydown', handleKeyPress);
+        return () => window.removeEventListener('keydown', handleKeyPress);
+    }, []);
+
+    // handle pinch zoom
+    useEffect(() => {
+        const handlePinch = (e: WheelEvent) => {
+            e.preventDefault();
+            e.stopPropagation();
+            const delta = e.deltaY;
+            const scaleFactor = delta > 0 ? 0.98 : 1.02;
+            setCanvasScale(prevScale => prevScale * scaleFactor);
+        };
+
+        drawingAreaRef.current?.addEventListener('wheel', handlePinch, {
+            passive: false,
+        });
+        return () => drawingAreaRef.current?.removeEventListener('wheel', handlePinch);
+    }, [drawingAreaRef]);
+
+    // updates the current position of the cursor
+    const updateCursorData = (e: React.PointerEvent) => {
+        const drawingArea = drawingAreaRef.current;
+        if (!drawingArea) return;
+        const { x, y } = PointerHandler.getPointRelativeToElement(drawingArea, e, 1);
+        setCursorData(data => ({
+            ...data,
+            x,
+            y,
+        }));
+    };
+
+    // Get AI Edit
+    const getEdit = async () => {
+        const img = currImg.current;
+        if (!img) return;
+        const canvas = canvasRef.current;
+        if (!canvas) return;
+        const ctx = ImageUtility.getCanvasContext(canvasRef);
+        if (!ctx) return;
+        setLoading(true);
+        setEdited(true);
+        try {
+            const canvasOriginalImg = ImageUtility.getCanvasImg(img);
+            if (!canvasOriginalImg) return;
+            const canvasMask = ImageUtility.getCanvasMask(canvas, canvasOriginalImg);
+            if (!canvasMask) return;
+            const maskBlob = await ImageUtility.canvasToBlob(canvasMask);
+            const imgBlob = await ImageUtility.canvasToBlob(canvasOriginalImg);
+            const res = await ImageUtility.getEdit(imgBlob, maskBlob, input !== '' ? input + ' in the same style' : 'Fill in the image in the same style', 2);
+            // const res = await ImageUtility.mockGetEdit(img.src);
+
+            // create first image
+            if (!newCollectionRef.current) {
+                if (!isNewCollection && imageRootDoc) {
+                    // if the parent hasn't been set yet
+                    if (!parentDoc.current) parentDoc.current = imageRootDoc;
+                } else {
+                    if (!(originalImg.current && imageRootDoc)) return;
+                    // create new collection and add it to the view
+                    newCollectionRef.current = Docs.Create.FreeformDocument([], {
+                        x: NumCast(imageRootDoc.x) + NumCast(imageRootDoc._width) + offsetX,
+                        y: NumCast(imageRootDoc.y),
+                        _width: newCollectionSize,
+                        _height: newCollectionSize,
+                        title: 'Image edit collection',
+                    });
+                    DocUtils.MakeLink(imageRootDoc, newCollectionRef.current, { link_relationship: 'Image Edit Version History', link_displayLine: false });
+
+                    // opening new tab
+                    CollectionDockingView.AddSplit(newCollectionRef.current, OpenWhereMod.right);
+
+                    // add the doc to the main freeform
+                    await createNewImgDoc(originalImg.current, true);
+                }
+            } else {
+                childrenDocs.current = [];
+            }
+
+            originalImg.current = currImg.current;
+            originalDoc.current = parentDoc.current;
+            const { urls } = res as APISuccess;
+            const imgUrls = await Promise.all(urls.map(url => ImageUtility.convertImgToCanvasUrl(url, canvasDims.width, canvasDims.height)));
+            const imgRes = await Promise.all(
+                imgUrls.map(async url => {
+                    const saveRes = await onSave(url);
+                    return [url, saveRes as Doc];
+                })
+            );
+            setEdits(imgRes);
+            const image = new Image();
+            image.src = imgUrls[0];
+            ImageUtility.drawImgToCanvas(image, canvasRef, canvasDims.width, canvasDims.height);
+            currImg.current = image;
+            parentDoc.current = imgRes[0][1] as Doc;
+        } catch (err) {
+            console.log(err);
+        }
+        setLoading(false);
+    };
+
+    // adjusts all the img positions to be aligned
+    const adjustImgPositions = () => {
+        if (!parentDoc.current) return;
+        const startY = NumCast(parentDoc.current.y);
+        const children = DocListCast(parentDoc.current.gen_fill_children);
+        const len = children.length;
+        let initialYPositions: number[] = [];
+        for (let i = 0; i < len; i++) {
+            initialYPositions.push(startY + i * offsetDistanceY);
+        }
+        children.forEach((doc, i) => {
+            if (len % 2 === 1) {
+                doc.y = initialYPositions[i] - Math.floor(len / 2) * offsetDistanceY;
+            } else {
+                doc.y = initialYPositions[i] - (len / 2 - 1 / 2) * offsetDistanceY;
+            }
+        });
+    };
+
+    // creates a new image document and returns its reference
+    const createNewImgDoc = async (img: HTMLImageElement, firstDoc: boolean): Promise<Doc | undefined> => {
+        if (!imageRootDoc) return;
+        const src = img.src;
+        const [result] = await Networking.PostToServer('/uploadRemoteImage', { sources: [src] });
+        const source = Utils.prepend(result.accessPaths.agnostic.client);
+
+        if (firstDoc) {
+            const x = 0;
+            const initialY = 0;
+            const newImg = Docs.Create.ImageDocument(source, {
+                x: x,
+                y: initialY,
+                _height: freeformRenderSize,
+                _width: freeformRenderSize,
+                data_nativeWidth: result.nativeWidth,
+                data_nativeHeight: result.nativeHeight,
+            });
+            if (isNewCollection && newCollectionRef.current) {
+                Doc.AddDocToList(newCollectionRef.current, undefined, newImg);
+            } else {
+                addDoc?.(newImg);
+            }
+            parentDoc.current = newImg;
+            return newImg;
+        } else {
+            if (!parentDoc.current) return;
+            const x = NumCast(parentDoc.current.x) + freeformRenderSize + offsetX;
+            const initialY = 0;
+
+            const newImg = Docs.Create.ImageDocument(source, {
+                x: x,
+                y: initialY,
+                _height: freeformRenderSize,
+                _width: freeformRenderSize,
+                data_nativeWidth: result.nativeWidth,
+                data_nativeHeight: result.nativeHeight,
+            });
+
+            const parentList = DocListCast(parentDoc.current.gen_fill_children);
+            if (parentList.length > 0) {
+                parentList.push(newImg);
+                parentDoc.current.gen_fill_children = new List<Doc>(parentList);
+            } else {
+                parentDoc.current.gen_fill_children = new List<Doc>([newImg]);
+            }
+
+            DocUtils.MakeLink(parentDoc.current, newImg, { link_relationship: `Image edit; Prompt: ${input}`, link_displayLine: true });
+            adjustImgPositions();
+
+            if (isNewCollection && newCollectionRef.current) {
+                Doc.AddDocToList(newCollectionRef.current, undefined, newImg);
+            } else {
+                addDoc?.(newImg);
+            }
+            return newImg;
+        }
+    };
+
+    // Saves an image to the collection
+    const onSave = async (src: string) => {
+        const img = new Image();
+        img.src = src;
+        if (!currImg.current || !originalImg.current || !imageRootDoc) return;
+        try {
+            const res = await createNewImgDoc(img, false);
+            return res;
+        } catch (err) {
+            console.log(err);
+        }
+    };
+
+    // Closes the editor view
+    const handleViewClose = () => {
+        ImageBox.setImageEditorOpen(false);
+        ImageBox.setImageEditorSource('');
+        if (newCollectionRef.current) {
+            newCollectionRef.current.fitContentOnce = true;
+        }
+        setEdits([]);
+    };
+
+    return (
+        <div className="generativeFillContainer" style={{ display: imageEditorOpen ? 'flex' : 'none' }}>
+            <div className="generativeFillControls">
+                <h1>Image Editor</h1>
+                <div style={{ display: 'flex', alignItems: 'center', gap: '1.5rem' }}>
+                    <FormControlLabel
+                        control={
+                            <Checkbox
+                                // disable once edited has been clicked (doesn't make sense to change after first edit)
+                                disabled={edited}
+                                checked={isNewCollection}
+                                onChange={e => {
+                                    setIsNewCollection(prev => !prev);
+                                }}
+                            />
+                        }
+                        label={'Create New Collection'}
+                        labelPlacement="end"
+                        sx={{ whiteSpace: 'nowrap' }}
+                    />
+                    <Buttons getEdit={getEdit} loading={loading} onReset={handleReset} />
+                    <IconButton color={activeColor} tooltip="close" icon={<CgClose size={'16px'} />} onClick={handleViewClose} />
+                </div>
+            </div>
+            {/* Main canvas for editing */}
+            <div
+                className="drawingArea" // this only works if pointerevents: none is set on the custom pointer
+                ref={drawingAreaRef}
+                onPointerOver={updateCursorData}
+                onPointerMove={updateCursorData}
+                onPointerDown={handlePointerDown}
+                onPointerUp={handlePointerUp}>
+                <canvas ref={canvasRef} width={canvasDims.width} height={canvasDims.height} style={{ transform: `scale(${canvasScale})` }} />
+                <canvas ref={canvasBackgroundRef} width={canvasDims.width} height={canvasDims.height} style={{ transform: `scale(${canvasScale})` }} />
+                <div
+                    className="pointer"
+                    style={{
+                        left: cursorData.x,
+                        top: cursorData.y,
+                        width: cursorData.width,
+                        height: cursorData.width,
+                    }}>
+                    <div className="innerPointer"></div>
+                </div>
+                {/* Icons */}
+                <div className="iconContainer">
+                    {/* Undo and Redo */}
+                    <IconButton
+                        style={{ cursor: 'pointer' }}
+                        onPointerDown={e => {
+                            e.stopPropagation();
+                            handleUndo();
+                        }}
+                        onPointerUp={e => {
+                            e.stopPropagation();
+                        }}
+                        color={activeColor}
+                        tooltip="Undo"
+                        icon={<IoMdUndo />}
+                    />
+                    <IconButton
+                        style={{ cursor: 'pointer' }}
+                        onPointerDown={e => {
+                            e.stopPropagation();
+                            handleRedo();
+                        }}
+                        onPointerUp={e => {
+                            e.stopPropagation();
+                        }}
+                        color={activeColor}
+                        tooltip="Redo"
+                        icon={<IoMdRedo />}
+                    />
+                    <div onPointerDown={e => e.stopPropagation()} style={{ height: 225, width: '100%', display: 'flex', justifyContent: 'center', cursor: 'pointer' }}>
+                        <Slider
+                            sx={{
+                                '& input[type="range"]': {
+                                    WebkitAppearance: 'slider-vertical',
+                                },
+                            }}
+                            orientation="vertical"
+                            min={25}
+                            max={500}
+                            defaultValue={150}
+                            size="small"
+                            valueLabelDisplay="auto"
+                            onChange={(e, val) => {
+                                setCursorData(prev => ({ ...prev, width: val as number }));
+                            }}
+                        />
+                    </div>
+                </div>
+                {/* Edits thumbnails*/}
+                <div className="editsBox">
+                    {edits.map((edit, i) => (
+                        <img
+                            key={i}
+                            width={75}
+                            src={edit[0] as string}
+                            style={{ cursor: 'pointer' }}
+                            onClick={async () => {
+                                const img = new Image();
+                                img.src = edit[0] as string;
+                                ImageUtility.drawImgToCanvas(img, canvasRef, canvasDims.width, canvasDims.height);
+                                currImg.current = img;
+                                parentDoc.current = edit[1] as Doc;
+                            }}
+                        />
+                    ))}
+                    {/* Original img thumbnail */}
+                    {edits.length > 0 && (
+                        <div style={{ position: 'relative' }}>
+                            <label
+                                style={{
+                                    position: 'absolute',
+                                    bottom: 10,
+                                    left: 10,
+                                    color: '#ffffff',
+                                    fontSize: '0.8rem',
+                                    letterSpacing: '1px',
+                                    textTransform: 'uppercase',
+                                }}>
+                                Original
+                            </label>
+                            <img
+                                width={75}
+                                src={originalImg.current?.src}
+                                style={{ cursor: 'pointer' }}
+                                onClick={() => {
+                                    if (!originalImg.current) return;
+                                    const img = new Image();
+                                    img.src = originalImg.current.src;
+                                    ImageUtility.drawImgToCanvas(img, canvasRef, canvasDims.width, canvasDims.height);
+                                    currImg.current = img;
+                                    parentDoc.current = originalDoc.current;
+                                }}
+                            />
+                        </div>
+                    )}
+                </div>
+            </div>
+            <div>
+                <TextField
+                    value={input}
+                    onChange={e => setInput(e.target.value)}
+                    disabled={isBrushing}
+                    type="text"
+                    label="Prompt"
+                    placeholder="Prompt..."
+                    InputLabelProps={{ style: { fontSize: '16px' } }}
+                    inputProps={{ style: { fontSize: '16px' } }}
+                    sx={{
+                        backgroundColor: '#ffffff',
+                        position: 'absolute',
+                        bottom: '16px',
+                        transform: 'translateX(calc(50vw - 50%))',
+                        width: 'calc(100vw - 64px)',
+                    }}
+                />
+            </div>
+        </div>
+    );
+};
+
+export default GenerativeFill;