import './GenerativeFill.scss'; import React = require('react'); import { useEffect, useRef, useState } from 'react'; import { APISuccess, ImageUtility } from './generativeFillUtils/ImageHandler'; import { BrushHandler } from './generativeFillUtils/BrushHandler'; import { IconButton, TextField } from '@mui/material'; import { CursorData, Point } from './generativeFillUtils/generativeFillInterfaces'; import { activeColor, canvasSize, eraserColor, freeformRenderSize, offsetDistanceY, offsetX } from './generativeFillUtils/generativeFillConstants'; import { PointerHandler } from './generativeFillUtils/PointerHandler'; import { BsBrush, BsEraser, BsX } from 'react-icons/bs'; import { AiOutlineUpload } from 'react-icons/ai'; import { CiUndo, CiRedo } from 'react-icons/ci'; import Buttons from './GenerativeFillButtons'; import { EditableText } from 'browndash-components'; import { MainView } from '../../MainView'; import { Doc } from '../../../../fields/Doc'; import { Networking } from '../../../Network'; import { Utils } from '../../../../Utils'; import { DocUtils, Docs } from '../../../documents/Documents'; import { DocCast, NumCast } from '../../../../fields/Types'; import { CollectionDockingView } from '../../collections/CollectionDockingView'; import { OpenWhere, OpenWhereMod } from '../DocumentView'; /** * For images not 1024x1024 fill in the rest in solid black, or a * reflected version of the image. */ /** * TODO: Look into img onload, sometimes the canvas doesn't update properly */ enum BrushStyle { ADD, SUBTRACT, MARQUEE, } interface ImageEdit { imgElement: HTMLImageElement; parent: ImageEdit; children: ImageEdit[]; } interface GenerativeFillProps { imageEditorOpen: boolean; imageEditorSource: string; imageRootDoc: Doc | undefined; addDoc: ((doc: Doc | Doc[], annotationKey?: string) => boolean) | undefined; } const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addDoc }: GenerativeFillProps) => { const canvasRef = useRef(null); const canvasBackgroundRef = useRef(null); const drawingAreaRef = useRef(null); const fileRef = useRef(null); const [cursorData, setCursorData] = useState({ x: 0, y: 0, width: 150, }); const [isBrushing, setIsBrushing] = useState(false); const [canvasScale, setCanvasScale] = useState(0.5); const [edits, setEdits] = useState([]); const [brushStyle, setBrushStyle] = useState(BrushStyle.ADD); const [input, setInput] = useState(''); const [loading, setLoading] = useState(false); // the current image in the main canvas const currImg = useRef(null); // the unedited version of each generation (parent) const originalImg = useRef(null); // stores history of data urls const undoStack = useRef([]); // stores redo stack const redoStack = useRef([]); // early stage properly, likely will get rid of const freeformPosition = useRef([0, 0]); // which urls were already saved to canvas const savedSrcs = useRef([]); // references to keep track of tree structure const newCollectionRef = useRef(null); const parentDoc = useRef(null); const childrenDocs = useRef([]); // Undo and Redo const handleUndo = () => { const ctx = ImageUtility.getCanvasContext(canvasRef); if (!ctx || !currImg.current || !canvasRef.current) return; const target = undoStack.current[undoStack.current.length - 1]; if (!target) { ImageUtility.drawImgToCanvas(currImg.current, canvasRef); } else { redoStack.current = [...redoStack.current, canvasRef.current.toDataURL()]; const img = new Image(); img.src = target; ImageUtility.drawImgToCanvas(img, canvasRef); undoStack.current = undoStack.current.slice(0, -1); } }; const handleRedo = () => { // TODO: handle undo as well const target = redoStack.current[redoStack.current.length - 1]; if (!target) { } else { const img = new Image(); img.src = target; ImageUtility.drawImgToCanvas(img, canvasRef); redoStack.current = redoStack.current.slice(0, -1); } }; const handleReset = () => { if (!canvasRef.current || !currImg.current) return; const ctx = ImageUtility.getCanvasContext(canvasRef); if (!ctx) return; ctx.clearRect(0, 0, canvasSize, canvasSize); undoStack.current = []; redoStack.current = []; ImageUtility.drawImgToCanvas(currImg.current, canvasRef, true); }; // initiate brushing const handlePointerDown = (e: React.PointerEvent) => { const canvas = canvasRef.current; if (!canvas) return; const ctx = ImageUtility.getCanvasContext(canvasRef); if (!ctx) return; undoStack.current = [...undoStack.current, canvasRef.current.toDataURL()]; redoStack.current = []; setIsBrushing(true); const { x, y } = PointerHandler.getPointRelativeToElement(canvas, e, canvasScale); BrushHandler.brushCircleOverlay(x, y, cursorData.width / 2 / canvasScale, ctx, eraserColor, brushStyle === BrushStyle.SUBTRACT); }; // stop brushing, push to undo stack const handlePointerUp = (e: React.PointerEvent) => { const ctx = ImageUtility.getCanvasContext(canvasBackgroundRef); if (!ctx) return; if (!isBrushing) return; setIsBrushing(false); }; // handles brushing on pointer movement useEffect(() => { if (!isBrushing) return; const canvas = canvasRef.current; if (!canvas) return; const ctx = ImageUtility.getCanvasContext(canvasRef); if (!ctx) return; const handlePointerMove = (e: PointerEvent) => { const currPoint = PointerHandler.getPointRelativeToElement(canvas, e, canvasScale); const lastPoint: Point = { x: currPoint.x - e.movementX / canvasScale, y: currPoint.y - e.movementY / canvasScale, }; BrushHandler.createBrushPathOverlay(lastPoint, currPoint, cursorData.width / 2 / canvasScale, ctx, eraserColor, brushStyle === BrushStyle.SUBTRACT); }; drawingAreaRef.current?.addEventListener('pointermove', handlePointerMove); return () => { drawingAreaRef.current?.removeEventListener('pointermove', handlePointerMove); }; }, [isBrushing]); // first load useEffect(() => { console.log('first load'); if (!imageEditorSource || imageEditorSource === '') return; const img = new Image(); img.src = imageEditorSource; ImageUtility.drawImgToCanvas(img, canvasRef); currImg.current = img; originalImg.current = img; freeformPosition.current = [0, 0]; return () => { console.log('cleanup'); newCollectionRef.current = null; parentDoc.current = null; childrenDocs.current = []; currImg.current = null; originalImg.current = null; freeformPosition.current = [0, 0]; undoStack.current = []; redoStack.current = []; }; }, [canvasRef, imageEditorSource]); // handles brush sizing useEffect(() => { const handleKeyPress = (e: KeyboardEvent) => { if (e.key === 'ArrowUp') { e.preventDefault(); e.stopPropagation(); setCursorData(data => ({ ...data, width: data.width + 5 })); } else if (e.key === 'ArrowDown') { e.preventDefault(); e.stopPropagation(); setCursorData(data => (data.width >= 20 ? { ...data, width: data.width - 5 } : data)); } }; window.addEventListener('keydown', handleKeyPress); return () => window.removeEventListener('keydown', handleKeyPress); }, []); // handle pinch zoom useEffect(() => { const handlePinch = (e: WheelEvent) => { e.preventDefault(); e.stopPropagation(); const delta = e.deltaY; const scaleFactor = delta > 0 ? 0.98 : 1.02; // Adjust the scale factor as per your requirement setCanvasScale(prevScale => prevScale * scaleFactor); }; drawingAreaRef.current?.addEventListener('wheel', handlePinch, { passive: false, }); return () => drawingAreaRef.current?.removeEventListener('wheel', handlePinch); }, [drawingAreaRef]); // updates the current position of the cursor const updateCursorData = (e: React.PointerEvent) => { const drawingArea = drawingAreaRef.current; if (!drawingArea) return; const { x, y } = PointerHandler.getPointRelativeToElement(drawingArea, e, 1); setCursorData(data => ({ ...data, x, y, })); }; // File upload const uploadImg = (e: React.ChangeEvent) => { if (e.target.files) { const file = e.target.files[0]; const image = new Image(); const imgUrl = URL.createObjectURL(file); image.src = imgUrl; ImageUtility.drawImgToCanvas(image, canvasRef); currImg.current = image; } }; // Get AI Edit const getEdit = async () => { const img = currImg.current; if (!img) return; const canvas = canvasRef.current; if (!canvas) return; const ctx = ImageUtility.getCanvasContext(canvasRef); if (!ctx) return; setLoading(true); // need to adjust later try { const maskBlob = await ImageUtility.canvasToBlob(canvas); const imgBlob = await ImageUtility.canvasToBlob(ImageUtility.getCanvasImg(img)); const res = await ImageUtility.getEdit(imgBlob, maskBlob, input !== '' ? input + ' in the same style' : 'Fill in the image in the same style', 2); // const res = await ImageUtility.mockGetEdit(img.src); // create first image if (!newCollectionRef.current) { if (!(originalImg.current && imageRootDoc)) return; console.log('creating first image'); // create new collection and add it to the view newCollectionRef.current = Docs.Create.FreeformDocument([], { x: NumCast(imageRootDoc.x) + NumCast(imageRootDoc._width) + 20, y: NumCast(imageRootDoc.y), _width: 1000, _height: 1000 }); DocUtils.MakeLink(imageRootDoc, newCollectionRef.current, { link_relationship: 'Image Edit Version History', link_displayLine: false }); addDoc?.(newCollectionRef.current); await createNewImgDoc(originalImg.current, true); } else { parentDoc.current = childrenDocs.current[childrenDocs.current.length - 1]; childrenDocs.current = []; } originalImg.current = currImg.current; const { urls } = res as APISuccess; const image = new Image(); image.src = urls[0]; setEdits(urls); ImageUtility.drawImgToCanvas(image, canvasRef); currImg.current = image; onSave(); freeformPosition.current[0] += 1; freeformPosition.current[1] = 0; } catch (err) { console.log(err); } setLoading(false); }; // adjusts all the img positions to be aligned const adjustImgPositions = () => { if (!parentDoc.current) return; const startY = NumCast(parentDoc.current.y); const len = childrenDocs.current.length; console.log(len); let initialYPositions: number[] = []; for (let i = 0; i < len; i++) { initialYPositions.push(startY + i * offsetDistanceY); } childrenDocs.current.forEach((doc, i) => { if (len % 2 === 1) { doc.y = initialYPositions[i] - Math.floor(len / 2) * offsetDistanceY; } else { doc.y = initialYPositions[i] - (len / 2 - 1 / 2) * offsetDistanceY; } }); }; // creates a new image document and returns its reference const createNewImgDoc = async (img: HTMLImageElement, firstDoc: boolean): Promise => { if (!newCollectionRef.current || !imageRootDoc) return; const src = img.src; const [result] = await Networking.PostToServer('/uploadRemoteImage', { sources: [src] }); const source = Utils.prepend(result.accessPaths.agnostic.client); if (firstDoc) { const x = 0; const initialY = 500; console.log('first doc'); const newImg = Docs.Create.ImageDocument(source, { x: x, y: initialY, _height: freeformRenderSize, _width: freeformRenderSize, data_nativeWidth: result.nativeWidth, data_nativeHeight: result.nativeHeight, }); Doc.AddDocToList(newCollectionRef.current, undefined, newImg); parentDoc.current = newImg; return newImg; } else { if (!parentDoc.current) return; const x = NumCast(parentDoc.current.x) + freeformRenderSize + offsetX; // dummy position console.log('creating child elements'); const initialY = 0; const newImg = Docs.Create.ImageDocument(source, { x: x, y: initialY, _height: freeformRenderSize, _width: freeformRenderSize, data_nativeWidth: result.nativeWidth, data_nativeHeight: result.nativeHeight, }); childrenDocs.current.push(newImg); DocUtils.MakeLink(parentDoc.current, newImg, { link_relationship: 'Image Edit', link_displayLine: true }); adjustImgPositions(); Doc.AddDocToList(newCollectionRef.current, undefined, newImg); return newImg; } }; // need to maybe call on every img click, not just when the save btn is clicked const onSave = async () => { setLoading(true); if (!currImg.current || !originalImg.current || !imageRootDoc) return; try { // if (!newCollectionRef.current) { // console.log('creating first image'); // // create new collection and add it to the view // newCollectionRef.current = Docs.Create.FreeformDocument([], { x: NumCast(imageRootDoc.x) + NumCast(imageRootDoc._width) + 20, y: NumCast(imageRootDoc.y), _width: 1000, _height: 1000 }); // addDoc?.(newCollectionRef.current); // await createNewImgDoc(originalImg.current, true); // } console.log('creating another image'); await createNewImgDoc(currImg.current, false); // CollectionDockingView.AddSplit(Doc.MakeCopy(DocCast(Doc.UserDoc().emptyPane)), OpenWhereMod.right); // CollectionDockingView.AddSplit(newCollection,OpenWhere.inParent) // mind mapping // this.props.addDocTab(); } catch (err) { console.log(err); } setLoading(false); }; const handleViewClose = () => { // if (newCollectionRef.current) { // CollectionDockingView.AddSplit(newCollectionRef.current, OpenWhereMod.right); // } MainView.Instance.setImageEditorOpen(false); MainView.Instance.setImageEditorSource(''); setEdits([]); }; return (

Generative Fill

{/* Main canvas for editing */}
{/* Icons */}
{ if (fileRef.current) { fileRef.current.click(); } }}> { setBrushStyle(BrushStyle.ADD); }}> {/* Undo and Redo */} { e.stopPropagation(); handleUndo(); }} onPointerUp={e => { e.stopPropagation(); }}> { e.stopPropagation(); handleRedo(); }} onPointerUp={e => { e.stopPropagation(); }}>
{/* Edits thumbnails*/}
{edits.map((edit, i) => ( { const img = new Image(); img.src = edit; ImageUtility.drawImgToCanvas(img, canvasRef); currImg.current = img; onSave(); }} /> ))} {/* Original img thumbnail */} {edits.length > 0 && (
{ if (!originalImg.current) return; const img = new Image(); img.src = originalImg.current.src; ImageUtility.drawImgToCanvas(img, canvasRef); currImg.current = img; }} />
)}
setInput(e.target.value)} disabled={isBrushing} type="text" label="Prompt" placeholder="Prompt..." InputLabelProps={{ style: { fontSize: '1.5rem' } }} inputProps={{ style: { fontSize: '1.5rem' } }} sx={{ backgroundColor: '#ffffff', position: 'absolute', bottom: '1rem', transform: 'translateX(calc(50vw - 50%))', width: 'calc(100vw - 4rem)', scale: 1.2, }} />
); }; export default GenerativeFill;