/* eslint-disable jsx-a11y/label-has-associated-control */ /* eslint-disable jsx-a11y/no-noninteractive-element-interactions */ /* eslint-disable jsx-a11y/img-redundant-alt */ /* eslint-disable jsx-a11y/click-events-have-key-events */ /* eslint-disable react/function-component-definition */ import { Checkbox, FormControlLabel, Slider, TextField } from '@mui/material'; import { IconButton } from 'browndash-components'; import * as React from 'react'; import { useEffect, useRef, useState } from 'react'; import { CgClose } from 'react-icons/cg'; import { IoMdRedo, IoMdUndo } from 'react-icons/io'; import { ClientUtils } from '../../../../ClientUtils'; import { Doc, DocListCast } from '../../../../fields/Doc'; import { List } from '../../../../fields/List'; import { NumCast } from '../../../../fields/Types'; import { Networking } from '../../../Network'; import { DocUtils } from '../../../documents/DocUtils'; import { Docs } from '../../../documents/Documents'; import { CollectionDockingView } from '../../collections/CollectionDockingView'; import { CollectionFreeFormView } from '../../collections/collectionFreeForm'; import { ImageEditorData } from '../ImageBox'; import { OpenWhereMod } from '../OpenWhere'; import './GenerativeFill.scss'; import Buttons from './GenerativeFillButtons'; import { BrushHandler } from './generativeFillUtils/BrushHandler'; import { APISuccess, ImageUtility } from './generativeFillUtils/ImageHandler'; import { PointerHandler } from './generativeFillUtils/PointerHandler'; import { activeColor, canvasSize, eraserColor, freeformRenderSize, newCollectionSize, offsetDistanceY, offsetX } from './generativeFillUtils/generativeFillConstants'; import { CursorData, ImageDimensions, Point } from './generativeFillUtils/generativeFillInterfaces'; import { DocumentView } from '../DocumentView'; // enum BrushStyle { // ADD, // SUBTRACT, // MARQUEE, // } interface GenerativeFillProps { imageEditorOpen: boolean; imageEditorSource: string; imageRootDoc: Doc | undefined; addDoc: ((doc: Doc | Doc[], annotationKey?: string) => boolean) | undefined; } // Added field on image doc: gen_fill_children: List of children Docs const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addDoc }: GenerativeFillProps) => { const canvasRef = useRef(null); const canvasBackgroundRef = useRef(null); const drawingAreaRef = useRef(null); const [cursorData, setCursorData] = useState({ x: 0, y: 0, width: 150, }); const [isBrushing, setIsBrushing] = useState(false); const [canvasScale, setCanvasScale] = useState(0.5); // format: array of [image source, corresponding image Doc] const [edits, setEdits] = useState<(string | Doc)[][]>([]); const [edited, setEdited] = useState(false); // const [brushStyle] = useState(BrushStyle.ADD); const [input, setInput] = useState(''); const [loading, setLoading] = useState(false); const [canvasDims, setCanvasDims] = useState({ width: canvasSize, height: canvasSize, }); // whether to create a new collection or not const [isNewCollection, setIsNewCollection] = useState(true); // the current image in the main canvas const currImg = useRef(null); // the unedited version of each generation (parent) const originalImg = useRef(null); const originalDoc = useRef(null); // stores history of data urls const undoStack = useRef([]); // stores redo stack const redoStack = useRef([]); // references to keep track of tree structure const newCollectionRef = useRef(null); const parentDoc = useRef(null); const childrenDocs = useRef([]); // Undo and Redo const handleUndo = () => { const ctx = ImageUtility.getCanvasContext(canvasRef); if (!ctx || !currImg.current || !canvasRef.current) return; const target = undoStack.current[undoStack.current.length - 1]; if (!target) { ImageUtility.drawImgToCanvas(currImg.current, canvasRef, canvasDims.width, canvasDims.height); } else { redoStack.current = [...redoStack.current, canvasRef.current.toDataURL()]; const img = new Image(); img.src = target; ImageUtility.drawImgToCanvas(img, canvasRef, canvasDims.width, canvasDims.height); undoStack.current = undoStack.current.slice(0, -1); } }; const handleRedo = () => { const ctx = ImageUtility.getCanvasContext(canvasRef); if (!ctx || !currImg.current || !canvasRef.current) return; const target = redoStack.current[redoStack.current.length - 1]; if (target) { undoStack.current = [...undoStack.current, canvasRef.current?.toDataURL()]; const img = new Image(); img.src = target; ImageUtility.drawImgToCanvas(img, canvasRef, canvasDims.width, canvasDims.height); redoStack.current = redoStack.current.slice(0, -1); } }; // resets any erase strokes const handleReset = () => { if (!canvasRef.current || !currImg.current) return; const ctx = ImageUtility.getCanvasContext(canvasRef); if (!ctx) return; ctx.clearRect(0, 0, canvasSize, canvasSize); undoStack.current = []; redoStack.current = []; ImageUtility.drawImgToCanvas(currImg.current, canvasRef, canvasDims.width, canvasDims.height); }; // initiate brushing const handlePointerDown = (e: React.PointerEvent) => { const canvas = canvasRef.current; if (!canvas) return; const ctx = ImageUtility.getCanvasContext(canvasRef); if (!ctx) return; undoStack.current = [...undoStack.current, canvasRef.current.toDataURL()]; redoStack.current = []; setIsBrushing(true); const { x, y } = PointerHandler.getPointRelativeToElement(canvas, e, canvasScale); BrushHandler.brushCircleOverlay(x, y, cursorData.width / 2 / canvasScale, ctx, eraserColor /* , brushStyle === BrushStyle.SUBTRACT */); }; // stop brushing, push to undo stack const handlePointerUp = () => { const ctx = ImageUtility.getCanvasContext(canvasBackgroundRef); if (!ctx) return; if (!isBrushing) return; setIsBrushing(false); }; // handles brushing on pointer movement useEffect(() => { if (!isBrushing) return undefined; const canvas = canvasRef.current; if (!canvas) return undefined; const ctx = ImageUtility.getCanvasContext(canvasRef); if (!ctx) return undefined; const handlePointerMove = (e: PointerEvent) => { const currPoint = PointerHandler.getPointRelativeToElement(canvas, e, canvasScale); const lastPoint: Point = { x: currPoint.x - e.movementX / canvasScale, y: currPoint.y - e.movementY / canvasScale, }; BrushHandler.createBrushPathOverlay(lastPoint, currPoint, cursorData.width / 2 / canvasScale, ctx, eraserColor /* , brushStyle === BrushStyle.SUBTRACT */); }; drawingAreaRef.current?.addEventListener('pointermove', handlePointerMove); return () => { drawingAreaRef.current?.removeEventListener('pointermove', handlePointerMove); }; }, [isBrushing]); // first load useEffect(() => { const loadInitial = async () => { if (!imageEditorSource || imageEditorSource === '') return; const img = new Image(); const res = await ImageUtility.urlToBase64(imageEditorSource); if (!res) return; img.src = `data:image/png;base64,${res}`; img.onload = () => { currImg.current = img; originalImg.current = img; const imgWidth = img.naturalWidth; const imgHeight = img.naturalHeight; const scale = Math.min(canvasSize / imgWidth, canvasSize / imgHeight); const width = imgWidth * scale; const height = imgHeight * scale; setCanvasDims({ width, height }); }; }; loadInitial(); // cleanup return () => { setInput(''); setEdited(false); newCollectionRef.current = null; parentDoc.current = null; childrenDocs.current = []; currImg.current = null; originalImg.current = null; originalDoc.current = null; undoStack.current = []; redoStack.current = []; ImageUtility.clearCanvas(canvasRef); }; }, [canvasRef, imageEditorSource]); // once the appropriate dimensions are set, draw the image to the canvas useEffect(() => { if (!currImg.current) return; ImageUtility.drawImgToCanvas(currImg.current, canvasRef, canvasDims.width, canvasDims.height); }, [canvasDims]); // handles brush sizing useEffect(() => { const handleKeyPress = (e: KeyboardEvent) => { if (e.key === 'ArrowUp') { e.preventDefault(); e.stopPropagation(); setCursorData(data => ({ ...data, width: data.width + 5 })); } else if (e.key === 'ArrowDown') { e.preventDefault(); e.stopPropagation(); setCursorData(data => (data.width >= 20 ? { ...data, width: data.width - 5 } : data)); } }; window.addEventListener('keydown', handleKeyPress); return () => window.removeEventListener('keydown', handleKeyPress); }, []); // handle pinch zoom useEffect(() => { const handlePinch = (e: WheelEvent) => { e.preventDefault(); e.stopPropagation(); const delta = e.deltaY; const scaleFactor = delta > 0 ? 0.98 : 1.02; setCanvasScale(prevScale => prevScale * scaleFactor); }; drawingAreaRef.current?.addEventListener('wheel', handlePinch, { passive: false, }); return () => drawingAreaRef.current?.removeEventListener('wheel', handlePinch); }, [drawingAreaRef]); // updates the current position of the cursor const updateCursorData = (e: React.PointerEvent) => { const drawingArea = drawingAreaRef.current; if (!drawingArea) return; const { x, y } = PointerHandler.getPointRelativeToElement(drawingArea, e, 1); setCursorData(data => ({ ...data, x, y, })); }; // Get AI Edit const getEdit = async () => { const img = currImg.current; if (!img) return; const canvas = canvasRef.current; if (!canvas) return; const ctx = ImageUtility.getCanvasContext(canvasRef); if (!ctx) return; setLoading(true); setEdited(true); try { const canvasOriginalImg = ImageUtility.getCanvasImg(img); if (!canvasOriginalImg) return; const canvasMask = ImageUtility.getCanvasMask(canvas, canvasOriginalImg); if (!canvasMask) return; const maskBlob = await ImageUtility.canvasToBlob(canvasMask); const imgBlob = await ImageUtility.canvasToBlob(canvasOriginalImg); const res = await ImageUtility.getEdit(imgBlob, maskBlob, input !== '' ? input + ' in the same style' : 'Fill in the image in the same style', 2); // const res = await ImageUtility.mockGetEdit(img.src); // create first image if (!newCollectionRef.current) { if (!isNewCollection && imageRootDoc) { // if the parent hasn't been set yet if (!parentDoc.current) parentDoc.current = imageRootDoc; } else { if (!(originalImg.current && imageRootDoc)) return; // create new collection and add it to the view newCollectionRef.current = Docs.Create.FreeformDocument([], { x: NumCast(imageRootDoc.x) + NumCast(imageRootDoc._width) + offsetX, y: NumCast(imageRootDoc.y), _width: newCollectionSize, _height: newCollectionSize, title: 'Image edit collection', }); DocUtils.MakeLink(imageRootDoc, newCollectionRef.current, { link_relationship: 'Image Edit Version History' }); // opening new tab CollectionDockingView.AddSplit(newCollectionRef.current, OpenWhereMod.right); // add the doc to the main freeform // eslint-disable-next-line no-use-before-define await createNewImgDoc(originalImg.current, true); } } else { childrenDocs.current = []; } originalImg.current = currImg.current; originalDoc.current = parentDoc.current; const { urls } = res as APISuccess; if (res.status !== 'error') { const imgUrls = await Promise.all(urls.map(url => ImageUtility.convertImgToCanvasUrl(url, canvasDims.width, canvasDims.height))); const imgRes = await Promise.all( imgUrls.map(async url => { // eslint-disable-next-line no-use-before-define const saveRes = await onSave(url); return [url, saveRes as Doc]; }) ); setEdits(imgRes); const image = new Image(); // eslint-disable-next-line prefer-destructuring image.src = imgUrls[0]; ImageUtility.drawImgToCanvas(image, canvasRef, canvasDims.width, canvasDims.height); currImg.current = image; parentDoc.current = imgRes[0][1] as Doc; } } catch (err) { console.log(err); } setLoading(false); }; // adjusts all the img positions to be aligned const adjustImgPositions = () => { if (!parentDoc.current) return; const startY = NumCast(parentDoc.current.y); const children = DocListCast(parentDoc.current.gen_fill_children); const len = children.length; const initialYPositions: number[] = []; for (let i = 0; i < len; i++) { initialYPositions.push(startY + i * offsetDistanceY); } children.forEach((doc, i) => { if (len % 2 === 1) { doc.y = initialYPositions[i] - Math.floor(len / 2) * offsetDistanceY; } else { doc.y = initialYPositions[i] - (len / 2 - 1 / 2) * offsetDistanceY; } }); }; // creates a new image document and returns its reference const createNewImgDoc = async (img: HTMLImageElement, firstDoc: boolean): Promise => { if (!imageRootDoc) return undefined; const { src } = img; const [result] = await Networking.PostToServer('/uploadRemoteImage', { sources: [src] }); const source = ClientUtils.prepend(result.accessPaths.agnostic.client); if (firstDoc) { const x = 0; const initialY = 0; const newImg = Docs.Create.ImageDocument(source, { x: x, y: initialY, _height: freeformRenderSize, _width: freeformRenderSize, data_nativeWidth: result.nativeWidth, data_nativeHeight: result.nativeHeight, }); if (isNewCollection && newCollectionRef.current) { Doc.AddDocToList(newCollectionRef.current, undefined, newImg); } else { addDoc?.(newImg); } parentDoc.current = newImg; return newImg; } if (!parentDoc.current) return undefined; const x = NumCast(parentDoc.current.x) + freeformRenderSize + offsetX; const initialY = 0; const newImg = Docs.Create.ImageDocument(source, { x: x, y: initialY, _height: freeformRenderSize, _width: freeformRenderSize, data_nativeWidth: result.nativeWidth, data_nativeHeight: result.nativeHeight, }); const parentList = DocListCast(parentDoc.current.gen_fill_children); if (parentList.length > 0) { parentList.push(newImg); parentDoc.current.gen_fill_children = new List(parentList); } else { parentDoc.current.gen_fill_children = new List([newImg]); } DocUtils.MakeLink(parentDoc.current, newImg, { link_relationship: `Image edit; Prompt: ${input}` }); adjustImgPositions(); if (isNewCollection && newCollectionRef.current) { Doc.AddDocToList(newCollectionRef.current, undefined, newImg); } else { addDoc?.(newImg); } return newImg; }; // Saves an image to the collection const onSave = async (src: string) => { const img = new Image(); img.src = src; if (!currImg.current || !originalImg.current || !imageRootDoc) return undefined; try { const res = await createNewImgDoc(img, false); return res; } catch (err) { console.log(err); } return undefined; }; // Closes the editor view const handleViewClose = () => { ImageEditorData.Open = false; ImageEditorData.Source = ''; if (newCollectionRef.current) { DocumentView.addViewRenderedCb(newCollectionRef.current, dv => (dv.ComponentView as CollectionFreeFormView)?.fitContentOnce()); } setEdits([]); }; return (

Image Editor

{ setIsNewCollection(prev => !prev); }} /> } label="Create New Collection" labelPlacement="end" sx={{ whiteSpace: 'nowrap' }} /> } onClick={handleViewClose} />
{/* Main canvas for editing */}
{/* Icons */}
{/* Undo and Redo */} { e.stopPropagation(); handleUndo(); }} onPointerUp={e => { e.stopPropagation(); }} color={activeColor} tooltip="Undo" icon={} /> { e.stopPropagation(); handleRedo(); }} onPointerUp={e => { e.stopPropagation(); }} color={activeColor} tooltip="Redo" icon={} />
e.stopPropagation()} style={{ height: 225, width: '100%', display: 'flex', justifyContent: 'center', cursor: 'pointer' }}> { setCursorData(prev => ({ ...prev, width: val as number })); }} />
{/* Edits thumbnails */}
{edits.map((edit, i) => ( image edits { const img = new Image(); img.src = edit[0] as string; ImageUtility.drawImgToCanvas(img, canvasRef, canvasDims.width, canvasDims.height); currImg.current = img; parentDoc.current = edit[1] as Doc; }} /> ))} {/* Original img thumbnail */} {edits.length > 0 && (
image stuff { if (!originalImg.current) return; const img = new Image(); img.src = originalImg.current.src; ImageUtility.drawImgToCanvas(img, canvasRef, canvasDims.width, canvasDims.height); currImg.current = img; parentDoc.current = originalDoc.current; }} />
)}
setInput(e.target.value)} disabled={isBrushing} type="text" label="Prompt" placeholder="Prompt..." InputLabelProps={{ style: { fontSize: '16px' } }} inputProps={{ style: { fontSize: '16px' } }} sx={{ backgroundColor: '#ffffff', position: 'absolute', bottom: '16px', transform: 'translateX(calc(50vw - 50%))', width: 'calc(100vw - 64px)', }} />
); }; export default GenerativeFill;