diff options
Diffstat (limited to 'src/client/views/nodes/generativeFill')
10 files changed, 1091 insertions, 0 deletions
diff --git a/src/client/views/nodes/generativeFill/GenerativeFill.scss b/src/client/views/nodes/generativeFill/GenerativeFill.scss new file mode 100644 index 000000000..c2669a950 --- /dev/null +++ b/src/client/views/nodes/generativeFill/GenerativeFill.scss @@ -0,0 +1,97 @@ +$navHeight: 5rem; +$canvasSize: 1024px; +$scale: 0.5; + +.generativeFillContainer { + position: absolute; + top: 0; + left: 0; + z-index: 9999; + height: 100vh; + width: 100vw; + display: flex; + flex-direction: column; + overflow: hidden; + + .generativeFillControls { + flex-shrink: 0; + height: $navHeight; + color: #000000; + background-color: #ffffff; + z-index: 999; + width: 100%; + display: flex; + gap: 3rem; + justify-content: space-between; + align-items: center; + border-bottom: 1px solid #c7cdd0; + padding: 0 2rem; + + h1 { + font-size: 1.5rem; + } + } + + .drawingArea { + cursor: none; + touch-action: none; + position: relative; + flex-grow: 1; + display: flex; + justify-content: center; + align-items: center; + width: 100%; + background-color: #f0f4f6; + + canvas { + display: block; + position: absolute; + transform-origin: 50% 50%; + } + + .pointer { + pointer-events: none; + position: absolute; + border-radius: 50%; + width: 50px; + height: 50px; + border: 1px solid #ffffff; + transform: translate(-50%, -50%); + display: flex; + justify-content: center; + align-items: center; + + .innerPointer { + width: 100%; + height: 100%; + border: 1px solid #000000; + border-radius: 50%; + } + } + + .iconContainer { + position: absolute; + top: 2rem; + left: 2rem; + display: flex; + flex-direction: column; + gap: 2rem; + } + + .editsBox { + position: absolute; + top: 2rem; + right: 2rem; + display: flex; + flex-direction: column; + gap: 1rem; + + img { + transition: all 0.2s ease-in-out; + &:hover { + opacity: 0.8; + } + } + } + } +} diff --git a/src/client/views/nodes/generativeFill/GenerativeFill.tsx b/src/client/views/nodes/generativeFill/GenerativeFill.tsx new file mode 100644 index 000000000..7230aa9b1 --- /dev/null +++ b/src/client/views/nodes/generativeFill/GenerativeFill.tsx @@ -0,0 +1,583 @@ +import './GenerativeFill.scss'; +import React = require('react'); +import { useEffect, useRef, useState } from 'react'; +import { APISuccess, ImageUtility } from './generativeFillUtils/ImageHandler'; +import { BrushHandler } from './generativeFillUtils/BrushHandler'; +import { IconButton } from 'browndash-components'; +import { Checkbox, FormControlLabel, Slider, TextField } from '@mui/material'; +import { CursorData, ImageDimensions, Point } from './generativeFillUtils/generativeFillInterfaces'; +import { activeColor, canvasSize, eraserColor, freeformRenderSize, newCollectionSize, offsetDistanceY, offsetX } from './generativeFillUtils/generativeFillConstants'; +import { PointerHandler } from './generativeFillUtils/PointerHandler'; +import { IoMdUndo, IoMdRedo } from 'react-icons/io'; +import { MainView } from '../../MainView'; +import { Doc, DocListCast } from '../../../../fields/Doc'; +import { Networking } from '../../../Network'; +import { Utils } from '../../../../Utils'; +import { DocUtils, Docs } from '../../../documents/Documents'; +import { NumCast } from '../../../../fields/Types'; +import { CollectionDockingView } from '../../collections/CollectionDockingView'; +import { OpenWhereMod } from '../DocumentView'; +import Buttons from './GenerativeFillButtons'; +import { List } from '../../../../fields/List'; +import { CgClose } from 'react-icons/cg'; + +enum BrushStyle { + ADD, + SUBTRACT, + MARQUEE, +} + +interface GenerativeFillProps { + imageEditorOpen: boolean; + imageEditorSource: string; + imageRootDoc: Doc | undefined; + addDoc: ((doc: Doc | Doc[], annotationKey?: string) => boolean) | undefined; +} + +// Added field on image doc: gen_fill_children: List of children Docs + +const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addDoc }: GenerativeFillProps) => { + const canvasRef = useRef<HTMLCanvasElement>(null); + const canvasBackgroundRef = useRef<HTMLCanvasElement>(null); + const drawingAreaRef = useRef<HTMLDivElement>(null); + const [cursorData, setCursorData] = useState<CursorData>({ + x: 0, + y: 0, + width: 150, + }); + const [isBrushing, setIsBrushing] = useState(false); + const [canvasScale, setCanvasScale] = useState(0.5); + // format: array of [image source, corresponding image Doc] + const [edits, setEdits] = useState<(string | Doc)[][]>([]); + const [edited, setEdited] = useState(false); + const [brushStyle, setBrushStyle] = useState<BrushStyle>(BrushStyle.ADD); + const [input, setInput] = useState(''); + const [loading, setLoading] = useState(false); + const [canvasDims, setCanvasDims] = useState<ImageDimensions>({ + width: canvasSize, + height: canvasSize, + }); + // whether to create a new collection or not + const [isNewCollection, setIsNewCollection] = useState(true); + // the current image in the main canvas + const currImg = useRef<HTMLImageElement | null>(null); + // the unedited version of each generation (parent) + const originalImg = useRef<HTMLImageElement | null>(null); + const originalDoc = useRef<Doc | null>(null); + // stores history of data urls + const undoStack = useRef<string[]>([]); + // stores redo stack + const redoStack = useRef<string[]>([]); + + // references to keep track of tree structure + const newCollectionRef = useRef<Doc | null>(null); + const parentDoc = useRef<Doc | null>(null); + const childrenDocs = useRef<Doc[]>([]); + + // Undo and Redo + const handleUndo = () => { + const ctx = ImageUtility.getCanvasContext(canvasRef); + if (!ctx || !currImg.current || !canvasRef.current) return; + + const target = undoStack.current[undoStack.current.length - 1]; + if (!target) { + ImageUtility.drawImgToCanvas(currImg.current, canvasRef, canvasDims.width, canvasDims.height); + } else { + redoStack.current = [...redoStack.current, canvasRef.current.toDataURL()]; + const img = new Image(); + img.src = target; + ImageUtility.drawImgToCanvas(img, canvasRef, canvasDims.width, canvasDims.height); + undoStack.current = undoStack.current.slice(0, -1); + } + }; + + const handleRedo = () => { + const ctx = ImageUtility.getCanvasContext(canvasRef); + if (!ctx || !currImg.current || !canvasRef.current) return; + + const target = redoStack.current[redoStack.current.length - 1]; + if (!target) { + } else { + undoStack.current = [...undoStack.current, canvasRef.current?.toDataURL()]; + const img = new Image(); + img.src = target; + ImageUtility.drawImgToCanvas(img, canvasRef, canvasDims.width, canvasDims.height); + redoStack.current = redoStack.current.slice(0, -1); + } + }; + + // resets any erase strokes + const handleReset = () => { + if (!canvasRef.current || !currImg.current) return; + const ctx = ImageUtility.getCanvasContext(canvasRef); + if (!ctx) return; + ctx.clearRect(0, 0, canvasSize, canvasSize); + undoStack.current = []; + redoStack.current = []; + ImageUtility.drawImgToCanvas(currImg.current, canvasRef, canvasDims.width, canvasDims.height); + }; + + // initiate brushing + const handlePointerDown = (e: React.PointerEvent) => { + const canvas = canvasRef.current; + if (!canvas) return; + const ctx = ImageUtility.getCanvasContext(canvasRef); + if (!ctx) return; + + undoStack.current = [...undoStack.current, canvasRef.current.toDataURL()]; + redoStack.current = []; + + setIsBrushing(true); + const { x, y } = PointerHandler.getPointRelativeToElement(canvas, e, canvasScale); + BrushHandler.brushCircleOverlay(x, y, cursorData.width / 2 / canvasScale, ctx, eraserColor, brushStyle === BrushStyle.SUBTRACT); + }; + + // stop brushing, push to undo stack + const handlePointerUp = (e: React.PointerEvent) => { + const ctx = ImageUtility.getCanvasContext(canvasBackgroundRef); + if (!ctx) return; + if (!isBrushing) return; + setIsBrushing(false); + }; + + // handles brushing on pointer movement + useEffect(() => { + if (!isBrushing) return; + const canvas = canvasRef.current; + if (!canvas) return; + const ctx = ImageUtility.getCanvasContext(canvasRef); + if (!ctx) return; + + const handlePointerMove = (e: PointerEvent) => { + const currPoint = PointerHandler.getPointRelativeToElement(canvas, e, canvasScale); + const lastPoint: Point = { + x: currPoint.x - e.movementX / canvasScale, + y: currPoint.y - e.movementY / canvasScale, + }; + BrushHandler.createBrushPathOverlay(lastPoint, currPoint, cursorData.width / 2 / canvasScale, ctx, eraserColor, brushStyle === BrushStyle.SUBTRACT); + }; + + drawingAreaRef.current?.addEventListener('pointermove', handlePointerMove); + return () => { + drawingAreaRef.current?.removeEventListener('pointermove', handlePointerMove); + }; + }, [isBrushing]); + + // first load + useEffect(() => { + if (!imageEditorSource || imageEditorSource === '') return; + const img = new Image(); + img.src = imageEditorSource; + currImg.current = img; + originalImg.current = img; + img.onload = () => { + const imgWidth = img.naturalWidth; + const imgHeight = img.naturalHeight; + const scale = Math.min(canvasSize / imgWidth, canvasSize / imgHeight); + const width = imgWidth * scale; + const height = imgHeight * scale; + setCanvasDims({ width, height }); + }; + + // cleanup + return () => { + setInput(''); + setEdited(false); + newCollectionRef.current = null; + parentDoc.current = null; + childrenDocs.current = []; + currImg.current = null; + originalImg.current = null; + originalDoc.current = null; + undoStack.current = []; + redoStack.current = []; + ImageUtility.clearCanvas(canvasRef); + }; + }, [canvasRef, imageEditorSource]); + + // once the appropriate dimensions are set, draw the image to the canvas + useEffect(() => { + if (!currImg.current) return; + ImageUtility.drawImgToCanvas(currImg.current, canvasRef, canvasDims.width, canvasDims.height); + }, [canvasDims]); + + // handles brush sizing + useEffect(() => { + const handleKeyPress = (e: KeyboardEvent) => { + if (e.key === 'ArrowUp') { + e.preventDefault(); + e.stopPropagation(); + setCursorData(data => ({ ...data, width: data.width + 5 })); + } else if (e.key === 'ArrowDown') { + e.preventDefault(); + e.stopPropagation(); + setCursorData(data => (data.width >= 20 ? { ...data, width: data.width - 5 } : data)); + } + }; + window.addEventListener('keydown', handleKeyPress); + return () => window.removeEventListener('keydown', handleKeyPress); + }, []); + + // handle pinch zoom + useEffect(() => { + const handlePinch = (e: WheelEvent) => { + e.preventDefault(); + e.stopPropagation(); + const delta = e.deltaY; + const scaleFactor = delta > 0 ? 0.98 : 1.02; + setCanvasScale(prevScale => prevScale * scaleFactor); + }; + + drawingAreaRef.current?.addEventListener('wheel', handlePinch, { + passive: false, + }); + return () => drawingAreaRef.current?.removeEventListener('wheel', handlePinch); + }, [drawingAreaRef]); + + // updates the current position of the cursor + const updateCursorData = (e: React.PointerEvent) => { + const drawingArea = drawingAreaRef.current; + if (!drawingArea) return; + const { x, y } = PointerHandler.getPointRelativeToElement(drawingArea, e, 1); + setCursorData(data => ({ + ...data, + x, + y, + })); + }; + + // Get AI Edit + const getEdit = async () => { + const img = currImg.current; + if (!img) return; + const canvas = canvasRef.current; + if (!canvas) return; + const ctx = ImageUtility.getCanvasContext(canvasRef); + if (!ctx) return; + setLoading(true); + setEdited(true); + try { + const canvasOriginalImg = ImageUtility.getCanvasImg(img); + if (!canvasOriginalImg) return; + const canvasMask = ImageUtility.getCanvasMask(canvas, canvasOriginalImg); + if (!canvasMask) return; + const maskBlob = await ImageUtility.canvasToBlob(canvasMask); + const imgBlob = await ImageUtility.canvasToBlob(canvasOriginalImg); + const res = await ImageUtility.getEdit(imgBlob, maskBlob, input !== '' ? input + ' in the same style' : 'Fill in the image in the same style', 2); + // const res = await ImageUtility.mockGetEdit(img.src); + + // create first image + if (!newCollectionRef.current) { + if (!isNewCollection && imageRootDoc) { + // if the parent hasn't been set yet + if (!parentDoc.current) parentDoc.current = imageRootDoc; + } else { + if (!(originalImg.current && imageRootDoc)) return; + // create new collection and add it to the view + newCollectionRef.current = Docs.Create.FreeformDocument([], { + x: NumCast(imageRootDoc.x) + NumCast(imageRootDoc._width) + offsetX, + y: NumCast(imageRootDoc.y), + _width: newCollectionSize, + _height: newCollectionSize, + title: 'Image edit collection', + }); + DocUtils.MakeLink(imageRootDoc, newCollectionRef.current, { link_relationship: 'Image Edit Version History', link_displayLine: false }); + + // opening new tab + CollectionDockingView.AddSplit(newCollectionRef.current, OpenWhereMod.right); + + // add the doc to the main freeform + await createNewImgDoc(originalImg.current, true); + } + } else { + childrenDocs.current = []; + } + + originalImg.current = currImg.current; + originalDoc.current = parentDoc.current; + const { urls } = res as APISuccess; + const imgUrls = await Promise.all(urls.map(url => ImageUtility.convertImgToCanvasUrl(url, canvasDims.width, canvasDims.height))); + const imgRes = await Promise.all( + imgUrls.map(async url => { + const saveRes = await onSave(url); + return [url, saveRes as Doc]; + }) + ); + setEdits(imgRes); + const image = new Image(); + image.src = imgUrls[0]; + ImageUtility.drawImgToCanvas(image, canvasRef, canvasDims.width, canvasDims.height); + currImg.current = image; + parentDoc.current = imgRes[0][1] as Doc; + } catch (err) { + console.log(err); + } + setLoading(false); + }; + + // adjusts all the img positions to be aligned + const adjustImgPositions = () => { + if (!parentDoc.current) return; + const startY = NumCast(parentDoc.current.y); + const children = DocListCast(parentDoc.current.gen_fill_children); + const len = children.length; + let initialYPositions: number[] = []; + for (let i = 0; i < len; i++) { + initialYPositions.push(startY + i * offsetDistanceY); + } + children.forEach((doc, i) => { + if (len % 2 === 1) { + doc.y = initialYPositions[i] - Math.floor(len / 2) * offsetDistanceY; + } else { + doc.y = initialYPositions[i] - (len / 2 - 1 / 2) * offsetDistanceY; + } + }); + }; + + // creates a new image document and returns its reference + const createNewImgDoc = async (img: HTMLImageElement, firstDoc: boolean): Promise<Doc | undefined> => { + if (!imageRootDoc) return; + const src = img.src; + const [result] = await Networking.PostToServer('/uploadRemoteImage', { sources: [src] }); + const source = Utils.prepend(result.accessPaths.agnostic.client); + + if (firstDoc) { + const x = 0; + const initialY = 0; + const newImg = Docs.Create.ImageDocument(source, { + x: x, + y: initialY, + _height: freeformRenderSize, + _width: freeformRenderSize, + data_nativeWidth: result.nativeWidth, + data_nativeHeight: result.nativeHeight, + }); + if (isNewCollection && newCollectionRef.current) { + Doc.AddDocToList(newCollectionRef.current, undefined, newImg); + } else { + addDoc?.(newImg); + } + parentDoc.current = newImg; + return newImg; + } else { + if (!parentDoc.current) return; + const x = NumCast(parentDoc.current.x) + freeformRenderSize + offsetX; + const initialY = 0; + + const newImg = Docs.Create.ImageDocument(source, { + x: x, + y: initialY, + _height: freeformRenderSize, + _width: freeformRenderSize, + data_nativeWidth: result.nativeWidth, + data_nativeHeight: result.nativeHeight, + }); + + const parentList = DocListCast(parentDoc.current.gen_fill_children); + if (parentList.length > 0) { + parentList.push(newImg); + parentDoc.current.gen_fill_children = new List<Doc>(parentList); + } else { + parentDoc.current.gen_fill_children = new List<Doc>([newImg]); + } + + DocUtils.MakeLink(parentDoc.current, newImg, { link_relationship: `Image edit; Prompt: ${input}`, link_displayLine: true }); + adjustImgPositions(); + + if (isNewCollection && newCollectionRef.current) { + Doc.AddDocToList(newCollectionRef.current, undefined, newImg); + } else { + addDoc?.(newImg); + } + return newImg; + } + }; + + // Saves an image to the collection + const onSave = async (src: string) => { + const img = new Image(); + img.src = src; + if (!currImg.current || !originalImg.current || !imageRootDoc) return; + try { + const res = await createNewImgDoc(img, false); + return res; + } catch (err) { + console.log(err); + } + }; + + // Closes the editor view + const handleViewClose = () => { + MainView.Instance.setImageEditorOpen(false); + MainView.Instance.setImageEditorSource(''); + if (newCollectionRef.current) { + newCollectionRef.current.fitContentOnce = true; + } + setEdits([]); + }; + + return ( + <div className="generativeFillContainer" style={{ display: imageEditorOpen ? 'flex' : 'none' }}> + <div className="generativeFillControls"> + <h1>Image Editor</h1> + <div style={{ display: 'flex', alignItems: 'center', gap: '1.5rem' }}> + <FormControlLabel + control={ + <Checkbox + // disable once edited has been clicked (doesn't make sense to change after first edit) + disabled={edited} + checked={isNewCollection} + onChange={e => { + setIsNewCollection(prev => !prev); + }} + /> + } + label={'Create New Collection'} + labelPlacement="end" + sx={{ whiteSpace: 'nowrap' }} + /> + <Buttons getEdit={getEdit} loading={loading} onReset={handleReset} /> + <IconButton color={activeColor} tooltip="close" icon={<CgClose size={'16px'} />} onClick={handleViewClose} /> + </div> + </div> + {/* Main canvas for editing */} + <div + className="drawingArea" // this only works if pointerevents: none is set on the custom pointer + ref={drawingAreaRef} + onPointerOver={updateCursorData} + onPointerMove={updateCursorData} + onPointerDown={handlePointerDown} + onPointerUp={handlePointerUp}> + <canvas ref={canvasRef} width={canvasDims.width} height={canvasDims.height} style={{ transform: `scale(${canvasScale})` }} /> + <canvas ref={canvasBackgroundRef} width={canvasDims.width} height={canvasDims.height} style={{ transform: `scale(${canvasScale})` }} /> + <div + className="pointer" + style={{ + left: cursorData.x, + top: cursorData.y, + width: cursorData.width, + height: cursorData.width, + }}> + <div className="innerPointer"></div> + </div> + {/* Icons */} + <div className="iconContainer"> + {/* Undo and Redo */} + <IconButton + style={{ cursor: 'pointer' }} + onPointerDown={e => { + e.stopPropagation(); + handleUndo(); + }} + onPointerUp={e => { + e.stopPropagation(); + }} + color={activeColor} + tooltip="Undo" + icon={<IoMdUndo />} + /> + <IconButton + style={{ cursor: 'pointer' }} + onPointerDown={e => { + e.stopPropagation(); + handleRedo(); + }} + onPointerUp={e => { + e.stopPropagation(); + }} + color={activeColor} + tooltip="Redo" + icon={<IoMdRedo />} + /> + <div onPointerDown={e => e.stopPropagation()} style={{ height: 225, width: '100%', display: 'flex', justifyContent: 'center', cursor: 'pointer' }}> + <Slider + sx={{ + '& input[type="range"]': { + WebkitAppearance: 'slider-vertical', + }, + }} + orientation="vertical" + min={25} + max={500} + defaultValue={150} + size="small" + valueLabelDisplay="auto" + onChange={(e, val) => { + setCursorData(prev => ({ ...prev, width: val as number })); + }} + /> + </div> + </div> + {/* Edits thumbnails*/} + <div className="editsBox"> + {edits.map((edit, i) => ( + <img + key={i} + width={75} + src={edit[0] as string} + style={{ cursor: 'pointer' }} + onClick={async () => { + const img = new Image(); + img.src = edit[0] as string; + ImageUtility.drawImgToCanvas(img, canvasRef, canvasDims.width, canvasDims.height); + currImg.current = img; + parentDoc.current = edit[1] as Doc; + }} + /> + ))} + {/* Original img thumbnail */} + {edits.length > 0 && ( + <div style={{ position: 'relative' }}> + <label + style={{ + position: 'absolute', + bottom: 10, + left: 10, + color: '#ffffff', + fontSize: '0.8rem', + letterSpacing: '1px', + textTransform: 'uppercase', + }}> + Original + </label> + <img + width={75} + src={originalImg.current?.src} + style={{ cursor: 'pointer' }} + onClick={() => { + if (!originalImg.current) return; + const img = new Image(); + img.src = originalImg.current.src; + ImageUtility.drawImgToCanvas(img, canvasRef, canvasDims.width, canvasDims.height); + currImg.current = img; + parentDoc.current = originalDoc.current; + }} + /> + </div> + )} + </div> + </div> + <div> + <TextField + value={input} + onChange={e => setInput(e.target.value)} + disabled={isBrushing} + type="text" + label="Prompt" + placeholder="Prompt..." + InputLabelProps={{ style: { fontSize: '16px' } }} + inputProps={{ style: { fontSize: '16px' } }} + sx={{ + backgroundColor: '#ffffff', + position: 'absolute', + bottom: '16px', + transform: 'translateX(calc(50vw - 50%))', + width: 'calc(100vw - 64px)', + }} + /> + </div> + </div> + ); +}; + +export default GenerativeFill; diff --git a/src/client/views/nodes/generativeFill/GenerativeFillButtons.scss b/src/client/views/nodes/generativeFill/GenerativeFillButtons.scss new file mode 100644 index 000000000..0180ef904 --- /dev/null +++ b/src/client/views/nodes/generativeFill/GenerativeFillButtons.scss @@ -0,0 +1,4 @@ +.generativeFillBtnContainer { + display: flex; + gap: 1rem; +} diff --git a/src/client/views/nodes/generativeFill/GenerativeFillButtons.tsx b/src/client/views/nodes/generativeFill/GenerativeFillButtons.tsx new file mode 100644 index 000000000..0dfcebea3 --- /dev/null +++ b/src/client/views/nodes/generativeFill/GenerativeFillButtons.tsx @@ -0,0 +1,42 @@ +import './GenerativeFillButtons.scss'; +import React = require('react'); +import ReactLoading from 'react-loading'; +import { activeColor } from './generativeFillUtils/generativeFillConstants'; +import { Button, Type } from 'browndash-components'; + +interface ButtonContainerProps { + getEdit: () => Promise<void>; + loading: boolean; + onReset: () => void; +} + +const Buttons = ({ loading, getEdit, onReset }: ButtonContainerProps) => { + return ( + <div className="generativeFillBtnContainer"> + <Button text="RESET" type={Type.PRIM} color={activeColor} onClick={onReset} /> + {loading ? ( + <Button + text="GET EDITS" + type={Type.TERT} + color={activeColor} + icon={<ReactLoading type="spin" color={'#ffffff'} width={20} height={20} />} + iconPlacement="right" + onClick={() => { + if (!loading) getEdit(); + }} + /> + ) : ( + <Button + text="GET EDITS" + type={Type.TERT} + color={activeColor} + onClick={() => { + if (!loading) getEdit(); + }} + /> + )} + </div> + ); +}; + +export default Buttons; diff --git a/src/client/views/nodes/generativeFill/generativeFillUtils/BrushHandler.ts b/src/client/views/nodes/generativeFill/generativeFillUtils/BrushHandler.ts new file mode 100644 index 000000000..f4ec70fbc --- /dev/null +++ b/src/client/views/nodes/generativeFill/generativeFillUtils/BrushHandler.ts @@ -0,0 +1,25 @@ +import { GenerativeFillMathHelpers } from './GenerativeFillMathHelpers'; +import { eraserColor } from './generativeFillConstants'; +import { Point } from './generativeFillInterfaces'; + +export class BrushHandler { + static brushCircleOverlay = (x: number, y: number, brushRadius: number, ctx: CanvasRenderingContext2D, fillColor: string, erase: boolean) => { + ctx.globalCompositeOperation = 'destination-out'; + ctx.fillStyle = fillColor; + ctx.shadowColor = eraserColor; + ctx.shadowBlur = 5; + ctx.beginPath(); + ctx.arc(x, y, brushRadius, 0, 2 * Math.PI); + ctx.fill(); + ctx.closePath(); + }; + + static createBrushPathOverlay = (startPoint: Point, endPoint: Point, brushRadius: number, ctx: CanvasRenderingContext2D, fillColor: string, erase: boolean) => { + const dist = GenerativeFillMathHelpers.distanceBetween(startPoint, endPoint); + + for (let i = 0; i < dist; i += 5) { + const s = i / dist; + BrushHandler.brushCircleOverlay(startPoint.x * (1 - s) + endPoint.x * s, startPoint.y * (1 - s) + endPoint.y * s, brushRadius, ctx, fillColor, erase); + } + }; +} diff --git a/src/client/views/nodes/generativeFill/generativeFillUtils/GenerativeFillMathHelpers.ts b/src/client/views/nodes/generativeFill/generativeFillUtils/GenerativeFillMathHelpers.ts new file mode 100644 index 000000000..97e03ff20 --- /dev/null +++ b/src/client/views/nodes/generativeFill/generativeFillUtils/GenerativeFillMathHelpers.ts @@ -0,0 +1,10 @@ +import { Point } from './generativeFillInterfaces'; + +export class GenerativeFillMathHelpers { + static distanceBetween = (p1: Point, p2: Point) => { + return Math.sqrt(Math.pow(p2.x - p1.x, 2) + Math.pow(p2.y - p1.y, 2)); + }; + static angleBetween = (p1: Point, p2: Point) => { + return Math.atan2(p2.x - p1.x, p2.y - p1.y); + }; +} diff --git a/src/client/views/nodes/generativeFill/generativeFillUtils/ImageHandler.ts b/src/client/views/nodes/generativeFill/generativeFillUtils/ImageHandler.ts new file mode 100644 index 000000000..2ede625f6 --- /dev/null +++ b/src/client/views/nodes/generativeFill/generativeFillUtils/ImageHandler.ts @@ -0,0 +1,286 @@ +import { RefObject } from 'react'; +import { bgColor, canvasSize } from './generativeFillConstants'; + +export interface APISuccess { + status: 'success'; + urls: string[]; +} + +export interface APIError { + status: 'error'; + message: string; +} + +export class ImageUtility { + /** + * + * @param canvas Canvas to convert + * @returns Blob of canvas + */ + static canvasToBlob = (canvas: HTMLCanvasElement): Promise<Blob> => { + return new Promise(resolve => { + canvas.toBlob(blob => { + if (blob) { + resolve(blob); + } + }, 'image/png'); + }); + }; + + // given a square api image, get the cropped img + static getCroppedImg = (img: HTMLImageElement, width: number, height: number): HTMLCanvasElement | undefined => { + // Create a new canvas element + const canvas = document.createElement('canvas'); + canvas.width = width; + canvas.height = height; + const ctx = canvas.getContext('2d'); + if (ctx) { + // Clear the canvas + ctx.clearRect(0, 0, canvas.width, canvas.height); + if (width < height) { + // horizontal padding, x offset + const xOffset = (canvasSize - width) / 2; + ctx.drawImage(img, xOffset, 0, canvas.width, canvas.height, 0, 0, canvas.width, canvas.height); + } else { + // vertical padding, y offset + const yOffset = (canvasSize - height) / 2; + ctx.drawImage(img, 0, yOffset, canvas.width, canvas.height, 0, 0, canvas.width, canvas.height); + } + return canvas; + } + }; + + // converts an image to a canvas data url + static convertImgToCanvasUrl = async (imageSrc: string, width: number, height: number): Promise<string> => { + return new Promise<string>((resolve, reject) => { + const img = new Image(); + img.onload = () => { + const canvas = this.getCroppedImg(img, width, height); + if (canvas) { + const dataUrl = canvas.toDataURL(); + resolve(dataUrl); + } + }; + img.onerror = error => { + reject(error); + }; + img.src = imageSrc; + }); + }; + + // calls the openai api to get image edits + static getEdit = async (imgBlob: Blob, maskBlob: Blob, prompt: string, n?: number): Promise<APISuccess | APIError> => { + const apiUrl = 'https://api.openai.com/v1/images/edits'; + const fd = new FormData(); + fd.append('image', imgBlob, 'image.png'); + fd.append('mask', maskBlob, 'mask.png'); + fd.append('prompt', prompt); + fd.append('size', '1024x1024'); + fd.append('n', n ? JSON.stringify(n) : '1'); + fd.append('response_format', 'b64_json'); + + try { + const res = await fetch(apiUrl, { + method: 'POST', + headers: { + Authorization: `Bearer ${process.env.OPENAI_KEY}`, + }, + body: fd, + }); + const data = await res.json(); + console.log(data.data); + return { + status: 'success', + urls: (data.data as { b64_json: string }[]).map(data => `data:image/png;base64,${data.b64_json}`), + }; + } catch (err) { + console.log(err); + return { status: 'error', message: 'API error.' }; + } + }; + + // mock api call + static mockGetEdit = async (mockSrc: string): Promise<APISuccess | APIError> => { + return { + status: 'success', + urls: [mockSrc, mockSrc, mockSrc], + }; + }; + + // Gets the canvas rendering context of a canvas + static getCanvasContext = (canvasRef: RefObject<HTMLCanvasElement>): CanvasRenderingContext2D | null => { + if (!canvasRef.current) return null; + const ctx = canvasRef.current.getContext('2d'); + if (!ctx) return null; + return ctx; + }; + + // Helper for downloading the canvas (for debugging) + static downloadCanvas = (canvas: HTMLCanvasElement) => { + const url = canvas.toDataURL(); + const downloadLink = document.createElement('a'); + downloadLink.href = url; + downloadLink.download = 'canvas'; + + downloadLink.click(); + downloadLink.remove(); + }; + + // Download the canvas (for debugging) + static downloadImageCanvas = (imgUrl: string) => { + const img = new Image(); + img.src = imgUrl; + img.onload = () => { + const canvas = document.createElement('canvas'); + canvas.width = canvasSize; + canvas.height = canvasSize; + const ctx = canvas.getContext('2d'); + ctx?.drawImage(img, 0, 0, canvasSize, canvasSize); + + this.downloadCanvas(canvas); + }; + }; + + // Clears the canvas + static clearCanvas = (canvasRef: React.RefObject<HTMLCanvasElement>) => { + const ctx = this.getCanvasContext(canvasRef); + if (!ctx || !canvasRef.current) return; + ctx.clearRect(0, 0, canvasRef.current.width, canvasRef.current.height); + }; + + // Draws the image to the current canvas + static drawImgToCanvas = (img: HTMLImageElement, canvasRef: React.RefObject<HTMLCanvasElement>, width: number, height: number) => { + const drawImg = (img: HTMLImageElement) => { + const ctx = this.getCanvasContext(canvasRef); + if (!ctx) return; + ctx.globalCompositeOperation = 'source-over'; + ctx.clearRect(0, 0, width, height); + ctx.drawImage(img, 0, 0, width, height); + }; + + if (img.complete) { + drawImg(img); + } else { + img.onload = () => { + drawImg(img); + }; + } + }; + + // Gets the image mask for the openai endpoint + static getCanvasMask = (srcCanvas: HTMLCanvasElement, paddedCanvas: HTMLCanvasElement): HTMLCanvasElement | undefined => { + const canvas = document.createElement('canvas'); + canvas.width = canvasSize; + canvas.height = canvasSize; + const ctx = canvas.getContext('2d'); + if (!ctx) return; + ctx?.clearRect(0, 0, canvasSize, canvasSize); + ctx.drawImage(paddedCanvas, 0, 0); + + // extract and set padding data + if (srcCanvas.height > srcCanvas.width) { + // horizontal padding, x offset + const xOffset = (canvasSize - srcCanvas.width) / 2; + ctx?.clearRect(xOffset, 0, srcCanvas.width, srcCanvas.height); + ctx.drawImage(srcCanvas, xOffset, 0, srcCanvas.width, srcCanvas.height); + } else { + // vertical padding, y offset + const yOffset = (canvasSize - srcCanvas.height) / 2; + ctx?.clearRect(0, yOffset, srcCanvas.width, srcCanvas.height); + ctx.drawImage(srcCanvas, 0, yOffset, srcCanvas.width, srcCanvas.height); + } + return canvas; + }; + + // Fills in the blank areas of the image with an image reflection (to fill in a square-shaped canvas) + static drawHorizontalReflection = (ctx: CanvasRenderingContext2D, canvas: HTMLCanvasElement, xOffset: number) => { + const imageData = ctx.getImageData(0, 0, canvas.width, canvas.height); + const data = imageData.data; + for (let i = 0; i < canvas.height; i++) { + for (let j = 0; j < xOffset; j++) { + const targetIdx = 4 * (i * canvas.width + j); + const sourceI = i; + const sourceJ = xOffset + (xOffset - j); + const sourceIdx = 4 * (sourceI * canvas.width + sourceJ); + data[targetIdx] = data[sourceIdx]; + data[targetIdx + 1] = data[sourceIdx + 1]; + data[targetIdx + 2] = data[sourceIdx + 2]; + } + } + for (let i = 0; i < canvas.height; i++) { + for (let j = canvas.width - 1; j >= canvas.width - 1 - xOffset; j--) { + const targetIdx = 4 * (i * canvas.width + j); + const sourceI = i; + const sourceJ = canvas.width - 1 - xOffset - (xOffset - (canvas.width - j)); + const sourceIdx = 4 * (sourceI * canvas.width + sourceJ); + data[targetIdx] = data[sourceIdx]; + data[targetIdx + 1] = data[sourceIdx + 1]; + data[targetIdx + 2] = data[sourceIdx + 2]; + } + } + ctx.putImageData(imageData, 0, 0); + }; + + // Fills in the blank areas of the image with an image reflection (to fill in a square-shaped canvas) + static drawVerticalReflection = (ctx: CanvasRenderingContext2D, canvas: HTMLCanvasElement, yOffset: number) => { + const imageData = ctx.getImageData(0, 0, canvas.width, canvas.height); + const data = imageData.data; + for (let j = 0; j < canvas.width; j++) { + for (let i = 0; i < yOffset; i++) { + const targetIdx = 4 * (i * canvas.width + j); + const sourceJ = j; + const sourceI = yOffset + (yOffset - i); + const sourceIdx = 4 * (sourceI * canvas.width + sourceJ); + data[targetIdx] = data[sourceIdx]; + data[targetIdx + 1] = data[sourceIdx + 1]; + data[targetIdx + 2] = data[sourceIdx + 2]; + } + } + for (let j = 0; j < canvas.width; j++) { + for (let i = canvas.height - 1; i >= canvas.height - 1 - yOffset; i--) { + const targetIdx = 4 * (i * canvas.width + j); + const sourceJ = j; + const sourceI = canvas.height - 1 - yOffset - (yOffset - (canvas.height - i)); + const sourceIdx = 4 * (sourceI * canvas.width + sourceJ); + data[targetIdx] = data[sourceIdx]; + data[targetIdx + 1] = data[sourceIdx + 1]; + data[targetIdx + 2] = data[sourceIdx + 2]; + } + } + ctx.putImageData(imageData, 0, 0); + }; + + // Gets the unaltered (besides filling in padding) version of the image for the api call + static getCanvasImg = (img: HTMLImageElement): HTMLCanvasElement | undefined => { + const canvas = document.createElement('canvas'); + canvas.width = canvasSize; + canvas.height = canvasSize; + const ctx = canvas.getContext('2d'); + if (!ctx) return; + // fix scaling + const scale = Math.min(canvasSize / img.width, canvasSize / img.height); + const width = Math.floor(img.width * scale); + const height = Math.floor(img.height * scale); + ctx?.clearRect(0, 0, canvasSize, canvasSize); + ctx.fillStyle = bgColor; + ctx.fillRect(0, 0, canvasSize, canvasSize); + + // extract and set padding data + if (img.naturalHeight > img.naturalWidth) { + // horizontal padding, x offset + const xOffset = Math.floor((canvasSize - width) / 2); + ctx.drawImage(img, xOffset, 0, width, height); + + // draw reflected image padding + this.drawHorizontalReflection(ctx, canvas, xOffset); + } else { + // vertical padding, y offset + const yOffset = Math.floor((canvasSize - height) / 2); + ctx.drawImage(img, 0, yOffset, width, height); + + // draw reflected image padding + this.drawVerticalReflection(ctx, canvas, yOffset); + } + return canvas; + }; +} diff --git a/src/client/views/nodes/generativeFill/generativeFillUtils/PointerHandler.ts b/src/client/views/nodes/generativeFill/generativeFillUtils/PointerHandler.ts new file mode 100644 index 000000000..9e620ad11 --- /dev/null +++ b/src/client/views/nodes/generativeFill/generativeFillUtils/PointerHandler.ts @@ -0,0 +1,15 @@ +import { Point } from "./generativeFillInterfaces"; + +export class PointerHandler { + static getPointRelativeToElement = ( + element: HTMLElement, + e: React.PointerEvent | PointerEvent, + scale: number + ): Point => { + const boundingBox = element.getBoundingClientRect(); + return { + x: (e.clientX - boundingBox.x) / scale, + y: (e.clientY - boundingBox.y) / scale, + }; + }; +} diff --git a/src/client/views/nodes/generativeFill/generativeFillUtils/generativeFillConstants.ts b/src/client/views/nodes/generativeFill/generativeFillUtils/generativeFillConstants.ts new file mode 100644 index 000000000..4772304bc --- /dev/null +++ b/src/client/views/nodes/generativeFill/generativeFillUtils/generativeFillConstants.ts @@ -0,0 +1,9 @@ +export const canvasSize = 1024; +export const freeformRenderSize = 300; +export const offsetDistanceY = freeformRenderSize + 400; +export const offsetX = 200; +export const newCollectionSize = 500; + +export const activeColor = '#1976d2'; +export const eraserColor = '#e1e9ec'; +export const bgColor = '#f0f4f6'; diff --git a/src/client/views/nodes/generativeFill/generativeFillUtils/generativeFillInterfaces.ts b/src/client/views/nodes/generativeFill/generativeFillUtils/generativeFillInterfaces.ts new file mode 100644 index 000000000..1e7801056 --- /dev/null +++ b/src/client/views/nodes/generativeFill/generativeFillUtils/generativeFillInterfaces.ts @@ -0,0 +1,20 @@ +export interface CursorData { + x: number; + y: number; + width: number; +} + +export interface Point { + x: number; + y: number; +} + +export enum BrushMode { + ADD, + SUBTRACT, +} + +export interface ImageDimensions { + width: number; + height: number; +} |