diff options
Diffstat (limited to 'src')
3 files changed, 156 insertions, 59 deletions
diff --git a/src/client/views/nodes/generativeFill/GenerativeFill.tsx b/src/client/views/nodes/generativeFill/GenerativeFill.tsx index f136982bc..4d475149d 100644 --- a/src/client/views/nodes/generativeFill/GenerativeFill.tsx +++ b/src/client/views/nodes/generativeFill/GenerativeFill.tsx @@ -42,10 +42,9 @@ enum BrushStyle { MARQUEE, } -interface ImageEdit { - imgElement: HTMLImageElement; - parent: ImageEdit; - children: ImageEdit[]; +interface ImageDimensions { + width: number; + height: number; } interface GenerativeFillProps { @@ -59,7 +58,6 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD const canvasRef = useRef<HTMLCanvasElement>(null); const canvasBackgroundRef = useRef<HTMLCanvasElement>(null); const drawingAreaRef = useRef<HTMLDivElement>(null); - const fileRef = useRef<HTMLInputElement>(null); const [cursorData, setCursorData] = useState<CursorData>({ x: 0, y: 0, @@ -72,6 +70,10 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD const [input, setInput] = useState(''); const [loading, setLoading] = useState(false); const [saveLoading, setSaveLoading] = useState(false); + const [canvasDims, setCanvasDims] = useState<ImageDimensions>({ + width: canvasSize, + height: canvasSize, + }); // the current image in the main canvas const currImg = useRef<HTMLImageElement | null>(null); // the unedited version of each generation (parent) @@ -91,7 +93,6 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD const newCollectionRef = useRef<Doc | null>(null); const parentDoc = useRef<Doc | null>(null); const childrenDocs = useRef<Doc[]>([]); - const addToExistingCollection = useRef<boolean>(false); // Undo and Redo const handleUndo = () => { @@ -100,12 +101,12 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD const target = undoStack.current[undoStack.current.length - 1]; if (!target) { - ImageUtility.drawImgToCanvas(currImg.current, canvasRef); + ImageUtility.drawImgToCanvas(currImg.current, canvasRef, canvasDims.width, canvasDims.height); } else { redoStack.current = [...redoStack.current, canvasRef.current.toDataURL()]; const img = new Image(); img.src = target; - ImageUtility.drawImgToCanvas(img, canvasRef); + ImageUtility.drawImgToCanvas(img, canvasRef, canvasDims.width, canvasDims.height); undoStack.current = undoStack.current.slice(0, -1); } }; @@ -117,7 +118,7 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD } else { const img = new Image(); img.src = target; - ImageUtility.drawImgToCanvas(img, canvasRef); + ImageUtility.drawImgToCanvas(img, canvasRef, canvasDims.width, canvasDims.height); redoStack.current = redoStack.current.slice(0, -1); } }; @@ -129,7 +130,7 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD ctx.clearRect(0, 0, canvasSize, canvasSize); undoStack.current = []; redoStack.current = []; - ImageUtility.drawImgToCanvas(currImg.current, canvasRef, true); + ImageUtility.drawImgToCanvas(currImg.current, canvasRef, canvasSize, canvasSize); }; // initiate brushing @@ -144,7 +145,6 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD setIsBrushing(true); const { x, y } = PointerHandler.getPointRelativeToElement(canvas, e, canvasScale); - BrushHandler.brushCircleOverlay(x, y, cursorData.width / 2 / canvasScale, ctx, eraserColor, brushStyle === BrushStyle.SUBTRACT); }; @@ -181,19 +181,21 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD // first load useEffect(() => { - console.log('first load'); if (!imageEditorSource || imageEditorSource === '') return; const img = new Image(); - console.log('source', imageEditorSource); img.src = imageEditorSource; - console.log('drawing image'); - ImageUtility.drawImgToCanvas(img, canvasRef); currImg.current = img; originalImg.current = img; - freeformPosition.current = [0, 0]; + img.onload = () => { + const imgWidth = img.naturalWidth; + const imgHeight = img.naturalHeight; + const scale = Math.min(canvasSize / imgWidth, canvasSize / imgHeight); + const width = imgWidth * scale; + const height = imgHeight * scale; + setCanvasDims({ width, height }); + }; return () => { - console.log('cleanup'); newCollectionRef.current = null; parentDoc.current = null; childrenDocs.current = []; @@ -202,9 +204,15 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD freeformPosition.current = [0, 0]; undoStack.current = []; redoStack.current = []; + ImageUtility.clearCanvas(canvasRef); }; }, [canvasRef, imageEditorSource]); + useEffect(() => { + if (!currImg.current) return; + ImageUtility.drawImgToCanvas(currImg.current, canvasRef, canvasDims.width, canvasDims.height); + }, [canvasDims]); + // handles brush sizing useEffect(() => { const handleKeyPress = (e: KeyboardEvent) => { @@ -261,15 +269,18 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD setLoading(true); // need to adjust later try { - const maskBlob = await ImageUtility.canvasToBlob(canvas); - const imgBlob = await ImageUtility.canvasToBlob(ImageUtility.getCanvasImg(img)); + const canvasOriginalImg = ImageUtility.getCanvasImg(img); + if (!canvasOriginalImg) return; + const canvasMask = ImageUtility.getCanvasMask(canvas); + if (!canvasMask) return; + // ImageUtility.downloadCanvas(canvasMask); + const maskBlob = await ImageUtility.canvasToBlob(canvasMask); + const imgBlob = await ImageUtility.canvasToBlob(canvasOriginalImg); const res = await ImageUtility.getEdit(imgBlob, maskBlob, input !== '' ? input + ' in the same style' : 'Fill in the image in the same style', 2); // const res = await ImageUtility.mockGetEdit(img.src); // create first image if (!newCollectionRef.current) { - if (addToExistingCollection.current) { - } if (!(originalImg.current && imageRootDoc)) return; console.log('creating first image'); // create new collection and add it to the view @@ -281,25 +292,30 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD title: 'Image edit collection', }); DocUtils.MakeLink(imageRootDoc, newCollectionRef.current, { link_relationship: 'Image Edit Version History', link_displayLine: false }); + + // opening new tab + newCollectionRef.current.fitContentOnce = true; + CollectionDockingView.AddSplit(newCollectionRef.current, OpenWhereMod.right); + // add the doc to the main freeform // addDoc?.(newCollectionRef.current); await createNewImgDoc(originalImg.current, true); } else { - parentDoc.current = childrenDocs.current[childrenDocs.current.length - 1]; + // parentDoc.current = childrenDocs.current[childrenDocs.current.length - 1]; childrenDocs.current = []; } originalImg.current = currImg.current; const { urls } = res as APISuccess; + const imgUrls = await Promise.all(urls.map(url => ImageUtility.convertImageToCanvasDataURL(url, canvasDims.width, canvasDims.height))); const image = new Image(); - image.src = urls[0]; - setEdits(urls); - ImageUtility.drawImgToCanvas(image, canvasRef); + image.src = imgUrls[0]; + + setEdits(imgUrls); + ImageUtility.drawImgToCanvas(image, canvasRef, canvasDims.width, canvasDims.height); currImg.current = image; onSave(); - freeformPosition.current[0] += 1; - freeformPosition.current[1] = 0; } catch (err) { console.log(err); } @@ -334,7 +350,6 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD if (firstDoc) { const x = 0; const initialY = 0; - console.log('first doc'); const newImg = Docs.Create.ImageDocument(source, { x: x, @@ -387,10 +402,10 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD }; const handleViewClose = () => { - if (newCollectionRef.current) { - newCollectionRef.current.fitContentOnce = true; - CollectionDockingView.AddSplit(newCollectionRef.current, OpenWhereMod.right); - } + // if (newCollectionRef.current) { + // newCollectionRef.current.fitContentOnce = true; + // CollectionDockingView.AddSplit(newCollectionRef.current, OpenWhereMod.right); + // } MainView.Instance.setImageEditorOpen(false); MainView.Instance.setImageEditorSource(''); setEdits([]); @@ -420,8 +435,8 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD onPointerMove={updateCursorData} onPointerDown={handlePointerDown} onPointerUp={handlePointerUp}> - <canvas ref={canvasRef} width={canvasSize} height={canvasSize} style={{ transform: `scale(${canvasScale})` }} /> - <canvas ref={canvasBackgroundRef} width={canvasSize} height={canvasSize} style={{ transform: `scale(${canvasScale})` }} /> + <canvas ref={canvasRef} width={canvasDims.width} height={canvasSize} style={{ transform: `scale(${canvasScale})` }} /> + <canvas ref={canvasBackgroundRef} width={canvasDims.height} height={canvasSize} style={{ transform: `scale(${canvasScale})` }} /> <div className="pointer" style={{ @@ -489,14 +504,13 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD {edits.map((edit, i) => ( <img key={i} - width={100} - height={100} + width={75} src={edit} onClick={async () => { // if (savedSrcs.current.has(edit)) return; const img = new Image(); img.src = edit; - ImageUtility.drawImgToCanvas(img, canvasRef); + ImageUtility.drawImgToCanvas(img, canvasRef, canvasDims.width, canvasDims.height); currImg.current = img; savedSrcs.current.add(edit); await onSave(); @@ -519,15 +533,14 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD Original </label> <img - width={100} - height={100} + width={75} src={originalImg.current?.src} style={{ cursor: 'pointer' }} onClick={() => { if (!originalImg.current) return; const img = new Image(); img.src = originalImg.current.src; - ImageUtility.drawImgToCanvas(img, canvasRef); + ImageUtility.drawImgToCanvas(img, canvasRef, canvasDims.width, canvasDims.height); currImg.current = img; }} /> diff --git a/src/client/views/nodes/generativeFill/generativeFillUtils/ImageHandler.ts b/src/client/views/nodes/generativeFill/generativeFillUtils/ImageHandler.ts index 1954ab3fb..8d32221bd 100644 --- a/src/client/views/nodes/generativeFill/generativeFillUtils/ImageHandler.ts +++ b/src/client/views/nodes/generativeFill/generativeFillUtils/ImageHandler.ts @@ -1,5 +1,5 @@ import { RefObject } from 'react'; -import { canvasSize } from './generativeFillConstants'; +import { bgColor, canvasSize } from './generativeFillConstants'; export interface APISuccess { status: 'success'; @@ -22,6 +22,47 @@ export class ImageUtility { }); }; + // given a square api image, get the cropped img + static getCroppedImg = (img: HTMLImageElement, width: number, height: number): HTMLCanvasElement | undefined => { + // Create a new canvas element + const canvas = document.createElement('canvas'); + canvas.width = width; + canvas.height = height; + const ctx = canvas.getContext('2d'); + if (ctx) { + // Clear the canvas + ctx.clearRect(0, 0, canvas.width, canvas.height); + if (width < height) { + // horizontal padding, x offset + const xOffset = (canvasSize - width) / 2; + console.log(xOffset); + ctx.drawImage(img, xOffset, 0, canvas.width, canvas.height, 0, 0, canvas.width, canvas.height); + } else { + // vertical padding, y offset + const yOffset = (canvasSize - height) / 2; + ctx.drawImage(img, 0, yOffset, canvas.width, canvas.height, 0, 0, canvas.width, canvas.height); + } + return canvas; + } + }; + + static convertImageToCanvasDataURL = async (imageSrc: string, width: number, height: number): Promise<string> => { + return new Promise<string>((resolve, reject) => { + const img = new Image(); + img.onload = () => { + const canvas = this.getCroppedImg(img, width, height); + if (canvas) { + const dataUrl = canvas.toDataURL(); + resolve(dataUrl); + } + }; + img.onerror = error => { + reject(error); + }; + img.src = imageSrc; + }); + }; + static getEdit = async (imgBlob: Blob, maskBlob: Blob, prompt: string, n?: number): Promise<APISuccess | APIError> => { const apiUrl = 'https://api.openai.com/v1/images/edits'; const fd = new FormData(); @@ -90,41 +131,83 @@ export class ImageUtility { }; }; - static drawImgToCanvas = (img: HTMLImageElement, canvasRef: React.RefObject<HTMLCanvasElement>, loaded?: boolean) => { - if (loaded) { + static clearCanvas = (canvasRef: React.RefObject<HTMLCanvasElement>) => { + const ctx = this.getCanvasContext(canvasRef); + if (!ctx || !canvasRef.current) return; + ctx.clearRect(0, 0, canvasRef.current.width, canvasRef.current.height); + }; + + static drawImgToCanvas = (img: HTMLImageElement, canvasRef: React.RefObject<HTMLCanvasElement>, width: number, height: number) => { + const drawImg = (img: HTMLImageElement) => { const ctx = this.getCanvasContext(canvasRef); if (!ctx) return; ctx.globalCompositeOperation = 'source-over'; - const scale = Math.max(canvasSize / img.width, canvasSize / img.height); - const width = img.width * scale; - const height = img.height * scale; - ctx.clearRect(0, 0, canvasSize, canvasSize); + ctx.clearRect(0, 0, width, height); ctx.drawImage(img, 0, 0, width, height); + }; + + if (img.complete) { + drawImg(img); } else { console.log('loading image'); img.onload = () => { - console.log('loaded'); - const ctx = this.getCanvasContext(canvasRef); - if (!ctx) return; - ctx.globalCompositeOperation = 'source-over'; - const scale = Math.max(canvasSize / img.width, canvasSize / img.height); - const width = img.width * scale; - const height = img.height * scale; - ctx.clearRect(0, 0, canvasSize, canvasSize); - ctx.drawImage(img, 0, 0, width, height); + drawImg(img); }; } }; // The image must be loaded! - static getCanvasImg = (img: HTMLImageElement): HTMLCanvasElement => { + static getCanvasMask = (srcCanvas: HTMLCanvasElement): HTMLCanvasElement | undefined => { const canvas = document.createElement('canvas'); canvas.width = canvasSize; canvas.height = canvasSize; const ctx = canvas.getContext('2d'); + if (!ctx) return; ctx?.clearRect(0, 0, canvasSize, canvasSize); - ctx?.drawImage(img, 0, 0, canvasSize, canvasSize); + ctx.fillStyle = bgColor; + ctx.fillRect(0, 0, canvasSize, canvasSize); + // extract and set padding data + if (srcCanvas.height > srcCanvas.width) { + // horizontal padding, x offset + const xOffset = (canvasSize - srcCanvas.width) / 2; + ctx?.clearRect(xOffset, 0, srcCanvas.width, srcCanvas.height); + ctx.drawImage(srcCanvas, xOffset, 0, srcCanvas.width, srcCanvas.height); + } else { + // vertical padding, y offset + const yOffset = (canvasSize - srcCanvas.height) / 2; + ctx?.clearRect(0, yOffset, srcCanvas.width, srcCanvas.height); + ctx.drawImage(srcCanvas, 0, yOffset, srcCanvas.width, srcCanvas.height); + } + return canvas; + }; + + // The image must be loaded! + static getCanvasImg = (img: HTMLImageElement): HTMLCanvasElement | undefined => { + const canvas = document.createElement('canvas'); + canvas.width = canvasSize; + canvas.height = canvasSize; + const ctx = canvas.getContext('2d'); + if (!ctx) return; + // fix scaling + const scale = Math.min(canvasSize / img.width, canvasSize / img.height); + const width = img.width * scale; + const height = img.height * scale; + ctx?.clearRect(0, 0, canvasSize, canvasSize); + ctx.fillStyle = bgColor; + ctx.fillRect(0, 0, canvasSize, canvasSize); + + // extract and set padding data + if (img.naturalHeight > img.naturalWidth) { + // horizontal padding, x offset + const xOffset = (canvasSize - width) / 2; + + ctx.drawImage(img, xOffset, 0, width, height); + } else { + // vertical padding, y offset + const yOffset = (canvasSize - height) / 2; + ctx.drawImage(img, 0, yOffset, width, height); + } return canvas; }; } diff --git a/src/client/views/nodes/generativeFill/generativeFillUtils/generativeFillConstants.ts b/src/client/views/nodes/generativeFill/generativeFillUtils/generativeFillConstants.ts index 412a4d238..286dc6e4c 100644 --- a/src/client/views/nodes/generativeFill/generativeFillUtils/generativeFillConstants.ts +++ b/src/client/views/nodes/generativeFill/generativeFillUtils/generativeFillConstants.ts @@ -1,9 +1,10 @@ // constants export const canvasSize = 512; export const freeformRenderSize = 300; -export const offsetDistanceY = freeformRenderSize + 200; +export const offsetDistanceY = freeformRenderSize + 300; export const offsetX = 200; export const newCollectionSize = 500; export const activeColor = '#1976d2'; export const eraserColor = '#e1e9ec'; +export const bgColor = '#f0f4f6'; |