diff options
3 files changed, 163 insertions, 77 deletions
diff --git a/src/client/views/nodes/generativeFill/GenerativeFill.tsx b/src/client/views/nodes/generativeFill/GenerativeFill.tsx index 97d1cbf20..92ce3a8a8 100644 --- a/src/client/views/nodes/generativeFill/GenerativeFill.tsx +++ b/src/client/views/nodes/generativeFill/GenerativeFill.tsx @@ -3,22 +3,23 @@ import React = require('react'); import { useEffect, useRef, useState } from 'react'; import { APISuccess, ImageUtility } from './generativeFillUtils/ImageHandler'; import { BrushHandler } from './generativeFillUtils/BrushHandler'; -import { Box, IconButton, Slider, TextField } from '@mui/material'; +import { Box, Checkbox, FormControlLabel, IconButton, Slider, TextField } from '@mui/material'; import { CursorData, ImageDimensions, Point } from './generativeFillUtils/generativeFillInterfaces'; import { activeColor, canvasSize, eraserColor, freeformRenderSize, newCollectionSize, offsetDistanceY, offsetX } from './generativeFillUtils/generativeFillConstants'; import { PointerHandler } from './generativeFillUtils/PointerHandler'; import { BsEraser, BsX } from 'react-icons/bs'; import { CiUndo, CiRedo } from 'react-icons/ci'; import { MainView } from '../../MainView'; -import { Doc } from '../../../../fields/Doc'; +import { Doc, DocListCast } from '../../../../fields/Doc'; import { Networking } from '../../../Network'; -import { Utils } from '../../../../Utils'; +import { Utils, returnEmptyDoclist } from '../../../../Utils'; import { DocUtils, Docs } from '../../../documents/Documents'; -import { NumCast } from '../../../../fields/Types'; +import { DocCast, NumCast } from '../../../../fields/Types'; import { CollectionDockingView } from '../../collections/CollectionDockingView'; import { OpenWhereMod } from '../DocumentView'; import { Oval } from 'react-loader-spinner'; import Buttons from './GenerativeFillButtons'; +import { List } from '../../../../fields/List'; enum BrushStyle { ADD, @@ -33,6 +34,8 @@ interface GenerativeFillProps { addDoc: ((doc: Doc | Doc[], annotationKey?: string) => boolean) | undefined; } +// New field on image doc: gen_fill_children => list of children Docs + const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addDoc }: GenerativeFillProps) => { const canvasRef = useRef<HTMLCanvasElement>(null); const canvasBackgroundRef = useRef<HTMLCanvasElement>(null); @@ -44,15 +47,18 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD }); const [isBrushing, setIsBrushing] = useState(false); const [canvasScale, setCanvasScale] = useState(0.5); - const [edits, setEdits] = useState<string[]>([]); + // format: array of [image source, corresponding image Doc] + const [edits, setEdits] = useState<(string | Doc)[][]>([]); + const [edited, setEdited] = useState(false); const [brushStyle, setBrushStyle] = useState<BrushStyle>(BrushStyle.ADD); const [input, setInput] = useState(''); const [loading, setLoading] = useState(false); - const [saveLoading, setSaveLoading] = useState(false); const [canvasDims, setCanvasDims] = useState<ImageDimensions>({ width: canvasSize, height: canvasSize, }); + // whether to create a new collection or not + const [isNewCollection, setIsNewCollection] = useState(true); // the current image in the main canvas const currImg = useRef<HTMLImageElement | null>(null); // the unedited version of each generation (parent) @@ -62,12 +68,6 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD // stores redo stack const redoStack = useRef<string[]>([]); - // early stage properly, likely will get rid of - const freeformPosition = useRef<number[]>([0, 0]); - - // which urls were already saved to canvas - const savedSrcs = useRef<Set<string>>(new Set()); - // references to keep track of tree structure const newCollectionRef = useRef<Doc | null>(null); const parentDoc = useRef<Doc | null>(null); @@ -178,13 +178,14 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD setCanvasDims({ width, height }); }; + // cleanup return () => { + setEdited(false); newCollectionRef.current = null; parentDoc.current = null; childrenDocs.current = []; currImg.current = null; originalImg.current = null; - freeformPosition.current = [0, 0]; undoStack.current = []; redoStack.current = []; ImageUtility.clearCanvas(canvasRef); @@ -250,10 +251,11 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD const ctx = ImageUtility.getCanvasContext(canvasRef); if (!ctx) return; setLoading(true); + setEdited(true); try { const canvasOriginalImg = ImageUtility.getCanvasImg(img); if (!canvasOriginalImg) return; - const canvasMask = ImageUtility.getCanvasMask(canvas); + const canvasMask = ImageUtility.getCanvasMask(canvas, canvasOriginalImg); if (!canvasMask) return; const maskBlob = await ImageUtility.canvasToBlob(canvasMask); const imgBlob = await ImageUtility.canvasToBlob(canvasOriginalImg); @@ -262,38 +264,46 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD // create first image if (!newCollectionRef.current) { - if (!(originalImg.current && imageRootDoc)) return; - // create new collection and add it to the view - newCollectionRef.current = Docs.Create.FreeformDocument([], { - x: NumCast(imageRootDoc.x) + NumCast(imageRootDoc._width) + offsetX, - y: NumCast(imageRootDoc.y), - _width: newCollectionSize, - _height: newCollectionSize, - title: 'Image edit collection', - }); - DocUtils.MakeLink(imageRootDoc, newCollectionRef.current, { link_relationship: 'Image Edit Version History', link_displayLine: false }); - - // opening new tab - CollectionDockingView.AddSplit(newCollectionRef.current, OpenWhereMod.right); - newCollectionRef.current.fitContentOnce = true; - - // add the doc to the main freeform - await createNewImgDoc(originalImg.current, true); + if (!isNewCollection && imageRootDoc) { + // new collection stays null + parentDoc.current = imageRootDoc; + } else { + if (!(originalImg.current && imageRootDoc)) return; + // create new collection and add it to the view + newCollectionRef.current = Docs.Create.FreeformDocument([], { + x: NumCast(imageRootDoc.x) + NumCast(imageRootDoc._width) + offsetX, + y: NumCast(imageRootDoc.y), + _width: newCollectionSize, + _height: newCollectionSize, + title: 'Image edit collection', + }); + DocUtils.MakeLink(imageRootDoc, newCollectionRef.current, { link_relationship: 'Image Edit Version History', link_displayLine: false }); + + // opening new tab + CollectionDockingView.AddSplit(newCollectionRef.current, OpenWhereMod.right); + + // add the doc to the main freeform + await createNewImgDoc(originalImg.current, true); + } } else { - parentDoc.current = childrenDocs.current[childrenDocs.current.length - 1]; childrenDocs.current = []; } originalImg.current = currImg.current; const { urls } = res as APISuccess; - const imgUrls = await Promise.all(urls.map(url => ImageUtility.convertImageToCanvasDataURL(url, canvasDims.width, canvasDims.height))); + const imgUrls = await Promise.all(urls.map(url => ImageUtility.convertImgToCanvasUrl(url, canvasDims.width, canvasDims.height))); + const imgRes = await Promise.all( + imgUrls.map(async url => { + const saveRes = await onSave(url); + return [url, saveRes as Doc]; + }) + ); + setEdits(imgRes); const image = new Image(); image.src = imgUrls[0]; - setEdits(imgUrls); ImageUtility.drawImgToCanvas(image, canvasRef, canvasDims.width, canvasDims.height); currImg.current = image; - onSave(); } catch (err) { console.log(err); } @@ -320,7 +330,7 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD // creates a new image document and returns its reference const createNewImgDoc = async (img: HTMLImageElement, firstDoc: boolean): Promise<Doc | undefined> => { - if (!newCollectionRef.current || !imageRootDoc) return; + if (!imageRootDoc) return; const src = img.src; const [result] = await Networking.PostToServer('/uploadRemoteImage', { sources: [src] }); const source = Utils.prepend(result.accessPaths.agnostic.client); @@ -337,8 +347,12 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD data_nativeWidth: result.nativeWidth, data_nativeHeight: result.nativeHeight, }); - - Doc.AddDocToList(newCollectionRef.current, undefined, newImg); + // add a new doc list field to newimg + if (isNewCollection && newCollectionRef.current) { + Doc.AddDocToList(newCollectionRef.current, undefined, newImg); + } else { + addDoc?.(newImg); + } parentDoc.current = newImg; return newImg; } else { @@ -354,30 +368,40 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD data_nativeWidth: result.nativeWidth, data_nativeHeight: result.nativeHeight, }); - + newImg.gen_fill_children = new List<Doc>([]); childrenDocs.current.push(newImg); - DocUtils.MakeLink(parentDoc.current, newImg, { link_relationship: 'Image Edit', link_displayLine: true }); + // DocListCast(parentDoc.current.gen_fill_children).push(newImg); + DocUtils.MakeLink(parentDoc.current, newImg, { link_relationship: `Image edit; Prompt: ${input}`, link_displayLine: true }); adjustImgPositions(); - Doc.AddDocToList(newCollectionRef.current, undefined, newImg); + if (isNewCollection && newCollectionRef.current) { + Doc.AddDocToList(newCollectionRef.current, undefined, newImg); + } else { + addDoc?.(newImg); + } return newImg; } }; - const onSave = async () => { - setSaveLoading(true); + const onSave = async (src: string) => { + const img = new Image(); + img.src = src; if (!currImg.current || !originalImg.current || !imageRootDoc) return; try { - await createNewImgDoc(currImg.current, false); + const res = await createNewImgDoc(img, false); + return res; } catch (err) { console.log(err); } - setSaveLoading(false); }; const handleViewClose = () => { MainView.Instance.setImageEditorOpen(false); MainView.Instance.setImageEditorSource(''); + if (newCollectionRef.current) { + newCollectionRef.current.fitContentOnce = true; + } + setEdits([]); }; @@ -386,15 +410,25 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD <div className="generativeFillControls"> <h1>AI Image Editor</h1> <div style={{ display: 'flex', alignItems: 'center', gap: '1.5rem' }}> + <FormControlLabel + control={ + <Checkbox + // disable once edited has been clicked + disabled={edited} + checked={isNewCollection} + onChange={e => { + setIsNewCollection(prev => !prev); + }} + /> + } + label={'Create New Collection'} + labelPlacement="end" + sx={{ whiteSpace: 'nowrap' }} + /> <Buttons canvasRef={canvasRef} currImg={currImg} getEdit={getEdit} loading={loading} onReset={handleReset} /> <IconButton onClick={handleViewClose}> <BsX color={activeColor} /> </IconButton> - {saveLoading && ( - <span style={{ height: '100%', display: 'flex', alignItems: 'center', gap: '8px' }}> - Saving image... <Oval height={20} width={20} color="#000000" visible={true} ariaLabel="oval-loading" secondaryColor="#ffffff89" strokeWidth={3} strokeWidthSecondary={3} /> - </span> - )} </div> </div> {/* Main canvas for editing */} @@ -475,14 +509,13 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD <img key={i} width={75} - src={edit} + src={edit[0] as string} onClick={async () => { const img = new Image(); - img.src = edit; + img.src = edit[0] as string; ImageUtility.drawImgToCanvas(img, canvasRef, canvasDims.width, canvasDims.height); currImg.current = img; - savedSrcs.current.add(edit); - await onSave(); + parentDoc.current = edit[1] as Doc; }} /> ))} diff --git a/src/client/views/nodes/generativeFill/GenerativeFillButtons.tsx b/src/client/views/nodes/generativeFill/GenerativeFillButtons.tsx index b4d56b408..e15af0a56 100644 --- a/src/client/views/nodes/generativeFill/GenerativeFillButtons.tsx +++ b/src/client/views/nodes/generativeFill/GenerativeFillButtons.tsx @@ -14,19 +14,10 @@ interface ButtonContainerProps { onReset: () => void; } -const Buttons = ({ canvasRef, currImg, loading, getEdit, onReset }: ButtonContainerProps) => { - +const Buttons = ({ loading, getEdit, onReset }: ButtonContainerProps) => { return ( <div className="generativeFillBtnContainer"> <Button onClick={onReset}>Reset</Button> - {/* <Button onClick={handleSave}>Save</Button> */} - {/* <Button - onClick={() => { - if (!canvasRef.current) return; - ImageUtility.downloadImageCanvas('/assets/firefly.png'); - }}> - Download Original - </Button> */} <Button variant="contained" onClick={() => { diff --git a/src/client/views/nodes/generativeFill/generativeFillUtils/ImageHandler.ts b/src/client/views/nodes/generativeFill/generativeFillUtils/ImageHandler.ts index 4847bfeed..4ff70c86c 100644 --- a/src/client/views/nodes/generativeFill/generativeFillUtils/ImageHandler.ts +++ b/src/client/views/nodes/generativeFill/generativeFillUtils/ImageHandler.ts @@ -45,7 +45,7 @@ export class ImageUtility { } }; - static convertImageToCanvasDataURL = async (imageSrc: string, width: number, height: number): Promise<string> => { + static convertImgToCanvasUrl = async (imageSrc: string, width: number, height: number): Promise<string> => { return new Promise<string>((resolve, reject) => { const img = new Image(); img.onload = () => { @@ -148,23 +148,23 @@ export class ImageUtility { if (img.complete) { drawImg(img); } else { - console.log('loading image'); img.onload = () => { drawImg(img); }; } }; - // The image must be loaded - static getCanvasMask = (srcCanvas: HTMLCanvasElement): HTMLCanvasElement | undefined => { + // The image must be loaded! + static getCanvasMask = (srcCanvas: HTMLCanvasElement, paddedCanvas: HTMLCanvasElement): HTMLCanvasElement | undefined => { const canvas = document.createElement('canvas'); canvas.width = canvasSize; canvas.height = canvasSize; const ctx = canvas.getContext('2d'); if (!ctx) return; ctx?.clearRect(0, 0, canvasSize, canvasSize); - ctx.fillStyle = '#000000'; - ctx.fillRect(0, 0, canvasSize, canvasSize); + // ctx.fillStyle = bgColor; + // ctx.fillRect(0, 0, canvasSize, canvasSize); + ctx.drawImage(paddedCanvas, 0, 0); // extract and set padding data if (srcCanvas.height > srcCanvas.width) { @@ -181,7 +181,63 @@ export class ImageUtility { return canvas; }; - // The image must be loaded + static drawHorizontalReflection = (ctx: CanvasRenderingContext2D, canvas: HTMLCanvasElement, xOffset: number) => { + const imageData = ctx.getImageData(0, 0, canvas.width, canvas.height); + const data = imageData.data; + for (let i = 0; i < canvas.height; i++) { + for (let j = 0; j < xOffset; j++) { + const targetIdx = 4 * (i * canvas.width + j); + const sourceI = i; + const sourceJ = xOffset + (xOffset - j); + const sourceIdx = 4 * (sourceI * canvas.width + sourceJ); + data[targetIdx] = data[sourceIdx]; + data[targetIdx + 1] = data[sourceIdx + 1]; + data[targetIdx + 2] = data[sourceIdx + 2]; + } + } + for (let i = 0; i < canvas.height; i++) { + for (let j = canvas.width - 1; j >= canvas.width - 1 - xOffset; j--) { + const targetIdx = 4 * (i * canvas.width + j); + const sourceI = i; + const sourceJ = canvas.width - 1 - xOffset - (xOffset - (canvas.width - j)); + const sourceIdx = 4 * (sourceI * canvas.width + sourceJ); + data[targetIdx] = data[sourceIdx]; + data[targetIdx + 1] = data[sourceIdx + 1]; + data[targetIdx + 2] = data[sourceIdx + 2]; + } + } + ctx.putImageData(imageData, 0, 0); + }; + + static drawVerticalReflection = (ctx: CanvasRenderingContext2D, canvas: HTMLCanvasElement, yOffset: number) => { + const imageData = ctx.getImageData(0, 0, canvas.width, canvas.height); + const data = imageData.data; + for (let j = 0; j < canvas.width; j++) { + for (let i = 0; i < yOffset; i++) { + const targetIdx = 4 * (i * canvas.width + j); + const sourceJ = j; + const sourceI = yOffset + (yOffset - i); + const sourceIdx = 4 * (sourceI * canvas.width + sourceJ); + data[targetIdx] = data[sourceIdx]; + data[targetIdx + 1] = data[sourceIdx + 1]; + data[targetIdx + 2] = data[sourceIdx + 2]; + } + } + for (let j = 0; j < canvas.width; j++) { + for (let i = canvas.height - 1; i >= canvas.height - 1 - yOffset; i--) { + const targetIdx = 4 * (i * canvas.width + j); + const sourceJ = j; + const sourceI = canvas.height - 1 - yOffset - (yOffset - (canvas.height - i)); + const sourceIdx = 4 * (sourceI * canvas.width + sourceJ); + data[targetIdx] = data[sourceIdx]; + data[targetIdx + 1] = data[sourceIdx + 1]; + data[targetIdx + 2] = data[sourceIdx + 2]; + } + } + ctx.putImageData(imageData, 0, 0); + }; + + // The image must be loaded! static getCanvasImg = (img: HTMLImageElement): HTMLCanvasElement | undefined => { const canvas = document.createElement('canvas'); canvas.width = canvasSize; @@ -190,21 +246,27 @@ export class ImageUtility { if (!ctx) return; // fix scaling const scale = Math.min(canvasSize / img.width, canvasSize / img.height); - const width = img.width * scale; - const height = img.height * scale; + const width = Math.floor(img.width * scale); + const height = Math.floor(img.height * scale); ctx?.clearRect(0, 0, canvasSize, canvasSize); - ctx.fillStyle = '#000000'; + ctx.fillStyle = bgColor; ctx.fillRect(0, 0, canvasSize, canvasSize); + // extract and set padding data if (img.naturalHeight > img.naturalWidth) { // horizontal padding, x offset - const xOffset = (canvasSize - width) / 2; - + const xOffset = Math.floor((canvasSize - width) / 2); ctx.drawImage(img, xOffset, 0, width, height); + + // draw reflected image padding + this.drawHorizontalReflection(ctx, canvas, xOffset); } else { // vertical padding, y offset - const yOffset = (canvasSize - height) / 2; + const yOffset = Math.floor((canvasSize - height) / 2); ctx.drawImage(img, 0, yOffset, width, height); + + // draw reflected image padding + this.drawVerticalReflection(ctx, canvas, yOffset); } return canvas; }; |