diff options
Diffstat (limited to 'src/client/views/nodes/imageEditor')
3 files changed, 35 insertions, 381 deletions
diff --git a/src/client/views/nodes/imageEditor/ImageEditor.tsx b/src/client/views/nodes/imageEditor/ImageEditor.tsx index abe235ad5..b56490bc3 100644 --- a/src/client/views/nodes/imageEditor/ImageEditor.tsx +++ b/src/client/views/nodes/imageEditor/ImageEditor.tsx @@ -269,7 +269,7 @@ const ImageEditor = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addDoc }; // Get AI Edit for Generative Fill - const getEdit = async () => { + const getEdit = async (useFirefly = true) => { const img = currImg.current; if (!img) return; const canvas = canvasRef.current; @@ -285,40 +285,41 @@ const ImageEditor = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addDoc if (!canvasMask) return; const maskBlob = await ImageUtility.canvasToBlob(canvasMask); const imgBlob = await ImageUtility.canvasToBlob(canvasOriginalImg); - const res = await ImageUtility.getEdit(imgBlob, maskBlob, input || 'Fill in the image in the same style', 2); - if (res.status == 'error') { - alert(res.message); - } - - // create first image - if (!newCollectionRef.current) { - createNewCollection(); + let imgUrls: string[] = []; + const setupCollection = async () => { + // create first image + if (!newCollectionRef.current) { + createNewCollection(); + } else { + childrenDocs.current = []; + } + if (!(originalImg.current && imageRootDoc)) return; + // add the doc to the main freeform + await createNewImgDoc(originalImg.current, true); + originalImg.current = currImg.current; + originalDoc.current = parentDoc.current; + }; + if (useFirefly) { + const res = await Networking.PostFormToServer(input || 'Fill in the image in the same style', imgBlob, maskBlob, img.width, img.height); + if (!res.ok) throw new Error(await res.text()); + const json = (await res.json()) as APISuccess; + imgUrls = json.urls ?? []; } else { - childrenDocs.current = []; - } - if (!(originalImg.current && imageRootDoc)) return; - // add the doc to the main freeform - await createNewImgDoc(originalImg.current, true); - originalImg.current = currImg.current; - originalDoc.current = parentDoc.current; - const { urls } = res as APISuccess; - if (res.status !== 'error') { - const imgUrls = await Promise.all(urls.map(url => ImageUtility.convertImgToCanvasUrl(url, canvasDims.width, canvasDims.height))); - const imgRes = await Promise.all( - imgUrls.map(async url => { - const saveRes = await onSave(url); - return { url, saveRes }; - }) - ); - setEdits(imgRes); - const image = new Image(); - image.src = imgUrls[0]; - ImageUtility.drawImgToCanvas(image, canvasRef, canvasDims.width, canvasDims.height); - currImg.current = image; - parentDoc.current = imgRes[0].saveRes ?? null; + const res = await ImageUtility.getEdit(imgBlob, maskBlob, input || 'Fill in the image in the same style', 2); + if (res.status == 'error') throw new Error(res.message); + const json = res as APISuccess; + imgUrls = await Promise.all((json.urls ?? []).map(url => ImageUtility.convertImgToCanvasUrl(url, canvasDims.width, canvasDims.height))); } - } catch (err) { - console.log(err); + + setupCollection(); + const imgRes = await Promise.all(imgUrls.map(async url => ({ url, saveRes: await onSave(url) }))); + setEdits(imgRes); + currImg.current = new Image(); + currImg.current.src = imgUrls[0]; + ImageUtility.drawImgToCanvas(currImg.current, canvasRef, canvasDims.width, canvasDims.height); + parentDoc.current = imgRes[0].saveRes ?? null; + } catch (err: unknown) { + alert('message' in (err as object) ? (err as { message: string }).message : err); } setLoading(false); }; @@ -561,7 +562,7 @@ const ImageEditor = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addDoc } // defines the tools and sets current tool - const genFillTool: ImageEditTool = { type: ImageToolType.GenerativeFill, btnText: 'GET EDITS', icon: 'fill', applyFunc: getEdit, sliderMin: 25, sliderMax: 500, sliderDefault: 150 }; + const genFillTool: ImageEditTool = { type: ImageToolType.GenerativeFill, btnText: 'GET EDITS', icon: 'fill', applyFunc: () => getEdit(), sliderMin: 25, sliderMax: 500, sliderDefault: 150 }; const cutTool: ImageEditTool = { type: ImageToolType.Cut, btnText: 'CUT IMAGE', icon: 'scissors', applyFunc: cutImage, sliderMin: 1, sliderMax: 50, sliderDefault: 5 }; const imageEditTools: ImageEditTool[] = [genFillTool, cutTool]; const [currToolType, setCurrToolType] = useState<ImageToolType>(ImageToolType.GenerativeFill); diff --git a/src/client/views/nodes/imageEditor/imageToolUtils/BrushHandler.ts b/src/client/views/nodes/imageEditor/imageToolUtils/BrushHandler.ts deleted file mode 100644 index 7139bebc3..000000000 --- a/src/client/views/nodes/imageEditor/imageToolUtils/BrushHandler.ts +++ /dev/null @@ -1,35 +0,0 @@ -import { GenerativeFillMathHelpers } from '../imageEditorUtils/GenerativeFillMathHelpers'; -import { eraserColor } from '../imageEditorUtils/imageEditorConstants'; -import { Point } from '../imageEditorUtils/imageEditorInterfaces'; -import { points } from '@turf/turf'; - -export enum BrushType { - GEN_FILL, - CUT, -} - -export class BrushHandler { - static brushCircleOverlay = (x: number, y: number, brushRadius: number, ctx: CanvasRenderingContext2D, fillColor: string /* , erase: boolean */) => { - ctx.globalCompositeOperation = 'destination-out'; - ctx.fillStyle = fillColor; - ctx.shadowColor = eraserColor; - ctx.shadowBlur = 5; - ctx.beginPath(); - ctx.arc(x, y, brushRadius, 0, 2 * Math.PI); - ctx.fill(); - ctx.closePath(); - }; - - static createBrushPathOverlay = (startPoint: Point, endPoint: Point, brushRadius: number, ctx: CanvasRenderingContext2D, fillColor: string, brushType: BrushType) => { - const dist = GenerativeFillMathHelpers.distanceBetween(startPoint, endPoint); - const pts: Point[] = []; - for (let i = 0; i < dist; i += 5) { - const s = i / dist; - const x = startPoint.x * (1 - s) + endPoint.x * s; - const y = startPoint.y * (1 - s) + endPoint.y * s; - pts.push({ x: startPoint.x, y: startPoint.y }); - BrushHandler.brushCircleOverlay(x, y, brushRadius, ctx, fillColor); - } - return pts; - }; -} diff --git a/src/client/views/nodes/imageEditor/imageToolUtils/ImageHandler.ts b/src/client/views/nodes/imageEditor/imageToolUtils/ImageHandler.ts deleted file mode 100644 index b9723b5be..000000000 --- a/src/client/views/nodes/imageEditor/imageToolUtils/ImageHandler.ts +++ /dev/null @@ -1,312 +0,0 @@ -import { RefObject } from 'react'; -import { bgColor, canvasSize } from '../imageEditorUtils/imageEditorConstants'; - -export interface APISuccess { - status: 'success'; - urls: string[]; -} - -export interface APIError { - status: 'error'; - message: string; -} - -export class ImageUtility { - /** - * - * @param canvas Canvas to convert - * @returns Blob of canvas - */ - static canvasToBlob = (canvas: HTMLCanvasElement): Promise<Blob> => - new Promise(resolve => { - canvas.toBlob(blob => { - if (blob) { - resolve(blob); - } - }, 'image/png'); - }); - - // given a square api image, get the cropped img - static getCroppedImg = (img: HTMLImageElement, width: number, height: number): HTMLCanvasElement | undefined => { - // Create a new canvas element - const canvas = document.createElement('canvas'); - canvas.width = width; - canvas.height = height; - const ctx = canvas.getContext('2d'); - if (ctx) { - // Clear the canvas - ctx.clearRect(0, 0, canvas.width, canvas.height); - if (width < height) { - // horizontal padding, x offset - const xOffset = (canvasSize - width) / 2; - ctx.drawImage(img, xOffset, 0, canvas.width, canvas.height, 0, 0, canvas.width, canvas.height); - } else { - // vertical padding, y offset - const yOffset = (canvasSize - height) / 2; - ctx.drawImage(img, 0, yOffset, canvas.width, canvas.height, 0, 0, canvas.width, canvas.height); - } - return canvas; - } - return undefined; - }; - - // converts an image to a canvas data url - static convertImgToCanvasUrl = async (imageSrc: string, width: number, height: number): Promise<string> => - new Promise<string>((resolve, reject) => { - const img = new Image(); - img.onload = () => { - const canvas = this.getCroppedImg(img, width, height); - if (canvas) { - const dataUrl = canvas.toDataURL(); - resolve(dataUrl); - } - }; - img.onerror = error => { - reject(error); - }; - img.src = imageSrc; - }); - - // calls the openai api to get image edits - static getEdit = async (imgBlob: Blob, maskBlob: Blob, prompt: string, n?: number): Promise<APISuccess | APIError> => { - const apiUrl = 'https://api.openai.com/v1/images/edits'; - const fd = new FormData(); - fd.append('image', imgBlob, 'image.png'); - fd.append('mask', maskBlob, 'mask.png'); - fd.append('prompt', prompt); - fd.append('size', '1024x1024'); - fd.append('n', n ? JSON.stringify(n) : '1'); - fd.append('response_format', 'b64_json'); - - try { - const res = await fetch(apiUrl, { - method: 'POST', - headers: { - Authorization: `Bearer ${process.env.OPENAI_KEY}`, - }, - body: fd, - }); - const data = await res.json(); - console.log(data.data); - return { - status: 'success', - urls: (data.data as { b64_json: string }[]).map(urlData => `data:image/png;base64,${urlData.b64_json}`), - }; - } catch (err) { - console.log(err); - return { status: 'error', message: 'API error.' }; - } - }; - - // mock api call - static mockGetEdit = async (mockSrc: string): Promise<APISuccess | APIError> => ({ - status: 'success', - urls: [mockSrc, mockSrc, mockSrc], - }); - - // Gets the canvas rendering context of a canvas - static getCanvasContext = (canvasRef: RefObject<HTMLCanvasElement>): CanvasRenderingContext2D | null => { - if (!canvasRef.current) return null; - const ctx = canvasRef.current.getContext('2d'); - if (!ctx) return null; - return ctx; - }; - - // Helper for downloading the canvas (for debugging) - static downloadCanvas = (canvas: HTMLCanvasElement) => { - const url = canvas.toDataURL(); - const downloadLink = document.createElement('a'); - downloadLink.href = url; - downloadLink.download = 'canvas'; - - downloadLink.click(); - downloadLink.remove(); - }; - - // Download the canvas (for debugging) - static downloadImageCanvas = (imgUrl: string) => { - const img = new Image(); - img.src = imgUrl; - img.onload = () => { - const canvas = document.createElement('canvas'); - canvas.width = canvasSize; - canvas.height = canvasSize; - const ctx = canvas.getContext('2d'); - ctx?.drawImage(img, 0, 0, canvasSize, canvasSize); - - this.downloadCanvas(canvas); - }; - }; - - // Clears the canvas - static clearCanvas = (canvasRef: React.RefObject<HTMLCanvasElement>) => { - const ctx = this.getCanvasContext(canvasRef); - if (!ctx || !canvasRef.current) return; - ctx.clearRect(0, 0, canvasRef.current.width, canvasRef.current.height); - }; - - // Draws the image to the current canvas - static drawImgToCanvas = (img: HTMLImageElement, canvasRef: React.RefObject<HTMLCanvasElement>, width: number, height: number) => { - const drawImg = (htmlImg: HTMLImageElement) => { - const ctx = this.getCanvasContext(canvasRef); - if (!ctx) return; - ctx.globalCompositeOperation = 'source-over'; - ctx.clearRect(0, 0, width, height); - ctx.drawImage(htmlImg, 0, 0, width, height); - }; - - if (img.complete) { - drawImg(img); - } else { - img.onload = () => { - drawImg(img); - }; - } - }; - - // Gets the image mask for the openai endpoint - static getCanvasMask = (srcCanvas: HTMLCanvasElement, paddedCanvas: HTMLCanvasElement): HTMLCanvasElement | undefined => { - const canvas = document.createElement('canvas'); - canvas.width = canvasSize; - canvas.height = canvasSize; - const ctx = canvas.getContext('2d'); - if (!ctx) return undefined; - ctx?.clearRect(0, 0, canvasSize, canvasSize); - ctx.drawImage(paddedCanvas, 0, 0); - - // extract and set padding data - if (srcCanvas.height > srcCanvas.width) { - // horizontal padding, x offset - const xOffset = (canvasSize - srcCanvas.width) / 2; - ctx?.clearRect(xOffset, 0, srcCanvas.width, srcCanvas.height); - ctx.drawImage(srcCanvas, xOffset, 0, srcCanvas.width, srcCanvas.height); - } else { - // vertical padding, y offset - const yOffset = (canvasSize - srcCanvas.height) / 2; - ctx?.clearRect(0, yOffset, srcCanvas.width, srcCanvas.height); - ctx.drawImage(srcCanvas, 0, yOffset, srcCanvas.width, srcCanvas.height); - } - return canvas; - }; - - // Fills in the blank areas of the image with an image reflection (to fill in a square-shaped canvas) - static drawHorizontalReflection = (ctx: CanvasRenderingContext2D, canvas: HTMLCanvasElement, xOffset: number) => { - const imageData = ctx.getImageData(0, 0, canvas.width, canvas.height); - const { data } = imageData; - for (let i = 0; i < canvas.height; i++) { - for (let j = 0; j < xOffset; j++) { - const targetIdx = 4 * (i * canvas.width + j); - const sourceI = i; - const sourceJ = xOffset + (xOffset - j); - const sourceIdx = 4 * (sourceI * canvas.width + sourceJ); - data[targetIdx] = data[sourceIdx]; - data[targetIdx + 1] = data[sourceIdx + 1]; - data[targetIdx + 2] = data[sourceIdx + 2]; - } - } - for (let i = 0; i < canvas.height; i++) { - for (let j = canvas.width - 1; j >= canvas.width - 1 - xOffset; j--) { - const targetIdx = 4 * (i * canvas.width + j); - const sourceI = i; - const sourceJ = canvas.width - 1 - xOffset - (xOffset - (canvas.width - j)); - const sourceIdx = 4 * (sourceI * canvas.width + sourceJ); - data[targetIdx] = data[sourceIdx]; - data[targetIdx + 1] = data[sourceIdx + 1]; - data[targetIdx + 2] = data[sourceIdx + 2]; - } - } - ctx.putImageData(imageData, 0, 0); - }; - - // Fills in the blank areas of the image with an image reflection (to fill in a square-shaped canvas) - static drawVerticalReflection = (ctx: CanvasRenderingContext2D, canvas: HTMLCanvasElement, yOffset: number) => { - const imageData = ctx.getImageData(0, 0, canvas.width, canvas.height); - const { data } = imageData; - for (let j = 0; j < canvas.width; j++) { - for (let i = 0; i < yOffset; i++) { - const targetIdx = 4 * (i * canvas.width + j); - const sourceJ = j; - const sourceI = yOffset + (yOffset - i); - const sourceIdx = 4 * (sourceI * canvas.width + sourceJ); - data[targetIdx] = data[sourceIdx]; - data[targetIdx + 1] = data[sourceIdx + 1]; - data[targetIdx + 2] = data[sourceIdx + 2]; - } - } - for (let j = 0; j < canvas.width; j++) { - for (let i = canvas.height - 1; i >= canvas.height - 1 - yOffset; i--) { - const targetIdx = 4 * (i * canvas.width + j); - const sourceJ = j; - const sourceI = canvas.height - 1 - yOffset - (yOffset - (canvas.height - i)); - const sourceIdx = 4 * (sourceI * canvas.width + sourceJ); - data[targetIdx] = data[sourceIdx]; - data[targetIdx + 1] = data[sourceIdx + 1]; - data[targetIdx + 2] = data[sourceIdx + 2]; - } - } - ctx.putImageData(imageData, 0, 0); - }; - - // Gets the unaltered (besides filling in padding) version of the image for the api call - static getCanvasImg = (img: HTMLImageElement): HTMLCanvasElement | undefined => { - const canvas = document.createElement('canvas'); - canvas.width = canvasSize; - canvas.height = canvasSize; - const ctx = canvas.getContext('2d'); - if (!ctx) return undefined; - // fix scaling - const scale = Math.min(canvasSize / img.width, canvasSize / img.height); - const width = Math.floor(img.width * scale); - const height = Math.floor(img.height * scale); - ctx?.clearRect(0, 0, canvasSize, canvasSize); - ctx.fillStyle = bgColor; - ctx.fillRect(0, 0, canvasSize, canvasSize); - - // extract and set padding data - if (img.naturalHeight > img.naturalWidth) { - // horizontal padding, x offset - const xOffset = Math.floor((canvasSize - width) / 2); - ctx.drawImage(img, xOffset, 0, width, height); - - // draw reflected image padding - this.drawHorizontalReflection(ctx, canvas, xOffset); - } else { - // vertical padding, y offset - const yOffset = Math.floor((canvasSize - height) / 2); - ctx.drawImage(img, 0, yOffset, width, height); - - // draw reflected image padding - this.drawVerticalReflection(ctx, canvas, yOffset); - } - return canvas; - }; - - /** - * Converts a url to base64 (tainted canvas workaround) - */ - static urlToBase64 = async (imageUrl: string): Promise<string | undefined> => { - try { - const res = await fetch(imageUrl); - const blob = await res.blob(); - - return new Promise<string>((resolve, reject) => { - const reader = new FileReader(); - reader.onload = () => { - const base64Data = reader.result?.toString().split(',')[1]; - if (base64Data) { - resolve(base64Data); - } else { - reject(new Error('Failed to convert.')); - } - }; - reader.onerror = () => { - reject(new Error('Error reading image data')); - }; - reader.readAsDataURL(blob); - }); - } catch (err) { - console.error(err); - } - return undefined; - }; -} |