From b6537cce6aa34eb33c052d7ec2cbbf804be08fba Mon Sep 17 00:00:00 2001 From: Sophie Zhang Date: Wed, 28 Jun 2023 11:47:41 -0400 Subject: added files --- .../generativeFillUtils/ImageHandler.ts | 146 +++++++++++++++++++++ 1 file changed, 146 insertions(+) create mode 100644 src/client/views/nodes/generativeFill/generativeFillUtils/ImageHandler.ts (limited to 'src/client/views/nodes/generativeFill/generativeFillUtils/ImageHandler.ts') diff --git a/src/client/views/nodes/generativeFill/generativeFillUtils/ImageHandler.ts b/src/client/views/nodes/generativeFill/generativeFillUtils/ImageHandler.ts new file mode 100644 index 000000000..1c726afbb --- /dev/null +++ b/src/client/views/nodes/generativeFill/generativeFillUtils/ImageHandler.ts @@ -0,0 +1,146 @@ +import { RefObject } from "react"; +import { OPENAI_KEY } from "../keys"; +import { canvasSize } from "./generativeFillConstants"; + +export interface APISuccess { + status: "success"; + urls: string[]; +} + +export interface APIError { + status: "error"; + message: string; +} + +export class ImageUtility { + static canvasToBlob = (canvas: HTMLCanvasElement): Promise => { + return new Promise((resolve) => { + canvas.toBlob((blob) => { + if (blob) { + resolve(blob); + } + }, "image/png"); + }); + }; + + static getEdit = async ( + imgBlob: Blob, + maskBlob: Blob, + prompt: string, + n?: number + ): Promise => { + const apiUrl = "https://api.openai.com/v1/images/edits"; + const fd = new FormData(); + fd.append("image", imgBlob, "image.png"); + fd.append("mask", maskBlob, "mask.png"); + fd.append("prompt", prompt); + fd.append("size", "1024x1024"); + fd.append("n", n ? JSON.stringify(n) : "1"); + fd.append("response_format", "b64_json"); + + try { + const res = await fetch(apiUrl, { + method: "POST", + headers: { + Authorization: `Bearer ${OPENAI_KEY}`, + }, + body: fd, + }); + const data = await res.json(); + console.log(data.data); + return { + status: "success", + urls: (data.data as { b64_json: string }[]).map( + (data) => `data:image/png;base64,${data.b64_json}` + ), + }; + } catch (err) { + console.log(err); + return { status: "error", message: "API error." }; + } + }; + + static mockGetEdit = async (): Promise => { + return { + status: "success", + urls: [ + "/assets/shiba.png", + "/assets/souffle-dalle.png", + "/assets/firefly.png", + ], + }; + }; + + static getCanvasContext = ( + canvasRef: RefObject + ): CanvasRenderingContext2D | null => { + if (!canvasRef.current) return null; + const ctx = canvasRef.current.getContext("2d"); + if (!ctx) return null; + return ctx; + }; + + static downloadCanvas = (canvas: HTMLCanvasElement) => { + const url = canvas.toDataURL(); + const downloadLink = document.createElement("a"); + downloadLink.href = url; + downloadLink.download = "canvas"; + + downloadLink.click(); + downloadLink.remove(); + }; + + static downloadImageCanvas = (imgUrl: string) => { + const img = new Image(); + img.src = imgUrl; + img.onload = () => { + const canvas = document.createElement("canvas"); + canvas.width = canvasSize; + canvas.height = canvasSize; + const ctx = canvas.getContext("2d"); + ctx?.drawImage(img, 0, 0, canvasSize, canvasSize); + + this.downloadCanvas(canvas); + }; + }; + + static drawImgToCanvas = ( + img: HTMLImageElement, + canvasRef: React.RefObject, + loaded?: boolean + ) => { + if (loaded) { + const ctx = this.getCanvasContext(canvasRef); + if (!ctx) return; + ctx.globalCompositeOperation = "source-over"; + const scale = Math.min(canvasSize / img.width, canvasSize / img.height); + const width = img.width * scale; + const height = img.height * scale; + ctx.clearRect(0, 0, canvasSize, canvasSize); + ctx.drawImage(img, 0, 0, width, height); + } else { + img.onload = () => { + const ctx = this.getCanvasContext(canvasRef); + if (!ctx) return; + ctx.globalCompositeOperation = "source-over"; + const scale = Math.min(canvasSize / img.width, canvasSize / img.height); + const width = img.width * scale; + const height = img.height * scale; + ctx.clearRect(0, 0, canvasSize, canvasSize); + ctx.drawImage(img, 0, 0, width, height); + }; + } + }; + + // The image must be loaded! + static getCanvasImg = (img: HTMLImageElement): HTMLCanvasElement => { + const canvas = document.createElement("canvas"); + canvas.width = canvasSize; + canvas.height = canvasSize; + const ctx = canvas.getContext("2d"); + ctx?.clearRect(0, 0, canvasSize, canvasSize); + ctx?.drawImage(img, 0, 0, canvasSize, canvasSize); + + return canvas; + }; +} -- cgit v1.2.3-70-g09d2 From 932ecd6092bd1b0ac3391309a550bac76cfb0e04 Mon Sep 17 00:00:00 2001 From: Sophie Zhang Date: Thu, 29 Jun 2023 13:56:41 -0400 Subject: undo and redo --- src/client/apis/gpt/GPT.ts | 2 +- src/client/views/MainView.tsx | 11 + src/client/views/nodes/ImageBox.tsx | 11 + .../views/nodes/formattedText/FormattedTextBox.tsx | 1 + .../views/nodes/generativeFill/GenerativeFill.scss | 4 +- .../views/nodes/generativeFill/GenerativeFill.tsx | 157 +++++++++++--- .../nodes/generativeFill/GenerativeFillButtons.tsx | 36 ++-- .../generativeFillUtils/ImageHandler.ts | 230 ++++++++++----------- .../generativeFillUtils/generativeFillConstants.ts | 5 +- 9 files changed, 273 insertions(+), 184 deletions(-) (limited to 'src/client/views/nodes/generativeFill/generativeFillUtils/ImageHandler.ts') diff --git a/src/client/apis/gpt/GPT.ts b/src/client/apis/gpt/GPT.ts index 4b3960902..18222b32a 100644 --- a/src/client/apis/gpt/GPT.ts +++ b/src/client/apis/gpt/GPT.ts @@ -39,7 +39,7 @@ const gptAPICall = async (inputText: string, callType: GPTCallType) => { temperature: opts.temp, prompt: `${opts.prompt}${inputText}`, }); - console.log(response.data.choices[0]); + // console.log(response.data.choices[0]); return response.data.choices[0].text; } catch (err) { console.log(err); diff --git a/src/client/views/MainView.tsx b/src/client/views/MainView.tsx index c0c473cfc..1fa1eff84 100644 --- a/src/client/views/MainView.tsx +++ b/src/client/views/MainView.tsx @@ -65,6 +65,7 @@ import { PreviewCursor } from './PreviewCursor'; import { PropertiesView } from './PropertiesView'; import { DashboardStyleProvider, DefaultStyleProvider } from './StyleProvider'; import { TopBar } from './topbar/TopBar'; +import GenerativeFill from './nodes/generativeFill/GenerativeFill'; const _global = (window /* browser */ || global) /* node */ as any; @observer @@ -72,6 +73,15 @@ export class MainView extends React.Component { public static Instance: MainView; public static Live: boolean = false; private _docBtnRef = React.createRef(); + // for ai image editor + @observable public imageEditorOpen: boolean = false; + @action public setImageEditorOpen = (open: boolean) => (this.imageEditorOpen = open); + @observable public imageEditorSource: string = ''; + @action public setImageEditorSource = (source: string) => (this.imageEditorSource = source); + @observable public imageRootDoc: Doc | undefined; + @action public setImageRootDoc = (doc: Doc) => (this.imageRootDoc = doc); + @observable public addDoc: ((doc: Doc | Doc[], annotationKey?: string) => boolean) | undefined; + @observable public LastButton: Opt; @observable private _windowWidth: number = 0; @observable private _windowHeight: number = 0; @@ -1014,6 +1024,7 @@ export class MainView extends React.Component { {this.snapLines} + ); } diff --git a/src/client/views/nodes/ImageBox.tsx b/src/client/views/nodes/ImageBox.tsx index 5b302e7ce..2c6f92681 100644 --- a/src/client/views/nodes/ImageBox.tsx +++ b/src/client/views/nodes/ImageBox.tsx @@ -38,6 +38,7 @@ import './ImageBox.scss'; import { PinProps, PresBox } from './trails'; import React = require('react'); import Color = require('color'); +import { MainView } from '../MainView'; export const pageSchema = createSchema({ googlePhotosUrl: 'string', @@ -248,6 +249,16 @@ export class ImageBox extends ViewBoxAnnotatableComponent Utils.CopyText(this.choosePath(field.url)), icon: 'copy' }); + funcs.push({ + description: 'Open Image Editor', + event: action(() => { + MainView.Instance.setImageEditorOpen(true); + MainView.Instance.setImageEditorSource(this.choosePath(field.url)); + MainView.Instance.addDoc = this.props.addDocument; + MainView.Instance.imageRootDoc = this.rootDoc; + }), + icon: 'pencil-alt', + }); if (!Doc.noviceMode) { funcs.push({ description: 'Export to Google Photos', event: () => GooglePhotos.Transactions.UploadImages([this.props.Document]), icon: 'caret-square-right' }); diff --git a/src/client/views/nodes/formattedText/FormattedTextBox.tsx b/src/client/views/nodes/formattedText/FormattedTextBox.tsx index 0dc186eaf..3fa5c099b 100644 --- a/src/client/views/nodes/formattedText/FormattedTextBox.tsx +++ b/src/client/views/nodes/formattedText/FormattedTextBox.tsx @@ -882,6 +882,7 @@ export class FormattedTextBox extends ViewBoxAnnotatableComponent { + console.log(this.gptRes); if (resIndex < this.gptRes.length) { this.dataDoc.text = (this.dataDoc.text as RichTextField)?.Text + this.gptRes[resIndex]; setTimeout(() => { diff --git a/src/client/views/nodes/generativeFill/GenerativeFill.scss b/src/client/views/nodes/generativeFill/GenerativeFill.scss index 92406ba9d..b1e570cf1 100644 --- a/src/client/views/nodes/generativeFill/GenerativeFill.scss +++ b/src/client/views/nodes/generativeFill/GenerativeFill.scss @@ -6,14 +6,14 @@ $scale: 0.5; position: absolute; top: 0; left: 0; - z-index: 999; + z-index: 9999; height: 100vh; width: 100vw; display: flex; flex-direction: column; overflow: hidden; - .controls { + .generativeFillControls { flex-shrink: 0; height: $navHeight; background-color: #ffffff; diff --git a/src/client/views/nodes/generativeFill/GenerativeFill.tsx b/src/client/views/nodes/generativeFill/GenerativeFill.tsx index 6dd80a5d1..0df05f4d7 100644 --- a/src/client/views/nodes/generativeFill/GenerativeFill.tsx +++ b/src/client/views/nodes/generativeFill/GenerativeFill.tsx @@ -3,14 +3,21 @@ import { APISuccess, ImageUtility } from './generativeFillUtils/ImageHandler'; import { BrushHandler } from './generativeFillUtils/BrushHandler'; import { IconButton, TextField } from '@mui/material'; import { CursorData, Point } from './generativeFillUtils/generativeFillInterfaces'; -import { activeColor, canvasSize, eraserColor } from './generativeFillUtils/generativeFillConstants'; +import { activeColor, canvasSize, eraserColor, freeformRenderSize } from './generativeFillUtils/generativeFillConstants'; import { PointerHandler } from './generativeFillUtils/PointerHandler'; -import { BsBrush, BsEraser } from 'react-icons/bs'; +import { BsBrush, BsEraser, BsX } from 'react-icons/bs'; import { AiOutlineUpload } from 'react-icons/ai'; import { CiUndo, CiRedo } from 'react-icons/ci'; import Buttons from './GenerativeFillButtons'; -import React from 'react'; +import React = require('react'); import './GenerativeFill.scss'; +import { EditableText } from 'browndash-components'; +import { MainView } from '../../MainView'; +import { Doc } from '../../../../fields/Doc'; +import { Networking } from '../../../Network'; +import { Utils } from '../../../../Utils'; +import { DocUtils, Docs } from '../../../documents/Documents'; +import { NumCast } from '../../../../fields/Types'; /** * For images not 1024x1024 fill in the rest in solid black, or a @@ -33,7 +40,14 @@ interface ImageEdit { children: ImageEdit[]; } -const GenerativeFill = () => { +interface GenerativeFillProps { + imageEditorOpen: boolean; + imageEditorSource: string; + imageRootDoc: Doc | undefined; + addDoc: ((doc: Doc | Doc[], annotationKey?: string) => boolean) | undefined; +} + +const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addDoc }: GenerativeFillProps) => { const canvasRef = useRef(null); const canvasBackgroundRef = useRef(null); const drawingAreaRef = useRef(null); @@ -51,8 +65,49 @@ const GenerativeFill = () => { const [loading, setLoading] = useState(false); // used to store the current image loaded to the main canvas const currImg = useRef(null); - // ref to store history - const undoStack = useRef([]); + const originalImg = useRef(null); + // stores history of data urls + const undoStack = useRef([]); + // stores redo stack + const redoStack = useRef([]); + + // Undo and Redo + const handleUndo = () => { + const ctx = ImageUtility.getCanvasContext(canvasRef); + if (!ctx || !currImg.current || !canvasRef.current) return; + + const target = undoStack.current[undoStack.current.length - 1]; + if (!target) { + ImageUtility.drawImgToCanvas(currImg.current, canvasRef); + } else { + redoStack.current = [...redoStack.current, canvasRef.current.toDataURL()]; + const img = new Image(); + img.src = target; + ImageUtility.drawImgToCanvas(img, canvasRef); + undoStack.current = undoStack.current.slice(0, -1); + } + }; + + const handleRedo = () => { + // TODO: handle undo as well + const target = redoStack.current[redoStack.current.length - 1]; + if (!target) { + } else { + const img = new Image(); + img.src = target; + ImageUtility.drawImgToCanvas(img, canvasRef); + redoStack.current = redoStack.current.slice(0, -1); + } + }; + + const handleReset = () => { + if (!canvasRef.current || !currImg.current) return; + const ctx = ImageUtility.getCanvasContext(canvasRef); + if (!ctx) return; + ctx.clearRect(0, 0, canvasSize, canvasSize); + undoStack.current = []; + ImageUtility.drawImgToCanvas(currImg.current, canvasRef, true); + }; // initiate brushing const handlePointerDown = (e: React.PointerEvent) => { @@ -61,6 +116,9 @@ const GenerativeFill = () => { const ctx = ImageUtility.getCanvasContext(canvasRef); if (!ctx) return; + undoStack.current = [...undoStack.current, canvasRef.current.toDataURL()]; + redoStack.current = []; + setIsBrushing(true); const { x, y } = PointerHandler.getPointRelativeToElement(canvas, e, canvasScale); @@ -98,17 +156,13 @@ const GenerativeFill = () => { // first load useEffect(() => { + if (!imageEditorSource || imageEditorSource === '') return; const img = new Image(); - img.src = '/assets/art.jpeg'; + img.src = imageEditorSource; ImageUtility.drawImgToCanvas(img, canvasRef); currImg.current = img; - }, [canvasRef]); - - useEffect(() => { - if (!canvasBackgroundRef.current) return; - const ctx = ImageUtility.getCanvasContext(canvasBackgroundRef); - if (!ctx) return; - }, [canvasBackgroundRef]); + originalImg.current = img; + }, [canvasRef, imageEditorSource]); // handles brush sizing useEffect(() => { @@ -180,16 +234,9 @@ const GenerativeFill = () => { const maskBlob = await ImageUtility.canvasToBlob(canvas); const imgBlob = await ImageUtility.canvasToBlob(ImageUtility.getCanvasImg(img)); - // const res = await ImageUtility.getEdit( - // imgBlob, - // maskBlob, - // input !== "" - // ? input + " in the same style" - // : "Fill in the image in the same style", - // 1 - // ); + const res = await ImageUtility.getEdit(imgBlob, maskBlob, input !== '' ? input + ' in the same style' : 'Fill in the image in the same style', 2); - const res = await ImageUtility.mockGetEdit(); + // const res = await ImageUtility.mockGetEdit(); const { urls } = res as APISuccess; const image = new Image(); image.src = urls[0]; @@ -202,11 +249,47 @@ const GenerativeFill = () => { } }; + const onSave = async () => { + if (!currImg.current || !imageRootDoc) return; + try { + const src = currImg.current.src; + console.log(src); + const [result] = await Networking.PostToServer('/uploadRemoteImage', { sources: [src] }); + const source = Utils.prepend(result.accessPaths.agnostic.client); + console.log(source); + const newImg = Docs.Create.ImageDocument(source, { + x: NumCast(imageRootDoc.x) + NumCast(imageRootDoc._width) + 20, + y: NumCast(imageRootDoc.y), + _height: freeformRenderSize, + _width: freeformRenderSize, + data_nativeWidth: result.nativeWidth, + data_nativeHeight: result.nativeHeight, + }); + + addDoc?.(newImg); + // Create link between prompt and image + DocUtils.MakeLink(imageRootDoc, newImg, { link_relationship: 'Image Edit' }); + console.log('done'); + } catch (err) { + console.log(err); + } + }; + return ( -
-
+
+

Generative Fill

- +
+ + { + MainView.Instance.setImageEditorOpen(false); + MainView.Instance.setImageEditorSource(''); + setEdits([]); + }}> + + +
{/* Main canvas for editing */}
{ }}> - { setBrushStyle(BrushStyle.SUBTRACT); }}> - + */} {/* Undo and Redo */} - {/* { e.stopPropagation(); - console.log(undoStack.current); + handleUndo(); }} onPointerUp={e => { e.stopPropagation(); }}> - {}}> + { + e.stopPropagation(); + handleRedo(); + }} + onPointerUp={e => { + e.stopPropagation(); + }}> - */} +
{/* Edits box */}
@@ -292,12 +382,15 @@ const GenerativeFill = () => { type="text" label="Prompt" placeholder="Prompt..." + InputLabelProps={{ style: { fontSize: '1.5rem' } }} + inputProps={{ style: { fontSize: '1.5rem' } }} sx={{ backgroundColor: '#ffffff', position: 'absolute', bottom: '1rem', transform: 'translateX(calc(50vw - 50%))', width: 'calc(100vw - 4rem)', + scale: 1.2, }} />
diff --git a/src/client/views/nodes/generativeFill/GenerativeFillButtons.tsx b/src/client/views/nodes/generativeFill/GenerativeFillButtons.tsx index 348e27a16..e8cb61ab5 100644 --- a/src/client/views/nodes/generativeFill/GenerativeFillButtons.tsx +++ b/src/client/views/nodes/generativeFill/GenerativeFillButtons.tsx @@ -3,44 +3,34 @@ import { ImageUtility } from './generativeFillUtils/ImageHandler'; import { canvasSize } from './generativeFillUtils/generativeFillConstants'; import { Oval } from 'react-loader-spinner'; import './GenerativeFillButtons.scss'; -import React from 'react'; +import React = require('react'); +import { Doc } from '../../../../fields/Doc'; interface ButtonContainerProps { canvasRef: React.RefObject; - backgroundref: React.RefObject; currImg: React.MutableRefObject; - undoStack: React.MutableRefObject; getEdit: () => Promise; loading: boolean; + onSave: () => Promise; + onReset: () => void; } -const Buttons = ({ canvasRef, backgroundref, currImg, undoStack, loading, getEdit }: ButtonContainerProps) => { - const handleReset = () => { - if (!canvasRef.current || !currImg.current) return; - const ctx = ImageUtility.getCanvasContext(canvasRef); - if (!ctx) return; - ctx.clearRect(0, 0, canvasSize, canvasSize); - ImageUtility.drawImgToCanvas(currImg.current, canvasRef, true); +const Buttons = ({ canvasRef, currImg, loading, getEdit, onSave, onReset }: ButtonContainerProps) => { + const handleSave = () => { + onSave(); }; return (
- - + + {/* - {/* */} + Download Original + */}
diff --git a/src/client/views/nodes/generativeFill/generativeFillUtils/ImageHandler.ts b/src/client/views/nodes/generativeFill/generativeFillUtils/ImageHandler.ts index 91f12f866..48055903c 100644 --- a/src/client/views/nodes/generativeFill/generativeFillUtils/ImageHandler.ts +++ b/src/client/views/nodes/generativeFill/generativeFillUtils/ImageHandler.ts @@ -52,10 +52,10 @@ export class ImageUtility { } }; - static mockGetEdit = async (): Promise => { + static mockGetEdit = async (mockSrc: string): Promise => { return { status: 'success', - urls: ['/assets/shiba.png', '/assets/souffle-dalle.png', '/assets/firefly.png'], + urls: [mockSrc, mockSrc, mockSrc], }; }; -- cgit v1.2.3-70-g09d2 From 314f62bb5335e3fb3f25501823d41cf0cdc53cac Mon Sep 17 00:00:00 2001 From: Sophie Zhang Date: Sun, 13 Aug 2023 17:02:58 -0400 Subject: update --- .../collectionFreeForm/CollectionFreeFormView.tsx | 18 +++- .../views/nodes/formattedText/FormattedTextBox.tsx | 15 +-- .../views/nodes/generativeFill/GenerativeFill.scss | 1 + .../views/nodes/generativeFill/GenerativeFill.tsx | 103 +++++++++--------- .../generativeFillUtils/BrushHandler.ts | 116 +++++++-------------- .../generativeFillUtils/ImageHandler.ts | 2 +- .../generativeFillUtils/generativeFillConstants.ts | 4 +- src/client/views/pdf/GPTPopup/GPTPopup.tsx | 4 +- 8 files changed, 118 insertions(+), 145 deletions(-) (limited to 'src/client/views/nodes/generativeFill/generativeFillUtils/ImageHandler.ts') diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx index ba31916a7..34f707b30 100644 --- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx +++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx @@ -237,7 +237,7 @@ export class CollectionFreeFormView extends CollectionSubView this.props.childClickScript || ScriptCast(this.Document.onChildClick); onChildDoubleClickHandler = () => this.props.childDoubleClickScript || ScriptCast(this.Document.onChildDoubleClick); elementFunc = () => this._layoutElements; - shrinkWrap = () => { + fitContentOnce = () => { if (this.props.DocumentView?.().nativeWidth) return; const vals = this.fitToContentVals; this.layoutDoc._freeform_panX = vals.bounds.cx; @@ -1592,6 +1592,14 @@ export class CollectionFreeFormView extends CollectionSubView (this._layoutElements = elements || []), { fireImmediately: true, name: 'doLayout' } ); + + this._disposers.fitContent = reaction( + () => this.rootDoc.fitContentOnce, + fitContentOnce => { + if (fitContentOnce) this.fitContentOnce(); + }, + { fireImmediately: true, name: 'fitContent' } + ); }) ); } @@ -1782,6 +1790,14 @@ export class CollectionFreeFormView extends CollectionSubView (this._showAnimTimeline = !this._showAnimTimeline)), icon: 'eye' }); this.props.renderDepth && optionItems.push({ description: 'Use Background Color as Default', event: () => (Cast(Doc.UserDoc().emptyCollection, Doc, null)._backgroundColor = StrCast(this.layoutDoc._backgroundColor)), icon: 'palette' }); + this.props.renderDepth && + optionItems.push({ + description: 'Fit Content Once', + event: () => { + this.fitContentOnce(); + }, + icon: 'object-group', + }); if (!Doc.noviceMode) { optionItems.push({ description: (!Doc.NativeWidth(this.layoutDoc) || !Doc.NativeHeight(this.layoutDoc) ? 'Freeze' : 'Unfreeze') + ' Aspect', event: this.toggleNativeDimensions, icon: 'snowflake' }); } diff --git a/src/client/views/nodes/formattedText/FormattedTextBox.tsx b/src/client/views/nodes/formattedText/FormattedTextBox.tsx index 9f4483e8d..072977150 100644 --- a/src/client/views/nodes/formattedText/FormattedTextBox.tsx +++ b/src/client/views/nodes/formattedText/FormattedTextBox.tsx @@ -937,6 +937,7 @@ export class FormattedTextBox extends ViewBoxAnnotatableComponent { console.log('Generate image from text: ', (this.dataDoc.text as RichTextField)?.Text); + GPTPopup.Instance?.setTextAnchor(this.getAnchor(false)); GPTPopup.Instance?.setImgTargetDoc(this.rootDoc); GPTPopup.Instance.setImgUrls([]); GPTPopup.Instance.setMode(GPTPopupMode.IMAGE); @@ -1270,13 +1271,13 @@ export class FormattedTextBox extends ViewBoxAnnotatableComponent([0, 0]); // which urls were already saved to canvas - const savedSrcs = useRef([]); + const savedSrcs = useRef>(new Set()); // references to keep track of tree structure const newCollectionRef = useRef(null); const parentDoc = useRef(null); const childrenDocs = useRef([]); + const addToExistingCollection = useRef(false); // Undo and Redo const handleUndo = () => { @@ -250,18 +248,6 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD })); }; - // File upload - const uploadImg = (e: React.ChangeEvent) => { - if (e.target.files) { - const file = e.target.files[0]; - const image = new Image(); - const imgUrl = URL.createObjectURL(file); - image.src = imgUrl; - ImageUtility.drawImgToCanvas(image, canvasRef); - currImg.current = image; - } - }; - // Get AI Edit const getEdit = async () => { const img = currImg.current; @@ -280,6 +266,8 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD // create first image if (!newCollectionRef.current) { + if (addToExistingCollection.current) { + } if (!(originalImg.current && imageRootDoc)) return; console.log('creating first image'); // create new collection and add it to the view @@ -292,7 +280,7 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD }); DocUtils.MakeLink(imageRootDoc, newCollectionRef.current, { link_relationship: 'Image Edit Version History', link_displayLine: false }); // add the doc to the main freeform - addDoc?.(newCollectionRef.current); + // addDoc?.(newCollectionRef.current); await createNewImgDoc(originalImg.current, true); } else { parentDoc.current = childrenDocs.current[childrenDocs.current.length - 1]; @@ -321,7 +309,6 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD if (!parentDoc.current) return; const startY = NumCast(parentDoc.current.y); const len = childrenDocs.current.length; - console.log(len); let initialYPositions: number[] = []; for (let i = 0; i < len; i++) { initialYPositions.push(startY + i * offsetDistanceY); @@ -398,9 +385,10 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD }; const handleViewClose = () => { - // if (newCollectionRef.current) { - // CollectionDockingView.AddSplit(newCollectionRef.current, OpenWhereMod.right); - // } + if (newCollectionRef.current) { + newCollectionRef.current.fitContentOnce = true; + CollectionDockingView.AddSplit(newCollectionRef.current, OpenWhereMod.right); + } MainView.Instance.setImageEditorOpen(false); MainView.Instance.setImageEditorSource(''); setEdits([]); @@ -409,12 +397,17 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD return (
-

Generative Fill

+

AI Image Editor

+ {saveLoading && ( + + Saving image... + + )}
{/* Main canvas for editing */} @@ -439,20 +432,11 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD
{/* Icons */}
- - { - if (fileRef.current) { - fileRef.current.click(); - } - }}> - - { setBrushStyle(BrushStyle.ADD); }}> - + {/* Undo and Redo */} + + { + setCursorData(prev => ({ ...prev, width: val as number })); + }} + /> +
{/* Edits thumbnails*/}
@@ -484,12 +490,14 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD width={100} height={100} src={edit} - onClick={() => { + onClick={async () => { + // if (savedSrcs.current.has(edit)) return; const img = new Image(); img.src = edit; ImageUtility.drawImgToCanvas(img, canvasRef); currImg.current = img; - onSave(); + savedSrcs.current.add(edit); + await onSave(); }} /> ))} @@ -521,14 +529,6 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD currImg.current = img; }} /> -
- {saveLoading && } -
)}
@@ -541,15 +541,14 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD type="text" label="Prompt" placeholder="Prompt..." - InputLabelProps={{ style: { fontSize: '1.5rem' } }} - inputProps={{ style: { fontSize: '1.5rem' } }} + InputLabelProps={{ style: { fontSize: '16px' } }} + inputProps={{ style: { fontSize: '16px' } }} sx={{ backgroundColor: '#ffffff', position: 'absolute', - bottom: '1rem', + bottom: '16px', transform: 'translateX(calc(50vw - 50%))', - width: 'calc(100vw - 4rem)', - scale: 1.2, + width: 'calc(100vw - 64px)', }} />
diff --git a/src/client/views/nodes/generativeFill/generativeFillUtils/BrushHandler.ts b/src/client/views/nodes/generativeFill/generativeFillUtils/BrushHandler.ts index c2716e083..f84f04190 100644 --- a/src/client/views/nodes/generativeFill/generativeFillUtils/BrushHandler.ts +++ b/src/client/views/nodes/generativeFill/generativeFillUtils/BrushHandler.ts @@ -1,87 +1,45 @@ -import { GenerativeFillMathHelpers } from "./GenerativeFillMathHelpers"; -import { eraserColor } from "./generativeFillConstants"; -import { Point } from "./generativeFillInterfaces"; +import { GenerativeFillMathHelpers } from './GenerativeFillMathHelpers'; +import { eraserColor } from './generativeFillConstants'; +import { Point } from './generativeFillInterfaces'; export class BrushHandler { - static brushCircle = ( - x: number, - y: number, - brushRadius: number, - ctx: CanvasRenderingContext2D - ) => { - ctx.globalCompositeOperation = "destination-out"; - ctx.shadowColor = "#ffffffeb"; - ctx.shadowBlur = 5; - ctx.beginPath(); - ctx.arc(x, y, brushRadius, 0, 2 * Math.PI); - ctx.fill(); - ctx.closePath(); - }; + static brushCircle = (x: number, y: number, brushRadius: number, ctx: CanvasRenderingContext2D) => { + ctx.globalCompositeOperation = 'destination-out'; + ctx.shadowColor = '#ffffffeb'; + ctx.shadowBlur = 5; + ctx.beginPath(); + ctx.arc(x, y, brushRadius, 0, 2 * Math.PI); + ctx.fill(); + ctx.closePath(); + }; - static brushCircleOverlay = ( - x: number, - y: number, - brushRadius: number, - ctx: CanvasRenderingContext2D, - fillColor: string, - erase: boolean - ) => { - ctx.globalCompositeOperation = "destination-out"; - // ctx.globalCompositeOperation = erase ? "destination-out" : "source-over"; - ctx.fillStyle = fillColor; - ctx.shadowColor = eraserColor; - ctx.shadowBlur = 5; - ctx.beginPath(); - ctx.arc(x, y, brushRadius, 0, 2 * Math.PI); - ctx.fill(); - ctx.closePath(); - }; + static brushCircleOverlay = (x: number, y: number, brushRadius: number, ctx: CanvasRenderingContext2D, fillColor: string, erase: boolean) => { + ctx.globalCompositeOperation = 'destination-out'; + // ctx.globalCompositeOperation = erase ? "destination-out" : "source-over"; + ctx.fillStyle = fillColor; + ctx.shadowColor = eraserColor; + ctx.shadowBlur = 5; + ctx.beginPath(); + ctx.arc(x, y, brushRadius, 0, 2 * Math.PI); + ctx.fill(); + ctx.closePath(); + }; - static createBrushPath = ( - startPoint: Point, - endPoint: Point, - brushRadius: number, - ctx: CanvasRenderingContext2D - ) => { - const dist = GenerativeFillMathHelpers.distanceBetween( - startPoint, - endPoint - ); + static createBrushPath = (startPoint: Point, endPoint: Point, brushRadius: number, ctx: CanvasRenderingContext2D) => { + const dist = GenerativeFillMathHelpers.distanceBetween(startPoint, endPoint); - for (let i = 0; i < dist; i += 5) { - const s = i / dist; - BrushHandler.brushCircle( - startPoint.x * (1 - s) + endPoint.x * s, - startPoint.y * (1 - s) + endPoint.y * s, - brushRadius, - ctx - ); - } - }; + for (let i = 0; i < dist; i += 5) { + const s = i / dist; + BrushHandler.brushCircle(startPoint.x * (1 - s) + endPoint.x * s, startPoint.y * (1 - s) + endPoint.y * s, brushRadius, ctx); + } + }; - static createBrushPathOverlay = ( - startPoint: Point, - endPoint: Point, - brushRadius: number, - ctx: CanvasRenderingContext2D, - fillColor: string, - erase: boolean - ) => { - const dist = GenerativeFillMathHelpers.distanceBetween( - startPoint, - endPoint - ); + static createBrushPathOverlay = (startPoint: Point, endPoint: Point, brushRadius: number, ctx: CanvasRenderingContext2D, fillColor: string, erase: boolean) => { + const dist = GenerativeFillMathHelpers.distanceBetween(startPoint, endPoint); - for (let i = 0; i < dist; i += 5) { - const s = i / dist; - BrushHandler.brushCircleOverlay( - startPoint.x * (1 - s) + endPoint.x * s, - startPoint.y * (1 - s) + endPoint.y * s, - brushRadius, - ctx, - fillColor, - erase - ); - } - }; + for (let i = 0; i < dist; i += 5) { + const s = i / dist; + BrushHandler.brushCircleOverlay(startPoint.x * (1 - s) + endPoint.x * s, startPoint.y * (1 - s) + endPoint.y * s, brushRadius, ctx, fillColor, erase); + } + }; } diff --git a/src/client/views/nodes/generativeFill/generativeFillUtils/ImageHandler.ts b/src/client/views/nodes/generativeFill/generativeFillUtils/ImageHandler.ts index 48055903c..45cf7196b 100644 --- a/src/client/views/nodes/generativeFill/generativeFillUtils/ImageHandler.ts +++ b/src/client/views/nodes/generativeFill/generativeFillUtils/ImageHandler.ts @@ -28,7 +28,7 @@ export class ImageUtility { fd.append('image', imgBlob, 'image.png'); fd.append('mask', maskBlob, 'mask.png'); fd.append('prompt', prompt); - fd.append('size', '1024x1024'); + fd.append('size', '512x512'); fd.append('n', n ? JSON.stringify(n) : '1'); fd.append('response_format', 'b64_json'); diff --git a/src/client/views/nodes/generativeFill/generativeFillUtils/generativeFillConstants.ts b/src/client/views/nodes/generativeFill/generativeFillUtils/generativeFillConstants.ts index 5a8d33742..412a4d238 100644 --- a/src/client/views/nodes/generativeFill/generativeFillUtils/generativeFillConstants.ts +++ b/src/client/views/nodes/generativeFill/generativeFillUtils/generativeFillConstants.ts @@ -1,9 +1,9 @@ // constants -export const canvasSize = 1024; +export const canvasSize = 512; export const freeformRenderSize = 300; export const offsetDistanceY = freeformRenderSize + 200; export const offsetX = 200; -export const newCollectionSize = 1000; +export const newCollectionSize = 500; export const activeColor = '#1976d2'; export const eraserColor = '#e1e9ec'; diff --git a/src/client/views/pdf/GPTPopup/GPTPopup.tsx b/src/client/views/pdf/GPTPopup/GPTPopup.tsx index aeee90d16..fc6fc1af8 100644 --- a/src/client/views/pdf/GPTPopup/GPTPopup.tsx +++ b/src/client/views/pdf/GPTPopup/GPTPopup.tsx @@ -124,9 +124,7 @@ export class GPTPopup extends React.Component { * Transfers the image urls to actual image docs */ private transferToImage = (source: string) => { - console.log('Text Anchor', this.textAnchor); - console.log('Whole doc anchor', this.imgTargetDoc); - const textAnchor = this.textAnchor ?? this.imgTargetDoc; + const textAnchor = this.imgTargetDoc; if (!textAnchor) return; const newDoc = Docs.Create.ImageDocument(source, { x: NumCast(textAnchor.x) + NumCast(textAnchor._width) + 10, -- cgit v1.2.3-70-g09d2 From 592e1a1b1000157db77bd812b8debfbcc45844f9 Mon Sep 17 00:00:00 2001 From: Sophie Zhang Date: Thu, 17 Aug 2023 17:52:12 -0400 Subject: revert --- src/client/views/nodes/ImageBox.tsx | 4 +- .../views/nodes/generativeFill/GenerativeFill.tsx | 115 ++++++++------------- .../generativeFillUtils/ImageHandler.ts | 2 + 3 files changed, 46 insertions(+), 75 deletions(-) (limited to 'src/client/views/nodes/generativeFill/generativeFillUtils/ImageHandler.ts') diff --git a/src/client/views/nodes/ImageBox.tsx b/src/client/views/nodes/ImageBox.tsx index 352e0fbdc..603994016 100644 --- a/src/client/views/nodes/ImageBox.tsx +++ b/src/client/views/nodes/ImageBox.tsx @@ -252,8 +252,8 @@ export class ImageBox extends ViewBoxAnnotatableComponent { - MainView.Instance.imageEditorOpen = true; - MainView.Instance.imageEditorSource = this.choosePath(field.url); + MainView.Instance.setImageEditorOpen(true); + MainView.Instance.setImageEditorSource(this.choosePath(field.url)); MainView.Instance.addDoc = this.props.addDocument; MainView.Instance.imageRootDoc = this.rootDoc; }), diff --git a/src/client/views/nodes/generativeFill/GenerativeFill.tsx b/src/client/views/nodes/generativeFill/GenerativeFill.tsx index 99068d647..f136982bc 100644 --- a/src/client/views/nodes/generativeFill/GenerativeFill.tsx +++ b/src/client/views/nodes/generativeFill/GenerativeFill.tsx @@ -3,7 +3,7 @@ import React = require('react'); import { useEffect, useRef, useState } from 'react'; import { APISuccess, ImageUtility } from './generativeFillUtils/ImageHandler'; import { BrushHandler } from './generativeFillUtils/BrushHandler'; -import { Box, Checkbox, FormControlLabel, IconButton, Slider, TextField } from '@mui/material'; +import { Box, IconButton, Slider, TextField } from '@mui/material'; import { CursorData, Point } from './generativeFillUtils/generativeFillInterfaces'; import { activeColor, canvasSize, eraserColor, freeformRenderSize, newCollectionSize, offsetDistanceY, offsetX } from './generativeFillUtils/generativeFillConstants'; import { PointerHandler } from './generativeFillUtils/PointerHandler'; @@ -19,7 +19,6 @@ import { Cast, DocCast, NumCast } from '../../../../fields/Types'; import { CollectionDockingView } from '../../collections/CollectionDockingView'; import { OpenWhere, OpenWhereMod } from '../DocumentView'; import { Oval } from 'react-loader-spinner'; -import { CheckBox } from '../../search/CheckBox'; /** * For images not 1024x1024 fill in the rest in solid black, or a @@ -73,7 +72,6 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD const [input, setInput] = useState(''); const [loading, setLoading] = useState(false); const [saveLoading, setSaveLoading] = useState(false); - const [isNewCollection, setIsNewCollection] = useState(false); // the current image in the main canvas const currImg = useRef(null); // the unedited version of each generation (parent) @@ -83,6 +81,9 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD // stores redo stack const redoStack = useRef([]); + // early stage properly, likely will get rid of + const freeformPosition = useRef([0, 0]); + // which urls were already saved to canvas const savedSrcs = useRef>(new Set()); @@ -90,6 +91,7 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD const newCollectionRef = useRef(null); const parentDoc = useRef(null); const childrenDocs = useRef([]); + const addToExistingCollection = useRef(false); // Undo and Redo const handleUndo = () => { @@ -109,13 +111,10 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD }; const handleRedo = () => { - const ctx = ImageUtility.getCanvasContext(canvasRef); - if (!ctx || !currImg.current || !canvasRef.current) return; - + // TODO: handle undo as well const target = redoStack.current[redoStack.current.length - 1]; if (!target) { } else { - undoStack.current = [...undoStack.current, canvasRef.current?.toDataURL()]; const img = new Image(); img.src = target; ImageUtility.drawImgToCanvas(img, canvasRef); @@ -185,10 +184,13 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD console.log('first load'); if (!imageEditorSource || imageEditorSource === '') return; const img = new Image(); + console.log('source', imageEditorSource); img.src = imageEditorSource; + console.log('drawing image'); ImageUtility.drawImgToCanvas(img, canvasRef); currImg.current = img; originalImg.current = img; + freeformPosition.current = [0, 0]; return () => { console.log('cleanup'); @@ -197,6 +199,7 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD childrenDocs.current = []; currImg.current = null; originalImg.current = null; + freeformPosition.current = [0, 0]; undoStack.current = []; redoStack.current = []; }; @@ -265,25 +268,24 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD // create first image if (!newCollectionRef.current) { - if (!isNewCollection) { - // newcollection should stay null - } else { - if (!(originalImg.current && imageRootDoc)) return; - // create new collection and add it to the view - newCollectionRef.current = Docs.Create.FreeformDocument([], { - x: NumCast(imageRootDoc.x) + NumCast(imageRootDoc._width) + offsetX, - y: NumCast(imageRootDoc.y), - _width: newCollectionSize, - _height: newCollectionSize, - title: 'Image edit collection', - }); - DocUtils.MakeLink(imageRootDoc, newCollectionRef.current, { link_relationship: 'Image Edit Version History', link_displayLine: false }); - // add the doc to the main freeform - addDoc?.(newCollectionRef.current); - await createNewImgDoc(originalImg.current, true); + if (addToExistingCollection.current) { } + if (!(originalImg.current && imageRootDoc)) return; + console.log('creating first image'); + // create new collection and add it to the view + newCollectionRef.current = Docs.Create.FreeformDocument([], { + x: NumCast(imageRootDoc.x) + NumCast(imageRootDoc._width) + offsetX, + y: NumCast(imageRootDoc.y), + _width: newCollectionSize, + _height: newCollectionSize, + title: 'Image edit collection', + }); + DocUtils.MakeLink(imageRootDoc, newCollectionRef.current, { link_relationship: 'Image Edit Version History', link_displayLine: false }); + // add the doc to the main freeform + // addDoc?.(newCollectionRef.current); + await createNewImgDoc(originalImg.current, true); } else { - // parentDoc.current = childrenDocs.current[childrenDocs.current.length - 1]; + parentDoc.current = childrenDocs.current[childrenDocs.current.length - 1]; childrenDocs.current = []; } @@ -292,24 +294,12 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD const { urls } = res as APISuccess; const image = new Image(); image.src = urls[0]; - - // need to crop images agh - // save to dash - - // const imageRes = await Promise.all( - // urls.map(async url => { - // const newImg = new Image(); - // newImg.src = url; - // await onSave(newImg); - // }) - // ); - - // map each url to [url, imgDoc] setEdits(urls); - ImageUtility.drawImgToCanvas(image, canvasRef); currImg.current = image; - // onSave(currImg.current); + onSave(); + freeformPosition.current[0] += 1; + freeformPosition.current[1] = 0; } catch (err) { console.log(err); } @@ -336,7 +326,7 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD // creates a new image document and returns its reference const createNewImgDoc = async (img: HTMLImageElement, firstDoc: boolean): Promise => { - if (!imageRootDoc) return; + if (!newCollectionRef.current || !imageRootDoc) return; const src = img.src; const [result] = await Networking.PostToServer('/uploadRemoteImage', { sources: [src] }); const source = Utils.prepend(result.accessPaths.agnostic.client); @@ -344,6 +334,7 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD if (firstDoc) { const x = 0; const initialY = 0; + console.log('first doc'); const newImg = Docs.Create.ImageDocument(source, { x: x, @@ -354,19 +345,16 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD data_nativeHeight: result.nativeHeight, }); - if (isNewCollection && newCollectionRef.current) { - Doc.AddDocToList(newCollectionRef.current, undefined, newImg); - } else { - addDoc?.(newImg); - } - + Doc.AddDocToList(newCollectionRef.current, undefined, newImg); parentDoc.current = newImg; return newImg; } else { if (!parentDoc.current) return; const x = NumCast(parentDoc.current.x) + freeformRenderSize + offsetX; // dummy position + console.log('creating child elements'); const initialY = 0; + const newImg = Docs.Create.ImageDocument(source, { x: x, y: initialY, @@ -377,26 +365,21 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD }); childrenDocs.current.push(newImg); - - if (isNewCollection && newCollectionRef.current) { - Doc.AddDocToList(newCollectionRef.current, undefined, newImg); - } else { - addDoc?.(newImg); - } - DocUtils.MakeLink(parentDoc.current, newImg, { link_relationship: 'Image Edit', link_displayLine: true }); adjustImgPositions(); + + Doc.AddDocToList(newCollectionRef.current, undefined, newImg); return newImg; } }; // need to maybe call on every img click, not just when the save btn is clicked - const onSave = async (img: HTMLImageElement) => { + const onSave = async () => { setSaveLoading(true); - // if (!currImg.current || !originalImg.current || !imageRootDoc) return; + if (!currImg.current || !originalImg.current || !imageRootDoc) return; try { console.log('creating another image'); - await createNewImgDoc(img, false); + await createNewImgDoc(currImg.current, false); } catch (err) { console.log(err); } @@ -406,7 +389,7 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD const handleViewClose = () => { if (newCollectionRef.current) { newCollectionRef.current.fitContentOnce = true; - // CollectionDockingView.AddSplit(newCollectionRef.current, OpenWhereMod.right); + CollectionDockingView.AddSplit(newCollectionRef.current, OpenWhereMod.right); } MainView.Instance.setImageEditorOpen(false); MainView.Instance.setImageEditorSource(''); @@ -418,19 +401,6 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD

AI Image Editor

- { - setIsNewCollection(prev => !prev); - }} - /> - } - label={'Create New Collection'} - labelPlacement="end" - sx={{ whiteSpace: 'nowrap' }} - /> @@ -523,14 +493,13 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD height={100} src={edit} onClick={async () => { + // if (savedSrcs.current.has(edit)) return; const img = new Image(); img.src = edit; ImageUtility.drawImgToCanvas(img, canvasRef); currImg.current = img; savedSrcs.current.add(edit); - undoStack.current = []; - redoStack.current = []; - await onSave(img); + await onSave(); }} /> ))} diff --git a/src/client/views/nodes/generativeFill/generativeFillUtils/ImageHandler.ts b/src/client/views/nodes/generativeFill/generativeFillUtils/ImageHandler.ts index 45cf7196b..1954ab3fb 100644 --- a/src/client/views/nodes/generativeFill/generativeFillUtils/ImageHandler.ts +++ b/src/client/views/nodes/generativeFill/generativeFillUtils/ImageHandler.ts @@ -101,7 +101,9 @@ export class ImageUtility { ctx.clearRect(0, 0, canvasSize, canvasSize); ctx.drawImage(img, 0, 0, width, height); } else { + console.log('loading image'); img.onload = () => { + console.log('loaded'); const ctx = this.getCanvasContext(canvasRef); if (!ctx) return; ctx.globalCompositeOperation = 'source-over'; -- cgit v1.2.3-70-g09d2 From 107f718199a5b0e15f62e5c7e7034b11047ff512 Mon Sep 17 00:00:00 2001 From: Sophie Zhang Date: Thu, 17 Aug 2023 19:11:42 -0400 Subject: baseline working version, different image sizes work --- .../views/nodes/generativeFill/GenerativeFill.tsx | 93 +++++++++------- .../generativeFillUtils/ImageHandler.ts | 119 +++++++++++++++++---- .../generativeFillUtils/generativeFillConstants.ts | 3 +- 3 files changed, 156 insertions(+), 59 deletions(-) (limited to 'src/client/views/nodes/generativeFill/generativeFillUtils/ImageHandler.ts') diff --git a/src/client/views/nodes/generativeFill/GenerativeFill.tsx b/src/client/views/nodes/generativeFill/GenerativeFill.tsx index f136982bc..4d475149d 100644 --- a/src/client/views/nodes/generativeFill/GenerativeFill.tsx +++ b/src/client/views/nodes/generativeFill/GenerativeFill.tsx @@ -42,10 +42,9 @@ enum BrushStyle { MARQUEE, } -interface ImageEdit { - imgElement: HTMLImageElement; - parent: ImageEdit; - children: ImageEdit[]; +interface ImageDimensions { + width: number; + height: number; } interface GenerativeFillProps { @@ -59,7 +58,6 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD const canvasRef = useRef(null); const canvasBackgroundRef = useRef(null); const drawingAreaRef = useRef(null); - const fileRef = useRef(null); const [cursorData, setCursorData] = useState({ x: 0, y: 0, @@ -72,6 +70,10 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD const [input, setInput] = useState(''); const [loading, setLoading] = useState(false); const [saveLoading, setSaveLoading] = useState(false); + const [canvasDims, setCanvasDims] = useState({ + width: canvasSize, + height: canvasSize, + }); // the current image in the main canvas const currImg = useRef(null); // the unedited version of each generation (parent) @@ -91,7 +93,6 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD const newCollectionRef = useRef(null); const parentDoc = useRef(null); const childrenDocs = useRef([]); - const addToExistingCollection = useRef(false); // Undo and Redo const handleUndo = () => { @@ -100,12 +101,12 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD const target = undoStack.current[undoStack.current.length - 1]; if (!target) { - ImageUtility.drawImgToCanvas(currImg.current, canvasRef); + ImageUtility.drawImgToCanvas(currImg.current, canvasRef, canvasDims.width, canvasDims.height); } else { redoStack.current = [...redoStack.current, canvasRef.current.toDataURL()]; const img = new Image(); img.src = target; - ImageUtility.drawImgToCanvas(img, canvasRef); + ImageUtility.drawImgToCanvas(img, canvasRef, canvasDims.width, canvasDims.height); undoStack.current = undoStack.current.slice(0, -1); } }; @@ -117,7 +118,7 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD } else { const img = new Image(); img.src = target; - ImageUtility.drawImgToCanvas(img, canvasRef); + ImageUtility.drawImgToCanvas(img, canvasRef, canvasDims.width, canvasDims.height); redoStack.current = redoStack.current.slice(0, -1); } }; @@ -129,7 +130,7 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD ctx.clearRect(0, 0, canvasSize, canvasSize); undoStack.current = []; redoStack.current = []; - ImageUtility.drawImgToCanvas(currImg.current, canvasRef, true); + ImageUtility.drawImgToCanvas(currImg.current, canvasRef, canvasSize, canvasSize); }; // initiate brushing @@ -144,7 +145,6 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD setIsBrushing(true); const { x, y } = PointerHandler.getPointRelativeToElement(canvas, e, canvasScale); - BrushHandler.brushCircleOverlay(x, y, cursorData.width / 2 / canvasScale, ctx, eraserColor, brushStyle === BrushStyle.SUBTRACT); }; @@ -181,19 +181,21 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD // first load useEffect(() => { - console.log('first load'); if (!imageEditorSource || imageEditorSource === '') return; const img = new Image(); - console.log('source', imageEditorSource); img.src = imageEditorSource; - console.log('drawing image'); - ImageUtility.drawImgToCanvas(img, canvasRef); currImg.current = img; originalImg.current = img; - freeformPosition.current = [0, 0]; + img.onload = () => { + const imgWidth = img.naturalWidth; + const imgHeight = img.naturalHeight; + const scale = Math.min(canvasSize / imgWidth, canvasSize / imgHeight); + const width = imgWidth * scale; + const height = imgHeight * scale; + setCanvasDims({ width, height }); + }; return () => { - console.log('cleanup'); newCollectionRef.current = null; parentDoc.current = null; childrenDocs.current = []; @@ -202,9 +204,15 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD freeformPosition.current = [0, 0]; undoStack.current = []; redoStack.current = []; + ImageUtility.clearCanvas(canvasRef); }; }, [canvasRef, imageEditorSource]); + useEffect(() => { + if (!currImg.current) return; + ImageUtility.drawImgToCanvas(currImg.current, canvasRef, canvasDims.width, canvasDims.height); + }, [canvasDims]); + // handles brush sizing useEffect(() => { const handleKeyPress = (e: KeyboardEvent) => { @@ -261,15 +269,18 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD setLoading(true); // need to adjust later try { - const maskBlob = await ImageUtility.canvasToBlob(canvas); - const imgBlob = await ImageUtility.canvasToBlob(ImageUtility.getCanvasImg(img)); + const canvasOriginalImg = ImageUtility.getCanvasImg(img); + if (!canvasOriginalImg) return; + const canvasMask = ImageUtility.getCanvasMask(canvas); + if (!canvasMask) return; + // ImageUtility.downloadCanvas(canvasMask); + const maskBlob = await ImageUtility.canvasToBlob(canvasMask); + const imgBlob = await ImageUtility.canvasToBlob(canvasOriginalImg); const res = await ImageUtility.getEdit(imgBlob, maskBlob, input !== '' ? input + ' in the same style' : 'Fill in the image in the same style', 2); // const res = await ImageUtility.mockGetEdit(img.src); // create first image if (!newCollectionRef.current) { - if (addToExistingCollection.current) { - } if (!(originalImg.current && imageRootDoc)) return; console.log('creating first image'); // create new collection and add it to the view @@ -281,25 +292,30 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD title: 'Image edit collection', }); DocUtils.MakeLink(imageRootDoc, newCollectionRef.current, { link_relationship: 'Image Edit Version History', link_displayLine: false }); + + // opening new tab + newCollectionRef.current.fitContentOnce = true; + CollectionDockingView.AddSplit(newCollectionRef.current, OpenWhereMod.right); + // add the doc to the main freeform // addDoc?.(newCollectionRef.current); await createNewImgDoc(originalImg.current, true); } else { - parentDoc.current = childrenDocs.current[childrenDocs.current.length - 1]; + // parentDoc.current = childrenDocs.current[childrenDocs.current.length - 1]; childrenDocs.current = []; } originalImg.current = currImg.current; const { urls } = res as APISuccess; + const imgUrls = await Promise.all(urls.map(url => ImageUtility.convertImageToCanvasDataURL(url, canvasDims.width, canvasDims.height))); const image = new Image(); - image.src = urls[0]; - setEdits(urls); - ImageUtility.drawImgToCanvas(image, canvasRef); + image.src = imgUrls[0]; + + setEdits(imgUrls); + ImageUtility.drawImgToCanvas(image, canvasRef, canvasDims.width, canvasDims.height); currImg.current = image; onSave(); - freeformPosition.current[0] += 1; - freeformPosition.current[1] = 0; } catch (err) { console.log(err); } @@ -334,7 +350,6 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD if (firstDoc) { const x = 0; const initialY = 0; - console.log('first doc'); const newImg = Docs.Create.ImageDocument(source, { x: x, @@ -387,10 +402,10 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD }; const handleViewClose = () => { - if (newCollectionRef.current) { - newCollectionRef.current.fitContentOnce = true; - CollectionDockingView.AddSplit(newCollectionRef.current, OpenWhereMod.right); - } + // if (newCollectionRef.current) { + // newCollectionRef.current.fitContentOnce = true; + // CollectionDockingView.AddSplit(newCollectionRef.current, OpenWhereMod.right); + // } MainView.Instance.setImageEditorOpen(false); MainView.Instance.setImageEditorSource(''); setEdits([]); @@ -420,8 +435,8 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD onPointerMove={updateCursorData} onPointerDown={handlePointerDown} onPointerUp={handlePointerUp}> - - + +
( { // if (savedSrcs.current.has(edit)) return; const img = new Image(); img.src = edit; - ImageUtility.drawImgToCanvas(img, canvasRef); + ImageUtility.drawImgToCanvas(img, canvasRef, canvasDims.width, canvasDims.height); currImg.current = img; savedSrcs.current.add(edit); await onSave(); @@ -519,15 +533,14 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD Original { if (!originalImg.current) return; const img = new Image(); img.src = originalImg.current.src; - ImageUtility.drawImgToCanvas(img, canvasRef); + ImageUtility.drawImgToCanvas(img, canvasRef, canvasDims.width, canvasDims.height); currImg.current = img; }} /> diff --git a/src/client/views/nodes/generativeFill/generativeFillUtils/ImageHandler.ts b/src/client/views/nodes/generativeFill/generativeFillUtils/ImageHandler.ts index 1954ab3fb..8d32221bd 100644 --- a/src/client/views/nodes/generativeFill/generativeFillUtils/ImageHandler.ts +++ b/src/client/views/nodes/generativeFill/generativeFillUtils/ImageHandler.ts @@ -1,5 +1,5 @@ import { RefObject } from 'react'; -import { canvasSize } from './generativeFillConstants'; +import { bgColor, canvasSize } from './generativeFillConstants'; export interface APISuccess { status: 'success'; @@ -22,6 +22,47 @@ export class ImageUtility { }); }; + // given a square api image, get the cropped img + static getCroppedImg = (img: HTMLImageElement, width: number, height: number): HTMLCanvasElement | undefined => { + // Create a new canvas element + const canvas = document.createElement('canvas'); + canvas.width = width; + canvas.height = height; + const ctx = canvas.getContext('2d'); + if (ctx) { + // Clear the canvas + ctx.clearRect(0, 0, canvas.width, canvas.height); + if (width < height) { + // horizontal padding, x offset + const xOffset = (canvasSize - width) / 2; + console.log(xOffset); + ctx.drawImage(img, xOffset, 0, canvas.width, canvas.height, 0, 0, canvas.width, canvas.height); + } else { + // vertical padding, y offset + const yOffset = (canvasSize - height) / 2; + ctx.drawImage(img, 0, yOffset, canvas.width, canvas.height, 0, 0, canvas.width, canvas.height); + } + return canvas; + } + }; + + static convertImageToCanvasDataURL = async (imageSrc: string, width: number, height: number): Promise => { + return new Promise((resolve, reject) => { + const img = new Image(); + img.onload = () => { + const canvas = this.getCroppedImg(img, width, height); + if (canvas) { + const dataUrl = canvas.toDataURL(); + resolve(dataUrl); + } + }; + img.onerror = error => { + reject(error); + }; + img.src = imageSrc; + }); + }; + static getEdit = async (imgBlob: Blob, maskBlob: Blob, prompt: string, n?: number): Promise => { const apiUrl = 'https://api.openai.com/v1/images/edits'; const fd = new FormData(); @@ -90,41 +131,83 @@ export class ImageUtility { }; }; - static drawImgToCanvas = (img: HTMLImageElement, canvasRef: React.RefObject, loaded?: boolean) => { - if (loaded) { + static clearCanvas = (canvasRef: React.RefObject) => { + const ctx = this.getCanvasContext(canvasRef); + if (!ctx || !canvasRef.current) return; + ctx.clearRect(0, 0, canvasRef.current.width, canvasRef.current.height); + }; + + static drawImgToCanvas = (img: HTMLImageElement, canvasRef: React.RefObject, width: number, height: number) => { + const drawImg = (img: HTMLImageElement) => { const ctx = this.getCanvasContext(canvasRef); if (!ctx) return; ctx.globalCompositeOperation = 'source-over'; - const scale = Math.max(canvasSize / img.width, canvasSize / img.height); - const width = img.width * scale; - const height = img.height * scale; - ctx.clearRect(0, 0, canvasSize, canvasSize); + ctx.clearRect(0, 0, width, height); ctx.drawImage(img, 0, 0, width, height); + }; + + if (img.complete) { + drawImg(img); } else { console.log('loading image'); img.onload = () => { - console.log('loaded'); - const ctx = this.getCanvasContext(canvasRef); - if (!ctx) return; - ctx.globalCompositeOperation = 'source-over'; - const scale = Math.max(canvasSize / img.width, canvasSize / img.height); - const width = img.width * scale; - const height = img.height * scale; - ctx.clearRect(0, 0, canvasSize, canvasSize); - ctx.drawImage(img, 0, 0, width, height); + drawImg(img); }; } }; // The image must be loaded! - static getCanvasImg = (img: HTMLImageElement): HTMLCanvasElement => { + static getCanvasMask = (srcCanvas: HTMLCanvasElement): HTMLCanvasElement | undefined => { const canvas = document.createElement('canvas'); canvas.width = canvasSize; canvas.height = canvasSize; const ctx = canvas.getContext('2d'); + if (!ctx) return; ctx?.clearRect(0, 0, canvasSize, canvasSize); - ctx?.drawImage(img, 0, 0, canvasSize, canvasSize); + ctx.fillStyle = bgColor; + ctx.fillRect(0, 0, canvasSize, canvasSize); + // extract and set padding data + if (srcCanvas.height > srcCanvas.width) { + // horizontal padding, x offset + const xOffset = (canvasSize - srcCanvas.width) / 2; + ctx?.clearRect(xOffset, 0, srcCanvas.width, srcCanvas.height); + ctx.drawImage(srcCanvas, xOffset, 0, srcCanvas.width, srcCanvas.height); + } else { + // vertical padding, y offset + const yOffset = (canvasSize - srcCanvas.height) / 2; + ctx?.clearRect(0, yOffset, srcCanvas.width, srcCanvas.height); + ctx.drawImage(srcCanvas, 0, yOffset, srcCanvas.width, srcCanvas.height); + } + return canvas; + }; + + // The image must be loaded! + static getCanvasImg = (img: HTMLImageElement): HTMLCanvasElement | undefined => { + const canvas = document.createElement('canvas'); + canvas.width = canvasSize; + canvas.height = canvasSize; + const ctx = canvas.getContext('2d'); + if (!ctx) return; + // fix scaling + const scale = Math.min(canvasSize / img.width, canvasSize / img.height); + const width = img.width * scale; + const height = img.height * scale; + ctx?.clearRect(0, 0, canvasSize, canvasSize); + ctx.fillStyle = bgColor; + ctx.fillRect(0, 0, canvasSize, canvasSize); + + // extract and set padding data + if (img.naturalHeight > img.naturalWidth) { + // horizontal padding, x offset + const xOffset = (canvasSize - width) / 2; + + ctx.drawImage(img, xOffset, 0, width, height); + } else { + // vertical padding, y offset + const yOffset = (canvasSize - height) / 2; + ctx.drawImage(img, 0, yOffset, width, height); + } return canvas; }; } diff --git a/src/client/views/nodes/generativeFill/generativeFillUtils/generativeFillConstants.ts b/src/client/views/nodes/generativeFill/generativeFillUtils/generativeFillConstants.ts index 412a4d238..286dc6e4c 100644 --- a/src/client/views/nodes/generativeFill/generativeFillUtils/generativeFillConstants.ts +++ b/src/client/views/nodes/generativeFill/generativeFillUtils/generativeFillConstants.ts @@ -1,9 +1,10 @@ // constants export const canvasSize = 512; export const freeformRenderSize = 300; -export const offsetDistanceY = freeformRenderSize + 200; +export const offsetDistanceY = freeformRenderSize + 300; export const offsetX = 200; export const newCollectionSize = 500; export const activeColor = '#1976d2'; export const eraserColor = '#e1e9ec'; +export const bgColor = '#f0f4f6'; -- cgit v1.2.3-70-g09d2 From 1f44bed14529f8bfc280f714ffd794813c1b7a73 Mon Sep 17 00:00:00 2001 From: Sophie Zhang Date: Thu, 17 Aug 2023 21:02:33 -0400 Subject: baseline before changes --- src/client/apis/gpt/GPT.ts | 1 - .../views/nodes/generativeFill/GenerativeFill.tsx | 61 ++++++---------------- .../generativeFillUtils/ImageHandler.ts | 10 ++-- .../generativeFillUtils/generativeFillConstants.ts | 2 +- .../generativeFillInterfaces.ts | 19 ++++--- 5 files changed, 32 insertions(+), 61 deletions(-) (limited to 'src/client/views/nodes/generativeFill/generativeFillUtils/ImageHandler.ts') diff --git a/src/client/apis/gpt/GPT.ts b/src/client/apis/gpt/GPT.ts index 6b4106f56..6bde7989b 100644 --- a/src/client/apis/gpt/GPT.ts +++ b/src/client/apis/gpt/GPT.ts @@ -39,7 +39,6 @@ const gptAPICall = async (inputText: string, callType: GPTCallType) => { temperature: opts.temp, prompt: `${opts.prompt}${inputText}`, }); - // console.log(response.data.choices[0]); return response.data.choices[0].text; } catch (err) { console.log(err); diff --git a/src/client/views/nodes/generativeFill/GenerativeFill.tsx b/src/client/views/nodes/generativeFill/GenerativeFill.tsx index 4d475149d..97d1cbf20 100644 --- a/src/client/views/nodes/generativeFill/GenerativeFill.tsx +++ b/src/client/views/nodes/generativeFill/GenerativeFill.tsx @@ -4,37 +4,21 @@ import { useEffect, useRef, useState } from 'react'; import { APISuccess, ImageUtility } from './generativeFillUtils/ImageHandler'; import { BrushHandler } from './generativeFillUtils/BrushHandler'; import { Box, IconButton, Slider, TextField } from '@mui/material'; -import { CursorData, Point } from './generativeFillUtils/generativeFillInterfaces'; +import { CursorData, ImageDimensions, Point } from './generativeFillUtils/generativeFillInterfaces'; import { activeColor, canvasSize, eraserColor, freeformRenderSize, newCollectionSize, offsetDistanceY, offsetX } from './generativeFillUtils/generativeFillConstants'; import { PointerHandler } from './generativeFillUtils/PointerHandler'; import { BsEraser, BsX } from 'react-icons/bs'; import { CiUndo, CiRedo } from 'react-icons/ci'; -import Buttons from './GenerativeFillButtons'; import { MainView } from '../../MainView'; import { Doc } from '../../../../fields/Doc'; import { Networking } from '../../../Network'; import { Utils } from '../../../../Utils'; import { DocUtils, Docs } from '../../../documents/Documents'; -import { Cast, DocCast, NumCast } from '../../../../fields/Types'; +import { NumCast } from '../../../../fields/Types'; import { CollectionDockingView } from '../../collections/CollectionDockingView'; -import { OpenWhere, OpenWhereMod } from '../DocumentView'; +import { OpenWhereMod } from '../DocumentView'; import { Oval } from 'react-loader-spinner'; - -/** - * For images not 1024x1024 fill in the rest in solid black, or a - * reflected version of the image. - * - * add a branch from image directly checkbox - */ - -/** - * - * - * CollectionDockingView.AddSplit(Doc.MakeCopy(DocCast(Doc.UserDoc().emptyPane)), OpenWhereMod.right); - * CollectionDockingView.AddSplit(newCollection,OpenWhere.inParent) - * mind mapping - * this.props.addDocTab(); - */ +import Buttons from './GenerativeFillButtons'; enum BrushStyle { ADD, @@ -42,11 +26,6 @@ enum BrushStyle { MARQUEE, } -interface ImageDimensions { - width: number; - height: number; -} - interface GenerativeFillProps { imageEditorOpen: boolean; imageEditorSource: string; @@ -112,10 +91,13 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD }; const handleRedo = () => { - // TODO: handle undo as well + const ctx = ImageUtility.getCanvasContext(canvasRef); + if (!ctx || !currImg.current || !canvasRef.current) return; + const target = redoStack.current[redoStack.current.length - 1]; if (!target) { } else { + undoStack.current = [...undoStack.current, canvasRef.current?.toDataURL()]; const img = new Image(); img.src = target; ImageUtility.drawImgToCanvas(img, canvasRef, canvasDims.width, canvasDims.height); @@ -123,6 +105,7 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD } }; + // resets any erase strokes const handleReset = () => { if (!canvasRef.current || !currImg.current) return; const ctx = ImageUtility.getCanvasContext(canvasRef); @@ -130,7 +113,7 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD ctx.clearRect(0, 0, canvasSize, canvasSize); undoStack.current = []; redoStack.current = []; - ImageUtility.drawImgToCanvas(currImg.current, canvasRef, canvasSize, canvasSize); + ImageUtility.drawImgToCanvas(currImg.current, canvasRef, canvasDims.width, canvasDims.height); }; // initiate brushing @@ -236,7 +219,7 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD e.preventDefault(); e.stopPropagation(); const delta = e.deltaY; - const scaleFactor = delta > 0 ? 0.98 : 1.02; // Adjust the scale factor as per your requirement + const scaleFactor = delta > 0 ? 0.98 : 1.02; setCanvasScale(prevScale => prevScale * scaleFactor); }; @@ -267,13 +250,11 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD const ctx = ImageUtility.getCanvasContext(canvasRef); if (!ctx) return; setLoading(true); - // need to adjust later try { const canvasOriginalImg = ImageUtility.getCanvasImg(img); if (!canvasOriginalImg) return; const canvasMask = ImageUtility.getCanvasMask(canvas); if (!canvasMask) return; - // ImageUtility.downloadCanvas(canvasMask); const maskBlob = await ImageUtility.canvasToBlob(canvasMask); const imgBlob = await ImageUtility.canvasToBlob(canvasOriginalImg); const res = await ImageUtility.getEdit(imgBlob, maskBlob, input !== '' ? input + ' in the same style' : 'Fill in the image in the same style', 2); @@ -282,7 +263,6 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD // create first image if (!newCollectionRef.current) { if (!(originalImg.current && imageRootDoc)) return; - console.log('creating first image'); // create new collection and add it to the view newCollectionRef.current = Docs.Create.FreeformDocument([], { x: NumCast(imageRootDoc.x) + NumCast(imageRootDoc._width) + offsetX, @@ -294,19 +274,17 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD DocUtils.MakeLink(imageRootDoc, newCollectionRef.current, { link_relationship: 'Image Edit Version History', link_displayLine: false }); // opening new tab - newCollectionRef.current.fitContentOnce = true; CollectionDockingView.AddSplit(newCollectionRef.current, OpenWhereMod.right); + newCollectionRef.current.fitContentOnce = true; // add the doc to the main freeform - // addDoc?.(newCollectionRef.current); await createNewImgDoc(originalImg.current, true); } else { - // parentDoc.current = childrenDocs.current[childrenDocs.current.length - 1]; + parentDoc.current = childrenDocs.current[childrenDocs.current.length - 1]; childrenDocs.current = []; } originalImg.current = currImg.current; - const { urls } = res as APISuccess; const imgUrls = await Promise.all(urls.map(url => ImageUtility.convertImageToCanvasDataURL(url, canvasDims.width, canvasDims.height))); const image = new Image(); @@ -366,8 +344,6 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD } else { if (!parentDoc.current) return; const x = NumCast(parentDoc.current.x) + freeformRenderSize + offsetX; - // dummy position - console.log('creating child elements'); const initialY = 0; const newImg = Docs.Create.ImageDocument(source, { @@ -388,12 +364,10 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD } }; - // need to maybe call on every img click, not just when the save btn is clicked const onSave = async () => { setSaveLoading(true); if (!currImg.current || !originalImg.current || !imageRootDoc) return; try { - console.log('creating another image'); await createNewImgDoc(currImg.current, false); } catch (err) { console.log(err); @@ -402,10 +376,6 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD }; const handleViewClose = () => { - // if (newCollectionRef.current) { - // newCollectionRef.current.fitContentOnce = true; - // CollectionDockingView.AddSplit(newCollectionRef.current, OpenWhereMod.right); - // } MainView.Instance.setImageEditorOpen(false); MainView.Instance.setImageEditorSource(''); setEdits([]); @@ -435,8 +405,8 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD onPointerMove={updateCursorData} onPointerDown={handlePointerDown} onPointerUp={handlePointerUp}> - - + +
{ - // if (savedSrcs.current.has(edit)) return; const img = new Image(); img.src = edit; ImageUtility.drawImgToCanvas(img, canvasRef, canvasDims.width, canvasDims.height); diff --git a/src/client/views/nodes/generativeFill/generativeFillUtils/ImageHandler.ts b/src/client/views/nodes/generativeFill/generativeFillUtils/ImageHandler.ts index 8d32221bd..4847bfeed 100644 --- a/src/client/views/nodes/generativeFill/generativeFillUtils/ImageHandler.ts +++ b/src/client/views/nodes/generativeFill/generativeFillUtils/ImageHandler.ts @@ -35,7 +35,6 @@ export class ImageUtility { if (width < height) { // horizontal padding, x offset const xOffset = (canvasSize - width) / 2; - console.log(xOffset); ctx.drawImage(img, xOffset, 0, canvas.width, canvas.height, 0, 0, canvas.width, canvas.height); } else { // vertical padding, y offset @@ -156,7 +155,7 @@ export class ImageUtility { } }; - // The image must be loaded! + // The image must be loaded static getCanvasMask = (srcCanvas: HTMLCanvasElement): HTMLCanvasElement | undefined => { const canvas = document.createElement('canvas'); canvas.width = canvasSize; @@ -164,7 +163,7 @@ export class ImageUtility { const ctx = canvas.getContext('2d'); if (!ctx) return; ctx?.clearRect(0, 0, canvasSize, canvasSize); - ctx.fillStyle = bgColor; + ctx.fillStyle = '#000000'; ctx.fillRect(0, 0, canvasSize, canvasSize); // extract and set padding data @@ -182,7 +181,7 @@ export class ImageUtility { return canvas; }; - // The image must be loaded! + // The image must be loaded static getCanvasImg = (img: HTMLImageElement): HTMLCanvasElement | undefined => { const canvas = document.createElement('canvas'); canvas.width = canvasSize; @@ -194,10 +193,9 @@ export class ImageUtility { const width = img.width * scale; const height = img.height * scale; ctx?.clearRect(0, 0, canvasSize, canvasSize); - ctx.fillStyle = bgColor; + ctx.fillStyle = '#000000'; ctx.fillRect(0, 0, canvasSize, canvasSize); - // extract and set padding data if (img.naturalHeight > img.naturalWidth) { // horizontal padding, x offset const xOffset = (canvasSize - width) / 2; diff --git a/src/client/views/nodes/generativeFill/generativeFillUtils/generativeFillConstants.ts b/src/client/views/nodes/generativeFill/generativeFillUtils/generativeFillConstants.ts index 286dc6e4c..1fe151b46 100644 --- a/src/client/views/nodes/generativeFill/generativeFillUtils/generativeFillConstants.ts +++ b/src/client/views/nodes/generativeFill/generativeFillUtils/generativeFillConstants.ts @@ -1,7 +1,7 @@ // constants export const canvasSize = 512; export const freeformRenderSize = 300; -export const offsetDistanceY = freeformRenderSize + 300; +export const offsetDistanceY = freeformRenderSize + 400; export const offsetX = 200; export const newCollectionSize = 500; diff --git a/src/client/views/nodes/generativeFill/generativeFillUtils/generativeFillInterfaces.ts b/src/client/views/nodes/generativeFill/generativeFillUtils/generativeFillInterfaces.ts index 9b9b9d3c2..83a21a1a5 100644 --- a/src/client/views/nodes/generativeFill/generativeFillUtils/generativeFillInterfaces.ts +++ b/src/client/views/nodes/generativeFill/generativeFillUtils/generativeFillInterfaces.ts @@ -1,16 +1,21 @@ // interfaces export interface CursorData { - x: number; - y: number; - width: number; + x: number; + y: number; + width: number; } export interface Point { - x: number; - y: number; + x: number; + y: number; } export enum BrushMode { - ADD, - SUBTRACT, + ADD, + SUBTRACT, +} + +export interface ImageDimensions { + width: number; + height: number; } -- cgit v1.2.3-70-g09d2 From bce4589c0bd871437c83bf44a6a818d01b9bf0f2 Mon Sep 17 00:00:00 2001 From: Sophie Zhang Date: Fri, 18 Aug 2023 16:52:12 -0400 Subject: reflected image padding --- .../views/nodes/generativeFill/GenerativeFill.tsx | 141 +++++++++++++-------- .../nodes/generativeFill/GenerativeFillButtons.tsx | 11 +- .../generativeFillUtils/ImageHandler.ts | 88 +++++++++++-- 3 files changed, 163 insertions(+), 77 deletions(-) (limited to 'src/client/views/nodes/generativeFill/generativeFillUtils/ImageHandler.ts') diff --git a/src/client/views/nodes/generativeFill/GenerativeFill.tsx b/src/client/views/nodes/generativeFill/GenerativeFill.tsx index 97d1cbf20..92ce3a8a8 100644 --- a/src/client/views/nodes/generativeFill/GenerativeFill.tsx +++ b/src/client/views/nodes/generativeFill/GenerativeFill.tsx @@ -3,22 +3,23 @@ import React = require('react'); import { useEffect, useRef, useState } from 'react'; import { APISuccess, ImageUtility } from './generativeFillUtils/ImageHandler'; import { BrushHandler } from './generativeFillUtils/BrushHandler'; -import { Box, IconButton, Slider, TextField } from '@mui/material'; +import { Box, Checkbox, FormControlLabel, IconButton, Slider, TextField } from '@mui/material'; import { CursorData, ImageDimensions, Point } from './generativeFillUtils/generativeFillInterfaces'; import { activeColor, canvasSize, eraserColor, freeformRenderSize, newCollectionSize, offsetDistanceY, offsetX } from './generativeFillUtils/generativeFillConstants'; import { PointerHandler } from './generativeFillUtils/PointerHandler'; import { BsEraser, BsX } from 'react-icons/bs'; import { CiUndo, CiRedo } from 'react-icons/ci'; import { MainView } from '../../MainView'; -import { Doc } from '../../../../fields/Doc'; +import { Doc, DocListCast } from '../../../../fields/Doc'; import { Networking } from '../../../Network'; -import { Utils } from '../../../../Utils'; +import { Utils, returnEmptyDoclist } from '../../../../Utils'; import { DocUtils, Docs } from '../../../documents/Documents'; -import { NumCast } from '../../../../fields/Types'; +import { DocCast, NumCast } from '../../../../fields/Types'; import { CollectionDockingView } from '../../collections/CollectionDockingView'; import { OpenWhereMod } from '../DocumentView'; import { Oval } from 'react-loader-spinner'; import Buttons from './GenerativeFillButtons'; +import { List } from '../../../../fields/List'; enum BrushStyle { ADD, @@ -33,6 +34,8 @@ interface GenerativeFillProps { addDoc: ((doc: Doc | Doc[], annotationKey?: string) => boolean) | undefined; } +// New field on image doc: gen_fill_children => list of children Docs + const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addDoc }: GenerativeFillProps) => { const canvasRef = useRef(null); const canvasBackgroundRef = useRef(null); @@ -44,15 +47,18 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD }); const [isBrushing, setIsBrushing] = useState(false); const [canvasScale, setCanvasScale] = useState(0.5); - const [edits, setEdits] = useState([]); + // format: array of [image source, corresponding image Doc] + const [edits, setEdits] = useState<(string | Doc)[][]>([]); + const [edited, setEdited] = useState(false); const [brushStyle, setBrushStyle] = useState(BrushStyle.ADD); const [input, setInput] = useState(''); const [loading, setLoading] = useState(false); - const [saveLoading, setSaveLoading] = useState(false); const [canvasDims, setCanvasDims] = useState({ width: canvasSize, height: canvasSize, }); + // whether to create a new collection or not + const [isNewCollection, setIsNewCollection] = useState(true); // the current image in the main canvas const currImg = useRef(null); // the unedited version of each generation (parent) @@ -62,12 +68,6 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD // stores redo stack const redoStack = useRef([]); - // early stage properly, likely will get rid of - const freeformPosition = useRef([0, 0]); - - // which urls were already saved to canvas - const savedSrcs = useRef>(new Set()); - // references to keep track of tree structure const newCollectionRef = useRef(null); const parentDoc = useRef(null); @@ -178,13 +178,14 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD setCanvasDims({ width, height }); }; + // cleanup return () => { + setEdited(false); newCollectionRef.current = null; parentDoc.current = null; childrenDocs.current = []; currImg.current = null; originalImg.current = null; - freeformPosition.current = [0, 0]; undoStack.current = []; redoStack.current = []; ImageUtility.clearCanvas(canvasRef); @@ -250,10 +251,11 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD const ctx = ImageUtility.getCanvasContext(canvasRef); if (!ctx) return; setLoading(true); + setEdited(true); try { const canvasOriginalImg = ImageUtility.getCanvasImg(img); if (!canvasOriginalImg) return; - const canvasMask = ImageUtility.getCanvasMask(canvas); + const canvasMask = ImageUtility.getCanvasMask(canvas, canvasOriginalImg); if (!canvasMask) return; const maskBlob = await ImageUtility.canvasToBlob(canvasMask); const imgBlob = await ImageUtility.canvasToBlob(canvasOriginalImg); @@ -262,38 +264,46 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD // create first image if (!newCollectionRef.current) { - if (!(originalImg.current && imageRootDoc)) return; - // create new collection and add it to the view - newCollectionRef.current = Docs.Create.FreeformDocument([], { - x: NumCast(imageRootDoc.x) + NumCast(imageRootDoc._width) + offsetX, - y: NumCast(imageRootDoc.y), - _width: newCollectionSize, - _height: newCollectionSize, - title: 'Image edit collection', - }); - DocUtils.MakeLink(imageRootDoc, newCollectionRef.current, { link_relationship: 'Image Edit Version History', link_displayLine: false }); - - // opening new tab - CollectionDockingView.AddSplit(newCollectionRef.current, OpenWhereMod.right); - newCollectionRef.current.fitContentOnce = true; - - // add the doc to the main freeform - await createNewImgDoc(originalImg.current, true); + if (!isNewCollection && imageRootDoc) { + // new collection stays null + parentDoc.current = imageRootDoc; + } else { + if (!(originalImg.current && imageRootDoc)) return; + // create new collection and add it to the view + newCollectionRef.current = Docs.Create.FreeformDocument([], { + x: NumCast(imageRootDoc.x) + NumCast(imageRootDoc._width) + offsetX, + y: NumCast(imageRootDoc.y), + _width: newCollectionSize, + _height: newCollectionSize, + title: 'Image edit collection', + }); + DocUtils.MakeLink(imageRootDoc, newCollectionRef.current, { link_relationship: 'Image Edit Version History', link_displayLine: false }); + + // opening new tab + CollectionDockingView.AddSplit(newCollectionRef.current, OpenWhereMod.right); + + // add the doc to the main freeform + await createNewImgDoc(originalImg.current, true); + } } else { - parentDoc.current = childrenDocs.current[childrenDocs.current.length - 1]; childrenDocs.current = []; } originalImg.current = currImg.current; const { urls } = res as APISuccess; - const imgUrls = await Promise.all(urls.map(url => ImageUtility.convertImageToCanvasDataURL(url, canvasDims.width, canvasDims.height))); + const imgUrls = await Promise.all(urls.map(url => ImageUtility.convertImgToCanvasUrl(url, canvasDims.width, canvasDims.height))); + const imgRes = await Promise.all( + imgUrls.map(async url => { + const saveRes = await onSave(url); + return [url, saveRes as Doc]; + }) + ); + setEdits(imgRes); const image = new Image(); image.src = imgUrls[0]; - setEdits(imgUrls); ImageUtility.drawImgToCanvas(image, canvasRef, canvasDims.width, canvasDims.height); currImg.current = image; - onSave(); } catch (err) { console.log(err); } @@ -320,7 +330,7 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD // creates a new image document and returns its reference const createNewImgDoc = async (img: HTMLImageElement, firstDoc: boolean): Promise => { - if (!newCollectionRef.current || !imageRootDoc) return; + if (!imageRootDoc) return; const src = img.src; const [result] = await Networking.PostToServer('/uploadRemoteImage', { sources: [src] }); const source = Utils.prepend(result.accessPaths.agnostic.client); @@ -337,8 +347,12 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD data_nativeWidth: result.nativeWidth, data_nativeHeight: result.nativeHeight, }); - - Doc.AddDocToList(newCollectionRef.current, undefined, newImg); + // add a new doc list field to newimg + if (isNewCollection && newCollectionRef.current) { + Doc.AddDocToList(newCollectionRef.current, undefined, newImg); + } else { + addDoc?.(newImg); + } parentDoc.current = newImg; return newImg; } else { @@ -354,30 +368,40 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD data_nativeWidth: result.nativeWidth, data_nativeHeight: result.nativeHeight, }); - + newImg.gen_fill_children = new List([]); childrenDocs.current.push(newImg); - DocUtils.MakeLink(parentDoc.current, newImg, { link_relationship: 'Image Edit', link_displayLine: true }); + // DocListCast(parentDoc.current.gen_fill_children).push(newImg); + DocUtils.MakeLink(parentDoc.current, newImg, { link_relationship: `Image edit; Prompt: ${input}`, link_displayLine: true }); adjustImgPositions(); - Doc.AddDocToList(newCollectionRef.current, undefined, newImg); + if (isNewCollection && newCollectionRef.current) { + Doc.AddDocToList(newCollectionRef.current, undefined, newImg); + } else { + addDoc?.(newImg); + } return newImg; } }; - const onSave = async () => { - setSaveLoading(true); + const onSave = async (src: string) => { + const img = new Image(); + img.src = src; if (!currImg.current || !originalImg.current || !imageRootDoc) return; try { - await createNewImgDoc(currImg.current, false); + const res = await createNewImgDoc(img, false); + return res; } catch (err) { console.log(err); } - setSaveLoading(false); }; const handleViewClose = () => { MainView.Instance.setImageEditorOpen(false); MainView.Instance.setImageEditorSource(''); + if (newCollectionRef.current) { + newCollectionRef.current.fitContentOnce = true; + } + setEdits([]); }; @@ -386,15 +410,25 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD

AI Image Editor

+ { + setIsNewCollection(prev => !prev); + }} + /> + } + label={'Create New Collection'} + labelPlacement="end" + sx={{ whiteSpace: 'nowrap' }} + /> - {saveLoading && ( - - Saving image... - - )}
{/* Main canvas for editing */} @@ -475,14 +509,13 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD { const img = new Image(); - img.src = edit; + img.src = edit[0] as string; ImageUtility.drawImgToCanvas(img, canvasRef, canvasDims.width, canvasDims.height); currImg.current = img; - savedSrcs.current.add(edit); - await onSave(); + parentDoc.current = edit[1] as Doc; }} /> ))} diff --git a/src/client/views/nodes/generativeFill/GenerativeFillButtons.tsx b/src/client/views/nodes/generativeFill/GenerativeFillButtons.tsx index b4d56b408..e15af0a56 100644 --- a/src/client/views/nodes/generativeFill/GenerativeFillButtons.tsx +++ b/src/client/views/nodes/generativeFill/GenerativeFillButtons.tsx @@ -14,19 +14,10 @@ interface ButtonContainerProps { onReset: () => void; } -const Buttons = ({ canvasRef, currImg, loading, getEdit, onReset }: ButtonContainerProps) => { - +const Buttons = ({ loading, getEdit, onReset }: ButtonContainerProps) => { return (
- {/* */} - {/* */}
{/* Main canvas for editing */} @@ -459,40 +458,34 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD
{/* Icons */}
- { - setBrushStyle(BrushStyle.ADD); - }}> - - {/* Undo and Redo */} { e.stopPropagation(); handleUndo(); }} onPointerUp={e => { e.stopPropagation(); - }}> - - + }} + color={activeColor} + tooltip="Undo" + icon={} + /> { e.stopPropagation(); handleRedo(); }} onPointerUp={e => { e.stopPropagation(); - }}> - - - + }} + color={activeColor} + tooltip="Redo" + icon={} + /> +
e.stopPropagation()} style={{ height: 225, width: '100%', display: 'flex', justifyContent: 'center', cursor: 'pointer' }}> { setCursorData(prev => ({ ...prev, width: val as number })); }} /> - +
{/* Edits thumbnails*/}
diff --git a/src/client/views/nodes/generativeFill/GenerativeFillButtons.tsx b/src/client/views/nodes/generativeFill/GenerativeFillButtons.tsx index 398ba5333..0dfcebea3 100644 --- a/src/client/views/nodes/generativeFill/GenerativeFillButtons.tsx +++ b/src/client/views/nodes/generativeFill/GenerativeFillButtons.tsx @@ -1,7 +1,8 @@ -import { Button } from '@mui/material'; -import { Oval } from 'react-loader-spinner'; import './GenerativeFillButtons.scss'; import React = require('react'); +import ReactLoading from 'react-loading'; +import { activeColor } from './generativeFillUtils/generativeFillConstants'; +import { Button, Type } from 'browndash-components'; interface ButtonContainerProps { getEdit: () => Promise; @@ -12,17 +13,28 @@ interface ButtonContainerProps { const Buttons = ({ loading, getEdit, onReset }: ButtonContainerProps) => { return (
- - +
); }; diff --git a/src/client/views/nodes/generativeFill/generativeFillUtils/BrushHandler.ts b/src/client/views/nodes/generativeFill/generativeFillUtils/BrushHandler.ts index ace7ebcae..f4ec70fbc 100644 --- a/src/client/views/nodes/generativeFill/generativeFillUtils/BrushHandler.ts +++ b/src/client/views/nodes/generativeFill/generativeFillUtils/BrushHandler.ts @@ -3,16 +3,6 @@ import { eraserColor } from './generativeFillConstants'; import { Point } from './generativeFillInterfaces'; export class BrushHandler { - static brushCircle = (x: number, y: number, brushRadius: number, ctx: CanvasRenderingContext2D) => { - ctx.globalCompositeOperation = 'destination-out'; - ctx.shadowColor = '#ffffffeb'; - ctx.shadowBlur = 5; - ctx.beginPath(); - ctx.arc(x, y, brushRadius, 0, 2 * Math.PI); - ctx.fill(); - ctx.closePath(); - }; - static brushCircleOverlay = (x: number, y: number, brushRadius: number, ctx: CanvasRenderingContext2D, fillColor: string, erase: boolean) => { ctx.globalCompositeOperation = 'destination-out'; ctx.fillStyle = fillColor; @@ -24,15 +14,6 @@ export class BrushHandler { ctx.closePath(); }; - static createBrushPath = (startPoint: Point, endPoint: Point, brushRadius: number, ctx: CanvasRenderingContext2D) => { - const dist = GenerativeFillMathHelpers.distanceBetween(startPoint, endPoint); - - for (let i = 0; i < dist; i += 5) { - const s = i / dist; - BrushHandler.brushCircle(startPoint.x * (1 - s) + endPoint.x * s, startPoint.y * (1 - s) + endPoint.y * s, brushRadius, ctx); - } - }; - static createBrushPathOverlay = (startPoint: Point, endPoint: Point, brushRadius: number, ctx: CanvasRenderingContext2D, fillColor: string, erase: boolean) => { const dist = GenerativeFillMathHelpers.distanceBetween(startPoint, endPoint); diff --git a/src/client/views/nodes/generativeFill/generativeFillUtils/GenerativeFillMathHelpers.ts b/src/client/views/nodes/generativeFill/generativeFillUtils/GenerativeFillMathHelpers.ts index 027b99a52..97e03ff20 100644 --- a/src/client/views/nodes/generativeFill/generativeFillUtils/GenerativeFillMathHelpers.ts +++ b/src/client/views/nodes/generativeFill/generativeFillUtils/GenerativeFillMathHelpers.ts @@ -1,11 +1,10 @@ -import { Point } from "./generativeFillInterfaces"; +import { Point } from './generativeFillInterfaces'; export class GenerativeFillMathHelpers { - // math helpers - static distanceBetween = (p1: Point, p2: Point) => { - return Math.sqrt(Math.pow(p2.x - p1.x, 2) + Math.pow(p2.y - p1.y, 2)); - }; - static angleBetween = (p1: Point, p2: Point) => { - return Math.atan2(p2.x - p1.x, p2.y - p1.y); - }; + static distanceBetween = (p1: Point, p2: Point) => { + return Math.sqrt(Math.pow(p2.x - p1.x, 2) + Math.pow(p2.y - p1.y, 2)); + }; + static angleBetween = (p1: Point, p2: Point) => { + return Math.atan2(p2.x - p1.x, p2.y - p1.y); + }; } diff --git a/src/client/views/nodes/generativeFill/generativeFillUtils/ImageHandler.ts b/src/client/views/nodes/generativeFill/generativeFillUtils/ImageHandler.ts index 4ff70c86c..c6ecedb6b 100644 --- a/src/client/views/nodes/generativeFill/generativeFillUtils/ImageHandler.ts +++ b/src/client/views/nodes/generativeFill/generativeFillUtils/ImageHandler.ts @@ -12,6 +12,11 @@ export interface APIError { } export class ImageUtility { + /** + * + * @param canvas Canvas to convert + * @returns Blob of canvas + */ static canvasToBlob = (canvas: HTMLCanvasElement): Promise => { return new Promise(resolve => { canvas.toBlob(blob => { @@ -45,6 +50,7 @@ export class ImageUtility { } }; + // converts an image to a canvas data url static convertImgToCanvasUrl = async (imageSrc: string, width: number, height: number): Promise => { return new Promise((resolve, reject) => { const img = new Image(); @@ -62,6 +68,7 @@ export class ImageUtility { }); }; + // calls the openai api to get image edits static getEdit = async (imgBlob: Blob, maskBlob: Blob, prompt: string, n?: number): Promise => { const apiUrl = 'https://api.openai.com/v1/images/edits'; const fd = new FormData(); @@ -92,6 +99,7 @@ export class ImageUtility { } }; + // mock api call static mockGetEdit = async (mockSrc: string): Promise => { return { status: 'success', @@ -99,6 +107,7 @@ export class ImageUtility { }; }; + // Gets the canvas rendering context of a canvas static getCanvasContext = (canvasRef: RefObject): CanvasRenderingContext2D | null => { if (!canvasRef.current) return null; const ctx = canvasRef.current.getContext('2d'); @@ -106,6 +115,7 @@ export class ImageUtility { return ctx; }; + // Helper for downloading the canvas (for debugging) static downloadCanvas = (canvas: HTMLCanvasElement) => { const url = canvas.toDataURL(); const downloadLink = document.createElement('a'); @@ -116,6 +126,7 @@ export class ImageUtility { downloadLink.remove(); }; + // Download the canvas (for debugging) static downloadImageCanvas = (imgUrl: string) => { const img = new Image(); img.src = imgUrl; @@ -130,12 +141,14 @@ export class ImageUtility { }; }; + // Clears the canvas static clearCanvas = (canvasRef: React.RefObject) => { const ctx = this.getCanvasContext(canvasRef); if (!ctx || !canvasRef.current) return; ctx.clearRect(0, 0, canvasRef.current.width, canvasRef.current.height); }; + // Draws the image to the current canvas static drawImgToCanvas = (img: HTMLImageElement, canvasRef: React.RefObject, width: number, height: number) => { const drawImg = (img: HTMLImageElement) => { const ctx = this.getCanvasContext(canvasRef); @@ -154,7 +167,7 @@ export class ImageUtility { } }; - // The image must be loaded! + // Gets the image mask for the openai endpoint static getCanvasMask = (srcCanvas: HTMLCanvasElement, paddedCanvas: HTMLCanvasElement): HTMLCanvasElement | undefined => { const canvas = document.createElement('canvas'); canvas.width = canvasSize; @@ -162,8 +175,6 @@ export class ImageUtility { const ctx = canvas.getContext('2d'); if (!ctx) return; ctx?.clearRect(0, 0, canvasSize, canvasSize); - // ctx.fillStyle = bgColor; - // ctx.fillRect(0, 0, canvasSize, canvasSize); ctx.drawImage(paddedCanvas, 0, 0); // extract and set padding data @@ -181,6 +192,7 @@ export class ImageUtility { return canvas; }; + // Fills in the blank areas of the image with an image reflection (to fill in a square-shaped canvas) static drawHorizontalReflection = (ctx: CanvasRenderingContext2D, canvas: HTMLCanvasElement, xOffset: number) => { const imageData = ctx.getImageData(0, 0, canvas.width, canvas.height); const data = imageData.data; @@ -209,6 +221,7 @@ export class ImageUtility { ctx.putImageData(imageData, 0, 0); }; + // Fills in the blank areas of the image with an image reflection (to fill in a square-shaped canvas) static drawVerticalReflection = (ctx: CanvasRenderingContext2D, canvas: HTMLCanvasElement, yOffset: number) => { const imageData = ctx.getImageData(0, 0, canvas.width, canvas.height); const data = imageData.data; @@ -237,7 +250,7 @@ export class ImageUtility { ctx.putImageData(imageData, 0, 0); }; - // The image must be loaded! + // Gets the unaltered (besides filling in padding) version of the image for the api call static getCanvasImg = (img: HTMLImageElement): HTMLCanvasElement | undefined => { const canvas = document.createElement('canvas'); canvas.width = canvasSize; diff --git a/src/client/views/nodes/generativeFill/generativeFillUtils/generativeFillConstants.ts b/src/client/views/nodes/generativeFill/generativeFillUtils/generativeFillConstants.ts index 1fe151b46..dc94a9368 100644 --- a/src/client/views/nodes/generativeFill/generativeFillUtils/generativeFillConstants.ts +++ b/src/client/views/nodes/generativeFill/generativeFillUtils/generativeFillConstants.ts @@ -1,4 +1,3 @@ -// constants export const canvasSize = 512; export const freeformRenderSize = 300; export const offsetDistanceY = freeformRenderSize + 400; diff --git a/src/client/views/nodes/generativeFill/generativeFillUtils/generativeFillInterfaces.ts b/src/client/views/nodes/generativeFill/generativeFillUtils/generativeFillInterfaces.ts index 83a21a1a5..1e7801056 100644 --- a/src/client/views/nodes/generativeFill/generativeFillUtils/generativeFillInterfaces.ts +++ b/src/client/views/nodes/generativeFill/generativeFillUtils/generativeFillInterfaces.ts @@ -1,4 +1,3 @@ -// interfaces export interface CursorData { x: number; y: number; -- cgit v1.2.3-70-g09d2 From c9fe812391b68b1337f50d4e572a18640e39ff3b Mon Sep 17 00:00:00 2001 From: Sophie Zhang Date: Fri, 18 Aug 2023 19:50:18 -0400 Subject: changed dims to 1024 --- .../views/nodes/generativeFill/generativeFillUtils/ImageHandler.ts | 2 +- .../nodes/generativeFill/generativeFillUtils/generativeFillConstants.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'src/client/views/nodes/generativeFill/generativeFillUtils/ImageHandler.ts') diff --git a/src/client/views/nodes/generativeFill/generativeFillUtils/ImageHandler.ts b/src/client/views/nodes/generativeFill/generativeFillUtils/ImageHandler.ts index c6ecedb6b..2ede625f6 100644 --- a/src/client/views/nodes/generativeFill/generativeFillUtils/ImageHandler.ts +++ b/src/client/views/nodes/generativeFill/generativeFillUtils/ImageHandler.ts @@ -75,7 +75,7 @@ export class ImageUtility { fd.append('image', imgBlob, 'image.png'); fd.append('mask', maskBlob, 'mask.png'); fd.append('prompt', prompt); - fd.append('size', '512x512'); + fd.append('size', '1024x1024'); fd.append('n', n ? JSON.stringify(n) : '1'); fd.append('response_format', 'b64_json'); diff --git a/src/client/views/nodes/generativeFill/generativeFillUtils/generativeFillConstants.ts b/src/client/views/nodes/generativeFill/generativeFillUtils/generativeFillConstants.ts index dc94a9368..4772304bc 100644 --- a/src/client/views/nodes/generativeFill/generativeFillUtils/generativeFillConstants.ts +++ b/src/client/views/nodes/generativeFill/generativeFillUtils/generativeFillConstants.ts @@ -1,4 +1,4 @@ -export const canvasSize = 512; +export const canvasSize = 1024; export const freeformRenderSize = 300; export const offsetDistanceY = freeformRenderSize + 400; export const offsetX = 200; -- cgit v1.2.3-70-g09d2