aboutsummaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
authoreleanor-park <eleanor_park@brown.edu>2024-10-20 12:44:15 -0400
committereleanor-park <eleanor_park@brown.edu>2024-10-20 12:44:15 -0400
commit3b17868560090756caf8b9b0f043ea163f2320e8 (patch)
treed748a5fa923f28e2f4e399a8846ea38aa06b78a7 /src
parentfc06a98deec3fa2b173f8ea30a4f4b1781447b19 (diff)
changes
Diffstat (limited to 'src')
-rw-r--r--src/client/views/MainView.tsx5
-rw-r--r--src/client/views/StyleProviderQuiz.tsx2
-rw-r--r--src/client/views/nodes/generativeFill/generativeFillUtils/generativeFillInterfaces.ts20
-rw-r--r--src/client/views/nodes/imageEditor/GenerativeFillButtons.scss (renamed from src/client/views/nodes/generativeFill/GenerativeFillButtons.scss)0
-rw-r--r--src/client/views/nodes/imageEditor/GenerativeFillButtons.tsx (renamed from src/client/views/nodes/generativeFill/GenerativeFillButtons.tsx)2
-rw-r--r--src/client/views/nodes/imageEditor/ImageEditor.scss (renamed from src/client/views/nodes/generativeFill/GenerativeFill.scss)45
-rw-r--r--src/client/views/nodes/imageEditor/ImageEditor.tsx (renamed from src/client/views/nodes/generativeFill/GenerativeFill.tsx)347
-rw-r--r--src/client/views/nodes/imageEditor/ImageEditorButtons.tsx69
-rw-r--r--src/client/views/nodes/imageEditor/imageEditingUtils/ImageHandler.ts (renamed from src/client/views/nodes/generativeFill/generativeFillUtils/ImageHandler.ts)2
-rw-r--r--src/client/views/nodes/imageEditor/imageEditorUtils/BrushHandler.ts (renamed from src/client/views/nodes/generativeFill/generativeFillUtils/BrushHandler.ts)4
-rw-r--r--src/client/views/nodes/imageEditor/imageEditorUtils/GenerativeFillMathHelpers.ts (renamed from src/client/views/nodes/generativeFill/generativeFillUtils/GenerativeFillMathHelpers.ts)2
-rw-r--r--src/client/views/nodes/imageEditor/imageEditorUtils/ImageHandler.ts312
-rw-r--r--src/client/views/nodes/imageEditor/imageEditorUtils/PointerHandler.ts (renamed from src/client/views/nodes/generativeFill/generativeFillUtils/PointerHandler.ts)2
-rw-r--r--src/client/views/nodes/imageEditor/imageEditorUtils/imageEditorConstants.ts (renamed from src/client/views/nodes/generativeFill/generativeFillUtils/generativeFillConstants.ts)0
-rw-r--r--src/client/views/nodes/imageEditor/imageEditorUtils/imageEditorInterfaces.ts38
-rw-r--r--src/client/views/nodes/imageEditor/imageToolUtils/BrushHandler.ts35
-rw-r--r--src/client/views/nodes/imageEditor/imageToolUtils/ImageHandler.ts312
-rw-r--r--src/client/views/smartdraw/AnnotationPalette.tsx2
18 files changed, 1027 insertions, 172 deletions
diff --git a/src/client/views/MainView.tsx b/src/client/views/MainView.tsx
index 7779d339f..7bc211036 100644
--- a/src/client/views/MainView.tsx
+++ b/src/client/views/MainView.tsx
@@ -73,7 +73,7 @@ import { OpenWhere, OpenWhereMod } from './nodes/OpenWhere';
import { TaskCompletionBox } from './nodes/TaskCompletedBox';
import { DashFieldViewMenu } from './nodes/formattedText/DashFieldView';
import { RichTextMenu } from './nodes/formattedText/RichTextMenu';
-import GenerativeFill from './nodes/generativeFill/GenerativeFill';
+import ImageEditorBox from './nodes/imageEditor/ImageEditor';
import { PresBox } from './nodes/trails';
import { AnchorMenu } from './pdf/AnchorMenu';
import { GPTPopup } from './pdf/GPTPopup/GPTPopup';
@@ -446,6 +446,7 @@ export class MainView extends ObservableReactComponent<object> {
fa.faAlignRight,
fa.faHeading,
fa.faRulerCombined,
+ fa.faFill,
fa.faFillDrip,
fa.faLink,
fa.faUnlink,
@@ -1146,7 +1147,7 @@ export class MainView extends ObservableReactComponent<object> {
<LightboxView key="lightbox" PanelWidth={this._windowWidth} addSplit={CollectionDockingView.AddSplit} PanelHeight={this._windowHeight} maxBorder={this.lightboxMaxBorder} />
<GPTPopup key="gptpopup" />
<SchemaCSVPopUp key="schemacsvpopup" />
- <GenerativeFill imageEditorOpen={ImageEditor.Open} imageEditorSource={ImageEditor.Source} imageRootDoc={ImageEditor.RootDoc} addDoc={ImageEditor.AddDoc} />
+ <ImageEditorBox imageEditorOpen={ImageEditor.Open} imageEditorSource={ImageEditor.Source} imageRootDoc={ImageEditor.RootDoc} addDoc={ImageEditor.AddDoc} />
</div>
);
}
diff --git a/src/client/views/StyleProviderQuiz.tsx b/src/client/views/StyleProviderQuiz.tsx
index 1f2ad1485..8a1515d21 100644
--- a/src/client/views/StyleProviderQuiz.tsx
+++ b/src/client/views/StyleProviderQuiz.tsx
@@ -17,7 +17,7 @@ import { AnchorMenu } from './pdf/AnchorMenu';
import { DocumentViewProps } from './nodes/DocumentView';
import { FieldViewProps } from './nodes/FieldView';
import { ImageBox } from './nodes/ImageBox';
-import { ImageUtility } from './nodes/generativeFill/generativeFillUtils/ImageHandler';
+import { ImageUtility } from './nodes/imageEditor/imageEditorUtils/ImageHandler';
import './StyleProviderQuiz.scss';
export namespace styleProviderQuiz {
diff --git a/src/client/views/nodes/generativeFill/generativeFillUtils/generativeFillInterfaces.ts b/src/client/views/nodes/generativeFill/generativeFillUtils/generativeFillInterfaces.ts
deleted file mode 100644
index 1e7801056..000000000
--- a/src/client/views/nodes/generativeFill/generativeFillUtils/generativeFillInterfaces.ts
+++ /dev/null
@@ -1,20 +0,0 @@
-export interface CursorData {
- x: number;
- y: number;
- width: number;
-}
-
-export interface Point {
- x: number;
- y: number;
-}
-
-export enum BrushMode {
- ADD,
- SUBTRACT,
-}
-
-export interface ImageDimensions {
- width: number;
- height: number;
-}
diff --git a/src/client/views/nodes/generativeFill/GenerativeFillButtons.scss b/src/client/views/nodes/imageEditor/GenerativeFillButtons.scss
index 0180ef904..0180ef904 100644
--- a/src/client/views/nodes/generativeFill/GenerativeFillButtons.scss
+++ b/src/client/views/nodes/imageEditor/GenerativeFillButtons.scss
diff --git a/src/client/views/nodes/generativeFill/GenerativeFillButtons.tsx b/src/client/views/nodes/imageEditor/GenerativeFillButtons.tsx
index fe22b273d..32ed6b307 100644
--- a/src/client/views/nodes/generativeFill/GenerativeFillButtons.tsx
+++ b/src/client/views/nodes/imageEditor/GenerativeFillButtons.tsx
@@ -3,7 +3,7 @@ import * as React from 'react';
import ReactLoading from 'react-loading';
import { Button, IconButton, Type } from 'browndash-components';
import { AiOutlineInfo } from 'react-icons/ai';
-import { activeColor } from './generativeFillUtils/generativeFillConstants';
+import { activeColor } from './imageEditorUtils/imageEditorConstants';
interface ButtonContainerProps {
onClick: () => Promise<void>;
diff --git a/src/client/views/nodes/generativeFill/GenerativeFill.scss b/src/client/views/nodes/imageEditor/ImageEditor.scss
index c2669a950..21c28f6da 100644
--- a/src/client/views/nodes/generativeFill/GenerativeFill.scss
+++ b/src/client/views/nodes/imageEditor/ImageEditor.scss
@@ -2,7 +2,7 @@ $navHeight: 5rem;
$canvasSize: 1024px;
$scale: 0.5;
-.generativeFillContainer {
+.imageEditorContainer {
position: absolute;
top: 0;
left: 0;
@@ -13,7 +13,7 @@ $scale: 0.5;
flex-direction: column;
overflow: hidden;
- .generativeFillControls {
+ .imageEditorTopBar {
flex-shrink: 0;
height: $navHeight;
color: #000000;
@@ -27,6 +27,12 @@ $scale: 0.5;
border-bottom: 1px solid #c7cdd0;
padding: 0 2rem;
+ .imageEditorControls {
+ display: flex;
+ align-items: center;
+ gap: 1.5rem;
+ }
+
h1 {
font-size: 1.5rem;
}
@@ -71,11 +77,31 @@ $scale: 0.5;
.iconContainer {
position: absolute;
- top: 2rem;
+ top: 3rem;
left: 2rem;
display: flex;
flex-direction: column;
- gap: 2rem;
+ gap: 4rem;
+
+ .imageToolsContainer {
+ display: flex;
+ flex-direction: column;
+ gap: 10px;
+ }
+
+ .undoRedoContainer {
+ justify-content: center;
+ display: flex;
+ flex-direction: row;
+ }
+
+ .sliderContainer {
+ height: 225px;
+ width: 100%;
+ display: flex;
+ justify-content: center;
+ cursor: pointer;
+ }
}
.editsBox {
@@ -86,7 +112,18 @@ $scale: 0.5;
flex-direction: column;
gap: 1rem;
+ .originalImageLabel {
+ position: absolute;
+ bottom: 10;
+ left: 10;
+ color: #ffffff;
+ font-size: 0.8rem;
+ letter-spacing: 1px;
+ text-transform: uppercase;
+ }
+
img {
+ cursor: pointer;
transition: all 0.2s ease-in-out;
&:hover {
opacity: 0.8;
diff --git a/src/client/views/nodes/generativeFill/GenerativeFill.tsx b/src/client/views/nodes/imageEditor/ImageEditor.tsx
index 261eb4bb4..86f7d8d29 100644
--- a/src/client/views/nodes/generativeFill/GenerativeFill.tsx
+++ b/src/client/views/nodes/imageEditor/ImageEditor.tsx
@@ -20,17 +20,18 @@ import { CollectionDockingView } from '../../collections/CollectionDockingView';
import { CollectionFreeFormView } from '../../collections/collectionFreeForm';
import { ImageEditorData } from '../ImageBox';
import { OpenWhereMod } from '../OpenWhere';
-import './GenerativeFill.scss';
-import { EditButtons, CutButtons } from './GenerativeFillButtons';
-import { BrushHandler, BrushType } from './generativeFillUtils/BrushHandler';
-import { APISuccess, ImageUtility } from './generativeFillUtils/ImageHandler';
-import { PointerHandler } from './generativeFillUtils/PointerHandler';
-import { activeColor, canvasSize, eraserColor, freeformRenderSize, newCollectionSize, offsetDistanceY, offsetX } from './generativeFillUtils/generativeFillConstants';
-import { CursorData, ImageDimensions, Point } from './generativeFillUtils/generativeFillInterfaces';
+import './ImageEditor.scss';
+import { ApplyFuncButtons, ImageToolButton } from './ImageEditorButtons';
+import { BrushHandler, BrushType } from './imageEditorUtils/BrushHandler';
+import { APISuccess, ImageUtility } from './imageEditorUtils/ImageHandler';
+import { PointerHandler } from './imageEditorUtils/PointerHandler';
+import { activeColor, canvasSize, eraserColor, freeformRenderSize, newCollectionSize, offsetDistanceY, offsetX } from './imageEditorUtils/imageEditorConstants';
+import { CursorData, ImageDimensions, ImageEditTool, ImageToolType, Point } from './imageEditorUtils/imageEditorInterfaces';
import { DocumentView } from '../DocumentView';
import { FontAwesomeIcon } from '@fortawesome/react-fontawesome';
import { ImageField } from '../../../../fields/URLField';
import { resolve } from 'url';
+import { DocData } from '../../../../fields/DocSymbols';
interface GenerativeFillProps {
imageEditorOpen: boolean;
@@ -41,7 +42,7 @@ interface GenerativeFillProps {
// Added field on image doc: gen_fill_children: List of children Docs
-const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addDoc }: GenerativeFillProps) => {
+const ImageEditor = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addDoc }: GenerativeFillProps) => {
const canvasRef = useRef<HTMLCanvasElement>(null);
const canvasBackgroundRef = useRef<HTMLCanvasElement>(null);
const drawingAreaRef = useRef<HTMLDivElement>(null);
@@ -55,6 +56,7 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD
// format: array of [image source, corresponding image Doc]
const [edits, setEdits] = useState<{ url: string; saveRes: Doc | undefined }[]>([]);
const [edited, setEdited] = useState(false);
+ const isFirstDoc = useRef(true);
// const [brushStyle] = useState<BrushStyle>(BrushStyle.ADD);
const [input, setInput] = useState('');
const [loading, setLoading] = useState(false);
@@ -82,6 +84,24 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD
// constants for image cutting
const cutPts = useRef<Point[]>([]);
+ /**
+ *
+ * @param type The new tool type we are changing to
+ */
+ const changeTool = (type: ImageToolType) => {
+ switch (type) {
+ case ImageToolType.GenerativeFill:
+ setCurrTool(genFillTool);
+ setCursorData(prev => ({ ...prev, width: genFillTool.sliderDefault as number }));
+ break;
+ case ImageToolType.Cut:
+ setCurrTool(cutTool);
+ setCursorData(prev => ({ ...prev, width: cutTool.sliderDefault as number }));
+ break;
+ default:
+ break;
+ }
+ };
// Undo and Redo
const handleUndo = () => {
const ctx = ImageUtility.getCanvasContext(canvasRef);
@@ -338,35 +358,32 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD
const img = currImg.current;
const canvas = canvasRef.current;
if (!canvas || !img) return;
- canvas.width = img.naturalWidth;
- canvas.height = img.naturalHeight;
const ctx = ImageUtility.getCanvasContext(canvasRef);
if (!ctx) return;
- ctx.globalCompositeOperation = 'source-over';
setLoading(true);
- setEdited(true);
// get the original image
const canvasOriginalImg = ImageUtility.getCanvasImg(img);
if (!canvasOriginalImg) return;
- // draw the image onto the canvas
- ctx.drawImage(img, 0, 0);
- // get the mask which i assume is the thing the user draws on
- // const canvasMask = ImageUtility.getCanvasMask(canvas, canvasOriginalImg);
- // if (!canvasMask) return;
- // canvasMask.width = canvas.width;
- // canvasMask.height = canvas.height;
- // now put the user's path around the mask
+ // NOTE: cutting two diff shapes can be made possible by having the user press a button to set a new shape!
+ let minX = img.width;
+ let maxX = 0;
+ let minY = img.height;
+ let maxY = 0;
if (cutPts.current.length) {
ctx.beginPath();
ctx.moveTo(cutPts.current[0].x, cutPts.current[0].y); // later check edge case where cutPts is empty
for (let i = 0; i < cutPts.current.length; i++) {
ctx.lineTo(cutPts.current[i].x, cutPts.current[i].y);
+ minX = Math.min(cutPts.current[i].x, minX);
+ minY = Math.min(cutPts.current[i].y, minY);
+ maxX = Math.max(cutPts.current[i].x, maxX);
+ maxY = Math.max(cutPts.current[i].y, maxY);
}
ctx.closePath();
- ctx.stroke();
+ ctx.globalCompositeOperation = 'destination-in';
ctx.fill();
- // ctx.clip();
}
+
const url = canvas.toDataURL(); // this does the same thing as convert img to canvasurl
if (!newCollectionRef.current) {
if (!isNewCollection && imageRootDoc) {
@@ -387,13 +404,33 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD
CollectionDockingView.AddSplit(newCollectionRef.current, OpenWhereMod.right);
}
}
+
const image = new Image();
image.src = url;
- await createNewImgDoc(image, true);
- // add the doc to the main freeform
- // eslint-disable-next-line no-use-before-define
- setLoading(false);
- cutPts.current.length = 0;
+ image.onload = async () => {
+ const croppedCanvas = document.createElement('canvas');
+ const croppedCtx = croppedCanvas.getContext('2d');
+ if (!croppedCtx) return;
+ croppedCanvas.width = maxX - minX;
+ croppedCanvas.height = maxY - minY;
+ croppedCtx.globalCompositeOperation = 'source-over';
+ croppedCtx.clearRect(0, 0, croppedCanvas.width, croppedCanvas.height);
+ croppedCtx.drawImage(image, -minX, -minY);
+ const croppedURL = croppedCanvas.toDataURL();
+ const croppedImage = new Image();
+ croppedImage.src = croppedURL;
+
+ currImg.current = croppedImage;
+ const newImgDoc = await createNewImgDoc(croppedImage, isFirstDoc.current);
+ if (isFirstDoc.current) isFirstDoc.current = false;
+ if (newImgDoc) {
+ const docData = newImgDoc[DocData];
+ docData.backgroundColor = 'transparent';
+ }
+ setEdits(prevEdits => [...prevEdits, { url: croppedURL, saveRes: undefined }]);
+ setLoading(false);
+ cutPts.current.length = 0;
+ };
};
// adjusts all the img positions to be aligned
@@ -416,7 +453,7 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD
};
// creates a new image document and returns its reference
- const createNewImgDoc = async (img: HTMLImageElement, firstDoc: boolean): Promise<Doc | undefined> => {
+ const createNewImgDoc = async (img: HTMLImageElement, firstDoc: boolean, parent?: Doc): Promise<Doc | undefined> => {
if (!imageRootDoc) return undefined;
const { src } = img;
const [result] = await Networking.PostToServer('/uploadRemoteImage', { sources: [src] });
@@ -497,12 +534,19 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD
setEdits([]);
};
- return (
- <div className="generativeFillContainer" style={{ display: imageEditorOpen ? 'flex' : 'none' }}>
- <div className="generativeFillControls">
+ // defines the tools and sets current tool
+ const genFillTool: ImageEditTool = { type: ImageToolType.GenerativeFill, name: 'Generative Fill', btnText: 'GET EDITS', icon: 'fill', applyFunc: getEdit, sliderMin: 25, sliderMax: 500, sliderDefault: 150 };
+ const cutTool: ImageEditTool = { type: ImageToolType.Cut, name: 'Cut', btnText: 'CUT IMAGE', icon: 'scissors', applyFunc: cutImage, sliderMin: 1, sliderMax: 50, sliderDefault: 5 };
+ const imageEditTools: ImageEditTool[] = [genFillTool, cutTool];
+ const [currTool, setCurrTool] = useState<ImageEditTool>(genFillTool);
+
+ // the top controls for making a new collection, resetting, and applying edits,
+ function renderControls() {
+ return (
+ <div className="imageEditorTopBar">
<h1>Image Editor</h1>
{/* <IconButton text="Cut out" icon={<FontAwesomeIcon icon="scissors" />} /> */}
- <div style={{ display: 'flex', alignItems: 'center', gap: '1.5rem' }}>
+ <div className="imageEditorControls">
<FormControlLabel
control={
<Checkbox
@@ -518,34 +562,62 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD
labelPlacement="end"
sx={{ whiteSpace: 'nowrap' }}
/>
- <EditButtons onClick={getEdit} loading={loading} onReset={handleReset} />
- <CutButtons onClick={cutImage} loading={loading} onReset={handleReset} />
+ <ApplyFuncButtons onClick={currTool.applyFunc} loading={loading} onReset={handleReset} btnText={currTool.btnText} />
<IconButton color={activeColor} tooltip="close" icon={<CgClose size="16px" />} onClick={handleViewClose} />
</div>
</div>
- {/* Main canvas for editing */}
- <div
- className="drawingArea" // this only works if pointerevents: none is set on the custom pointer
- ref={drawingAreaRef}
- onPointerOver={updateCursorData}
- onPointerMove={updateCursorData}
- onPointerDown={handlePointerDown}
- onPointerUp={handlePointerUp}>
- <canvas ref={canvasRef} width={canvasDims.width} height={canvasDims.height} style={{ transform: `scale(${canvasScale})` }} />
- <canvas ref={canvasBackgroundRef} width={canvasDims.width} height={canvasDims.height} style={{ transform: `scale(${canvasScale})` }} />
- <div
- className="pointer"
- style={{
- left: cursorData.x,
- top: cursorData.y,
- width: cursorData.width,
- height: cursorData.width,
- }}>
- <div className="innerPointer" />
+ );
+ }
+
+ // the side icons including tool type, the slider, and undo/redo
+ function renderSideIcons() {
+ return (
+ <div className="iconContainer">
+ <div className="imageToolsContainer">
+ {imageEditTools.map(tool => {
+ return ImageToolButton(tool, tool.type === currTool.type, changeTool);
+ })}
</div>
- {/* Icons */}
- <div className="iconContainer">
- {/* Undo and Redo */}
+ <div className="sliderContainer" onPointerDown={e => e.stopPropagation()}>
+ {currTool.type === ImageToolType.GenerativeFill && (
+ <Slider
+ sx={{
+ '& input[type="range"]': {
+ WebkitAppearance: 'slider-vertical',
+ },
+ }}
+ orientation="vertical"
+ min={genFillTool.sliderMin}
+ max={genFillTool.sliderMax}
+ defaultValue={genFillTool.sliderDefault}
+ size="small"
+ valueLabelDisplay="auto"
+ onChange={(e, val) => {
+ setCursorData(prev => ({ ...prev, width: val as number }));
+ }}
+ />
+ )}
+ {currTool.type === ImageToolType.Cut && (
+ <Slider
+ sx={{
+ '& input[type="range"]': {
+ WebkitAppearance: 'slider-vertical',
+ },
+ }}
+ orientation="vertical"
+ min={cutTool.sliderMin}
+ max={cutTool.sliderMax}
+ defaultValue={cutTool.sliderDefault}
+ size="small"
+ valueLabelDisplay="auto"
+ onChange={(e, val) => {
+ setCursorData(prev => ({ ...prev, width: val as number }));
+ }}
+ />
+ )}
+ </div>
+ {/* Undo and Redo */}
+ <div className="undoRedoContainer">
<IconButton
style={{ cursor: 'pointer' }}
onPointerDown={e => {
@@ -572,99 +644,77 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD
tooltip="Redo"
icon={<IoMdRedo />}
/>
- <div onPointerDown={e => e.stopPropagation()} style={{ height: 225, width: '100%', display: 'flex', justifyContent: 'center', cursor: 'pointer' }}>
- <Slider
- sx={{
- '& input[type="range"]': {
- WebkitAppearance: 'slider-vertical',
- },
- }}
- orientation="vertical"
- min={25}
- max={500}
- defaultValue={150}
- size="small"
- valueLabelDisplay="auto"
- onChange={(e: any, val: any) => {
- setCursorData(prev => ({ ...prev, width: val as number }));
- }}
- />
- </div>
- <div onPointerDown={e => e.stopPropagation()} style={{ height: 225, width: '100%', display: 'flex', justifyContent: 'center', cursor: 'pointer' }}>
- <Slider
- sx={{
- '& input[type="range"]': {
- WebkitAppearance: 'slider-vertical',
- },
- }}
- orientation="vertical"
- min={1}
- max={500}
- defaultValue={150}
- size="small"
- valueLabelDisplay="auto"
- onChange={(e: any, val: any) => {
- setCursorData(prev => ({ ...prev, width: val as number }));
- }}
- />
- </div>
</div>
- {/* Edits thumbnails */}
- <div className="editsBox">
- {edits.map((edit, i) => (
+ </div>
+ );
+ }
+
+ // circular pointer for drawing/erasing
+ function renderPointer() {
+ return (
+ <div
+ className="pointer"
+ style={{
+ left: cursorData.x,
+ top: cursorData.y,
+ width: cursorData.width,
+ height: cursorData.width,
+ }}>
+ <div className="innerPointer" />
+ </div>
+ );
+ }
+
+ // the previews for each edit
+ function renderEditThumbnails() {
+ return (
+ <div className="editsBox">
+ {edits.map((edit, i) => (
+ <img
+ // eslint-disable-next-line react/no-array-index-key
+ key={i}
+ alt="image edits"
+ width={75}
+ src={edit.url}
+ onClick={async () => {
+ const img = new Image();
+ img.src = edit.url;
+ ImageUtility.drawImgToCanvas(img, canvasRef, img.width, img.height);
+ currImg.current = img;
+ parentDoc.current = edit.saveRes ?? null;
+ }}
+ />
+ ))}
+ {/* Original img thumbnail */}
+ {edits.length > 0 && (
+ <div style={{ position: 'relative' }}>
+ <label className="originalImageLabel">Original</label>
<img
- // eslint-disable-next-line react/no-array-index-key
- key={i}
- alt="image edits"
+ alt="image stuff"
width={75}
- src={edit.url}
- style={{ cursor: 'pointer' }}
- onClick={async () => {
+ src={originalImg.current?.src}
+ onClick={() => {
+ if (!originalImg.current) return;
const img = new Image();
- img.src = edit.url;
+ img.src = originalImg.current.src;
ImageUtility.drawImgToCanvas(img, canvasRef, canvasDims.width, canvasDims.height);
currImg.current = img;
- parentDoc.current = edit.saveRes ?? null;
+ if (!parentDoc.current) parentDoc.current = originalDoc.current;
}}
/>
- ))}
- {/* Original img thumbnail */}
- {edits.length > 0 && (
- <div style={{ position: 'relative' }}>
- <label
- style={{
- position: 'absolute',
- bottom: 10,
- left: 10,
- color: '#ffffff',
- fontSize: '0.8rem',
- letterSpacing: '1px',
- textTransform: 'uppercase',
- }}>
- Original
- </label>
- <img
- alt="image stuff"
- width={75}
- src={originalImg.current?.src}
- style={{ cursor: 'pointer' }}
- onClick={() => {
- if (!originalImg.current) return;
- const img = new Image();
- img.src = originalImg.current.src;
- ImageUtility.drawImgToCanvas(img, canvasRef, canvasDims.width, canvasDims.height);
- currImg.current = img;
- parentDoc.current = originalDoc.current;
- }}
- />
- </div>
- )}
- </div>
+ </div>
+ )}
</div>
+ );
+ }
+
+ // the prompt box for generative fill
+ function renderPromptBox() {
+ return (
<div>
<TextField
value={input}
- onChange={(e: any) => setInput(e.target.value)}
+ onChange={e => setInput(e.target.value)}
disabled={isBrushing}
type="text"
label="Prompt"
@@ -680,8 +730,29 @@ const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addD
}}
/>
</div>
+ );
+ }
+
+ return (
+ <div className="imageEditorContainer" style={{ display: imageEditorOpen ? 'flex' : 'none' }}>
+ {renderControls()}
+ {/* Main canvas for editing */}
+ <div
+ className="drawingArea" // this only works if pointerevents: none is set on the custom pointer
+ ref={drawingAreaRef}
+ onPointerOver={updateCursorData}
+ onPointerMove={updateCursorData}
+ onPointerDown={handlePointerDown}
+ onPointerUp={handlePointerUp}>
+ <canvas ref={canvasRef} width={canvasDims.width} height={canvasDims.height} style={{ transform: `scale(${canvasScale})` }} />
+ <canvas ref={canvasBackgroundRef} width={canvasDims.width} height={canvasDims.height} style={{ transform: `scale(${canvasScale})` }} />
+ {renderPointer()}
+ {renderSideIcons()}
+ {renderEditThumbnails()}
+ </div>
+ {currTool.type === ImageToolType.GenerativeFill && renderPromptBox()}
</div>
);
};
-export default GenerativeFill;
+export default ImageEditor;
diff --git a/src/client/views/nodes/imageEditor/ImageEditorButtons.tsx b/src/client/views/nodes/imageEditor/ImageEditorButtons.tsx
new file mode 100644
index 000000000..e90babf9b
--- /dev/null
+++ b/src/client/views/nodes/imageEditor/ImageEditorButtons.tsx
@@ -0,0 +1,69 @@
+import './GenerativeFillButtons.scss';
+import * as React from 'react';
+import ReactLoading from 'react-loading';
+import { Button, IconButton, Type } from 'browndash-components';
+import { AiOutlineInfo } from 'react-icons/ai';
+import { bgColor } from './imageEditorUtils/imageEditorConstants';
+import { ImageEditTool, ImageToolType } from './imageEditorUtils/imageEditorInterfaces';
+import { FontAwesomeIcon } from '@fortawesome/react-fontawesome';
+import { SettingsManager } from '../../../util/SettingsManager';
+
+interface ButtonContainerProps {
+ onClick: () => Promise<void>;
+ loading: boolean;
+ onReset: () => void;
+ btnText: string;
+}
+
+export function ApplyFuncButtons({ loading, onClick: getEdit, onReset, btnText }: ButtonContainerProps) {
+ return (
+ <div className="generativeFillBtnContainer">
+ <Button text="RESET" type={Type.PRIM} color={SettingsManager.userVariantColor} onClick={onReset} />
+ {loading ? (
+ <Button
+ text={btnText}
+ type={Type.TERT}
+ color={SettingsManager.userVariantColor}
+ icon={<ReactLoading type="spin" color="#ffffff" width={20} height={20} />}
+ iconPlacement="right"
+ onClick={() => {
+ if (!loading) getEdit();
+ }}
+ />
+ ) : (
+ <Button
+ text={btnText}
+ type={Type.TERT}
+ color={SettingsManager.userVariantColor}
+ onClick={() => {
+ if (!loading) getEdit();
+ }}
+ />
+ )}
+ <IconButton
+ type={Type.SEC}
+ color={SettingsManager.userVariantColor}
+ tooltip="Open Documentation"
+ icon={<AiOutlineInfo size="16px" />}
+ onClick={() => window.open('https://brown-dash.github.io/Dash-Documentation/features/generativeai/#editing', '_blank')}
+ />
+ </div>
+ );
+}
+
+export function ImageToolButton(tool: ImageEditTool, isActive: boolean, selectTool: (type: ImageToolType) => void) {
+ return (
+ <div className="imageEditorButtonContainer">
+ <Button
+ style={{ width: '100%', border: '1px' }}
+ text={tool.name}
+ type={Type.TERT}
+ color={isActive ? SettingsManager.userVariantColor : bgColor}
+ icon={<FontAwesomeIcon icon={tool.icon} />}
+ onClick={() => {
+ selectTool(tool.type);
+ }}
+ />
+ </div>
+ );
+}
diff --git a/src/client/views/nodes/generativeFill/generativeFillUtils/ImageHandler.ts b/src/client/views/nodes/imageEditor/imageEditingUtils/ImageHandler.ts
index 24dba1778..514e8a94f 100644
--- a/src/client/views/nodes/generativeFill/generativeFillUtils/ImageHandler.ts
+++ b/src/client/views/nodes/imageEditor/imageEditingUtils/ImageHandler.ts
@@ -1,5 +1,5 @@
import { RefObject } from 'react';
-import { bgColor, canvasSize } from './generativeFillConstants';
+import { bgColor, canvasSize } from '../ImageEditorUtils/imageEditorConstants';
export interface APISuccess {
status: 'success';
diff --git a/src/client/views/nodes/generativeFill/generativeFillUtils/BrushHandler.ts b/src/client/views/nodes/imageEditor/imageEditorUtils/BrushHandler.ts
index 8a66d7347..a9fe02d4f 100644
--- a/src/client/views/nodes/generativeFill/generativeFillUtils/BrushHandler.ts
+++ b/src/client/views/nodes/imageEditor/imageEditorUtils/BrushHandler.ts
@@ -1,6 +1,6 @@
import { GenerativeFillMathHelpers } from './GenerativeFillMathHelpers';
-import { eraserColor } from './generativeFillConstants';
-import { Point } from './generativeFillInterfaces';
+import { eraserColor } from './imageEditorConstants';
+import { Point } from './imageEditorInterfaces';
import { points } from '@turf/turf';
export enum BrushType {
diff --git a/src/client/views/nodes/generativeFill/generativeFillUtils/GenerativeFillMathHelpers.ts b/src/client/views/nodes/imageEditor/imageEditorUtils/GenerativeFillMathHelpers.ts
index 6da8c3da0..f820300f3 100644
--- a/src/client/views/nodes/generativeFill/generativeFillUtils/GenerativeFillMathHelpers.ts
+++ b/src/client/views/nodes/imageEditor/imageEditorUtils/GenerativeFillMathHelpers.ts
@@ -1,4 +1,4 @@
-import { Point } from './generativeFillInterfaces';
+import { Point } from './imageEditorInterfaces';
export class GenerativeFillMathHelpers {
static distanceBetween = (p1: Point, p2: Point) => Math.sqrt((p2.x - p1.x) ** 2 + (p2.y - p1.y) ** 2);
diff --git a/src/client/views/nodes/imageEditor/imageEditorUtils/ImageHandler.ts b/src/client/views/nodes/imageEditor/imageEditorUtils/ImageHandler.ts
new file mode 100644
index 000000000..ece0f4d7f
--- /dev/null
+++ b/src/client/views/nodes/imageEditor/imageEditorUtils/ImageHandler.ts
@@ -0,0 +1,312 @@
+import { RefObject } from 'react';
+import { bgColor, canvasSize } from './imageEditorConstants';
+
+export interface APISuccess {
+ status: 'success';
+ urls: string[];
+}
+
+export interface APIError {
+ status: 'error';
+ message: string;
+}
+
+export class ImageUtility {
+ /**
+ *
+ * @param canvas Canvas to convert
+ * @returns Blob of canvas
+ */
+ static canvasToBlob = (canvas: HTMLCanvasElement): Promise<Blob> =>
+ new Promise(resolve => {
+ canvas.toBlob(blob => {
+ if (blob) {
+ resolve(blob);
+ }
+ }, 'image/png');
+ });
+
+ // given a square api image, get the cropped img
+ static getCroppedImg = (img: HTMLImageElement, width: number, height: number): HTMLCanvasElement | undefined => {
+ // Create a new canvas element
+ const canvas = document.createElement('canvas');
+ canvas.width = width;
+ canvas.height = height;
+ const ctx = canvas.getContext('2d');
+ if (ctx) {
+ // Clear the canvas
+ ctx.clearRect(0, 0, canvas.width, canvas.height);
+ if (width < height) {
+ // horizontal padding, x offset
+ const xOffset = (canvasSize - width) / 2;
+ ctx.drawImage(img, xOffset, 0, canvas.width, canvas.height, 0, 0, canvas.width, canvas.height);
+ } else {
+ // vertical padding, y offset
+ const yOffset = (canvasSize - height) / 2;
+ ctx.drawImage(img, 0, yOffset, canvas.width, canvas.height, 0, 0, canvas.width, canvas.height);
+ }
+ return canvas;
+ }
+ return undefined;
+ };
+
+ // converts an image to a canvas data url
+ static convertImgToCanvasUrl = async (imageSrc: string, width: number, height: number): Promise<string> =>
+ new Promise<string>((resolve, reject) => {
+ const img = new Image();
+ img.onload = () => {
+ const canvas = this.getCroppedImg(img, width, height);
+ if (canvas) {
+ const dataUrl = canvas.toDataURL();
+ resolve(dataUrl);
+ }
+ };
+ img.onerror = error => {
+ reject(error);
+ };
+ img.src = imageSrc;
+ });
+
+ // calls the openai api to get image edits
+ static getEdit = async (imgBlob: Blob, maskBlob: Blob, prompt: string, n?: number): Promise<APISuccess | APIError> => {
+ const apiUrl = 'https://api.openai.com/v1/images/edits';
+ const fd = new FormData();
+ fd.append('image', imgBlob, 'image.png');
+ fd.append('mask', maskBlob, 'mask.png');
+ fd.append('prompt', prompt);
+ fd.append('size', '1024x1024');
+ fd.append('n', n ? JSON.stringify(n) : '1');
+ fd.append('response_format', 'b64_json');
+
+ try {
+ const res = await fetch(apiUrl, {
+ method: 'POST',
+ headers: {
+ Authorization: `Bearer ${process.env.OPENAI_KEY}`,
+ },
+ body: fd,
+ });
+ const data = await res.json();
+ console.log(data.data);
+ return {
+ status: 'success',
+ urls: (data.data as { b64_json: string }[]).map(urlData => `data:image/png;base64,${urlData.b64_json}`),
+ };
+ } catch (err) {
+ console.log(err);
+ return { status: 'error', message: 'API error.' };
+ }
+ };
+
+ // mock api call
+ static mockGetEdit = async (mockSrc: string): Promise<APISuccess | APIError> => ({
+ status: 'success',
+ urls: [mockSrc, mockSrc, mockSrc],
+ });
+
+ // Gets the canvas rendering context of a canvas
+ static getCanvasContext = (canvasRef: RefObject<HTMLCanvasElement>): CanvasRenderingContext2D | null => {
+ if (!canvasRef.current) return null;
+ const ctx = canvasRef.current.getContext('2d');
+ if (!ctx) return null;
+ return ctx;
+ };
+
+ // Helper for downloading the canvas (for debugging)
+ static downloadCanvas = (canvas: HTMLCanvasElement) => {
+ const url = canvas.toDataURL();
+ const downloadLink = document.createElement('a');
+ downloadLink.href = url;
+ downloadLink.download = 'canvas';
+
+ downloadLink.click();
+ downloadLink.remove();
+ };
+
+ // Download the canvas (for debugging)
+ static downloadImageCanvas = (imgUrl: string) => {
+ const img = new Image();
+ img.src = imgUrl;
+ img.onload = () => {
+ const canvas = document.createElement('canvas');
+ canvas.width = canvasSize;
+ canvas.height = canvasSize;
+ const ctx = canvas.getContext('2d');
+ ctx?.drawImage(img, 0, 0, canvasSize, canvasSize);
+
+ this.downloadCanvas(canvas);
+ };
+ };
+
+ // Clears the canvas
+ static clearCanvas = (canvasRef: React.RefObject<HTMLCanvasElement>) => {
+ const ctx = this.getCanvasContext(canvasRef);
+ if (!ctx || !canvasRef.current) return;
+ ctx.clearRect(0, 0, canvasRef.current.width, canvasRef.current.height);
+ };
+
+ // Draws the image to the current canvas
+ static drawImgToCanvas = (img: HTMLImageElement, canvasRef: React.RefObject<HTMLCanvasElement>, width: number, height: number) => {
+ const drawImg = (htmlImg: HTMLImageElement) => {
+ const ctx = this.getCanvasContext(canvasRef);
+ if (!ctx) return;
+ ctx.globalCompositeOperation = 'source-over';
+ ctx.clearRect(0, 0, canvasRef.current?.width || width, canvasRef.current?.height || height);
+ ctx.drawImage(htmlImg, 0, 0, width, height);
+ };
+
+ if (img.complete) {
+ drawImg(img);
+ } else {
+ img.onload = () => {
+ drawImg(img);
+ };
+ }
+ };
+
+ // Gets the image mask for the openai endpoint
+ static getCanvasMask = (srcCanvas: HTMLCanvasElement, paddedCanvas: HTMLCanvasElement): HTMLCanvasElement | undefined => {
+ const canvas = document.createElement('canvas');
+ canvas.width = canvasSize;
+ canvas.height = canvasSize;
+ const ctx = canvas.getContext('2d');
+ if (!ctx) return undefined;
+ ctx?.clearRect(0, 0, canvasSize, canvasSize);
+ ctx.drawImage(paddedCanvas, 0, 0);
+
+ // extract and set padding data
+ if (srcCanvas.height > srcCanvas.width) {
+ // horizontal padding, x offset
+ const xOffset = (canvasSize - srcCanvas.width) / 2;
+ ctx?.clearRect(xOffset, 0, srcCanvas.width, srcCanvas.height);
+ ctx.drawImage(srcCanvas, xOffset, 0, srcCanvas.width, srcCanvas.height);
+ } else {
+ // vertical padding, y offset
+ const yOffset = (canvasSize - srcCanvas.height) / 2;
+ ctx?.clearRect(0, yOffset, srcCanvas.width, srcCanvas.height);
+ ctx.drawImage(srcCanvas, 0, yOffset, srcCanvas.width, srcCanvas.height);
+ }
+ return canvas;
+ };
+
+ // Fills in the blank areas of the image with an image reflection (to fill in a square-shaped canvas)
+ static drawHorizontalReflection = (ctx: CanvasRenderingContext2D, canvas: HTMLCanvasElement, xOffset: number) => {
+ const imageData = ctx.getImageData(0, 0, canvas.width, canvas.height);
+ const { data } = imageData;
+ for (let i = 0; i < canvas.height; i++) {
+ for (let j = 0; j < xOffset; j++) {
+ const targetIdx = 4 * (i * canvas.width + j);
+ const sourceI = i;
+ const sourceJ = xOffset + (xOffset - j);
+ const sourceIdx = 4 * (sourceI * canvas.width + sourceJ);
+ data[targetIdx] = data[sourceIdx];
+ data[targetIdx + 1] = data[sourceIdx + 1];
+ data[targetIdx + 2] = data[sourceIdx + 2];
+ }
+ }
+ for (let i = 0; i < canvas.height; i++) {
+ for (let j = canvas.width - 1; j >= canvas.width - 1 - xOffset; j--) {
+ const targetIdx = 4 * (i * canvas.width + j);
+ const sourceI = i;
+ const sourceJ = canvas.width - 1 - xOffset - (xOffset - (canvas.width - j));
+ const sourceIdx = 4 * (sourceI * canvas.width + sourceJ);
+ data[targetIdx] = data[sourceIdx];
+ data[targetIdx + 1] = data[sourceIdx + 1];
+ data[targetIdx + 2] = data[sourceIdx + 2];
+ }
+ }
+ ctx.putImageData(imageData, 0, 0);
+ };
+
+ // Fills in the blank areas of the image with an image reflection (to fill in a square-shaped canvas)
+ static drawVerticalReflection = (ctx: CanvasRenderingContext2D, canvas: HTMLCanvasElement, yOffset: number) => {
+ const imageData = ctx.getImageData(0, 0, canvas.width, canvas.height);
+ const { data } = imageData;
+ for (let j = 0; j < canvas.width; j++) {
+ for (let i = 0; i < yOffset; i++) {
+ const targetIdx = 4 * (i * canvas.width + j);
+ const sourceJ = j;
+ const sourceI = yOffset + (yOffset - i);
+ const sourceIdx = 4 * (sourceI * canvas.width + sourceJ);
+ data[targetIdx] = data[sourceIdx];
+ data[targetIdx + 1] = data[sourceIdx + 1];
+ data[targetIdx + 2] = data[sourceIdx + 2];
+ }
+ }
+ for (let j = 0; j < canvas.width; j++) {
+ for (let i = canvas.height - 1; i >= canvas.height - 1 - yOffset; i--) {
+ const targetIdx = 4 * (i * canvas.width + j);
+ const sourceJ = j;
+ const sourceI = canvas.height - 1 - yOffset - (yOffset - (canvas.height - i));
+ const sourceIdx = 4 * (sourceI * canvas.width + sourceJ);
+ data[targetIdx] = data[sourceIdx];
+ data[targetIdx + 1] = data[sourceIdx + 1];
+ data[targetIdx + 2] = data[sourceIdx + 2];
+ }
+ }
+ ctx.putImageData(imageData, 0, 0);
+ };
+
+ // Gets the unaltered (besides filling in padding) version of the image for the api call
+ static getCanvasImg = (img: HTMLImageElement): HTMLCanvasElement | undefined => {
+ const canvas = document.createElement('canvas');
+ canvas.width = canvasSize;
+ canvas.height = canvasSize;
+ const ctx = canvas.getContext('2d');
+ if (!ctx) return undefined;
+ // fix scaling
+ const scale = Math.min(canvasSize / img.width, canvasSize / img.height);
+ const width = Math.floor(img.width * scale);
+ const height = Math.floor(img.height * scale);
+ ctx?.clearRect(0, 0, canvasSize, canvasSize);
+ ctx.fillStyle = bgColor;
+ ctx.fillRect(0, 0, canvasSize, canvasSize);
+
+ // extract and set padding data
+ if (img.naturalHeight > img.naturalWidth) {
+ // horizontal padding, x offset
+ const xOffset = Math.floor((canvasSize - width) / 2);
+ ctx.drawImage(img, xOffset, 0, width, height);
+
+ // draw reflected image padding
+ this.drawHorizontalReflection(ctx, canvas, xOffset);
+ } else {
+ // vertical padding, y offset
+ const yOffset = Math.floor((canvasSize - height) / 2);
+ ctx.drawImage(img, 0, yOffset, width, height);
+
+ // draw reflected image padding
+ this.drawVerticalReflection(ctx, canvas, yOffset);
+ }
+ return canvas;
+ };
+
+ /**
+ * Converts a url to base64 (tainted canvas workaround)
+ */
+ static urlToBase64 = async (imageUrl: string): Promise<string | undefined> => {
+ try {
+ const res = await fetch(imageUrl);
+ const blob = await res.blob();
+
+ return new Promise<string>((resolve, reject) => {
+ const reader = new FileReader();
+ reader.onload = () => {
+ const base64Data = reader.result?.toString().split(',')[1];
+ if (base64Data) {
+ resolve(base64Data);
+ } else {
+ reject(new Error('Failed to convert.'));
+ }
+ };
+ reader.onerror = () => {
+ reject(new Error('Error reading image data'));
+ };
+ reader.readAsDataURL(blob);
+ });
+ } catch (err) {
+ console.error(err);
+ }
+ return undefined;
+ };
+}
diff --git a/src/client/views/nodes/generativeFill/generativeFillUtils/PointerHandler.ts b/src/client/views/nodes/imageEditor/imageEditorUtils/PointerHandler.ts
index 260923a64..e86f46636 100644
--- a/src/client/views/nodes/generativeFill/generativeFillUtils/PointerHandler.ts
+++ b/src/client/views/nodes/imageEditor/imageEditorUtils/PointerHandler.ts
@@ -1,4 +1,4 @@
-import { Point } from './generativeFillInterfaces';
+import { Point } from './imageEditorInterfaces';
export class PointerHandler {
static getPointRelativeToElement = (element: HTMLElement, e: React.PointerEvent | PointerEvent, scale: number): Point => {
diff --git a/src/client/views/nodes/generativeFill/generativeFillUtils/generativeFillConstants.ts b/src/client/views/nodes/imageEditor/imageEditorUtils/imageEditorConstants.ts
index 4772304bc..4772304bc 100644
--- a/src/client/views/nodes/generativeFill/generativeFillUtils/generativeFillConstants.ts
+++ b/src/client/views/nodes/imageEditor/imageEditorUtils/imageEditorConstants.ts
diff --git a/src/client/views/nodes/imageEditor/imageEditorUtils/imageEditorInterfaces.ts b/src/client/views/nodes/imageEditor/imageEditorUtils/imageEditorInterfaces.ts
new file mode 100644
index 000000000..f4ae7d9c4
--- /dev/null
+++ b/src/client/views/nodes/imageEditor/imageEditorUtils/imageEditorInterfaces.ts
@@ -0,0 +1,38 @@
+import { IconProp } from '@fortawesome/fontawesome-svg-core';
+
+export interface CursorData {
+ x: number;
+ y: number;
+ width: number;
+}
+
+export interface Point {
+ x: number;
+ y: number;
+}
+
+export enum ImageToolType {
+ GenerativeFill = 'genFill',
+ Cut = 'cut',
+}
+
+export interface ImageEditTool {
+ type: ImageToolType;
+ name: string;
+ btnText: string;
+ icon: IconProp;
+ applyFunc: () => Promise<void>;
+ sliderMin?: number;
+ sliderMax?: number;
+ sliderDefault?: number;
+}
+
+export enum BrushMode {
+ ADD,
+ SUBTRACT,
+}
+
+export interface ImageDimensions {
+ width: number;
+ height: number;
+}
diff --git a/src/client/views/nodes/imageEditor/imageToolUtils/BrushHandler.ts b/src/client/views/nodes/imageEditor/imageToolUtils/BrushHandler.ts
new file mode 100644
index 000000000..7139bebc3
--- /dev/null
+++ b/src/client/views/nodes/imageEditor/imageToolUtils/BrushHandler.ts
@@ -0,0 +1,35 @@
+import { GenerativeFillMathHelpers } from '../imageEditorUtils/GenerativeFillMathHelpers';
+import { eraserColor } from '../imageEditorUtils/imageEditorConstants';
+import { Point } from '../imageEditorUtils/imageEditorInterfaces';
+import { points } from '@turf/turf';
+
+export enum BrushType {
+ GEN_FILL,
+ CUT,
+}
+
+export class BrushHandler {
+ static brushCircleOverlay = (x: number, y: number, brushRadius: number, ctx: CanvasRenderingContext2D, fillColor: string /* , erase: boolean */) => {
+ ctx.globalCompositeOperation = 'destination-out';
+ ctx.fillStyle = fillColor;
+ ctx.shadowColor = eraserColor;
+ ctx.shadowBlur = 5;
+ ctx.beginPath();
+ ctx.arc(x, y, brushRadius, 0, 2 * Math.PI);
+ ctx.fill();
+ ctx.closePath();
+ };
+
+ static createBrushPathOverlay = (startPoint: Point, endPoint: Point, brushRadius: number, ctx: CanvasRenderingContext2D, fillColor: string, brushType: BrushType) => {
+ const dist = GenerativeFillMathHelpers.distanceBetween(startPoint, endPoint);
+ const pts: Point[] = [];
+ for (let i = 0; i < dist; i += 5) {
+ const s = i / dist;
+ const x = startPoint.x * (1 - s) + endPoint.x * s;
+ const y = startPoint.y * (1 - s) + endPoint.y * s;
+ pts.push({ x: startPoint.x, y: startPoint.y });
+ BrushHandler.brushCircleOverlay(x, y, brushRadius, ctx, fillColor);
+ }
+ return pts;
+ };
+}
diff --git a/src/client/views/nodes/imageEditor/imageToolUtils/ImageHandler.ts b/src/client/views/nodes/imageEditor/imageToolUtils/ImageHandler.ts
new file mode 100644
index 000000000..b9723b5be
--- /dev/null
+++ b/src/client/views/nodes/imageEditor/imageToolUtils/ImageHandler.ts
@@ -0,0 +1,312 @@
+import { RefObject } from 'react';
+import { bgColor, canvasSize } from '../imageEditorUtils/imageEditorConstants';
+
+export interface APISuccess {
+ status: 'success';
+ urls: string[];
+}
+
+export interface APIError {
+ status: 'error';
+ message: string;
+}
+
+export class ImageUtility {
+ /**
+ *
+ * @param canvas Canvas to convert
+ * @returns Blob of canvas
+ */
+ static canvasToBlob = (canvas: HTMLCanvasElement): Promise<Blob> =>
+ new Promise(resolve => {
+ canvas.toBlob(blob => {
+ if (blob) {
+ resolve(blob);
+ }
+ }, 'image/png');
+ });
+
+ // given a square api image, get the cropped img
+ static getCroppedImg = (img: HTMLImageElement, width: number, height: number): HTMLCanvasElement | undefined => {
+ // Create a new canvas element
+ const canvas = document.createElement('canvas');
+ canvas.width = width;
+ canvas.height = height;
+ const ctx = canvas.getContext('2d');
+ if (ctx) {
+ // Clear the canvas
+ ctx.clearRect(0, 0, canvas.width, canvas.height);
+ if (width < height) {
+ // horizontal padding, x offset
+ const xOffset = (canvasSize - width) / 2;
+ ctx.drawImage(img, xOffset, 0, canvas.width, canvas.height, 0, 0, canvas.width, canvas.height);
+ } else {
+ // vertical padding, y offset
+ const yOffset = (canvasSize - height) / 2;
+ ctx.drawImage(img, 0, yOffset, canvas.width, canvas.height, 0, 0, canvas.width, canvas.height);
+ }
+ return canvas;
+ }
+ return undefined;
+ };
+
+ // converts an image to a canvas data url
+ static convertImgToCanvasUrl = async (imageSrc: string, width: number, height: number): Promise<string> =>
+ new Promise<string>((resolve, reject) => {
+ const img = new Image();
+ img.onload = () => {
+ const canvas = this.getCroppedImg(img, width, height);
+ if (canvas) {
+ const dataUrl = canvas.toDataURL();
+ resolve(dataUrl);
+ }
+ };
+ img.onerror = error => {
+ reject(error);
+ };
+ img.src = imageSrc;
+ });
+
+ // calls the openai api to get image edits
+ static getEdit = async (imgBlob: Blob, maskBlob: Blob, prompt: string, n?: number): Promise<APISuccess | APIError> => {
+ const apiUrl = 'https://api.openai.com/v1/images/edits';
+ const fd = new FormData();
+ fd.append('image', imgBlob, 'image.png');
+ fd.append('mask', maskBlob, 'mask.png');
+ fd.append('prompt', prompt);
+ fd.append('size', '1024x1024');
+ fd.append('n', n ? JSON.stringify(n) : '1');
+ fd.append('response_format', 'b64_json');
+
+ try {
+ const res = await fetch(apiUrl, {
+ method: 'POST',
+ headers: {
+ Authorization: `Bearer ${process.env.OPENAI_KEY}`,
+ },
+ body: fd,
+ });
+ const data = await res.json();
+ console.log(data.data);
+ return {
+ status: 'success',
+ urls: (data.data as { b64_json: string }[]).map(urlData => `data:image/png;base64,${urlData.b64_json}`),
+ };
+ } catch (err) {
+ console.log(err);
+ return { status: 'error', message: 'API error.' };
+ }
+ };
+
+ // mock api call
+ static mockGetEdit = async (mockSrc: string): Promise<APISuccess | APIError> => ({
+ status: 'success',
+ urls: [mockSrc, mockSrc, mockSrc],
+ });
+
+ // Gets the canvas rendering context of a canvas
+ static getCanvasContext = (canvasRef: RefObject<HTMLCanvasElement>): CanvasRenderingContext2D | null => {
+ if (!canvasRef.current) return null;
+ const ctx = canvasRef.current.getContext('2d');
+ if (!ctx) return null;
+ return ctx;
+ };
+
+ // Helper for downloading the canvas (for debugging)
+ static downloadCanvas = (canvas: HTMLCanvasElement) => {
+ const url = canvas.toDataURL();
+ const downloadLink = document.createElement('a');
+ downloadLink.href = url;
+ downloadLink.download = 'canvas';
+
+ downloadLink.click();
+ downloadLink.remove();
+ };
+
+ // Download the canvas (for debugging)
+ static downloadImageCanvas = (imgUrl: string) => {
+ const img = new Image();
+ img.src = imgUrl;
+ img.onload = () => {
+ const canvas = document.createElement('canvas');
+ canvas.width = canvasSize;
+ canvas.height = canvasSize;
+ const ctx = canvas.getContext('2d');
+ ctx?.drawImage(img, 0, 0, canvasSize, canvasSize);
+
+ this.downloadCanvas(canvas);
+ };
+ };
+
+ // Clears the canvas
+ static clearCanvas = (canvasRef: React.RefObject<HTMLCanvasElement>) => {
+ const ctx = this.getCanvasContext(canvasRef);
+ if (!ctx || !canvasRef.current) return;
+ ctx.clearRect(0, 0, canvasRef.current.width, canvasRef.current.height);
+ };
+
+ // Draws the image to the current canvas
+ static drawImgToCanvas = (img: HTMLImageElement, canvasRef: React.RefObject<HTMLCanvasElement>, width: number, height: number) => {
+ const drawImg = (htmlImg: HTMLImageElement) => {
+ const ctx = this.getCanvasContext(canvasRef);
+ if (!ctx) return;
+ ctx.globalCompositeOperation = 'source-over';
+ ctx.clearRect(0, 0, width, height);
+ ctx.drawImage(htmlImg, 0, 0, width, height);
+ };
+
+ if (img.complete) {
+ drawImg(img);
+ } else {
+ img.onload = () => {
+ drawImg(img);
+ };
+ }
+ };
+
+ // Gets the image mask for the openai endpoint
+ static getCanvasMask = (srcCanvas: HTMLCanvasElement, paddedCanvas: HTMLCanvasElement): HTMLCanvasElement | undefined => {
+ const canvas = document.createElement('canvas');
+ canvas.width = canvasSize;
+ canvas.height = canvasSize;
+ const ctx = canvas.getContext('2d');
+ if (!ctx) return undefined;
+ ctx?.clearRect(0, 0, canvasSize, canvasSize);
+ ctx.drawImage(paddedCanvas, 0, 0);
+
+ // extract and set padding data
+ if (srcCanvas.height > srcCanvas.width) {
+ // horizontal padding, x offset
+ const xOffset = (canvasSize - srcCanvas.width) / 2;
+ ctx?.clearRect(xOffset, 0, srcCanvas.width, srcCanvas.height);
+ ctx.drawImage(srcCanvas, xOffset, 0, srcCanvas.width, srcCanvas.height);
+ } else {
+ // vertical padding, y offset
+ const yOffset = (canvasSize - srcCanvas.height) / 2;
+ ctx?.clearRect(0, yOffset, srcCanvas.width, srcCanvas.height);
+ ctx.drawImage(srcCanvas, 0, yOffset, srcCanvas.width, srcCanvas.height);
+ }
+ return canvas;
+ };
+
+ // Fills in the blank areas of the image with an image reflection (to fill in a square-shaped canvas)
+ static drawHorizontalReflection = (ctx: CanvasRenderingContext2D, canvas: HTMLCanvasElement, xOffset: number) => {
+ const imageData = ctx.getImageData(0, 0, canvas.width, canvas.height);
+ const { data } = imageData;
+ for (let i = 0; i < canvas.height; i++) {
+ for (let j = 0; j < xOffset; j++) {
+ const targetIdx = 4 * (i * canvas.width + j);
+ const sourceI = i;
+ const sourceJ = xOffset + (xOffset - j);
+ const sourceIdx = 4 * (sourceI * canvas.width + sourceJ);
+ data[targetIdx] = data[sourceIdx];
+ data[targetIdx + 1] = data[sourceIdx + 1];
+ data[targetIdx + 2] = data[sourceIdx + 2];
+ }
+ }
+ for (let i = 0; i < canvas.height; i++) {
+ for (let j = canvas.width - 1; j >= canvas.width - 1 - xOffset; j--) {
+ const targetIdx = 4 * (i * canvas.width + j);
+ const sourceI = i;
+ const sourceJ = canvas.width - 1 - xOffset - (xOffset - (canvas.width - j));
+ const sourceIdx = 4 * (sourceI * canvas.width + sourceJ);
+ data[targetIdx] = data[sourceIdx];
+ data[targetIdx + 1] = data[sourceIdx + 1];
+ data[targetIdx + 2] = data[sourceIdx + 2];
+ }
+ }
+ ctx.putImageData(imageData, 0, 0);
+ };
+
+ // Fills in the blank areas of the image with an image reflection (to fill in a square-shaped canvas)
+ static drawVerticalReflection = (ctx: CanvasRenderingContext2D, canvas: HTMLCanvasElement, yOffset: number) => {
+ const imageData = ctx.getImageData(0, 0, canvas.width, canvas.height);
+ const { data } = imageData;
+ for (let j = 0; j < canvas.width; j++) {
+ for (let i = 0; i < yOffset; i++) {
+ const targetIdx = 4 * (i * canvas.width + j);
+ const sourceJ = j;
+ const sourceI = yOffset + (yOffset - i);
+ const sourceIdx = 4 * (sourceI * canvas.width + sourceJ);
+ data[targetIdx] = data[sourceIdx];
+ data[targetIdx + 1] = data[sourceIdx + 1];
+ data[targetIdx + 2] = data[sourceIdx + 2];
+ }
+ }
+ for (let j = 0; j < canvas.width; j++) {
+ for (let i = canvas.height - 1; i >= canvas.height - 1 - yOffset; i--) {
+ const targetIdx = 4 * (i * canvas.width + j);
+ const sourceJ = j;
+ const sourceI = canvas.height - 1 - yOffset - (yOffset - (canvas.height - i));
+ const sourceIdx = 4 * (sourceI * canvas.width + sourceJ);
+ data[targetIdx] = data[sourceIdx];
+ data[targetIdx + 1] = data[sourceIdx + 1];
+ data[targetIdx + 2] = data[sourceIdx + 2];
+ }
+ }
+ ctx.putImageData(imageData, 0, 0);
+ };
+
+ // Gets the unaltered (besides filling in padding) version of the image for the api call
+ static getCanvasImg = (img: HTMLImageElement): HTMLCanvasElement | undefined => {
+ const canvas = document.createElement('canvas');
+ canvas.width = canvasSize;
+ canvas.height = canvasSize;
+ const ctx = canvas.getContext('2d');
+ if (!ctx) return undefined;
+ // fix scaling
+ const scale = Math.min(canvasSize / img.width, canvasSize / img.height);
+ const width = Math.floor(img.width * scale);
+ const height = Math.floor(img.height * scale);
+ ctx?.clearRect(0, 0, canvasSize, canvasSize);
+ ctx.fillStyle = bgColor;
+ ctx.fillRect(0, 0, canvasSize, canvasSize);
+
+ // extract and set padding data
+ if (img.naturalHeight > img.naturalWidth) {
+ // horizontal padding, x offset
+ const xOffset = Math.floor((canvasSize - width) / 2);
+ ctx.drawImage(img, xOffset, 0, width, height);
+
+ // draw reflected image padding
+ this.drawHorizontalReflection(ctx, canvas, xOffset);
+ } else {
+ // vertical padding, y offset
+ const yOffset = Math.floor((canvasSize - height) / 2);
+ ctx.drawImage(img, 0, yOffset, width, height);
+
+ // draw reflected image padding
+ this.drawVerticalReflection(ctx, canvas, yOffset);
+ }
+ return canvas;
+ };
+
+ /**
+ * Converts a url to base64 (tainted canvas workaround)
+ */
+ static urlToBase64 = async (imageUrl: string): Promise<string | undefined> => {
+ try {
+ const res = await fetch(imageUrl);
+ const blob = await res.blob();
+
+ return new Promise<string>((resolve, reject) => {
+ const reader = new FileReader();
+ reader.onload = () => {
+ const base64Data = reader.result?.toString().split(',')[1];
+ if (base64Data) {
+ resolve(base64Data);
+ } else {
+ reject(new Error('Failed to convert.'));
+ }
+ };
+ reader.onerror = () => {
+ reject(new Error('Error reading image data'));
+ };
+ reader.readAsDataURL(blob);
+ });
+ } catch (err) {
+ console.error(err);
+ }
+ return undefined;
+ };
+}
diff --git a/src/client/views/smartdraw/AnnotationPalette.tsx b/src/client/views/smartdraw/AnnotationPalette.tsx
index f1e2e4f41..dc0c01d36 100644
--- a/src/client/views/smartdraw/AnnotationPalette.tsx
+++ b/src/client/views/smartdraw/AnnotationPalette.tsx
@@ -185,7 +185,7 @@ export class AnnotationPalette extends ObservableReactComponent<AnnotationPalett
docData.drawingColored = this._opts.autoColor;
docData.drawingSize = this._opts.size;
docData.drawingData = this._gptRes[cIndex];
- docData.width = this._opts.size;
+ focusedDrawing.width = this._opts.size;
docData.x = this._opts.x;
docData.y = this._opts.y;
await AnnotationPalette.addToPalette(focusedDrawing);