From 3b17868560090756caf8b9b0f043ea163f2320e8 Mon Sep 17 00:00:00 2001 From: eleanor-park Date: Sun, 20 Oct 2024 12:44:15 -0400 Subject: changes --- .../nodes/imageEditor/GenerativeFillButtons.scss | 4 + .../nodes/imageEditor/GenerativeFillButtons.tsx | 72 ++ .../views/nodes/imageEditor/ImageEditor.scss | 134 ++++ src/client/views/nodes/imageEditor/ImageEditor.tsx | 758 +++++++++++++++++++++ .../views/nodes/imageEditor/ImageEditorButtons.tsx | 69 ++ .../imageEditor/imageEditingUtils/ImageHandler.ts | 312 +++++++++ .../imageEditor/imageEditorUtils/BrushHandler.ts | 35 + .../imageEditorUtils/GenerativeFillMathHelpers.ts | 6 + .../imageEditor/imageEditorUtils/ImageHandler.ts | 312 +++++++++ .../imageEditor/imageEditorUtils/PointerHandler.ts | 11 + .../imageEditorUtils/imageEditorConstants.ts | 9 + .../imageEditorUtils/imageEditorInterfaces.ts | 38 ++ .../imageEditor/imageToolUtils/BrushHandler.ts | 35 + .../imageEditor/imageToolUtils/ImageHandler.ts | 312 +++++++++ 14 files changed, 2107 insertions(+) create mode 100644 src/client/views/nodes/imageEditor/GenerativeFillButtons.scss create mode 100644 src/client/views/nodes/imageEditor/GenerativeFillButtons.tsx create mode 100644 src/client/views/nodes/imageEditor/ImageEditor.scss create mode 100644 src/client/views/nodes/imageEditor/ImageEditor.tsx create mode 100644 src/client/views/nodes/imageEditor/ImageEditorButtons.tsx create mode 100644 src/client/views/nodes/imageEditor/imageEditingUtils/ImageHandler.ts create mode 100644 src/client/views/nodes/imageEditor/imageEditorUtils/BrushHandler.ts create mode 100644 src/client/views/nodes/imageEditor/imageEditorUtils/GenerativeFillMathHelpers.ts create mode 100644 src/client/views/nodes/imageEditor/imageEditorUtils/ImageHandler.ts create mode 100644 src/client/views/nodes/imageEditor/imageEditorUtils/PointerHandler.ts create mode 100644 src/client/views/nodes/imageEditor/imageEditorUtils/imageEditorConstants.ts create mode 100644 src/client/views/nodes/imageEditor/imageEditorUtils/imageEditorInterfaces.ts create mode 100644 src/client/views/nodes/imageEditor/imageToolUtils/BrushHandler.ts create mode 100644 src/client/views/nodes/imageEditor/imageToolUtils/ImageHandler.ts (limited to 'src/client/views/nodes/imageEditor') diff --git a/src/client/views/nodes/imageEditor/GenerativeFillButtons.scss b/src/client/views/nodes/imageEditor/GenerativeFillButtons.scss new file mode 100644 index 000000000..0180ef904 --- /dev/null +++ b/src/client/views/nodes/imageEditor/GenerativeFillButtons.scss @@ -0,0 +1,4 @@ +.generativeFillBtnContainer { + display: flex; + gap: 1rem; +} diff --git a/src/client/views/nodes/imageEditor/GenerativeFillButtons.tsx b/src/client/views/nodes/imageEditor/GenerativeFillButtons.tsx new file mode 100644 index 000000000..32ed6b307 --- /dev/null +++ b/src/client/views/nodes/imageEditor/GenerativeFillButtons.tsx @@ -0,0 +1,72 @@ +import './GenerativeFillButtons.scss'; +import * as React from 'react'; +import ReactLoading from 'react-loading'; +import { Button, IconButton, Type } from 'browndash-components'; +import { AiOutlineInfo } from 'react-icons/ai'; +import { activeColor } from './imageEditorUtils/imageEditorConstants'; + +interface ButtonContainerProps { + onClick: () => Promise; + loading: boolean; + onReset: () => void; +} + +export function EditButtons({ loading, onClick: getEdit, onReset }: ButtonContainerProps) { + return ( +
+
+ ); +} + +export function CutButtons({ loading, onClick: cutImage, onReset }: ButtonContainerProps) { + return ( +
+
+ ); +} diff --git a/src/client/views/nodes/imageEditor/ImageEditor.scss b/src/client/views/nodes/imageEditor/ImageEditor.scss new file mode 100644 index 000000000..21c28f6da --- /dev/null +++ b/src/client/views/nodes/imageEditor/ImageEditor.scss @@ -0,0 +1,134 @@ +$navHeight: 5rem; +$canvasSize: 1024px; +$scale: 0.5; + +.imageEditorContainer { + position: absolute; + top: 0; + left: 0; + z-index: 9999; + height: 100vh; + width: 100vw; + display: flex; + flex-direction: column; + overflow: hidden; + + .imageEditorTopBar { + flex-shrink: 0; + height: $navHeight; + color: #000000; + background-color: #ffffff; + z-index: 999; + width: 100%; + display: flex; + gap: 3rem; + justify-content: space-between; + align-items: center; + border-bottom: 1px solid #c7cdd0; + padding: 0 2rem; + + .imageEditorControls { + display: flex; + align-items: center; + gap: 1.5rem; + } + + h1 { + font-size: 1.5rem; + } + } + + .drawingArea { + cursor: none; + touch-action: none; + position: relative; + flex-grow: 1; + display: flex; + justify-content: center; + align-items: center; + width: 100%; + background-color: #f0f4f6; + + canvas { + display: block; + position: absolute; + transform-origin: 50% 50%; + } + + .pointer { + pointer-events: none; + position: absolute; + border-radius: 50%; + width: 50px; + height: 50px; + border: 1px solid #ffffff; + transform: translate(-50%, -50%); + display: flex; + justify-content: center; + align-items: center; + + .innerPointer { + width: 100%; + height: 100%; + border: 1px solid #000000; + border-radius: 50%; + } + } + + .iconContainer { + position: absolute; + top: 3rem; + left: 2rem; + display: flex; + flex-direction: column; + gap: 4rem; + + .imageToolsContainer { + display: flex; + flex-direction: column; + gap: 10px; + } + + .undoRedoContainer { + justify-content: center; + display: flex; + flex-direction: row; + } + + .sliderContainer { + height: 225px; + width: 100%; + display: flex; + justify-content: center; + cursor: pointer; + } + } + + .editsBox { + position: absolute; + top: 2rem; + right: 2rem; + display: flex; + flex-direction: column; + gap: 1rem; + + .originalImageLabel { + position: absolute; + bottom: 10; + left: 10; + color: #ffffff; + font-size: 0.8rem; + letter-spacing: 1px; + text-transform: uppercase; + } + + img { + cursor: pointer; + transition: all 0.2s ease-in-out; + &:hover { + opacity: 0.8; + } + } + } + } +} diff --git a/src/client/views/nodes/imageEditor/ImageEditor.tsx b/src/client/views/nodes/imageEditor/ImageEditor.tsx new file mode 100644 index 000000000..86f7d8d29 --- /dev/null +++ b/src/client/views/nodes/imageEditor/ImageEditor.tsx @@ -0,0 +1,758 @@ +/* eslint-disable jsx-a11y/label-has-associated-control */ +/* eslint-disable jsx-a11y/no-noninteractive-element-interactions */ +/* eslint-disable jsx-a11y/img-redundant-alt */ +/* eslint-disable jsx-a11y/click-events-have-key-events */ +/* eslint-disable react/function-component-definition */ +import { Checkbox, FormControlLabel, Slider, TextField } from '@mui/material'; +import { IconButton } from 'browndash-components'; +import * as React from 'react'; +import { useEffect, useRef, useState } from 'react'; +import { CgClose } from 'react-icons/cg'; +import { IoMdRedo, IoMdUndo } from 'react-icons/io'; +import { ClientUtils } from '../../../../ClientUtils'; +import { Doc, DocListCast } from '../../../../fields/Doc'; +import { List } from '../../../../fields/List'; +import { NumCast } from '../../../../fields/Types'; +import { Networking } from '../../../Network'; +import { DocUtils } from '../../../documents/DocUtils'; +import { Docs } from '../../../documents/Documents'; +import { CollectionDockingView } from '../../collections/CollectionDockingView'; +import { CollectionFreeFormView } from '../../collections/collectionFreeForm'; +import { ImageEditorData } from '../ImageBox'; +import { OpenWhereMod } from '../OpenWhere'; +import './ImageEditor.scss'; +import { ApplyFuncButtons, ImageToolButton } from './ImageEditorButtons'; +import { BrushHandler, BrushType } from './imageEditorUtils/BrushHandler'; +import { APISuccess, ImageUtility } from './imageEditorUtils/ImageHandler'; +import { PointerHandler } from './imageEditorUtils/PointerHandler'; +import { activeColor, canvasSize, eraserColor, freeformRenderSize, newCollectionSize, offsetDistanceY, offsetX } from './imageEditorUtils/imageEditorConstants'; +import { CursorData, ImageDimensions, ImageEditTool, ImageToolType, Point } from './imageEditorUtils/imageEditorInterfaces'; +import { DocumentView } from '../DocumentView'; +import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; +import { ImageField } from '../../../../fields/URLField'; +import { resolve } from 'url'; +import { DocData } from '../../../../fields/DocSymbols'; + +interface GenerativeFillProps { + imageEditorOpen: boolean; + imageEditorSource: string; + imageRootDoc: Doc | undefined; + addDoc: ((doc: Doc | Doc[], annotationKey?: string) => boolean) | undefined; +} + +// Added field on image doc: gen_fill_children: List of children Docs + +const ImageEditor = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addDoc }: GenerativeFillProps) => { + const canvasRef = useRef(null); + const canvasBackgroundRef = useRef(null); + const drawingAreaRef = useRef(null); + const [cursorData, setCursorData] = useState({ + x: 0, + y: 0, + width: 150, + }); + const [isBrushing, setIsBrushing] = useState(false); + const [canvasScale, setCanvasScale] = useState(0.5); + // format: array of [image source, corresponding image Doc] + const [edits, setEdits] = useState<{ url: string; saveRes: Doc | undefined }[]>([]); + const [edited, setEdited] = useState(false); + const isFirstDoc = useRef(true); + // const [brushStyle] = useState(BrushStyle.ADD); + const [input, setInput] = useState(''); + const [loading, setLoading] = useState(false); + const [canvasDims, setCanvasDims] = useState({ + width: canvasSize, + height: canvasSize, + }); + // whether to create a new collection or not + const [isNewCollection, setIsNewCollection] = useState(true); + // the current image in the main canvas + const currImg = useRef(null); + // the unedited version of each generation (parent) + const originalImg = useRef(null); + const originalDoc = useRef(null); + // stores history of data urls + const undoStack = useRef([]); + // stores redo stack + const redoStack = useRef([]); + + // references to keep track of tree structure + const newCollectionRef = useRef(null); + const parentDoc = useRef(null); + const childrenDocs = useRef([]); + + // constants for image cutting + const cutPts = useRef([]); + + /** + * + * @param type The new tool type we are changing to + */ + const changeTool = (type: ImageToolType) => { + switch (type) { + case ImageToolType.GenerativeFill: + setCurrTool(genFillTool); + setCursorData(prev => ({ ...prev, width: genFillTool.sliderDefault as number })); + break; + case ImageToolType.Cut: + setCurrTool(cutTool); + setCursorData(prev => ({ ...prev, width: cutTool.sliderDefault as number })); + break; + default: + break; + } + }; + // Undo and Redo + const handleUndo = () => { + const ctx = ImageUtility.getCanvasContext(canvasRef); + if (!ctx || !currImg.current || !canvasRef.current) return; + + const target = undoStack.current[undoStack.current.length - 1]; + if (!target) { + ImageUtility.drawImgToCanvas(currImg.current, canvasRef, canvasDims.width, canvasDims.height); + } else { + redoStack.current = [...redoStack.current, canvasRef.current.toDataURL()]; + const img = new Image(); + img.src = target; + ImageUtility.drawImgToCanvas(img, canvasRef, canvasDims.width, canvasDims.height); + undoStack.current = undoStack.current.slice(0, -1); + } + }; + + const handleRedo = () => { + const ctx = ImageUtility.getCanvasContext(canvasRef); + if (!ctx || !currImg.current || !canvasRef.current) return; + + const target = redoStack.current[redoStack.current.length - 1]; + if (target) { + undoStack.current = [...undoStack.current, canvasRef.current?.toDataURL()]; + const img = new Image(); + img.src = target; + ImageUtility.drawImgToCanvas(img, canvasRef, canvasDims.width, canvasDims.height); + redoStack.current = redoStack.current.slice(0, -1); + } + }; + + // resets any erase strokes + const handleReset = () => { + if (!canvasRef.current || !currImg.current) return; + const ctx = ImageUtility.getCanvasContext(canvasRef); + if (!ctx) return; + ctx.clearRect(0, 0, canvasSize, canvasSize); + undoStack.current = []; + redoStack.current = []; + ImageUtility.drawImgToCanvas(currImg.current, canvasRef, canvasDims.width, canvasDims.height); + }; + + // initiate brushing + const handlePointerDown = (e: React.PointerEvent) => { + const canvas = canvasRef.current; + if (!canvas) return; + const ctx = ImageUtility.getCanvasContext(canvasRef); + if (!ctx) return; + + undoStack.current = [...undoStack.current, canvasRef.current.toDataURL()]; + redoStack.current = []; + + setIsBrushing(true); + const { x, y } = PointerHandler.getPointRelativeToElement(canvas, e, canvasScale); + BrushHandler.brushCircleOverlay(x, y, cursorData.width / 2 / canvasScale, ctx, eraserColor /* , brushStyle === BrushStyle.SUBTRACT */); + }; + + // stop brushing, push to undo stack + const handlePointerUp = () => { + const ctx = ImageUtility.getCanvasContext(canvasBackgroundRef); + if (!ctx) return; + if (!isBrushing) return; + setIsBrushing(false); + }; + + // handles brushing on pointer movement + useEffect(() => { + if (!isBrushing) return undefined; + const canvas = canvasRef.current; + if (!canvas) return undefined; + const ctx = ImageUtility.getCanvasContext(canvasRef); + if (!ctx) return undefined; + + const handlePointerMove = (e: PointerEvent) => { + const currPoint = PointerHandler.getPointRelativeToElement(canvas, e, canvasScale); + const lastPoint: Point = { + x: currPoint.x - e.movementX / canvasScale, + y: currPoint.y - e.movementY / canvasScale, + }; + const pts = BrushHandler.createBrushPathOverlay(lastPoint, currPoint, cursorData.width / 2 / canvasScale, ctx, eraserColor, BrushType.CUT); + cutPts.current.push(...pts); + }; + + drawingAreaRef.current?.addEventListener('pointermove', handlePointerMove); + return () => { + drawingAreaRef.current?.removeEventListener('pointermove', handlePointerMove); + }; + }, [isBrushing]); + + // first load + useEffect(() => { + const loadInitial = async () => { + if (!imageEditorSource || imageEditorSource === '') return; + const img = new Image(); + const res = await ImageUtility.urlToBase64(imageEditorSource); + if (!res) return; + img.src = `data:image/png;base64,${res}`; + + img.onload = () => { + currImg.current = img; + originalImg.current = img; + const imgWidth = img.naturalWidth; + const imgHeight = img.naturalHeight; + const scale = Math.min(canvasSize / imgWidth, canvasSize / imgHeight); + const width = imgWidth * scale; + const height = imgHeight * scale; + setCanvasDims({ width, height }); + }; + }; + + loadInitial(); + + // cleanup + return () => { + setInput(''); + setEdited(false); + newCollectionRef.current = null; + parentDoc.current = null; + childrenDocs.current = []; + currImg.current = null; + originalImg.current = null; + originalDoc.current = null; + undoStack.current = []; + redoStack.current = []; + ImageUtility.clearCanvas(canvasRef); + }; + }, [canvasRef, imageEditorSource]); + + // once the appropriate dimensions are set, draw the image to the canvas + useEffect(() => { + if (!currImg.current) return; + ImageUtility.drawImgToCanvas(currImg.current, canvasRef, canvasDims.width, canvasDims.height); + }, [canvasDims]); + + // handles brush sizing + useEffect(() => { + const handleKeyPress = (e: KeyboardEvent) => { + if (e.key === 'ArrowUp') { + e.preventDefault(); + e.stopPropagation(); + setCursorData(data => ({ ...data, width: data.width + 5 })); + } else if (e.key === 'ArrowDown') { + e.preventDefault(); + e.stopPropagation(); + setCursorData(data => (data.width >= 20 ? { ...data, width: data.width - 5 } : data)); + } + }; + window.addEventListener('keydown', handleKeyPress); + return () => window.removeEventListener('keydown', handleKeyPress); + }, []); + + // handle pinch zoom + useEffect(() => { + const handlePinch = (e: WheelEvent) => { + e.preventDefault(); + e.stopPropagation(); + const delta = e.deltaY; + const scaleFactor = delta > 0 ? 0.98 : 1.02; + setCanvasScale(prevScale => prevScale * scaleFactor); + }; + + drawingAreaRef.current?.addEventListener('wheel', handlePinch, { + passive: false, + }); + return () => drawingAreaRef.current?.removeEventListener('wheel', handlePinch); + }, [drawingAreaRef]); + + // updates the current position of the cursor + const updateCursorData = (e: React.PointerEvent) => { + const drawingArea = drawingAreaRef.current; + if (!drawingArea) return; + const { x, y } = PointerHandler.getPointRelativeToElement(drawingArea, e, 1); + setCursorData(data => ({ + ...data, + x, + y, + })); + }; + + // Get AI Edit + const getEdit = async () => { + const img = currImg.current; + if (!img) return; + const canvas = canvasRef.current; + if (!canvas) return; + const ctx = ImageUtility.getCanvasContext(canvasRef); + if (!ctx) return; + setLoading(true); + setEdited(true); + try { + const canvasOriginalImg = ImageUtility.getCanvasImg(img); + if (!canvasOriginalImg) return; + const canvasMask = ImageUtility.getCanvasMask(canvas, canvasOriginalImg); + if (!canvasMask) return; + const maskBlob = await ImageUtility.canvasToBlob(canvasMask); + const imgBlob = await ImageUtility.canvasToBlob(canvasOriginalImg); + const res = await ImageUtility.getEdit(imgBlob, maskBlob, input !== '' ? input + ' in the same style' : 'Fill in the image in the same style', 2); + + // create first image + if (!newCollectionRef.current) { + if (!isNewCollection && imageRootDoc) { + // if the parent hasn't been set yet + if (!parentDoc.current) parentDoc.current = imageRootDoc; + } else { + if (!(originalImg.current && imageRootDoc)) return; + // create new collection and add it to the view + newCollectionRef.current = Docs.Create.FreeformDocument([], { + x: NumCast(imageRootDoc.x) + NumCast(imageRootDoc._width) + offsetX, + y: NumCast(imageRootDoc.y), + _width: newCollectionSize, + _height: newCollectionSize, + title: 'Image edit collection', + }); + DocUtils.MakeLink(imageRootDoc, newCollectionRef.current, { link_relationship: 'Image Edit Version History' }); + + // opening new tab + CollectionDockingView.AddSplit(newCollectionRef.current, OpenWhereMod.right); + + // add the doc to the main freeform + // eslint-disable-next-line no-use-before-define + await createNewImgDoc(originalImg.current, true); + } + } else { + childrenDocs.current = []; + } + + originalImg.current = currImg.current; + originalDoc.current = parentDoc.current; + const { urls } = res as APISuccess; + if (res.status !== 'error') { + const imgUrls = await Promise.all(urls.map(url => ImageUtility.convertImgToCanvasUrl(url, canvasDims.width, canvasDims.height))); + const imgRes = await Promise.all( + imgUrls.map(async url => { + // eslint-disable-next-line no-use-before-define + const saveRes = await onSave(url); + return { url, saveRes }; + }) + ); + setEdits(imgRes); + const image = new Image(); + // eslint-disable-next-line prefer-destructuring + image.src = imgUrls[0]; + ImageUtility.drawImgToCanvas(image, canvasRef, canvasDims.width, canvasDims.height); + currImg.current = image; + parentDoc.current = imgRes[0].saveRes ?? null; + } + } catch (err) { + console.log(err); + } + setLoading(false); + }; + + const cutImage = async () => { + const img = currImg.current; + const canvas = canvasRef.current; + if (!canvas || !img) return; + const ctx = ImageUtility.getCanvasContext(canvasRef); + if (!ctx) return; + setLoading(true); + // get the original image + const canvasOriginalImg = ImageUtility.getCanvasImg(img); + if (!canvasOriginalImg) return; + // NOTE: cutting two diff shapes can be made possible by having the user press a button to set a new shape! + let minX = img.width; + let maxX = 0; + let minY = img.height; + let maxY = 0; + if (cutPts.current.length) { + ctx.beginPath(); + ctx.moveTo(cutPts.current[0].x, cutPts.current[0].y); // later check edge case where cutPts is empty + for (let i = 0; i < cutPts.current.length; i++) { + ctx.lineTo(cutPts.current[i].x, cutPts.current[i].y); + minX = Math.min(cutPts.current[i].x, minX); + minY = Math.min(cutPts.current[i].y, minY); + maxX = Math.max(cutPts.current[i].x, maxX); + maxY = Math.max(cutPts.current[i].y, maxY); + } + ctx.closePath(); + ctx.globalCompositeOperation = 'destination-in'; + ctx.fill(); + } + + const url = canvas.toDataURL(); // this does the same thing as convert img to canvasurl + if (!newCollectionRef.current) { + if (!isNewCollection && imageRootDoc) { + // if the parent hasn't been set yet + if (!parentDoc.current) parentDoc.current = imageRootDoc; + } else { + if (!(originalImg.current && imageRootDoc)) return; + // create new collection and add it to the view + newCollectionRef.current = Docs.Create.FreeformDocument([], { + x: NumCast(imageRootDoc.x) + NumCast(imageRootDoc._width) + offsetX, + y: NumCast(imageRootDoc.y), + _width: newCollectionSize, + _height: newCollectionSize, + title: 'Image edit collection', + }); + DocUtils.MakeLink(imageRootDoc, newCollectionRef.current, { link_relationship: 'Image Edit Version History' }); + // opening new tab + CollectionDockingView.AddSplit(newCollectionRef.current, OpenWhereMod.right); + } + } + + const image = new Image(); + image.src = url; + image.onload = async () => { + const croppedCanvas = document.createElement('canvas'); + const croppedCtx = croppedCanvas.getContext('2d'); + if (!croppedCtx) return; + croppedCanvas.width = maxX - minX; + croppedCanvas.height = maxY - minY; + croppedCtx.globalCompositeOperation = 'source-over'; + croppedCtx.clearRect(0, 0, croppedCanvas.width, croppedCanvas.height); + croppedCtx.drawImage(image, -minX, -minY); + const croppedURL = croppedCanvas.toDataURL(); + const croppedImage = new Image(); + croppedImage.src = croppedURL; + + currImg.current = croppedImage; + const newImgDoc = await createNewImgDoc(croppedImage, isFirstDoc.current); + if (isFirstDoc.current) isFirstDoc.current = false; + if (newImgDoc) { + const docData = newImgDoc[DocData]; + docData.backgroundColor = 'transparent'; + } + setEdits(prevEdits => [...prevEdits, { url: croppedURL, saveRes: undefined }]); + setLoading(false); + cutPts.current.length = 0; + }; + }; + + // adjusts all the img positions to be aligned + const adjustImgPositions = () => { + if (!parentDoc.current) return; + const startY = NumCast(parentDoc.current.y); + const children = DocListCast(parentDoc.current.gen_fill_children); + const len = children.length; + const initialYPositions: number[] = []; + for (let i = 0; i < len; i++) { + initialYPositions.push(startY + i * offsetDistanceY); + } + children.forEach((doc, i) => { + if (len % 2 === 1) { + doc.y = initialYPositions[i] - Math.floor(len / 2) * offsetDistanceY; + } else { + doc.y = initialYPositions[i] - (len / 2 - 1 / 2) * offsetDistanceY; + } + }); + }; + + // creates a new image document and returns its reference + const createNewImgDoc = async (img: HTMLImageElement, firstDoc: boolean, parent?: Doc): Promise => { + if (!imageRootDoc) return undefined; + const { src } = img; + const [result] = await Networking.PostToServer('/uploadRemoteImage', { sources: [src] }); + const source = ClientUtils.prepend(result.accessPaths.agnostic.client); + + if (firstDoc) { + const x = 0; + const initialY = 0; + const newImg = Docs.Create.ImageDocument(source, { + x: x, + y: initialY, + _height: freeformRenderSize, + _width: freeformRenderSize, + data_nativeWidth: result.nativeWidth, + data_nativeHeight: result.nativeHeight, + }); + if (isNewCollection && newCollectionRef.current) { + Doc.AddDocToList(newCollectionRef.current, undefined, newImg); + } else { + addDoc?.(newImg); + } + parentDoc.current = newImg; + return newImg; + } + if (!parentDoc.current) return undefined; + const x = NumCast(parentDoc.current.x) + freeformRenderSize + offsetX; + const initialY = 0; + + const newImg = Docs.Create.ImageDocument(source, { + x: x, + y: initialY, + _height: freeformRenderSize, + _width: freeformRenderSize, + data_nativeWidth: result.nativeWidth, + data_nativeHeight: result.nativeHeight, + }); + + const parentList = DocListCast(parentDoc.current.gen_fill_children); + if (parentList.length > 0) { + parentList.push(newImg); + parentDoc.current.gen_fill_children = new List(parentList); + } else { + parentDoc.current.gen_fill_children = new List([newImg]); + } + + DocUtils.MakeLink(parentDoc.current, newImg, { link_relationship: `Image edit; Prompt: ${input}` }); + adjustImgPositions(); + + if (isNewCollection && newCollectionRef.current) { + Doc.AddDocToList(newCollectionRef.current, undefined, newImg); + } else { + addDoc?.(newImg); + } + return newImg; + }; + + // Saves an image to the collection + const onSave = async (src: string) => { + const img = new Image(); + img.src = src; + if (!currImg.current || !originalImg.current || !imageRootDoc) return undefined; + try { + const res = await createNewImgDoc(img, false); + return res; + } catch (err) { + console.log(err); + } + return undefined; + }; + + // Closes the editor view + const handleViewClose = () => { + ImageEditorData.Open = false; + ImageEditorData.Source = ''; + if (newCollectionRef.current) { + DocumentView.addViewRenderedCb(newCollectionRef.current, dv => (dv.ComponentView as CollectionFreeFormView)?.fitContentOnce()); + } + setEdits([]); + }; + + // defines the tools and sets current tool + const genFillTool: ImageEditTool = { type: ImageToolType.GenerativeFill, name: 'Generative Fill', btnText: 'GET EDITS', icon: 'fill', applyFunc: getEdit, sliderMin: 25, sliderMax: 500, sliderDefault: 150 }; + const cutTool: ImageEditTool = { type: ImageToolType.Cut, name: 'Cut', btnText: 'CUT IMAGE', icon: 'scissors', applyFunc: cutImage, sliderMin: 1, sliderMax: 50, sliderDefault: 5 }; + const imageEditTools: ImageEditTool[] = [genFillTool, cutTool]; + const [currTool, setCurrTool] = useState(genFillTool); + + // the top controls for making a new collection, resetting, and applying edits, + function renderControls() { + return ( +
+

Image Editor

+ {/* } /> */} +
+ { + setIsNewCollection(prev => !prev); + }} + /> + } + label="Create New Collection" + labelPlacement="end" + sx={{ whiteSpace: 'nowrap' }} + /> + + } onClick={handleViewClose} /> +
+
+ ); + } + + // the side icons including tool type, the slider, and undo/redo + function renderSideIcons() { + return ( +
+
+ {imageEditTools.map(tool => { + return ImageToolButton(tool, tool.type === currTool.type, changeTool); + })} +
+
e.stopPropagation()}> + {currTool.type === ImageToolType.GenerativeFill && ( + { + setCursorData(prev => ({ ...prev, width: val as number })); + }} + /> + )} + {currTool.type === ImageToolType.Cut && ( + { + setCursorData(prev => ({ ...prev, width: val as number })); + }} + /> + )} +
+ {/* Undo and Redo */} +
+ { + e.stopPropagation(); + handleUndo(); + }} + onPointerUp={e => { + e.stopPropagation(); + }} + color={activeColor} + tooltip="Undo" + icon={} + /> + { + e.stopPropagation(); + handleRedo(); + }} + onPointerUp={e => { + e.stopPropagation(); + }} + color={activeColor} + tooltip="Redo" + icon={} + /> +
+
+ ); + } + + // circular pointer for drawing/erasing + function renderPointer() { + return ( +
+
+
+ ); + } + + // the previews for each edit + function renderEditThumbnails() { + return ( +
+ {edits.map((edit, i) => ( + image edits { + const img = new Image(); + img.src = edit.url; + ImageUtility.drawImgToCanvas(img, canvasRef, img.width, img.height); + currImg.current = img; + parentDoc.current = edit.saveRes ?? null; + }} + /> + ))} + {/* Original img thumbnail */} + {edits.length > 0 && ( +
+ + image stuff { + if (!originalImg.current) return; + const img = new Image(); + img.src = originalImg.current.src; + ImageUtility.drawImgToCanvas(img, canvasRef, canvasDims.width, canvasDims.height); + currImg.current = img; + if (!parentDoc.current) parentDoc.current = originalDoc.current; + }} + /> +
+ )} +
+ ); + } + + // the prompt box for generative fill + function renderPromptBox() { + return ( +
+ setInput(e.target.value)} + disabled={isBrushing} + type="text" + label="Prompt" + placeholder="Prompt..." + InputLabelProps={{ style: { fontSize: '16px' } }} + inputProps={{ style: { fontSize: '16px' } }} + sx={{ + backgroundColor: '#ffffff', + position: 'absolute', + bottom: '16px', + transform: 'translateX(calc(50vw - 50%))', + width: 'calc(100vw - 64px)', + }} + /> +
+ ); + } + + return ( +
+ {renderControls()} + {/* Main canvas for editing */} +
+ + + {renderPointer()} + {renderSideIcons()} + {renderEditThumbnails()} +
+ {currTool.type === ImageToolType.GenerativeFill && renderPromptBox()} +
+ ); +}; + +export default ImageEditor; diff --git a/src/client/views/nodes/imageEditor/ImageEditorButtons.tsx b/src/client/views/nodes/imageEditor/ImageEditorButtons.tsx new file mode 100644 index 000000000..e90babf9b --- /dev/null +++ b/src/client/views/nodes/imageEditor/ImageEditorButtons.tsx @@ -0,0 +1,69 @@ +import './GenerativeFillButtons.scss'; +import * as React from 'react'; +import ReactLoading from 'react-loading'; +import { Button, IconButton, Type } from 'browndash-components'; +import { AiOutlineInfo } from 'react-icons/ai'; +import { bgColor } from './imageEditorUtils/imageEditorConstants'; +import { ImageEditTool, ImageToolType } from './imageEditorUtils/imageEditorInterfaces'; +import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; +import { SettingsManager } from '../../../util/SettingsManager'; + +interface ButtonContainerProps { + onClick: () => Promise; + loading: boolean; + onReset: () => void; + btnText: string; +} + +export function ApplyFuncButtons({ loading, onClick: getEdit, onReset, btnText }: ButtonContainerProps) { + return ( +
+
+ ); +} + +export function ImageToolButton(tool: ImageEditTool, isActive: boolean, selectTool: (type: ImageToolType) => void) { + return ( +
+
+ ); +} diff --git a/src/client/views/nodes/imageEditor/imageEditingUtils/ImageHandler.ts b/src/client/views/nodes/imageEditor/imageEditingUtils/ImageHandler.ts new file mode 100644 index 000000000..514e8a94f --- /dev/null +++ b/src/client/views/nodes/imageEditor/imageEditingUtils/ImageHandler.ts @@ -0,0 +1,312 @@ +import { RefObject } from 'react'; +import { bgColor, canvasSize } from '../ImageEditorUtils/imageEditorConstants'; + +export interface APISuccess { + status: 'success'; + urls: string[]; +} + +export interface APIError { + status: 'error'; + message: string; +} + +export class ImageUtility { + /** + * + * @param canvas Canvas to convert + * @returns Blob of canvas + */ + static canvasToBlob = (canvas: HTMLCanvasElement): Promise => + new Promise(resolve => { + canvas.toBlob(blob => { + if (blob) { + resolve(blob); + } + }, 'image/png'); + }); + + // given a square api image, get the cropped img + static getCroppedImg = (img: HTMLImageElement, width: number, height: number): HTMLCanvasElement | undefined => { + // Create a new canvas element + const canvas = document.createElement('canvas'); + canvas.width = width; + canvas.height = height; + const ctx = canvas.getContext('2d'); + if (ctx) { + // Clear the canvas + ctx.clearRect(0, 0, canvas.width, canvas.height); + if (width < height) { + // horizontal padding, x offset + const xOffset = (canvasSize - width) / 2; + ctx.drawImage(img, xOffset, 0, canvas.width, canvas.height, 0, 0, canvas.width, canvas.height); + } else { + // vertical padding, y offset + const yOffset = (canvasSize - height) / 2; + ctx.drawImage(img, 0, yOffset, canvas.width, canvas.height, 0, 0, canvas.width, canvas.height); + } + return canvas; + } + return undefined; + }; + + // converts an image to a canvas data url + static convertImgToCanvasUrl = async (imageSrc: string, width: number, height: number): Promise => + new Promise((resolve, reject) => { + const img = new Image(); + img.onload = () => { + const canvas = this.getCroppedImg(img, width, height); + if (canvas) { + const dataUrl = canvas.toDataURL(); + resolve(dataUrl); + } + }; + img.onerror = error => { + reject(error); + }; + img.src = imageSrc; + }); + + // calls the openai api to get image edits + static getEdit = async (imgBlob: Blob, maskBlob: Blob, prompt: string, n?: number): Promise => { + const apiUrl = 'https://api.openai.com/v1/images/edits'; + const fd = new FormData(); + fd.append('image', imgBlob, 'image.png'); + fd.append('mask', maskBlob, 'mask.png'); + fd.append('prompt', prompt); + fd.append('size', '1024x1024'); + fd.append('n', n ? JSON.stringify(n) : '1'); + fd.append('response_format', 'b64_json'); + + try { + const res = await fetch(apiUrl, { + method: 'POST', + headers: { + Authorization: `Bearer ${process.env.OPENAI_KEY}`, + }, + body: fd, + }); + const data = await res.json(); + console.log(data.data); + return { + status: 'success', + urls: (data.data as { b64_json: string }[]).map(urlData => `data:image/png;base64,${urlData.b64_json}`), + }; + } catch (err) { + console.log(err); + return { status: 'error', message: 'API error.' }; + } + }; + + // mock api call + static mockGetEdit = async (mockSrc: string): Promise => ({ + status: 'success', + urls: [mockSrc, mockSrc, mockSrc], + }); + + // Gets the canvas rendering context of a canvas + static getCanvasContext = (canvasRef: RefObject): CanvasRenderingContext2D | null => { + if (!canvasRef.current) return null; + const ctx = canvasRef.current.getContext('2d'); + if (!ctx) return null; + return ctx; + }; + + // Helper for downloading the canvas (for debugging) + static downloadCanvas = (canvas: HTMLCanvasElement) => { + const url = canvas.toDataURL(); + const downloadLink = document.createElement('a'); + downloadLink.href = url; + downloadLink.download = 'canvas'; + + downloadLink.click(); + downloadLink.remove(); + }; + + // Download the canvas (for debugging) + static downloadImageCanvas = (imgUrl: string) => { + const img = new Image(); + img.src = imgUrl; + img.onload = () => { + const canvas = document.createElement('canvas'); + canvas.width = canvasSize; + canvas.height = canvasSize; + const ctx = canvas.getContext('2d'); + ctx?.drawImage(img, 0, 0, canvasSize, canvasSize); + + this.downloadCanvas(canvas); + }; + }; + + // Clears the canvas + static clearCanvas = (canvasRef: React.RefObject) => { + const ctx = this.getCanvasContext(canvasRef); + if (!ctx || !canvasRef.current) return; + ctx.clearRect(0, 0, canvasRef.current.width, canvasRef.current.height); + }; + + // Draws the image to the current canvas + static drawImgToCanvas = (img: HTMLImageElement, canvasRef: React.RefObject, width: number, height: number) => { + const drawImg = (htmlImg: HTMLImageElement) => { + const ctx = this.getCanvasContext(canvasRef); + if (!ctx) return; + ctx.globalCompositeOperation = 'source-over'; + ctx.clearRect(0, 0, width, height); + ctx.drawImage(htmlImg, 0, 0, width, height); + }; + + if (img.complete) { + drawImg(img); + } else { + img.onload = () => { + drawImg(img); + }; + } + }; + + // Gets the image mask for the openai endpoint + static getCanvasMask = (srcCanvas: HTMLCanvasElement, paddedCanvas: HTMLCanvasElement): HTMLCanvasElement | undefined => { + const canvas = document.createElement('canvas'); + canvas.width = canvasSize; + canvas.height = canvasSize; + const ctx = canvas.getContext('2d'); + if (!ctx) return undefined; + ctx?.clearRect(0, 0, canvasSize, canvasSize); + ctx.drawImage(paddedCanvas, 0, 0); + + // extract and set padding data + if (srcCanvas.height > srcCanvas.width) { + // horizontal padding, x offset + const xOffset = (canvasSize - srcCanvas.width) / 2; + ctx?.clearRect(xOffset, 0, srcCanvas.width, srcCanvas.height); + ctx.drawImage(srcCanvas, xOffset, 0, srcCanvas.width, srcCanvas.height); + } else { + // vertical padding, y offset + const yOffset = (canvasSize - srcCanvas.height) / 2; + ctx?.clearRect(0, yOffset, srcCanvas.width, srcCanvas.height); + ctx.drawImage(srcCanvas, 0, yOffset, srcCanvas.width, srcCanvas.height); + } + return canvas; + }; + + // Fills in the blank areas of the image with an image reflection (to fill in a square-shaped canvas) + static drawHorizontalReflection = (ctx: CanvasRenderingContext2D, canvas: HTMLCanvasElement, xOffset: number) => { + const imageData = ctx.getImageData(0, 0, canvas.width, canvas.height); + const { data } = imageData; + for (let i = 0; i < canvas.height; i++) { + for (let j = 0; j < xOffset; j++) { + const targetIdx = 4 * (i * canvas.width + j); + const sourceI = i; + const sourceJ = xOffset + (xOffset - j); + const sourceIdx = 4 * (sourceI * canvas.width + sourceJ); + data[targetIdx] = data[sourceIdx]; + data[targetIdx + 1] = data[sourceIdx + 1]; + data[targetIdx + 2] = data[sourceIdx + 2]; + } + } + for (let i = 0; i < canvas.height; i++) { + for (let j = canvas.width - 1; j >= canvas.width - 1 - xOffset; j--) { + const targetIdx = 4 * (i * canvas.width + j); + const sourceI = i; + const sourceJ = canvas.width - 1 - xOffset - (xOffset - (canvas.width - j)); + const sourceIdx = 4 * (sourceI * canvas.width + sourceJ); + data[targetIdx] = data[sourceIdx]; + data[targetIdx + 1] = data[sourceIdx + 1]; + data[targetIdx + 2] = data[sourceIdx + 2]; + } + } + ctx.putImageData(imageData, 0, 0); + }; + + // Fills in the blank areas of the image with an image reflection (to fill in a square-shaped canvas) + static drawVerticalReflection = (ctx: CanvasRenderingContext2D, canvas: HTMLCanvasElement, yOffset: number) => { + const imageData = ctx.getImageData(0, 0, canvas.width, canvas.height); + const { data } = imageData; + for (let j = 0; j < canvas.width; j++) { + for (let i = 0; i < yOffset; i++) { + const targetIdx = 4 * (i * canvas.width + j); + const sourceJ = j; + const sourceI = yOffset + (yOffset - i); + const sourceIdx = 4 * (sourceI * canvas.width + sourceJ); + data[targetIdx] = data[sourceIdx]; + data[targetIdx + 1] = data[sourceIdx + 1]; + data[targetIdx + 2] = data[sourceIdx + 2]; + } + } + for (let j = 0; j < canvas.width; j++) { + for (let i = canvas.height - 1; i >= canvas.height - 1 - yOffset; i--) { + const targetIdx = 4 * (i * canvas.width + j); + const sourceJ = j; + const sourceI = canvas.height - 1 - yOffset - (yOffset - (canvas.height - i)); + const sourceIdx = 4 * (sourceI * canvas.width + sourceJ); + data[targetIdx] = data[sourceIdx]; + data[targetIdx + 1] = data[sourceIdx + 1]; + data[targetIdx + 2] = data[sourceIdx + 2]; + } + } + ctx.putImageData(imageData, 0, 0); + }; + + // Gets the unaltered (besides filling in padding) version of the image for the api call + static getCanvasImg = (img: HTMLImageElement): HTMLCanvasElement | undefined => { + const canvas = document.createElement('canvas'); + canvas.width = canvasSize; + canvas.height = canvasSize; + const ctx = canvas.getContext('2d'); + if (!ctx) return undefined; + // fix scaling + const scale = Math.min(canvasSize / img.width, canvasSize / img.height); + const width = Math.floor(img.width * scale); + const height = Math.floor(img.height * scale); + ctx?.clearRect(0, 0, canvasSize, canvasSize); + ctx.fillStyle = bgColor; + ctx.fillRect(0, 0, canvasSize, canvasSize); + + // extract and set padding data + if (img.naturalHeight > img.naturalWidth) { + // horizontal padding, x offset + const xOffset = Math.floor((canvasSize - width) / 2); + ctx.drawImage(img, xOffset, 0, width, height); + + // draw reflected image padding + this.drawHorizontalReflection(ctx, canvas, xOffset); + } else { + // vertical padding, y offset + const yOffset = Math.floor((canvasSize - height) / 2); + ctx.drawImage(img, 0, yOffset, width, height); + + // draw reflected image padding + this.drawVerticalReflection(ctx, canvas, yOffset); + } + return canvas; + }; + + /** + * Converts a url to base64 (tainted canvas workaround) + */ + static urlToBase64 = async (imageUrl: string): Promise => { + try { + const res = await fetch(imageUrl); + const blob = await res.blob(); + + return new Promise((resolve, reject) => { + const reader = new FileReader(); + reader.onload = () => { + const base64Data = reader.result?.toString().split(',')[1]; + if (base64Data) { + resolve(base64Data); + } else { + reject(new Error('Failed to convert.')); + } + }; + reader.onerror = () => { + reject(new Error('Error reading image data')); + }; + reader.readAsDataURL(blob); + }); + } catch (err) { + console.error(err); + } + return undefined; + }; +} diff --git a/src/client/views/nodes/imageEditor/imageEditorUtils/BrushHandler.ts b/src/client/views/nodes/imageEditor/imageEditorUtils/BrushHandler.ts new file mode 100644 index 000000000..a9fe02d4f --- /dev/null +++ b/src/client/views/nodes/imageEditor/imageEditorUtils/BrushHandler.ts @@ -0,0 +1,35 @@ +import { GenerativeFillMathHelpers } from './GenerativeFillMathHelpers'; +import { eraserColor } from './imageEditorConstants'; +import { Point } from './imageEditorInterfaces'; +import { points } from '@turf/turf'; + +export enum BrushType { + GEN_FILL, + CUT, +} + +export class BrushHandler { + static brushCircleOverlay = (x: number, y: number, brushRadius: number, ctx: CanvasRenderingContext2D, fillColor: string /* , erase: boolean */) => { + ctx.globalCompositeOperation = 'destination-out'; + ctx.fillStyle = fillColor; + ctx.shadowColor = eraserColor; + ctx.shadowBlur = 5; + ctx.beginPath(); + ctx.arc(x, y, brushRadius, 0, 2 * Math.PI); + ctx.fill(); + ctx.closePath(); + }; + + static createBrushPathOverlay = (startPoint: Point, endPoint: Point, brushRadius: number, ctx: CanvasRenderingContext2D, fillColor: string, brushType: BrushType) => { + const dist = GenerativeFillMathHelpers.distanceBetween(startPoint, endPoint); + const pts: Point[] = []; + for (let i = 0; i < dist; i += 5) { + const s = i / dist; + const x = startPoint.x * (1 - s) + endPoint.x * s; + const y = startPoint.y * (1 - s) + endPoint.y * s; + pts.push({ x: startPoint.x, y: startPoint.y }); + BrushHandler.brushCircleOverlay(x, y, brushRadius, ctx, fillColor); + } + return pts; + }; +} diff --git a/src/client/views/nodes/imageEditor/imageEditorUtils/GenerativeFillMathHelpers.ts b/src/client/views/nodes/imageEditor/imageEditorUtils/GenerativeFillMathHelpers.ts new file mode 100644 index 000000000..f820300f3 --- /dev/null +++ b/src/client/views/nodes/imageEditor/imageEditorUtils/GenerativeFillMathHelpers.ts @@ -0,0 +1,6 @@ +import { Point } from './imageEditorInterfaces'; + +export class GenerativeFillMathHelpers { + static distanceBetween = (p1: Point, p2: Point) => Math.sqrt((p2.x - p1.x) ** 2 + (p2.y - p1.y) ** 2); + static angleBetween = (p1: Point, p2: Point) => Math.atan2(p2.x - p1.x, p2.y - p1.y); +} diff --git a/src/client/views/nodes/imageEditor/imageEditorUtils/ImageHandler.ts b/src/client/views/nodes/imageEditor/imageEditorUtils/ImageHandler.ts new file mode 100644 index 000000000..ece0f4d7f --- /dev/null +++ b/src/client/views/nodes/imageEditor/imageEditorUtils/ImageHandler.ts @@ -0,0 +1,312 @@ +import { RefObject } from 'react'; +import { bgColor, canvasSize } from './imageEditorConstants'; + +export interface APISuccess { + status: 'success'; + urls: string[]; +} + +export interface APIError { + status: 'error'; + message: string; +} + +export class ImageUtility { + /** + * + * @param canvas Canvas to convert + * @returns Blob of canvas + */ + static canvasToBlob = (canvas: HTMLCanvasElement): Promise => + new Promise(resolve => { + canvas.toBlob(blob => { + if (blob) { + resolve(blob); + } + }, 'image/png'); + }); + + // given a square api image, get the cropped img + static getCroppedImg = (img: HTMLImageElement, width: number, height: number): HTMLCanvasElement | undefined => { + // Create a new canvas element + const canvas = document.createElement('canvas'); + canvas.width = width; + canvas.height = height; + const ctx = canvas.getContext('2d'); + if (ctx) { + // Clear the canvas + ctx.clearRect(0, 0, canvas.width, canvas.height); + if (width < height) { + // horizontal padding, x offset + const xOffset = (canvasSize - width) / 2; + ctx.drawImage(img, xOffset, 0, canvas.width, canvas.height, 0, 0, canvas.width, canvas.height); + } else { + // vertical padding, y offset + const yOffset = (canvasSize - height) / 2; + ctx.drawImage(img, 0, yOffset, canvas.width, canvas.height, 0, 0, canvas.width, canvas.height); + } + return canvas; + } + return undefined; + }; + + // converts an image to a canvas data url + static convertImgToCanvasUrl = async (imageSrc: string, width: number, height: number): Promise => + new Promise((resolve, reject) => { + const img = new Image(); + img.onload = () => { + const canvas = this.getCroppedImg(img, width, height); + if (canvas) { + const dataUrl = canvas.toDataURL(); + resolve(dataUrl); + } + }; + img.onerror = error => { + reject(error); + }; + img.src = imageSrc; + }); + + // calls the openai api to get image edits + static getEdit = async (imgBlob: Blob, maskBlob: Blob, prompt: string, n?: number): Promise => { + const apiUrl = 'https://api.openai.com/v1/images/edits'; + const fd = new FormData(); + fd.append('image', imgBlob, 'image.png'); + fd.append('mask', maskBlob, 'mask.png'); + fd.append('prompt', prompt); + fd.append('size', '1024x1024'); + fd.append('n', n ? JSON.stringify(n) : '1'); + fd.append('response_format', 'b64_json'); + + try { + const res = await fetch(apiUrl, { + method: 'POST', + headers: { + Authorization: `Bearer ${process.env.OPENAI_KEY}`, + }, + body: fd, + }); + const data = await res.json(); + console.log(data.data); + return { + status: 'success', + urls: (data.data as { b64_json: string }[]).map(urlData => `data:image/png;base64,${urlData.b64_json}`), + }; + } catch (err) { + console.log(err); + return { status: 'error', message: 'API error.' }; + } + }; + + // mock api call + static mockGetEdit = async (mockSrc: string): Promise => ({ + status: 'success', + urls: [mockSrc, mockSrc, mockSrc], + }); + + // Gets the canvas rendering context of a canvas + static getCanvasContext = (canvasRef: RefObject): CanvasRenderingContext2D | null => { + if (!canvasRef.current) return null; + const ctx = canvasRef.current.getContext('2d'); + if (!ctx) return null; + return ctx; + }; + + // Helper for downloading the canvas (for debugging) + static downloadCanvas = (canvas: HTMLCanvasElement) => { + const url = canvas.toDataURL(); + const downloadLink = document.createElement('a'); + downloadLink.href = url; + downloadLink.download = 'canvas'; + + downloadLink.click(); + downloadLink.remove(); + }; + + // Download the canvas (for debugging) + static downloadImageCanvas = (imgUrl: string) => { + const img = new Image(); + img.src = imgUrl; + img.onload = () => { + const canvas = document.createElement('canvas'); + canvas.width = canvasSize; + canvas.height = canvasSize; + const ctx = canvas.getContext('2d'); + ctx?.drawImage(img, 0, 0, canvasSize, canvasSize); + + this.downloadCanvas(canvas); + }; + }; + + // Clears the canvas + static clearCanvas = (canvasRef: React.RefObject) => { + const ctx = this.getCanvasContext(canvasRef); + if (!ctx || !canvasRef.current) return; + ctx.clearRect(0, 0, canvasRef.current.width, canvasRef.current.height); + }; + + // Draws the image to the current canvas + static drawImgToCanvas = (img: HTMLImageElement, canvasRef: React.RefObject, width: number, height: number) => { + const drawImg = (htmlImg: HTMLImageElement) => { + const ctx = this.getCanvasContext(canvasRef); + if (!ctx) return; + ctx.globalCompositeOperation = 'source-over'; + ctx.clearRect(0, 0, canvasRef.current?.width || width, canvasRef.current?.height || height); + ctx.drawImage(htmlImg, 0, 0, width, height); + }; + + if (img.complete) { + drawImg(img); + } else { + img.onload = () => { + drawImg(img); + }; + } + }; + + // Gets the image mask for the openai endpoint + static getCanvasMask = (srcCanvas: HTMLCanvasElement, paddedCanvas: HTMLCanvasElement): HTMLCanvasElement | undefined => { + const canvas = document.createElement('canvas'); + canvas.width = canvasSize; + canvas.height = canvasSize; + const ctx = canvas.getContext('2d'); + if (!ctx) return undefined; + ctx?.clearRect(0, 0, canvasSize, canvasSize); + ctx.drawImage(paddedCanvas, 0, 0); + + // extract and set padding data + if (srcCanvas.height > srcCanvas.width) { + // horizontal padding, x offset + const xOffset = (canvasSize - srcCanvas.width) / 2; + ctx?.clearRect(xOffset, 0, srcCanvas.width, srcCanvas.height); + ctx.drawImage(srcCanvas, xOffset, 0, srcCanvas.width, srcCanvas.height); + } else { + // vertical padding, y offset + const yOffset = (canvasSize - srcCanvas.height) / 2; + ctx?.clearRect(0, yOffset, srcCanvas.width, srcCanvas.height); + ctx.drawImage(srcCanvas, 0, yOffset, srcCanvas.width, srcCanvas.height); + } + return canvas; + }; + + // Fills in the blank areas of the image with an image reflection (to fill in a square-shaped canvas) + static drawHorizontalReflection = (ctx: CanvasRenderingContext2D, canvas: HTMLCanvasElement, xOffset: number) => { + const imageData = ctx.getImageData(0, 0, canvas.width, canvas.height); + const { data } = imageData; + for (let i = 0; i < canvas.height; i++) { + for (let j = 0; j < xOffset; j++) { + const targetIdx = 4 * (i * canvas.width + j); + const sourceI = i; + const sourceJ = xOffset + (xOffset - j); + const sourceIdx = 4 * (sourceI * canvas.width + sourceJ); + data[targetIdx] = data[sourceIdx]; + data[targetIdx + 1] = data[sourceIdx + 1]; + data[targetIdx + 2] = data[sourceIdx + 2]; + } + } + for (let i = 0; i < canvas.height; i++) { + for (let j = canvas.width - 1; j >= canvas.width - 1 - xOffset; j--) { + const targetIdx = 4 * (i * canvas.width + j); + const sourceI = i; + const sourceJ = canvas.width - 1 - xOffset - (xOffset - (canvas.width - j)); + const sourceIdx = 4 * (sourceI * canvas.width + sourceJ); + data[targetIdx] = data[sourceIdx]; + data[targetIdx + 1] = data[sourceIdx + 1]; + data[targetIdx + 2] = data[sourceIdx + 2]; + } + } + ctx.putImageData(imageData, 0, 0); + }; + + // Fills in the blank areas of the image with an image reflection (to fill in a square-shaped canvas) + static drawVerticalReflection = (ctx: CanvasRenderingContext2D, canvas: HTMLCanvasElement, yOffset: number) => { + const imageData = ctx.getImageData(0, 0, canvas.width, canvas.height); + const { data } = imageData; + for (let j = 0; j < canvas.width; j++) { + for (let i = 0; i < yOffset; i++) { + const targetIdx = 4 * (i * canvas.width + j); + const sourceJ = j; + const sourceI = yOffset + (yOffset - i); + const sourceIdx = 4 * (sourceI * canvas.width + sourceJ); + data[targetIdx] = data[sourceIdx]; + data[targetIdx + 1] = data[sourceIdx + 1]; + data[targetIdx + 2] = data[sourceIdx + 2]; + } + } + for (let j = 0; j < canvas.width; j++) { + for (let i = canvas.height - 1; i >= canvas.height - 1 - yOffset; i--) { + const targetIdx = 4 * (i * canvas.width + j); + const sourceJ = j; + const sourceI = canvas.height - 1 - yOffset - (yOffset - (canvas.height - i)); + const sourceIdx = 4 * (sourceI * canvas.width + sourceJ); + data[targetIdx] = data[sourceIdx]; + data[targetIdx + 1] = data[sourceIdx + 1]; + data[targetIdx + 2] = data[sourceIdx + 2]; + } + } + ctx.putImageData(imageData, 0, 0); + }; + + // Gets the unaltered (besides filling in padding) version of the image for the api call + static getCanvasImg = (img: HTMLImageElement): HTMLCanvasElement | undefined => { + const canvas = document.createElement('canvas'); + canvas.width = canvasSize; + canvas.height = canvasSize; + const ctx = canvas.getContext('2d'); + if (!ctx) return undefined; + // fix scaling + const scale = Math.min(canvasSize / img.width, canvasSize / img.height); + const width = Math.floor(img.width * scale); + const height = Math.floor(img.height * scale); + ctx?.clearRect(0, 0, canvasSize, canvasSize); + ctx.fillStyle = bgColor; + ctx.fillRect(0, 0, canvasSize, canvasSize); + + // extract and set padding data + if (img.naturalHeight > img.naturalWidth) { + // horizontal padding, x offset + const xOffset = Math.floor((canvasSize - width) / 2); + ctx.drawImage(img, xOffset, 0, width, height); + + // draw reflected image padding + this.drawHorizontalReflection(ctx, canvas, xOffset); + } else { + // vertical padding, y offset + const yOffset = Math.floor((canvasSize - height) / 2); + ctx.drawImage(img, 0, yOffset, width, height); + + // draw reflected image padding + this.drawVerticalReflection(ctx, canvas, yOffset); + } + return canvas; + }; + + /** + * Converts a url to base64 (tainted canvas workaround) + */ + static urlToBase64 = async (imageUrl: string): Promise => { + try { + const res = await fetch(imageUrl); + const blob = await res.blob(); + + return new Promise((resolve, reject) => { + const reader = new FileReader(); + reader.onload = () => { + const base64Data = reader.result?.toString().split(',')[1]; + if (base64Data) { + resolve(base64Data); + } else { + reject(new Error('Failed to convert.')); + } + }; + reader.onerror = () => { + reject(new Error('Error reading image data')); + }; + reader.readAsDataURL(blob); + }); + } catch (err) { + console.error(err); + } + return undefined; + }; +} diff --git a/src/client/views/nodes/imageEditor/imageEditorUtils/PointerHandler.ts b/src/client/views/nodes/imageEditor/imageEditorUtils/PointerHandler.ts new file mode 100644 index 000000000..e86f46636 --- /dev/null +++ b/src/client/views/nodes/imageEditor/imageEditorUtils/PointerHandler.ts @@ -0,0 +1,11 @@ +import { Point } from './imageEditorInterfaces'; + +export class PointerHandler { + static getPointRelativeToElement = (element: HTMLElement, e: React.PointerEvent | PointerEvent, scale: number): Point => { + const boundingBox = element.getBoundingClientRect(); + return { + x: (e.clientX - boundingBox.x) / scale, + y: (e.clientY - boundingBox.y) / scale, + }; + }; +} diff --git a/src/client/views/nodes/imageEditor/imageEditorUtils/imageEditorConstants.ts b/src/client/views/nodes/imageEditor/imageEditorUtils/imageEditorConstants.ts new file mode 100644 index 000000000..4772304bc --- /dev/null +++ b/src/client/views/nodes/imageEditor/imageEditorUtils/imageEditorConstants.ts @@ -0,0 +1,9 @@ +export const canvasSize = 1024; +export const freeformRenderSize = 300; +export const offsetDistanceY = freeformRenderSize + 400; +export const offsetX = 200; +export const newCollectionSize = 500; + +export const activeColor = '#1976d2'; +export const eraserColor = '#e1e9ec'; +export const bgColor = '#f0f4f6'; diff --git a/src/client/views/nodes/imageEditor/imageEditorUtils/imageEditorInterfaces.ts b/src/client/views/nodes/imageEditor/imageEditorUtils/imageEditorInterfaces.ts new file mode 100644 index 000000000..f4ae7d9c4 --- /dev/null +++ b/src/client/views/nodes/imageEditor/imageEditorUtils/imageEditorInterfaces.ts @@ -0,0 +1,38 @@ +import { IconProp } from '@fortawesome/fontawesome-svg-core'; + +export interface CursorData { + x: number; + y: number; + width: number; +} + +export interface Point { + x: number; + y: number; +} + +export enum ImageToolType { + GenerativeFill = 'genFill', + Cut = 'cut', +} + +export interface ImageEditTool { + type: ImageToolType; + name: string; + btnText: string; + icon: IconProp; + applyFunc: () => Promise; + sliderMin?: number; + sliderMax?: number; + sliderDefault?: number; +} + +export enum BrushMode { + ADD, + SUBTRACT, +} + +export interface ImageDimensions { + width: number; + height: number; +} diff --git a/src/client/views/nodes/imageEditor/imageToolUtils/BrushHandler.ts b/src/client/views/nodes/imageEditor/imageToolUtils/BrushHandler.ts new file mode 100644 index 000000000..7139bebc3 --- /dev/null +++ b/src/client/views/nodes/imageEditor/imageToolUtils/BrushHandler.ts @@ -0,0 +1,35 @@ +import { GenerativeFillMathHelpers } from '../imageEditorUtils/GenerativeFillMathHelpers'; +import { eraserColor } from '../imageEditorUtils/imageEditorConstants'; +import { Point } from '../imageEditorUtils/imageEditorInterfaces'; +import { points } from '@turf/turf'; + +export enum BrushType { + GEN_FILL, + CUT, +} + +export class BrushHandler { + static brushCircleOverlay = (x: number, y: number, brushRadius: number, ctx: CanvasRenderingContext2D, fillColor: string /* , erase: boolean */) => { + ctx.globalCompositeOperation = 'destination-out'; + ctx.fillStyle = fillColor; + ctx.shadowColor = eraserColor; + ctx.shadowBlur = 5; + ctx.beginPath(); + ctx.arc(x, y, brushRadius, 0, 2 * Math.PI); + ctx.fill(); + ctx.closePath(); + }; + + static createBrushPathOverlay = (startPoint: Point, endPoint: Point, brushRadius: number, ctx: CanvasRenderingContext2D, fillColor: string, brushType: BrushType) => { + const dist = GenerativeFillMathHelpers.distanceBetween(startPoint, endPoint); + const pts: Point[] = []; + for (let i = 0; i < dist; i += 5) { + const s = i / dist; + const x = startPoint.x * (1 - s) + endPoint.x * s; + const y = startPoint.y * (1 - s) + endPoint.y * s; + pts.push({ x: startPoint.x, y: startPoint.y }); + BrushHandler.brushCircleOverlay(x, y, brushRadius, ctx, fillColor); + } + return pts; + }; +} diff --git a/src/client/views/nodes/imageEditor/imageToolUtils/ImageHandler.ts b/src/client/views/nodes/imageEditor/imageToolUtils/ImageHandler.ts new file mode 100644 index 000000000..b9723b5be --- /dev/null +++ b/src/client/views/nodes/imageEditor/imageToolUtils/ImageHandler.ts @@ -0,0 +1,312 @@ +import { RefObject } from 'react'; +import { bgColor, canvasSize } from '../imageEditorUtils/imageEditorConstants'; + +export interface APISuccess { + status: 'success'; + urls: string[]; +} + +export interface APIError { + status: 'error'; + message: string; +} + +export class ImageUtility { + /** + * + * @param canvas Canvas to convert + * @returns Blob of canvas + */ + static canvasToBlob = (canvas: HTMLCanvasElement): Promise => + new Promise(resolve => { + canvas.toBlob(blob => { + if (blob) { + resolve(blob); + } + }, 'image/png'); + }); + + // given a square api image, get the cropped img + static getCroppedImg = (img: HTMLImageElement, width: number, height: number): HTMLCanvasElement | undefined => { + // Create a new canvas element + const canvas = document.createElement('canvas'); + canvas.width = width; + canvas.height = height; + const ctx = canvas.getContext('2d'); + if (ctx) { + // Clear the canvas + ctx.clearRect(0, 0, canvas.width, canvas.height); + if (width < height) { + // horizontal padding, x offset + const xOffset = (canvasSize - width) / 2; + ctx.drawImage(img, xOffset, 0, canvas.width, canvas.height, 0, 0, canvas.width, canvas.height); + } else { + // vertical padding, y offset + const yOffset = (canvasSize - height) / 2; + ctx.drawImage(img, 0, yOffset, canvas.width, canvas.height, 0, 0, canvas.width, canvas.height); + } + return canvas; + } + return undefined; + }; + + // converts an image to a canvas data url + static convertImgToCanvasUrl = async (imageSrc: string, width: number, height: number): Promise => + new Promise((resolve, reject) => { + const img = new Image(); + img.onload = () => { + const canvas = this.getCroppedImg(img, width, height); + if (canvas) { + const dataUrl = canvas.toDataURL(); + resolve(dataUrl); + } + }; + img.onerror = error => { + reject(error); + }; + img.src = imageSrc; + }); + + // calls the openai api to get image edits + static getEdit = async (imgBlob: Blob, maskBlob: Blob, prompt: string, n?: number): Promise => { + const apiUrl = 'https://api.openai.com/v1/images/edits'; + const fd = new FormData(); + fd.append('image', imgBlob, 'image.png'); + fd.append('mask', maskBlob, 'mask.png'); + fd.append('prompt', prompt); + fd.append('size', '1024x1024'); + fd.append('n', n ? JSON.stringify(n) : '1'); + fd.append('response_format', 'b64_json'); + + try { + const res = await fetch(apiUrl, { + method: 'POST', + headers: { + Authorization: `Bearer ${process.env.OPENAI_KEY}`, + }, + body: fd, + }); + const data = await res.json(); + console.log(data.data); + return { + status: 'success', + urls: (data.data as { b64_json: string }[]).map(urlData => `data:image/png;base64,${urlData.b64_json}`), + }; + } catch (err) { + console.log(err); + return { status: 'error', message: 'API error.' }; + } + }; + + // mock api call + static mockGetEdit = async (mockSrc: string): Promise => ({ + status: 'success', + urls: [mockSrc, mockSrc, mockSrc], + }); + + // Gets the canvas rendering context of a canvas + static getCanvasContext = (canvasRef: RefObject): CanvasRenderingContext2D | null => { + if (!canvasRef.current) return null; + const ctx = canvasRef.current.getContext('2d'); + if (!ctx) return null; + return ctx; + }; + + // Helper for downloading the canvas (for debugging) + static downloadCanvas = (canvas: HTMLCanvasElement) => { + const url = canvas.toDataURL(); + const downloadLink = document.createElement('a'); + downloadLink.href = url; + downloadLink.download = 'canvas'; + + downloadLink.click(); + downloadLink.remove(); + }; + + // Download the canvas (for debugging) + static downloadImageCanvas = (imgUrl: string) => { + const img = new Image(); + img.src = imgUrl; + img.onload = () => { + const canvas = document.createElement('canvas'); + canvas.width = canvasSize; + canvas.height = canvasSize; + const ctx = canvas.getContext('2d'); + ctx?.drawImage(img, 0, 0, canvasSize, canvasSize); + + this.downloadCanvas(canvas); + }; + }; + + // Clears the canvas + static clearCanvas = (canvasRef: React.RefObject) => { + const ctx = this.getCanvasContext(canvasRef); + if (!ctx || !canvasRef.current) return; + ctx.clearRect(0, 0, canvasRef.current.width, canvasRef.current.height); + }; + + // Draws the image to the current canvas + static drawImgToCanvas = (img: HTMLImageElement, canvasRef: React.RefObject, width: number, height: number) => { + const drawImg = (htmlImg: HTMLImageElement) => { + const ctx = this.getCanvasContext(canvasRef); + if (!ctx) return; + ctx.globalCompositeOperation = 'source-over'; + ctx.clearRect(0, 0, width, height); + ctx.drawImage(htmlImg, 0, 0, width, height); + }; + + if (img.complete) { + drawImg(img); + } else { + img.onload = () => { + drawImg(img); + }; + } + }; + + // Gets the image mask for the openai endpoint + static getCanvasMask = (srcCanvas: HTMLCanvasElement, paddedCanvas: HTMLCanvasElement): HTMLCanvasElement | undefined => { + const canvas = document.createElement('canvas'); + canvas.width = canvasSize; + canvas.height = canvasSize; + const ctx = canvas.getContext('2d'); + if (!ctx) return undefined; + ctx?.clearRect(0, 0, canvasSize, canvasSize); + ctx.drawImage(paddedCanvas, 0, 0); + + // extract and set padding data + if (srcCanvas.height > srcCanvas.width) { + // horizontal padding, x offset + const xOffset = (canvasSize - srcCanvas.width) / 2; + ctx?.clearRect(xOffset, 0, srcCanvas.width, srcCanvas.height); + ctx.drawImage(srcCanvas, xOffset, 0, srcCanvas.width, srcCanvas.height); + } else { + // vertical padding, y offset + const yOffset = (canvasSize - srcCanvas.height) / 2; + ctx?.clearRect(0, yOffset, srcCanvas.width, srcCanvas.height); + ctx.drawImage(srcCanvas, 0, yOffset, srcCanvas.width, srcCanvas.height); + } + return canvas; + }; + + // Fills in the blank areas of the image with an image reflection (to fill in a square-shaped canvas) + static drawHorizontalReflection = (ctx: CanvasRenderingContext2D, canvas: HTMLCanvasElement, xOffset: number) => { + const imageData = ctx.getImageData(0, 0, canvas.width, canvas.height); + const { data } = imageData; + for (let i = 0; i < canvas.height; i++) { + for (let j = 0; j < xOffset; j++) { + const targetIdx = 4 * (i * canvas.width + j); + const sourceI = i; + const sourceJ = xOffset + (xOffset - j); + const sourceIdx = 4 * (sourceI * canvas.width + sourceJ); + data[targetIdx] = data[sourceIdx]; + data[targetIdx + 1] = data[sourceIdx + 1]; + data[targetIdx + 2] = data[sourceIdx + 2]; + } + } + for (let i = 0; i < canvas.height; i++) { + for (let j = canvas.width - 1; j >= canvas.width - 1 - xOffset; j--) { + const targetIdx = 4 * (i * canvas.width + j); + const sourceI = i; + const sourceJ = canvas.width - 1 - xOffset - (xOffset - (canvas.width - j)); + const sourceIdx = 4 * (sourceI * canvas.width + sourceJ); + data[targetIdx] = data[sourceIdx]; + data[targetIdx + 1] = data[sourceIdx + 1]; + data[targetIdx + 2] = data[sourceIdx + 2]; + } + } + ctx.putImageData(imageData, 0, 0); + }; + + // Fills in the blank areas of the image with an image reflection (to fill in a square-shaped canvas) + static drawVerticalReflection = (ctx: CanvasRenderingContext2D, canvas: HTMLCanvasElement, yOffset: number) => { + const imageData = ctx.getImageData(0, 0, canvas.width, canvas.height); + const { data } = imageData; + for (let j = 0; j < canvas.width; j++) { + for (let i = 0; i < yOffset; i++) { + const targetIdx = 4 * (i * canvas.width + j); + const sourceJ = j; + const sourceI = yOffset + (yOffset - i); + const sourceIdx = 4 * (sourceI * canvas.width + sourceJ); + data[targetIdx] = data[sourceIdx]; + data[targetIdx + 1] = data[sourceIdx + 1]; + data[targetIdx + 2] = data[sourceIdx + 2]; + } + } + for (let j = 0; j < canvas.width; j++) { + for (let i = canvas.height - 1; i >= canvas.height - 1 - yOffset; i--) { + const targetIdx = 4 * (i * canvas.width + j); + const sourceJ = j; + const sourceI = canvas.height - 1 - yOffset - (yOffset - (canvas.height - i)); + const sourceIdx = 4 * (sourceI * canvas.width + sourceJ); + data[targetIdx] = data[sourceIdx]; + data[targetIdx + 1] = data[sourceIdx + 1]; + data[targetIdx + 2] = data[sourceIdx + 2]; + } + } + ctx.putImageData(imageData, 0, 0); + }; + + // Gets the unaltered (besides filling in padding) version of the image for the api call + static getCanvasImg = (img: HTMLImageElement): HTMLCanvasElement | undefined => { + const canvas = document.createElement('canvas'); + canvas.width = canvasSize; + canvas.height = canvasSize; + const ctx = canvas.getContext('2d'); + if (!ctx) return undefined; + // fix scaling + const scale = Math.min(canvasSize / img.width, canvasSize / img.height); + const width = Math.floor(img.width * scale); + const height = Math.floor(img.height * scale); + ctx?.clearRect(0, 0, canvasSize, canvasSize); + ctx.fillStyle = bgColor; + ctx.fillRect(0, 0, canvasSize, canvasSize); + + // extract and set padding data + if (img.naturalHeight > img.naturalWidth) { + // horizontal padding, x offset + const xOffset = Math.floor((canvasSize - width) / 2); + ctx.drawImage(img, xOffset, 0, width, height); + + // draw reflected image padding + this.drawHorizontalReflection(ctx, canvas, xOffset); + } else { + // vertical padding, y offset + const yOffset = Math.floor((canvasSize - height) / 2); + ctx.drawImage(img, 0, yOffset, width, height); + + // draw reflected image padding + this.drawVerticalReflection(ctx, canvas, yOffset); + } + return canvas; + }; + + /** + * Converts a url to base64 (tainted canvas workaround) + */ + static urlToBase64 = async (imageUrl: string): Promise => { + try { + const res = await fetch(imageUrl); + const blob = await res.blob(); + + return new Promise((resolve, reject) => { + const reader = new FileReader(); + reader.onload = () => { + const base64Data = reader.result?.toString().split(',')[1]; + if (base64Data) { + resolve(base64Data); + } else { + reject(new Error('Failed to convert.')); + } + }; + reader.onerror = () => { + reject(new Error('Error reading image data')); + }; + reader.readAsDataURL(blob); + }); + } catch (err) { + console.error(err); + } + return undefined; + }; +} -- cgit v1.2.3-70-g09d2 From d0569fcb79389b3c4f3d60e2a84f6f9220ca9cf3 Mon Sep 17 00:00:00 2001 From: eleanor-park <113556828+eleanor-park@users.noreply.github.com> Date: Sun, 20 Oct 2024 23:52:20 -0400 Subject: added different cut types, haven't been able to run & test --- .../views/nodes/imageEditor/ImageEditor.scss | 6 +++ src/client/views/nodes/imageEditor/ImageEditor.tsx | 56 ++++++++++++++++++++-- .../imageEditorUtils/imageEditorInterfaces.ts | 6 ++- 3 files changed, 61 insertions(+), 7 deletions(-) (limited to 'src/client/views/nodes/imageEditor') diff --git a/src/client/views/nodes/imageEditor/ImageEditor.scss b/src/client/views/nodes/imageEditor/ImageEditor.scss index 21c28f6da..49146aa23 100644 --- a/src/client/views/nodes/imageEditor/ImageEditor.scss +++ b/src/client/views/nodes/imageEditor/ImageEditor.scss @@ -89,6 +89,12 @@ $scale: 0.5; gap: 10px; } + .cutToolsContainer { + display: grid; + gap: 5px; + grid-template-columns: 1fr 1fr; + } + .undoRedoContainer { justify-content: center; display: flex; diff --git a/src/client/views/nodes/imageEditor/ImageEditor.tsx b/src/client/views/nodes/imageEditor/ImageEditor.tsx index 86f7d8d29..d9f46876e 100644 --- a/src/client/views/nodes/imageEditor/ImageEditor.tsx +++ b/src/client/views/nodes/imageEditor/ImageEditor.tsx @@ -25,13 +25,14 @@ import { ApplyFuncButtons, ImageToolButton } from './ImageEditorButtons'; import { BrushHandler, BrushType } from './imageEditorUtils/BrushHandler'; import { APISuccess, ImageUtility } from './imageEditorUtils/ImageHandler'; import { PointerHandler } from './imageEditorUtils/PointerHandler'; -import { activeColor, canvasSize, eraserColor, freeformRenderSize, newCollectionSize, offsetDistanceY, offsetX } from './imageEditorUtils/imageEditorConstants'; -import { CursorData, ImageDimensions, ImageEditTool, ImageToolType, Point } from './imageEditorUtils/imageEditorInterfaces'; +import { activeColor, bgColor, canvasSize, eraserColor, freeformRenderSize, newCollectionSize, offsetDistanceY, offsetX } from './imageEditorUtils/imageEditorConstants'; +import { BrushMode, CursorData, ImageDimensions, ImageEditTool, ImageToolType, Point } from './imageEditorUtils/imageEditorInterfaces'; import { DocumentView } from '../DocumentView'; import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; import { ImageField } from '../../../../fields/URLField'; import { resolve } from 'url'; import { DocData } from '../../../../fields/DocSymbols'; +import { SettingsManager } from '../../../util/SettingsManager'; interface GenerativeFillProps { imageEditorOpen: boolean; @@ -57,13 +58,13 @@ const ImageEditor = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addDoc const [edits, setEdits] = useState<{ url: string; saveRes: Doc | undefined }[]>([]); const [edited, setEdited] = useState(false); const isFirstDoc = useRef(true); - // const [brushStyle] = useState(BrushStyle.ADD); const [input, setInput] = useState(''); const [loading, setLoading] = useState(false); const [canvasDims, setCanvasDims] = useState({ width: canvasSize, height: canvasSize, }); + const [cutType, setCutType] = useState(BrushMode.IN); // whether to create a new collection or not const [isNewCollection, setIsNewCollection] = useState(true); // the current image in the main canvas @@ -84,6 +85,7 @@ const ImageEditor = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addDoc // constants for image cutting const cutPts = useRef([]); + /** * * @param type The new tool type we are changing to @@ -380,8 +382,21 @@ const ImageEditor = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addDoc maxY = Math.max(cutPts.current[i].y, maxY); } ctx.closePath(); - ctx.globalCompositeOperation = 'destination-in'; - ctx.fill(); + switch (cutType) { // may need to move this before the drawing at all for the line cases + case BrushMode.IN: + ctx.globalCompositeOperation = 'destination-in'; + ctx.fill(); + break; + case BrushMode.OUT: + ctx.globalCompositeOperation = 'destination-out'; + ctx.fill(); + break; + case BrushMode.LINE_OUT: + ctx.globalCompositeOperation = 'destination-out'; + break; + case BrushMode.LINE_IN: + ctx.globalCompositeOperation = 'destination-in'; + } } const url = canvas.toDataURL(); // this does the same thing as convert img to canvasurl @@ -578,6 +593,37 @@ const ImageEditor = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addDoc return ImageToolButton(tool, tool.type === currTool.type, changeTool); })}
+ {currTool.type == ImageToolType.Cut && +
+
}
e.stopPropagation()}> {currTool.type === ImageToolType.GenerativeFill && ( Date: Mon, 28 Oct 2024 23:10:24 -0400 Subject: basic structure of 4 cut modes implemented, many bugs to fix --- .../views/nodes/imageEditor/ImageEditor.scss | 59 +++-- src/client/views/nodes/imageEditor/ImageEditor.tsx | 265 +++++++++++---------- .../views/nodes/imageEditor/ImageEditorButtons.tsx | 2 +- .../imageEditorUtils/imageEditorInterfaces.ts | 3 +- 4 files changed, 180 insertions(+), 149 deletions(-) (limited to 'src/client/views/nodes/imageEditor') diff --git a/src/client/views/nodes/imageEditor/ImageEditor.scss b/src/client/views/nodes/imageEditor/ImageEditor.scss index 49146aa23..c691e6a18 100644 --- a/src/client/views/nodes/imageEditor/ImageEditor.scss +++ b/src/client/views/nodes/imageEditor/ImageEditor.scss @@ -75,38 +75,47 @@ $scale: 0.5; } } - .iconContainer { + .sideControlsContainer { + width: 160px; position: absolute; - top: 3rem; - left: 2rem; - display: flex; - flex-direction: column; - gap: 4rem; + left: 0; + height: 100%; - .imageToolsContainer { + .sideControls { + position: absolute; + width: 120px; + top: 3rem; + left: 2rem; display: flex; flex-direction: column; - gap: 10px; - } + gap: 1rem; - .cutToolsContainer { - display: grid; - gap: 5px; - grid-template-columns: 1fr 1fr; - } + .imageToolsContainer { + display: flex; + flex-direction: column; + gap: 10px; + } - .undoRedoContainer { - justify-content: center; - display: flex; - flex-direction: row; - } + .cutToolsContainer { + display: grid; + gap: 5px; + grid-template-columns: 1fr 1fr; + } - .sliderContainer { - height: 225px; - width: 100%; - display: flex; - justify-content: center; - cursor: pointer; + .undoRedoContainer { + justify-content: center; + display: flex; + flex-direction: row; + } + + .sliderContainer { + margin: 3rem 0; + height: 225px; + width: 100%; + display: flex; + justify-content: center; + cursor: pointer; + } } } diff --git a/src/client/views/nodes/imageEditor/ImageEditor.tsx b/src/client/views/nodes/imageEditor/ImageEditor.tsx index d9f46876e..980f3e566 100644 --- a/src/client/views/nodes/imageEditor/ImageEditor.tsx +++ b/src/client/views/nodes/imageEditor/ImageEditor.tsx @@ -4,7 +4,7 @@ /* eslint-disable jsx-a11y/click-events-have-key-events */ /* eslint-disable react/function-component-definition */ import { Checkbox, FormControlLabel, Slider, TextField } from '@mui/material'; -import { IconButton } from 'browndash-components'; +import { Button, IconButton, Type } from 'browndash-components'; import * as React from 'react'; import { useEffect, useRef, useState } from 'react'; import { CgClose } from 'react-icons/cg'; @@ -85,7 +85,6 @@ const ImageEditor = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addDoc // constants for image cutting const cutPts = useRef([]); - /** * * @param type The new tool type we are changing to @@ -356,7 +355,7 @@ const ImageEditor = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addDoc setLoading(false); }; - const cutImage = async () => { + const cutImage = async (currCutType: BrushMode, brushWidth: number, prevEdits: { url: string; saveRes: Doc | undefined }[]) => { const img = currImg.current; const canvas = canvasRef.current; if (!canvas || !img) return; @@ -381,21 +380,22 @@ const ImageEditor = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addDoc maxX = Math.max(cutPts.current[i].x, maxX); maxY = Math.max(cutPts.current[i].y, maxY); } - ctx.closePath(); - switch (cutType) { // may need to move this before the drawing at all for the line cases + switch (currCutType) { case BrushMode.IN: + ctx.closePath(); ctx.globalCompositeOperation = 'destination-in'; ctx.fill(); break; case BrushMode.OUT: + ctx.closePath(); ctx.globalCompositeOperation = 'destination-out'; ctx.fill(); break; - case BrushMode.LINE_OUT: - ctx.globalCompositeOperation = 'destination-out'; - break; case BrushMode.LINE_IN: ctx.globalCompositeOperation = 'destination-in'; + ctx.lineWidth = brushWidth + 20; + ctx.stroke(); + break; } } @@ -423,31 +423,41 @@ const ImageEditor = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addDoc const image = new Image(); image.src = url; image.onload = async () => { - const croppedCanvas = document.createElement('canvas'); - const croppedCtx = croppedCanvas.getContext('2d'); - if (!croppedCtx) return; - croppedCanvas.width = maxX - minX; - croppedCanvas.height = maxY - minY; - croppedCtx.globalCompositeOperation = 'source-over'; - croppedCtx.clearRect(0, 0, croppedCanvas.width, croppedCanvas.height); - croppedCtx.drawImage(image, -minX, -minY); - const croppedURL = croppedCanvas.toDataURL(); - const croppedImage = new Image(); - croppedImage.src = croppedURL; - - currImg.current = croppedImage; - const newImgDoc = await createNewImgDoc(croppedImage, isFirstDoc.current); - if (isFirstDoc.current) isFirstDoc.current = false; + let finalImg: HTMLImageElement | undefined = image; + let finalImgURL: string = url; + if (currCutType == BrushMode.IN) { + const croppedData = cropImage(image, minX, maxY, minY, maxY); + finalImg = croppedData; + finalImgURL = croppedData.src; + } + currImg.current = finalImg; + const newImgDoc = await createNewImgDoc(finalImg, isFirstDoc.current); if (newImgDoc) { const docData = newImgDoc[DocData]; docData.backgroundColor = 'transparent'; + if (isFirstDoc.current) isFirstDoc.current = false; + setEdits([...prevEdits, { url: finalImgURL, saveRes: undefined }]); } - setEdits(prevEdits => [...prevEdits, { url: croppedURL, saveRes: undefined }]); setLoading(false); cutPts.current.length = 0; }; }; + const cropImage = (image: HTMLImageElement, minX: number, maxX: number, minY: number, maxY: number) => { + const croppedCanvas = document.createElement('canvas'); + const croppedCtx = croppedCanvas.getContext('2d'); + if (!croppedCtx) return image; + croppedCanvas.width = maxX - minX; + croppedCanvas.height = maxY - minY; + croppedCtx.globalCompositeOperation = 'source-over'; + croppedCtx.clearRect(0, 0, croppedCanvas.width, croppedCanvas.height); + croppedCtx.drawImage(image, -minX, -minY); + const croppedURL = croppedCanvas.toDataURL(); + const croppedImage = new Image(); + croppedImage.src = croppedURL; + return croppedImage; + }; + // adjusts all the img positions to be aligned const adjustImgPositions = () => { if (!parentDoc.current) return; @@ -577,7 +587,7 @@ const ImageEditor = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addDoc labelPlacement="end" sx={{ whiteSpace: 'nowrap' }} /> - + currTool.applyFunc(cutType, cursorData.width, edits)} loading={loading} onReset={handleReset} btnText={currTool.btnText} /> } onClick={handleViewClose} />
@@ -587,109 +597,120 @@ const ImageEditor = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addDoc // the side icons including tool type, the slider, and undo/redo function renderSideIcons() { return ( -
-
- {imageEditTools.map(tool => { - return ImageToolButton(tool, tool.type === currTool.type, changeTool); - })} -
- {currTool.type == ImageToolType.Cut && -
-
} -
e.stopPropagation()}> - {currTool.type === ImageToolType.GenerativeFill && ( - +
+
+ {imageEditTools.map(tool => { + return ImageToolButton(tool, tool.type === currTool.type, changeTool); + })} +
+ {currTool.type == ImageToolType.Cut && ( +
+
+ )} +
e.stopPropagation()}> + {currTool.type === ImageToolType.GenerativeFill && ( + { + setCursorData(prev => ({ ...prev, width: val as number })); + }} + /> + )} + {currTool.type === ImageToolType.Cut && ( + { + setCursorData(prev => ({ ...prev, width: val as number })); + }} + /> + )} +
+ {/* Undo and Redo */} +
+ { + e.stopPropagation(); + handleUndo(); }} - orientation="vertical" - min={genFillTool.sliderMin} - max={genFillTool.sliderMax} - defaultValue={genFillTool.sliderDefault} - size="small" - valueLabelDisplay="auto" - onChange={(e, val) => { - setCursorData(prev => ({ ...prev, width: val as number })); + onPointerUp={e => { + e.stopPropagation(); }} + color={activeColor} + tooltip="Undo" + icon={} /> - )} - {currTool.type === ImageToolType.Cut && ( - { + e.stopPropagation(); + handleRedo(); }} - orientation="vertical" - min={cutTool.sliderMin} - max={cutTool.sliderMax} - defaultValue={cutTool.sliderDefault} - size="small" - valueLabelDisplay="auto" - onChange={(e, val) => { - setCursorData(prev => ({ ...prev, width: val as number })); + onPointerUp={e => { + e.stopPropagation(); }} + color={activeColor} + tooltip="Redo" + icon={} /> - )} -
- {/* Undo and Redo */} -
- { - e.stopPropagation(); - handleUndo(); - }} - onPointerUp={e => { - e.stopPropagation(); - }} - color={activeColor} - tooltip="Undo" - icon={} - /> - { - e.stopPropagation(); - handleRedo(); - }} - onPointerUp={e => { - e.stopPropagation(); - }} - color={activeColor} - tooltip="Redo" - icon={} - /> +
); diff --git a/src/client/views/nodes/imageEditor/ImageEditorButtons.tsx b/src/client/views/nodes/imageEditor/ImageEditorButtons.tsx index e90babf9b..cb963616b 100644 --- a/src/client/views/nodes/imageEditor/ImageEditorButtons.tsx +++ b/src/client/views/nodes/imageEditor/ImageEditorButtons.tsx @@ -55,7 +55,7 @@ export function ImageToolButton(tool: ImageEditTool, isActive: boolean, selectTo return (
diff --git a/src/client/views/nodes/imageEditor/imageEditorUtils/imageEditorInterfaces.ts b/src/client/views/nodes/imageEditor/imageEditorUtils/imageEditorInterfaces.ts index 660af7c42..75659fc53 100644 --- a/src/client/views/nodes/imageEditor/imageEditorUtils/imageEditorInterfaces.ts +++ b/src/client/views/nodes/imageEditor/imageEditorUtils/imageEditorInterfaces.ts @@ -22,7 +22,7 @@ export interface ImageEditTool { name: string; btnText: string; icon: IconProp; - applyFunc: (currCutType: BrushMode, brushWidth: number, prevEdits: { url: string; saveRes: Doc | undefined }[]) => Promise; + applyFunc: (currCutType: BrushMode, brushWidth: number, prevEdits: { url: string; saveRes: Doc | undefined }[], isFirstDoc: boolean) => Promise; sliderMin?: number; sliderMax?: number; sliderDefault?: number; -- cgit v1.2.3-70-g09d2 From b5944e87f9d4f3149161de4de0d76db486461c76 Mon Sep 17 00:00:00 2001 From: eleanor-park Date: Wed, 30 Oct 2024 19:38:18 -0400 Subject: integrated image stickers with palette --- src/client/documents/Documents.ts | 2 +- src/client/util/CurrentUserUtils.ts | 4 +- src/client/views/DocumentDecorations.tsx | 4 +- src/client/views/LightboxView.tsx | 8 +- src/client/views/Main.tsx | 4 +- .../collectionFreeForm/CollectionFreeFormView.tsx | 8 +- src/client/views/nodes/ImageBox.tsx | 8 +- src/client/views/nodes/imageEditor/ImageEditor.tsx | 2 +- src/client/views/smartdraw/AnnotationPalette.scss | 56 ---- src/client/views/smartdraw/AnnotationPalette.tsx | 361 -------------------- src/client/views/smartdraw/SmartDrawHandler.tsx | 2 +- src/client/views/smartdraw/StickerPalette.scss | 56 ++++ src/client/views/smartdraw/StickerPalette.tsx | 362 +++++++++++++++++++++ src/fields/Doc.ts | 2 +- 14 files changed, 443 insertions(+), 436 deletions(-) delete mode 100644 src/client/views/smartdraw/AnnotationPalette.scss delete mode 100644 src/client/views/smartdraw/AnnotationPalette.tsx create mode 100644 src/client/views/smartdraw/StickerPalette.scss create mode 100644 src/client/views/smartdraw/StickerPalette.tsx (limited to 'src/client/views/nodes/imageEditor') diff --git a/src/client/documents/Documents.ts b/src/client/documents/Documents.ts index 5f2a592ae..b79e8930b 100644 --- a/src/client/documents/Documents.ts +++ b/src/client/documents/Documents.ts @@ -1044,7 +1044,7 @@ export namespace Docs { } export function AnnoPaletteDocument(options?: DocumentOptions) { - return InstanceFromProto(Prototypes.get(DocumentType.ANNOPALETTE), new List([Doc.MyAnnos]), { ...(options || {}) }); + return InstanceFromProto(Prototypes.get(DocumentType.ANNOPALETTE), new List([Doc.MyStickers]), { ...(options || {}) }); } export function DockDocument(documents: Array, config: string, options: DocumentOptions, id?: string) { diff --git a/src/client/util/CurrentUserUtils.ts b/src/client/util/CurrentUserUtils.ts index 274bc79be..bb3d232fb 100644 --- a/src/client/util/CurrentUserUtils.ts +++ b/src/client/util/CurrentUserUtils.ts @@ -160,9 +160,9 @@ export class CurrentUserUtils { return DocUtils.AssignDocField(doc, field, (opts,items) => Docs.Create.TreeDocument(items??[], opts), reqdOpts, templates, reqdScripts); } - static setupAnnoPalette(doc: Doc, field="myAnnos") { + static setupAnnoPalette(doc: Doc, field="myStickers") { const reqdOpts:DocumentOptions = { - title: "Saved Annotations", _xMargin: 0, _layout_showTitle: "title", hidden: false, _chromeHidden: true, + title: "Stickers", _xMargin: 0, _layout_showTitle: "title", hidden: false, _chromeHidden: true, _dragOnlyWithinContainer: true, layout_hideContextMenu: true, isSystem: true, _forceActive: true, _layout_autoHeight: true, _width: 500, _height: 300, _layout_fitWidth: true, _columnWidth: 35, ignoreClick: true, _lockedPosition: true, }; diff --git a/src/client/views/DocumentDecorations.tsx b/src/client/views/DocumentDecorations.tsx index 62f2de776..24ebca751 100644 --- a/src/client/views/DocumentDecorations.tsx +++ b/src/client/views/DocumentDecorations.tsx @@ -232,9 +232,9 @@ export class DocumentDecorations extends ObservableReactComponent { }[] = []; private _savedState: LightboxSavedState = {}; private _history: { doc: Doc; target?: Doc }[] = []; - private _annoPaletteView: AnnotationPalette | null = null; + private _annoPaletteView: StickerPalette | null = null; @observable private _future: Doc[] = []; @observable private _layoutTemplate: Opt = undefined; @observable private _layoutTemplateString: Opt = undefined; @@ -317,7 +317,7 @@ export class LightboxView extends ObservableReactComponent { - {this._showPalette && (this._annoPaletteView = r)} Document={DocCast(Doc.UserDoc().myLightboxDrawings)} />} + {this._showPalette && (this._annoPaletteView = r)} Document={DocCast(Doc.UserDoc().myLightboxDrawings)} />} {this.renderNavBtn(0, undefined, this._props.PanelHeight / 2 - 12.5, 'chevron-left', this._doc && this._history.length ? true : false, this.previous)} {this.renderNavBtn( this._props.PanelWidth - Math.min(this._props.PanelWidth / 4, this._props.maxBorder[0]), @@ -331,7 +331,7 @@ export class LightboxView extends ObservableReactComponent { {toggleBtn('lightboxView-navBtn', 'toggle reading view', BoolCast(this._doc?._layout_fitWidth), 'book-open', 'book', this.toggleFitWidth)} {toggleBtn('lightboxView-tabBtn', 'open document in a tab', false, 'file-export', '', this.downloadDoc)} - {toggleBtn('lightboxView-paletteBtn', 'toggle annotation palette', this._showPalette === true, 'palette', '', this.togglePalette)} + {toggleBtn('lightboxView-paletteBtn', 'toggle sticker palette', this._showPalette === true, 'palette', '', this.togglePalette)} {toggleBtn('lightboxView-penBtn', 'toggle pen annotation', Doc.ActiveTool === InkTool.Pen, 'pen', '', this.togglePen)} {toggleBtn('lightboxView-exploreBtn', 'toggle navigate only mode', SnappingManager.ExploreMode, 'globe-americas', '', this.toggleExplore)} diff --git a/src/client/views/Main.tsx b/src/client/views/Main.tsx index 2a59f6dc3..dda543470 100644 --- a/src/client/views/Main.tsx +++ b/src/client/views/Main.tsx @@ -64,7 +64,7 @@ import { ImportElementBox } from './nodes/importBox/ImportElementBox'; import { PresBox, PresElementBox } from './nodes/trails'; import { FaceRecognitionHandler } from './search/FaceRecognitionHandler'; import { SearchBox } from './search/SearchBox'; -import { AnnotationPalette } from './smartdraw/AnnotationPalette'; +import { StickerPalette } from './smartdraw/StickerPalette'; dotenv.config(); @@ -117,7 +117,7 @@ FieldLoader.ServerLoadStatus = { requested: 0, retrieved: 0, message: 'cache' }; KeyValueBox.Init(); PresBox.Init(TabDocView.AllTabDocs); DocumentContentsView.Init(KeyValueBox.LayoutString(), { - AnnotationPalette, + StickerPalette: StickerPalette, FormattedTextBox, ImageBox, FontIconBox, diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx index d2bc8f2c2..8d73601d1 100644 --- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx +++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx @@ -42,7 +42,7 @@ import { FocusViewOptions } from '../../nodes/FocusViewOptions'; import { FormattedTextBox } from '../../nodes/formattedText/FormattedTextBox'; import { OpenWhere, OpenWhereMod } from '../../nodes/OpenWhere'; import { PinDocView, PinProps } from '../../PinFuncs'; -import { AnnotationPalette } from '../../smartdraw/AnnotationPalette'; +import { StickerPalette } from '../../smartdraw/StickerPalette'; import { DrawingOptions, SmartDrawHandler } from '../../smartdraw/SmartDrawHandler'; import { StyleProp } from '../../StyleProp'; import { CollectionSubView, SubCollectionViewProps } from '../CollectionSubView'; @@ -1992,9 +1992,9 @@ export class CollectionFreeFormView extends CollectionSubView await AnnotationPalette.addToPalette(this.Document), 'save to palette')), - icon: this.Document.savedAsAnno ? 'clipboard-check' : 'file-arrow-down', + description: this.Document.savedAsSticker ? 'Sticker Saved!' : 'Save to Stickers', + event: action(undoable(async () => await StickerPalette.addToPalette(this.Document), 'save to palette')), + icon: this.Document.savedAsSticker ? 'clipboard-check' : 'file-arrow-down', }); this._props.renderDepth && optionItems.push({ diff --git a/src/client/views/nodes/ImageBox.tsx b/src/client/views/nodes/ImageBox.tsx index 0dfc0ec28..f98ff6d6e 100644 --- a/src/client/views/nodes/ImageBox.tsx +++ b/src/client/views/nodes/ImageBox.tsx @@ -32,6 +32,7 @@ import { MarqueeAnnotator } from '../MarqueeAnnotator'; import { OverlayView } from '../OverlayView'; import { AnchorMenu } from '../pdf/AnchorMenu'; import { PinDocView, PinProps } from '../PinFuncs'; +import { StickerPalette } from '../smartdraw/StickerPalette'; import { StyleProp } from '../StyleProp'; import { DocumentView } from './DocumentView'; import { FieldView, FieldViewProps } from './FieldView'; @@ -320,6 +321,11 @@ export class ImageBox extends ViewBoxAnnotatableComponent() { }), icon: 'pencil-alt', }); + funcs.push({ + description: this.Document.savedAsSticker ? 'Sticker Saved!' : 'Save to Stickers', + event: action(undoable(async () => await StickerPalette.addToPalette(this.Document), 'save to palette')), + icon: this.Document.savedAsSticker ? 'clipboard-check' : 'file-arrow-down', + }); ContextMenu.Instance?.addItem({ description: 'Options...', subitems: funcs, icon: 'asterisk' }); } }; @@ -338,7 +344,7 @@ export class ImageBox extends ViewBoxAnnotatableComponent() { @computed get nativeSize() { TraceMobx(); - if (this.paths.length && this.paths[0].includes('icon-hi')) return { nativeWidth: NumCast(this.layoutDoc._width), nativeHeight: NumCast(this.layoutDoc._height), nativeOrientation: 0} + if (this.paths.length && this.paths[0].includes('icon-hi')) return { nativeWidth: NumCast(this.layoutDoc._width), nativeHeight: NumCast(this.layoutDoc._height), nativeOrientation: 0 }; const nativeWidth = NumCast(this.dataDoc[this.fieldKey + '_nativeWidth'], NumCast(this.layoutDoc[this.fieldKey + '_nativeWidth'], 500)); const nativeHeight = NumCast(this.dataDoc[this.fieldKey + '_nativeHeight'], NumCast(this.layoutDoc[this.fieldKey + '_nativeHeight'], 500)); const nativeOrientation = NumCast(this.dataDoc[this.fieldKey + '_nativeOrientation'], 1); diff --git a/src/client/views/nodes/imageEditor/ImageEditor.tsx b/src/client/views/nodes/imageEditor/ImageEditor.tsx index e6d651ee9..9642dcc31 100644 --- a/src/client/views/nodes/imageEditor/ImageEditor.tsx +++ b/src/client/views/nodes/imageEditor/ImageEditor.tsx @@ -435,7 +435,7 @@ const ImageEditor = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addDoc const newImgDoc = await createNewImgDoc(finalImg, firstDoc); if (newImgDoc) { const docData = newImgDoc[DocData]; - docData.backgroundColor = 'transparent'; + // docData.backgroundColor = 'transparent'; if (firstDoc) setIsFirstDoc(false); setEdits([...prevEdits, { url: finalImgURL, saveRes: undefined }]); } diff --git a/src/client/views/smartdraw/AnnotationPalette.scss b/src/client/views/smartdraw/AnnotationPalette.scss deleted file mode 100644 index 4f11e8afc..000000000 --- a/src/client/views/smartdraw/AnnotationPalette.scss +++ /dev/null @@ -1,56 +0,0 @@ -.annotation-palette { - display: flex; - flex-direction: column; - align-items: center; - position: absolute; - right: 14px; - top: 50px; - border-radius: 5px; - margin: auto; -} - -.palette-create { - display: flex; - flex-direction: row; - width: 170px; - - .palette-create-input { - color: black; - width: 170px; - } -} - -.palette-create-options { - display: flex; - flex-direction: row; - justify-content: space-between; - width: 170px; - margin-top: 5px; - - .palette-color { - display: flex; - flex-direction: column; - align-items: center; - width: 40px; - } - - .palette-detail, - .palette-size { - display: flex; - flex-direction: column; - align-items: center; - width: 60px; - } -} - -.palette-buttons { - width: 100%; - display: flex; - flex-direction: row; - justify-content: space-between; -} - -.palette-save-reset { - display: flex; - flex-direction: row; -} diff --git a/src/client/views/smartdraw/AnnotationPalette.tsx b/src/client/views/smartdraw/AnnotationPalette.tsx deleted file mode 100644 index dc0c01d36..000000000 --- a/src/client/views/smartdraw/AnnotationPalette.tsx +++ /dev/null @@ -1,361 +0,0 @@ -import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; -import { Slider, Switch } from '@mui/material'; -import { Button } from 'browndash-components'; -import { action, makeObservable, observable } from 'mobx'; -import { observer } from 'mobx-react'; -import * as React from 'react'; -import { AiOutlineSend } from 'react-icons/ai'; -import ReactLoading from 'react-loading'; -import { returnEmptyFilter, returnFalse, returnTrue } from '../../../ClientUtils'; -import { emptyFunction } from '../../../Utils'; -import { Doc, DocListCast, returnEmptyDoclist } from '../../../fields/Doc'; -import { DocData } from '../../../fields/DocSymbols'; -import { ImageCast, NumCast } from '../../../fields/Types'; -import { ImageField } from '../../../fields/URLField'; -import { DocumentType } from '../../documents/DocumentTypes'; -import { Docs } from '../../documents/Documents'; -import { makeUserTemplateButtonOrImage } from '../../util/DropConverter'; -import { SettingsManager } from '../../util/SettingsManager'; -import { Transform } from '../../util/Transform'; -import { undoBatch } from '../../util/UndoManager'; -import { ObservableReactComponent } from '../ObservableReactComponent'; -import { DefaultStyleProvider, returnEmptyDocViewList } from '../StyleProvider'; -import { DocumentView, DocumentViewInternal } from '../nodes/DocumentView'; -import { FieldView } from '../nodes/FieldView'; -import './AnnotationPalette.scss'; -import { DrawingOptions, SmartDrawHandler } from './SmartDrawHandler'; - -interface AnnotationPaletteProps { - Document: Doc; -} - -/** - * The AnnotationPalette can be toggled in the lightbox view of a document. The goal of the palette - * is to offer an easy way for users to save then drag and drop repeated annotations onto a document. - * These annotations can be of any annotation type and operate similarly to user templates. - * - * On the "add" side of the palette, there is a way to create a drawing annotation with GPT. Users can - * enter the item to draw, toggle different settings, then GPT will generate three versions of the drawing - * to choose from. These drawings can then be saved to the palette as annotations. - */ -@observer -export class AnnotationPalette extends ObservableReactComponent { - @observable private _paletteMode: 'create' | 'view' = 'view'; - @observable private _userInput: string = ''; - @observable private _isLoading: boolean = false; - @observable private _canInteract: boolean = true; - @observable private _showRegenerate: boolean = false; - @observable private _docView: DocumentView | null = null; - @observable private _docCarouselView: DocumentView | null = null; - @observable private _opts: DrawingOptions = { text: '', complexity: 5, size: 200, autoColor: true, x: 0, y: 0 }; - private _gptRes: string[] = []; - - constructor(props: AnnotationPaletteProps) { - super(props); - makeObservable(this); - } - - public static LayoutString(fieldKey: string) { - return FieldView.LayoutString(AnnotationPalette, fieldKey); - } - - Contains = (view: DocumentView) => { - return (this._docView && (view.containerViewPath?.() ?? []).concat(view).includes(this._docView)) || (this._docCarouselView && (view.containerViewPath?.() ?? []).concat(view).includes(this._docCarouselView)); - }; - - return170 = () => 170; - - @action - handleKeyPress = async (event: React.KeyboardEvent) => { - if (event.key === 'Enter') { - await this.generateDrawings(); - } - }; - - @action - setPaletteMode = (mode: 'create' | 'view') => { - this._paletteMode = mode; - }; - - @action - setUserInput = (input: string) => { - if (!this._isLoading) this._userInput = input; - }; - - @action - setDetail = (detail: number) => { - if (this._canInteract) this._opts.complexity = detail; - }; - - @action - setColor = (autoColor: boolean) => { - if (this._canInteract) this._opts.autoColor = autoColor; - }; - - @action - setSize = (size: number) => { - if (this._canInteract) this._opts.size = size; - }; - - @action - resetPalette = (changePaletteMode: boolean) => { - if (changePaletteMode) this.setPaletteMode('view'); - this.setUserInput(''); - this.setDetail(5); - this.setColor(true); - this.setSize(200); - this._showRegenerate = false; - this._canInteract = true; - this._opts = { text: '', complexity: 5, size: 200, autoColor: true, x: 0, y: 0 }; - this._gptRes = []; - this._props.Document[DocData].data = undefined; - }; - - /** - * Adds a doc to the annotation palette. Gets a snapshot of the document to use as a preview in the palette. When this - * preview is dragged onto a parent document, a copy of that document is added as an annotation. - */ - public static addToPalette = async (doc: Doc) => { - if (!doc.savedAsAnno) { - const docView = DocumentView.getDocumentView(doc); - await docView?.ComponentView?.updateIcon?.(true); - const { clone } = await Doc.MakeClone(doc); - clone.title = doc.title; - const image = ImageCast(doc.icon, ImageCast(clone[Doc.LayoutFieldKey(clone)]))?.url?.href; - Doc.AddDocToList(Doc.MyAnnos, 'data', makeUserTemplateButtonOrImage(clone, image)); - doc.savedAsAnno = true; - } - }; - - public static getIcon(group: Doc) { - const docView = DocumentView.getDocumentView(group); - if (docView) { - docView.ComponentView?.updateIcon?.(true); - return new Promise(res => setTimeout(() => res(ImageCast(docView.Document.icon)), 1000)); - } - return undefined; - } - - /** - * Calls the draw with GPT functions in SmartDrawHandler to allow users to generate drawings straight from - * the annotation palette. - */ - @undoBatch - generateDrawings = action(async () => { - this._isLoading = true; - this._props.Document[DocData].data = undefined; - for (let i = 0; i < 3; i++) { - try { - SmartDrawHandler.Instance.AddDrawing = this.addDrawing; - this._canInteract = false; - if (this._showRegenerate) { - await SmartDrawHandler.Instance.regenerate(this._opts, this._gptRes[i], this._userInput); - } else { - await SmartDrawHandler.Instance.drawWithGPT({ X: 0, Y: 0 }, this._userInput, this._opts.complexity, this._opts.size, this._opts.autoColor); - } - } catch (e) { - console.log('Error generating drawing', e); - } - } - this._opts.text !== '' ? (this._opts.text = `${this._opts.text} ~~~ ${this._userInput}`) : (this._opts.text = this._userInput); - this._userInput = ''; - this._isLoading = false; - this._showRegenerate = true; - }); - - @action - addDrawing = (drawing: Doc, opts: DrawingOptions, gptRes: string) => { - this._gptRes.push(gptRes); - drawing[DocData].freeform_fitContentsToBox = true; - Doc.AddDocToList(this._props.Document, 'data', drawing); - }; - - /** - * Saves the currently showing, newly generated drawing to the annotation palette and sets the metadata. - * AddToPalette() is generically used to add any document to the palette, while this defines the behavior for when a user - * presses the "save drawing" button. - */ - saveDrawing = async () => { - const cIndex = NumCast(this._props.Document.carousel_index); - const focusedDrawing = DocListCast(this._props.Document.data)[cIndex]; - const docData = focusedDrawing[DocData]; - docData.title = this._opts.text.match(/^(.*?)~~~.*$/)?.[1] || this._opts.text; - docData.drawingInput = this._opts.text; - docData.drawingComplexity = this._opts.complexity; - docData.drawingColored = this._opts.autoColor; - docData.drawingSize = this._opts.size; - docData.drawingData = this._gptRes[cIndex]; - focusedDrawing.width = this._opts.size; - docData.x = this._opts.x; - docData.y = this._opts.y; - await AnnotationPalette.addToPalette(focusedDrawing); - this.resetPalette(true); - }; - - render() { - return ( -
e.stopPropagation()}> - {this._paletteMode === 'view' && ( - <> - (this._docView = r)} - Document={Doc.MyAnnos} - addDocument={undefined} - addDocTab={DocumentViewInternal.addDocTabFunc} - pinToPres={DocumentView.PinDoc} - containerViewPath={returnEmptyDocViewList} - styleProvider={DefaultStyleProvider} - removeDocument={returnFalse} - ScreenToLocalTransform={Transform.Identity} - PanelWidth={this.return170} - PanelHeight={this.return170} - renderDepth={0} - isContentActive={returnTrue} - focus={emptyFunction} - whenChildContentsActiveChanged={emptyFunction} - childFilters={returnEmptyFilter} - childFiltersByRanges={returnEmptyFilter} - searchFilterDocs={returnEmptyDoclist} - /> -
-
-
- Color - this.setColor(!this._opts.autoColor)} - /> -
-
- Detail - { - this.setDetail(val as number); - }} - valueLabelDisplay="auto" - /> -
-
- Size - { - this.setSize(val as number); - }} - valueLabelDisplay="auto" - /> -
-
- (this._docCarouselView = r)} - Document={this._props.Document} - addDocument={undefined} - addDocTab={DocumentViewInternal.addDocTabFunc} - pinToPres={DocumentView.PinDoc} - containerViewPath={returnEmptyDocViewList} - styleProvider={DefaultStyleProvider} - removeDocument={returnFalse} - ScreenToLocalTransform={Transform.Identity} - PanelWidth={this.return170} - PanelHeight={this.return170} - renderDepth={1} - isContentActive={returnTrue} - focus={emptyFunction} - whenChildContentsActiveChanged={emptyFunction} - childFilters={returnEmptyFilter} - childFiltersByRanges={returnEmptyFilter} - searchFilterDocs={returnEmptyDoclist} - /> -
-
- - - )} - - ); - } -} - -Docs.Prototypes.TemplateMap.set(DocumentType.ANNOPALETTE, { - layout: { view: AnnotationPalette, dataField: 'data' }, - options: { acl: '' }, -}); diff --git a/src/client/views/smartdraw/SmartDrawHandler.tsx b/src/client/views/smartdraw/SmartDrawHandler.tsx index b4635673c..0078dfc76 100644 --- a/src/client/views/smartdraw/SmartDrawHandler.tsx +++ b/src/client/views/smartdraw/SmartDrawHandler.tsx @@ -83,7 +83,7 @@ export class SmartDrawHandler extends ObservableReactComponent { /** * AddDrawing and RemoveDrawing are defined by the other classes that call the smart draw functions (i.e. CollectionFreeForm, FormattedTextBox, AnnotationPalette) to define how a drawing document should be added - or removed in their respective locations (to the freeform canvs, to the annotation palette's preview, etc.) + or removed in their respective locations (to the freeform canvas, to the sticker palette's preview, etc.) */ public AddDrawing: (doc: Doc, opts: DrawingOptions, gptRes: string) => void = unimplementedFunction; public RemoveDrawing: (useLastContainer: boolean, doc?: Doc) => void = unimplementedFunction; diff --git a/src/client/views/smartdraw/StickerPalette.scss b/src/client/views/smartdraw/StickerPalette.scss new file mode 100644 index 000000000..ca99410cf --- /dev/null +++ b/src/client/views/smartdraw/StickerPalette.scss @@ -0,0 +1,56 @@ +.sticker-palette { + display: flex; + flex-direction: column; + align-items: center; + position: absolute; + right: 14px; + top: 50px; + border-radius: 5px; + margin: auto; +} + +.palette-create { + display: flex; + flex-direction: row; + width: 170px; + + .palette-create-input { + color: black; + width: 170px; + } +} + +.palette-create-options { + display: flex; + flex-direction: row; + justify-content: space-between; + width: 170px; + margin-top: 5px; + + .palette-color { + display: flex; + flex-direction: column; + align-items: center; + width: 40px; + } + + .palette-detail, + .palette-size { + display: flex; + flex-direction: column; + align-items: center; + width: 60px; + } +} + +.palette-buttons { + width: 100%; + display: flex; + flex-direction: row; + justify-content: space-between; +} + +.palette-save-reset { + display: flex; + flex-direction: row; +} diff --git a/src/client/views/smartdraw/StickerPalette.tsx b/src/client/views/smartdraw/StickerPalette.tsx new file mode 100644 index 000000000..468d0fd13 --- /dev/null +++ b/src/client/views/smartdraw/StickerPalette.tsx @@ -0,0 +1,362 @@ +import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; +import { Slider, Switch } from '@mui/material'; +import { Button } from 'browndash-components'; +import { action, makeObservable, observable } from 'mobx'; +import { observer } from 'mobx-react'; +import * as React from 'react'; +import { AiOutlineSend } from 'react-icons/ai'; +import ReactLoading from 'react-loading'; +import { returnEmptyFilter, returnFalse, returnTrue } from '../../../ClientUtils'; +import { emptyFunction } from '../../../Utils'; +import { Doc, DocListCast, returnEmptyDoclist } from '../../../fields/Doc'; +import { DocData } from '../../../fields/DocSymbols'; +import { ImageCast, NumCast } from '../../../fields/Types'; +import { ImageField } from '../../../fields/URLField'; +import { DocumentType } from '../../documents/DocumentTypes'; +import { Docs } from '../../documents/Documents'; +import { makeUserTemplateButtonOrImage } from '../../util/DropConverter'; +import { SettingsManager } from '../../util/SettingsManager'; +import { Transform } from '../../util/Transform'; +import { undoBatch } from '../../util/UndoManager'; +import { ObservableReactComponent } from '../ObservableReactComponent'; +import { DefaultStyleProvider, returnEmptyDocViewList } from '../StyleProvider'; +import { DocumentView, DocumentViewInternal } from '../nodes/DocumentView'; +import { FieldView } from '../nodes/FieldView'; +import './StickerPalette.scss'; +import { DrawingOptions, SmartDrawHandler } from './SmartDrawHandler'; + +interface StickerPaletteProps { + Document: Doc; +} + +/** + * The StickerPalette can be toggled in the lightbox view of a document. The goal of the palette + * is to offer an easy way for users to create stickers and drag and drop them onto a document. + * These stickers can technically be of any document type and operate similarly to user templates. + * However, the palette is designed to be geared toward ink stickers and image stickers. + * + * On the "add" side of the palette, there is a way to create a drawing sticker with GPT. Users can + * enter the item to draw, toggle different settings, then GPT will generate three versions of the drawing + * to choose from. These drawings can then be saved to the palette as stickers. + */ +@observer +export class StickerPalette extends ObservableReactComponent { + @observable private _paletteMode: 'create' | 'view' = 'view'; + @observable private _userInput: string = ''; + @observable private _isLoading: boolean = false; + @observable private _canInteract: boolean = true; + @observable private _showRegenerate: boolean = false; + @observable private _docView: DocumentView | null = null; + @observable private _docCarouselView: DocumentView | null = null; + @observable private _opts: DrawingOptions = { text: '', complexity: 5, size: 200, autoColor: true, x: 0, y: 0 }; + private _gptRes: string[] = []; + + constructor(props: StickerPaletteProps) { + super(props); + makeObservable(this); + } + + public static LayoutString(fieldKey: string) { + return FieldView.LayoutString(StickerPalette, fieldKey); + } + + Contains = (view: DocumentView) => { + return (this._docView && (view.containerViewPath?.() ?? []).concat(view).includes(this._docView)) || (this._docCarouselView && (view.containerViewPath?.() ?? []).concat(view).includes(this._docCarouselView)); + }; + + return170 = () => 170; + + @action + handleKeyPress = async (event: React.KeyboardEvent) => { + if (event.key === 'Enter') { + await this.generateDrawings(); + } + }; + + @action + setPaletteMode = (mode: 'create' | 'view') => { + this._paletteMode = mode; + }; + + @action + setUserInput = (input: string) => { + if (!this._isLoading) this._userInput = input; + }; + + @action + setDetail = (detail: number) => { + if (this._canInteract) this._opts.complexity = detail; + }; + + @action + setColor = (autoColor: boolean) => { + if (this._canInteract) this._opts.autoColor = autoColor; + }; + + @action + setSize = (size: number) => { + if (this._canInteract) this._opts.size = size; + }; + + @action + resetPalette = (changePaletteMode: boolean) => { + if (changePaletteMode) this.setPaletteMode('view'); + this.setUserInput(''); + this.setDetail(5); + this.setColor(true); + this.setSize(200); + this._showRegenerate = false; + this._canInteract = true; + this._opts = { text: '', complexity: 5, size: 200, autoColor: true, x: 0, y: 0 }; + this._gptRes = []; + this._props.Document[DocData].data = undefined; + }; + + /** + * Adds a doc to the sticker palette. Gets a snapshot of the document to use as a preview in the palette. When this + * preview is dragged onto a parent document, a copy of that document is added as a sticker. + */ + public static addToPalette = async (doc: Doc) => { + if (!doc.savedAsSticker) { + const docView = DocumentView.getDocumentView(doc); + await docView?.ComponentView?.updateIcon?.(true); + const { clone } = await Doc.MakeClone(doc); + clone.title = doc.title; + const image = ImageCast(doc.icon, ImageCast(clone[Doc.LayoutFieldKey(clone)]))?.url?.href; + Doc.AddDocToList(Doc.MyStickers, 'data', makeUserTemplateButtonOrImage(clone, image)); + doc.savedAsSticker = true; + } + }; + + public static getIcon(group: Doc) { + const docView = DocumentView.getDocumentView(group); + if (docView) { + docView.ComponentView?.updateIcon?.(true); + return new Promise(res => setTimeout(() => res(ImageCast(docView.Document.icon)), 1000)); + } + return undefined; + } + + /** + * Calls the draw with GPT functions in SmartDrawHandler to allow users to generate drawings straight from + * the sticker palette. + */ + @undoBatch + generateDrawings = action(async () => { + this._isLoading = true; + this._props.Document[DocData].data = undefined; + for (let i = 0; i < 3; i++) { + try { + SmartDrawHandler.Instance.AddDrawing = this.addDrawing; + this._canInteract = false; + if (this._showRegenerate) { + await SmartDrawHandler.Instance.regenerate(this._opts, this._gptRes[i], this._userInput); + } else { + await SmartDrawHandler.Instance.drawWithGPT({ X: 0, Y: 0 }, this._userInput, this._opts.complexity, this._opts.size, this._opts.autoColor); + } + } catch (e) { + console.log('Error generating drawing', e); + } + } + this._opts.text !== '' ? (this._opts.text = `${this._opts.text} ~~~ ${this._userInput}`) : (this._opts.text = this._userInput); + this._userInput = ''; + this._isLoading = false; + this._showRegenerate = true; + }); + + @action + addDrawing = (drawing: Doc, opts: DrawingOptions, gptRes: string) => { + this._gptRes.push(gptRes); + drawing[DocData].freeform_fitContentsToBox = true; + Doc.AddDocToList(this._props.Document, 'data', drawing); + }; + + /** + * Saves the currently showing, newly generated drawing to the sticker palette and sets the metadata. + * AddToPalette() is generically used to add any document to the palette, while this defines the behavior for when a user + * presses the "save drawing" button. + */ + saveDrawing = async () => { + const cIndex = NumCast(this._props.Document.carousel_index); + const focusedDrawing = DocListCast(this._props.Document.data)[cIndex]; + const docData = focusedDrawing[DocData]; + docData.title = this._opts.text.match(/^(.*?)~~~.*$/)?.[1] || this._opts.text; + docData.drawingInput = this._opts.text; + docData.drawingComplexity = this._opts.complexity; + docData.drawingColored = this._opts.autoColor; + docData.drawingSize = this._opts.size; + docData.drawingData = this._gptRes[cIndex]; + focusedDrawing.width = this._opts.size; + docData.x = this._opts.x; + docData.y = this._opts.y; + await StickerPalette.addToPalette(focusedDrawing); + this.resetPalette(true); + }; + + render() { + return ( +
e.stopPropagation()}> + {this._paletteMode === 'view' && ( + <> + (this._docView = r)} + Document={Doc.MyStickers} + addDocument={undefined} + addDocTab={DocumentViewInternal.addDocTabFunc} + pinToPres={DocumentView.PinDoc} + containerViewPath={returnEmptyDocViewList} + styleProvider={DefaultStyleProvider} + removeDocument={returnFalse} + ScreenToLocalTransform={Transform.Identity} + PanelWidth={this.return170} + PanelHeight={this.return170} + renderDepth={0} + isContentActive={returnTrue} + focus={emptyFunction} + whenChildContentsActiveChanged={emptyFunction} + childFilters={returnEmptyFilter} + childFiltersByRanges={returnEmptyFilter} + searchFilterDocs={returnEmptyDoclist} + /> +
+
+
+ Color + this.setColor(!this._opts.autoColor)} + /> +
+
+ Detail + { + this.setDetail(val as number); + }} + valueLabelDisplay="auto" + /> +
+
+ Size + { + this.setSize(val as number); + }} + valueLabelDisplay="auto" + /> +
+
+ (this._docCarouselView = r)} + Document={this._props.Document} + addDocument={undefined} + addDocTab={DocumentViewInternal.addDocTabFunc} + pinToPres={DocumentView.PinDoc} + containerViewPath={returnEmptyDocViewList} + styleProvider={DefaultStyleProvider} + removeDocument={returnFalse} + ScreenToLocalTransform={Transform.Identity} + PanelWidth={this.return170} + PanelHeight={this.return170} + renderDepth={1} + isContentActive={returnTrue} + focus={emptyFunction} + whenChildContentsActiveChanged={emptyFunction} + childFilters={returnEmptyFilter} + childFiltersByRanges={returnEmptyFilter} + searchFilterDocs={returnEmptyDoclist} + /> +
+
+ + + )} + + ); + } +} + +Docs.Prototypes.TemplateMap.set(DocumentType.ANNOPALETTE, { + layout: { view: StickerPalette, dataField: 'data' }, + options: { acl: '' }, +}); diff --git a/src/fields/Doc.ts b/src/fields/Doc.ts index 81241f9fe..c77c2a77d 100644 --- a/src/fields/Doc.ts +++ b/src/fields/Doc.ts @@ -236,7 +236,7 @@ export class Doc extends RefField { public static get MyPublishedDocs() { return DocListCast(Doc.ActiveDashboard?.myPublishedDocs).concat(DocListCast(DocCast(Doc.UserDoc().myPublishedDocs)?.data)); } // prettier-ignore public static get MyDashboards() { return DocCast(Doc.UserDoc().myDashboards); } // prettier-ignore public static get MyTemplates() { return DocCast(Doc.UserDoc().myTemplates); } // prettier-ignore - public static get MyAnnos() { return DocCast(Doc.UserDoc().myAnnos); } // prettier-ignore + public static get MyStickers() { return DocCast(Doc.UserDoc().myStickers); } // prettier-ignore public static get MyLightboxDrawings() { return DocCast(Doc.UserDoc().myLightboxDrawings); } // prettier-ignore public static get MyImports() { return DocCast(Doc.UserDoc().myImports); } // prettier-ignore public static get MyFilesystem() { return DocCast(Doc.UserDoc().myFilesystem); } // prettier-ignore -- cgit v1.2.3-70-g09d2 From f7bc3b892bbb3f559fa8bac1ebe9d2d747a5728c Mon Sep 17 00:00:00 2001 From: eleanor-park Date: Wed, 30 Oct 2024 20:14:53 -0400 Subject: blending changes --- src/client/views/nodes/imageEditor/ImageEditor.tsx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'src/client/views/nodes/imageEditor') diff --git a/src/client/views/nodes/imageEditor/ImageEditor.tsx b/src/client/views/nodes/imageEditor/ImageEditor.tsx index 9642dcc31..8f6a7e023 100644 --- a/src/client/views/nodes/imageEditor/ImageEditor.tsx +++ b/src/client/views/nodes/imageEditor/ImageEditor.tsx @@ -435,7 +435,8 @@ const ImageEditor = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addDoc const newImgDoc = await createNewImgDoc(finalImg, firstDoc); if (newImgDoc) { const docData = newImgDoc[DocData]; - // docData.backgroundColor = 'transparent'; + docData.backgroundColor = 'transparent'; + docData.disableMixBlend = true; if (firstDoc) setIsFirstDoc(false); setEdits([...prevEdits, { url: finalImgURL, saveRes: undefined }]); } -- cgit v1.2.3-70-g09d2 From 8847d6cf5a0eaeeddaefc71d0202247b4d2d096b Mon Sep 17 00:00:00 2001 From: eleanor-park Date: Sun, 3 Nov 2024 16:32:28 -0500 Subject: working on smart draw width bugs --- src/client/util/CurrentUserUtils.ts | 2 +- src/client/views/LightboxView.tsx | 3 +-- .../views/collections/collectionFreeForm/CollectionFreeFormView.tsx | 2 +- src/client/views/nodes/imageEditor/ImageEditor.tsx | 2 +- src/client/views/smartdraw/SmartDrawHandler.tsx | 2 +- src/fields/InkField.ts | 2 +- 6 files changed, 6 insertions(+), 7 deletions(-) (limited to 'src/client/views/nodes/imageEditor') diff --git a/src/client/util/CurrentUserUtils.ts b/src/client/util/CurrentUserUtils.ts index 009cd7de7..99faafe38 100644 --- a/src/client/util/CurrentUserUtils.ts +++ b/src/client/util/CurrentUserUtils.ts @@ -779,7 +779,7 @@ pie title Minerals in my tap water { title: " Size", toolTip: "Size of area pencil eraser", btnType: ButtonType.NumberSliderButton, toolType: "eraserWidth", ignoreClick: true, scripts: {script: '{ return setInkProperty(this.toolType, value, _readOnly_);}'}, funcs: {hidden:"NotRadiusEraser()"}, numBtnMin: 1, linearBtnWidth:40}, { title: "Mask", toolTip: "Make Stroke a Stencil Mask", btnType: ButtonType.ToggleButton, icon: "user-circle",toolType: "inkMask", scripts: {onClick:'{ return setInkProperty(this.toolType, value, _readOnly_);}'}, funcs: {hidden:"IsNoviceMode()" } }, { title: "Labels", toolTip: "Show Labels Inside Shapes", btnType: ButtonType.ToggleButton, icon: "text-width", toolType: "labels", scripts: {onClick:'{ return setInkProperty(this.toolType, value, _readOnly_);}'}}, - { title: "Smart Draw", toolTip: "Draw with GPT", btnType: ButtonType.ToggleButton, icon: "user-pen", toolType: "smartdraw", scripts: {onClick:'{ return setActiveTool(this.toolType, false, _readOnly_);}'}, funcs: {hidden: "IsNoviceMode()"}}, + { title: "Smart Draw", toolTip: "Draw with GPT", btnType: ButtonType.ToggleButton, icon: "user-pen", toolType: InkTool.SmartDraw, scripts: {onClick:'{ return setActiveTool(this.toolType, false, _readOnly_);}'}, funcs: {hidden: "IsNoviceMode()"}}, ]; } diff --git a/src/client/views/LightboxView.tsx b/src/client/views/LightboxView.tsx index 65de85008..ce6b26323 100644 --- a/src/client/views/LightboxView.tsx +++ b/src/client/views/LightboxView.tsx @@ -35,7 +35,7 @@ type LightboxSavedState = { [key: string]: FieldResult; }; // prettier-ignore @observer export class LightboxView extends ObservableReactComponent { /** - * Determines whether a DocumentView is descendant of the lightbox view (or any of its pop-ups like the annotationPalette) + * Determines whether a DocumentView is descendant of the lightbox view (or any of its pop-ups like the stickerPalette) * @param view * @returns true if a DocumentView is descendant of the lightbox view */ @@ -211,7 +211,6 @@ export class LightboxView extends ObservableReactComponent { }; togglePalette = () => { this._showPalette = !this._showPalette; - // if (this._showPalette === false) AnnotationPalette.Instance.resetPalette(true); }; togglePen = () => { Doc.ActiveTool = Doc.ActiveTool === InkTool.Ink ? InkTool.None : InkTool.Ink; diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx index 65b2702c6..4f62b6fbb 100644 --- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx +++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx @@ -1285,7 +1285,7 @@ export class CollectionFreeFormView extends CollectionSubView { const docData = doc[DocData]; docData.title = opts.text.match(/^(.*?)~~~.*$/)?.[1] || opts.text; - docData.width = opts.size; + docData._width = opts.size; docData.drawingInput = opts.text; docData.drawingComplexity = opts.complexity; docData.drawingColored = opts.autoColor; diff --git a/src/client/views/nodes/imageEditor/ImageEditor.tsx b/src/client/views/nodes/imageEditor/ImageEditor.tsx index 8f6a7e023..c8fe5adc3 100644 --- a/src/client/views/nodes/imageEditor/ImageEditor.tsx +++ b/src/client/views/nodes/imageEditor/ImageEditor.tsx @@ -426,7 +426,7 @@ const ImageEditor = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addDoc image.onload = async () => { let finalImg: HTMLImageElement | undefined = image; let finalImgURL: string = url; - if (currCutType == BrushMode.IN) { + if (currCutType == BrushMode.IN || currCutType == BrushMode.LINE_IN) { const croppedData = cropImage(image, minX, maxX, minY, maxY); finalImg = croppedData; finalImgURL = croppedData.src; diff --git a/src/client/views/smartdraw/SmartDrawHandler.tsx b/src/client/views/smartdraw/SmartDrawHandler.tsx index 4c5be047b..83360bb6e 100644 --- a/src/client/views/smartdraw/SmartDrawHandler.tsx +++ b/src/client/views/smartdraw/SmartDrawHandler.tsx @@ -82,7 +82,7 @@ export class SmartDrawHandler extends ObservableReactComponent { /** * AddDrawing and RemoveDrawing are defined by the other classes that call the smart draw functions (i.e. - CollectionFreeForm, FormattedTextBox, AnnotationPalette) to define how a drawing document should be added + CollectionFreeForm, FormattedTextBox, StickerPalette) to define how a drawing document should be added or removed in their respective locations (to the freeform canvas, to the sticker palette's preview, etc.) */ public AddDrawing: (doc: Doc, opts: DrawingOptions, gptRes: string) => void = unimplementedFunction; diff --git a/src/fields/InkField.ts b/src/fields/InkField.ts index f3ff5f11f..702301b31 100644 --- a/src/fields/InkField.ts +++ b/src/fields/InkField.ts @@ -11,7 +11,7 @@ export enum InkTool { None = 'None', Ink = 'Ink', Eraser = 'Eraser', // not a real tool, but a class of tools - SmartDraw = 'Smartdraw', + SmartDraw = 'smartdraw', } export enum InkInkTool { -- cgit v1.2.3-70-g09d2 From 0fbe7696fa23cd0621d4053b430cec01d72f6801 Mon Sep 17 00:00:00 2001 From: eleanor-park Date: Mon, 25 Nov 2024 22:54:11 -0500 Subject: bug fixes to image cropping --- src/client/views/nodes/imageEditor/ImageEditor.tsx | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) (limited to 'src/client/views/nodes/imageEditor') diff --git a/src/client/views/nodes/imageEditor/ImageEditor.tsx b/src/client/views/nodes/imageEditor/ImageEditor.tsx index 5c7e09645..5c4c83eef 100644 --- a/src/client/views/nodes/imageEditor/ImageEditor.tsx +++ b/src/client/views/nodes/imageEditor/ImageEditor.tsx @@ -451,21 +451,13 @@ const ImageEditor = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addDoc const croppedCanvas = document.createElement('canvas'); const croppedCtx = croppedCanvas.getContext('2d'); if (!croppedCtx) return image; - croppedCanvas.width = maxX - minX; - croppedCanvas.height = maxY - minY; + const cropWidth = Math.abs(maxX - minX); + const cropHeight = Math.abs(maxY - minY); + croppedCanvas.width = cropWidth; + croppedCanvas.height = cropHeight; croppedCtx.globalCompositeOperation = 'source-over'; - croppedCtx.clearRect(0, 0, croppedCanvas.width, croppedCanvas.height); - croppedCtx.drawImage( - image, - minX, - minY, - maxX - minX, - maxY - minY, // Source image crop area - 0, - 0, - maxX - minX, - maxY - minY // Destination area on the canvas - ); + croppedCtx.clearRect(0, 0, cropWidth, cropHeight); + croppedCtx.drawImage(image, minX, minY, cropWidth, cropHeight, 0, 0, cropWidth, cropHeight); const croppedURL = croppedCanvas.toDataURL(); const croppedImage = new Image(); croppedImage.src = croppedURL; -- cgit v1.2.3-70-g09d2 From e6a4b7c05899ee6d6a821666b72023856081d5ca Mon Sep 17 00:00:00 2001 From: eleanor-park Date: Fri, 13 Dec 2024 17:45:34 -0500 Subject: added comments, refactored duplicate code, and resolved lint errors --- src/client/views/nodes/imageEditor/ImageEditor.tsx | 137 +++++++++++---------- .../imageEditorUtils/imageEditorInterfaces.ts | 18 +-- 2 files changed, 82 insertions(+), 73 deletions(-) (limited to 'src/client/views/nodes/imageEditor') diff --git a/src/client/views/nodes/imageEditor/ImageEditor.tsx b/src/client/views/nodes/imageEditor/ImageEditor.tsx index 5c4c83eef..a742673e0 100644 --- a/src/client/views/nodes/imageEditor/ImageEditor.tsx +++ b/src/client/views/nodes/imageEditor/ImageEditor.tsx @@ -22,11 +22,11 @@ import { ImageEditorData } from '../ImageBox'; import { OpenWhereMod } from '../OpenWhere'; import './ImageEditor.scss'; import { ApplyFuncButtons, ImageToolButton } from './ImageEditorButtons'; -import { BrushHandler, BrushType } from './imageEditorUtils/BrushHandler'; +import { BrushHandler } from './imageEditorUtils/BrushHandler'; import { APISuccess, ImageUtility } from './imageEditorUtils/ImageHandler'; import { PointerHandler } from './imageEditorUtils/PointerHandler'; -import { activeColor, bgColor, canvasSize, eraserColor, freeformRenderSize, newCollectionSize, offsetDistanceY, offsetX } from './imageEditorUtils/imageEditorConstants'; -import { BrushMode, CursorData, ImageDimensions, ImageEditTool, ImageToolType, Point } from './imageEditorUtils/imageEditorInterfaces'; +import { activeColor, bgColor, brushWidthOffset, canvasSize, eraserColor, freeformRenderSize, newCollectionSize, offsetDistanceY, offsetX } from './imageEditorUtils/imageEditorConstants'; +import { CutMode, CursorData, ImageDimensions, ImageEditTool, ImageToolType, Point } from './imageEditorUtils/imageEditorInterfaces'; import { DocumentView } from '../DocumentView'; import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; import { ImageField } from '../../../../fields/URLField'; @@ -64,7 +64,7 @@ const ImageEditor = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addDoc width: canvasSize, height: canvasSize, }); - const [cutType, setCutType] = useState(BrushMode.IN); + const [cutType, setCutType] = useState(CutMode.IN); // whether to create a new collection or not const [isNewCollection, setIsNewCollection] = useState(true); // the current image in the main canvas @@ -183,7 +183,7 @@ const ImageEditor = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addDoc x: currPoint.x - e.movementX / canvasScale, y: currPoint.y - e.movementY / canvasScale, }; - const pts = BrushHandler.createBrushPathOverlay(lastPoint, currPoint, cursorData.width / 2 / canvasScale, ctx, eraserColor, BrushType.CUT); + const pts = BrushHandler.createBrushPathOverlay(lastPoint, currPoint, cursorData.width / 2 / canvasScale, ctx, eraserColor); cutPts.current.push(...pts); }; @@ -283,7 +283,7 @@ const ImageEditor = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addDoc })); }; - // Get AI Edit + // Get AI Edit for Generative Fill const getEdit = async () => { const img = currImg.current; if (!img) return; @@ -304,32 +304,14 @@ const ImageEditor = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addDoc // create first image if (!newCollectionRef.current) { - if (!isNewCollection && imageRootDoc) { - // if the parent hasn't been set yet - if (!parentDoc.current) parentDoc.current = imageRootDoc; - } else { - if (!(originalImg.current && imageRootDoc)) return; - // create new collection and add it to the view - newCollectionRef.current = Docs.Create.FreeformDocument([], { - x: NumCast(imageRootDoc.x) + NumCast(imageRootDoc._width) + offsetX, - y: NumCast(imageRootDoc.y), - _width: newCollectionSize, - _height: newCollectionSize, - title: 'Image edit collection', - }); - DocUtils.MakeLink(imageRootDoc, newCollectionRef.current, { link_relationship: 'Image Edit Version History' }); - - // opening new tab - CollectionDockingView.AddSplit(newCollectionRef.current, OpenWhereMod.right); - - // add the doc to the main freeform - // eslint-disable-next-line no-use-before-define - await createNewImgDoc(originalImg.current, true); - } + createNewCollection(); } else { childrenDocs.current = []; } - + if (!(originalImg.current && imageRootDoc)) return; + // add the doc to the main freeform + // eslint-disable-next-line no-use-before-define + await createNewImgDoc(originalImg.current, true); originalImg.current = currImg.current; originalDoc.current = parentDoc.current; const { urls } = res as APISuccess; @@ -356,23 +338,32 @@ const ImageEditor = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addDoc setLoading(false); }; - const cutImage = async (currCutType: BrushMode, brushWidth: number, prevEdits: { url: string; saveRes: Doc | undefined }[], firstDoc: boolean) => { + /** + * This function performs image cutting based on the inputted BrushMode. There are currently four ways to cut images: + * 1. By outlining the area that should be kept (BrushMode.IN) + * 2. By outlining the area that should be removed (BrushMode.OUT) + * 3. By drawing in the area that should be kept (where the image is brushed, the image will remain and everything else will be removed) (BrushMode.DRAW_IN) + * 4. By drawing the area that she be removed, so this operates as an eraser (BrushMode.ERASE) + * @param currCutType BrushMode enum that determines what kind of cutting operation to perform + * @param firstDoc boolean for whether it's the first edited image. This is for positioning of the edited images when they render on the canvas. + */ + const cutImage = async (currCutType: CutMode, brushWidth: number, prevEdits: { url: string; saveRes: Doc | undefined }[], firstDoc: boolean) => { const img = currImg.current; const canvas = canvasRef.current; if (!canvas || !img) return; const ctx = ImageUtility.getCanvasContext(canvasRef); if (!ctx) return; - setLoading(true); // get the original image const canvasOriginalImg = ImageUtility.getCanvasImg(img); if (!canvasOriginalImg) return; - // NOTE: cutting two diff shapes can be made possible by having the user press a button to set a new shape! + setLoading(true); const currPts = [...cutPts.current]; - if (currCutType !== BrushMode.LINE_OUT) handleReset(); // gets rid of the visible brush strokes (mostly needed for line_in) unless it's erasing (which depends on the brush strokes) + if (currCutType !== CutMode.ERASE) handleReset(); // gets rid of the visible brush strokes (mostly needed for line_in) unless it's erasing (which depends on the brush strokes) let minX = img.width; let maxX = 0; let minY = img.height; let maxY = 0; + // currPts is populated by the brush strokes' points, so this code is drawing a path along the points if (currPts.length) { ctx.beginPath(); ctx.moveTo(currPts[0].x, currPts[0].y); @@ -383,44 +374,30 @@ const ImageEditor = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addDoc maxX = Math.max(currPts[i].x, maxX); maxY = Math.max(currPts[i].y, maxY); } - switch (currCutType) { - case BrushMode.IN: + switch ( + currCutType // use different canvas operations depending on the type of cutting we're applying + ) { + case CutMode.IN: ctx.closePath(); ctx.globalCompositeOperation = 'destination-in'; ctx.fill(); break; - case BrushMode.OUT: + case CutMode.OUT: ctx.closePath(); ctx.globalCompositeOperation = 'destination-out'; ctx.fill(); break; - case BrushMode.LINE_IN: + case CutMode.DRAW_IN: ctx.globalCompositeOperation = 'destination-in'; - ctx.lineWidth = brushWidth + 10; // added offset because width gets cut off a little bit + ctx.lineWidth = brushWidth + brushWidthOffset; // added offset because width gets cut off a little bit ctx.stroke(); break; } } - const url = canvas.toDataURL(); // this does the same thing as convert img to canvasurl + const url = canvas.toDataURL(); if (!newCollectionRef.current) { - if (!isNewCollection && imageRootDoc) { - // if the parent hasn't been set yet - if (!parentDoc.current) parentDoc.current = imageRootDoc; - } else { - if (!(originalImg.current && imageRootDoc)) return; - // create new collection and add it to the view - newCollectionRef.current = Docs.Create.FreeformDocument([], { - x: NumCast(imageRootDoc.x) + NumCast(imageRootDoc._width) + offsetX, - y: NumCast(imageRootDoc.y), - _width: newCollectionSize, - _height: newCollectionSize, - title: 'Image edit collection', - }); - DocUtils.MakeLink(imageRootDoc, newCollectionRef.current, { link_relationship: 'Image Edit Version History' }); - // opening new tab - CollectionDockingView.AddSplit(newCollectionRef.current, OpenWhereMod.right); - } + createNewCollection(); } const image = new Image(); @@ -428,7 +405,8 @@ const ImageEditor = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addDoc image.onload = async () => { let finalImg: HTMLImageElement | undefined = image; let finalImgURL: string = url; - if (currCutType == BrushMode.IN || currCutType == BrushMode.LINE_IN) { + // crop the image for these brush modes to remove excess blank space around the image contents + if (currCutType == CutMode.IN || currCutType == CutMode.DRAW_IN) { const croppedData = cropImage(image, minX, maxX, minY, maxY); finalImg = croppedData; finalImgURL = croppedData.src; @@ -436,6 +414,7 @@ const ImageEditor = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addDoc currImg.current = finalImg; const newImgDoc = await createNewImgDoc(finalImg, firstDoc); if (newImgDoc) { + // set the image to transparent to remove the background / brushstrokes const docData = newImgDoc[DocData]; docData.backgroundColor = 'transparent'; docData.disableMixBlend = true; @@ -447,6 +426,34 @@ const ImageEditor = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addDoc }; }; + /** + * Creates a new collection to put the image edits on. Adds to a new tab on the right if "Create New Collection" is checked. + * @returns + */ + const createNewCollection = () => { + if (!isNewCollection && imageRootDoc) { + // if the parent hasn't been set yet + if (!parentDoc.current) parentDoc.current = imageRootDoc; + } else { + if (!(originalImg.current && imageRootDoc)) return; + // create new collection and add it to the view + newCollectionRef.current = Docs.Create.FreeformDocument([], { + x: NumCast(imageRootDoc.x) + NumCast(imageRootDoc._width) + offsetX, + y: NumCast(imageRootDoc.y), + _width: newCollectionSize, + _height: newCollectionSize, + title: 'Image edit collection', + }); + DocUtils.MakeLink(imageRootDoc, newCollectionRef.current, { link_relationship: 'Image Edit Version History' }); + // opening new tab + CollectionDockingView.AddSplit(newCollectionRef.current, OpenWhereMod.right); + } + }; + + /** + * This function crops an image based on the inputted dimensions. This is used to automatically adjust the images that are + * edited to be smaller than the original (i.e. for cutting into a small part of the image) + */ const cropImage = (image: HTMLImageElement, minX: number, maxX: number, minY: number, maxY: number) => { const croppedCanvas = document.createElement('canvas'); const croppedCtx = croppedCanvas.getContext('2d'); @@ -617,36 +624,36 @@ const ImageEditor = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addDoc style={{ width: '100%' }} text="Keep in" type={Type.TERT} - color={cutType == BrushMode.IN ? SettingsManager.userColor : bgColor} + color={cutType == CutMode.IN ? SettingsManager.userColor : bgColor} onClick={() => { - setCutType(BrushMode.IN); + setCutType(CutMode.IN); }} />