diff options
Diffstat (limited to 'src')
-rw-r--r-- | src/client/util/bezierFit.ts | 1 | ||||
-rw-r--r-- | src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx | 2 | ||||
-rw-r--r-- | src/client/views/nodes/ImageBox.tsx | 5 | ||||
-rw-r--r-- | src/client/views/nodes/imageEditor/imageEditingUtils/ImageHandler.ts | 312 | ||||
-rw-r--r-- | src/client/views/smartdraw/DrawingFillHandler.tsx | 18 | ||||
-rw-r--r-- | src/client/views/smartdraw/FireflyConstants.ts | 20 | ||||
-rw-r--r-- | src/client/views/smartdraw/SmartDrawHandler.scss | 3 | ||||
-rw-r--r-- | src/client/views/smartdraw/SmartDrawHandler.tsx | 34 | ||||
-rw-r--r-- | src/server/ApiManagers/FireflyManager.ts | 7 |
9 files changed, 51 insertions, 351 deletions
diff --git a/src/client/util/bezierFit.ts b/src/client/util/bezierFit.ts index 84b27e84c..7ef370d48 100644 --- a/src/client/util/bezierFit.ts +++ b/src/client/util/bezierFit.ts @@ -703,6 +703,7 @@ export function SVGToBezier(name: SVGType, attributes: any): Point[] { coordList.push({ X: parseInt(match[1]), Y: parseInt(match[2]) }); coordList.push({ X: parseInt(match[1]), Y: parseInt(match[2]) }); coordList.push({ X: parseInt(match[3]), Y: parseInt(match[4]) }); + coordList.push({ X: parseInt(match[3]), Y: parseInt(match[4]) }); lastPt = { X: parseInt(match[3]), Y: parseInt(match[4]) }; } else if (match[0].startsWith('C')) { coordList.push({ X: parseInt(match[5]), Y: parseInt(match[6]) }); diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx index 4bccdd286..ef0b80720 100644 --- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx +++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx @@ -1995,7 +1995,7 @@ export class CollectionFreeFormView extends CollectionSubView<Partial<collection }); this.layoutDoc.drawingData != undefined && optionItems.push({ - description: 'Show Drawing Editor', + description: 'Regenerate AI Drawing', event: action(() => { SmartDrawHandler.Instance.CreateDrawingDoc = this.createDrawingDoc; SmartDrawHandler.Instance.AddDrawing = this.addDrawing; diff --git a/src/client/views/nodes/ImageBox.tsx b/src/client/views/nodes/ImageBox.tsx index 8f6a90e61..7ce429f0f 100644 --- a/src/client/views/nodes/ImageBox.tsx +++ b/src/client/views/nodes/ImageBox.tsx @@ -355,9 +355,8 @@ export class ImageBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { this.layoutDoc.ai_generated && funcs.push({ description: 'Regenerate AI Image', - event: action(() => { - console.log('COOOORDS', this.dataDoc.width as number, this.dataDoc.y as number); - !SmartDrawHandler.Instance.ShowRegenerate ? SmartDrawHandler.Instance.displayRegenerate(this.dataDoc.x as number, (this.dataDoc.y as number) - 10) : SmartDrawHandler.Instance.hideRegenerate(); + event: action(e => { + !SmartDrawHandler.Instance.ShowRegenerate ? SmartDrawHandler.Instance.displayRegenerate(e?.x || 0, e?.y || 0) : SmartDrawHandler.Instance.hideRegenerate(); }), icon: 'pen-to-square', }); diff --git a/src/client/views/nodes/imageEditor/imageEditingUtils/ImageHandler.ts b/src/client/views/nodes/imageEditor/imageEditingUtils/ImageHandler.ts deleted file mode 100644 index 514e8a94f..000000000 --- a/src/client/views/nodes/imageEditor/imageEditingUtils/ImageHandler.ts +++ /dev/null @@ -1,312 +0,0 @@ -import { RefObject } from 'react'; -import { bgColor, canvasSize } from '../ImageEditorUtils/imageEditorConstants'; - -export interface APISuccess { - status: 'success'; - urls: string[]; -} - -export interface APIError { - status: 'error'; - message: string; -} - -export class ImageUtility { - /** - * - * @param canvas Canvas to convert - * @returns Blob of canvas - */ - static canvasToBlob = (canvas: HTMLCanvasElement): Promise<Blob> => - new Promise(resolve => { - canvas.toBlob(blob => { - if (blob) { - resolve(blob); - } - }, 'image/png'); - }); - - // given a square api image, get the cropped img - static getCroppedImg = (img: HTMLImageElement, width: number, height: number): HTMLCanvasElement | undefined => { - // Create a new canvas element - const canvas = document.createElement('canvas'); - canvas.width = width; - canvas.height = height; - const ctx = canvas.getContext('2d'); - if (ctx) { - // Clear the canvas - ctx.clearRect(0, 0, canvas.width, canvas.height); - if (width < height) { - // horizontal padding, x offset - const xOffset = (canvasSize - width) / 2; - ctx.drawImage(img, xOffset, 0, canvas.width, canvas.height, 0, 0, canvas.width, canvas.height); - } else { - // vertical padding, y offset - const yOffset = (canvasSize - height) / 2; - ctx.drawImage(img, 0, yOffset, canvas.width, canvas.height, 0, 0, canvas.width, canvas.height); - } - return canvas; - } - return undefined; - }; - - // converts an image to a canvas data url - static convertImgToCanvasUrl = async (imageSrc: string, width: number, height: number): Promise<string> => - new Promise<string>((resolve, reject) => { - const img = new Image(); - img.onload = () => { - const canvas = this.getCroppedImg(img, width, height); - if (canvas) { - const dataUrl = canvas.toDataURL(); - resolve(dataUrl); - } - }; - img.onerror = error => { - reject(error); - }; - img.src = imageSrc; - }); - - // calls the openai api to get image edits - static getEdit = async (imgBlob: Blob, maskBlob: Blob, prompt: string, n?: number): Promise<APISuccess | APIError> => { - const apiUrl = 'https://api.openai.com/v1/images/edits'; - const fd = new FormData(); - fd.append('image', imgBlob, 'image.png'); - fd.append('mask', maskBlob, 'mask.png'); - fd.append('prompt', prompt); - fd.append('size', '1024x1024'); - fd.append('n', n ? JSON.stringify(n) : '1'); - fd.append('response_format', 'b64_json'); - - try { - const res = await fetch(apiUrl, { - method: 'POST', - headers: { - Authorization: `Bearer ${process.env.OPENAI_KEY}`, - }, - body: fd, - }); - const data = await res.json(); - console.log(data.data); - return { - status: 'success', - urls: (data.data as { b64_json: string }[]).map(urlData => `data:image/png;base64,${urlData.b64_json}`), - }; - } catch (err) { - console.log(err); - return { status: 'error', message: 'API error.' }; - } - }; - - // mock api call - static mockGetEdit = async (mockSrc: string): Promise<APISuccess | APIError> => ({ - status: 'success', - urls: [mockSrc, mockSrc, mockSrc], - }); - - // Gets the canvas rendering context of a canvas - static getCanvasContext = (canvasRef: RefObject<HTMLCanvasElement>): CanvasRenderingContext2D | null => { - if (!canvasRef.current) return null; - const ctx = canvasRef.current.getContext('2d'); - if (!ctx) return null; - return ctx; - }; - - // Helper for downloading the canvas (for debugging) - static downloadCanvas = (canvas: HTMLCanvasElement) => { - const url = canvas.toDataURL(); - const downloadLink = document.createElement('a'); - downloadLink.href = url; - downloadLink.download = 'canvas'; - - downloadLink.click(); - downloadLink.remove(); - }; - - // Download the canvas (for debugging) - static downloadImageCanvas = (imgUrl: string) => { - const img = new Image(); - img.src = imgUrl; - img.onload = () => { - const canvas = document.createElement('canvas'); - canvas.width = canvasSize; - canvas.height = canvasSize; - const ctx = canvas.getContext('2d'); - ctx?.drawImage(img, 0, 0, canvasSize, canvasSize); - - this.downloadCanvas(canvas); - }; - }; - - // Clears the canvas - static clearCanvas = (canvasRef: React.RefObject<HTMLCanvasElement>) => { - const ctx = this.getCanvasContext(canvasRef); - if (!ctx || !canvasRef.current) return; - ctx.clearRect(0, 0, canvasRef.current.width, canvasRef.current.height); - }; - - // Draws the image to the current canvas - static drawImgToCanvas = (img: HTMLImageElement, canvasRef: React.RefObject<HTMLCanvasElement>, width: number, height: number) => { - const drawImg = (htmlImg: HTMLImageElement) => { - const ctx = this.getCanvasContext(canvasRef); - if (!ctx) return; - ctx.globalCompositeOperation = 'source-over'; - ctx.clearRect(0, 0, width, height); - ctx.drawImage(htmlImg, 0, 0, width, height); - }; - - if (img.complete) { - drawImg(img); - } else { - img.onload = () => { - drawImg(img); - }; - } - }; - - // Gets the image mask for the openai endpoint - static getCanvasMask = (srcCanvas: HTMLCanvasElement, paddedCanvas: HTMLCanvasElement): HTMLCanvasElement | undefined => { - const canvas = document.createElement('canvas'); - canvas.width = canvasSize; - canvas.height = canvasSize; - const ctx = canvas.getContext('2d'); - if (!ctx) return undefined; - ctx?.clearRect(0, 0, canvasSize, canvasSize); - ctx.drawImage(paddedCanvas, 0, 0); - - // extract and set padding data - if (srcCanvas.height > srcCanvas.width) { - // horizontal padding, x offset - const xOffset = (canvasSize - srcCanvas.width) / 2; - ctx?.clearRect(xOffset, 0, srcCanvas.width, srcCanvas.height); - ctx.drawImage(srcCanvas, xOffset, 0, srcCanvas.width, srcCanvas.height); - } else { - // vertical padding, y offset - const yOffset = (canvasSize - srcCanvas.height) / 2; - ctx?.clearRect(0, yOffset, srcCanvas.width, srcCanvas.height); - ctx.drawImage(srcCanvas, 0, yOffset, srcCanvas.width, srcCanvas.height); - } - return canvas; - }; - - // Fills in the blank areas of the image with an image reflection (to fill in a square-shaped canvas) - static drawHorizontalReflection = (ctx: CanvasRenderingContext2D, canvas: HTMLCanvasElement, xOffset: number) => { - const imageData = ctx.getImageData(0, 0, canvas.width, canvas.height); - const { data } = imageData; - for (let i = 0; i < canvas.height; i++) { - for (let j = 0; j < xOffset; j++) { - const targetIdx = 4 * (i * canvas.width + j); - const sourceI = i; - const sourceJ = xOffset + (xOffset - j); - const sourceIdx = 4 * (sourceI * canvas.width + sourceJ); - data[targetIdx] = data[sourceIdx]; - data[targetIdx + 1] = data[sourceIdx + 1]; - data[targetIdx + 2] = data[sourceIdx + 2]; - } - } - for (let i = 0; i < canvas.height; i++) { - for (let j = canvas.width - 1; j >= canvas.width - 1 - xOffset; j--) { - const targetIdx = 4 * (i * canvas.width + j); - const sourceI = i; - const sourceJ = canvas.width - 1 - xOffset - (xOffset - (canvas.width - j)); - const sourceIdx = 4 * (sourceI * canvas.width + sourceJ); - data[targetIdx] = data[sourceIdx]; - data[targetIdx + 1] = data[sourceIdx + 1]; - data[targetIdx + 2] = data[sourceIdx + 2]; - } - } - ctx.putImageData(imageData, 0, 0); - }; - - // Fills in the blank areas of the image with an image reflection (to fill in a square-shaped canvas) - static drawVerticalReflection = (ctx: CanvasRenderingContext2D, canvas: HTMLCanvasElement, yOffset: number) => { - const imageData = ctx.getImageData(0, 0, canvas.width, canvas.height); - const { data } = imageData; - for (let j = 0; j < canvas.width; j++) { - for (let i = 0; i < yOffset; i++) { - const targetIdx = 4 * (i * canvas.width + j); - const sourceJ = j; - const sourceI = yOffset + (yOffset - i); - const sourceIdx = 4 * (sourceI * canvas.width + sourceJ); - data[targetIdx] = data[sourceIdx]; - data[targetIdx + 1] = data[sourceIdx + 1]; - data[targetIdx + 2] = data[sourceIdx + 2]; - } - } - for (let j = 0; j < canvas.width; j++) { - for (let i = canvas.height - 1; i >= canvas.height - 1 - yOffset; i--) { - const targetIdx = 4 * (i * canvas.width + j); - const sourceJ = j; - const sourceI = canvas.height - 1 - yOffset - (yOffset - (canvas.height - i)); - const sourceIdx = 4 * (sourceI * canvas.width + sourceJ); - data[targetIdx] = data[sourceIdx]; - data[targetIdx + 1] = data[sourceIdx + 1]; - data[targetIdx + 2] = data[sourceIdx + 2]; - } - } - ctx.putImageData(imageData, 0, 0); - }; - - // Gets the unaltered (besides filling in padding) version of the image for the api call - static getCanvasImg = (img: HTMLImageElement): HTMLCanvasElement | undefined => { - const canvas = document.createElement('canvas'); - canvas.width = canvasSize; - canvas.height = canvasSize; - const ctx = canvas.getContext('2d'); - if (!ctx) return undefined; - // fix scaling - const scale = Math.min(canvasSize / img.width, canvasSize / img.height); - const width = Math.floor(img.width * scale); - const height = Math.floor(img.height * scale); - ctx?.clearRect(0, 0, canvasSize, canvasSize); - ctx.fillStyle = bgColor; - ctx.fillRect(0, 0, canvasSize, canvasSize); - - // extract and set padding data - if (img.naturalHeight > img.naturalWidth) { - // horizontal padding, x offset - const xOffset = Math.floor((canvasSize - width) / 2); - ctx.drawImage(img, xOffset, 0, width, height); - - // draw reflected image padding - this.drawHorizontalReflection(ctx, canvas, xOffset); - } else { - // vertical padding, y offset - const yOffset = Math.floor((canvasSize - height) / 2); - ctx.drawImage(img, 0, yOffset, width, height); - - // draw reflected image padding - this.drawVerticalReflection(ctx, canvas, yOffset); - } - return canvas; - }; - - /** - * Converts a url to base64 (tainted canvas workaround) - */ - static urlToBase64 = async (imageUrl: string): Promise<string | undefined> => { - try { - const res = await fetch(imageUrl); - const blob = await res.blob(); - - return new Promise<string>((resolve, reject) => { - const reader = new FileReader(); - reader.onload = () => { - const base64Data = reader.result?.toString().split(',')[1]; - if (base64Data) { - resolve(base64Data); - } else { - reject(new Error('Failed to convert.')); - } - }; - reader.onerror = () => { - reject(new Error('Error reading image data')); - }; - reader.readAsDataURL(blob); - }); - } catch (err) { - console.error(err); - } - return undefined; - }; -} diff --git a/src/client/views/smartdraw/DrawingFillHandler.tsx b/src/client/views/smartdraw/DrawingFillHandler.tsx index 1a470f995..7a95e27c2 100644 --- a/src/client/views/smartdraw/DrawingFillHandler.tsx +++ b/src/client/views/smartdraw/DrawingFillHandler.tsx @@ -1,5 +1,6 @@ import { imageUrlToBase64 } from '../../../ClientUtils'; import { Doc } from '../../../fields/Doc'; +import { DocData } from '../../../fields/DocSymbols'; import { ImageCast } from '../../../fields/Types'; import { Upload } from '../../../server/SharedMediaTypes'; import { gptDescribeImage } from '../../apis/gpt/GPT'; @@ -7,19 +8,32 @@ import { Docs } from '../../documents/Documents'; import { Networking } from '../../Network'; import { DocumentView, DocumentViewInternal } from '../nodes/DocumentView'; import { OpenWhere } from '../nodes/OpenWhere'; +import { AspectRatioLimits, FireflyDimensionsMap, FireflyImageDimensions } from './FireflyConstants'; export class DrawingFillHandler { static drawingToImage = (drawing: Doc, strength: number, prompt: string) => DocumentView.GetDocImage(drawing)?.then(imageField => { if (imageField) { + const aspectRatio = (drawing.width as number) / (drawing.height as number); + let dims: { width: number; height: number }; + if (aspectRatio > AspectRatioLimits[FireflyImageDimensions.Widescreen]) { + dims = FireflyDimensionsMap[FireflyImageDimensions.Widescreen]; + } else if (aspectRatio > AspectRatioLimits[FireflyImageDimensions.Landscape]) { + dims = FireflyDimensionsMap[FireflyImageDimensions.Landscape]; + } else if (aspectRatio < AspectRatioLimits[FireflyImageDimensions.Portrait]) { + dims = FireflyDimensionsMap[FireflyImageDimensions.Portrait]; + } else { + dims = FireflyDimensionsMap[FireflyImageDimensions.Square]; + } const { href } = ImageCast(imageField).url; const hrefParts = href.split('.'); const structureUrl = `${hrefParts.slice(0, -1).join('.')}_o.${hrefParts.lastElement()}`; imageUrlToBase64(structureUrl) .then((hrefBase64: string) => gptDescribeImage(hrefBase64)) .then((prompt: string) => { - Networking.PostToServer('/queryFireflyImageFromStructure', { prompt, structureUrl, strength }).then((info: Upload.ImageInformation) => - DocumentViewInternal.addDocTabFunc(Docs.Create.ImageDocument(info.accessPaths.agnostic.client, {}), OpenWhere.addRight)) // prettier-ignore + Networking.PostToServer('/queryFireflyImageFromStructure', { prompt: prompt, width: dims.width, height: dims.height, structureUrl, strength }).then((info: Upload.ImageInformation) => + DocumentViewInternal.addDocTabFunc(Docs.Create.ImageDocument(info.accessPaths.agnostic.client, { ai_generated: true, nativeWidth: dims.width, nativeHeight: dims.height }), OpenWhere.addRight) + ); // prettier-ignore }); } return false; diff --git a/src/client/views/smartdraw/FireflyConstants.ts b/src/client/views/smartdraw/FireflyConstants.ts new file mode 100644 index 000000000..f51305fba --- /dev/null +++ b/src/client/views/smartdraw/FireflyConstants.ts @@ -0,0 +1,20 @@ +export enum FireflyImageDimensions { + Square = 'square', + Landscape = 'landscape', + Portrait = 'portrait', + Widescreen = 'widescreen', +} + +export const FireflyDimensionsMap = { + square: { width: 2048, height: 2048 }, + landscape: { width: 2304, height: 1792 }, + portrait: { width: 1792, height: 2304 }, + widescreen: { width: 2688, height: 1536 }, +}; + +export const AspectRatioLimits = { + square: 1, + landscape: 1.167, + portrait: 0.875, + widescreen: 1.472, +}; diff --git a/src/client/views/smartdraw/SmartDrawHandler.scss b/src/client/views/smartdraw/SmartDrawHandler.scss index 4b21c92a5..cca7d77c7 100644 --- a/src/client/views/smartdraw/SmartDrawHandler.scss +++ b/src/client/views/smartdraw/SmartDrawHandler.scss @@ -1,6 +1,5 @@ .smart-draw-handler { position: absolute; - width: 265px; .smart-draw-main { display: flex; @@ -16,9 +15,11 @@ display: flex; flex-direction: row; justify-content: center; + margin-top: 3px; } .smartdraw-options-container { + width: 265px; padding: 5px; font-weight: bolder; text-align: center; diff --git a/src/client/views/smartdraw/SmartDrawHandler.tsx b/src/client/views/smartdraw/SmartDrawHandler.tsx index 8cff2174f..6c9470480 100644 --- a/src/client/views/smartdraw/SmartDrawHandler.tsx +++ b/src/client/views/smartdraw/SmartDrawHandler.tsx @@ -25,6 +25,7 @@ import { ActiveInkArrowEnd, ActiveInkArrowStart, ActiveInkDash, ActiveInkFillCol import './SmartDrawHandler.scss'; import { Networking } from '../../Network'; import { OpenWhere } from '../nodes/OpenWhere'; +import { FireflyDimensionsMap, FireflyImageDimensions } from './FireflyConstants'; export interface DrawingOptions { text: string; @@ -35,13 +36,6 @@ export interface DrawingOptions { y: number; } -enum FireflyImageDimensions { - Square = 'square', - Landscape = 'landscape', - Portrait = 'portrait', - Widescreen = 'widescreen', -} - /** * The SmartDrawHandler allows users to generate drawings with GPT from text input. Users are able to enter * the item to draw, how complex they want the drawing to be, how large the drawing should be, and whether @@ -274,30 +268,12 @@ export class SmartDrawHandler extends ObservableReactComponent<object> { */ createImageWithFirefly = (input: string, seed?: number) => { this._lastInput.text = input; - let width = 0; - let height = 0; - switch (this._imgDims) { - case FireflyImageDimensions.Square: - width = 2048; - height = 2048; - break; - case FireflyImageDimensions.Landscape: - width = 2304; - height = 1792; - case FireflyImageDimensions.Portrait: - width = 1792; - height = 2304; - break; - case FireflyImageDimensions.Widescreen: - width = 2688; - height = 1536; - break; - } - return Networking.PostToServer('/queryFireflyImage', { prompt: input, width: width, height: height, seed: seed }).then(img => { + const dims = FireflyDimensionsMap[this._imgDims]; + return Networking.PostToServer('/queryFireflyImage', { prompt: input, width: dims.width, height: dims.height, seed: seed }).then(img => { const imgDoc: Doc = Docs.Create.ImageDocument(img.accessPaths.agnostic.client, { title: input.match(/^(.*?)~~~.*$/)?.[1] || input, - nativeWidth: width, - nativeHeight: height, + nativeWidth: dims.width, + nativeHeight: dims.height, ai_generated: true, firefly_seed: img.accessPaths.agnostic.client.match(/\/(\d+)upload/)[1], firefly_prompt: input, diff --git a/src/server/ApiManagers/FireflyManager.ts b/src/server/ApiManagers/FireflyManager.ts index a41492745..4c4aac5e0 100644 --- a/src/server/ApiManagers/FireflyManager.ts +++ b/src/server/ApiManagers/FireflyManager.ts @@ -20,7 +20,7 @@ export default class FireflyManager extends ApiManager { return undefined; }); - generateImageFromStructure = (prompt: string = 'a realistic illustration of a cat coding', structureUrl: string, strength: number) => + generateImageFromStructure = (prompt: string = 'a realistic illustration of a cat coding', width: number = 2048, height: number = 2048, structureUrl: string, strength: number = 50) => this.getBearerToken().then(response => response?.json().then((data: { access_token: string }) => fetch('https://firefly-api.adobe.io/v3/images/generate', { @@ -32,7 +32,8 @@ export default class FireflyManager extends ApiManager { ['Authorization', `Bearer ${data.access_token}`], ], body: JSON.stringify({ - prompt, + prompt: prompt, + size: { width: width, height: height }, structure: !structureUrl ? undefined : { @@ -226,7 +227,7 @@ export default class FireflyManager extends ApiManager { subscription: '/queryFireflyImageFromStructure', secureHandler: async ({ req, res }) => this.uploadImageToDropbox(req.body.structureUrl).then(uploadUrl => - this.generateImageFromStructure(req.body.prompt, uploadUrl, req.body.strength).then(fire => + this.generateImageFromStructure(req.body.prompt, req.body.width, req.body.height, uploadUrl, req.body.strength).then(fire => DashUploadUtils.UploadImage(JSON.parse(fire ?? '').url).then(info => { if (info instanceof Error) _invalid(res, info.message); else _success(res, info); |