import { Button, IconButton } from '@dash/components'; import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; import { Checkbox, FormControlLabel, Radio, RadioGroup, Slider, Switch } from '@mui/material'; import { action, makeObservable, observable, runInAction } from 'mobx'; import { observer } from 'mobx-react'; import React from 'react'; import { AiOutlineSend } from 'react-icons/ai'; import ReactLoading from 'react-loading'; import { INode, parse } from 'svgson'; import { imageUrlToBase64, setupMoveUpEvents } from '../../../ClientUtils'; import { unimplementedFunction } from '../../../Utils'; import { Doc, DocListCast } from '../../../fields/Doc'; import { InkData, InkField, InkTool } from '../../../fields/InkField'; import { List } from '../../../fields/List'; import { BoolCast, ImageCast, NumCast, StrCast } from '../../../fields/Types'; import { PointData } from '../../../pen-gestures/GestureTypes'; import { Upload } from '../../../server/SharedMediaTypes'; import { Networking } from '../../Network'; import { GPTCallType, gptAPICall, gptDrawingColor } from '../../apis/gpt/GPT'; import { DocumentType } from '../../documents/DocumentTypes'; import { Docs } from '../../documents/Documents'; import { SettingsManager } from '../../util/SettingsManager'; import { undoable } from '../../util/UndoManager'; import { SVGToBezier, SVGType } from '../../util/bezierFit'; import { InkingStroke } from '../InkingStroke'; import { ObservableReactComponent } from '../ObservableReactComponent'; import { MarqueeView } from '../collections/collectionFreeForm'; import { ActiveInkArrowEnd, ActiveInkArrowStart, ActiveInkBezierApprox, ActiveInkColor, ActiveInkDash, ActiveInkFillColor, ActiveInkWidth, ActiveIsInkMask, DocumentView } from '../nodes/DocumentView'; import { FireflyDimensionsMap, FireflyImageData, FireflyImageDimensions } from './FireflyConstants'; import './SmartDrawHandler.scss'; export interface DrawingOptions { text?: string; complexity?: number; size?: number; autoColor?: boolean; x?: number; y?: number; } type svgparsedData = [PointData[], string, string]; /** * The SmartDrawHandler allows users to generate drawings with GPT from text input. Users are able to enter * the item to draw, how complex they want the drawing to be, how large the drawing should be, and whether * it will be colored. If the drawing is colored, GPT will automatically define the stroke and fill of each * stroke. Drawings are retrieved from GPT as SVG code then converted into Dash-supported Beziers. * * The handler is selected from the ink tools menu. To generate a drawing, users can click anywhere on the freeform * canvas and a popup will appear that prompts them to create a drawing. Once the drawing is created, users have * the option to regenerate or edit the drawing. * * When each drawing is created, it is added to Dash as a group of ink strokes. The group is tagged with metadata * for user input, the drawing's SVG code, and its settings (size, complexity). In the context menu -> 'Options', * users can then show the drawing editor and regenerate/edit them at any point in the future. */ @observer export class SmartDrawHandler extends ObservableReactComponent { // eslint-disable-next-line no-use-before-define static Instance: SmartDrawHandler; private _lastInput: DrawingOptions = { text: '', complexity: 5, size: 350, autoColor: true, x: 0, y: 0 }; private _lastResponse: string = ''; private _selectedDocs: Doc[] = []; @observable private _display: boolean = false; @observable private _pageX: number = 0; @observable private _pageY: number = 0; @observable private _scale: number = 0; @observable private _yRelativeToTop: boolean = true; @observable private _isLoading: boolean = false; @observable private _userInput: string = ''; @observable private _regenInput: string = ''; @observable private _showOptions: boolean = false; @observable private _showEditBox: boolean = false; @observable private _complexity: number = 5; @observable private _size: number = 200; @observable private _autoColor: boolean = true; @observable private _imgDims: FireflyImageDimensions = FireflyImageDimensions.Square; @observable private _canInteract: boolean = true; @observable private _generateDrawing: boolean = true; @observable private _generateImage: boolean = true; @observable public ShowRegenerate: boolean = false; constructor(props: object) { super(props); makeObservable(this); SmartDrawHandler.Instance = this; } /** * AddDrawing and RemoveDrawing are defined by the other classes that call the smart draw functions (i.e. CollectionFreeForm, FormattedTextBox, StickerPalette) to define how a drawing document should be added or removed in their respective locations (to the freeform canvas, to the sticker palette's preview, etc.) */ public AddDrawing: (doc: Doc, opts: DrawingOptions, gptRes: string, x?: number, y?: number) => void = unimplementedFunction; public RemoveDrawing: (useLastContainer: boolean, doc?: Doc) => void = unimplementedFunction; /** * This creates the ink document that represents a drawing, so it goes through the strokes that make up the drawing, * creates ink documents for each stroke, then adds the strokes to a collection. This can also be redefined by other * classes to customize the way the drawing docs get created. For example, the freeform canvas has a different way of * defining document bounds, so CreateDrawingDoc is redefined when that class calls gpt draw functions. */ public static CreateDrawingDoc: (strokeList: [InkData, string, string][], opts: DrawingOptions, gptRes: string, containerDoc?: Doc) => Doc | undefined = (strokeList: [InkData, string, string][], opts: DrawingOptions) => { const drawing: Doc[] = []; strokeList.forEach((stroke: [InkData, string, string]) => { const bounds = InkField.getBounds(stroke[0]); const inkWidth = Math.min(5, ActiveInkWidth()); const inkDoc = Docs.Create.InkDocument( stroke[0], { title: 'stroke', x: bounds.left - inkWidth / 2, y: bounds.top - inkWidth / 2, _width: bounds.width + inkWidth, _height: bounds.height + inkWidth, stroke_showLabel: false}, // prettier-ignore inkWidth, opts.autoColor ? stroke[1] : ActiveInkColor(), ActiveInkBezierApprox(), stroke[2] === 'none' ? ActiveInkFillColor() : stroke[2], ActiveInkArrowStart(), ActiveInkArrowEnd(), ActiveInkDash(), ActiveIsInkMask() ); drawing.push(inkDoc); }); return MarqueeView.getCollection(drawing, undefined, true, { left: 1, top: 1, width: 1, height: 1 }); }; @action displaySmartDrawHandler = (x: number, y: number, scale: number) => { [this._pageX, this._pageY] = [x, y]; this._display = true; this._scale = scale; }; /** * This is called in two places: 1. In this class, where the regenerate popup shows as soon as a * drawing is created to replace the original smart draw popup. 2. From the context menu to make * the regenerate popup show by user command. */ @action displayRegenerate = (x: number, y: number) => { this._selectedDocs = [DocumentView.SelectedDocs()?.lastElement()]; [this._pageX, this._pageY] = [x, y]; this._display = false; this.ShowRegenerate = true; this._showEditBox = false; const docData = this._selectedDocs[0]; this._lastResponse = StrCast(docData.$drawingData); this._lastInput = { text: StrCast(docData.$ai_drawing_input), complexity: NumCast(docData.$ai_drawing_complexity), size: NumCast(docData.$ai_drawing_size), autoColor: BoolCast(docData.$ai_drawing_colored), x: this._pageX, y: this._pageY }; }; /** * Hides the smart draw handler and resets its fields to their default. */ @action hideSmartDrawHandler = () => { if (this._display) { this.ShowRegenerate = false; this._display = false; this._isLoading = false; this._showOptions = false; this._userInput = ''; this._complexity = 5; this._size = 350; this._autoColor = true; Doc.ActiveTool = InkTool.None; } }; /** * Hides the popup that allows users to regenerate a drawing and resets its corresponding fields. */ @action hideRegenerate = () => { if (!this._isLoading) { this.ShowRegenerate = false; this._isLoading = false; this._regenInput = ''; this._lastInput = { text: '', complexity: 5, size: 350, autoColor: true, x: 0, y: 0 }; } }; /** * This allows users to press the return/enter key to send input. */ handleKeyPress = (e: React.KeyboardEvent) => { if (e.key === 'Enter') { this.handleSendClick(this._pageX, this._pageY); } }; /** * This is called when a user hits "send" on the draw with GPT popup. It calls the drawWithGPT or regenerate * functions depending on what mode is currently displayed, then sets various observable fields that facilitate * what the user sees. */ @action handleSendClick = async (X: number, Y: number) => { if ((!this.ShowRegenerate && this._userInput == '') || (!this._generateImage && !this._generateDrawing)) return; this._isLoading = true; this._canInteract = false; if (this.ShowRegenerate) { await this.regenerate(this._selectedDocs, undefined, undefined, this._regenInput).then(action(() => (this._showEditBox = false))); } else { this._showOptions = false; try { if (this._generateImage) { await this.createImageWithFirefly(this._userInput); } if (this._generateDrawing) { await this.drawWithGPT({ X, Y }, this._userInput, this._complexity, this._size, this._autoColor); } this.hideSmartDrawHandler(); } catch (err) { console.error('GPT call failed', err); } } runInAction(() => { this._isLoading = false; this._canInteract = true; }); }; /** * Calls GPT API to create a drawing based on user input. */ drawWithGPT = async (screenPt: { X: number; Y: number }, input: string, complexity: number, size: number, autoColor: boolean) => { if (input) { this._lastInput = { text: input, complexity: complexity, size: size, autoColor: autoColor, x: screenPt.X, y: screenPt.Y }; const res = await gptAPICall(`"${input}", "${complexity}", "${size}"`, GPTCallType.DRAW, undefined, true); if (res) { const strokeData = await this.parseSvg(res, { X: 0, Y: 0 }, false, autoColor); const drawingDoc = strokeData && SmartDrawHandler.CreateDrawingDoc(strokeData.data, strokeData.lastInput, strokeData.lastRes); drawingDoc && this.AddDrawing(drawingDoc, this._lastInput, res, screenPt.X, screenPt.Y); drawingDoc && this._selectedDocs.push(drawingDoc); return strokeData; } else { console.error('GPT call failed'); } } return undefined; }; /** * Calls Firefly API to create an image based on user input */ createImageWithFirefly = (input: string, seed?: number): Promise => { this._lastInput.text = input; return SmartDrawHandler.CreateWithFirefly(input, this._imgDims, seed).then(doc => { doc instanceof Doc && this.AddDrawing(doc, this._lastInput, input, this._pageX, this._pageY); return doc; }); }; /** * Calls Firefly API to create an image based on user input */ recreateImageWithFirefly = (input: string, seed?: number): Promise => { this._lastInput.text = input; return SmartDrawHandler.ReCreateWithFirefly(input, this._imgDims, seed); }; public static ReCreateWithFirefly(input: string, imgDims: FireflyImageDimensions, seed?: number): Promise { const dims = FireflyDimensionsMap[imgDims]; return Networking.PostToServer('/queryFireflyImage', { prompt: input, width: dims.width, height: dims.height, seed }) .then(res => { const img = res as Upload.FileInformation; const error = res as { error: string }; if ('error' in error) { alert('recreate image failed: ' + error.error); return undefined; } return { prompt: input, seed, pathname: img.accessPaths.agnostic.client }; }) .catch(e => { alert('recreate image failed: ' + e.toString()); return undefined; }); } public static CreateWithFirefly(input: string, imgDims: FireflyImageDimensions, seed?: number): Promise { const dims = FireflyDimensionsMap[imgDims]; return Networking.PostToServer('/queryFireflyImage', { prompt: input, width: dims.width, height: dims.height, seed }) .then(res => { const img = res as Upload.FileInformation; const error = res as { error: string }; if ('error' in error) { alert('create image failed: ' + error.error); return undefined; } const newseed = img.accessPaths.agnostic.client.match(/\/(\d+)upload/)?.[1]; return Docs.Create.ImageDocument(img.accessPaths.agnostic.client, { title: input, nativeWidth: dims.width, nativeHeight: dims.height, tags: new List(['@ai']), _width: Math.min(400, dims.width), _height: (Math.min(400, dims.width) * dims.height) / dims.width, ai: 'firefly', ai_firefly_seed: +(newseed ?? 0), ai_firefly_prompt: input, }); }) .catch(e => { alert('create image failed: ' + e.toString()); return undefined; }); } /** * Regenerates drawings with the option to add a specific regenerate prompt/request. * @param doc the drawing Docs to regenerate */ @action regenerate = (drawingDocs: Doc[], lastInput?: DrawingOptions, lastResponse?: string, regenInput?: string, changeInPlace?: boolean) => { if (lastInput) this._lastInput = lastInput; if (lastResponse) this._lastResponse = lastResponse; if (regenInput) this._regenInput = regenInput; return Promise.all( drawingDocs.map(async doc => { switch (doc.type) { case DocumentType.IMG: { const func = changeInPlace ? this.recreateImageWithFirefly : this.createImageWithFirefly; const newPrompt = doc.ai_firefly_prompt ? `${doc.ai_firefly_prompt} ~~~ ${this._regenInput}` : this._regenInput; return this._regenInput ? func(newPrompt, NumCast(doc?.ai_firefly_seed)) : func(this._lastInput.text || StrCast(doc.ai_firefly_prompt)); } case DocumentType.COL: { try { const res = await (async () => { if (this._regenInput) { const prompt = `This is your previously generated svg code: ${this._lastResponse} for the user input "${this._lastInput.text}". Please regenerate it with the provided specifications.`; this._lastInput.text = `${this._lastInput.text} ~~~ ${this._regenInput}`; return gptAPICall(`"${this._regenInput}"`, GPTCallType.DRAW, prompt, true); } return gptAPICall(`"${this._lastInput.text}", "${this._lastInput.complexity}", "${this._lastInput.size}"`, GPTCallType.DRAW, undefined, true); })(); if (res) { const strokeData = await this.parseSvg(res, { X: this._lastInput.x ?? 0, Y: this._lastInput.y ?? 0 }, true, lastInput?.autoColor || this._autoColor); this.RemoveDrawing !== unimplementedFunction && this.RemoveDrawing(true, doc); const drawingDoc = strokeData && SmartDrawHandler.CreateDrawingDoc(strokeData.data, strokeData.lastInput, strokeData.lastRes); drawingDoc && this.AddDrawing(drawingDoc, this._lastInput, res); } else { console.error('GPT call failed'); } } catch (err) { console.error('Error regenerating drawing', err); } break; } } }) ); }; /** * Parses the svg code that GPT returns into Bezier curves, with coordinates and colors. */ parseSvg = async (res: string, startPoint: { X: number; Y: number }, regenerate: boolean, autoColor: boolean) => { const svg = res.match(/]*>([\s\S]*?)<\/svg>/g); if (svg) { this._lastResponse = svg[0]; const svgObject = await parse(svg[0]); console.log(res, svgObject); const svgStrokes: INode[] = svgObject.children; const strokeData: [InkData, string, string][] = []; const tl = { X: Number.MAX_SAFE_INTEGER, Y: Number.MAX_SAFE_INTEGER }; let last: PointData = { X: 0, Y: 0 }; svgStrokes.forEach(child => { const convertedBezier: InkData = SVGToBezier(child.name as SVGType, child.attributes, last); last = convertedBezier.lastElement(); strokeData.push([ convertedBezier.map(point => { if (point.X < tl.X) tl.X = point.X; if (point.Y < tl.Y) tl.Y = point.Y; return { X: point.X, Y: point.Y }; }), (regenerate ? this._lastInput.autoColor : autoColor) ? child.attributes.stroke : '', (regenerate ? this._lastInput.autoColor : autoColor) ? child.attributes.fill : '', ]); }); const mapStroke = (pd: PointData): PointData => ({ X: startPoint.X + (pd.X - tl.X) * this._scale, Y: startPoint.Y + (pd.Y - tl.Y) * this._scale }); return { data: strokeData.map(sdata => [sdata[0].map(mapStroke), sdata[1], sdata[2]] as svgparsedData), lastInput: this._lastInput, lastRes: svg[0], }; } }; /** * Sends request to GPT API to recolor a selected ink document or group of ink documents. */ colorWithGPT = async (drawing: Doc) => { const img = await DocumentView.GetDocImage(drawing); const { href } = ImageCast(img).url; const hrefParts = href.split('.'); const hrefComplete = `${hrefParts[0]}_o.${hrefParts[1]}`; try { const hrefBase64 = await imageUrlToBase64(hrefComplete); const strokes = DocListCast(drawing.$data); const coords: string[] = []; strokes.forEach((stroke, i) => { const inkingStroke = DocumentView.getDocumentView(stroke)?.ComponentView as InkingStroke; const { inkData } = inkingStroke.inkScaledData(); coords.push(`${i + 1}. ${inkData.filter((point, index) => index % 4 === 0 || index == inkData.length - 1).map(point => `(${point.X.toString()}, ${point.Y.toString()})`)}`); }); const colorResponse = await gptDrawingColor(hrefBase64, coords).then(response => gptAPICall(response, GPTCallType.COLOR, undefined)); this.colorStrokes(colorResponse, drawing); } catch (error) { console.log('GPT call failed', error); } }; /** * Function that parses the GPT color response and sets the selected stroke(s) to the new color. */ colorStrokes = undoable((res: string, drawing: Doc) => { const colorList = res.match(/\{.*?\}/g); const strokes = DocListCast(drawing.$data); colorList?.forEach((colors, index) => { const strokeAndFill = colors.match(/#[0-9A-Fa-f]{6}/g); if (strokeAndFill && strokeAndFill.length == 2) { strokes[index].$color = strokeAndFill[0]; const inkStroke = DocumentView.getDocumentView(strokes[index])?.ComponentView as InkingStroke; const { inkData } = inkStroke.inkScaledData(); InkingStroke.IsClosed(inkData) ? (strokes[index].$fillColor = strokeAndFill[1]) : (strokes[index].$fillColor = undefined); } }); }, 'color strokes'); renderGenerateOutputOptions = () => (
Generate Ink this._canInteract && (this._generateDrawing = !this._generateDrawing)} />
Generate Image this._canInteract && (this._generateImage = !this._generateImage))} />
); renderGenerateDrawing = () => (
Drawing Options
Auto color this._canInteract && (this._autoColor = !this._autoColor))} />
Complexity this._canInteract && (this._complexity = val as number))} valueLabelDisplay="auto" />
Size (in pixels) this._canInteract && (this._size = val as number))} valueLabelDisplay="auto" />
); renderGenerateImage = () => (
Image Options
{Object.values(FireflyImageDimensions).map(dim => ( } onChange={() => this._canInteract && (this._imgDims = dim)} label={dim} /> ))}
); renderDisplay = () => { return (
setupMoveUpEvents( this, e, action(me => { this._pageX = this._pageX + me.movementX; this._pageY = this._pageY + me.movementY; return false; }), () => {}, () => {} ) } style={{ display: this._display ? '' : 'none', left: this._pageX, ...(this._yRelativeToTop ? { top: Math.max(0, this._pageY) } : { bottom: this._pageY }), background: SettingsManager.userBackgroundColor, color: SettingsManager.userColor, }}>
{ this.hideSmartDrawHandler(); this.hideRegenerate(); }} icon={} color={SettingsManager.userColor} /> this._canInteract && (this._userInput = e.target.value))} placeholder="Enter item to draw" onKeyDown={this.handleKeyPress} /> } color={SettingsManager.userColor} onClick={action(() => (this._showOptions = !this._showOptions))} />
{this._showOptions && (
{this.renderGenerateOutputOptions()} {this._generateDrawing ? this.renderGenerateDrawing() : null} {this._generateImage ? this.renderGenerateImage() : null}
)}
); }; renderRegenerateEditBox = () => (
this._canInteract && (this._regenInput = e.target.value))} onKeyDown={this.handleKeyPress} placeholder="Edit instructions" onPointerDown={e => e.stopPropagation()} />
); startDragging = (e: PointerEvent) => { setupMoveUpEvents( this, e, action(me => { this._pageX = this._pageX + me.movementX; this._pageY = this._pageY + me.movementY; return false; }), () => {}, () => {} ); }; renderRegenerate = () => (
setupMoveUpEvents( this, e, action(me => { this._pageX = this._pageX + me.movementX; this._pageY = this._pageY + me.movementY; return false; }), () => {}, () => {} ) } style={{ padding: 10, left: this._pageX, ...(this._yRelativeToTop ? { top: Math.max(0, this._pageY) } : { bottom: this._pageY }), background: SettingsManager.userBackgroundColor, color: SettingsManager.userColor, }}>
: } color={SettingsManager.userColor} onClick={() => this.handleSendClick(this._pageX, this._pageY)} /> } color={SettingsManager.userColor} onClick={action(() => (this._showEditBox = !this._showEditBox))} /> {this._showEditBox ? this.renderRegenerateEditBox() : null}
); render() { return this._display ? this.renderDisplay() // : this.ShowRegenerate ? this.renderRegenerate() : null; } }