import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; import { action, makeObservable, observable } from 'mobx'; import { observer } from 'mobx-react'; import React from 'react'; import { SettingsManager } from '../../util/SettingsManager'; import { ObservableReactComponent } from '../ObservableReactComponent'; import { Button, IconButton } from 'browndash-components'; import ReactLoading from 'react-loading'; import { AiOutlineSend } from 'react-icons/ai'; import { gptAPICall, GPTCallType, gptDrawingColor } from '../../apis/gpt/GPT'; import { InkData, InkField, InkTool } from '../../../fields/InkField'; import { SVGToBezier } from '../../util/bezierFit'; import { INode, parse } from 'svgson'; import { Slider, Switch } from '@mui/material'; import { Doc, DocListCast } from '../../../fields/Doc'; import { DocData } from '../../../fields/DocSymbols'; import { ActiveArrowEnd, ActiveArrowStart, ActiveDash, ActiveFillColor, ActiveInkBezierApprox, ActiveInkColor, ActiveInkWidth, ActiveIsInkMask, DocumentView } from '../nodes/DocumentView'; import { BoolCast, ImageCast, NumCast, StrCast } from '../../../fields/Types'; import './SmartDrawHandler.scss'; import { unimplementedFunction } from '../../../Utils'; import { Docs } from '../../documents/Documents'; import { MarqueeView } from '../collections/collectionFreeForm'; import { ImageField, URLField } from '../../../fields/URLField'; import { CollectionCardView } from '../collections/CollectionCardDeckView'; import { InkingStroke } from '../InkingStroke'; import { undoBatch } from '../../util/UndoManager'; export interface DrawingOptions { text: string; complexity: number; size: number; autoColor: boolean; x: number; y: number; } @observer // eslint-disable-next-line @typescript-eslint/no-empty-object-type export class SmartDrawHandler extends ObservableReactComponent<{}> { static Instance: SmartDrawHandler; @observable private _display: boolean = false; @observable private _pageX: number = 0; @observable private _pageY: number = 0; @observable private _yRelativeToTop: boolean = true; @observable private _isLoading: boolean = false; @observable private _userInput: string = ''; @observable private _showOptions: boolean = false; @observable private _showEditBox: boolean = false; @observable public _showRegenerate: boolean = false; @observable private _complexity: number = 5; @observable private _size: number = 200; @observable private _autoColor: boolean = true; @observable private _regenInput: string = ''; @observable private _canInteract: boolean = true; private _lastInput: DrawingOptions = { text: '', complexity: 5, size: 350, autoColor: true, x: 0, y: 0 }; private _lastResponse: string = ''; private _selectedDoc: Doc | undefined = undefined; private _errorOccurredOnce = false; public RemoveDrawing: (doc?: Doc) => void = unimplementedFunction; public CreateDrawingDoc: (strokeList: [InkData, string, string][], opts: DrawingOptions, gptRes: string, containerDoc?: Doc) => Doc | undefined = (strokeList: [InkData, string, string][], opts: DrawingOptions) => { const drawing: Doc[] = []; strokeList.forEach((stroke: [InkData, string, string]) => { const bounds = InkField.getBounds(stroke[0]); const inkWidth = Math.min(5, ActiveInkWidth()); const inkDoc = Docs.Create.InkDocument( stroke[0], { title: 'stroke', x: bounds.left - inkWidth / 2, y: bounds.top - inkWidth / 2, _width: bounds.width + inkWidth, _height: bounds.height + inkWidth, stroke_showLabel: BoolCast(Doc.UserDoc().activeInkHideTextLabels)}, // prettier-ignore inkWidth, opts.autoColor ? stroke[1] : ActiveInkColor(), ActiveInkBezierApprox(), stroke[2] === 'none' ? ActiveFillColor() : stroke[2], ActiveArrowStart(), ActiveArrowEnd(), ActiveDash(), ActiveIsInkMask() ); drawing.push(inkDoc); }); const collection = MarqueeView.getCollection(drawing, undefined, true, { left: 1, top: 1, width: 1, height: 1 }); return collection; }; public AddDrawing: (doc: Doc, opts: DrawingOptions, gptRes: string) => void = unimplementedFunction; // eslint-disable-next-line @typescript-eslint/no-explicit-any constructor(props: any) { super(props); makeObservable(this); SmartDrawHandler.Instance = this; } @action setUserInput = (input: string) => { if (this._canInteract) this._userInput = input; }; @action setRegenInput = (input: string) => { if (this._canInteract) this._regenInput = input; }; @action setShowOptions = () => { this._showOptions = !this._showOptions; }; @action setComplexity = (val: number) => { if (this._canInteract) this._complexity = val; }; @action setSize = (val: number) => { if (this._canInteract) this._size = val; }; @action setAutoColor = () => { if (this._canInteract) this._autoColor = !this._autoColor; }; @action setEdit = () => { this._showEditBox = !this._showEditBox; }; @action displaySmartDrawHandler = (x: number, y: number) => { this._pageX = x; this._pageY = y; this._display = true; }; @action displayRegenerate = (x: number, y: number) => { this._selectedDoc = DocumentView.SelectedDocs()?.lastElement(); this._pageX = x; this._pageY = y; this._display = false; this._showRegenerate = true; this._showEditBox = false; const docData = this._selectedDoc[DocData]; this._lastResponse = StrCast(docData.drawingData); this._lastInput = { text: StrCast(docData.drawingInput), complexity: NumCast(docData.drawingComplexity), size: NumCast(docData.drawingSize), autoColor: BoolCast(docData.drawingColored), x: this._pageX, y: this._pageY }; }; @action hideSmartDrawHandler = () => { this._showRegenerate = false; this._display = false; this._isLoading = false; this._showOptions = false; this._userInput = ''; this._complexity = 5; this._size = 350; this._autoColor = true; Doc.ActiveTool = InkTool.None; }; @action hideRegenerate = () => { if (!this._isLoading) { this._showRegenerate = false; this._isLoading = false; this._regenInput = ''; this._lastInput = { text: '', complexity: 5, size: 350, autoColor: true, x: 0, y: 0 }; } }; @action handleKeyPress = async (event: React.KeyboardEvent) => { if (event.key === 'Enter') { await this.handleSendClick(); } }; @action handleSendClick = async () => { this._isLoading = true; this._canInteract = false; if (this._showRegenerate) { await this.regenerate(); this._regenInput = ''; this._showEditBox = false; } else { this._showOptions = false; try { await this.drawWithGPT({ X: this._pageX, Y: this._pageY }, this._userInput, this._complexity, this._size, this._autoColor); this.hideSmartDrawHandler(); this._showRegenerate = true; } catch (err) { if (this._errorOccurredOnce) { console.error('GPT call failed', err); this._errorOccurredOnce = false; } else { this._errorOccurredOnce = true; await this.drawWithGPT({ X: this._pageX, Y: this._pageY }, this._userInput, this._complexity, this._size, this._autoColor); } } } this._isLoading = false; this._canInteract = true; }; /** * Calls GPT API to create a drawing based on user input */ @action drawWithGPT = async (startPt: { X: number; Y: number }, input: string, complexity: number, size: number, autoColor: boolean) => { if (input === '') return; this._lastInput = { text: input, complexity: complexity, size: size, autoColor: autoColor, x: startPt.X, y: startPt.Y }; const res = await gptAPICall(`"${input}", "${complexity}", "${size}"`, GPTCallType.DRAW, undefined, true); if (!res) { console.error('GPT call failed'); return; } console.log(res); const strokeData = await this.parseSvg(res, startPt, false, autoColor); const drawingDoc = strokeData && this.CreateDrawingDoc(strokeData?.data, strokeData?.lastInput, strokeData?.lastRes); // eslint-disable-next-line @typescript-eslint/no-unused-expressions drawingDoc && this.AddDrawing(drawingDoc, this._lastInput, res); this._errorOccurredOnce = false; return strokeData; }; /** * Regenerates drawings with the option to add a specific regenerate prompt/request. */ @action regenerate = async (lastInput?: DrawingOptions, lastResponse?: string, regenInput?: string) => { if (lastInput) this._lastInput = lastInput; if (lastResponse) this._lastResponse = lastResponse; if (regenInput) this._regenInput = regenInput; try { let res; if (this._regenInput !== '') { const prompt: string = `This is your previously generated svg code: ${this._lastResponse} for the user input "${this._lastInput.text}". Please regenerate it with the provided specifications.`; res = await gptAPICall(`"${this._regenInput}"`, GPTCallType.DRAW, prompt, true); this._lastInput.text = `${this._lastInput.text} ~~~ ${this._regenInput}`; } else { res = await gptAPICall(`"${this._lastInput.text}", "${this._lastInput.complexity}", "${this._lastInput.size}"`, GPTCallType.DRAW, undefined, true); } if (!res) { console.error('GPT call failed'); return; } console.log(res); const strokeData = await this.parseSvg(res, { X: this._lastInput.x, Y: this._lastInput.y }, true, lastInput?.autoColor || this._autoColor); // eslint-disable-next-line @typescript-eslint/no-unused-expressions this.RemoveDrawing !== unimplementedFunction && this.RemoveDrawing(this._selectedDoc); const drawingDoc = strokeData && this.CreateDrawingDoc(strokeData?.data, strokeData?.lastInput, strokeData?.lastRes); // eslint-disable-next-line @typescript-eslint/no-unused-expressions drawingDoc && this.AddDrawing(drawingDoc, this._lastInput, res); return strokeData; } catch (err) { console.error('Error regenerating drawing', err); } }; /** * Parses the svg code that GPT returns into Bezier curves. */ @action parseSvg = async (res: string, startPoint: { X: number; Y: number }, regenerate: boolean, autoColor: boolean) => { const svg = res.match(/]*>([\s\S]*?)<\/svg>/g); if (svg) { this._lastResponse = svg[0]; const svgObject = await parse(svg[0]); const svgStrokes: INode[] = svgObject.children; const strokeData: [InkData, string, string][] = []; // eslint-disable-next-line @typescript-eslint/no-explicit-any svgStrokes.forEach((child: any) => { const convertedBezier: InkData = SVGToBezier(child.name, child.attributes); strokeData.push([ convertedBezier.map(point => { return { X: point.X + startPoint.X - this._size / 1.5, Y: point.Y + startPoint.Y - this._size / 2 }; }), (regenerate ? this._lastInput.autoColor : autoColor) ? child.attributes.stroke : undefined, (regenerate ? this._lastInput.autoColor : autoColor) ? child.attributes.fill : undefined, ]); }); return { data: strokeData, lastInput: this._lastInput, lastRes: svg[0] }; } }; /** * Sends request to GPT API to recolor a selected ink document or group of ink documents. */ colorWithGPT = async (drawing: Doc) => { const img = await this.getIcon(drawing); const { href } = (img as URLField).url; const hrefParts = href.split('.'); const hrefComplete = `${hrefParts[0]}_o.${hrefParts[1]}`; try { const hrefBase64 = await CollectionCardView.imageUrlToBase64(hrefComplete); const strokes = DocListCast(drawing[DocData].data); const coords: string[] = []; strokes.forEach((stroke, i) => { const inkingStroke = DocumentView.getDocumentView(stroke)?.ComponentView as InkingStroke; const { inkData } = inkingStroke.inkScaledData(); coords.push( `${i + 1}. ${inkData .filter((point, index) => { return index % 4 === 0 || index == inkData.length - 1; }) .map(point => { return `(${point.X.toString()}, ${point.Y.toString()})`; })}` ); }); const response = await gptDrawingColor(hrefBase64, coords); console.log(response); const colorResponse = await gptAPICall(response, GPTCallType.COLOR, undefined); console.log(colorResponse); this.colorStrokes(colorResponse, drawing); } catch (error) { console.log('GPT call failed', error); } }; /** * Function that parses the GPT color response and sets the selected stroke(s) to the new color. */ @undoBatch colorStrokes = (res: string, drawing: Doc) => { const colorList = res.match(/\{.*?\}/g); const strokes = DocListCast(drawing[DocData].data); colorList?.forEach((colors, index) => { const strokeAndFill = colors.match(/#[0-9A-Fa-f]{6}/g); if (strokeAndFill && strokeAndFill.length == 2) { strokes[index][DocData].color = strokeAndFill[0]; const inkStroke = DocumentView.getDocumentView(strokes[index])?.ComponentView as InkingStroke; const { inkData } = inkStroke.inkScaledData(); // eslint-disable-next-line @typescript-eslint/no-unused-expressions InkingStroke.IsClosed(inkData) ? (strokes[index][DocData].fillColor = strokeAndFill[1]) : (strokes[index][DocData].fillColor = undefined); } }); }; /** * Gets an image snapshot of a doc. In this class, it's used to snapshot a selected ink stroke/group to use for GPT color. */ async getIcon(doc: Doc) { const docView = DocumentView.getDocumentView(doc); console.log(doc); if (docView) { console.log(docView); docView.ComponentView?.updateIcon?.(); return new Promise(res => setTimeout(() => res(ImageCast(docView.Document.icon)), 1000)); } return undefined; } render() { if (this._display) { return (
{ this.hideSmartDrawHandler(); this.hideRegenerate(); }} icon={} color={SettingsManager.userColor} style={{ width: '19px' }} /> { this.setUserInput(e.target.value); }} placeholder="Enter item to draw" onKeyDown={this.handleKeyPress} /> } color={SettingsManager.userColor} style={{ width: '14px' }} onClick={this.setShowOptions} />
{this._showOptions && ( <>
Auto color
Complexity { this.setComplexity(val as number); }} valueLabelDisplay="auto" />
Size (in pixels) { this.setSize(val as number); }} valueLabelDisplay="auto" />
)}
); } else if (this._showRegenerate) { return (
: } color={SettingsManager.userColor} onClick={this.handleSendClick} /> } color={SettingsManager.userColor} onClick={this.setEdit} /> {this._showEditBox && (
{ this.setRegenInput(e.target.value); }} onKeyDown={this.handleKeyPress} placeholder="Edit instructions" />
)}
); } else { return <>; } } }