diff options
Diffstat (limited to 'src/client/views/smartdraw/SmartDrawHandler.tsx')
| -rw-r--r-- | src/client/views/smartdraw/SmartDrawHandler.tsx | 444 |
1 files changed, 444 insertions, 0 deletions
diff --git a/src/client/views/smartdraw/SmartDrawHandler.tsx b/src/client/views/smartdraw/SmartDrawHandler.tsx new file mode 100644 index 000000000..8271c0959 --- /dev/null +++ b/src/client/views/smartdraw/SmartDrawHandler.tsx @@ -0,0 +1,444 @@ +import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; +import { Slider, Switch } from '@mui/material'; +import { Button, IconButton } from 'browndash-components'; +import { action, makeObservable, observable } from 'mobx'; +import { observer } from 'mobx-react'; +import React from 'react'; +import { AiOutlineSend } from 'react-icons/ai'; +import ReactLoading from 'react-loading'; +import { INode, parse } from 'svgson'; +import { unimplementedFunction } from '../../../Utils'; +import { Doc, DocListCast } from '../../../fields/Doc'; +import { DocData } from '../../../fields/DocSymbols'; +import { InkData, InkField, InkTool } from '../../../fields/InkField'; +import { BoolCast, ImageCast, NumCast, StrCast } from '../../../fields/Types'; +import { ImageField } from '../../../fields/URLField'; +import { GPTCallType, gptAPICall, gptDrawingColor } from '../../apis/gpt/GPT'; +import { Docs } from '../../documents/Documents'; +import { SettingsManager } from '../../util/SettingsManager'; +import { undoable } from '../../util/UndoManager'; +import { SVGToBezier, SVGType } from '../../util/bezierFit'; +import { InkingStroke } from '../InkingStroke'; +import { ObservableReactComponent } from '../ObservableReactComponent'; +import { CollectionCardView } from '../collections/CollectionCardDeckView'; +import { MarqueeView } from '../collections/collectionFreeForm'; +import { ActiveArrowEnd, ActiveArrowStart, ActiveDash, ActiveFillColor, ActiveInkBezierApprox, ActiveInkColor, ActiveInkWidth, ActiveIsInkMask, DocumentView } from '../nodes/DocumentView'; +import './SmartDrawHandler.scss'; + +export interface DrawingOptions { + text: string; + complexity: number; + size: number; + autoColor: boolean; + x: number; + y: number; +} + +@observer +export class SmartDrawHandler extends ObservableReactComponent<object> { + static Instance: SmartDrawHandler; + + private _lastInput: DrawingOptions = { text: '', complexity: 5, size: 350, autoColor: true, x: 0, y: 0 }; + private _lastResponse: string = ''; + private _selectedDoc: Doc | undefined = undefined; + private _errorOccurredOnce = false; + + @observable private _display: boolean = false; + @observable private _pageX: number = 0; + @observable private _pageY: number = 0; + @observable private _yRelativeToTop: boolean = true; + @observable private _isLoading: boolean = false; + @observable private _userInput: string = ''; + @observable private _showOptions: boolean = false; + @observable private _showEditBox: boolean = false; + @observable private _complexity: number = 5; + @observable private _size: number = 200; + @observable private _autoColor: boolean = true; + @observable private _regenInput: string = ''; + @observable private _canInteract: boolean = true; + + @observable public ShowRegenerate: boolean = false; + + constructor(props: object) { + super(props); + makeObservable(this); + SmartDrawHandler.Instance = this; + } + + public AddDrawing: (doc: Doc, opts: DrawingOptions, gptRes: string) => void = unimplementedFunction; + public RemoveDrawing: (doc?: Doc) => void = unimplementedFunction; + public CreateDrawingDoc: (strokeList: [InkData, string, string][], opts: DrawingOptions, gptRes: string, containerDoc?: Doc) => Doc | undefined = (strokeList: [InkData, string, string][], opts: DrawingOptions) => { + const drawing: Doc[] = []; + strokeList.forEach((stroke: [InkData, string, string]) => { + const bounds = InkField.getBounds(stroke[0]); + const inkWidth = Math.min(5, ActiveInkWidth()); + const inkDoc = Docs.Create.InkDocument( + stroke[0], + { title: 'stroke', + x: bounds.left - inkWidth / 2, + y: bounds.top - inkWidth / 2, + _width: bounds.width + inkWidth, + _height: bounds.height + inkWidth, + stroke_showLabel: BoolCast(Doc.UserDoc().activeInkHideTextLabels)}, // prettier-ignore + inkWidth, + opts.autoColor ? stroke[1] : ActiveInkColor(), + ActiveInkBezierApprox(), + stroke[2] === 'none' ? ActiveFillColor() : stroke[2], + ActiveArrowStart(), + ActiveArrowEnd(), + ActiveDash(), + ActiveIsInkMask() + ); + drawing.push(inkDoc); + }); + + return MarqueeView.getCollection(drawing, undefined, true, { left: 1, top: 1, width: 1, height: 1 }); + }; + + @action + displaySmartDrawHandler = (x: number, y: number) => { + [this._pageX, this._pageY] = [x, y]; + this._display = true; + }; + + @action + displayRegenerate = (x: number, y: number) => { + this._selectedDoc = DocumentView.SelectedDocs()?.lastElement(); + [this._pageX, this._pageY] = [x, y]; + this._display = false; + this.ShowRegenerate = true; + this._showEditBox = false; + const docData = this._selectedDoc[DocData]; + this._lastResponse = StrCast(docData.drawingData); + this._lastInput = { text: StrCast(docData.drawingInput), complexity: NumCast(docData.drawingComplexity), size: NumCast(docData.drawingSize), autoColor: BoolCast(docData.drawingColored), x: this._pageX, y: this._pageY }; + }; + + @action + hideSmartDrawHandler = () => { + this.ShowRegenerate = false; + this._display = false; + this._isLoading = false; + this._showOptions = false; + this._userInput = ''; + this._complexity = 5; + this._size = 350; + this._autoColor = true; + Doc.ActiveTool = InkTool.None; + }; + + @action + hideRegenerate = () => { + if (!this._isLoading) { + this.ShowRegenerate = false; + this._isLoading = false; + this._regenInput = ''; + this._lastInput = { text: '', complexity: 5, size: 350, autoColor: true, x: 0, y: 0 }; + } + }; + + handleKeyPress = (event: React.KeyboardEvent) => { + if (event.key === 'Enter') { + this.handleSendClick(); + } + }; + + @action + handleSendClick = async () => { + this._isLoading = true; + this._canInteract = false; + if (this.ShowRegenerate) { + await this.regenerate(); + this._regenInput = ''; + this._showEditBox = false; + } else { + this._showOptions = false; + try { + await this.drawWithGPT({ X: this._pageX, Y: this._pageY }, this._userInput, this._complexity, this._size, this._autoColor); + this.hideSmartDrawHandler(); + this.ShowRegenerate = true; + } catch (err) { + if (this._errorOccurredOnce) { + console.error('GPT call failed', err); + this._errorOccurredOnce = false; + } else { + this._errorOccurredOnce = true; + await this.drawWithGPT({ X: this._pageX, Y: this._pageY }, this._userInput, this._complexity, this._size, this._autoColor); + } + } + } + this._isLoading = false; + this._canInteract = true; + }; + + /** + * Calls GPT API to create a drawing based on user input + */ + @action + drawWithGPT = async (startPt: { X: number; Y: number }, input: string, complexity: number, size: number, autoColor: boolean) => { + if (input === '') return; + this._lastInput = { text: input, complexity: complexity, size: size, autoColor: autoColor, x: startPt.X, y: startPt.Y }; + const res = await gptAPICall(`"${input}", "${complexity}", "${size}"`, GPTCallType.DRAW, undefined, true); + if (!res) { + console.error('GPT call failed'); + return; + } + const strokeData = await this.parseSvg(res, startPt, false, autoColor); + const drawingDoc = strokeData && this.CreateDrawingDoc(strokeData?.data, strokeData?.lastInput, strokeData?.lastRes); + drawingDoc && this.AddDrawing(drawingDoc, this._lastInput, res); + + this._errorOccurredOnce = false; + return strokeData; + }; + + /** + * Regenerates drawings with the option to add a specific regenerate prompt/request. + */ + @action + regenerate = async (lastInput?: DrawingOptions, lastResponse?: string, regenInput?: string) => { + if (lastInput) this._lastInput = lastInput; + if (lastResponse) this._lastResponse = lastResponse; + if (regenInput) this._regenInput = regenInput; + + try { + let res; + if (this._regenInput !== '') { + const prompt: string = `This is your previously generated svg code: ${this._lastResponse} for the user input "${this._lastInput.text}". Please regenerate it with the provided specifications.`; + res = await gptAPICall(`"${this._regenInput}"`, GPTCallType.DRAW, prompt, true); + this._lastInput.text = `${this._lastInput.text} ~~~ ${this._regenInput}`; + } else { + res = await gptAPICall(`"${this._lastInput.text}", "${this._lastInput.complexity}", "${this._lastInput.size}"`, GPTCallType.DRAW, undefined, true); + } + if (!res) { + console.error('GPT call failed'); + return; + } + const strokeData = await this.parseSvg(res, { X: this._lastInput.x, Y: this._lastInput.y }, true, lastInput?.autoColor || this._autoColor); + this.RemoveDrawing !== unimplementedFunction && this.RemoveDrawing(this._selectedDoc); + const drawingDoc = strokeData && this.CreateDrawingDoc(strokeData?.data, strokeData?.lastInput, strokeData?.lastRes); + drawingDoc && this.AddDrawing(drawingDoc, this._lastInput, res); + return strokeData; + } catch (err) { + console.error('Error regenerating drawing', err); + } + }; + + /** + * Parses the svg code that GPT returns into Bezier curves. + */ + @action + parseSvg = async (res: string, startPoint: { X: number; Y: number }, regenerate: boolean, autoColor: boolean) => { + const svg = res.match(/<svg[^>]*>([\s\S]*?)<\/svg>/g); + if (svg) { + this._lastResponse = svg[0]; + const svgObject = await parse(svg[0]); + const svgStrokes: INode[] = svgObject.children; + const strokeData: [InkData, string, string][] = []; + svgStrokes.forEach(child => { + const convertedBezier: InkData = SVGToBezier(child.name as SVGType, child.attributes); + strokeData.push([ + convertedBezier.map(point => ({ X: point.X + startPoint.X - this._size / 1.5, Y: point.Y + startPoint.Y - this._size / 2 })), + (regenerate ? this._lastInput.autoColor : autoColor) ? child.attributes.stroke : '', + (regenerate ? this._lastInput.autoColor : autoColor) ? child.attributes.fill : '', + ]); + }); + return { data: strokeData, lastInput: this._lastInput, lastRes: svg[0] }; + } + }; + + /** + * Sends request to GPT API to recolor a selected ink document or group of ink documents. + */ + colorWithGPT = async (drawing: Doc) => { + const img = await DocumentView.GetDocImage(drawing); + const { href } = ImageCast(img).url; + const hrefParts = href.split('.'); + const hrefComplete = `${hrefParts[0]}_o.${hrefParts[1]}`; + try { + const hrefBase64 = await CollectionCardView.imageUrlToBase64(hrefComplete); + const strokes = DocListCast(drawing[DocData].data); + const coords: string[] = []; + strokes.forEach((stroke, i) => { + const inkingStroke = DocumentView.getDocumentView(stroke)?.ComponentView as InkingStroke; + const { inkData } = inkingStroke.inkScaledData(); + coords.push(`${i + 1}. ${inkData.filter((point, index) => index % 4 === 0 || index == inkData.length - 1).map(point => `(${point.X.toString()}, ${point.Y.toString()})`)}`); + }); + const colorResponse = await gptDrawingColor(hrefBase64, coords).then(response => gptAPICall(response, GPTCallType.COLOR, undefined)); + this.colorStrokes(colorResponse, drawing); + } catch (error) { + console.log('GPT call failed', error); + } + }; + + /** + * Function that parses the GPT color response and sets the selected stroke(s) to the new color. + */ + colorStrokes = undoable((res: string, drawing: Doc) => { + const colorList = res.match(/\{.*?\}/g); + const strokes = DocListCast(drawing[DocData].data); + colorList?.forEach((colors, index) => { + const strokeAndFill = colors.match(/#[0-9A-Fa-f]{6}/g); + if (strokeAndFill && strokeAndFill.length == 2) { + strokes[index][DocData].color = strokeAndFill[0]; + const inkStroke = DocumentView.getDocumentView(strokes[index])?.ComponentView as InkingStroke; + const { inkData } = inkStroke.inkScaledData(); + InkingStroke.IsClosed(inkData) ? (strokes[index][DocData].fillColor = strokeAndFill[1]) : (strokes[index][DocData].fillColor = undefined); + } + }); + }, 'color strokes'); + + renderDisplay() { + return ( + <div + id="label-handler" + className="smart-draw-handler" + style={{ + display: this._display ? '' : 'none', + left: this._pageX, + ...(this._yRelativeToTop ? { top: Math.max(0, this._pageY) } : { bottom: this._pageY }), + background: SettingsManager.userBackgroundColor, + color: SettingsManager.userColor, + }}> + <div> + <IconButton + tooltip="Cancel" + onClick={() => { + this.hideSmartDrawHandler(); + this.hideRegenerate(); + }} + icon={<FontAwesomeIcon icon="xmark" />} + color={SettingsManager.userColor} + style={{ width: '19px' }} + /> + <input + aria-label="Smart Draw Input" + className="smartdraw-input" + type="text" + autoFocus + style={{ color: 'black' }} + value={this._userInput} + onChange={action(e => this._canInteract && (this._userInput = e.target.value))} + placeholder="Enter item to draw" + onKeyDown={this.handleKeyPress} + /> + <IconButton + tooltip="Advanced Options" + icon={<FontAwesomeIcon icon={this._showOptions ? 'caret-down' : 'caret-right'} />} + color={SettingsManager.userColor} + style={{ width: '14px' }} + onClick={action(() => (this._showOptions = !this._showOptions))} + /> + <Button + style={{ alignSelf: 'flex-end' }} + text="Send" + icon={this._isLoading ? <ReactLoading type="spin" color={SettingsManager.userVariantColor} width={16} height={20} /> : <AiOutlineSend />} + iconPlacement="right" + color={SettingsManager.userColor} + onClick={this.handleSendClick} + /> + </div> + {this._showOptions && ( + <div style={{ display: 'flex', flexDirection: 'row', justifyContent: 'space-around' }}> + <div style={{ display: 'flex', flexDirection: 'column', justifyContent: 'center', width: '30%' }}> + Auto color + <Switch + sx={{ + '& .MuiSwitch-switchBase.Mui-checked': { color: SettingsManager.userColor }, + '& .MuiSwitch-switchBase.Mui-checked + .MuiSwitch-track': { backgroundColor: SettingsManager.userVariantColor }, + }} + defaultChecked={true} + value={this._autoColor} + size="small" + onChange={action(e => this._canInteract && (this._autoColor = !this._autoColor))} + /> + </div> + <div style={{ display: 'flex', flexDirection: 'column', justifyContent: 'center', width: '31%' }}> + Complexity + <Slider + sx={{ + '& .MuiSlider-track': { color: SettingsManager.userVariantColor }, + '& .MuiSlider-rail': { color: SettingsManager.userColor }, + '& .MuiSlider-thumb': { color: SettingsManager.userColor, '&.Mui-focusVisible, &:hover, &.Mui-active': { boxShadow: `0px 0px 0px 8px${SettingsManager.userColor.slice(0, 7)}10` } }, + }} + style={{ width: '80%' }} + min={1} + max={10} + step={1} + size="small" + value={this._complexity} + onChange={action((e, val) => this._canInteract && (this._complexity = val as number))} + valueLabelDisplay="auto" + /> + </div> + <div style={{ display: 'flex', flexDirection: 'column', justifyContent: 'center', width: '39%' }}> + Size (in pixels) + <Slider + sx={{ + '& .MuiSlider-track': { color: SettingsManager.userVariantColor }, + '& .MuiSlider-rail': { color: SettingsManager.userColor }, + '& .MuiSlider-thumb': { color: SettingsManager.userColor, '&.Mui-focusVisible, &:hover, &.Mui-active': { boxShadow: `0px 0px 0px 8px${SettingsManager.userColor.slice(0, 7)}20` } }, + }} + style={{ width: '80%' }} + min={50} + max={700} + step={10} + size="small" + value={this._size} + onChange={action((e, val) => this._canInteract && (this._size = val as number))} + valueLabelDisplay="auto" + /> + </div> + </div> + )} + </div> + ); + } + + renderRegenerate() { + return ( + <div + className="smart-draw-handler" + style={{ + left: this._pageX, + ...(this._yRelativeToTop ? { top: Math.max(0, this._pageY) } : { bottom: this._pageY }), + background: SettingsManager.userBackgroundColor, + color: SettingsManager.userColor, + }}> + <div style={{ display: 'flex', flexDirection: 'row' }}> + <IconButton + tooltip="Regenerate" + icon={this._isLoading && this._regenInput === '' ? <ReactLoading type="spin" color={SettingsManager.userVariantColor} width={16} height={20} /> : <FontAwesomeIcon icon={'rotate'} />} + color={SettingsManager.userColor} + onClick={this.handleSendClick} + /> + <IconButton tooltip="Edit with GPT" icon={<FontAwesomeIcon icon="pen-to-square" />} color={SettingsManager.userColor} onClick={action(() => (this._showEditBox = !this._showEditBox))} /> + {this._showEditBox && ( + <div style={{ display: 'flex', flexDirection: 'row' }}> + <input + aria-label="Edit instructions input" + className="smartdraw-input" + type="text" + style={{ color: 'black' }} + value={this._regenInput} + onChange={action(e => this._canInteract && (this._regenInput = e.target.value))} + onKeyDown={this.handleKeyPress} + placeholder="Edit instructions" + /> + <Button + style={{ alignSelf: 'flex-end' }} + text="Send" + icon={this._isLoading && this._regenInput !== '' ? <ReactLoading type="spin" color={SettingsManager.userVariantColor} width={16} height={20} /> : <AiOutlineSend />} + iconPlacement="right" + color={SettingsManager.userColor} + onClick={this.handleSendClick} + /> + </div> + )} + </div> + </div> + ); + } + + render() { + return this._display ? this.renderDisplay() : this.ShowRegenerate ? this.renderRegenerate() : null; + } +} |