diff options
Diffstat (limited to 'src/client/views/smartdraw/SmartDrawHandler.tsx')
| -rw-r--r-- | src/client/views/smartdraw/SmartDrawHandler.tsx | 72 |
1 files changed, 45 insertions, 27 deletions
diff --git a/src/client/views/smartdraw/SmartDrawHandler.tsx b/src/client/views/smartdraw/SmartDrawHandler.tsx index fbf471900..ca308015d 100644 --- a/src/client/views/smartdraw/SmartDrawHandler.tsx +++ b/src/client/views/smartdraw/SmartDrawHandler.tsx @@ -27,16 +27,19 @@ import { ActiveInkArrowEnd, ActiveInkArrowStart, ActiveInkBezierApprox, ActiveIn import { FireflyDimensionsMap, FireflyImageData, FireflyImageDimensions } from './FireflyConstants'; import './SmartDrawHandler.scss'; import { Upload } from '../../../server/SharedMediaTypes'; +import { PointData } from '../../../pen-gestures/GestureTypes'; export interface DrawingOptions { - text: string; - complexity: number; - size: number; - autoColor: boolean; - x: number; - y: number; + text?: string; + complexity?: number; + size?: number; + autoColor?: boolean; + x?: number; + y?: number; } +type svgparsedData = [PointData[], string, string]; + /** * The SmartDrawHandler allows users to generate drawings with GPT from text input. Users are able to enter * the item to draw, how complex they want the drawing to be, how large the drawing should be, and whether @@ -102,7 +105,7 @@ export class SmartDrawHandler extends ObservableReactComponent<object> { * classes to customize the way the drawing docs get created. For example, the freeform canvas has a different way of * defining document bounds, so CreateDrawingDoc is redefined when that class calls gpt draw functions. */ - public CreateDrawingDoc: (strokeList: [InkData, string, string][], opts: DrawingOptions, gptRes: string, containerDoc?: Doc) => Doc | undefined = (strokeList: [InkData, string, string][], opts: DrawingOptions) => { + public static CreateDrawingDoc: (strokeList: [InkData, string, string][], opts: DrawingOptions, gptRes: string, containerDoc?: Doc) => Doc | undefined = (strokeList: [InkData, string, string][], opts: DrawingOptions) => { const drawing: Doc[] = []; strokeList.forEach((stroke: [InkData, string, string]) => { const bounds = InkField.getBounds(stroke[0]); @@ -114,7 +117,7 @@ export class SmartDrawHandler extends ObservableReactComponent<object> { y: bounds.top - inkWidth / 2, _width: bounds.width + inkWidth, _height: bounds.height + inkWidth, - stroke_showLabel: !BoolCast(Doc.UserDoc().activeHideTextLabels)}, // prettier-ignore + stroke_showLabel: false}, // prettier-ignore inkWidth, opts.autoColor ? stroke[1] : ActiveInkColor(), ActiveInkBezierApprox(), @@ -188,9 +191,9 @@ export class SmartDrawHandler extends ObservableReactComponent<object> { /** * This allows users to press the return/enter key to send input. */ - handleKeyPress = (event: React.KeyboardEvent) => { - if (event.key === 'Enter') { - this.handleSendClick(); + handleKeyPress = (e: React.KeyboardEvent) => { + if (e.key === 'Enter') { + this.handleSendClick(this._pageX, this._pageY); } }; @@ -200,7 +203,7 @@ export class SmartDrawHandler extends ObservableReactComponent<object> { * what the user sees. */ @action - handleSendClick = async () => { + handleSendClick = async (X: number, Y: number) => { if ((!this.ShowRegenerate && this._userInput == '') || (!this._generateImage && !this._generateDrawing)) return; this._isLoading = true; this._canInteract = false; @@ -213,7 +216,7 @@ export class SmartDrawHandler extends ObservableReactComponent<object> { await this.createImageWithFirefly(this._userInput); } if (this._generateDrawing) { - await this.drawWithGPT({ X: this._pageX, Y: this._pageY }, this._userInput, this._complexity, this._size, this._autoColor); + await this.drawWithGPT({ X, Y }, this._userInput, this._complexity, this._size, this._autoColor); } this.hideSmartDrawHandler(); } catch (err) { @@ -229,14 +232,14 @@ export class SmartDrawHandler extends ObservableReactComponent<object> { /** * Calls GPT API to create a drawing based on user input. */ - drawWithGPT = async (startPt: { X: number; Y: number }, input: string, complexity: number, size: number, autoColor: boolean) => { + drawWithGPT = async (screenPt: { X: number; Y: number }, input: string, complexity: number, size: number, autoColor: boolean) => { if (input) { - this._lastInput = { text: input, complexity: complexity, size: size, autoColor: autoColor, x: startPt.X, y: startPt.Y }; + this._lastInput = { text: input, complexity: complexity, size: size, autoColor: autoColor, x: screenPt.X, y: screenPt.Y }; const res = await gptAPICall(`"${input}", "${complexity}", "${size}"`, GPTCallType.DRAW, undefined, true); if (res) { - const strokeData = await this.parseSvg(res, startPt, false, autoColor); - const drawingDoc = strokeData && this.CreateDrawingDoc(strokeData.data, strokeData.lastInput, strokeData.lastRes); - drawingDoc && this.AddDrawing(drawingDoc, this._lastInput, res); + const strokeData = await this.parseSvg(res, { X: 0, Y: 0 }, false, autoColor); + const drawingDoc = strokeData && SmartDrawHandler.CreateDrawingDoc(strokeData.data, strokeData.lastInput, strokeData.lastRes); + drawingDoc && this.AddDrawing(drawingDoc, this._lastInput, res, screenPt.X, screenPt.Y); drawingDoc && this._selectedDocs.push(drawingDoc); return strokeData; } else { @@ -334,9 +337,9 @@ export class SmartDrawHandler extends ObservableReactComponent<object> { return gptAPICall(`"${this._lastInput.text}", "${this._lastInput.complexity}", "${this._lastInput.size}"`, GPTCallType.DRAW, undefined, true); })(); if (res) { - const strokeData = await this.parseSvg(res, { X: this._lastInput.x, Y: this._lastInput.y }, true, lastInput?.autoColor || this._autoColor); + const strokeData = await this.parseSvg(res, { X: this._lastInput.x ?? 0, Y: this._lastInput.y ?? 0 }, true, lastInput?.autoColor || this._autoColor); this.RemoveDrawing !== unimplementedFunction && this.RemoveDrawing(true, doc); - const drawingDoc = strokeData && this.CreateDrawingDoc(strokeData.data, strokeData.lastInput, strokeData.lastRes); + const drawingDoc = strokeData && SmartDrawHandler.CreateDrawingDoc(strokeData.data, strokeData.lastInput, strokeData.lastRes); drawingDoc && this.AddDrawing(drawingDoc, this._lastInput, res); } else { console.error('GPT call failed'); @@ -356,20 +359,35 @@ export class SmartDrawHandler extends ObservableReactComponent<object> { */ parseSvg = async (res: string, startPoint: { X: number; Y: number }, regenerate: boolean, autoColor: boolean) => { const svg = res.match(/<svg[^>]*>([\s\S]*?)<\/svg>/g); + if (svg) { this._lastResponse = svg[0]; const svgObject = await parse(svg[0]); + console.log(res, svgObject); const svgStrokes: INode[] = svgObject.children; const strokeData: [InkData, string, string][] = []; + + const tl = { X: Number.MAX_SAFE_INTEGER, Y: Number.MAX_SAFE_INTEGER }; + let last: PointData = { X: 0, Y: 0 }; svgStrokes.forEach(child => { - const convertedBezier: InkData = SVGToBezier(child.name as SVGType, child.attributes); + const convertedBezier: InkData = SVGToBezier(child.name as SVGType, child.attributes, last); + last = convertedBezier.lastElement(); strokeData.push([ - convertedBezier.map(point => ({ X: startPoint.X + (point.X - startPoint.X) * this._scale, Y: startPoint.Y + (point.Y - startPoint.Y) * this._scale })), + convertedBezier.map(point => { + if (point.X < tl.X) tl.X = point.X; + if (point.Y < tl.Y) tl.Y = point.Y; + return { X: point.X, Y: point.Y }; + }), (regenerate ? this._lastInput.autoColor : autoColor) ? child.attributes.stroke : '', (regenerate ? this._lastInput.autoColor : autoColor) ? child.attributes.fill : '', ]); }); - return { data: strokeData, lastInput: this._lastInput, lastRes: svg[0] }; + const mapStroke = (pd: PointData): PointData => ({ X: startPoint.X + (pd.X - tl.X) * this._scale, Y: startPoint.Y + (pd.Y - tl.Y) * this._scale }); + return { + data: strokeData.map(sdata => [sdata[0].map(mapStroke), sdata[1], sdata[2]] as svgparsedData), + lastInput: this._lastInput, + lastRes: svg[0], + }; } }; @@ -439,7 +457,7 @@ export class SmartDrawHandler extends ObservableReactComponent<object> { }, }} checked={this._generateImage} - onChange={() => this._canInteract && (this._generateImage = !this._generateImage)} + onChange={action(() => this._canInteract && (this._generateImage = !this._generateImage))} /> </div> </div> @@ -566,7 +584,7 @@ export class SmartDrawHandler extends ObservableReactComponent<object> { icon={this._isLoading ? <ReactLoading type="spin" color={SettingsManager.userVariantColor} width={16} height={20} /> : <AiOutlineSend />} iconPlacement="right" color={SettingsManager.userColor} - onClick={this.handleSendClick} + onClick={() => this.handleSendClick(this._pageX, this._pageY)} /> </div> {this._showOptions && ( @@ -598,7 +616,7 @@ export class SmartDrawHandler extends ObservableReactComponent<object> { icon={this._isLoading && this._regenInput !== '' ? <ReactLoading type="spin" color={SettingsManager.userVariantColor} width={16} height={20} /> : <AiOutlineSend />} iconPlacement="right" color={SettingsManager.userColor} - onClick={this.handleSendClick} + onClick={() => this.handleSendClick(this._pageX, this._pageY)} /> </div> ); @@ -644,7 +662,7 @@ export class SmartDrawHandler extends ObservableReactComponent<object> { tooltip="Regenerate" icon={this._isLoading && this._regenInput === '' ? <ReactLoading type="spin" color={SettingsManager.userVariantColor} width={16} height={20} /> : <FontAwesomeIcon icon={'rotate'} />} color={SettingsManager.userColor} - onClick={this.handleSendClick} + onClick={() => this.handleSendClick(this._pageX, this._pageY)} /> <IconButton tooltip="Edit with GPT" icon={<FontAwesomeIcon icon="pen-to-square" />} color={SettingsManager.userColor} onClick={action(() => (this._showEditBox = !this._showEditBox))} /> {this._showEditBox ? this.renderRegenerateEditBox() : null} |