diff options
Diffstat (limited to 'src/client/views/smartdraw')
| -rw-r--r-- | src/client/views/smartdraw/AnnotationPalette.scss | 56 | ||||
| -rw-r--r-- | src/client/views/smartdraw/AnnotationPalette.tsx | 361 | ||||
| -rw-r--r-- | src/client/views/smartdraw/SmartDrawHandler.scss | 44 | ||||
| -rw-r--r-- | src/client/views/smartdraw/SmartDrawHandler.tsx | 491 |
4 files changed, 952 insertions, 0 deletions
diff --git a/src/client/views/smartdraw/AnnotationPalette.scss b/src/client/views/smartdraw/AnnotationPalette.scss new file mode 100644 index 000000000..4f11e8afc --- /dev/null +++ b/src/client/views/smartdraw/AnnotationPalette.scss @@ -0,0 +1,56 @@ +.annotation-palette { + display: flex; + flex-direction: column; + align-items: center; + position: absolute; + right: 14px; + top: 50px; + border-radius: 5px; + margin: auto; +} + +.palette-create { + display: flex; + flex-direction: row; + width: 170px; + + .palette-create-input { + color: black; + width: 170px; + } +} + +.palette-create-options { + display: flex; + flex-direction: row; + justify-content: space-between; + width: 170px; + margin-top: 5px; + + .palette-color { + display: flex; + flex-direction: column; + align-items: center; + width: 40px; + } + + .palette-detail, + .palette-size { + display: flex; + flex-direction: column; + align-items: center; + width: 60px; + } +} + +.palette-buttons { + width: 100%; + display: flex; + flex-direction: row; + justify-content: space-between; +} + +.palette-save-reset { + display: flex; + flex-direction: row; +} diff --git a/src/client/views/smartdraw/AnnotationPalette.tsx b/src/client/views/smartdraw/AnnotationPalette.tsx new file mode 100644 index 000000000..f1e2e4f41 --- /dev/null +++ b/src/client/views/smartdraw/AnnotationPalette.tsx @@ -0,0 +1,361 @@ +import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; +import { Slider, Switch } from '@mui/material'; +import { Button } from 'browndash-components'; +import { action, makeObservable, observable } from 'mobx'; +import { observer } from 'mobx-react'; +import * as React from 'react'; +import { AiOutlineSend } from 'react-icons/ai'; +import ReactLoading from 'react-loading'; +import { returnEmptyFilter, returnFalse, returnTrue } from '../../../ClientUtils'; +import { emptyFunction } from '../../../Utils'; +import { Doc, DocListCast, returnEmptyDoclist } from '../../../fields/Doc'; +import { DocData } from '../../../fields/DocSymbols'; +import { ImageCast, NumCast } from '../../../fields/Types'; +import { ImageField } from '../../../fields/URLField'; +import { DocumentType } from '../../documents/DocumentTypes'; +import { Docs } from '../../documents/Documents'; +import { makeUserTemplateButtonOrImage } from '../../util/DropConverter'; +import { SettingsManager } from '../../util/SettingsManager'; +import { Transform } from '../../util/Transform'; +import { undoBatch } from '../../util/UndoManager'; +import { ObservableReactComponent } from '../ObservableReactComponent'; +import { DefaultStyleProvider, returnEmptyDocViewList } from '../StyleProvider'; +import { DocumentView, DocumentViewInternal } from '../nodes/DocumentView'; +import { FieldView } from '../nodes/FieldView'; +import './AnnotationPalette.scss'; +import { DrawingOptions, SmartDrawHandler } from './SmartDrawHandler'; + +interface AnnotationPaletteProps { + Document: Doc; +} + +/** + * The AnnotationPalette can be toggled in the lightbox view of a document. The goal of the palette + * is to offer an easy way for users to save then drag and drop repeated annotations onto a document. + * These annotations can be of any annotation type and operate similarly to user templates. + * + * On the "add" side of the palette, there is a way to create a drawing annotation with GPT. Users can + * enter the item to draw, toggle different settings, then GPT will generate three versions of the drawing + * to choose from. These drawings can then be saved to the palette as annotations. + */ +@observer +export class AnnotationPalette extends ObservableReactComponent<AnnotationPaletteProps> { + @observable private _paletteMode: 'create' | 'view' = 'view'; + @observable private _userInput: string = ''; + @observable private _isLoading: boolean = false; + @observable private _canInteract: boolean = true; + @observable private _showRegenerate: boolean = false; + @observable private _docView: DocumentView | null = null; + @observable private _docCarouselView: DocumentView | null = null; + @observable private _opts: DrawingOptions = { text: '', complexity: 5, size: 200, autoColor: true, x: 0, y: 0 }; + private _gptRes: string[] = []; + + constructor(props: AnnotationPaletteProps) { + super(props); + makeObservable(this); + } + + public static LayoutString(fieldKey: string) { + return FieldView.LayoutString(AnnotationPalette, fieldKey); + } + + Contains = (view: DocumentView) => { + return (this._docView && (view.containerViewPath?.() ?? []).concat(view).includes(this._docView)) || (this._docCarouselView && (view.containerViewPath?.() ?? []).concat(view).includes(this._docCarouselView)); + }; + + return170 = () => 170; + + @action + handleKeyPress = async (event: React.KeyboardEvent) => { + if (event.key === 'Enter') { + await this.generateDrawings(); + } + }; + + @action + setPaletteMode = (mode: 'create' | 'view') => { + this._paletteMode = mode; + }; + + @action + setUserInput = (input: string) => { + if (!this._isLoading) this._userInput = input; + }; + + @action + setDetail = (detail: number) => { + if (this._canInteract) this._opts.complexity = detail; + }; + + @action + setColor = (autoColor: boolean) => { + if (this._canInteract) this._opts.autoColor = autoColor; + }; + + @action + setSize = (size: number) => { + if (this._canInteract) this._opts.size = size; + }; + + @action + resetPalette = (changePaletteMode: boolean) => { + if (changePaletteMode) this.setPaletteMode('view'); + this.setUserInput(''); + this.setDetail(5); + this.setColor(true); + this.setSize(200); + this._showRegenerate = false; + this._canInteract = true; + this._opts = { text: '', complexity: 5, size: 200, autoColor: true, x: 0, y: 0 }; + this._gptRes = []; + this._props.Document[DocData].data = undefined; + }; + + /** + * Adds a doc to the annotation palette. Gets a snapshot of the document to use as a preview in the palette. When this + * preview is dragged onto a parent document, a copy of that document is added as an annotation. + */ + public static addToPalette = async (doc: Doc) => { + if (!doc.savedAsAnno) { + const docView = DocumentView.getDocumentView(doc); + await docView?.ComponentView?.updateIcon?.(true); + const { clone } = await Doc.MakeClone(doc); + clone.title = doc.title; + const image = ImageCast(doc.icon, ImageCast(clone[Doc.LayoutFieldKey(clone)]))?.url?.href; + Doc.AddDocToList(Doc.MyAnnos, 'data', makeUserTemplateButtonOrImage(clone, image)); + doc.savedAsAnno = true; + } + }; + + public static getIcon(group: Doc) { + const docView = DocumentView.getDocumentView(group); + if (docView) { + docView.ComponentView?.updateIcon?.(true); + return new Promise<ImageField | undefined>(res => setTimeout(() => res(ImageCast(docView.Document.icon)), 1000)); + } + return undefined; + } + + /** + * Calls the draw with GPT functions in SmartDrawHandler to allow users to generate drawings straight from + * the annotation palette. + */ + @undoBatch + generateDrawings = action(async () => { + this._isLoading = true; + this._props.Document[DocData].data = undefined; + for (let i = 0; i < 3; i++) { + try { + SmartDrawHandler.Instance.AddDrawing = this.addDrawing; + this._canInteract = false; + if (this._showRegenerate) { + await SmartDrawHandler.Instance.regenerate(this._opts, this._gptRes[i], this._userInput); + } else { + await SmartDrawHandler.Instance.drawWithGPT({ X: 0, Y: 0 }, this._userInput, this._opts.complexity, this._opts.size, this._opts.autoColor); + } + } catch (e) { + console.log('Error generating drawing', e); + } + } + this._opts.text !== '' ? (this._opts.text = `${this._opts.text} ~~~ ${this._userInput}`) : (this._opts.text = this._userInput); + this._userInput = ''; + this._isLoading = false; + this._showRegenerate = true; + }); + + @action + addDrawing = (drawing: Doc, opts: DrawingOptions, gptRes: string) => { + this._gptRes.push(gptRes); + drawing[DocData].freeform_fitContentsToBox = true; + Doc.AddDocToList(this._props.Document, 'data', drawing); + }; + + /** + * Saves the currently showing, newly generated drawing to the annotation palette and sets the metadata. + * AddToPalette() is generically used to add any document to the palette, while this defines the behavior for when a user + * presses the "save drawing" button. + */ + saveDrawing = async () => { + const cIndex = NumCast(this._props.Document.carousel_index); + const focusedDrawing = DocListCast(this._props.Document.data)[cIndex]; + const docData = focusedDrawing[DocData]; + docData.title = this._opts.text.match(/^(.*?)~~~.*$/)?.[1] || this._opts.text; + docData.drawingInput = this._opts.text; + docData.drawingComplexity = this._opts.complexity; + docData.drawingColored = this._opts.autoColor; + docData.drawingSize = this._opts.size; + docData.drawingData = this._gptRes[cIndex]; + docData.width = this._opts.size; + docData.x = this._opts.x; + docData.y = this._opts.y; + await AnnotationPalette.addToPalette(focusedDrawing); + this.resetPalette(true); + }; + + render() { + return ( + <div className="annotation-palette" style={{ zIndex: 1000 }} onClick={e => e.stopPropagation()}> + {this._paletteMode === 'view' && ( + <> + <DocumentView + ref={r => (this._docView = r)} + Document={Doc.MyAnnos} + addDocument={undefined} + addDocTab={DocumentViewInternal.addDocTabFunc} + pinToPres={DocumentView.PinDoc} + containerViewPath={returnEmptyDocViewList} + styleProvider={DefaultStyleProvider} + removeDocument={returnFalse} + ScreenToLocalTransform={Transform.Identity} + PanelWidth={this.return170} + PanelHeight={this.return170} + renderDepth={0} + isContentActive={returnTrue} + focus={emptyFunction} + whenChildContentsActiveChanged={emptyFunction} + childFilters={returnEmptyFilter} + childFiltersByRanges={returnEmptyFilter} + searchFilterDocs={returnEmptyDoclist} + /> + <Button text="Add" icon={<FontAwesomeIcon icon="square-plus" />} color={SettingsManager.userColor} onClick={() => this.setPaletteMode('create')} /> + </> + )} + {this._paletteMode === 'create' && ( + <> + <div className="palette-create"> + <input + className="palette-create-input" + aria-label="label-input" + id="new-label" + type="text" + value={this._userInput} + onChange={e => { + this.setUserInput(e.target.value); + }} + placeholder={this._showRegenerate ? '(Optional) Enter edits' : 'Enter item to draw'} + onKeyDown={this.handleKeyPress} + /> + <Button + style={{ alignSelf: 'flex-end' }} + tooltip={this._showRegenerate ? 'Regenerate' : 'Send'} + icon={this._isLoading ? <ReactLoading type="spin" color={SettingsManager.userVariantColor} width={16} height={20} /> : this._showRegenerate ? <FontAwesomeIcon icon={'rotate'} /> : <AiOutlineSend />} + iconPlacement="right" + color={SettingsManager.userColor} + onClick={this.generateDrawings} + /> + </div> + <div className="palette-create-options"> + <div className="palette-color"> + Color + <Switch + sx={{ + '& .MuiSwitch-switchBase.Mui-checked': { + color: SettingsManager.userColor, + }, + '& .MuiSwitch-switchBase.Mui-checked + .MuiSwitch-track': { + backgroundColor: SettingsManager.userVariantColor, + }, + }} + defaultChecked={true} + value={this._opts.autoColor} + size="small" + onChange={() => this.setColor(!this._opts.autoColor)} + /> + </div> + <div className="palette-detail"> + Detail + <Slider + sx={{ + '& .MuiSlider-thumb': { + color: SettingsManager.userColor, + '&.Mui-focusVisible, &:hover, &.Mui-active': { + boxShadow: `0px 0px 0px 8px${SettingsManager.userColor.slice(0, 7)}10`, + }, + }, + '& .MuiSlider-track': { + color: SettingsManager.userVariantColor, + }, + '& .MuiSlider-rail': { + color: SettingsManager.userColor, + }, + }} + style={{ width: '80%' }} + min={1} + max={10} + step={1} + size="small" + value={this._opts.complexity} + onChange={(e, val) => { + this.setDetail(val as number); + }} + valueLabelDisplay="auto" + /> + </div> + <div className="palette-size"> + Size + <Slider + sx={{ + '& .MuiSlider-thumb': { + color: SettingsManager.userColor, + '&.Mui-focusVisible, &:hover, &.Mui-active': { + boxShadow: `0px 0px 0px 8px${SettingsManager.userColor.slice(0, 7)}20`, + }, + }, + '& .MuiSlider-track': { + color: SettingsManager.userVariantColor, + }, + '& .MuiSlider-rail': { + color: SettingsManager.userColor, + }, + }} + style={{ width: '80%' }} + min={50} + max={500} + step={10} + size="small" + value={this._opts.size} + onChange={(e, val) => { + this.setSize(val as number); + }} + valueLabelDisplay="auto" + /> + </div> + </div> + <DocumentView + ref={r => (this._docCarouselView = r)} + Document={this._props.Document} + addDocument={undefined} + addDocTab={DocumentViewInternal.addDocTabFunc} + pinToPres={DocumentView.PinDoc} + containerViewPath={returnEmptyDocViewList} + styleProvider={DefaultStyleProvider} + removeDocument={returnFalse} + ScreenToLocalTransform={Transform.Identity} + PanelWidth={this.return170} + PanelHeight={this.return170} + renderDepth={1} + isContentActive={returnTrue} + focus={emptyFunction} + whenChildContentsActiveChanged={emptyFunction} + childFilters={returnEmptyFilter} + childFiltersByRanges={returnEmptyFilter} + searchFilterDocs={returnEmptyDoclist} + /> + <div className="palette-buttons"> + <Button text="Back" tooltip="Back to All Annotations" icon={<FontAwesomeIcon icon="reply" />} color={SettingsManager.userColor} onClick={() => this.resetPalette(true)} /> + <div className="palette-save-reset"> + <Button tooltip="Save" icon={<FontAwesomeIcon icon="file-arrow-down" />} color={SettingsManager.userColor} onClick={this.saveDrawing} /> + <Button tooltip="Reset" icon={<FontAwesomeIcon icon="rotate-left" />} color={SettingsManager.userColor} onClick={() => this.resetPalette(false)} /> + </div> + </div> + </> + )} + </div> + ); + } +} + +Docs.Prototypes.TemplateMap.set(DocumentType.ANNOPALETTE, { + layout: { view: AnnotationPalette, dataField: 'data' }, + options: { acl: '' }, +}); diff --git a/src/client/views/smartdraw/SmartDrawHandler.scss b/src/client/views/smartdraw/SmartDrawHandler.scss new file mode 100644 index 000000000..0e8bd3349 --- /dev/null +++ b/src/client/views/smartdraw/SmartDrawHandler.scss @@ -0,0 +1,44 @@ +.smart-draw-handler { + position: absolute; +} + +.smartdraw-input { + color: black; +} + +.smartdraw-options { + display: flex; + flex-direction: row; + justify-content: space-around; + + .auto-color { + display: flex; + flex-direction: column; + justify-content: center; + width: 30%; + } + + .complexity { + display: flex; + flex-direction: column; + justify-content: center; + width: 31%; + } + + .size { + display: flex; + flex-direction: column; + justify-content: center; + width: 39%; + + .size-slider { + width: 80%; + } + } +} + +.regenerate-box, +.edit-box { + display: flex; + flex-direction: row; +} diff --git a/src/client/views/smartdraw/SmartDrawHandler.tsx b/src/client/views/smartdraw/SmartDrawHandler.tsx new file mode 100644 index 000000000..75ef55060 --- /dev/null +++ b/src/client/views/smartdraw/SmartDrawHandler.tsx @@ -0,0 +1,491 @@ +import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; +import { Slider, Switch } from '@mui/material'; +import { Button, IconButton } from 'browndash-components'; +import { action, makeObservable, observable, runInAction } from 'mobx'; +import { observer } from 'mobx-react'; +import React from 'react'; +import { AiOutlineSend } from 'react-icons/ai'; +import ReactLoading from 'react-loading'; +import { INode, parse } from 'svgson'; +import { unimplementedFunction } from '../../../Utils'; +import { Doc, DocListCast } from '../../../fields/Doc'; +import { DocData } from '../../../fields/DocSymbols'; +import { InkData, InkField, InkTool } from '../../../fields/InkField'; +import { BoolCast, ImageCast, NumCast, StrCast } from '../../../fields/Types'; +import { GPTCallType, gptAPICall, gptDrawingColor } from '../../apis/gpt/GPT'; +import { Docs } from '../../documents/Documents'; +import { SettingsManager } from '../../util/SettingsManager'; +import { undoable } from '../../util/UndoManager'; +import { SVGToBezier, SVGType } from '../../util/bezierFit'; +import { InkingStroke } from '../InkingStroke'; +import { ObservableReactComponent } from '../ObservableReactComponent'; +import { CollectionCardView } from '../collections/CollectionCardDeckView'; +import { MarqueeView } from '../collections/collectionFreeForm'; +import { ActiveArrowEnd, ActiveArrowStart, ActiveDash, ActiveFillColor, ActiveInkBezierApprox, ActiveInkColor, ActiveInkWidth, ActiveIsInkMask, DocumentView } from '../nodes/DocumentView'; +import './SmartDrawHandler.scss'; + +export interface DrawingOptions { + text: string; + complexity: number; + size: number; + autoColor: boolean; + x: number; + y: number; +} + +/** + * The SmartDrawHandler allows users to generate drawings with GPT from text input. Users are able to enter + * the item to draw, how complex they want the drawing to be, how large the drawing should be, and whether + * it will be colored. If the drawing is colored, GPT will automatically define the stroke and fill of each + * stroke. Drawings are retrieved from GPT as SVG code then converted into Dash-supported Beziers. + * + * The handler is selected from the ink tools menu. To generate a drawing, users can click anywhere on the freeform + * canvas and a popup will appear that prompts them to create a drawing. Once the drawing is created, users have + * the option to regenerate or edit the drawing. + * + * When each drawing is created, it is added to Dash as a group of ink strokes. The group is tagged with metadata + * for user input, the drawing's SVG code, and its settings (size, complexity). In the context menu -> 'Options', + * users can then show the drawing editor and regenerate/edit them at any point in the future. + */ + +@observer +export class SmartDrawHandler extends ObservableReactComponent<object> { + // eslint-disable-next-line no-use-before-define + static Instance: SmartDrawHandler; + + private _lastInput: DrawingOptions = { text: '', complexity: 5, size: 350, autoColor: true, x: 0, y: 0 }; + private _lastResponse: string = ''; + private _selectedDoc: Doc | undefined = undefined; + private _errorOccurredOnce = false; + + @observable private _display: boolean = false; + @observable private _pageX: number = 0; + @observable private _pageY: number = 0; + @observable private _yRelativeToTop: boolean = true; + @observable private _isLoading: boolean = false; + @observable private _userInput: string = ''; + @observable private _showOptions: boolean = false; + @observable private _showEditBox: boolean = false; + @observable private _complexity: number = 5; + @observable private _size: number = 200; + @observable private _autoColor: boolean = true; + @observable private _regenInput: string = ''; + @observable private _canInteract: boolean = true; + + @observable public ShowRegenerate: boolean = false; + + constructor(props: object) { + super(props); + makeObservable(this); + SmartDrawHandler.Instance = this; + } + + /** + * AddDrawing and RemoveDrawing are defined by the other classes that call the smart draw functions (i.e. + CollectionFreeForm, FormattedTextBox, AnnotationPalette) to define how a drawing document should be added + or removed in their respective locations (to the freeform canvs, to the annotation palette's preview, etc.) + */ + public AddDrawing: (doc: Doc, opts: DrawingOptions, gptRes: string) => void = unimplementedFunction; + public RemoveDrawing: (useLastContainer: boolean, doc?: Doc) => void = unimplementedFunction; + /** + * This creates the ink document that represents a drawing, so it goes through the strokes that make up the drawing, + * creates ink documents for each stroke, then adds the strokes to a collection. This can also be redefined by other + * classes to customize the way the drawing docs get created. For example, the freeform canvas has a different way of + * defining document bounds, so CreateDrawingDoc is redefined when that class calls gpt draw functions. + */ + public CreateDrawingDoc: (strokeList: [InkData, string, string][], opts: DrawingOptions, gptRes: string, containerDoc?: Doc) => Doc | undefined = (strokeList: [InkData, string, string][], opts: DrawingOptions) => { + const drawing: Doc[] = []; + strokeList.forEach((stroke: [InkData, string, string]) => { + const bounds = InkField.getBounds(stroke[0]); + const inkWidth = Math.min(5, ActiveInkWidth()); + const inkDoc = Docs.Create.InkDocument( + stroke[0], + { title: 'stroke', + x: bounds.left - inkWidth / 2, + y: bounds.top - inkWidth / 2, + _width: bounds.width + inkWidth, + _height: bounds.height + inkWidth, + stroke_showLabel: BoolCast(Doc.UserDoc().activeInkHideTextLabels)}, // prettier-ignore + inkWidth, + opts.autoColor ? stroke[1] : ActiveInkColor(), + ActiveInkBezierApprox(), + stroke[2] === 'none' ? ActiveFillColor() : stroke[2], + ActiveArrowStart(), + ActiveArrowEnd(), + ActiveDash(), + ActiveIsInkMask() + ); + drawing.push(inkDoc); + }); + + return MarqueeView.getCollection(drawing, undefined, true, { left: 1, top: 1, width: 1, height: 1 }); + }; + + @action + displaySmartDrawHandler = (x: number, y: number) => { + [this._pageX, this._pageY] = [x, y]; + this._display = true; + }; + + /** + * This is called in two places: 1. In this class, where the regenerate popup shows as soon as a + * drawing is created to replace the original smart draw popup. 2. From the context menu to make + * the regenerate popup show by user command. + */ + @action + displayRegenerate = (x: number, y: number) => { + this._selectedDoc = DocumentView.SelectedDocs()?.lastElement(); + [this._pageX, this._pageY] = [x, y]; + this._display = false; + this.ShowRegenerate = true; + this._showEditBox = false; + const docData = this._selectedDoc[DocData]; + this._lastResponse = StrCast(docData.drawingData); + this._lastInput = { text: StrCast(docData.drawingInput), complexity: NumCast(docData.drawingComplexity), size: NumCast(docData.drawingSize), autoColor: BoolCast(docData.drawingColored), x: this._pageX, y: this._pageY }; + }; + + /** + * Hides the smart draw handler and resets its fields to their default. + */ + @action + hideSmartDrawHandler = () => { + if (this._display) { + this.ShowRegenerate = false; + this._display = false; + this._isLoading = false; + this._showOptions = false; + this._userInput = ''; + this._complexity = 5; + this._size = 350; + this._autoColor = true; + Doc.ActiveTool = InkTool.None; + } + }; + + /** + * Hides the popup that allows users to regenerate a drawing and resets its corresponding fields. + */ + @action + hideRegenerate = () => { + if (!this._isLoading) { + this.ShowRegenerate = false; + this._isLoading = false; + this._regenInput = ''; + this._lastInput = { text: '', complexity: 5, size: 350, autoColor: true, x: 0, y: 0 }; + } + }; + + /** + * This allows users to press the return/enter key to send input. + */ + handleKeyPress = (event: React.KeyboardEvent) => { + if (event.key === 'Enter') { + this.handleSendClick(); + } + }; + + /** + * This is called when a user hits "send" on the draw with GPT popup. It calls the drawWithGPT or regenerate + * functions depending on what mode is currently displayed, then sets various observable fields that facilitate + * what the user sees. + */ + @action + handleSendClick = async () => { + this._isLoading = true; + this._canInteract = false; + if (this.ShowRegenerate) { + await this.regenerate(); + runInAction(() => { + this._regenInput = ''; + this._showEditBox = false; + }); + } else { + runInAction(() => { + this._showOptions = false; + }); + try { + await this.drawWithGPT({ X: this._pageX, Y: this._pageY }, this._userInput, this._complexity, this._size, this._autoColor); + this.hideSmartDrawHandler(); + + runInAction(() => { + this.ShowRegenerate = true; + }); + } catch (err) { + if (this._errorOccurredOnce) { + console.error('GPT call failed', err); + this._errorOccurredOnce = false; + } else { + this._errorOccurredOnce = true; + await this.drawWithGPT({ X: this._pageX, Y: this._pageY }, this._userInput, this._complexity, this._size, this._autoColor); + } + } + } + runInAction(() => { + this._isLoading = false; + this._canInteract = true; + }); + }; + + /** + * Calls GPT API to create a drawing based on user input. + */ + @action + drawWithGPT = async (startPt: { X: number; Y: number }, input: string, complexity: number, size: number, autoColor: boolean) => { + if (input === '') return; + this._lastInput = { text: input, complexity: complexity, size: size, autoColor: autoColor, x: startPt.X, y: startPt.Y }; + const res = await gptAPICall(`"${input}", "${complexity}", "${size}"`, GPTCallType.DRAW, undefined, true); + if (!res) { + console.error('GPT call failed'); + return; + } + const strokeData = await this.parseSvg(res, startPt, false, autoColor); + const drawingDoc = strokeData && this.CreateDrawingDoc(strokeData.data, strokeData.lastInput, strokeData.lastRes); + drawingDoc && this.AddDrawing(drawingDoc, this._lastInput, res); + + this._errorOccurredOnce = false; + return strokeData; + }; + + /** + * Regenerates drawings with the option to add a specific regenerate prompt/request. + */ + @action + regenerate = async (lastInput?: DrawingOptions, lastResponse?: string, regenInput?: string) => { + if (lastInput) this._lastInput = lastInput; + if (lastResponse) this._lastResponse = lastResponse; + if (regenInput) this._regenInput = regenInput; + + try { + let res; + if (this._regenInput !== '') { + const prompt: string = `This is your previously generated svg code: ${this._lastResponse} for the user input "${this._lastInput.text}". Please regenerate it with the provided specifications.`; + res = await gptAPICall(`"${this._regenInput}"`, GPTCallType.DRAW, prompt, true); + this._lastInput.text = `${this._lastInput.text} ~~~ ${this._regenInput}`; + } else { + res = await gptAPICall(`"${this._lastInput.text}", "${this._lastInput.complexity}", "${this._lastInput.size}"`, GPTCallType.DRAW, undefined, true); + } + if (!res) { + console.error('GPT call failed'); + return; + } + const strokeData = await this.parseSvg(res, { X: this._lastInput.x, Y: this._lastInput.y }, true, lastInput?.autoColor || this._autoColor); + this.RemoveDrawing !== unimplementedFunction && this.RemoveDrawing(true, this._selectedDoc); + const drawingDoc = strokeData && this.CreateDrawingDoc(strokeData.data, strokeData.lastInput, strokeData.lastRes); + drawingDoc && this.AddDrawing(drawingDoc, this._lastInput, res); + return strokeData; + } catch (err) { + console.error('Error regenerating drawing', err); + } + }; + + /** + * Parses the svg code that GPT returns into Bezier curves, with coordinates and colors. + */ + @action + parseSvg = async (res: string, startPoint: { X: number; Y: number }, regenerate: boolean, autoColor: boolean) => { + const svg = res.match(/<svg[^>]*>([\s\S]*?)<\/svg>/g); + if (svg) { + this._lastResponse = svg[0]; + const svgObject = await parse(svg[0]); + const svgStrokes: INode[] = svgObject.children; + const strokeData: [InkData, string, string][] = []; + svgStrokes.forEach(child => { + const convertedBezier: InkData = SVGToBezier(child.name as SVGType, child.attributes); + strokeData.push([ + convertedBezier.map(point => ({ X: point.X + startPoint.X - this._size / 1.5, Y: point.Y + startPoint.Y - this._size / 2 })), + (regenerate ? this._lastInput.autoColor : autoColor) ? child.attributes.stroke : '', + (regenerate ? this._lastInput.autoColor : autoColor) ? child.attributes.fill : '', + ]); + }); + return { data: strokeData, lastInput: this._lastInput, lastRes: svg[0] }; + } + }; + + /** + * Sends request to GPT API to recolor a selected ink document or group of ink documents. + */ + colorWithGPT = async (drawing: Doc) => { + const img = await DocumentView.GetDocImage(drawing); + const { href } = ImageCast(img).url; + const hrefParts = href.split('.'); + const hrefComplete = `${hrefParts[0]}_o.${hrefParts[1]}`; + try { + const hrefBase64 = await CollectionCardView.imageUrlToBase64(hrefComplete); + const strokes = DocListCast(drawing[DocData].data); + const coords: string[] = []; + strokes.forEach((stroke, i) => { + const inkingStroke = DocumentView.getDocumentView(stroke)?.ComponentView as InkingStroke; + const { inkData } = inkingStroke.inkScaledData(); + coords.push(`${i + 1}. ${inkData.filter((point, index) => index % 4 === 0 || index == inkData.length - 1).map(point => `(${point.X.toString()}, ${point.Y.toString()})`)}`); + }); + const colorResponse = await gptDrawingColor(hrefBase64, coords).then(response => gptAPICall(response, GPTCallType.COLOR, undefined)); + this.colorStrokes(colorResponse, drawing); + } catch (error) { + console.log('GPT call failed', error); + } + }; + + /** + * Function that parses the GPT color response and sets the selected stroke(s) to the new color. + */ + colorStrokes = undoable((res: string, drawing: Doc) => { + const colorList = res.match(/\{.*?\}/g); + const strokes = DocListCast(drawing[DocData].data); + colorList?.forEach((colors, index) => { + const strokeAndFill = colors.match(/#[0-9A-Fa-f]{6}/g); + if (strokeAndFill && strokeAndFill.length == 2) { + strokes[index][DocData].color = strokeAndFill[0]; + const inkStroke = DocumentView.getDocumentView(strokes[index])?.ComponentView as InkingStroke; + const { inkData } = inkStroke.inkScaledData(); + InkingStroke.IsClosed(inkData) ? (strokes[index][DocData].fillColor = strokeAndFill[1]) : (strokes[index][DocData].fillColor = undefined); + } + }); + }, 'color strokes'); + + renderDisplay() { + return ( + <div + id="label-handler" + className="smart-draw-handler" + style={{ + display: this._display ? '' : 'none', + left: this._pageX, + ...(this._yRelativeToTop ? { top: Math.max(0, this._pageY) } : { bottom: this._pageY }), + background: SettingsManager.userBackgroundColor, + color: SettingsManager.userColor, + }}> + <div> + <IconButton + tooltip="Cancel" + onClick={() => { + this.hideSmartDrawHandler(); + this.hideRegenerate(); + }} + icon={<FontAwesomeIcon icon="xmark" />} + color={SettingsManager.userColor} + /> + <input + aria-label="Smart Draw Input" + className="smartdraw-input" + type="text" + autoFocus + value={this._userInput} + onChange={action(e => this._canInteract && (this._userInput = e.target.value))} + placeholder="Enter item to draw" + onKeyDown={this.handleKeyPress} + /> + <IconButton tooltip="Advanced Options" icon={<FontAwesomeIcon icon={this._showOptions ? 'caret-down' : 'caret-right'} />} color={SettingsManager.userColor} onClick={action(() => (this._showOptions = !this._showOptions))} /> + <Button + style={{ alignSelf: 'flex-end' }} + text="Send" + icon={this._isLoading ? <ReactLoading type="spin" color={SettingsManager.userVariantColor} width={16} height={20} /> : <AiOutlineSend />} + iconPlacement="right" + color={SettingsManager.userColor} + onClick={this.handleSendClick} + /> + </div> + {this._showOptions && ( + <div className="smartdraw-options"> + <div className="auto-color"> + Auto color + <Switch + sx={{ + '& .MuiSwitch-switchBase.Mui-checked': { color: SettingsManager.userColor }, + '& .MuiSwitch-switchBase.Mui-checked + .MuiSwitch-track': { backgroundColor: SettingsManager.userVariantColor }, + }} + defaultChecked={true} + value={this._autoColor} + size="small" + onChange={action(() => this._canInteract && (this._autoColor = !this._autoColor))} + /> + </div> + <div className="complexity"> + Complexity + <Slider + sx={{ + '& .MuiSlider-track': { color: SettingsManager.userVariantColor }, + '& .MuiSlider-rail': { color: SettingsManager.userColor }, + '& .MuiSlider-thumb': { color: SettingsManager.userColor, '&.Mui-focusVisible, &:hover, &.Mui-active': { boxShadow: `0px 0px 0px 8px${SettingsManager.userColor.slice(0, 7)}10` } }, + }} + style={{ width: '80%' }} + min={1} + max={10} + step={1} + size="small" + value={this._complexity} + onChange={action((e, val) => this._canInteract && (this._complexity = val as number))} + valueLabelDisplay="auto" + /> + </div> + <div className="size"> + Size (in pixels) + <Slider + className="size-slider" + sx={{ + '& .MuiSlider-track': { color: SettingsManager.userVariantColor }, + '& .MuiSlider-rail': { color: SettingsManager.userColor }, + '& .MuiSlider-thumb': { color: SettingsManager.userColor, '&.Mui-focusVisible, &:hover, &.Mui-active': { boxShadow: `0px 0px 0px 8px${SettingsManager.userColor.slice(0, 7)}20` } }, + }} + min={50} + max={700} + step={10} + size="small" + value={this._size} + onChange={action((e, val) => this._canInteract && (this._size = val as number))} + valueLabelDisplay="auto" + /> + </div> + </div> + )} + </div> + ); + } + + renderRegenerate() { + return ( + <div + className="smart-draw-handler" + style={{ + left: this._pageX, + ...(this._yRelativeToTop ? { top: Math.max(0, this._pageY) } : { bottom: this._pageY }), + background: SettingsManager.userBackgroundColor, + color: SettingsManager.userColor, + }}> + <div className="regenerate-box"> + <IconButton + tooltip="Regenerate" + icon={this._isLoading && this._regenInput === '' ? <ReactLoading type="spin" color={SettingsManager.userVariantColor} width={16} height={20} /> : <FontAwesomeIcon icon={'rotate'} />} + color={SettingsManager.userColor} + onClick={this.handleSendClick} + /> + <IconButton tooltip="Edit with GPT" icon={<FontAwesomeIcon icon="pen-to-square" />} color={SettingsManager.userColor} onClick={action(() => (this._showEditBox = !this._showEditBox))} /> + {this._showEditBox && ( + <div className="edit-box"> + <input + aria-label="Edit instructions input" + className="smartdraw-input" + type="text" + value={this._regenInput} + onChange={action(e => this._canInteract && (this._regenInput = e.target.value))} + onKeyDown={this.handleKeyPress} + placeholder="Edit instructions" + /> + <Button + style={{ alignSelf: 'flex-end' }} + text="Send" + icon={this._isLoading && this._regenInput !== '' ? <ReactLoading type="spin" color={SettingsManager.userVariantColor} width={16} height={20} /> : <AiOutlineSend />} + iconPlacement="right" + color={SettingsManager.userColor} + onClick={this.handleSendClick} + /> + </div> + )} + </div> + </div> + ); + } + + render() { + return this._display ? this.renderDisplay() : this.ShowRegenerate ? this.renderRegenerate() : null; + } +} |