path: root/src/client/views/smartdraw/SmartDrawHandler.tsx

import { FontAwesomeIcon } from '@fortawesome/react-fontawesome';
import { Slider, Switch } from '@mui/material';
import { Button, IconButton } from 'browndash-components';
import { action, makeObservable, observable, runInAction } from 'mobx';
import { observer } from 'mobx-react';
import React from 'react';
import { AiOutlineSend } from 'react-icons/ai';
import ReactLoading from 'react-loading';
import { INode, parse } from 'svgson';
import { unimplementedFunction } from '../../../Utils';
import { Doc, DocListCast } from '../../../fields/Doc';
import { DocData } from '../../../fields/DocSymbols';
import { InkData, InkField, InkTool } from '../../../fields/InkField';
import { BoolCast, ImageCast, NumCast, StrCast } from '../../../fields/Types';
import { GPTCallType, gptAPICall, gptDrawingColor } from '../../apis/gpt/GPT';
import { Docs } from '../../documents/Documents';
import { SettingsManager } from '../../util/SettingsManager';
import { undoable } from '../../util/UndoManager';
import { SVGToBezier, SVGType } from '../../util/bezierFit';
import { InkingStroke } from '../InkingStroke';
import { ObservableReactComponent } from '../ObservableReactComponent';
import { CollectionCardView } from '../collections/CollectionCardDeckView';
import { MarqueeView } from '../collections/collectionFreeForm';
import { ActiveArrowEnd, ActiveArrowStart, ActiveDash, ActiveFillColor, ActiveInkBezierApprox, ActiveInkColor, ActiveInkWidth, ActiveIsInkMask, DocumentView } from '../nodes/DocumentView';
import './SmartDrawHandler.scss';

export interface DrawingOptions {
    text: string;
    complexity: number;
    size: number;
    autoColor: boolean;
    x: number;
    y: number;
}

/**
 * The SmartDrawHandler allows users to generate drawings with GPT from text input. Users are able to enter
 * the item to draw, how complex they want the drawing to be, how large the drawing should be, and whether
 * it will be colored. If the drawing is colored, GPT will automatically define the stroke and fill of each
 * stroke. Drawings are retrieved from GPT as SVG code then converted into Dash-supported Beziers.
 *
 * The handler is selected from the ink tools menu. To generate a drawing, users can click anywhere on the freeform
 * canvas and a popup will appear that prompts them to create a drawing. Once the drawing is created, users have
 * the option to regenerate or edit the drawing.
 *
 * When each drawing is created, it is added to Dash as a group of ink strokes. The group is tagged with metadata
 * for user input, the drawing's SVG code, and its settings (size, complexity). In the context menu -> 'Options',
 * users can then show the drawing editor and regenerate/edit them at any point in the future.
 */

@observer
export class SmartDrawHandler extends ObservableReactComponent<object> {
    // eslint-disable-next-line no-use-before-define
    static Instance: SmartDrawHandler;

    private _lastInput: DrawingOptions = { text: '', complexity: 5, size: 350, autoColor: true, x: 0, y: 0 };
    private _lastResponse: string = '';
    private _selectedDoc: Doc | undefined = undefined;
    private _errorOccurredOnce = false;

    @observable private _display: boolean = false;
    @observable private _pageX: number = 0;
    @observable private _pageY: number = 0;
    @observable private _yRelativeToTop: boolean = true;
    @observable private _isLoading: boolean = false;
    @observable private _userInput: string = '';
    @observable private _showOptions: boolean = false;
    @observable private _showEditBox: boolean = false;
    @observable private _complexity: number = 5;
    @observable private _size: number = 200;
    @observable private _autoColor: boolean = true;
    @observable private _regenInput: string = '';
    @observable private _canInteract: boolean = true;

    @observable public ShowRegenerate: boolean = false;

    constructor(props: object) {
        super(props);
        makeObservable(this);
        SmartDrawHandler.Instance = this;
    }

    /**
     *  AddDrawing and RemoveDrawing are defined by the other classes that call the smart draw functions (i.e. 
        CollectionFreeForm, FormattedTextBox, AnnotationPalette) to define how a drawing document should be added
        or removed in their respective locations (to the freeform canvs, to the annotation palette's preview, etc.)
    */
    public AddDrawing: (doc: Doc, opts: DrawingOptions, gptRes: string) => void = unimplementedFunction;
    public RemoveDrawing: (useLastContainer: boolean, doc?: Doc) => void = unimplementedFunction;
    /**
     * This creates the ink document that represents a drawing, so it goes through the strokes that make up the drawing,
     * creates ink documents for each stroke, then adds the strokes to a collection. This can also be redefined by other
     * classes to customize the way the drawing docs get created. For example, the freeform canvas has a different way of
     * defining document bounds, so CreateDrawingDoc is redefined when that class calls gpt draw functions.
     */
    public CreateDrawingDoc: (strokeList: [InkData, string, string][], opts: DrawingOptions, gptRes: string, containerDoc?: Doc) => Doc | undefined = (strokeList: [InkData, string, string][], opts: DrawingOptions) => {
        const drawing: Doc[] = [];
        strokeList.forEach((stroke: [InkData, string, string]) => {
            const bounds = InkField.getBounds(stroke[0]);
            const inkWidth = Math.min(5, ActiveInkWidth());
            const inkDoc = Docs.Create.InkDocument(
                stroke[0],
                { title: 'stroke',
                        x: bounds.left - inkWidth / 2,
                        y: bounds.top - inkWidth / 2,
                        _width: bounds.width + inkWidth,
                        _height: bounds.height + inkWidth,
                        stroke_showLabel: BoolCast(Doc.UserDoc().activeInkHideTextLabels)}, // prettier-ignore
                inkWidth,
                opts.autoColor ? stroke[1] : ActiveInkColor(),
                ActiveInkBezierApprox(),
                stroke[2] === 'none' ? ActiveFillColor() : stroke[2],
                ActiveArrowStart(),
                ActiveArrowEnd(),
                ActiveDash(),
                ActiveIsInkMask()
            );
            drawing.push(inkDoc);
        });

        return MarqueeView.getCollection(drawing, undefined, true, { left: 1, top: 1, width: 1, height: 1 });
    };

    @action
    displaySmartDrawHandler = (x: number, y: number) => {
        [this._pageX, this._pageY] = [x, y];
        this._display = true;
    };

    /**
     * This is called in two places: 1. In this class, where the regenerate popup shows as soon as a
     * drawing is created to replace the original smart draw popup. 2. From the context menu to make
     * the regenerate popup show by user command.
     */
    @action
    displayRegenerate = (x: number, y: number) => {
        this._selectedDoc = DocumentView.SelectedDocs()?.lastElement();
        [this._pageX, this._pageY] = [x, y];
        this._display = false;
        this.ShowRegenerate = true;
        this._showEditBox = false;
        const docData = this._selectedDoc[DocData];
        this._lastResponse = StrCast(docData.drawingData);
        this._lastInput = { text: StrCast(docData.drawingInput), complexity: NumCast(docData.drawingComplexity), size: NumCast(docData.drawingSize), autoColor: BoolCast(docData.drawingColored), x: this._pageX, y: this._pageY };
    };

    /**
     * Hides the smart draw handler and resets its fields to their default.
     */
    @action
    hideSmartDrawHandler = () => {
        if (this._display) {
            this.ShowRegenerate = false;
            this._display = false;
            this._isLoading = false;
            this._showOptions = false;
            this._userInput = '';
            this._complexity = 5;
            this._size = 350;
            this._autoColor = true;
            Doc.ActiveTool = InkTool.None;
        }
    };

    /**
     * Hides the popup that allows users to regenerate a drawing and resets its corresponding fields.
     */
    @action
    hideRegenerate = () => {
        if (!this._isLoading) {
            this.ShowRegenerate = false;
            this._isLoading = false;
            this._regenInput = '';
            this._lastInput = { text: '', complexity: 5, size: 350, autoColor: true, x: 0, y: 0 };
        }
    };

    /**
     * This allows users to press the return/enter key to send input.
     */
    handleKeyPress = (event: React.KeyboardEvent) => {
        if (event.key === 'Enter') {
            this.handleSendClick();
        }
    };

    /**
     * This is called when a user hits "send" on the draw with GPT popup. It calls the drawWithGPT or regenerate
     * functions depending on what mode is currently displayed, then sets various observable fields that facilitate
     * what the user sees.
     */
    @action
    handleSendClick = async () => {
        this._isLoading = true;
        this._canInteract = false;
        if (this.ShowRegenerate) {
            await this.regenerate();
            runInAction(() => {
                this._regenInput = '';
                this._showEditBox = false;
            });
        } else {
            runInAction(() => {
                this._showOptions = false;
            });
            try {
                await this.drawWithGPT({ X: this._pageX, Y: this._pageY }, this._userInput, this._complexity, this._size, this._autoColor);
                this.hideSmartDrawHandler();

                runInAction(() => {
                    this.ShowRegenerate = true;
                });
            } catch (err) {
                if (this._errorOccurredOnce) {
                    console.error('GPT call failed', err);
                    this._errorOccurredOnce = false;
                } else {
                    this._errorOccurredOnce = true;
                    await this.drawWithGPT({ X: this._pageX, Y: this._pageY }, this._userInput, this._complexity, this._size, this._autoColor);
                }
            }
        }
        runInAction(() => {
            this._isLoading = false;
            this._canInteract = true;
        });
    };

    /**
     * Calls GPT API to create a drawing based on user input.
     */
    @action
    drawWithGPT = async (startPt: { X: number; Y: number }, input: string, complexity: number, size: number, autoColor: boolean) => {
        if (input === '') return;
        this._lastInput = { text: input, complexity: complexity, size: size, autoColor: autoColor, x: startPt.X, y: startPt.Y };
        const res = await gptAPICall(`"${input}", "${complexity}", "${size}"`, GPTCallType.DRAW, undefined, true);
        if (!res) {
            console.error('GPT call failed');
            return;
        }
        const strokeData = await this.parseSvg(res, startPt, false, autoColor);
        const drawingDoc = strokeData && this.CreateDrawingDoc(strokeData.data, strokeData.lastInput, strokeData.lastRes);
        drawingDoc && this.AddDrawing(drawingDoc, this._lastInput, res);

        this._errorOccurredOnce = false;
        return strokeData;
    };

    /**
     * Regenerates drawings with the option to add a specific regenerate prompt/request.
     */
    @action
    regenerate = async (lastInput?: DrawingOptions, lastResponse?: string, regenInput?: string) => {
        if (lastInput) this._lastInput = lastInput;
        if (lastResponse) this._lastResponse = lastResponse;
        if (regenInput) this._regenInput = regenInput;

        try {
            let res;
            if (this._regenInput !== '') {
                const prompt: string = `This is your previously generated svg code: ${this._lastResponse} for the user input "${this._lastInput.text}". Please regenerate it with the provided specifications.`;
                res = await gptAPICall(`"${this._regenInput}"`, GPTCallType.DRAW, prompt, true);
                this._lastInput.text = `${this._lastInput.text} ~~~ ${this._regenInput}`;
            } else {
                res = await gptAPICall(`"${this._lastInput.text}", "${this._lastInput.complexity}", "${this._lastInput.size}"`, GPTCallType.DRAW, undefined, true);
            }
            if (!res) {
                console.error('GPT call failed');
                return;
            }
            const strokeData = await this.parseSvg(res, { X: this._lastInput.x, Y: this._lastInput.y }, true, lastInput?.autoColor || this._autoColor);
            this.RemoveDrawing !== unimplementedFunction && this.RemoveDrawing(true, this._selectedDoc);
            const drawingDoc = strokeData && this.CreateDrawingDoc(strokeData.data, strokeData.lastInput, strokeData.lastRes);
            drawingDoc && this.AddDrawing(drawingDoc, this._lastInput, res);
            return strokeData;
        } catch (err) {
            console.error('Error regenerating drawing', err);
        }
    };

    /**
     * Parses the svg code that GPT returns into Bezier curves, with coordinates and colors.
     */
    @action
    parseSvg = async (res: string, startPoint: { X: number; Y: number }, regenerate: boolean, autoColor: boolean) => {
        const svg = res.match(/<svg[^>]*>([\s\S]*?)<\/svg>/g);
        if (svg) {
            this._lastResponse = svg[0];
            const svgObject = await parse(svg[0]);
            const svgStrokes: INode[] = svgObject.children;
            const strokeData: [InkData, string, string][] = [];
            svgStrokes.forEach(child => {
                const convertedBezier: InkData = SVGToBezier(child.name as SVGType, child.attributes);
                strokeData.push([
                    convertedBezier.map(point => ({ X: point.X + startPoint.X - this._size / 1.5, Y: point.Y + startPoint.Y - this._size / 2 })),
                    (regenerate ? this._lastInput.autoColor : autoColor) ? child.attributes.stroke : '',
                    (regenerate ? this._lastInput.autoColor : autoColor) ? child.attributes.fill : '',
                ]);
            });
            return { data: strokeData, lastInput: this._lastInput, lastRes: svg[0] };
        }
    };

    /**
     * Sends request to GPT API to recolor a selected ink document or group of ink documents.
     */
    colorWithGPT = async (drawing: Doc) => {
        const img = await DocumentView.GetDocImage(drawing);
        const { href } = ImageCast(img).url;
        const hrefParts = href.split('.');
        const hrefComplete = `${hrefParts[0]}_o.${hrefParts[1]}`;
        try {
            const hrefBase64 = await CollectionCardView.imageUrlToBase64(hrefComplete);
            const strokes = DocListCast(drawing[DocData].data);
            const coords: string[] = [];
            strokes.forEach((stroke, i) => {
                const inkingStroke = DocumentView.getDocumentView(stroke)?.ComponentView as InkingStroke;
                const { inkData } = inkingStroke.inkScaledData();
                coords.push(`${i + 1}. ${inkData.filter((point, index) => index % 4 === 0 || index == inkData.length - 1).map(point => `(${point.X.toString()}, ${point.Y.toString()})`)}`);
            });
            const colorResponse = await gptDrawingColor(hrefBase64, coords).then(response => gptAPICall(response, GPTCallType.COLOR, undefined));
            this.colorStrokes(colorResponse, drawing);
        } catch (error) {
            console.log('GPT call failed', error);
        }
    };

    /**
     * Function that parses the GPT color response and sets the selected stroke(s) to the new color.
     */
    colorStrokes = undoable((res: string, drawing: Doc) => {
        const colorList = res.match(/\{.*?\}/g);
        const strokes = DocListCast(drawing[DocData].data);
        colorList?.forEach((colors, index) => {
            const strokeAndFill = colors.match(/#[0-9A-Fa-f]{6}/g);
            if (strokeAndFill && strokeAndFill.length == 2) {
                strokes[index][DocData].color = strokeAndFill[0];
                const inkStroke = DocumentView.getDocumentView(strokes[index])?.ComponentView as InkingStroke;
                const { inkData } = inkStroke.inkScaledData();
                InkingStroke.IsClosed(inkData) ? (strokes[index][DocData].fillColor = strokeAndFill[1]) : (strokes[index][DocData].fillColor = undefined);
            }
        });
    }, 'color strokes');

    renderDisplay() {
        return (
            <div
                id="label-handler"
                className="smart-draw-handler"
                style={{
                    display: this._display ? '' : 'none',
                    left: this._pageX,
                    ...(this._yRelativeToTop ? { top: Math.max(0, this._pageY) } : { bottom: this._pageY }),
                    background: SettingsManager.userBackgroundColor,
                    color: SettingsManager.userColor,
                }}>
                <div>
                    <IconButton
                        tooltip="Cancel"
                        onClick={() => {
                            this.hideSmartDrawHandler();
                            this.hideRegenerate();
                        }}
                        icon={<FontAwesomeIcon icon="xmark" />}
                        color={SettingsManager.userColor}
                    />
                    <input
                        aria-label="Smart Draw Input"
                        className="smartdraw-input"
                        type="text"
                        autoFocus
                        value={this._userInput}
                        onChange={action(e => this._canInteract && (this._userInput = e.target.value))}
                        placeholder="Enter item to draw"
                        onKeyDown={this.handleKeyPress}
                    />
                    <IconButton tooltip="Advanced Options" icon={<FontAwesomeIcon icon={this._showOptions ? 'caret-down' : 'caret-right'} />} color={SettingsManager.userColor} onClick={action(() => (this._showOptions = !this._showOptions))} />
                    <Button
                        style={{ alignSelf: 'flex-end' }}
                        text="Send"
                        icon={this._isLoading ? <ReactLoading type="spin" color={SettingsManager.userVariantColor} width={16} height={20} /> : <AiOutlineSend />}
                        iconPlacement="right"
                        color={SettingsManager.userColor}
                        onClick={this.handleSendClick}
                    />
                </div>
                {this._showOptions && (
                    <div className="smartdraw-options">
                        <div className="auto-color">
                            Auto color
                            <Switch
                                sx={{
                                    '& .MuiSwitch-switchBase.Mui-checked': { color: SettingsManager.userColor },
                                    '& .MuiSwitch-switchBase.Mui-checked + .MuiSwitch-track': { backgroundColor: SettingsManager.userVariantColor },
                                }}
                                defaultChecked={true}
                                value={this._autoColor}
                                size="small"
                                onChange={action(() => this._canInteract && (this._autoColor = !this._autoColor))}
                            />
                        </div>
                        <div className="complexity">
                            Complexity
                            <Slider
                                sx={{
                                    '& .MuiSlider-track': { color: SettingsManager.userVariantColor },
                                    '& .MuiSlider-rail': { color: SettingsManager.userColor },
                                    '& .MuiSlider-thumb': { color: SettingsManager.userColor, '&.Mui-focusVisible, &:hover, &.Mui-active': { boxShadow: `0px 0px 0px 8px${SettingsManager.userColor.slice(0, 7)}10` } },
                                }}
                                style={{ width: '80%' }}
                                min={1}
                                max={10}
                                step={1}
                                size="small"
                                value={this._complexity}
                                onChange={action((e, val) => this._canInteract && (this._complexity = val as number))}
                                valueLabelDisplay="auto"
                            />
                        </div>
                        <div className="size">
                            Size (in pixels)
                            <Slider
                                className="size-slider"
                                sx={{
                                    '& .MuiSlider-track': { color: SettingsManager.userVariantColor },
                                    '& .MuiSlider-rail': { color: SettingsManager.userColor },
                                    '& .MuiSlider-thumb': { color: SettingsManager.userColor, '&.Mui-focusVisible, &:hover, &.Mui-active': { boxShadow: `0px 0px 0px 8px${SettingsManager.userColor.slice(0, 7)}20` } },
                                }}
                                min={50}
                                max={700}
                                step={10}
                                size="small"
                                value={this._size}
                                onChange={action((e, val) => this._canInteract && (this._size = val as number))}
                                valueLabelDisplay="auto"
                            />
                        </div>
                    </div>
                )}
            </div>
        );
    }

    renderRegenerate() {
        return (
            <div
                className="smart-draw-handler"
                style={{
                    left: this._pageX,
                    ...(this._yRelativeToTop ? { top: Math.max(0, this._pageY) } : { bottom: this._pageY }),
                    background: SettingsManager.userBackgroundColor,
                    color: SettingsManager.userColor,
                }}>
                <div className="regenerate-box">
                    <IconButton
                        tooltip="Regenerate"
                        icon={this._isLoading && this._regenInput === '' ? <ReactLoading type="spin" color={SettingsManager.userVariantColor} width={16} height={20} /> : <FontAwesomeIcon icon={'rotate'} />}
                        color={SettingsManager.userColor}
                        onClick={this.handleSendClick}
                    />
                    <IconButton tooltip="Edit with GPT" icon={<FontAwesomeIcon icon="pen-to-square" />} color={SettingsManager.userColor} onClick={action(() => (this._showEditBox = !this._showEditBox))} />
                    {this._showEditBox && (
                        <div className="edit-box">
                            <input
                                aria-label="Edit instructions input"
                                className="smartdraw-input"
                                type="text"
                                value={this._regenInput}
                                onChange={action(e => this._canInteract && (this._regenInput = e.target.value))}
                                onKeyDown={this.handleKeyPress}
                                placeholder="Edit instructions"
                            />
                            <Button
                                style={{ alignSelf: 'flex-end' }}
                                text="Send"
                                icon={this._isLoading && this._regenInput !== '' ? <ReactLoading type="spin" color={SettingsManager.userVariantColor} width={16} height={20} /> : <AiOutlineSend />}
                                iconPlacement="right"
                                color={SettingsManager.userColor}
                                onClick={this.handleSendClick}
                            />
                        </div>
                    )}
                </div>
            </div>
        );
    }

    render() {
        return this._display ? this.renderDisplay() : this.ShowRegenerate ? this.renderRegenerate() : null;
    }
}