From 692076b1356309111c4f2cb69cbdbf4be1a825bd Mon Sep 17 00:00:00 2001 From: eleanor-park Date: Sun, 22 Sep 2024 15:40:19 -0400 Subject: small bug fixes for smart draw --- src/client/util/Scripting.ts | 4 +- src/client/util/bezierFit.ts | 4 +- src/client/views/DocumentButtonBar.tsx | 15 ---- src/client/views/InkStrokeProperties.ts | 35 ++++------ src/client/views/PropertiesView.tsx | 6 +- .../collectionFreeForm/CollectionFreeFormView.tsx | 16 +++-- src/client/views/smartdraw/AnnotationPalette.scss | 46 +++++++++++++ src/client/views/smartdraw/AnnotationPalette.tsx | 56 ++++++++++----- src/client/views/smartdraw/SmartDrawHandler.scss | 41 +++++++++++ src/client/views/smartdraw/SmartDrawHandler.tsx | 80 ++++++++++++++++------ 10 files changed, 213 insertions(+), 90 deletions(-) (limited to 'src') diff --git a/src/client/util/Scripting.ts b/src/client/util/Scripting.ts index c63d3d7cb..cb314e3f1 100644 --- a/src/client/util/Scripting.ts +++ b/src/client/util/Scripting.ts @@ -1,7 +1,7 @@ // export const ts = (window as any).ts; // import * as typescriptlib from '!!raw-loader!../../../node_modules/typescript/lib/lib.d.ts' // import * as typescriptes5 from '!!raw-loader!../../../node_modules/typescript/lib/lib.es5.d.ts' -import typescriptlib from 'type_decls.d'; +// import typescriptlib from 'type_decls.d'; import * as ts from 'typescript'; import { Doc, FieldType } from '../../fields/Doc'; import { RefField } from '../../fields/RefField'; @@ -248,7 +248,7 @@ export function CompileScript(script: string, options: ScriptOptions = {}): Comp const funcScript = `(function(${paramString})${reqTypes} { ${body} })`; host.writeFile('file.ts', funcScript); - if (typecheck) host.writeFile('node_modules/typescript/lib/lib.d.ts', typescriptlib); + // if (typecheck) host.writeFile('node_modules/typescript/lib/lib.d.ts', typescriptlib); const program = ts.createProgram(['file.ts'], {}, host); const testResult = program.emit(); const outputText = host.readFile('file.js'); diff --git a/src/client/util/bezierFit.ts b/src/client/util/bezierFit.ts index 4c7f4a0ba..4aef28e6b 100644 --- a/src/client/util/bezierFit.ts +++ b/src/client/util/bezierFit.ts @@ -696,7 +696,7 @@ export function SVGToBezier(name: SVGType, attributes: any): Point[] { const matches: RegExpMatchArray[] = Array.from( attributes.d.matchAll(/Q(-?\d+\.?\d*),(-?\d+\.?\d*) (-?\d+\.?\d*),(-?\d+\.?\d*)|C(-?\d+\.?\d*),(-?\d+\.?\d*) (-?\d+\.?\d*),(-?\d+\.?\d*) (-?\d+\.?\d*),(-?\d+\.?\d*)|L(-?\d+\.?\d*),(-?\d+\.?\d*)/g) ); - let lastPt: Point = { X: 0, Y: 0 }; + let lastPt: Point = startPt; matches.forEach(match => { if (match[0].startsWith('Q')) { coordList.push({ X: parseInt(match[1]), Y: parseInt(match[2]) }); @@ -711,7 +711,7 @@ export function SVGToBezier(name: SVGType, attributes: any): Point[] { coordList.push({ X: parseInt(match[9]), Y: parseInt(match[10]) }); lastPt = { X: parseInt(match[9]), Y: parseInt(match[10]) }; } else { - coordList.push(lastPt || { X: parseInt(startPt[1]), Y: parseInt(startPt[2]) }); + coordList.push(lastPt); coordList.push({ X: parseInt(match[11]), Y: parseInt(match[12]) }); coordList.push({ X: parseInt(match[11]), Y: parseInt(match[12]) }); coordList.push({ X: parseInt(match[11]), Y: parseInt(match[12]) }); diff --git a/src/client/views/DocumentButtonBar.tsx b/src/client/views/DocumentButtonBar.tsx index 096f058ad..04c1d359c 100644 --- a/src/client/views/DocumentButtonBar.tsx +++ b/src/client/views/DocumentButtonBar.tsx @@ -239,20 +239,6 @@ export class DocumentButtonBar extends ObservableReactComponent<{ views: () => ( ); } - saveAnno = undoable(async (targetDoc: Doc) => await AnnotationPalette.addToPalette(targetDoc), 'save to palette'); - - @computed - get saveAnnoButton() { - const targetDoc = this.view0?.Document; - return !targetDoc ? null : ( - {targetDoc.savedAsAnno ? 'Saved as Annotation!' : 'Save to Annotation Palette'}}> -
this.saveAnno(targetDoc)}> - -
- - ); - } - @computed get shareButton() { const targetDoc = this.view0?.Document; @@ -473,7 +459,6 @@ export class DocumentButtonBar extends ObservableReactComponent<{ views: () => (
{this.templateButton}
{!DocumentView.Selected().some(v => v.allLinks.length) ? null :
{this.followLinkButton}
}
{this.pinButton}
-
{this.saveAnnoButton}
{this.recordButton}
{this.calendarButton}
{this.keywordButton}
diff --git a/src/client/views/InkStrokeProperties.ts b/src/client/views/InkStrokeProperties.ts index 13807c25f..5cacde0d4 100644 --- a/src/client/views/InkStrokeProperties.ts +++ b/src/client/views/InkStrokeProperties.ts @@ -499,31 +499,20 @@ export class InkStrokeProperties { const inkView = DocumentView.getDocumentView(inkDoc); const inkStroke = inkView?.ComponentView as InkingStroke; const { inkData } = inkStroke.inkScaledData(); - - const result = inkData.length > 2 && GestureUtils.GestureRecognizer.Recognize([inkData]); - console.log(result); - if (result && (result.Name === 'line' ? result.Score > 0.92 : result.Score > 0.85)) { - switch (result.Name) { - case Gestures.Line: - case Gestures.Triangle: - case Gestures.Rectangle: - case Gestures.Circle: - GestureOverlay.makeBezierPolygon(inkData, result.Name, true); - break; - default: - } - } else { - const polylinePoints = inkData.filter((pt, index) => { return index % 4 === 0 || pt === inkData.lastElement()}).map(pt => { return { x: pt.X, y: pt.Y }; }); // prettier-ignore - if (polylinePoints.length > 2) { - const toKeep = simplify(polylinePoints, tolerance).map(pt => {return { X: pt.x, Y: pt.y }}); // prettier-ignore - for (var i = 4; i < inkData.length - 3; i += 4) { - const contains = toKeep.find(pt => pt.X === inkData[i].X && pt.Y === inkData[i].Y); - if (!contains) { - this._currentPoint = i; - inkView && this.deletePoints(inkView, false); - } + const polylinePoints = inkData.filter((pt, index) => { return index % 4 === 0 || pt === inkData.lastElement()}).map(pt => { return { x: pt.X, y: pt.Y }; }); // prettier-ignore + if (polylinePoints.length > 2) { + const toKeep = simplify(polylinePoints, tolerance).map(pt => {return { X: pt.x, Y: pt.y }}); // prettier-ignore + for (var i = 4; i < inkData.length - 3; i += 4) { + const contains = toKeep.find(pt => pt.X === inkData[i].X && pt.Y === inkData[i].Y); + if (!contains) { + this._currentPoint = i; + inkView && this.deletePoints(inkView, false); } } + // close the curve if the first and last points are really close (based on tolerance) + if (!InkingStroke.IsClosed(inkData) && Math.sqrt((inkData.lastElement().X - inkData[0].X) ** 2 + (inkData.lastElement().Y - inkData[0].Y) ** 2) <= tolerance * 2) { + inkData[inkData.length - 1] = inkData[0]; + } } }); }, 'smooth ink stroke'); diff --git a/src/client/views/PropertiesView.tsx b/src/client/views/PropertiesView.tsx index 9299705ee..39ed16f5d 100644 --- a/src/client/views/PropertiesView.tsx +++ b/src/client/views/PropertiesView.tsx @@ -991,12 +991,12 @@ export class PropertiesView extends ObservableReactComponent { !isNaN(val) && (this.smoothAmt = val); }, - 20, + 10, 1 )} @@ -1021,7 +1021,7 @@ export class PropertiesView extends ObservableReactComponent { + removeDrawing = (useLastContainer: boolean, doc?: Doc) => { this._batch = UndoManager.StartBatch('regenerateDrawing'); - if (doc) { + if (useLastContainer && this._drawingContainer) { + this._props.removeDocument?.(this._drawingContainer); + } else if (doc) { const docData = doc[DocData]; const children = DocListCast(docData.data); this._props.removeDocument?.(doc); this._props.removeDocument?.(children); - } else { - if (this._drawingContainer) this._props.removeDocument?.(this._drawingContainer); } this._drawing = []; }; @@ -1294,6 +1296,7 @@ export class CollectionFreeFormView extends CollectionSubView { const docData = doc[DocData]; docData.title = opts.text.match(/^(.*?)~~~.*$/)?.[1] || opts.text; + docData.width = opts.size; docData.drawingInput = opts.text; docData.drawingComplexity = opts.complexity; docData.drawingColored = opts.autoColor; @@ -2010,6 +2013,11 @@ export class CollectionFreeFormView extends CollectionSubView await AnnotationPalette.addToPalette(this.Document), 'save to palette')), + icon: this.Document.savedAsAnno ? 'clipboard-check' : 'file-arrow-down', + }); this._props.renderDepth && optionItems.push({ description: 'Use Background Color as Default', diff --git a/src/client/views/smartdraw/AnnotationPalette.scss b/src/client/views/smartdraw/AnnotationPalette.scss index 9f875f61a..4f11e8afc 100644 --- a/src/client/views/smartdraw/AnnotationPalette.scss +++ b/src/client/views/smartdraw/AnnotationPalette.scss @@ -8,3 +8,49 @@ border-radius: 5px; margin: auto; } + +.palette-create { + display: flex; + flex-direction: row; + width: 170px; + + .palette-create-input { + color: black; + width: 170px; + } +} + +.palette-create-options { + display: flex; + flex-direction: row; + justify-content: space-between; + width: 170px; + margin-top: 5px; + + .palette-color { + display: flex; + flex-direction: column; + align-items: center; + width: 40px; + } + + .palette-detail, + .palette-size { + display: flex; + flex-direction: column; + align-items: center; + width: 60px; + } +} + +.palette-buttons { + width: 100%; + display: flex; + flex-direction: row; + justify-content: space-between; +} + +.palette-save-reset { + display: flex; + flex-direction: row; +} diff --git a/src/client/views/smartdraw/AnnotationPalette.tsx b/src/client/views/smartdraw/AnnotationPalette.tsx index a2d6cc88d..0c8dbf12d 100644 --- a/src/client/views/smartdraw/AnnotationPalette.tsx +++ b/src/client/views/smartdraw/AnnotationPalette.tsx @@ -23,11 +23,22 @@ import { DocumentView, DocumentViewInternal } from '../nodes/DocumentView'; import { FieldView } from '../nodes/FieldView'; import './AnnotationPalette.scss'; import { DrawingOptions, SmartDrawHandler } from './SmartDrawHandler'; +import { ImageField } from '../../../fields/URLField'; +import { Copy } from '../../../fields/FieldSymbols'; interface AnnotationPaletteProps { Document: Doc; } +/** + * The AnnotationPalette can be toggled in the lightbox view of a document. The goal of the palette + * is to offer an easy way for users to save then drag and drop repeated annotations onto a document. + * These annotations can be of any annotation type and operate similarly to user templates. + * + * On the "add" side of the palette, there is a way to create a drawing annotation with GPT. Users can + * enter the item to draw, toggle different settings, then GPT will generate three versions of the drawing + * to choose from. These drawings can then be saved to the palette as annotations. + */ @observer export class AnnotationPalette extends ObservableReactComponent { @observable private _paletteMode: 'create' | 'view' = 'view'; @@ -107,20 +118,25 @@ export class AnnotationPalette extends ObservableReactComponent { if (!doc.savedAsAnno) { - Doc.MakeClone(doc).then(cloneMap => - DocumentView.getDocumentView(doc) - ?.ComponentView?.updateIcon?.(true) - .then(() => { - const { clone } = cloneMap; - clone.title = doc.title; - const image = ImageCast(doc.icon, ImageCast(clone[Doc.LayoutFieldKey(clone)]))?.url?.href; - Doc.AddDocToList(Doc.MyAnnos, 'data', makeUserTemplateImage(clone, image)); - doc.savedAsAnno = true; - }) - ); + const docView = DocumentView.getDocumentView(doc); + await docView?.ComponentView?.updateIcon?.(true); + const { clone } = await Doc.MakeClone(doc); + clone.title = doc.title; + const image = ImageCast(doc.icon, ImageCast(clone[Doc.LayoutFieldKey(clone)]))?.url?.href; + Doc.AddDocToList(Doc.MyAnnos, 'data', makeUserTemplateImage(clone, image)); + doc.savedAsAnno = true; } }; + public static getIcon(group: Doc) { + const docView = DocumentView.getDocumentView(group); + if (docView) { + docView.ComponentView?.updateIcon?.(true); + return new Promise(res => setTimeout(() => res(ImageCast(docView.Document.icon)), 1000)); + } + return undefined; + } + /** * Calls the draw with GPT functions in SmartDrawHandler to allow users to generate drawings straight from * the annotation palette. @@ -172,6 +188,8 @@ export class AnnotationPalette extends ObservableReactComponent -
+
{ this.setUserInput(e.target.value); @@ -228,8 +246,8 @@ export class AnnotationPalette extends ObservableReactComponent
-
-
+
+
Color this.setColor(!this._opts.autoColor)} />
-
+
Detail
-
+
Size -
+
diff --git a/src/client/views/smartdraw/SmartDrawHandler.scss b/src/client/views/smartdraw/SmartDrawHandler.scss index 6d402a80f..0e8bd3349 100644 --- a/src/client/views/smartdraw/SmartDrawHandler.scss +++ b/src/client/views/smartdraw/SmartDrawHandler.scss @@ -1,3 +1,44 @@ .smart-draw-handler { position: absolute; } + +.smartdraw-input { + color: black; +} + +.smartdraw-options { + display: flex; + flex-direction: row; + justify-content: space-around; + + .auto-color { + display: flex; + flex-direction: column; + justify-content: center; + width: 30%; + } + + .complexity { + display: flex; + flex-direction: column; + justify-content: center; + width: 31%; + } + + .size { + display: flex; + flex-direction: column; + justify-content: center; + width: 39%; + + .size-slider { + width: 80%; + } + } +} + +.regenerate-box, +.edit-box { + display: flex; + flex-direction: row; +} diff --git a/src/client/views/smartdraw/SmartDrawHandler.tsx b/src/client/views/smartdraw/SmartDrawHandler.tsx index 8271c0959..e362c0c89 100644 --- a/src/client/views/smartdraw/SmartDrawHandler.tsx +++ b/src/client/views/smartdraw/SmartDrawHandler.tsx @@ -12,7 +12,6 @@ import { Doc, DocListCast } from '../../../fields/Doc'; import { DocData } from '../../../fields/DocSymbols'; import { InkData, InkField, InkTool } from '../../../fields/InkField'; import { BoolCast, ImageCast, NumCast, StrCast } from '../../../fields/Types'; -import { ImageField } from '../../../fields/URLField'; import { GPTCallType, gptAPICall, gptDrawingColor } from '../../apis/gpt/GPT'; import { Docs } from '../../documents/Documents'; import { SettingsManager } from '../../util/SettingsManager'; @@ -34,6 +33,21 @@ export interface DrawingOptions { y: number; } +/** + * The SmartDrawHandler allows users to generate drawings with GPT from text input. Users are able to enter + * the item to draw, how complex they want the drawing to be, how large the drawing should be, and whether + * it will be colored. If the drawing is colored, GPT will automatically define the stroke and fill of each + * stroke. Drawings are retrieved from GPT as SVG code then converted into Dash-supported Beziers. + * + * The handler is selected from the ink tools menu. To generate a drawing, users can click anywhere on the freeform + * canvas and a popup will appear that prompts them to create a drawing. Once the drawing is created, users have + * the option to regenerate or edit the drawing. + * + * When each drawing is created, it is added to Dash as a group of ink strokes. The group is tagged with metadata + * for user input, the drawing's SVG code, and its settings (size, complexity). In the context menu -> 'Options', + * users can then show the drawing editor and regenerate/edit them at any point in the future. + */ + @observer export class SmartDrawHandler extends ObservableReactComponent { static Instance: SmartDrawHandler; @@ -65,8 +79,19 @@ export class SmartDrawHandler extends ObservableReactComponent { SmartDrawHandler.Instance = this; } + /** + * AddDrawing and RemoveDrawing are defined by the other classes that call the smart draw functions (i.e. + CollectionFreeForm, FormattedTextBox, AnnotationPalette) to define how a drawing document should be added + or removed in their respective locations (to the freeform canvs, to the annotation palette's preview, etc.) + */ public AddDrawing: (doc: Doc, opts: DrawingOptions, gptRes: string) => void = unimplementedFunction; - public RemoveDrawing: (doc?: Doc) => void = unimplementedFunction; + public RemoveDrawing: (useLastContainer: boolean, doc?: Doc) => void = unimplementedFunction; + /** + * This creates the ink document that represents a drawing, so it goes through the strokes that make up the drawing, + * creates ink documents for each stroke, then adds the strokes to a collection. This can also be redefined by other + * classes to customize the way the drawing docs get created. For example, the freeform canvas has a different way of + * defining document bounds, so CreateDrawingDoc is redefined when that class calls gpt draw functions. + */ public CreateDrawingDoc: (strokeList: [InkData, string, string][], opts: DrawingOptions, gptRes: string, containerDoc?: Doc) => Doc | undefined = (strokeList: [InkData, string, string][], opts: DrawingOptions) => { const drawing: Doc[] = []; strokeList.forEach((stroke: [InkData, string, string]) => { @@ -101,6 +126,11 @@ export class SmartDrawHandler extends ObservableReactComponent { this._display = true; }; + /** + * This is called in two places: 1. In this class, where the regenerate popup shows as soon as a + * drawing is created to replace the original smart draw popup. 2. From the context menu to make + * the regenerate popup show by user command. + */ @action displayRegenerate = (x: number, y: number) => { this._selectedDoc = DocumentView.SelectedDocs()?.lastElement(); @@ -113,6 +143,9 @@ export class SmartDrawHandler extends ObservableReactComponent { this._lastInput = { text: StrCast(docData.drawingInput), complexity: NumCast(docData.drawingComplexity), size: NumCast(docData.drawingSize), autoColor: BoolCast(docData.drawingColored), x: this._pageX, y: this._pageY }; }; + /** + * Hides the smart draw handler and resets its fields to their default. + */ @action hideSmartDrawHandler = () => { this.ShowRegenerate = false; @@ -126,6 +159,9 @@ export class SmartDrawHandler extends ObservableReactComponent { Doc.ActiveTool = InkTool.None; }; + /** + * Hides the popup that allows users to regenerate a drawing and resets its corresponding fields. + */ @action hideRegenerate = () => { if (!this._isLoading) { @@ -136,12 +172,20 @@ export class SmartDrawHandler extends ObservableReactComponent { } }; + /** + * This allows users to press the return/enter key to send input. + */ handleKeyPress = (event: React.KeyboardEvent) => { if (event.key === 'Enter') { this.handleSendClick(); } }; + /** + * This is called when a user hits "send" on the draw with GPT popup. It calls the drawWithGPT or regenerate + * functions depending on what mode is currently displayed, then sets various observable fields that facilitate + * what the user sees. + */ @action handleSendClick = async () => { this._isLoading = true; @@ -171,7 +215,7 @@ export class SmartDrawHandler extends ObservableReactComponent { }; /** - * Calls GPT API to create a drawing based on user input + * Calls GPT API to create a drawing based on user input. */ @action drawWithGPT = async (startPt: { X: number; Y: number }, input: string, complexity: number, size: number, autoColor: boolean) => { @@ -182,6 +226,7 @@ export class SmartDrawHandler extends ObservableReactComponent { console.error('GPT call failed'); return; } + console.log(res); const strokeData = await this.parseSvg(res, startPt, false, autoColor); const drawingDoc = strokeData && this.CreateDrawingDoc(strokeData?.data, strokeData?.lastInput, strokeData?.lastRes); drawingDoc && this.AddDrawing(drawingDoc, this._lastInput, res); @@ -213,7 +258,7 @@ export class SmartDrawHandler extends ObservableReactComponent { return; } const strokeData = await this.parseSvg(res, { X: this._lastInput.x, Y: this._lastInput.y }, true, lastInput?.autoColor || this._autoColor); - this.RemoveDrawing !== unimplementedFunction && this.RemoveDrawing(this._selectedDoc); + this.RemoveDrawing !== unimplementedFunction && this.RemoveDrawing(true, this._selectedDoc); const drawingDoc = strokeData && this.CreateDrawingDoc(strokeData?.data, strokeData?.lastInput, strokeData?.lastRes); drawingDoc && this.AddDrawing(drawingDoc, this._lastInput, res); return strokeData; @@ -223,7 +268,7 @@ export class SmartDrawHandler extends ObservableReactComponent { }; /** - * Parses the svg code that GPT returns into Bezier curves. + * Parses the svg code that GPT returns into Bezier curves, with coordinates and colors. */ @action parseSvg = async (res: string, startPoint: { X: number; Y: number }, regenerate: boolean, autoColor: boolean) => { @@ -307,26 +352,18 @@ export class SmartDrawHandler extends ObservableReactComponent { }} icon={} color={SettingsManager.userColor} - style={{ width: '19px' }} /> this._canInteract && (this._userInput = e.target.value))} placeholder="Enter item to draw" onKeyDown={this.handleKeyPress} /> - } - color={SettingsManager.userColor} - style={{ width: '14px' }} - onClick={action(() => (this._showOptions = !this._showOptions))} - /> + } color={SettingsManager.userColor} onClick={action(() => (this._showOptions = !this._showOptions))} />