From fd5278045e8c2e280d81cb965c0b2cc5afb59be8 Mon Sep 17 00:00:00 2001 From: eleanor-park Date: Wed, 7 Aug 2024 18:09:47 -0400 Subject: problem with setting smooth amount --- src/client/apis/gpt/GPT.ts | 45 +++++- src/client/cognitive_services/CognitiveServices.ts | 18 ++- src/client/views/GestureOverlay.tsx | 150 ++++++++--------- src/client/views/InkStrokeProperties.ts | 43 ++++- src/client/views/InkTranscription.tsx | 4 +- src/client/views/InkingStroke.tsx | 4 +- src/client/views/PropertiesButtons.tsx | 4 +- src/client/views/PropertiesView.scss | 6 + src/client/views/PropertiesView.tsx | 180 +++++++++++++++++++-- .../collectionFreeForm/CollectionFreeFormView.tsx | 106 ++++++------ .../collectionFreeForm/MarqueeOptionsMenu.tsx | 2 + .../collections/collectionFreeForm/MarqueeView.tsx | 80 ++++++++- .../views/nodes/formattedText/FormattedTextBox.tsx | 8 + src/client/views/pdf/AnchorMenu.tsx | 38 +++++ src/client/views/pdf/PDFViewer.tsx | 10 +- src/client/views/smartdraw/AnnotationPalette.tsx | 44 +---- src/client/views/smartdraw/SmartDrawHandler.tsx | 155 ++++++++++++++---- 17 files changed, 662 insertions(+), 235 deletions(-) (limited to 'src') diff --git a/src/client/apis/gpt/GPT.ts b/src/client/apis/gpt/GPT.ts index a780596fa..ee8d5e9b2 100644 --- a/src/client/apis/gpt/GPT.ts +++ b/src/client/apis/gpt/GPT.ts @@ -13,6 +13,7 @@ enum GPTCallType { MERMAID = 'mermaid', DATA = 'data', DRAW = 'draw', + COLOR = 'color', } type GPTCallOpts = { @@ -57,9 +58,15 @@ const callTypeMap: { [type: string]: GPTCallOpts } = { draw: { model: 'gpt-4o', maxTokens: 1024, - temp: 0.5, + temp: 0.8, prompt: 'Given an item, a level of complexity from 1-10, and a size in pixels, generate a detailed and colored line drawing representation of it. Make sure every element has the stroke field filled out. More complex drawings will have much more detail and strokes. The drawing should be in SVG format with no additional text or comments. For path coordinates, make sure you format with a comma between numbers, like M100,200 C150,250 etc. The only supported commands are line, ellipse, circle, rect, polygon, and path with M, Q, C, and L so only use those.', }, + color: { + model: 'gpt-4o', + maxTokens: 1024, + temp: 0.5, + prompt: 'You will be coloring drawings. You will be given what the drawing is, then a list of descriptions for parts of the drawing. Based on each description, respond with the stroke and fill color that it should be. Follow the rules: 1. Avoid using black for stroke color 2. Make the stroke color 1-3 shades darker than the fill color 3. Use the same colors when possible. Format as {#abcdef #abcdef}, making sure theres a color for each description, and do not include any additional text.', + }, }; let lastCall = ''; @@ -187,4 +194,38 @@ const gptHandwriting = async (src: string): Promise => { } }; -export { gptAPICall, gptImageCall, GPTCallType, gptImageLabel, gptGetEmbedding, gptHandwriting }; +const gptDrawingColor = async (image: string, coords: string[]): Promise => { + try { + const response = await openai.chat.completions.create({ + model: 'gpt-4o', + temperature: 0, + messages: [ + { + role: 'user', + content: [ + { + type: 'text', + text: `Identify what the drawing in the image represents in 1-5 words. Then, given a list of a list of coordinates, where each list is the coordinates for one stroke of the drawing, determine which part of the drawing it is. Return just what the item it is, followed by ~~~ then only your descriptions in a list like [description, description, ...]. Here are the coordinates: ${coords}`, + }, + { + type: 'image_url', + image_url: { + url: `${image}`, + detail: 'low', + }, + }, + ], + }, + ], + }); + if (response.choices[0].message.content) { + return response.choices[0].message.content; + } + return 'Missing labels'; + } catch (err) { + console.log(err); + return 'Error connecting with API'; + } +}; + +export { gptAPICall, gptImageCall, GPTCallType, gptImageLabel, gptGetEmbedding, gptHandwriting, gptDrawingColor }; diff --git a/src/client/cognitive_services/CognitiveServices.ts b/src/client/cognitive_services/CognitiveServices.ts index 9f46b8685..9f7701c54 100644 --- a/src/client/cognitive_services/CognitiveServices.ts +++ b/src/client/cognitive_services/CognitiveServices.ts @@ -47,14 +47,14 @@ export namespace CognitiveServices { const ExecuteQuery = async (service: Service, manager: APIManager, data: D): Promise => { const apiKey = process.env[service.toUpperCase()]; if (apiKey) { - console.log(data) + console.log(data); console.log(`No API key found for ${service}: ensure youe root directory has .env file with _CLIENT_${service.toUpperCase()}.`); return undefined; } let results: any; try { - results = await manager.requester("has", manager.converter(data), service).then(json => JSON.parse(json)); + results = await manager.requester('has', manager.converter(data), service).then(json => JSON.parse(json)); } catch (e) { throw e; } @@ -138,12 +138,14 @@ export namespace CognitiveServices { points: points.map(({ X: x, Y: y }) => `${x},${y}`).join(','), language: 'en-US', })); - console.log(JSON.stringify({ - version: 1, - language: 'en-US', - unit: 'mm', - strokes, - })) + console.log( + JSON.stringify({ + version: 1, + language: 'en-US', + unit: 'mm', + strokes, + }) + ); return JSON.stringify({ version: 1, language: 'en-US', diff --git a/src/client/views/GestureOverlay.tsx b/src/client/views/GestureOverlay.tsx index 649208989..7eac583dd 100644 --- a/src/client/views/GestureOverlay.tsx +++ b/src/client/views/GestureOverlay.tsx @@ -141,7 +141,7 @@ export class GestureOverlay extends ObservableReactComponent { - const xs = this._points.map(p => p.X); - const ys = this._points.map(p => p.Y); + public static makeBezierPolygon = (points: { X: number; Y: number }[], shape: string, gesture: boolean) => { + const xs = points.map(p => p.X); + const ys = points.map(p => p.Y); let right = Math.max(...xs); let left = Math.min(...xs); let bottom = Math.max(...ys); let top = Math.min(...ys); - const firstx = this._points[0].X; - const firsty = this._points[0].Y; - let lastx = this._points[this._points.length - 2].X; - let lasty = this._points[this._points.length - 2].Y; + const firstx = points[0].X; + const firsty = points[0].Y; + let lastx = points[points.length - 2].X; + let lasty = points[points.length - 2].Y; let fourth = (lastx - firstx) / 4; if (isNaN(fourth) || fourth === 0) { fourth = 0.01; @@ -223,15 +223,15 @@ export class GestureOverlay extends ObservableReactComponent right) { const temp = right; @@ -245,47 +245,47 @@ export class GestureOverlay extends ObservableReactComponent 10) { lasty = firsty; } - this._points.push({ X: firstx, Y: firsty }); - this._points.push({ X: firstx, Y: firsty }); + points.push({ X: firstx, Y: firsty }); + points.push({ X: firstx, Y: firsty }); - this._points.push({ X: lastx, Y: lasty }); - this._points.push({ X: lastx, Y: lasty }); + points.push({ X: lastx, Y: lasty }); + points.push({ X: lastx, Y: lasty }); break; case Gestures.Arrow: { @@ -347,16 +347,16 @@ export class GestureOverlay extends ObservableReactComponent { diff --git a/src/client/views/InkStrokeProperties.ts b/src/client/views/InkStrokeProperties.ts index 3920ecc2a..35d628a4e 100644 --- a/src/client/views/InkStrokeProperties.ts +++ b/src/client/views/InkStrokeProperties.ts @@ -5,8 +5,8 @@ import { Doc, NumListCast, Opt } from '../../fields/Doc'; import { InkData, InkField, InkTool } from '../../fields/InkField'; import { List } from '../../fields/List'; import { listSpec } from '../../fields/Schema'; -import { Cast, NumCast } from '../../fields/Types'; -import { PointData } from '../../pen-gestures/GestureTypes'; +import { Cast, NumCast, toList } from '../../fields/Types'; +import { Gestures, PointData } from '../../pen-gestures/GestureTypes'; import { Point } from '../../pen-gestures/ndollar'; import { DocumentType } from '../documents/DocumentTypes'; import { undoBatch } from '../util/UndoManager'; @@ -14,6 +14,9 @@ import { FitOneCurve } from '../util/bezierFit'; import { InkingStroke } from './InkingStroke'; import { CollectionFreeFormView } from './collections/collectionFreeForm'; import { DocumentView } from './nodes/DocumentView'; +import simplify from 'simplify-js'; +import { GestureUtils } from '../../pen-gestures/GestureUtils'; +import { GestureOverlay } from './GestureOverlay'; export class InkStrokeProperties { // eslint-disable-next-line no-use-before-define @@ -487,4 +490,40 @@ export class InkStrokeProperties { } return inkCopy; }); + + @undoBatch + smoothInkStrokes = (inkDocs: Doc[], tolerance: number = 5) => { + inkDocs.forEach(inkDoc => { + const inkView = DocumentView.getDocumentView(inkDoc); + const inkStroke = inkView?.ComponentView as InkingStroke; + const { inkData } = inkStroke.inkScaledData(); + + const result = inkData.length > 2 && GestureUtils.GestureRecognizer.Recognize([inkData]); + console.log(result); + let polygonPoints: { X: number; Y: number }[] | undefined = undefined; + if (result && (result.Name === 'line' ? result.Score > 0.92 : result.Score > 0.85)) { + switch (result.Name) { + case Gestures.Line: + case Gestures.Triangle: + case Gestures.Rectangle: + case Gestures.Circle: + GestureOverlay.makeBezierPolygon(inkData, result.Name, true); + break; + default: + } + } else { + const polylinePoints = inkData.filter((pt, index) => { return index % 4 === 0 || pt === inkData.lastElement()}).map(pt => { return { x: pt.X, y: pt.Y }; }); // prettier-ignore + if (polylinePoints.length > 2) { + const toKeep = simplify(polylinePoints, tolerance).map(pt => {return { X: pt.x, Y: pt.y }}); // prettier-ignore + for (var i = 4; i < inkData.length - 3; i += 4) { + const contains = toKeep.find(pt => pt.X === inkData[i].X && pt.Y === inkData[i].Y); + if (!contains) { + this._currentPoint = i; + inkView && this.deletePoints(inkView, false); + } + } + } + } + }); + }; } diff --git a/src/client/views/InkTranscription.tsx b/src/client/views/InkTranscription.tsx index 3f90df7d1..1277a6fe7 100644 --- a/src/client/views/InkTranscription.tsx +++ b/src/client/views/InkTranscription.tsx @@ -6,7 +6,7 @@ import { InkData, InkField, InkTool } from '../../fields/InkField'; import { Cast, DateCast, ImageCast, NumCast, StrCast } from '../../fields/Types'; import { aggregateBounds } from '../../Utils'; import { DocumentType } from '../documents/DocumentTypes'; -import { CollectionFreeFormView } from './collections/collectionFreeForm'; +import { CollectionFreeFormView, MarqueeView } from './collections/collectionFreeForm'; import { InkingStroke } from './InkingStroke'; import './InkTranscription.scss'; import { Docs } from '../documents/Documents'; @@ -387,7 +387,7 @@ export class InkTranscription extends React.Component { ); docView.props.removeDocument?.(selected); // Gets a collection based on the selected nodes using a marquee view ref - const newCollection = marqViewRef?.getCollection(selected, undefined, true); + const newCollection = MarqueeView.getCollection(selected, undefined, true, { left: 1, top: 1, width: 1, height: 1 }); if (newCollection) { newCollection.width = NumCast(newCollection._width); newCollection.height = NumCast(newCollection._height); diff --git a/src/client/views/InkingStroke.tsx b/src/client/views/InkingStroke.tsx index ce1c07f2f..48b26d903 100644 --- a/src/client/views/InkingStroke.tsx +++ b/src/client/views/InkingStroke.tsx @@ -104,10 +104,10 @@ export class InkingStroke extends ViewBoxAnnotatableComponent() * analyzes the ink stroke and saves the analysis of the stroke to the 'inkAnalysis' field, * and the recognized words to the 'handwriting' */ - analyzeStrokes=()=> { + analyzeStrokes = () => { const data: InkData = this.inkScaledData().inkData ?? []; CognitiveServices.Inking.Appliers.ConcatenateHandwriting(this.dataDoc, ['inkAnalysis', 'handwriting'], [data]); - } + }; /** * Toggles whether the ink stroke is displayed as an overlay mask or as a regular stroke. diff --git a/src/client/views/PropertiesButtons.tsx b/src/client/views/PropertiesButtons.tsx index edf6df2b9..de1800700 100644 --- a/src/client/views/PropertiesButtons.tsx +++ b/src/client/views/PropertiesButtons.tsx @@ -15,7 +15,7 @@ import { MdClosedCaption, MdClosedCaptionDisabled, MdGridOff, MdGridOn, MdSubtit import { RxWidth } from 'react-icons/rx'; import { TbEditCircle, TbEditCircleOff, TbHandOff, TbHandStop, TbHighlight, TbHighlightOff } from 'react-icons/tb'; import { TfiBarChart } from 'react-icons/tfi'; -import { Doc, Opt } from '../../fields/Doc'; +import { Doc, DocListCast, Opt } from '../../fields/Doc'; import { DocData } from '../../fields/DocSymbols'; import { ScriptField } from '../../fields/ScriptField'; import { BoolCast, ScriptCast } from '../../fields/Types'; @@ -30,6 +30,8 @@ import { Colors } from './global/globalEnums'; import { DocumentView } from './nodes/DocumentView'; import { OpenWhere } from './nodes/OpenWhere'; import { FormattedTextBox } from './nodes/formattedText/FormattedTextBox'; +import { MarqueeOptionsMenu } from './collections/collectionFreeForm'; +import { InkStrokeProperties } from './InkStrokeProperties'; @observer export class PropertiesButtons extends React.Component<{}, {}> { diff --git a/src/client/views/PropertiesView.scss b/src/client/views/PropertiesView.scss index 840df41e7..aa825a6e9 100644 --- a/src/client/views/PropertiesView.scss +++ b/src/client/views/PropertiesView.scss @@ -638,3 +638,9 @@ padding-left: 8px; background-color: rgb(51, 51, 51); } + +.smooth, +.color, +.smooth-slider { + margin-top: 3px; +} diff --git a/src/client/views/PropertiesView.tsx b/src/client/views/PropertiesView.tsx index 024db82a4..ac2625f32 100644 --- a/src/client/views/PropertiesView.tsx +++ b/src/client/views/PropertiesView.tsx @@ -5,7 +5,7 @@ import { IconLookup } from '@fortawesome/fontawesome-svg-core'; import { faAnchor, faArrowRight, faWindowMaximize } from '@fortawesome/free-solid-svg-icons'; import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; import { Checkbox, Tooltip } from '@mui/material'; -import { Colors, EditableText, IconButton, NumberInput, Size, Slider, Type } from 'browndash-components'; +import { Colors, EditableText, IconButton, NumberInput, Size, Slider, Toggle, ToggleType, Type } from 'browndash-components'; import { concat } from 'lodash'; import { IReactionDisposer, action, computed, makeObservable, observable, reaction } from 'mobx'; import { observer } from 'mobx-react'; @@ -14,13 +14,13 @@ import { ColorResult, SketchPicker } from 'react-color'; import * as Icons from 'react-icons/bs'; // {BsCollectionFill, BsFillFileEarmarkImageFill} from "react-icons/bs" import { ClientUtils, returnEmptyDoclist, returnEmptyFilter, returnEmptyString, returnFalse, returnTrue, setupMoveUpEvents } from '../../ClientUtils'; import { emptyFunction } from '../../Utils'; -import { Doc, Field, FieldResult, FieldType, HierarchyMapping, NumListCast, Opt, ReverseHierarchyMap, StrListCast } from '../../fields/Doc'; +import { Doc, DocListCast, Field, FieldResult, FieldType, HierarchyMapping, NumListCast, Opt, ReverseHierarchyMap, StrListCast } from '../../fields/Doc'; import { AclAdmin, DocAcl, DocData } from '../../fields/DocSymbols'; import { Id } from '../../fields/FieldSymbols'; import { InkField } from '../../fields/InkField'; import { List } from '../../fields/List'; import { ComputedField } from '../../fields/ScriptField'; -import { Cast, DocCast, NumCast, StrCast } from '../../fields/Types'; +import { BoolCast, Cast, DocCast, NumCast, StrCast } from '../../fields/Types'; import { GetEffectiveAcl, SharingPermissions, normalizeEmail } from '../../fields/util'; import { CollectionViewType, DocumentType } from '../documents/DocumentTypes'; import { GroupManager } from '../util/GroupManager'; @@ -43,6 +43,10 @@ import { DocumentView } from './nodes/DocumentView'; import { StyleProviderFuncType } from './nodes/FieldView'; import { OpenWhere } from './nodes/OpenWhere'; import { PresBox, PresEffect, PresEffectDirection } from './nodes/trails'; +import { InkingStroke } from './InkingStroke'; +import { SettingsManager } from '../util/SettingsManager'; +import { MarqueeOptionsMenu } from './collections/collectionFreeForm'; +import { SmartDrawHandler } from './smartdraw/SmartDrawHandler'; const _global = (window /* browser */ || global) /* node */ as any; @@ -117,6 +121,10 @@ export class PropertiesView extends ObservableReactComponent { + doc[DocData].stroke_width = Math.round(value * 100) / 100; + }) + } else { + this.selectedDoc && (this.selectedDoc[DocData].stroke_width = Math.round(value * 100) / 100); + } + } // prettier-ignore @computed get hgtInput() { return this.inputBoxDuo( @@ -843,10 +860,32 @@ export class PropertiesView extends ObservableReactComponent { + const inkStroke = DocumentView.getDocumentView(doc)?.ComponentView as InkingStroke; + const { inkData } = inkStroke.inkScaledData(); + if (InkingStroke.IsClosed(inkData)) { + doc[DocData].fillColor = value || undefined; + } + }); + } else { + this.selectedDoc && (this.selectedDoc[DocData].fillColor = value || undefined); + } + } + @computed get colorStk() { return this.containsInkDoc ? StrCast(this.inkDoc?.[DocData].color) : StrCast(this.selectedDoc?.[DocData].color); } // prettier-ignore + set colorStk(value) { + if (this.containsInkDoc) { + const childDocs = DocListCast(this.selectedDoc[DocData].data); + childDocs.forEach(doc => { + doc[DocData].color = value || undefined; + }); + } else { + this.selectedDoc && (this.selectedDoc[DocData].color = value || undefined); + } + } colorButton(value: string, type: string, setter: () => void) { return ( @@ -917,10 +956,69 @@ export class PropertiesView extends ObservableReactComponent +
+ } + iconPlacement="left" + align="flex-start" + fillWidth + toggleType={ToggleType.BUTTON} + onClick={undoable(() => { + InkStrokeProperties.Instance.smoothInkStrokes(this.containsInkDoc ? DocListCast(targetDoc.data) : [targetDoc], this.smoothAmt); + }, 'smoothStrokes')} + /> +
+
+ {this.getNumber( + 'Smooth Amount', + '', + 1, + Math.max(20, this.smoothAmt), + this.smoothAmt, + (val: number) => { + !isNaN(val) && (this.smoothAmt = val); + }, + 20, + 1 + )} +
+ {!targetDoc.layout_isSvg && ( +
+ } + iconPlacement="left" + align="flex-start" + fillWidth + toggleType={ToggleType.BUTTON} + onClick={undoable(() => { + SmartDrawHandler.Instance.colorWithGPT(targetDoc); + }, 'smoothStrokes')} + /> +
+ )} + + ); + } + + @computed get dashdStk() { return this.containsInkDoc? this.inkDoc?.stroke_dash || '' : this.selectedDoc?.stroke_dash || ''; } // prettier-ignore set dashdStk(value) { value && (this._lastDash = value); - this.selectedDoc && (this.selectedDoc[DocData].stroke_dash = value ? this._lastDash : undefined); + if (this.containsInkDoc) { + const childDocs = DocListCast(this.selectedDoc[DocData].data); + childDocs.forEach(doc => { + doc[DocData].stroke_dash = value ? this._lastDash : undefined; + }); + } else { + this.selectedDoc && (this.selectedDoc[DocData].stroke_dash = value ? this._lastDash : undefined); + } } @computed get widthStk() { return this.getField('stroke_width') || '1'; } // prettier-ignore set widthStk(value) { @@ -930,13 +1028,31 @@ export class PropertiesView extends ObservableReactComponent { + doc[DocData].stroke_startMarker = value; + }); + } else { + this.selectedDoc && (this.selectedDoc[DocData].stroke_startMarker = value); + } } @computed get markTail() { return this.getField('stroke_endMarker') || ''; } // prettier-ignore set markTail(value) { - this.selectedDoc && (this.selectedDoc[DocData].stroke_endMarker = value); + if (this.containsInkDoc) { + const childDocs = DocListCast(this.selectedDoc[DocData].data); + childDocs.forEach(doc => { + doc[DocData].stroke_endMarker = value; + }); + } else { + this.selectedDoc && (this.selectedDoc[DocData].stroke_endMarker = value); + } } regInput = (key: string, value: any, setter: (val: string) => {}) => ( @@ -1036,6 +1152,16 @@ export class PropertiesView extends ObservableReactComponent {this.widthAndDash} {this.strokeAndFill} + {this.smoothAndColor} + + ); + } + + @computed get inkEditor() { + return ( +
+ {this.widthAndDash} + {this.strokeAndFill}
); } @@ -1164,6 +1290,7 @@ export class PropertiesView extends ObservableReactComponent @@ -1177,6 +1304,30 @@ export class PropertiesView extends ObservableReactComponent { + const childDocs: Doc[] = DocListCast(selectedDoc[DocData].data); + for (var i = 0; i < childDocs.length; i++) { + if (DocumentView.getDocumentView(childDocs[i])?.layoutDoc?.layout_isSvg) { + this.inkDoc = childDocs[i]; + this.containsInkDoc = true; + return true; + } + } + this.containsInkDoc = false; + return false; + }; + + @computed get inkCollectionSubMenu() { + return ( + // prettier-ignore + <> + { this.openAppearance = bool; }} onDoubleClick={this.CloseAll}> + {this.isGroup && this.containsInk(this.selectedDoc) ? this.appearanceEditor : null} + + + ); + } + @computed get fieldsSubMenu() { return ( { const points = segment.reduce((data, curve) => [...data, ...curve.points.map(p => intersect.inkView.ComponentView?.ptToScreen?.({ X: p.x, Y: p.y }) ?? { X: 0, Y: 0 })], [] as PointData[]); - const bounds = InkField.getBounds(points); - const B = this.screenToFreeformContentsXf.transformBounds(bounds.left, bounds.top, bounds.width, bounds.height); - const inkWidth = ActiveInkWidth() * this.ScreenToLocalBoxXf().Scale; - return Docs.Create.InkDocument( - points, - { title: 'stroke', - x: B.x - inkWidth / 2, - y: B.y - inkWidth / 2, - _width: B.width + inkWidth, - _height: B.height + inkWidth, - stroke_showLabel: BoolCast(Doc.UserDoc().activeInkHideTextLabels)}, // prettier-ignore - inkWidth, - ActiveInkColor(), - ActiveInkBezierApprox(), - ActiveFillColor(), - ActiveArrowStart(), - ActiveArrowEnd(), - ActiveDash(), - ActiveIsInkMask() - ); + return this.createInkDoc(points); }); newStrokes && this.addDocument?.(newStrokes); // setTimeout(() => this._eraserLock--); @@ -1276,18 +1240,43 @@ export class CollectionFreeFormView extends CollectionSubView { + const bounds = InkField.getBounds(points); + const B = transformedBounds || this.screenToFreeformContentsXf.transformBounds(bounds.left, bounds.top, bounds.width, bounds.height); + const inkWidth = ActiveInkWidth() * this.ScreenToLocalBoxXf().Scale; + return Docs.Create.InkDocument( + points, + { title: 'stroke', + x: B.x - inkWidth / 2, + y: B.y - inkWidth / 2, + _width: B.width + inkWidth, + _height: B.height + inkWidth, + stroke_showLabel: BoolCast(Doc.UserDoc().activeInkHideTextLabels)}, // prettier-ignore + inkWidth, + ActiveInkColor(), + ActiveInkBezierApprox(), + ActiveFillColor(), + ActiveArrowStart(), + ActiveArrowEnd(), + ActiveDash(), + ActiveIsInkMask() + ); + }; + @action - showSmartDraw = (e: PointerEvent, doubleTap?: boolean) => { - SmartDrawHandler.Instance.displaySmartDrawHandler(e.pageX, e.pageY, this.createDrawing, this.removeDrawing); + showSmartDraw = (e: PointerEvent) => { + SmartDrawHandler.Instance.CreateDrawingDoc = this.createDrawingDoc; + SmartDrawHandler.Instance.RemoveDrawing = this.removeDrawing; + SmartDrawHandler.Instance.AddDrawing = this.addDrawing; + SmartDrawHandler.Instance.displaySmartDrawHandler(e.pageX, e.pageY); }; _drawing: Doc[] = []; _drawingContainer: Doc | undefined = undefined; @undoBatch - createDrawing = (strokeData: [InkData, string, string][], opts: DrawingOptions, gptRes: string, containerDoc?: Doc) => { + createDrawingDoc = (strokeData: [InkData, string, string][], opts: DrawingOptions, gptRes: string) => { this._drawing = []; const xf = this.screenToFreeformContentsXf; - // this._drawingContainer = undefined; strokeData.forEach((stroke: [InkData, string, string]) => { const bounds = InkField.getBounds(stroke[0]); const B = xf.transformBounds(bounds.left, bounds.top, bounds.width, bounds.height); @@ -1310,20 +1299,9 @@ export class CollectionFreeFormView extends CollectionSubView { @@ -1339,6 +1317,19 @@ export class CollectionFreeFormView extends CollectionSubView { + const docData = doc[DocData]; + docData.title = opts.text.match(/^(.*?)~~~.*$/)?.[1] || opts.text; + docData.drawingInput = opts.text; + docData.drawingComplexity = opts.complexity; + docData.drawingColored = opts.autoColor; + docData.drawingSize = opts.size; + docData.drawingData = gptRes; + this._drawingContainer = doc; + this.addDocument(doc); + this._batch?.end(); + }; + @action zoom = (pointX: number, pointY: number, deltaY: number): void => { if (this.Document.isGroup || this.Document[(this._props.viewField ?? '_') + 'freeform_noZoom']) return; @@ -2045,7 +2036,10 @@ export class CollectionFreeFormView extends CollectionSubView { - !SmartDrawHandler.Instance._showRegenerate ? SmartDrawHandler.Instance.displayRegenerate(this._downX, this._downY - 10, this.createDrawing, this.removeDrawing) : SmartDrawHandler.Instance.hideRegenerate(); + SmartDrawHandler.Instance.CreateDrawingDoc = this.createDrawingDoc; + SmartDrawHandler.Instance.AddDrawing = this.addDrawing; + SmartDrawHandler.Instance.RemoveDrawing = this.removeDrawing; + !SmartDrawHandler.Instance._showRegenerate ? SmartDrawHandler.Instance.displayRegenerate(this._downX, this._downY - 10) : SmartDrawHandler.Instance.hideRegenerate(); }), icon: 'pen-to-square', }); diff --git a/src/client/views/collections/collectionFreeForm/MarqueeOptionsMenu.tsx b/src/client/views/collections/collectionFreeForm/MarqueeOptionsMenu.tsx index b3fdd9379..76c37dff0 100644 --- a/src/client/views/collections/collectionFreeForm/MarqueeOptionsMenu.tsx +++ b/src/client/views/collections/collectionFreeForm/MarqueeOptionsMenu.tsx @@ -21,6 +21,7 @@ export class MarqueeOptionsMenu extends AntimodeMenu { public pinWithView: (e: KeyboardEvent | React.PointerEvent | undefined) => void = unimplementedFunction; public classifyImages: (e: React.MouseEvent | undefined) => void = unimplementedFunction; public groupImages: () => void = unimplementedFunction; + public smoothStrokes: (docs?: Doc[]) => void = unimplementedFunction; public isShown = () => this._opacity > 0; constructor(props: any) { super(props); @@ -41,6 +42,7 @@ export class MarqueeOptionsMenu extends AntimodeMenu { } color={this.userColor} /> } color={this.userColor} /> } color={this.userColor} /> + this.smoothStrokes} icon={} color={this.userColor} /> ); return this.getElement(buttons); diff --git a/src/client/views/collections/collectionFreeForm/MarqueeView.tsx b/src/client/views/collections/collectionFreeForm/MarqueeView.tsx index 8560323c9..92c0da983 100644 --- a/src/client/views/collections/collectionFreeForm/MarqueeView.tsx +++ b/src/client/views/collections/collectionFreeForm/MarqueeView.tsx @@ -4,14 +4,14 @@ import { action, computed, makeObservable, observable } from 'mobx'; import { observer } from 'mobx-react'; import * as React from 'react'; import { ClientUtils, lightOrDark, returnFalse } from '../../../../ClientUtils'; -import { intersectRect, numberRange } from '../../../../Utils'; +import { intersectRect, numberRange, unimplementedFunction } from '../../../../Utils'; import { Doc, NumListCast, Opt } from '../../../../fields/Doc'; import { AclAdmin, AclAugment, AclEdit, DocData } from '../../../../fields/DocSymbols'; import { Id } from '../../../../fields/FieldSymbols'; import { InkData, InkField, InkTool } from '../../../../fields/InkField'; import { List } from '../../../../fields/List'; import { RichTextField } from '../../../../fields/RichTextField'; -import { Cast, FieldValue, ImageCast, NumCast, StrCast } from '../../../../fields/Types'; +import { BoolCast, Cast, DocCast, FieldValue, ImageCast, NumCast, StrCast } from '../../../../fields/Types'; import { ImageField } from '../../../../fields/URLField'; import { GetEffectiveAcl } from '../../../../fields/util'; import { gptGetEmbedding, gptImageLabel } from '../../../apis/gpt/GPT'; @@ -26,7 +26,7 @@ import { ContextMenu } from '../../ContextMenu'; import { ObservableReactComponent } from '../../ObservableReactComponent'; import { MarqueeViewBounds } from '../../PinFuncs'; import { PreviewCursor } from '../../PreviewCursor'; -import { DocumentView } from '../../nodes/DocumentView'; +import { ActiveArrowEnd, ActiveArrowStart, ActiveDash, ActiveFillColor, ActiveInkBezierApprox, ActiveInkColor, ActiveInkWidth, ActiveIsInkMask, DocumentView } from '../../nodes/DocumentView'; import { OpenWhere } from '../../nodes/OpenWhere'; import { pasteImageBitmap } from '../../nodes/WebBoxRenderer'; import { FormattedTextBox } from '../../nodes/formattedText/FormattedTextBox'; @@ -36,7 +36,12 @@ import { CollectionFreeFormView } from './CollectionFreeFormView'; import { ImageLabelHandler } from './ImageLabelHandler'; import { MarqueeOptionsMenu } from './MarqueeOptionsMenu'; import './MarqueeView.scss'; -import { collectionOf } from '@turf/turf'; +import { collectionOf, points } from '@turf/turf'; +import { InkingStroke } from '../../InkingStroke'; +import { GestureUtils } from '../../../../pen-gestures/GestureUtils'; +import { Gestures } from '../../../../pen-gestures/GestureTypes'; +import { GestureOverlay } from '../../GestureOverlay'; +import { InkStrokeProperties } from '../../InkStrokeProperties'; interface MarqueeViewProps { getContainerTransform: () => Transform; @@ -89,6 +94,8 @@ export class MarqueeView extends ObservableReactComponent Doc | void = unimplementedFunction; + componentDidMount() { this._props.setPreviewCursor?.(this.setPreviewCursor); } @@ -277,6 +284,7 @@ export class MarqueeView extends ObservableReactComponent { + docs && docs.length > 0 ? (this._selectedDocs = docs) : (this._selectedDocs = this.marqueeSelect(false, DocumentType.INK)); + if (this._selectedDocs.length == 0) return; + + this._selectedDocs.forEach(stroke => { + const docView = DocumentView.getDocumentView(stroke); + const inkStroke = docView?.ComponentView as InkingStroke; + const { inkData } = inkStroke.inkScaledData(); + + const result = inkData.length > 2 && GestureUtils.GestureRecognizer.Recognize([inkData]); + console.log(result); + let polygonPoints: { X: number; Y: number }[] | undefined = undefined; + if (result && (result.Name === 'line' ? result.Score > 0.9 : result.Score > 0.8)) { + switch (result.Name) { + case Gestures.Line: + case Gestures.Triangle: + case Gestures.Rectangle: + case Gestures.Circle: + GestureOverlay.makeBezierPolygon(inkData, result.Name, true); + break; + default: + } + } else { + const distances: number[] = []; + for (var i = 0; i < inkData.length - 3; i += 4) { + distances.push(Math.sqrt((inkData[i].X - inkData[i + 3].X) ** 2 + (inkData[i].Y - inkData[i + 3].Y) ** 2)); + } + const avgDist = (NumCast(stroke.width) + NumCast(stroke.height)) / 2; + // const avgDist = distances.reduce((a, b) => a + b) / distances.length; + if (Math.sqrt((inkData.lastElement().X - inkData[0].X) ** 2 + (inkData.lastElement().Y - inkData[0].Y) ** 2) < avgDist) { + inkData.pop(); + inkData.push({ X: inkData[0].X, Y: inkData[0].Y }); + } + // const editedPoints: InkData = []; + // const toDelete: number[] = []; + + // distances.forEach((dist, i) => { + // if (dist < avgDist / 3) { + // toDelete.unshift(i * 4); + // } + // }); + // toDelete.forEach(pt => { + // InkStrokeProperties.Instance._currentPoint = pt; + // docView && InkStrokeProperties.Instance.deletePoints(docView, false); + // }); + + // for (var i = 0; i < distances.length; i++) { + // if (distances[i] > avgDist / 3) { + // editedPoints.push(...inkData.slice(i * 4, i * 4 + 4)); + // } else { + // if (i !== distances.length) { + // editedPoints.push(...inkData.slice(i * 4, i * 4 + 2)); + // editedPoints.push(...inkData.slice(i * 4 + 6, i * 4 + 8)); + // i++; + // } + // } + // } + // inkData.length = 0; + // inkData.push(...editedPoints); + } + }); + }); + @undoBatch syntaxHighlight = action((e: KeyboardEvent | React.PointerEvent | undefined) => { const selected = this.marqueeSelect(false); diff --git a/src/client/views/nodes/formattedText/FormattedTextBox.tsx b/src/client/views/nodes/formattedText/FormattedTextBox.tsx index 9f2a9b8e1..1e700d240 100644 --- a/src/client/views/nodes/formattedText/FormattedTextBox.tsx +++ b/src/client/views/nodes/formattedText/FormattedTextBox.tsx @@ -295,6 +295,14 @@ export class FormattedTextBox extends ViewBoxAnnotatableComponent this.getAnchor(true), targetCreator), e.pageX, e.pageY); }); + AnchorMenu.Instance.AddDrawingAnnotation = (drawing: Doc) => { + const container = DocCast(this._props.Document.embedContainer); + const docView = DocumentView.getDocumentView?.(container); + docView?.ComponentView?._props.addDocument?.(drawing); + drawing.x = NumCast(this._props.Document.x) + (this._props.Document.width as number); + drawing.y = NumCast(this._props.Document.y); + }; + AnchorMenu.Instance.setSelectedText(window.getSelection()?.toString() ?? ''); const coordsB = this._editorView!.coordsAtPos(this._editorView!.state.selection.to); this._props.rootSelected?.() && AnchorMenu.Instance.jumpTo(coordsB.left, coordsB.bottom); diff --git a/src/client/views/pdf/AnchorMenu.tsx b/src/client/views/pdf/AnchorMenu.tsx index df990b0c0..ea574493a 100644 --- a/src/client/views/pdf/AnchorMenu.tsx +++ b/src/client/views/pdf/AnchorMenu.tsx @@ -15,6 +15,11 @@ import { LinkPopup } from '../linking/LinkPopup'; import { DocumentView } from '../nodes/DocumentView'; import './AnchorMenu.scss'; import { GPTPopup, GPTPopupMode } from './GPTPopup/GPTPopup'; +import { DrawingOptions, SmartDrawHandler } from '../smartdraw/SmartDrawHandler'; +import { InkData, InkField } from '../../../fields/InkField'; +import { DocData } from '../../../fields/DocSymbols'; +import { undoBatch } from '../../util/UndoManager'; +import ReactLoading from 'react-loading'; @observer export class AnchorMenu extends AntimodeMenu { @@ -38,6 +43,7 @@ export class AnchorMenu extends AntimodeMenu { // GPT additions @observable private _selectedText: string = ''; + @observable private _isLoading: boolean = false; @action public setSelectedText = (txt: string) => { this._selectedText = txt.trim(); @@ -61,6 +67,7 @@ export class AnchorMenu extends AntimodeMenu { public get Active() { return this._left > 0; } + public AddDrawingAnnotation: (doc: Doc) => void = unimplementedFunction; public addToCollection: ((doc: Doc | Doc[], annotationKey?: string | undefined) => boolean) | undefined; componentWillUnmount() { @@ -137,6 +144,29 @@ export class AnchorMenu extends AntimodeMenu { this.addToCollection?.(newCol); }; + gptDraw = async (e: React.PointerEvent) => { + try { + SmartDrawHandler.Instance.AddDrawing = this.createDrawingAnnotation; + this._isLoading = true; + await SmartDrawHandler.Instance.drawWithGPT({ X: e.clientX, Y: e.clientY }, this._selectedText, 5, 100, true); + this._isLoading = false; + } catch (err) { + console.error(err); + } + }; + + @undoBatch + createDrawingAnnotation = action((drawing: Doc, opts: DrawingOptions, gptRes: string) => { + this.AddDrawingAnnotation(drawing); + const docData = drawing[DocData]; + docData.title = opts.text.match(/^(.*?)~~~.*$/)?.[1] || opts.text; + docData.drawingInput = opts.text; + docData.drawingComplexity = opts.complexity; + docData.drawingColored = opts.autoColor; + docData.drawingSize = opts.size; + docData.drawingData = gptRes; + }); + pointerDown = (e: React.PointerEvent) => { setupMoveUpEvents( this, @@ -239,6 +269,14 @@ export class AnchorMenu extends AntimodeMenu { icon={} color={SettingsManager.userColor} /> + {this._selectedText && ( + this.gptDraw(e)} + icon={this._isLoading ? : } + color={SettingsManager.userColor} + /> + )} {AnchorMenu.Instance.OnAudio === unimplementedFunction ? null : ( { GPTPopup.Instance.addDoc = this._props.sidebarAddDoc; // allows for creating collection AnchorMenu.Instance.addToCollection = this._props.DocumentView?.()._props.addDocument; + AnchorMenu.Instance.AddDrawingAnnotation = this.addDrawingAnnotation; + }; + + addDrawingAnnotation = (drawing: Doc) => { + // drawing[DocData].x = this._props.pdfBox.ScreenToLocalBoxXf().TranslateX + // const scaleX = this._mainCont.current.offsetWidth / boundingRect.width; + drawing.y = (drawing.y as number) + (this._props.Document.data_sidebar_panY as number); + this._props.addDocument?.(drawing); }; @action diff --git a/src/client/views/smartdraw/AnnotationPalette.tsx b/src/client/views/smartdraw/AnnotationPalette.tsx index ec4279e3e..7e4d46204 100644 --- a/src/client/views/smartdraw/AnnotationPalette.tsx +++ b/src/client/views/smartdraw/AnnotationPalette.tsx @@ -64,7 +64,7 @@ export class AnnotationPalette extends ObservableReactComponent { if (event.key === 'Enter') { - await this.generateDrawing(); + await this.generateDrawings(); } }; @@ -130,15 +130,14 @@ export class AnnotationPalette extends ObservableReactComponent { + generateDrawings = action(async () => { this._isLoading = true; this._props.Document[DocData].data = undefined; for (var i = 0; i < 3; i++) { try { - SmartDrawHandler.Instance._addFunc = this.createDrawing; + SmartDrawHandler.Instance.AddDrawing = this.addDrawing; this._canInteract = false; if (this._showRegenerate) { - SmartDrawHandler.Instance._deleteFunc = unimplementedFunction; await SmartDrawHandler.Instance.regenerate(this._opts, this._gptRes[i], this._userInput); } else { await SmartDrawHandler.Instance.drawWithGPT({ X: 0, Y: 0 }, this._userInput, this._opts.complexity, this._opts.size, this._opts.autoColor); @@ -154,39 +153,10 @@ export class AnnotationPalette extends ObservableReactComponent { - this._opts = opts; + addDrawing = (drawing: Doc, opts: DrawingOptions, gptRes: string) => { this._gptRes.push(gptRes); - const drawing: Doc[] = []; - - strokeList.forEach((stroke: [InkData, string, string]) => { - const bounds = InkField.getBounds(stroke[0]); - const inkWidth = Math.min(5, ActiveInkWidth()); - const inkDoc = Docs.Create.InkDocument( - stroke[0], - { title: 'stroke', - x: bounds.left - inkWidth / 2, - y: bounds.top - inkWidth / 2, - _width: bounds.width + inkWidth, - _height: bounds.height + inkWidth, - stroke_showLabel: BoolCast(Doc.UserDoc().activeInkHideTextLabels)}, // prettier-ignore - inkWidth, - opts.autoColor ? stroke[1] : ActiveInkColor(), - ActiveInkBezierApprox(), - stroke[2] === 'none' ? ActiveFillColor() : stroke[2], - ActiveArrowStart(), - ActiveArrowEnd(), - ActiveDash(), - ActiveIsInkMask() - ); - drawing.push(inkDoc); - }); - - const collection = MarqueeView.getCollection(drawing, undefined, true, { left: 1, top: 1, width: 1, height: 1 }); - if (collection) { - collection[DocData].freeform_fitContentsToBox = true; - Doc.AddDocToList(this._props.Document, 'data', collection); - } + drawing[DocData].freeform_fitContentsToBox = true; + Doc.AddDocToList(this._props.Document, 'data', drawing); }; saveDrawing = async () => { @@ -262,7 +232,7 @@ export class AnnotationPalette extends ObservableReactComponent : this._showRegenerate ? : } iconPlacement="right" color={SettingsManager.userColor} - onClick={this.generateDrawing} + onClick={this.generateDrawings} />
diff --git a/src/client/views/smartdraw/SmartDrawHandler.tsx b/src/client/views/smartdraw/SmartDrawHandler.tsx index c842551c3..52df598ee 100644 --- a/src/client/views/smartdraw/SmartDrawHandler.tsx +++ b/src/client/views/smartdraw/SmartDrawHandler.tsx @@ -7,17 +7,23 @@ import { ObservableReactComponent } from '../ObservableReactComponent'; import { Button, IconButton } from 'browndash-components'; import ReactLoading from 'react-loading'; import { AiOutlineSend } from 'react-icons/ai'; -import { gptAPICall, GPTCallType } from '../../apis/gpt/GPT'; -import { InkData, InkTool } from '../../../fields/InkField'; +import { gptAPICall, GPTCallType, gptDrawingColor } from '../../apis/gpt/GPT'; +import { InkData, InkField, InkTool } from '../../../fields/InkField'; import { SVGToBezier } from '../../util/bezierFit'; const { parse } = require('svgson'); import { Slider, Switch } from '@mui/material'; -import { Doc } from '../../../fields/Doc'; +import { Doc, DocListCast } from '../../../fields/Doc'; import { DocData } from '../../../fields/DocSymbols'; -import { DocumentView } from '../nodes/DocumentView'; -import { BoolCast, NumCast, StrCast } from '../../../fields/Types'; +import { ActiveArrowEnd, ActiveArrowStart, ActiveDash, ActiveFillColor, ActiveInkBezierApprox, ActiveInkColor, ActiveInkWidth, ActiveIsInkMask, DocumentView } from '../nodes/DocumentView'; +import { BoolCast, ImageCast, NumCast, StrCast } from '../../../fields/Types'; import './SmartDrawHandler.scss'; import { unimplementedFunction } from '../../../Utils'; +import { Docs } from '../../documents/Documents'; +import { MarqueeView } from '../collections/collectionFreeForm'; +import { ImageField, URLField } from '../../../fields/URLField'; +import { CollectionCardView } from '../collections/CollectionCardDeckView'; +import { InkingStroke } from '../InkingStroke'; +import { undoBatch } from '../../util/UndoManager'; export interface DrawingOptions { text: string; @@ -46,13 +52,43 @@ export class SmartDrawHandler extends ObservableReactComponent<{}> { @observable private _autoColor: boolean = true; @observable private _regenInput: string = ''; @observable private _canInteract: boolean = true; - public _addFunc: (strokeList: [InkData, string, string][], opts: DrawingOptions, gptRes: string, containerDoc?: Doc) => void = () => {}; - public _deleteFunc: (doc?: Doc) => void = () => {}; + private _lastInput: DrawingOptions = { text: '', complexity: 5, size: 350, autoColor: true, x: 0, y: 0 }; private _lastResponse: string = ''; private _selectedDoc: Doc | undefined = undefined; private _errorOccurredOnce = false; + public RemoveDrawing: (doc?: Doc) => void = unimplementedFunction; + public CreateDrawingDoc: (strokeList: [InkData, string, string][], opts: DrawingOptions, gptRes: string, containerDoc?: Doc) => Doc | undefined = (strokeList: [InkData, string, string][], opts: DrawingOptions, gptRes: string) => { + const drawing: Doc[] = []; + strokeList.forEach((stroke: [InkData, string, string]) => { + const bounds = InkField.getBounds(stroke[0]); + const inkWidth = Math.min(5, ActiveInkWidth()); + const inkDoc = Docs.Create.InkDocument( + stroke[0], + { title: 'stroke', + x: bounds.left - inkWidth / 2, + y: bounds.top - inkWidth / 2, + _width: bounds.width + inkWidth, + _height: bounds.height + inkWidth, + stroke_showLabel: BoolCast(Doc.UserDoc().activeInkHideTextLabels)}, // prettier-ignore + inkWidth, + opts.autoColor ? stroke[1] : ActiveInkColor(), + ActiveInkBezierApprox(), + stroke[2] === 'none' ? ActiveFillColor() : stroke[2], + ActiveArrowStart(), + ActiveArrowEnd(), + ActiveDash(), + ActiveIsInkMask() + ); + drawing.push(inkDoc); + }); + + const collection = MarqueeView.getCollection(drawing, undefined, true, { left: 1, top: 1, width: 1, height: 1 }); + return collection; + }; + public AddDrawing: (doc: Doc, opts: DrawingOptions, gptRes: string) => void = unimplementedFunction; + constructor(props: any) { super(props); makeObservable(this); @@ -90,25 +126,26 @@ export class SmartDrawHandler extends ObservableReactComponent<{}> { }; @action - displaySmartDrawHandler = (x: number, y: number, addFunc: (strokeData: [InkData, string, string][], opts: DrawingOptions, gptRes: string, containerDoc?: Doc) => void, deleteFunc: (doc?: Doc) => void) => { + setEdit = () => { + this._showEditBox = !this._showEditBox; + }; + + @action + displaySmartDrawHandler = (x: number, y: number) => { this._pageX = x; this._pageY = y; this._display = true; - this._addFunc = addFunc; - this._deleteFunc = deleteFunc; }; @action - displayRegenerate = (x: number, y: number, addFunc: (strokeData: [InkData, string, string][], opts: DrawingOptions, gptRes: string, containerDoc?: Doc) => void, deleteFunc: (doc?: Doc) => void) => { + displayRegenerate = (x: number, y: number) => { this._selectedDoc = DocumentView.SelectedDocs()?.lastElement(); - const docData = this._selectedDoc[DocData]; - this._addFunc = addFunc; - this._deleteFunc = deleteFunc; this._pageX = x; this._pageY = y; this._display = false; this._showRegenerate = true; this._showEditBox = false; + const docData = this._selectedDoc[DocData]; this._lastResponse = StrCast(docData.drawingData); this._lastInput = { text: StrCast(docData.drawingInput), complexity: NumCast(docData.drawingComplexity), size: NumCast(docData.drawingSize), autoColor: BoolCast(docData.drawingColored), x: this._pageX, y: this._pageY }; }; @@ -155,8 +192,8 @@ export class SmartDrawHandler extends ObservableReactComponent<{}> { this._showOptions = false; try { await this.drawWithGPT({ X: this._pageX, Y: this._pageY }, this._userInput, this._complexity, this._size, this._autoColor); - this._showRegenerate = true; this.hideSmartDrawHandler(); + this._showRegenerate = true; } catch (err) { if (this._errorOccurredOnce) { console.error('GPT call failed', err); @@ -181,16 +218,14 @@ export class SmartDrawHandler extends ObservableReactComponent<{}> { return; } console.log(res); - const strokeData = await this.parseResponse(res, startPt, false, autoColor); + const strokeData = await this.parseSvg(res, startPt, false, autoColor); + const drawingDoc = strokeData && this.CreateDrawingDoc(strokeData?.data, strokeData?.lastInput, strokeData?.lastRes); + drawingDoc && this.AddDrawing(drawingDoc, this._lastInput, res); + this._errorOccurredOnce = false; return strokeData; }; - @action - edit = () => { - this._showEditBox = !this._showEditBox; - }; - @action regenerate = async (lastInput?: DrawingOptions, lastResponse?: string, regenInput?: string) => { if (lastInput) this._lastInput = lastInput; @@ -211,14 +246,18 @@ export class SmartDrawHandler extends ObservableReactComponent<{}> { return; } console.log(res); - await this.parseResponse(res, { X: this._lastInput.x, Y: this._lastInput.y }, true, lastInput?.autoColor || this._autoColor); + const strokeData = await this.parseSvg(res, { X: this._lastInput.x, Y: this._lastInput.y }, true, lastInput?.autoColor || this._autoColor); + this.RemoveDrawing !== unimplementedFunction && this.RemoveDrawing(this._selectedDoc); + const drawingDoc = strokeData && this.CreateDrawingDoc(strokeData?.data, strokeData?.lastInput, strokeData?.lastRes); + drawingDoc && this.AddDrawing(drawingDoc, this._lastInput, res); + return strokeData; } catch (err) { - console.error('GPT call failed', err); + console.error('Error regenerating drawing', err); } }; @action - parseResponse = async (res: string, startPoint: { X: number; Y: number }, regenerate: boolean, autoColor: boolean) => { + parseSvg = async (res: string, startPoint: { X: number; Y: number }, regenerate: boolean, autoColor: boolean) => { const svg = res.match(/]*>([\s\S]*?)<\/svg>/g); if (svg) { this._lastResponse = svg[0]; @@ -235,16 +274,70 @@ export class SmartDrawHandler extends ObservableReactComponent<{}> { (regenerate ? this._lastInput.autoColor : autoColor) ? child.attributes.fill : undefined, ]); }); - if (regenerate) { - if (this._deleteFunc !== unimplementedFunction) this._deleteFunc(this._selectedDoc); - this._addFunc(strokeData, this._lastInput, svg[0]); - } else { - this._addFunc(strokeData, this._lastInput, svg[0]); - } return { data: strokeData, lastInput: this._lastInput, lastRes: svg[0] }; } }; + colorWithGPT = async (drawing: Doc) => { + const img = await this.getIcon(drawing); + const { href } = (img as URLField).url; + const hrefParts = href.split('.'); + const hrefComplete = `${hrefParts[0]}_o.${hrefParts[1]}`; + try { + const hrefBase64 = await CollectionCardView.imageUrlToBase64(hrefComplete); + const strokes = DocListCast(drawing[DocData].data); + const coords: string[] = []; + strokes.forEach((stroke, i) => { + const inkingStroke = DocumentView.getDocumentView(stroke)?.ComponentView as InkingStroke; + const { inkData } = inkingStroke.inkScaledData(); + coords.push( + `${i + 1}. ${inkData + .filter((point, index) => { + return index % 4 === 0 || index == inkData.length - 1; + }) + .map(point => { + return `(${point.X.toString()}, ${point.Y.toString()})`; + })}` + ); + }); + const response = await gptDrawingColor(hrefBase64, coords); + console.log(response); + const colorResponse = await gptAPICall(response, GPTCallType.COLOR, undefined); + console.log(colorResponse); + this.colorStrokes(colorResponse, drawing); + } catch (error) { + console.log('GPT call failed'); + } + }; + + @undoBatch + colorStrokes = (res: string, drawing: Doc) => { + const colorList = res.match(/\{.*?\}/g); + const strokes = DocListCast(drawing[DocData].data); + colorList?.forEach((colors, index) => { + const strokeAndFill = colors.match(/#[0-9A-Fa-f]{6}/g); + if (strokeAndFill && strokeAndFill.length == 2) { + strokes[index][DocData].color = strokeAndFill[0]; + const inkStroke = DocumentView.getDocumentView(strokes[index])?.ComponentView as InkingStroke; + const { inkData } = inkStroke.inkScaledData(); + if (InkingStroke.IsClosed(inkData)) { + strokes[index][DocData].fillColor = strokeAndFill[1]; + } + } + }); + }; + + async getIcon(doc: Doc) { + const docView = DocumentView.getDocumentView(doc); + console.log(doc); + if (docView) { + console.log(docView); + docView.ComponentView?.updateIcon?.(); + return new Promise(res => setTimeout(() => res(ImageCast(docView.Document.icon)), 1000)); + } + return undefined; + } + render() { if (this._display) { return ( @@ -397,7 +490,7 @@ export class SmartDrawHandler extends ObservableReactComponent<{}> { color={SettingsManager.userColor} onClick={this.handleSendClick} /> - } color={SettingsManager.userColor} onClick={this.edit} /> + } color={SettingsManager.userColor} onClick={this.setEdit} /> {this._showEditBox && (