diff options
Diffstat (limited to 'src')
49 files changed, 2577 insertions, 717 deletions
diff --git a/src/client/apis/gpt/GPT.ts b/src/client/apis/gpt/GPT.ts index 8dd3fd6e2..20c810651 100644 --- a/src/client/apis/gpt/GPT.ts +++ b/src/client/apis/gpt/GPT.ts @@ -12,6 +12,8 @@ enum GPTCallType { DESCRIBE = 'describe', MERMAID = 'mermaid', DATA = 'data', + DRAW = 'draw', + COLOR = 'color', } type GPTCallOpts = { @@ -53,6 +55,18 @@ const callTypeMap: { [type: string]: GPTCallOpts } = { temp: 0, prompt: 'List unique differences between the content of the UserAnswer and Rubric. Before each difference, label it and provide any additional information the UserAnswer missed and explain it in second person without separating it into UserAnswer and Rubric content and additional information. If there are no differences, say correct', }, + draw: { + model: 'gpt-4o', + maxTokens: 1024, + temp: 0.8, + prompt: 'Given an item, a level of complexity from 1-10, and a size in pixels, generate a detailed and colored line drawing representation of it. Make sure every element has the stroke field filled out. More complex drawings will have much more detail and strokes. The drawing should be in SVG format with no additional text or comments. For path coordinates, make sure you format with a comma between numbers, like M100,200 C150,250 etc. The only supported commands are line, ellipse, circle, rect, polygon, and path with M, Q, C, and L so only use those.', + }, + color: { + model: 'gpt-4o', + maxTokens: 1024, + temp: 0.5, + prompt: 'You will be coloring drawings. You will be given what the drawing is, then a list of descriptions for parts of the drawing. Based on each description, respond with the stroke and fill color that it should be. Follow the rules: 1. Avoid using black for stroke color 2. Make the stroke color 1-3 shades darker than the fill color 3. Use the same colors when possible. Format as {#abcdef #abcdef}, making sure theres a color for each description, and do not include any additional text.', + }, }; let lastCall = ''; @@ -63,10 +77,10 @@ let lastResp = ''; * @param inputText Text to process * @returns AI Output */ -const gptAPICall = async (inputTextIn: string, callType: GPTCallType, prompt?: any) => { +const gptAPICall = async (inputTextIn: string, callType: GPTCallType, prompt?: any, dontCache?: boolean) => { const inputText = [GPTCallType.SUMMARY, GPTCallType.FLASHCARD, GPTCallType.QUIZ].includes(callType) ? inputTextIn + '.' : inputTextIn; const opts: GPTCallOpts = callTypeMap[callType]; - if (lastCall === inputText) return lastResp; + if (lastCall === inputText && dontCache !== true) return lastResp; try { lastCall = inputText; @@ -149,5 +163,69 @@ const gptImageLabel = async (src: string): Promise<string> => { return 'Error connecting with API'; } }; +const gptHandwriting = async (src: string): Promise<string> => { + try { + const response = await openai.chat.completions.create({ + model: 'gpt-4o', + temperature: 0, + messages: [ + { + role: 'user', + content: [ + { type: 'text', text: 'What is this does this handwriting say. Only return the text' }, + { + type: 'image_url', + image_url: { + url: `${src}`, + detail: 'low', + }, + }, + ], + }, + ], + }); + if (response.choices[0].message.content) { + return response.choices[0].message.content; + } + return 'Missing labels'; + } catch (err) { + console.log(err); + return 'Error connecting with API'; + } +}; + +const gptDrawingColor = async (image: string, coords: string[]): Promise<string> => { + try { + const response = await openai.chat.completions.create({ + model: 'gpt-4o', + temperature: 0, + messages: [ + { + role: 'user', + content: [ + { + type: 'text', + text: `Identify what the drawing in the image represents in 1-5 words. Then, given a list of a list of coordinates, where each list is the coordinates for one stroke of the drawing, determine which part of the drawing it is. Return just what the item it is, followed by ~~~ then only your descriptions in a list like [description, description, ...]. Here are the coordinates: ${coords}`, + }, + { + type: 'image_url', + image_url: { + url: `${image}`, + detail: 'low', + }, + }, + ], + }, + ], + }); + if (response.choices[0].message.content) { + return response.choices[0].message.content; + } + return 'Missing labels'; + } catch (err) { + console.log(err); + return 'Error connecting with API'; + } +}; -export { gptAPICall, gptImageCall, GPTCallType, gptImageLabel, gptGetEmbedding }; +export { gptAPICall, gptImageCall, GPTCallType, gptImageLabel, gptGetEmbedding, gptHandwriting, gptDrawingColor }; diff --git a/src/client/cognitive_services/CognitiveServices.ts b/src/client/cognitive_services/CognitiveServices.ts index 9808b6a01..3ee61cbfb 100644 --- a/src/client/cognitive_services/CognitiveServices.ts +++ b/src/client/cognitive_services/CognitiveServices.ts @@ -336,7 +336,7 @@ export namespace CognitiveServices { 'Ocp-Apim-Subscription-Key': apiKey, }, }; - return request.post(options); + return rp.post(options); }, }; diff --git a/src/client/documents/DocumentTypes.ts b/src/client/documents/DocumentTypes.ts index b055546fc..5a5c3d5bf 100644 --- a/src/client/documents/DocumentTypes.ts +++ b/src/client/documents/DocumentTypes.ts @@ -29,6 +29,7 @@ export enum DocumentType { FUNCPLOT = 'funcplot', // function plotter MAP = 'map', DATAVIZ = 'dataviz', + ANNOPALETTE = 'annopalette', LOADING = 'loading', SIMULATION = 'simulation', // physics simulation diff --git a/src/client/documents/Documents.ts b/src/client/documents/Documents.ts index af181b031..91bba4755 100644 --- a/src/client/documents/Documents.ts +++ b/src/client/documents/Documents.ts @@ -773,7 +773,7 @@ export namespace Docs { export function ComparisonDocument(text: string, options: DocumentOptions = { title: 'Comparison Box' }) { return InstanceFromProto(Prototypes.get(DocumentType.COMPARISON), text, options); } - export function DiagramDocument(options: DocumentOptions = { title: 'bruh box' }) { + export function DiagramDocument(options: DocumentOptions = { title: '' }) { return InstanceFromProto(Prototypes.get(DocumentType.DIAGRAM), undefined, options); } @@ -1027,6 +1027,10 @@ export namespace Docs { return InstanceFromProto(Prototypes.get(DocumentType.DATAVIZ), new CsvField(url), { title: 'Data Viz', type: 'dataviz', ...options }, undefined, undefined, undefined, overwriteDoc); } + export function AnnoPaletteDocument(options?: DocumentOptions) { + return InstanceFromProto(Prototypes.get(DocumentType.ANNOPALETTE), new List([Doc.MyAnnos]), { ...(options || {}) }); + } + export function DockDocument(documents: Array<Doc>, config: string, options: DocumentOptions, id?: string) { const ret = InstanceFromProto(Prototypes.get(DocumentType.COL), new List(documents), { treeView_FreezeChildren: 'remove|add', ...options, type_collection: CollectionViewType.Docking, dockingConfig: config }, id); documents.map(c => Doc.SetContainer(c, ret)); diff --git a/src/client/util/CurrentUserUtils.ts b/src/client/util/CurrentUserUtils.ts index 14fb65252..cdf5505f8 100644 --- a/src/client/util/CurrentUserUtils.ts +++ b/src/client/util/CurrentUserUtils.ts @@ -158,6 +158,26 @@ export class CurrentUserUtils { const reqdScripts = { dropConverter: "convertToButtons(dragData)" }; return DocUtils.AssignDocField(doc, field, (opts,items) => Docs.Create.TreeDocument(items??[], opts), reqdOpts, templates, reqdScripts); } + + static setupAnnoPalette(doc: Doc, field="myAnnos") { + const reqdOpts:DocumentOptions = { + title: "Saved Annotations", _xMargin: 0, _layout_showTitle: "title", hidden: false, + _dragOnlyWithinContainer: true, layout_hideContextMenu: true, isSystem: true, _forceActive: true, + _layout_autoHeight: true, _width: 500, _height: 300, _layout_fitWidth: true, _columnWidth: 35, ignoreClick: true, _lockedPosition: true, + }; + const reqdScripts = { dropConverter: "convertToButtons(dragData)" }; + const savedAnnos = DocCast(doc[field]); + return DocUtils.AssignDocField(doc, field, (opts,items) => Docs.Create.MasonryDocument(items??[], opts), reqdOpts, DocListCast(savedAnnos?.data), reqdScripts); + } + + static setupLightboxDrawingPreviews(doc: Doc, field="myLightboxDrawings") { + const reqdOpts:DocumentOptions = { + title: "Preview", _header_height: 0, _layout_fitWidth: true, childLayoutFitWidth: true, + }; + const reqdScripts = {}; + const drawings = DocCast(doc[field]); + return DocUtils.AssignDocField(doc, field, (opts,items) => Docs.Create.CarouselDocument(items??[], opts), reqdOpts, DocListCast(drawings?.data), reqdScripts); + } // setup templates for different document types when they are iconified from Document Decorations static setupDefaultIconTemplates(doc: Doc, field="template_icons") { @@ -365,7 +385,8 @@ pie title Minerals in my tap water {key: "Button", creator: Docs.Create.ButtonDocument, opts: { _width: 150, _height: 50, _xPadding: 10, _yPadding: 10, title_custom: true, waitForDoubleClickToClick: 'never'}, scripts: {onClick: FollowLinkScript()?.script.originalScript ?? ""}}, {key: "Script", creator: opts => Docs.Create.ScriptingDocument(null, opts), opts: { _width: 200, _height: 250, }}, {key: "DataViz", creator: opts => Docs.Create.DataVizDocument("/users/rz/Downloads/addresses.csv", opts), opts: { _width: 300, _height: 300 }}, - {key: "Chat", creator: Docs.Create.ChatDocument, opts: { _width: 300, _height: 300, }}, + {key: "Chat", creator: Docs.Create.ChatDocument, opts: { _width: 300, _height: 300, }}, + {key: "AnnoPalette",creator: Docs.Create.AnnoPaletteDocument, opts: {_width: 300, _height: 300, _dropAction: dropActionType.move }}, {key: "Header", creator: headerTemplate, opts: { _width: 300, _height: 120, _header_pointerEvents: "all", _header_height: 50, _header_fontSize: 9,_layout_autoHeightMargins: 50, _layout_autoHeight: true, treeView_HideUnrendered: true}}, {key: "ViewSlide", creator: slideView, opts: { _width: 400, _height: 300, _xMargin: 3, _yMargin: 3,}}, {key: "Trail", creator: Docs.Create.PresDocument, opts: { _width: 400, _height: 30, _type_collection: CollectionViewType.Stacking, dropAction: dropActionType.embed, treeView_HideTitle: true, _layout_fitWidth:true, layout_boxShadow: "0 0" }}, @@ -392,11 +413,11 @@ pie title Minerals in my tap water { toolTip: "Tap or drag to create a plotly node", title: "Plotly", icon: "rocket", dragFactory: doc.emptyPlotly as Doc, clickFactory: DocCast(doc.emptyMermaids)}, { toolTip: "Tap or drag to create a physics simulation",title: "Simulation", icon: "rocket",dragFactory: doc.emptySimulation as Doc, clickFactory: DocCast(doc.emptySimulation), funcs: { hidden: "IsNoviceMode()"}}, { toolTip: "Tap or drag to create a note board", title: "Notes", icon: "book", dragFactory: doc.emptyNoteboard as Doc, clickFactory: DocCast(doc.emptyNoteboard)}, - { toolTip: "Tap or drag to create an iamge", title: "Image", icon: "image", dragFactory: doc.emptyImage as Doc, clickFactory: DocCast(doc.emptyImage)}, + { toolTip: "Tap or drag to create an image", title: "Image", icon: "image", dragFactory: doc.emptyImage as Doc, clickFactory: DocCast(doc.emptyImage)}, { toolTip: "Tap or drag to create a collection", title: "Col", icon: "folder", dragFactory: doc.emptyCollection as Doc, clickFactory: DocCast(doc.emptyTab)}, { toolTip: "Tap or drag to create a webpage", title: "Web", icon: "globe-asia", dragFactory: doc.emptyWebpage as Doc, clickFactory: DocCast(doc.emptyWebpage)}, { toolTip: "Tap or drag to create a comparison box", title: "Compare", icon: "columns", dragFactory: doc.emptyComparison as Doc, clickFactory: DocCast(doc.emptyComparison)}, - { toolTip: "Tap or drag to create a diagram", title: "Diagram", icon: "tree", dragFactory: doc.emptyDiagram as Doc, clickFactory: DocCast(doc.emptyDiagram)}, + { toolTip: "Tap or drag to create a diagram", title: "Diagram", icon: "tree", dragFactory: doc.emptyDiagram as Doc, clickFactory: DocCast(doc.emptyDiagram)}, { toolTip: "Tap or drag to create an audio recorder", title: "Audio", icon: "microphone", dragFactory: doc.emptyAudio as Doc, clickFactory: DocCast(doc.emptyAudio), openFactoryLocation: OpenWhere.overlay}, { toolTip: "Tap or drag to create a map", title: "Map", icon: "map-marker-alt", dragFactory: doc.emptyMap as Doc, clickFactory: DocCast(doc.emptyMap)}, { toolTip: "Tap or drag to create a chat assistant", title: "Assistant Chat", icon: "book",dragFactory: doc.emptyChat as Doc, clickFactory: DocCast(doc.emptyChat)}, @@ -405,9 +426,10 @@ pie title Minerals in my tap water { toolTip: "Tap or drag to create a button", title: "Button", icon: "circle", dragFactory: doc.emptyButton as Doc, clickFactory: DocCast(doc.emptyButton)}, { toolTip: "Tap or drag to create a scripting box", title: "Script", icon: "terminal", dragFactory: doc.emptyScript as Doc, clickFactory: DocCast(doc.emptyScript), funcs: { hidden: "IsNoviceMode()"}}, { toolTip: "Tap or drag to create a data viz node", title: "DataViz", icon: "chart-bar", dragFactory: doc.emptyDataViz as Doc, clickFactory: DocCast(doc.emptyDataViz)}, - { toolTip: "Tap or drag to create a bullet slide", title: "PPT Slide", icon: "person-chalkboard",dragFactory: doc.emptySlide as Doc,clickFactory: DocCast(doc.emptySlide), openFactoryLocation: OpenWhere.overlay, funcs: { hidden: "IsNoviceMode()"}}, - { toolTip: "Tap or drag to create a view slide", title: "View Slide", icon: "address-card", dragFactory: doc.emptyViewSlide as Doc,clickFactory: DocCast(doc.emptyViewSlide), openFactoryLocation: OpenWhere.overlay,funcs: { hidden: "IsNoviceMode()"}}, - { toolTip: "Tap or drag to create a data note", title: "DataNote", icon: "window-maximize",dragFactory: doc.emptyHeader as Doc,clickFactory: DocCast(doc.emptyHeader), openFactoryAsDelegate: true, funcs: { hidden: "IsNoviceMode()"} }, + { toolTip: "Tap or drag to create a bullet slide", title: "PPT Slide", icon: "person-chalkboard", dragFactory: doc.emptySlide as Doc, clickFactory: DocCast(doc.emptySlide), openFactoryLocation: OpenWhere.overlay,funcs: { hidden: "IsNoviceMode()"}}, + { toolTip: "Tap or drag to create a view slide", title: "View Slide", icon: "address-card", dragFactory: doc.emptyViewSlide as Doc, clickFactory: DocCast(doc.emptyViewSlide), openFactoryLocation: OpenWhere.overlay,funcs: { hidden: "IsNoviceMode()"}}, + { toolTip: "Tap or drag to create a data note", title: "DataNote", icon: "window-maximize", dragFactory: doc.emptyHeader as Doc, clickFactory: DocCast(doc.emptyHeader), openFactoryAsDelegate: true, funcs: { hidden: "IsNoviceMode()"} }, + { toolTip: "Tap or drag to create an annotation palette",title: "Annotation Palette", icon: "palette", dragFactory: doc.emptyAnnoPalette as Doc, clickFactory: DocCast(doc.emptyAnnoPalette)}, { toolTip: "Toggle a Calculator REPL", title: "replviewer", icon: "calculator", clickFactory: '<ScriptingRepl />' as unknown as Doc, openFactoryLocation: OpenWhere.overlay}, // hack: clickFactory is not a Doc but will get interpreted as a custom UI by the openDoc() onClick script // { toolTip: "Toggle an UndoStack", title: "undostacker", icon: "calculator", clickFactory: "<UndoStack />" as any, openFactoryLocation: OpenWhere.overlay}, ].map(tuple => ( @@ -738,6 +760,7 @@ pie title Minerals in my tap water static inkTools():Button[] { return [ { title: "Pen", toolTip: "Pen (Ctrl+P)", btnType: ButtonType.ToggleButton, icon: "pen-nib", toolType: "pen", scripts: {onClick:'{ return setActiveTool(this.toolType, false, _readOnly_);}' }}, + { title: "Highlight", toolTip: "Highlight (Ctrl+H)", btnType: ButtonType.ToggleButton, icon: "highlighter",toolType: "highlighter", scripts: {onClick:'{ return setActiveTool(this.toolType, false, _readOnly_);}' }}, { title: "Write", toolTip: "Write (Ctrl+Shift+P)", btnType: ButtonType.ToggleButton, icon: "pen", toolType: "write", scripts: {onClick:'{ return setActiveTool(this.toolType, false, _readOnly_);}' }, funcs: {hidden:"IsNoviceMode()" }}, { title: "Eraser", toolTip: "Eraser (Ctrl+E)", btnType: ButtonType.MultiToggleButton, scripts: {onClick: '{ return setActiveTool(this.toolType, false, _readOnly_);}'}, funcs: {toolType:"activeEraserTool()"}, subMenu: [ @@ -750,9 +773,10 @@ pie title Minerals in my tap water { title: "Square", toolTip: "Square (double tap to lock mode)", btnType: ButtonType.ToggleButton, icon: "square", toolType: Gestures.Rectangle, scripts: {onClick:`{ return setActiveTool(this.toolType, false, _readOnly_);}`, onDoubleClick:`{ return setActiveTool(this.toolType, true, _readOnly_);}`} }, { title: "Line", toolTip: "Line (double tap to lock mode)", btnType: ButtonType.ToggleButton, icon: "minus", toolType: Gestures.Line, scripts: {onClick:`{ return setActiveTool(this.toolType, false, _readOnly_);}`, onDoubleClick:`{ return setActiveTool(this.toolType, true, _readOnly_);}`} }, { title: "Mask", toolTip: "Mask", btnType: ButtonType.ToggleButton, icon: "user-circle",toolType: "inkMask", scripts: {onClick:'{ return setInkProperty(this.toolType, value, _readOnly_);}'}, funcs: {hidden:"IsNoviceMode()" } }, - { title: "Labels", toolTip: "Lab els", btnType: ButtonType.ToggleButton, icon: "text-width", toolType: "labels", scripts: {onClick:'{ return setInkProperty(this.toolType, value, _readOnly_);}'}, }, + { title: "Labels", toolTip: "Labels", btnType: ButtonType.ToggleButton, icon: "text-width", toolType: "labels", scripts: {onClick:'{ return setInkProperty(this.toolType, value, _readOnly_);}'}, }, { title: "Width", toolTip: "Stroke width", btnType: ButtonType.NumberSliderButton, toolType: "strokeWidth", ignoreClick: true, scripts: {script: '{ return setInkProperty(this.toolType, value, _readOnly_);}'}, numBtnMin: 1}, { title: "Ink", toolTip: "Stroke color", btnType: ButtonType.ColorButton, icon: "pen", toolType: "strokeColor", ignoreClick: true, scripts: {script: '{ return setInkProperty(this.toolType, value, _readOnly_);}'} }, + { title: "Smart Draw", toolTip: "Draw with GPT", btnType: ButtonType.ToggleButton, icon: "user-pen", toolType: "smartdraw", scripts: {onClick:'{ return setActiveTool(this.toolType, false, _readOnly_);}'}, funcs: {hidden: "IsNoviceMode()"}}, ]; } @@ -992,6 +1016,8 @@ pie title Minerals in my tap water this.setupTopbarButtons(doc); this.setupDockedButtons(doc); // the bottom bar of font icons this.setupLeftSidebarMenu(doc); // the left-side column of buttons that open their contents in a flyout panel on the left + this.setupAnnoPalette(doc); + this.setupLightboxDrawingPreviews(doc); this.setupDocTemplates(doc); // sets up the template menu of templates // sthis.setupFieldInfos(doc); // sets up the collection of field info descriptions for each possible DocumentOption DocUtils.AssignDocField(doc, "globalScriptDatabase", () => Docs.Prototypes.MainScriptDocument(), {}); diff --git a/src/client/util/DropConverter.ts b/src/client/util/DropConverter.ts index eb2011b77..57c79cff3 100644 --- a/src/client/util/DropConverter.ts +++ b/src/client/util/DropConverter.ts @@ -87,6 +87,32 @@ export function makeUserTemplateButton(doc: Doc) { dbox.onDragStart = ScriptField.MakeFunction('getCopy(this.dragFactory)'); return dbox; } + +/** + * Similar to makeUserTemplateButton, but rather than creating a draggable button for the template, it takes in + * an ImageField that will display. + */ +export function makeUserTemplateImage(doc: Doc, image: ImageField) { + const layoutDoc = doc; // doc.layout instanceof Doc && doc.layout.isTemplateForField ? doc.layout : doc; + if (layoutDoc.type !== DocumentType.FONTICON) { + !layoutDoc.isTemplateDoc && makeTemplate(layoutDoc); + } + layoutDoc.isTemplateDoc = true; + const dbox = Docs.Create.ImageDocument(image, { + _nativeWidth: 100, + _nativeHeight: 100, + _width: 100, + _height: 100, + title: StrCast(layoutDoc.title), + isSystem: false, + }); + dbox.title = ComputedField.MakeFunction('this.dragFactory.title'); + dbox.dragFactory = layoutDoc; + dbox.dropPropertiesToRemove = doc.dropPropertiesToRemove instanceof ObjectField ? ObjectField.MakeCopy(doc.dropPropertiesToRemove) : undefined; + dbox.onDragStart = ScriptField.MakeFunction('getCopy(this.dragFactory)'); + return dbox; +} + export function convertDropDataToButtons(data: DragManager.DocumentDragData) { data?.draggedDocuments.forEach((doc, i) => { let dbox = doc; diff --git a/src/client/util/SettingsManager.tsx b/src/client/util/SettingsManager.tsx index fde8869e3..9200d68db 100644 --- a/src/client/util/SettingsManager.tsx +++ b/src/client/util/SettingsManager.tsx @@ -85,7 +85,7 @@ export class SettingsManager extends React.Component<object> { if (this._playgroundMode) { DocServer.Control.makeReadOnly(); addStyleSheetRule(SettingsManager._settingsStyle, 'topbar-inner-container', { background: 'red !important' }); - } else ClientUtils.CurrentUserEmail() !== 'guest' && DocServer.Control.makeEditable(); + } else if (ClientUtils.CurrentUserEmail() !== 'guest') DocServer.Control.makeEditable(); }), 'set playgorund mode' ); diff --git a/src/client/util/bezierFit.ts b/src/client/util/bezierFit.ts index d6f3f2340..4c7f4a0ba 100644 --- a/src/client/util/bezierFit.ts +++ b/src/client/util/bezierFit.ts @@ -4,6 +4,15 @@ /* eslint-disable camelcase */ import { Point } from '../../pen-gestures/ndollar'; +export enum SVGType { + Rect = 'rect', + Path = 'path', + Circle = 'circle', + Ellipse = 'ellipse', + Line = 'line', + Polygon = 'polygon', +} + class SmartRect { minx: number = 0; miny: number = 0; @@ -557,6 +566,12 @@ function FitCubic(d: Point[], first: number, last: number, tHat1: Point, tHat2: const negThatCenter = new Point(-tHatCenter.X, -tHatCenter.Y); FitCubic(d, splitPoint2D, last, negThatCenter, tHat2, error, result); } +/** + * Convert polyline coordinates to a (multi) segment bezier curve + * @param d - polyline coordinates + * @param error - how much error to allow in fitting (measured in pixels) + * @returns + */ export function FitCurve(d: Point[], error: number) { const tHat1 = ComputeLeftTangent(d, 0); // Unit tangent vectors at endpoints const tHat2 = ComputeRightTangent(d, d.length - 1); @@ -586,6 +601,151 @@ export function FitOneCurve(d: Point[], tHat1?: Point, tHat2?: Point) { return { finalCtrls, error }; } +// alpha determines how far away the tangents are, or the "tightness" of the bezier +export function GenerateControlPoints(coordinates: Point[], alpha = 0.1) { + const firstEnd = coordinates.length ? [coordinates[0], coordinates[0]] : []; + const lastEnd = coordinates.length ? [coordinates.lastElement(), coordinates.lastElement()] : []; + const points: Point[] = coordinates.slice(1, coordinates.length - 1).flatMap((pt, index, inkData) => { + const prevPt: Point = index === 0 ? firstEnd[0] : inkData[index - 1]; + const nextPt: Point = index === inkData.length - 1 ? lastEnd[0] : inkData[index + 1]; + if (prevPt.X === nextPt.X) { + const verticalDist = nextPt.Y - prevPt.Y; + return [{ X: pt.X, Y: pt.Y - alpha * verticalDist }, pt, pt, { X: pt.X, Y: pt.Y + alpha * verticalDist }]; + } else if (prevPt.Y === nextPt.Y) { + const horizDist = nextPt.X - prevPt.X; + return [{ X: pt.X - alpha * horizDist, Y: pt.Y }, pt, pt, { X: pt.X + alpha * horizDist, Y: pt.Y }]; + } + // tangent vectors between the adjacent points + const tanX = nextPt.X - prevPt.X; + const tanY = nextPt.Y - prevPt.Y; + const ctrlPt1: Point = { X: pt.X - alpha * tanX, Y: pt.Y - alpha * tanY }; + const ctrlPt2: Point = { X: pt.X + alpha * tanX, Y: pt.Y + alpha * tanY }; + return [ctrlPt1, pt, pt, ctrlPt2]; + }); + return [...firstEnd, ...points, ...lastEnd]; +} + +export function SVGToBezier(name: SVGType, attributes: any): Point[] { + switch (name) { + case 'line': { + const x1 = parseInt(attributes.x1); + const x2 = parseInt(attributes.x2); + const y1 = parseInt(attributes.y1); + const y2 = parseInt(attributes.y2); + return [ + { X: x1, Y: y1 }, + { X: x1, Y: y1 }, + { X: x2, Y: y2 }, + { X: x2, Y: y2 }, + ]; + } + case 'circle': + case 'ellipse': { + const c = 0.551915024494; + const centerX = parseInt(attributes.cx); + const centerY = parseInt(attributes.cy); + const radiusX = parseInt(attributes.rx) || parseInt(attributes.r); + const radiusY = parseInt(attributes.ry) || parseInt(attributes.r); + return [ + { X: centerX, Y: centerY + radiusY }, + { X: centerX + c * radiusX, Y: centerY + radiusY }, + { X: centerX + radiusX, Y: centerY + c * radiusY }, + { X: centerX + radiusX, Y: centerY }, + { X: centerX + radiusX, Y: centerY }, + { X: centerX + radiusX, Y: centerY - c * radiusY }, + { X: centerX + c * radiusX, Y: centerY - radiusY }, + { X: centerX, Y: centerY - radiusY }, + { X: centerX, Y: centerY - radiusY }, + { X: centerX - c * radiusX, Y: centerY - radiusY }, + { X: centerX - radiusX, Y: centerY - c * radiusY }, + { X: centerX - radiusX, Y: centerY }, + { X: centerX - radiusX, Y: centerY }, + { X: centerX - radiusX, Y: centerY + c * radiusY }, + { X: centerX - c * radiusX, Y: centerY + radiusY }, + { X: centerX, Y: centerY + radiusY }, + ]; + } + case 'rect': { + const x = parseInt(attributes.x); + const y = parseInt(attributes.y); + const width = parseInt(attributes.width); + const height = parseInt(attributes.height); + return [ + { X: x, Y: y }, + { X: x, Y: y }, + { X: x + width, Y: y }, + { X: x + width, Y: y }, + { X: x + width, Y: y }, + { X: x + width, Y: y }, + { X: x + width, Y: y + height }, + { X: x + width, Y: y + height }, + { X: x + width, Y: y + height }, + { X: x + width, Y: y + height }, + { X: x, Y: y + height }, + { X: x, Y: y + height }, + { X: x, Y: y + height }, + { X: x, Y: y + height }, + { X: x, Y: y }, + { X: x, Y: y }, + ]; + } + case 'path': { + const coordList: Point[] = []; + const startPt = attributes.d.match(/M(-?\d+\.?\d*),(-?\d+\.?\d*)/); + coordList.push({ X: parseInt(startPt[1]), Y: parseInt(startPt[2]) }); + const matches: RegExpMatchArray[] = Array.from( + attributes.d.matchAll(/Q(-?\d+\.?\d*),(-?\d+\.?\d*) (-?\d+\.?\d*),(-?\d+\.?\d*)|C(-?\d+\.?\d*),(-?\d+\.?\d*) (-?\d+\.?\d*),(-?\d+\.?\d*) (-?\d+\.?\d*),(-?\d+\.?\d*)|L(-?\d+\.?\d*),(-?\d+\.?\d*)/g) + ); + let lastPt: Point = { X: 0, Y: 0 }; + matches.forEach(match => { + if (match[0].startsWith('Q')) { + coordList.push({ X: parseInt(match[1]), Y: parseInt(match[2]) }); + coordList.push({ X: parseInt(match[1]), Y: parseInt(match[2]) }); + coordList.push({ X: parseInt(match[3]), Y: parseInt(match[4]) }); + coordList.push({ X: parseInt(match[3]), Y: parseInt(match[4]) }); + lastPt = { X: parseInt(match[3]), Y: parseInt(match[4]) }; + } else if (match[0].startsWith('C')) { + coordList.push({ X: parseInt(match[5]), Y: parseInt(match[6]) }); + coordList.push({ X: parseInt(match[7]), Y: parseInt(match[8]) }); + coordList.push({ X: parseInt(match[9]), Y: parseInt(match[10]) }); + coordList.push({ X: parseInt(match[9]), Y: parseInt(match[10]) }); + lastPt = { X: parseInt(match[9]), Y: parseInt(match[10]) }; + } else { + coordList.push(lastPt || { X: parseInt(startPt[1]), Y: parseInt(startPt[2]) }); + coordList.push({ X: parseInt(match[11]), Y: parseInt(match[12]) }); + coordList.push({ X: parseInt(match[11]), Y: parseInt(match[12]) }); + coordList.push({ X: parseInt(match[11]), Y: parseInt(match[12]) }); + lastPt = { X: parseInt(match[11]), Y: parseInt(match[12]) }; + } + }); + const hasZ = attributes.d.match(/Z/); + if (hasZ) { + coordList.push(lastPt); + coordList.push({ X: parseInt(startPt[1]), Y: parseInt(startPt[2]) }); + coordList.push({ X: parseInt(startPt[1]), Y: parseInt(startPt[2]) }); + } else { + coordList.pop(); + } + return coordList; + } + case 'polygon': { + const coords: RegExpMatchArray[] = Array.from(attributes.points.matchAll(/(-?\d+\.?\d*),(-?\d+\.?\d*)/g)); + let list: Point[] = []; + coords.forEach(coord => { + list.push({ X: parseInt(coord[1]), Y: parseInt(coord[2]) }); + list.push({ X: parseInt(coord[1]), Y: parseInt(coord[2]) }); + list.push({ X: parseInt(coord[1]), Y: parseInt(coord[2]) }); + list.push({ X: parseInt(coord[1]), Y: parseInt(coord[2]) }); + }); + const firstPts = list.splice(0, 2); + list = list.concat(firstPts); + return list; + } + default: + return []; + } +} + /* static double GetTValueFromSValue (const BezierRep &parent, double t, double endT, bool left, double influenceDistance, double &excess) { double dist = 0; diff --git a/src/client/views/DocumentButtonBar.tsx b/src/client/views/DocumentButtonBar.tsx index 58b7f207c..096f058ad 100644 --- a/src/client/views/DocumentButtonBar.tsx +++ b/src/client/views/DocumentButtonBar.tsx @@ -28,7 +28,7 @@ import { DocumentLinksButton } from './nodes/DocumentLinksButton'; import { DocumentView } from './nodes/DocumentView'; import { OpenWhere } from './nodes/OpenWhere'; import { DashFieldView } from './nodes/formattedText/DashFieldView'; -import { DocData } from '../../fields/DocSymbols'; +import { AnnotationPalette } from './smartdraw/AnnotationPalette'; @observer export class DocumentButtonBar extends ObservableReactComponent<{ views: () => (DocumentView | undefined)[]; stack?: unknown }> { @@ -239,6 +239,20 @@ export class DocumentButtonBar extends ObservableReactComponent<{ views: () => ( ); } + saveAnno = undoable(async (targetDoc: Doc) => await AnnotationPalette.addToPalette(targetDoc), 'save to palette'); + + @computed + get saveAnnoButton() { + const targetDoc = this.view0?.Document; + return !targetDoc ? null : ( + <Tooltip title={<div className="dash-tooltip">{targetDoc.savedAsAnno ? 'Saved as Annotation!' : 'Save to Annotation Palette'}</div>}> + <div className="documentButtonBar-icon" style={{ color: 'white' }} onClick={() => this.saveAnno(targetDoc)}> + <FontAwesomeIcon className="documentdecorations-icon" icon={targetDoc.savedAsAnno ? 'clipboard-check' : 'file-arrow-down'} /> + </div> + </Tooltip> + ); + } + @computed get shareButton() { const targetDoc = this.view0?.Document; @@ -459,6 +473,7 @@ export class DocumentButtonBar extends ObservableReactComponent<{ views: () => ( <div className="documentButtonBar-button">{this.templateButton}</div> {!DocumentView.Selected().some(v => v.allLinks.length) ? null : <div className="documentButtonBar-button">{this.followLinkButton}</div>} <div className="documentButtonBar-button">{this.pinButton}</div> + <div className="documentButtonBar-button">{this.saveAnnoButton}</div> <div className="documentButtonBar-button">{this.recordButton}</div> <div className="documentButtonBar-button">{this.calendarButton}</div> <div className="documentButtonBar-button">{this.keywordButton}</div> diff --git a/src/client/views/DocumentDecorations.tsx b/src/client/views/DocumentDecorations.tsx index da35459bb..8552003eb 100644 --- a/src/client/views/DocumentDecorations.tsx +++ b/src/client/views/DocumentDecorations.tsx @@ -231,6 +231,11 @@ export class DocumentDecorations extends ObservableReactComponent<DocumentDecora if (iconViewDoc.activeFrame) { iconViewDoc.opacity = 0; // bcz: hacky ... allows inkMasks and other documents to be "turned off" without removing them from the animated collection which allows them to function properly in a presenation. } else { + // to mark annotations as no longer saved if they're deleted from the palette + const dragFactory: Doc = DocCast(iconView.Document.dragFactory); + if (dragFactory && DocCast(dragFactory.cloneOf).savedAsAnno) { + DocCast(dragFactory.cloneOf).savedAsAnno = undefined; + } iconView._props.removeDocument?.(iconView.Document); } }); diff --git a/src/client/views/GestureOverlay.tsx b/src/client/views/GestureOverlay.tsx index 3a2738c3b..befd19f6e 100644 --- a/src/client/views/GestureOverlay.tsx +++ b/src/client/views/GestureOverlay.tsx @@ -136,7 +136,7 @@ export class GestureOverlay extends ObservableReactComponent<React.PropsWithChil // if any of the shape is activated in the CollectionFreeFormViewChrome if (this.InkShape) { - this.makeBezierPolygon(this.InkShape, false); + GestureOverlay.makeBezierPolygon(this._points, this.InkShape, false); this.dispatchGesture(this.InkShape); this.primCreated(); } @@ -151,13 +151,18 @@ export class GestureOverlay extends ObservableReactComponent<React.PropsWithChil case Gestures.Triangle: case Gestures.Rectangle: case Gestures.Circle: - this.makeBezierPolygon(result.Name, true); + GestureOverlay.makeBezierPolygon(this._points, result.Name, true); actionPerformed = this.dispatchGesture(result.Name); break; case Gestures.Scribble: console.log('scribble'); break; - default: + case Gestures.RightAngle: + console.log('RightAngle'); + break; + default: { + /* empty */ + } } } @@ -191,17 +196,17 @@ export class GestureOverlay extends ObservableReactComponent<React.PropsWithChil this._points.length = 0; }; - makeBezierPolygon = (shape: string, gesture: boolean) => { - const xs = this._points.map(p => p.X); - const ys = this._points.map(p => p.Y); + public static makeBezierPolygon = (points: { X: number; Y: number }[], shape: string, gesture: boolean) => { + const xs = points.map(p => p.X); + const ys = points.map(p => p.Y); let right = Math.max(...xs); let left = Math.min(...xs); let bottom = Math.max(...ys); let top = Math.min(...ys); - const firstx = this._points[0].X; - const firsty = this._points[0].Y; - let lastx = this._points[this._points.length - 2].X; - let lasty = this._points[this._points.length - 2].Y; + const firstx = points[0].X; + const firsty = points[0].Y; + let lastx = points[points.length - 2].X; + let lasty = points[points.length - 2].Y; let fourth = (lastx - firstx) / 4; if (isNaN(fourth) || fourth === 0) { fourth = 0.01; @@ -212,15 +217,15 @@ export class GestureOverlay extends ObservableReactComponent<React.PropsWithChil } // const b = firsty - m * firstx; if (shape === 'noRec') { - return false; + return undefined; } if (!gesture) { // if shape options is activated in inkOptionMenu // take second to last point because _point[length-1] is _points[0] - right = this._points[this._points.length - 2].X; - left = this._points[0].X; - bottom = this._points[this._points.length - 2].Y; - top = this._points[0].Y; + right = points[points.length - 2].X; + left = points[0].X; + bottom = points[points.length - 2].Y; + top = points[0].Y; if (shape !== Gestures.Arrow && shape !== Gestures.Line && shape !== Gestures.Circle) { if (left > right) { const temp = right; @@ -234,47 +239,47 @@ export class GestureOverlay extends ObservableReactComponent<React.PropsWithChil } } } - this._points.length = 0; + points.length = 0; switch (shape) { case Gestures.Rectangle: - this._points.push({ X: left, Y: top }); - this._points.push({ X: left, Y: top }); - this._points.push({ X: right, Y: top }); - this._points.push({ X: right, Y: top }); - - this._points.push({ X: right, Y: top }); - this._points.push({ X: right, Y: top }); - this._points.push({ X: right, Y: bottom }); - this._points.push({ X: right, Y: bottom }); - - this._points.push({ X: right, Y: bottom }); - this._points.push({ X: right, Y: bottom }); - this._points.push({ X: left, Y: bottom }); - this._points.push({ X: left, Y: bottom }); - - this._points.push({ X: left, Y: bottom }); - this._points.push({ X: left, Y: bottom }); - this._points.push({ X: left, Y: top }); - this._points.push({ X: left, Y: top }); + points.push({ X: left, Y: top }); // curr pt + points.push({ X: left, Y: top }); // curr first ctrl pt + points.push({ X: right, Y: top }); // next ctrl pt + points.push({ X: right, Y: top }); // next pt + + points.push({ X: right, Y: top }); // next pt + points.push({ X: right, Y: top }); // next first ctrl pt + points.push({ X: right, Y: bottom }); // next next ctrl pt + points.push({ X: right, Y: bottom }); // next next pt + + points.push({ X: right, Y: bottom }); + points.push({ X: right, Y: bottom }); + points.push({ X: left, Y: bottom }); + points.push({ X: left, Y: bottom }); + + points.push({ X: left, Y: bottom }); + points.push({ X: left, Y: bottom }); + points.push({ X: left, Y: top }); + points.push({ X: left, Y: top }); break; case Gestures.Triangle: - this._points.push({ X: left, Y: bottom }); - this._points.push({ X: left, Y: bottom }); + points.push({ X: left, Y: bottom }); + points.push({ X: left, Y: bottom }); - this._points.push({ X: right, Y: bottom }); - this._points.push({ X: right, Y: bottom }); - this._points.push({ X: right, Y: bottom }); - this._points.push({ X: right, Y: bottom }); + points.push({ X: right, Y: bottom }); + points.push({ X: right, Y: bottom }); + points.push({ X: right, Y: bottom }); + points.push({ X: right, Y: bottom }); - this._points.push({ X: (right + left) / 2, Y: top }); - this._points.push({ X: (right + left) / 2, Y: top }); - this._points.push({ X: (right + left) / 2, Y: top }); - this._points.push({ X: (right + left) / 2, Y: top }); + points.push({ X: (right + left) / 2, Y: top }); + points.push({ X: (right + left) / 2, Y: top }); + points.push({ X: (right + left) / 2, Y: top }); + points.push({ X: (right + left) / 2, Y: top }); - this._points.push({ X: left, Y: bottom }); - this._points.push({ X: left, Y: bottom }); + points.push({ X: left, Y: bottom }); + points.push({ X: left, Y: bottom }); break; case Gestures.Circle: @@ -288,25 +293,25 @@ export class GestureOverlay extends ObservableReactComponent<React.PropsWithChil const radius = Math.max(centerX - Math.min(left, right), centerY - Math.min(top, bottom)); // Dividing the circle into four equal sections, and fitting each section to a cubic Bézier curve. - this._points.push({ X: centerX, Y: centerY + radius }); - this._points.push({ X: centerX + c * radius, Y: centerY + radius }); - this._points.push({ X: centerX + radius, Y: centerY + c * radius }); - this._points.push({ X: centerX + radius, Y: centerY }); - - this._points.push({ X: centerX + radius, Y: centerY }); - this._points.push({ X: centerX + radius, Y: centerY - c * radius }); - this._points.push({ X: centerX + c * radius, Y: centerY - radius }); - this._points.push({ X: centerX, Y: centerY - radius }); - - this._points.push({ X: centerX, Y: centerY - radius }); - this._points.push({ X: centerX - c * radius, Y: centerY - radius }); - this._points.push({ X: centerX - radius, Y: centerY - c * radius }); - this._points.push({ X: centerX - radius, Y: centerY }); - - this._points.push({ X: centerX - radius, Y: centerY }); - this._points.push({ X: centerX - radius, Y: centerY + c * radius }); - this._points.push({ X: centerX - c * radius, Y: centerY + radius }); - this._points.push({ X: centerX, Y: centerY + radius }); + points.push({ X: centerX, Y: centerY + radius }); // curr pt + points.push({ X: centerX + c * radius, Y: centerY + radius }); // curr first ctrl pt + points.push({ X: centerX + radius, Y: centerY + c * radius }); // next pt ctrl pt + points.push({ X: centerX + radius, Y: centerY }); // next pt + + points.push({ X: centerX + radius, Y: centerY }); // next pt + points.push({ X: centerX + radius, Y: centerY - c * radius }); // next first ctrl pt + points.push({ X: centerX + c * radius, Y: centerY - radius }); + points.push({ X: centerX, Y: centerY - radius }); + + points.push({ X: centerX, Y: centerY - radius }); + points.push({ X: centerX - c * radius, Y: centerY - radius }); + points.push({ X: centerX - radius, Y: centerY - c * radius }); + points.push({ X: centerX - radius, Y: centerY }); + + points.push({ X: centerX - radius, Y: centerY }); + points.push({ X: centerX - radius, Y: centerY + c * radius }); + points.push({ X: centerX - c * radius, Y: centerY + radius }); + points.push({ X: centerX, Y: centerY + radius }); } break; @@ -317,11 +322,11 @@ export class GestureOverlay extends ObservableReactComponent<React.PropsWithChil if (Math.abs(firsty - lasty) < 10 && Math.abs(firstx - lastx) > 10) { lasty = firsty; } - this._points.push({ X: firstx, Y: firsty }); - this._points.push({ X: firstx, Y: firsty }); + points.push({ X: firstx, Y: firsty }); + points.push({ X: firstx, Y: firsty }); - this._points.push({ X: lastx, Y: lasty }); - this._points.push({ X: lastx, Y: lasty }); + points.push({ X: lastx, Y: lasty }); + points.push({ X: lastx, Y: lasty }); break; case Gestures.Arrow: { @@ -336,16 +341,16 @@ export class GestureOverlay extends ObservableReactComponent<React.PropsWithChil const y3 = y2 + (L2 / L1) * ((y1 - y2) * Math.cos(angle) - (x1 - x2) * Math.sin(angle)); const x4 = x2 + (L2 / L1) * ((x1 - x2) * Math.cos(angle) - (y1 - y2) * Math.sin(angle)); const y4 = y2 + (L2 / L1) * ((y1 - y2) * Math.cos(angle) + (x1 - x2) * Math.sin(angle)); - this._points.push({ X: x1, Y: y1 }); - this._points.push({ X: x2, Y: y2 }); - this._points.push({ X: x3, Y: y3 }); - this._points.push({ X: x4, Y: y4 }); - this._points.push({ X: x2, Y: y2 }); + points.push({ X: x1, Y: y1 }); + points.push({ X: x2, Y: y2 }); + points.push({ X: x3, Y: y3 }); + points.push({ X: x4, Y: y4 }); + points.push({ X: x2, Y: y2 }); } break; default: } - return false; + return points; }; dispatchGesture = (gesture: Gestures, stroke?: InkData, text?: string) => { diff --git a/src/client/views/GlobalKeyHandler.ts b/src/client/views/GlobalKeyHandler.ts index a85a03aab..d7d8e9506 100644 --- a/src/client/views/GlobalKeyHandler.ts +++ b/src/client/views/GlobalKeyHandler.ts @@ -162,7 +162,7 @@ export class KeyManager { case 'delete': case 'backspace': if (document.activeElement?.tagName !== 'INPUT' && document.activeElement?.tagName !== 'TEXTAREA') { - if (DocumentView.LightboxDoc()) { + if (DocumentView.LightboxDoc() && !DocumentView.Selected().length) { DocumentView.SetLightboxDoc(undefined); DocumentView.DeselectAll(); } else if (!window.getSelection()?.toString()) DocumentDecorations.Instance.onCloseClick(true); diff --git a/src/client/views/InkStrokeProperties.ts b/src/client/views/InkStrokeProperties.ts index 3920ecc2a..13807c25f 100644 --- a/src/client/views/InkStrokeProperties.ts +++ b/src/client/views/InkStrokeProperties.ts @@ -1,16 +1,19 @@ import { Bezier } from 'bezier-js'; import * as _ from 'lodash'; import { action, makeObservable, observable, reaction, runInAction } from 'mobx'; +import simplify from 'simplify-js'; import { Doc, NumListCast, Opt } from '../../fields/Doc'; import { InkData, InkField, InkTool } from '../../fields/InkField'; import { List } from '../../fields/List'; import { listSpec } from '../../fields/Schema'; import { Cast, NumCast } from '../../fields/Types'; -import { PointData } from '../../pen-gestures/GestureTypes'; +import { Gestures, PointData } from '../../pen-gestures/GestureTypes'; +import { GestureUtils } from '../../pen-gestures/GestureUtils'; import { Point } from '../../pen-gestures/ndollar'; import { DocumentType } from '../documents/DocumentTypes'; -import { undoBatch } from '../util/UndoManager'; +import { undoable } from '../util/UndoManager'; import { FitOneCurve } from '../util/bezierFit'; +import { GestureOverlay } from './GestureOverlay'; import { InkingStroke } from './InkingStroke'; import { CollectionFreeFormView } from './collections/collectionFreeForm'; import { DocumentView } from './nodes/DocumentView'; @@ -89,8 +92,7 @@ export class InkStrokeProperties { * @param i index of first control point of segment being split * @param control The list of all control points of the ink. */ - @undoBatch - addPoints = (inkView: DocumentView, t: number, i: number, controls: { X: number; Y: number }[]) => { + addPoints = undoable((inkView: DocumentView, t: number, i: number, controls: { X: number; Y: number }[]) => { this.applyFunction(inkView, (view: DocumentView /* , ink: InkData */) => { const doc = view.Document; const array = [controls[i], controls[i + 1], controls[i + 2], controls[i + 3]]; @@ -106,7 +108,7 @@ export class InkStrokeProperties { return controls; }); - }; + }, 'add ink points'); /** * Scales a handle point of a control point that is adjacent to a newly added one. @@ -161,8 +163,7 @@ export class InkStrokeProperties { /** * Deletes the current control point of the selected ink instance. */ - @undoBatch - deletePoints = (inkView: DocumentView, preserve: boolean) => + deletePoints = undoable((inkView: DocumentView, preserve: boolean) => { this.applyFunction( inkView, (view: DocumentView, ink: InkData) => { @@ -201,6 +202,7 @@ export class InkStrokeProperties { }, true ); + }, 'delete ink points'); /** * Rotates ink stroke(s) about a point @@ -208,8 +210,7 @@ export class InkStrokeProperties { * @param angle The angle at which to rotate the ink in radians. * @param scrpt The center point of the rotation in screen coordinates */ - @undoBatch - rotateInk = (inkStrokes: DocumentView[], angle: number, scrpt: PointData) => { + rotateInk = undoable((inkStrokes: DocumentView[], angle: number, scrpt: PointData) => { this.applyFunction(inkStrokes, (view: DocumentView, ink: InkData, xScale: number, yScale: number /* , inkStrokeWidth: number */) => { const inkCenterPt = view.ComponentView?.ptFromScreen?.(scrpt); return !inkCenterPt @@ -221,7 +222,7 @@ export class InkStrokeProperties { return { X: newX + inkCenterPt.X, Y: newY + inkCenterPt.Y }; }); }); - }; + }, 'rotate ink'); /** * Rotates ink stroke(s) about a point @@ -229,8 +230,7 @@ export class InkStrokeProperties { * @param angle The angle at which to rotate the ink in radians. * @param scrpt The center point of the rotation in screen coordinates */ - @undoBatch - stretchInk = (inkStrokes: DocumentView[], scaling: number, scrpt: PointData, scrVec: PointData, scaleUniformly: boolean) => { + stretchInk = undoable((inkStrokes: DocumentView[], scaling: number, scrpt: PointData, scrVec: PointData, scaleUniformly: boolean) => { this.applyFunction(inkStrokes, (view: DocumentView, ink: InkData) => { const ptFromScreen = view.ComponentView?.ptFromScreen; const ptToScreen = view.ComponentView?.ptToScreen; @@ -244,77 +244,77 @@ export class InkStrokeProperties { return ptFromScreen(newscrpt); }); }); - }; + }, 'stretch ink'); /** * Handles the movement/scaling of a control point. */ - @undoBatch - moveControlPtHandle = (inkView: DocumentView, deltaX: number, deltaY: number, controlIndex: number, origInk?: InkData) => + moveControlPtHandle = undoable((inkView: DocumentView, deltaX: number, deltaY: number, controlIndex: number, origInk?: InkData) => { inkView && - this.applyFunction(inkView, (view: DocumentView, ink: InkData) => { - const order = controlIndex % 4; - const closed = InkingStroke.IsClosed(ink); - const brokenIndices = Cast(inkView.Document.brokenInkIndices, listSpec('number'), []); - if (origInk && this._currentPoint > 0 && this._currentPoint < ink.length - 1 && brokenIndices.findIndex(value => value === controlIndex) === -1) { - const cptBefore = ink[controlIndex]; - const cpt = { X: cptBefore.X + deltaX, Y: cptBefore.Y + deltaY }; - const newink = origInk.slice(); - const start = this._currentPoint === 0 ? 0 : this._currentPoint - 4; - const splicedPoints = origInk.slice(start, start + (this._currentPoint === 0 || this._currentPoint === ink.length - 1 ? 4 : 8)); - const { nearestT, nearestSeg } = InkStrokeProperties.nearestPtToStroke(splicedPoints, cpt); - if ((nearestSeg === 0 && nearestT < 1e-1) || (nearestSeg === 4 && 1 - nearestT < 1e-1) || nearestSeg < 0) return ink.slice(); - const samplesLeft: Point[] = []; - const samplesRight: Point[] = []; - let startDir = { x: 0, y: 0 }; - let endDir = { x: 0, y: 0 }; - for (let i = 0; i < nearestSeg / 4 + 1; i++) { - const bez = new Bezier(splicedPoints.slice(i * 4, i * 4 + 4).map(p => ({ x: p.X, y: p.Y }))); - if (i === 0) startDir = bez.derivative(_.isEqual(bez.derivative(0), { x: 0, y: 0, t: 0 }) ? 1e-8 : 0); - if (i === nearestSeg / 4) endDir = bez.derivative(nearestT); - for (let t = 0; t < (i === nearestSeg / 4 ? nearestT + 0.05 : 1); t += 0.05) { - const pt = bez.compute(i !== nearestSeg / 4 ? t : Math.min(nearestT, t)); - samplesLeft.push(new Point(pt.x, pt.y)); + this.applyFunction(inkView, (view: DocumentView, ink: InkData) => { + const order = controlIndex % 4; + const closed = InkingStroke.IsClosed(ink); + const brokenIndices = Cast(inkView.Document.brokenInkIndices, listSpec('number'), []); + if (origInk && this._currentPoint > 0 && this._currentPoint < ink.length - 1 && brokenIndices.findIndex(value => value === controlIndex) === -1) { + const cptBefore = ink[controlIndex]; + const cpt = { X: cptBefore.X + deltaX, Y: cptBefore.Y + deltaY }; + const newink = origInk.slice(); + const start = this._currentPoint === 0 ? 0 : this._currentPoint - 4; + const splicedPoints = origInk.slice(start, start + (this._currentPoint === 0 || this._currentPoint === ink.length - 1 ? 4 : 8)); + const { nearestT, nearestSeg } = InkStrokeProperties.nearestPtToStroke(splicedPoints, cpt); + if ((nearestSeg === 0 && nearestT < 1e-1) || (nearestSeg === 4 && 1 - nearestT < 1e-1) || nearestSeg < 0) return ink.slice(); + const samplesLeft: Point[] = []; + const samplesRight: Point[] = []; + let startDir = { x: 0, y: 0 }; + let endDir = { x: 0, y: 0 }; + for (let i = 0; i < nearestSeg / 4 + 1; i++) { + const bez = new Bezier(splicedPoints.slice(i * 4, i * 4 + 4).map(p => ({ x: p.X, y: p.Y }))); + if (i === 0) startDir = bez.derivative(_.isEqual(bez.derivative(0), { x: 0, y: 0, t: 0 }) ? 1e-8 : 0); + if (i === nearestSeg / 4) endDir = bez.derivative(nearestT); + for (let t = 0; t < (i === nearestSeg / 4 ? nearestT + 0.05 : 1); t += 0.05) { + const pt = bez.compute(i !== nearestSeg / 4 ? t : Math.min(nearestT, t)); + samplesLeft.push(new Point(pt.x, pt.y)); + } } - } - let { finalCtrls } = FitOneCurve(samplesLeft, { X: startDir.x, Y: startDir.y }, { X: endDir.x, Y: endDir.y }); - for (let i = nearestSeg / 4; i < splicedPoints.length / 4; i++) { - const bez = new Bezier(splicedPoints.slice(i * 4, i * 4 + 4).map(p => ({ x: p.X, y: p.Y }))); - if (i === nearestSeg / 4) startDir = bez.derivative(nearestT); - if (i === splicedPoints.length / 4 - 1) endDir = bez.derivative(_.isEqual(bez.derivative(1), { x: 0, y: 0, t: 1 }) ? 1 - 1e-8 : 1); - for (let t = i === nearestSeg / 4 ? nearestT : 0; t < (i === nearestSeg / 4 ? 1 + 0.05 + 1e-7 : 1 + 1e-7); t += 0.05) { - const pt = bez.compute(Math.min(1, t)); - samplesRight.push(new Point(pt.x, pt.y)); + let { finalCtrls } = FitOneCurve(samplesLeft, { X: startDir.x, Y: startDir.y }, { X: endDir.x, Y: endDir.y }); + for (let i = nearestSeg / 4; i < splicedPoints.length / 4; i++) { + const bez = new Bezier(splicedPoints.slice(i * 4, i * 4 + 4).map(p => ({ x: p.X, y: p.Y }))); + if (i === nearestSeg / 4) startDir = bez.derivative(nearestT); + if (i === splicedPoints.length / 4 - 1) endDir = bez.derivative(_.isEqual(bez.derivative(1), { x: 0, y: 0, t: 1 }) ? 1 - 1e-8 : 1); + for (let t = i === nearestSeg / 4 ? nearestT : 0; t < (i === nearestSeg / 4 ? 1 + 0.05 + 1e-7 : 1 + 1e-7); t += 0.05) { + const pt = bez.compute(Math.min(1, t)); + samplesRight.push(new Point(pt.x, pt.y)); + } } + const { finalCtrls: rightCtrls /* , error: errorRight */ } = FitOneCurve(samplesRight, { X: startDir.x, Y: startDir.y }, { X: endDir.x, Y: endDir.y }); + finalCtrls = finalCtrls.concat(rightCtrls); + newink.splice(this._currentPoint - 4, 8, ...finalCtrls); + return newink; } - const { finalCtrls: rightCtrls /* , error: errorRight */ } = FitOneCurve(samplesRight, { X: startDir.x, Y: startDir.y }, { X: endDir.x, Y: endDir.y }); - finalCtrls = finalCtrls.concat(rightCtrls); - newink.splice(this._currentPoint - 4, 8, ...finalCtrls); - return newink; - } - return ink.map((pt, i) => { - const leftHandlePoint = order === 0 && i === controlIndex + 1; - const rightHandlePoint = order === 0 && controlIndex !== 0 && i === controlIndex - 2; - if (controlIndex === i || (order === 0 && controlIndex !== 0 && i === controlIndex - 1) || (order === 3 && i === controlIndex - 1)) { - return { X: pt.X + deltaX, Y: pt.Y + deltaY }; - } - if ( - controlIndex === i || - leftHandlePoint || - rightHandlePoint || - (order === 0 && controlIndex !== 0 && i === controlIndex - 1) || - ((order === 0 || order === 3) && (controlIndex === 0 || controlIndex === ink.length - 1) && (i === 1 || i === ink.length - 2) && closed) || - (order === 3 && i === controlIndex - 1) || - (order === 3 && controlIndex !== ink.length - 1 && i === controlIndex + 1) || - (order === 3 && controlIndex !== ink.length - 1 && i === controlIndex + 2) || - (ink[0].X === ink[ink.length - 1].X && ink[0].Y === ink[ink.length - 1].Y && (i === 0 || i === ink.length - 1) && (controlIndex === 0 || controlIndex === ink.length - 1)) - ) { - return { X: pt.X + deltaX, Y: pt.Y + deltaY }; - } - return pt; + return ink.map((pt, i) => { + const leftHandlePoint = order === 0 && i === controlIndex + 1; + const rightHandlePoint = order === 0 && controlIndex !== 0 && i === controlIndex - 2; + if (controlIndex === i || (order === 0 && controlIndex !== 0 && i === controlIndex - 1) || (order === 3 && i === controlIndex - 1)) { + return { X: pt.X + deltaX, Y: pt.Y + deltaY }; + } + if ( + controlIndex === i || + leftHandlePoint || + rightHandlePoint || + (order === 0 && controlIndex !== 0 && i === controlIndex - 1) || + ((order === 0 || order === 3) && (controlIndex === 0 || controlIndex === ink.length - 1) && (i === 1 || i === ink.length - 2) && closed) || + (order === 3 && i === controlIndex - 1) || + (order === 3 && controlIndex !== ink.length - 1 && i === controlIndex + 1) || + (order === 3 && controlIndex !== ink.length - 1 && i === controlIndex + 2) || + (ink[0].X === ink[ink.length - 1].X && ink[0].Y === ink[ink.length - 1].Y && (i === 0 || i === ink.length - 1) && (controlIndex === 0 || controlIndex === ink.length - 1)) + ) { + return { X: pt.X + deltaX, Y: pt.Y + deltaY }; + } + return pt; + }); }); - }); + }, 'move ink ctrl pt'); public static nearestPtToStroke(ctrlPoints: { X: number; Y: number }[], refInkSpacePt: { X: number; Y: number }, excludeSegs?: number[]) { let distance = Number.MAX_SAFE_INTEGER; @@ -467,8 +467,7 @@ export class InkStrokeProperties { /** * Handles the movement/scaling of a handle point. */ - @undoBatch - moveTangentHandle = (inkView: DocumentView, deltaX: number, deltaY: number, handleIndex: number, oppositeHandleIndex: number, controlIndex: number) => + moveTangentHandle = undoable((inkView: DocumentView, deltaX: number, deltaY: number, handleIndex: number, oppositeHandleIndex: number, controlIndex: number) => { this.applyFunction(inkView, (view: DocumentView, ink: InkData) => { const doc = view.Document; const closed = InkingStroke.IsClosed(ink); @@ -487,4 +486,45 @@ export class InkStrokeProperties { } return inkCopy; }); + }, 'move ink tangent'); + + /** + * Function that "smooths" ink strokes by using the gesture recognizer to detect shapes and + * removing excess control points with the simplify-js package. + * @param inkDocs + * @param tolerance Determines how strong the smooth effect will be + */ + smoothInkStrokes = undoable((inkDocs: Doc[], tolerance: number = 5) => { + inkDocs.forEach(inkDoc => { + const inkView = DocumentView.getDocumentView(inkDoc); + const inkStroke = inkView?.ComponentView as InkingStroke; + const { inkData } = inkStroke.inkScaledData(); + + const result = inkData.length > 2 && GestureUtils.GestureRecognizer.Recognize([inkData]); + console.log(result); + if (result && (result.Name === 'line' ? result.Score > 0.92 : result.Score > 0.85)) { + switch (result.Name) { + case Gestures.Line: + case Gestures.Triangle: + case Gestures.Rectangle: + case Gestures.Circle: + GestureOverlay.makeBezierPolygon(inkData, result.Name, true); + break; + default: + } + } else { + const polylinePoints = inkData.filter((pt, index) => { return index % 4 === 0 || pt === inkData.lastElement()}).map(pt => { return { x: pt.X, y: pt.Y }; }); // prettier-ignore + if (polylinePoints.length > 2) { + const toKeep = simplify(polylinePoints, tolerance).map(pt => {return { X: pt.x, Y: pt.y }}); // prettier-ignore + for (var i = 4; i < inkData.length - 3; i += 4) { + const contains = toKeep.find(pt => pt.X === inkData[i].X && pt.Y === inkData[i].Y); + if (!contains) { + this._currentPoint = i; + inkView && this.deletePoints(inkView, false); + } + } + } + } + }); + }, 'smooth ink stroke'); } diff --git a/src/client/views/InkTranscription.scss b/src/client/views/InkTranscription.scss index bbb0a1afa..18d6b8b10 100644 --- a/src/client/views/InkTranscription.scss +++ b/src/client/views/InkTranscription.scss @@ -2,4 +2,9 @@ .error-msg { display: none !important; } + .ms-editor{ + .smartguide{ + top:1000px; + } + } }
\ No newline at end of file diff --git a/src/client/views/InkTranscription.tsx b/src/client/views/InkTranscription.tsx index 33db72960..485a0e7c4 100644 --- a/src/client/views/InkTranscription.tsx +++ b/src/client/views/InkTranscription.tsx @@ -1,350 +1,431 @@ -// import * as iink from 'iink-js'; -// import { action, observable } from 'mobx'; -// import * as React from 'react'; -// import { Doc, DocListCast } from '../../fields/Doc'; -// import { InkData, InkField, InkTool } from '../../fields/InkField'; -// import { Cast, DateCast, NumCast } from '../../fields/Types'; -// import { aggregateBounds } from '../../Utils'; -// import { DocumentType } from '../documents/DocumentTypes'; -// import { CollectionFreeFormView } from './collections/collectionFreeForm'; -// import { InkingStroke } from './InkingStroke'; -// import './InkTranscription.scss'; - -// /** -// * Class component that handles inking in writing mode -// */ -// export class InkTranscription extends React.Component { -// static Instance: InkTranscription; - -// @observable _mathRegister: any= undefined; -// @observable _mathRef: any= undefined; -// @observable _textRegister: any= undefined; -// @observable _textRef: any= undefined; -// private lastJiix: any; -// private currGroup?: Doc; - -// constructor(props: Readonly<{}>) { -// super(props); - -// InkTranscription.Instance = this; -// } - -// componentWillUnmount() { -// this._mathRef.removeEventListener('exported', (e: any) => this.exportInk(e, this._mathRef)); -// this._textRef.removeEventListener('exported', (e: any) => this.exportInk(e, this._textRef)); -// } - -// @action -// setMathRef = (r: any) => { -// if (!this._mathRegister) { -// this._mathRegister = r -// ? iink.register(r, { -// recognitionParams: { -// type: 'MATH', -// protocol: 'WEBSOCKET', -// server: { -// host: 'cloud.myscript.com', -// applicationKey: process.env.IINKJS_APP, -// hmacKey: process.env.IINKJS_HMAC, -// websocket: { -// pingEnabled: false, -// autoReconnect: true, -// }, -// }, -// iink: { -// math: { -// mimeTypes: ['application/x-latex', 'application/vnd.myscript.jiix'], -// }, -// export: { -// jiix: { -// strokes: true, -// }, -// }, -// }, -// }, -// }) -// : null; -// } - -// r?.addEventListener('exported', (e: any) => this.exportInk(e, this._mathRef)); - -// return (this._mathRef = r); -// }; - -// @action -// setTextRef = (r: any) => { -// if (!this._textRegister) { -// this._textRegister = r -// ? iink.register(r, { -// recognitionParams: { -// type: 'TEXT', -// protocol: 'WEBSOCKET', -// server: { -// host: 'cloud.myscript.com', -// applicationKey: '7277ec34-0c2e-4ee1-9757-ccb657e3f89f', -// hmacKey: 'f5cb18f2-1f95-4ddb-96ac-3f7c888dffc1', -// websocket: { -// pingEnabled: false, -// autoReconnect: true, -// }, -// }, -// iink: { -// text: { -// mimeTypes: ['text/plain'], -// }, -// export: { -// jiix: { -// strokes: true, -// }, -// }, -// }, -// }, -// }) -// : null; -// } - -// r?.addEventListener('exported', (e: any) => this.exportInk(e, this._textRef)); - -// return (this._textRef = r); -// }; - -// /** -// * Handles processing Dash Doc data for ink transcription. -// * -// * @param groupDoc the group which contains the ink strokes we want to transcribe -// * @param inkDocs the ink docs contained within the selected group -// * @param math boolean whether to do math transcription or not -// */ -// transcribeInk = (groupDoc: Doc | undefined, inkDocs: Doc[], math: boolean) => { -// if (!groupDoc) return; -// const validInks = inkDocs.filter(s => s.type === DocumentType.INK); - -// const strokes: InkData[] = []; -// const times: number[] = []; -// validInks -// .filter(i => Cast(i[Doc.LayoutFieldKey(i)], InkField)) -// .forEach(i => { -// const d = Cast(i[Doc.LayoutFieldKey(i)], InkField, null); -// const inkStroke = DocumentManager.Instance.getDocumentView(i)?.ComponentView as InkingStroke; -// strokes.push(d.inkData.map(pd => inkStroke.ptToScreen({ X: pd.X, Y: pd.Y }))); -// times.push(DateCast(i.author_date).getDate().getTime()); -// }); - -// this.currGroup = groupDoc; - -// const pointerData = { events: strokes.map((stroke, i) => this.inkJSON(stroke, times[i])) }; -// const processGestures = false; - -// if (math) { -// this._mathRef.editor.pointerEvents(pointerData, processGestures); -// } else { -// this._textRef.editor.pointerEvents(pointerData, processGestures); -// } -// }; - -// /** -// * Converts the Dash Ink Data to JSON. -// * -// * @param stroke The dash ink data -// * @param time the time of the stroke -// * @returns json object representation of ink data -// */ -// inkJSON = (stroke: InkData, time: number) => { -// return { -// pointerType: 'PEN', -// pointerId: 1, -// x: stroke.map(point => point.X), -// y: stroke.map(point => point.Y), -// t: new Array(stroke.length).fill(time), -// p: new Array(stroke.length).fill(1.0), -// }; -// }; - -// /** -// * Creates subgroups for each word for the whole text transcription -// * @param wordInkDocMap the mapping of words to ink strokes (Ink Docs) -// */ -// subgroupsTranscriptions = (wordInkDocMap: Map<string, Doc[]>) => { -// // iterate through the keys of wordInkDocMap -// wordInkDocMap.forEach(async (inkDocs: Doc[], word: string) => { -// const selected = inkDocs.slice(); -// if (!selected) { -// return; -// } -// const ctx = await Cast(selected[0].embedContainer, Doc); -// if (!ctx) { -// return; -// } -// const docView: CollectionFreeFormView = DocumentManager.Instance.getDocumentView(ctx)?.ComponentView as CollectionFreeFormView; - -// if (!docView) return; -// const marqViewRef = docView._marqueeViewRef.current; -// if (!marqViewRef) return; -// this.groupInkDocs(selected, docView, word); -// }); -// }; - -// /** -// * Event listener function for when the 'exported' event is heard. -// * -// * @param e the event objects -// * @param ref the ref to the editor -// */ -// exportInk = (e: any, ref: any) => { -// const exports = e.detail.exports; -// if (exports) { -// if (exports['application/x-latex']) { -// const latex = exports['application/x-latex']; -// if (this.currGroup) { -// this.currGroup.text = latex; -// this.currGroup.title = latex; -// } - -// ref.editor.clear(); -// } else if (exports['text/plain']) { -// if (exports['application/vnd.myscript.jiix']) { -// this.lastJiix = JSON.parse(exports['application/vnd.myscript.jiix']); -// // map timestamp to strokes -// const timestampWord = new Map<number, string>(); -// this.lastJiix.words.map((word: any) => { -// if (word.items) { -// word.items.forEach((i: { id: string; timestamp: string; X: Array<number>; Y: Array<number>; F: Array<number> }) => { -// const ms = Date.parse(i.timestamp); -// timestampWord.set(ms, word.label); -// }); -// } -// }); - -// const wordInkDocMap = new Map<string, Doc[]>(); -// if (this.currGroup) { -// const docList = DocListCast(this.currGroup.data); -// docList.forEach((inkDoc: Doc) => { -// // just having the times match up and be a unique value (actual timestamp doesn't matter) -// const ms = DateCast(inkDoc.author_date).getDate().getTime() + 14400000; -// const word = timestampWord.get(ms); -// if (!word) { -// return; -// } -// const entry = wordInkDocMap.get(word); -// if (entry) { -// entry.push(inkDoc); -// wordInkDocMap.set(word, entry); -// } else { -// const newEntry = [inkDoc]; -// wordInkDocMap.set(word, newEntry); -// } -// }); -// if (this.lastJiix.words.length > 1) this.subgroupsTranscriptions(wordInkDocMap); -// } -// } -// const text = exports['text/plain']; - -// if (this.currGroup) { -// this.currGroup.text = text; // transcription text -// this.currGroup.icon_fieldKey = 'transcription'; // use the transcription icon template when iconifying -// this.currGroup.title = text.split('\n')[0]; -// } - -// ref.editor.clear(); -// } -// } -// }; - -// /** -// * Creates the ink grouping once the user leaves the writing mode. -// */ -// createInkGroup() { -// // TODO nda - if document being added to is a inkGrouping then we can just add to that group -// if (Doc.ActiveTool === InkTool.Write) { -// CollectionFreeFormView.collectionsWithUnprocessedInk.forEach(ffView => { -// // TODO: nda - will probably want to go through ffView unprocessed docs and then see if any of the inksToGroup docs are in it and only use those -// const selected = ffView.unprocessedDocs; -// const newCollection = this.groupInkDocs( -// selected.filter(doc => doc.embedContainer), -// ffView -// ); -// ffView.unprocessedDocs = []; - -// InkTranscription.Instance.transcribeInk(newCollection, selected, false); -// }); -// } -// CollectionFreeFormView.collectionsWithUnprocessedInk.clear(); -// } - -// /** -// * Creates the groupings for a given list of ink docs on a specific doc view -// * @param selected: the list of ink docs to create a grouping of -// * @param docView: the view in which we want the grouping to be created -// * @param word: optional param if the group we are creating is a word (subgrouping individual words) -// * @returns a new collection Doc or undefined if the grouping fails -// */ -// groupInkDocs(selected: Doc[], docView: CollectionFreeFormView, word?: string): Doc | undefined { -// const bounds: { x: number; y: number; width?: number; height?: number }[] = []; - -// // calculate the necessary bounds from the selected ink docs -// selected.map( -// action(d => { -// const x = NumCast(d.x); -// const y = NumCast(d.y); -// const width = NumCast(d._width); -// const height = NumCast(d._height); -// bounds.push({ x, y, width, height }); -// }) -// ); - -// // calculate the aggregated bounds -// const aggregBounds = aggregateBounds(bounds, 0, 0); -// const marqViewRef = docView._marqueeViewRef.current; - -// // set the vals for bounds in marqueeView -// if (marqViewRef) { -// marqViewRef._downX = aggregBounds.x; -// marqViewRef._downY = aggregBounds.y; -// marqViewRef._lastX = aggregBounds.r; -// marqViewRef._lastY = aggregBounds.b; -// } - -// // map through all the selected ink strokes and create the groupings -// selected.map( -// action(d => { -// const dx = NumCast(d.x); -// const dy = NumCast(d.y); -// delete d.x; -// delete d.y; -// delete d.activeFrame; -// delete d._timecodeToShow; // bcz: this should be automatic somehow.. along with any other properties that were logically associated with the original collection -// delete d._timecodeToHide; // bcz: this should be automatic somehow.. along with any other properties that were logically associated with the original collection -// // calculate pos based on bounds -// if (marqViewRef?.Bounds) { -// d.x = dx - marqViewRef.Bounds.left - marqViewRef.Bounds.width / 2; -// d.y = dy - marqViewRef.Bounds.top - marqViewRef.Bounds.height / 2; -// } -// return d; -// }) -// ); -// docView.props.removeDocument?.(selected); -// // Gets a collection based on the selected nodes using a marquee view ref -// const newCollection = marqViewRef?.getCollection(selected, undefined, true); -// if (newCollection) { -// newCollection.width = NumCast(newCollection._width); -// newCollection.height = NumCast(newCollection._height); -// // if the grouping we are creating is an individual word -// if (word) { -// newCollection.title = word; -// } -// } - -// // nda - bug: when deleting a stroke before leaving writing mode, delete the stroke from unprocessed ink docs -// newCollection && docView.props.addDocument?.(newCollection); -// return newCollection; -// } - -// render() { -// return ( -// <div className="ink-transcription"> -// <div className="math-editor" ref={this.setMathRef} touch-action="none"></div> -// <div className="text-editor" ref={this.setTextRef} touch-action="none"></div> -// </div> -// ); -// } -// } +import * as iink from 'iink-ts'; +import { action, observable } from 'mobx'; +import * as React from 'react'; +import { Doc, DocListCast } from '../../fields/Doc'; +import { InkData, InkField, InkTool } from '../../fields/InkField'; +import { Cast, DateCast, ImageCast, NumCast, StrCast } from '../../fields/Types'; +import { aggregateBounds } from '../../Utils'; +import { DocumentType } from '../documents/DocumentTypes'; +import { CollectionFreeFormView, MarqueeView } from './collections/collectionFreeForm'; +import { InkingStroke } from './InkingStroke'; +import './InkTranscription.scss'; +import { Docs } from '../documents/Documents'; +import { DocumentView } from './nodes/DocumentView'; +import { Number } from 'mongoose'; +import { NumberArray } from 'd3'; +import { ImageField } from '../../fields/URLField'; +import { gptHandwriting } from '../apis/gpt/GPT'; +import * as fs from 'fs'; +import { URLField } from '../../fields/URLField'; +/** + * Class component that handles inking in writing mode + */ +export class InkTranscription extends React.Component { + static Instance: InkTranscription; + + @observable _mathRegister: any = undefined; + @observable _mathRef: any = undefined; + @observable _textRegister: any = undefined; + @observable _textRef: any = undefined; + @observable iinkEditor: any = undefined; + private lastJiix: any; + private currGroup?: Doc; + private collectionFreeForm?: CollectionFreeFormView; + + constructor(props: Readonly<{}>) { + super(props); + + InkTranscription.Instance = this; + } + + componentWillUnmount() { + // this._mathRef.removeEventListener('exported', (e: any) => this.exportInk(e, this._mathRef)); + // this._textRef.removeEventListener('exported', (e: any) => this.exportInk(e, this._textRef)); + } + + @action + setMathRef = async (r: any) => { + if (!this._textRegister && r) { + let editor; + const options = { + configuration: { + server: { + scheme: 'https', + host: 'cloud.myscript.com', + applicationKey: 'c0901093-5ac5-4454-8e64-0def0f13f2ca', + hmacKey: 'f6465cca-1856-4492-a6a4-e2395841be2f', + protocol: 'WEBSOCKET', + }, + recognition: { + type: 'TEXT', + }, + }, + }; + + editor = new iink.Editor(r, options as any); + + await editor.initialize(); + + this._textRegister = r; + r?.addEventListener('exported', (e: any) => this.exportInk(e, this._textRef)); + + return (this._textRef = r); + } + }; + @action + setTextRef = async (r: any) => { + if (!this._textRegister && r) { + let editor; + const options = { + configuration: { + server: { + scheme: 'https', + host: 'cloud.myscript.com', + applicationKey: 'c0901093-5ac5-4454-8e64-0def0f13f2ca', + hmacKey: 'f6465cca-1856-4492-a6a4-e2395841be2f', + protocol: 'WEBSOCKET', + }, + recognition: { + type: 'TEXT', + }, + }, + }; + + editor = new iink.Editor(r, options as any); + + await editor.initialize(); + this.iinkEditor = editor; + this._textRegister = r; + r?.addEventListener('exported', (e: any) => this.exportInk(e, this._textRef)); + + return (this._textRef = r); + } + }; + + /** + * Handles processing Dash Doc data for ink transcription. + * + * @param groupDoc the group which contains the ink strokes we want to transcribe + * @param inkDocs the ink docs contained within the selected group + * @param math boolean whether to do math transcription or not + */ + transcribeInk = (groupDoc: Doc | undefined, inkDocs: Doc[], math: boolean) => { + if (!groupDoc) return; + const validInks = inkDocs.filter(s => s.type === DocumentType.INK); + + const strokes: InkData[] = []; + + const times: number[] = []; + validInks + .filter(i => Cast(i[Doc.LayoutFieldKey(i)], InkField)) + .forEach(i => { + const d = Cast(i[Doc.LayoutFieldKey(i)], InkField, null); + const inkStroke = DocumentView.getDocumentView(i)?.ComponentView as InkingStroke; + strokes.push(d.inkData.map(pd => inkStroke.ptToScreen({ X: pd.X, Y: pd.Y }))); + times.push(DateCast(i.author_date).getDate().getTime()); + }); + console.log(strokes); + console.log( + `this.Multistrokes.push( + new Multistroke( + Gestures.Scribble, + useBoundedRotationInvariance, + new Array([ + ` + + this.convertPointsToString(strokes) + + ` + ]) + ) + ) + ` + ); + //console.log(this.convertPointsToString(strokes)); + //console.log(this.convertPointsToString2(strokes)); + this.currGroup = groupDoc; + const pointerData = strokes.map((stroke, i) => this.inkJSON(stroke, times[i])); + const processGestures = false; + if (math) { + console.log('math'); + this.iinkEditor.importPointEvents(pointerData); + } else { + this.iinkEditor.importPointEvents(pointerData); + } + }; + convertPointsToString(points: InkData[]): string { + return points[0].map(point => `new Point(${point.X}, ${point.Y})`).join(','); + } + convertPointsToString2(points: InkData[]): string { + return points[0].map(point => `(${point.X},${point.Y})`).join(','); + } + + /** + * Converts the Dash Ink Data to JSON. + * + * @param stroke The dash ink data + * @param time the time of the stroke + * @returns json object representation of ink data + */ + inkJSON = (stroke: InkData, time: number) => { + interface strokeData { + x: number; + y: number; + t: number; + p: number; + } + let strokeObjects: strokeData[] = []; + stroke.forEach(point => { + let tempObject: strokeData = { + x: point.X, + y: point.Y, + t: time, + p: 1.0, + }; + strokeObjects.push(tempObject); + }); + return { + pointerType: 'PEN', + pointerId: 1, + pointers: strokeObjects, + }; + }; + + /** + * Creates subgroups for each word for the whole text transcription + * @param wordInkDocMap the mapping of words to ink strokes (Ink Docs) + */ + subgroupsTranscriptions = (wordInkDocMap: Map<string, Doc[]>) => { + // iterate through the keys of wordInkDocMap + wordInkDocMap.forEach(async (inkDocs: Doc[], word: string) => { + const selected = inkDocs.slice(); + if (!selected) { + return; + } + const ctx = await Cast(selected[0].embedContainer, Doc); + if (!ctx) { + return; + } + const docView: CollectionFreeFormView = DocumentView.getDocumentView(ctx)?.ComponentView as CollectionFreeFormView; + // DocumentManager.Instance.getDocumentView(ctx)?.ComponentView as CollectionFreeFormView; + + if (!docView) return; + const marqViewRef = docView._marqueeViewRef.current; + if (!marqViewRef) return; + this.groupInkDocs(selected, docView, word); + }); + }; + + /** + * Event listener function for when the 'exported' event is heard. + * + * @param e the event objects + * @param ref the ref to the editor + */ + exportInk = async (e: any, ref: any) => { + const exports = e.detail['application/vnd.myscript.jiix']; + if (exports) { + if (exports['type'] == 'Math') { + const latex = exports['application/x-latex']; + if (this.currGroup) { + this.currGroup.text = latex; + this.currGroup.title = latex; + } + + ref.editor.clear(); + } else if (exports['type'] == 'Text') { + if (exports['application/vnd.myscript.jiix']) { + this.lastJiix = JSON.parse(exports['application/vnd.myscript.jiix']); + // map timestamp to strokes + const timestampWord = new Map<number, string>(); + this.lastJiix.words.map((word: any) => { + if (word.items) { + word.items.forEach((i: { id: string; timestamp: string; X: Array<number>; Y: Array<number>; F: Array<number> }) => { + const ms = Date.parse(i.timestamp); + timestampWord.set(ms, word.label); + }); + } + }); + + const wordInkDocMap = new Map<string, Doc[]>(); + if (this.currGroup) { + const docList = DocListCast(this.currGroup.data); + docList.forEach((inkDoc: Doc) => { + // just having the times match up and be a unique value (actual timestamp doesn't matter) + const ms = DateCast(inkDoc.author_date).getDate().getTime() + 14400000; + const word = timestampWord.get(ms); + if (!word) { + return; + } + const entry = wordInkDocMap.get(word); + if (entry) { + entry.push(inkDoc); + wordInkDocMap.set(word, entry); + } else { + const newEntry = [inkDoc]; + wordInkDocMap.set(word, newEntry); + } + }); + if (this.lastJiix.words.length > 1) this.subgroupsTranscriptions(wordInkDocMap); + } + } + const text = exports['label']; + + if (this.currGroup && text) { + DocumentView.getDocumentView(this.currGroup)?.ComponentView?.updateIcon?.(); + this.currGroup.transcription = text; + this.currGroup.title = text; + let image = await this.getIcon(); + const pathname = image?.url.href as string; + console.log(image?.url); + console.log(image); + const { href } = (image as URLField).url; + const hrefParts = href.split('.'); + const hrefComplete = `${hrefParts[0]}_o.${hrefParts[1]}`; + let response; + try { + const hrefBase64 = await this.imageUrlToBase64(hrefComplete); + response = await gptHandwriting(hrefBase64); + console.log(response); + } catch (error) { + console.log('bad things have happened'); + } + const textBoxText = 'iink: ' + text + '\n' + '\n' + 'ChatGPT: ' + response; + if (!this.currGroup.hasTextBox) { + const newDoc = Docs.Create.TextDocument(textBoxText, { title: '', x: this.currGroup.x as number, y: (this.currGroup.y as number) + (this.currGroup.height as number) }); + newDoc.height = 200; + this.collectionFreeForm?.addDocument(newDoc); + this.currGroup.hasTextBox = true; + } + ref.editor.clear(); + } + } + } + }; + async getIcon() { + const docView = DocumentView.getDocumentView(this.currGroup); + console.log(this.currGroup); + if (docView) { + console.log(docView); + docView.ComponentView?.updateIcon?.(); + return new Promise<ImageField | undefined>(res => setTimeout(() => res(ImageCast(docView.Document.icon)), 1000)); + } + return undefined; + } + imageUrlToBase64 = async (imageUrl: string): Promise<string> => { + try { + const response = await fetch(imageUrl); + const blob = await response.blob(); + + return new Promise((resolve, reject) => { + const reader = new FileReader(); + reader.readAsDataURL(blob); + reader.onloadend = () => resolve(reader.result as string); + reader.onerror = error => reject(error); + }); + } catch (error) { + console.error('Error:', error); + throw error; + } + }; + + /** + * Creates the ink grouping once the user leaves the writing mode. + */ + createInkGroup() { + // TODO nda - if document being added to is a inkGrouping then we can just add to that group + if (Doc.ActiveTool === InkTool.Write) { + CollectionFreeFormView.collectionsWithUnprocessedInk.forEach(ffView => { + // TODO: nda - will probably want to go through ffView unprocessed docs and then see if any of the inksToGroup docs are in it and only use those + const selected = ffView.unprocessedDocs; + const newCollection = this.groupInkDocs( + selected.filter(doc => doc.embedContainer), + ffView + ); + ffView.unprocessedDocs = []; + + InkTranscription.Instance.transcribeInk(newCollection, selected, false); + }); + } + CollectionFreeFormView.collectionsWithUnprocessedInk.clear(); + } + + /** + * Creates the groupings for a given list of ink docs on a specific doc view + * @param selected: the list of ink docs to create a grouping of + * @param docView: the view in which we want the grouping to be created + * @param word: optional param if the group we are creating is a word (subgrouping individual words) + * @returns a new collection Doc or undefined if the grouping fails + */ + groupInkDocs(selected: Doc[], docView: CollectionFreeFormView, word?: string): Doc | undefined { + this.collectionFreeForm = docView; + const bounds: { x: number; y: number; width?: number; height?: number }[] = []; + + // calculate the necessary bounds from the selected ink docs + selected.map( + action(d => { + const x = NumCast(d.x); + const y = NumCast(d.y); + const width = NumCast(d._width); + const height = NumCast(d._height); + bounds.push({ x, y, width, height }); + }) + ); + + // calculate the aggregated bounds + const aggregBounds = aggregateBounds(bounds, 0, 0); + const marqViewRef = docView._marqueeViewRef.current; + + // set the vals for bounds in marqueeView + if (marqViewRef) { + marqViewRef._downX = aggregBounds.x; + marqViewRef._downY = aggregBounds.y; + marqViewRef._lastX = aggregBounds.r; + marqViewRef._lastY = aggregBounds.b; + } + + // map through all the selected ink strokes and create the groupings + selected.map( + action(d => { + const dx = NumCast(d.x); + const dy = NumCast(d.y); + delete d.x; + delete d.y; + delete d.activeFrame; + delete d._timecodeToShow; // bcz: this should be automatic somehow.. along with any other properties that were logically associated with the original collection + delete d._timecodeToHide; // bcz: this should be automatic somehow.. along with any other properties that were logically associated with the original collection + // calculate pos based on bounds + if (marqViewRef?.Bounds) { + d.x = dx - marqViewRef.Bounds.left - marqViewRef.Bounds.width / 2; + d.y = dy - marqViewRef.Bounds.top - marqViewRef.Bounds.height / 2; + } + return d; + }) + ); + docView.props.removeDocument?.(selected); + // Gets a collection based on the selected nodes using a marquee view ref + const newCollection = MarqueeView.getCollection(selected, undefined, true, { top: 1, left: 1, width: 1, height: 1 }); + if (newCollection) { + newCollection.width = NumCast(newCollection._width); + newCollection.height = NumCast(newCollection._height); + // if the grouping we are creating is an individual word + if (word) { + newCollection.title = word; + } + } + + // nda - bug: when deleting a stroke before leaving writing mode, delete the stroke from unprocessed ink docs + newCollection && docView.props.addDocument?.(newCollection); + if (newCollection) { + newCollection.hasTextBox = false; + } + return newCollection; + } + + render() { + return ( + <div className="ink-transcription"> + <div className="math-editor" ref={this.setMathRef} touch-action="none"></div> + <div className="text-editor" ref={this.setTextRef} touch-action="none"></div> + </div> + ); + } +} diff --git a/src/client/views/InkingStroke.tsx b/src/client/views/InkingStroke.tsx index 2e82371cb..32bbd57c6 100644 --- a/src/client/views/InkingStroke.tsx +++ b/src/client/views/InkingStroke.tsx @@ -107,10 +107,10 @@ export class InkingStroke extends ViewBoxAnnotatableComponent<FieldViewProps>() * analyzes the ink stroke and saves the analysis of the stroke to the 'inkAnalysis' field, * and the recognized words to the 'handwriting' */ - analyzeStrokes() { - const data: InkData = Cast(this.dataDoc[this.fieldKey], InkField)?.inkData ?? []; + analyzeStrokes = () => { + const data: InkData = this.inkScaledData().inkData ?? []; CognitiveServices.Inking.Appliers.ConcatenateHandwriting(this.dataDoc, ['inkAnalysis', 'handwriting'], [data]); - } + }; /** * Toggles whether the ink stroke is displayed as an overlay mask or as a regular stroke. diff --git a/src/client/views/LightboxView.scss b/src/client/views/LightboxView.scss index 6da5c0338..3e65843df 100644 --- a/src/client/views/LightboxView.scss +++ b/src/client/views/LightboxView.scss @@ -1,7 +1,7 @@ .lightboxView-navBtn { margin: auto; position: absolute; - right: 10; + right: 19; top: 10; background: transparent; border-radius: 8; @@ -16,7 +16,7 @@ .lightboxView-tabBtn { margin: auto; position: absolute; - right: 45; + right: 54; top: 10; background: transparent; border-radius: 8; @@ -28,10 +28,26 @@ opacity: 1; } } +.lightboxView-paletteBtn { + margin: auto; + position: absolute; + right: 89; + top: 10; + background: transparent; + border-radius: 8; + opacity: 0.7; + width: 25; + flex-direction: column; + display: flex; + &:hover { + opacity: 1; + } +} + .lightboxView-penBtn { margin: auto; position: absolute; - right: 80; + right: 124; top: 10; background: transparent; border-radius: 8; @@ -46,7 +62,7 @@ .lightboxView-exploreBtn { margin: auto; position: absolute; - right: 115; + right: 159; top: 10; background: transparent; border-radius: 8; diff --git a/src/client/views/LightboxView.tsx b/src/client/views/LightboxView.tsx index b8b73e7dd..77a2c1b86 100644 --- a/src/client/views/LightboxView.tsx +++ b/src/client/views/LightboxView.tsx @@ -10,7 +10,7 @@ import { emptyFunction } from '../../Utils'; import { CreateLinkToActiveAudio, Doc, DocListCast, FieldResult, Opt, returnEmptyDoclist } from '../../fields/Doc'; import { Id } from '../../fields/FieldSymbols'; import { InkTool } from '../../fields/InkField'; -import { BoolCast, Cast, NumCast, toList } from '../../fields/Types'; +import { BoolCast, Cast, DocCast, NumCast, toList } from '../../fields/Types'; import { ScriptingGlobals } from '../util/ScriptingGlobals'; import { SnappingManager } from '../util/SnappingManager'; import { Transform } from '../util/Transform'; @@ -21,6 +21,7 @@ import { OverlayView } from './OverlayView'; import { DefaultStyleProvider, returnEmptyDocViewList, wavyBorderPath } from './StyleProvider'; import { DocumentView } from './nodes/DocumentView'; import { OpenWhere, OpenWhereMod } from './nodes/OpenWhere'; +import { AnnotationPalette } from './smartdraw/AnnotationPalette'; interface LightboxViewProps { PanelWidth: number; @@ -38,7 +39,12 @@ export class LightboxView extends ObservableReactComponent<LightboxViewProps> { * @param view * @returns true if a DocumentView is descendant of the lightbox view */ - public static Contains(view?:DocumentView) { return view && LightboxView.Instance?._docView && (view.containerViewPath?.() ?? []).concat(view).includes(LightboxView.Instance?._docView); } // prettier-ignore + public static Contains(view?: DocumentView) { + return ( + (view && LightboxView.Instance?._docView && (view.containerViewPath?.() ?? []).concat(view).includes(LightboxView.Instance?._docView)) || + (view && LightboxView.Instance?._annoPaletteView?.Contains(view)) || undefined + ); + } // prettier-ignore public static LightboxDoc = () => LightboxView.Instance?._doc; // eslint-disable-next-line no-use-before-define static Instance: LightboxView; @@ -51,12 +57,14 @@ export class LightboxView extends ObservableReactComponent<LightboxViewProps> { }[] = []; private _savedState: LightboxSavedState = {}; private _history: { doc: Doc; target?: Doc }[] = []; + private _annoPaletteView: AnnotationPalette | null = null; @observable private _future: Doc[] = []; @observable private _layoutTemplate: Opt<Doc> = undefined; @observable private _layoutTemplateString: Opt<string> = undefined; @observable private _doc: Opt<Doc> = undefined; @observable private _docTarget: Opt<Doc> = undefined; @observable private _docView: Opt<DocumentView> = undefined; + @observable private _showPalette: boolean = false; @computed get leftBorder() { return Math.min(this._props.PanelWidth / 4, this._props.maxBorder[0]); } // prettier-ignore @computed get topBorder() { return Math.min(this._props.PanelHeight / 4, this._props.maxBorder[1]); } // prettier-ignore @@ -69,6 +77,7 @@ export class LightboxView extends ObservableReactComponent<LightboxViewProps> { DocumentView._lightboxContains = LightboxView.Contains; DocumentView._lightboxDoc = LightboxView.LightboxDoc; } + /** * Sets the root Doc to render in the lightbox view. * @param doc @@ -101,6 +110,7 @@ export class LightboxView extends ObservableReactComponent<LightboxViewProps> { this._history = []; Doc.ActiveTool = InkTool.None; SnappingManager.SetExploreMode(false); + this._showPalette = false; } DocumentView.DeselectAll(); if (future) { @@ -200,6 +210,10 @@ export class LightboxView extends ObservableReactComponent<LightboxViewProps> { toggleFitWidth = () => { this._doc && (this._doc._layout_fitWidth = !this._doc._layout_fitWidth); }; + togglePalette = () => { + this._showPalette = !this._showPalette; + // if (this._showPalette === false) AnnotationPalette.Instance.resetPalette(true); + }; togglePen = () => { Doc.ActiveTool = Doc.ActiveTool === InkTool.Pen ? InkTool.None : InkTool.Pen; }; @@ -304,6 +318,7 @@ export class LightboxView extends ObservableReactComponent<LightboxViewProps> { </GestureOverlay> </div> + {this._showPalette && <AnnotationPalette ref={r => (this._annoPaletteView = r)} Document={DocCast(Doc.UserDoc().myLightboxDrawings)} />} {this.renderNavBtn(0, undefined, this._props.PanelHeight / 2 - 12.5, 'chevron-left', this._doc && this._history.length ? true : false, this.previous)} {this.renderNavBtn( this._props.PanelWidth - Math.min(this._props.PanelWidth / 4, this._props.maxBorder[0]), @@ -316,7 +331,8 @@ export class LightboxView extends ObservableReactComponent<LightboxViewProps> { )} <LightboxTourBtn lightboxDoc={this.lightboxDoc} navBtn={this.renderNavBtn} future={this.future} stepInto={this.stepInto} /> {toggleBtn('lightboxView-navBtn', 'toggle reading view', BoolCast(this._doc?._layout_fitWidth), 'book-open', 'book', this.toggleFitWidth)} - {toggleBtn('lightboxView-tabBtn', 'open document in a tab', false, 'file-download', '', this.downloadDoc)} + {toggleBtn('lightboxView-tabBtn', 'open document in a tab', false, 'file-export', '', this.downloadDoc)} + {toggleBtn('lightboxView-paletteBtn', 'toggle annotation palette', this._showPalette === true, 'palette', '', this.togglePalette)} {toggleBtn('lightboxView-penBtn', 'toggle pen annotation', Doc.ActiveTool === InkTool.Pen, 'pen', '', this.togglePen)} {toggleBtn('lightboxView-exploreBtn', 'toggle navigate only mode', SnappingManager.ExploreMode, 'globe-americas', '', this.toggleExplore)} </div> diff --git a/src/client/views/Main.tsx b/src/client/views/Main.tsx index f7cd0e925..94294e97a 100644 --- a/src/client/views/Main.tsx +++ b/src/client/views/Main.tsx @@ -4,6 +4,8 @@ // } import * as dotenv from 'dotenv'; // see https://github.com/motdotla/dotenv#how-do-i-use-dotenv-with-import +import { Node } from 'prosemirror-model'; +import { EditorView } from 'prosemirror-view'; import * as React from 'react'; import * as ReactDOM from 'react-dom/client'; import { AssignAllExtensions } from '../../extensions/Extensions'; @@ -23,6 +25,8 @@ import { CollectionView } from './collections/CollectionView'; import { TabDocView } from './collections/TabDocView'; import { CollectionFreeFormView } from './collections/collectionFreeForm'; import { CollectionFreeFormInfoUI } from './collections/collectionFreeForm/CollectionFreeFormInfoUI'; +import { FaceCollectionBox, UniqueFaceBox } from './collections/collectionFreeForm/FaceCollectionBox'; +import { ImageLabelBox } from './collections/collectionFreeForm/ImageLabelBox'; import { CollectionSchemaView } from './collections/collectionSchema/CollectionSchemaView'; import { SchemaRowBox } from './collections/collectionSchema/SchemaRowBox'; import './global/globalScripts'; @@ -60,12 +64,9 @@ import { FormattedTextBox } from './nodes/formattedText/FormattedTextBox'; import { SummaryView } from './nodes/formattedText/SummaryView'; import { ImportElementBox } from './nodes/importBox/ImportElementBox'; import { PresBox, PresElementBox } from './nodes/trails'; -import { SearchBox } from './search/SearchBox'; -import { ImageLabelBox } from './collections/collectionFreeForm/ImageLabelBox'; import { FaceRecognitionHandler } from './search/FaceRecognitionHandler'; -import { FaceCollectionBox, UniqueFaceBox } from './collections/collectionFreeForm/FaceCollectionBox'; -import { Node } from 'prosemirror-model'; -import { EditorView } from 'prosemirror-view'; +import { SearchBox } from './search/SearchBox'; +import { AnnotationPalette } from './smartdraw/AnnotationPalette'; dotenv.config(); @@ -118,6 +119,7 @@ FieldLoader.ServerLoadStatus = { requested: 0, retrieved: 0, message: 'cache' }; KeyValueBox.Init(); PresBox.Init(TabDocView.AllTabDocs); DocumentContentsView.Init(KeyValueBox.LayoutString(), { + AnnotationPalette, FormattedTextBox, ImageBox, FontIconBox, diff --git a/src/client/views/MainView.tsx b/src/client/views/MainView.tsx index 9f1c7da3d..2adeefa37 100644 --- a/src/client/views/MainView.tsx +++ b/src/client/views/MainView.tsx @@ -74,6 +74,8 @@ import { PresBox } from './nodes/trails'; import { AnchorMenu } from './pdf/AnchorMenu'; import { GPTPopup } from './pdf/GPTPopup/GPTPopup'; import { TopBar } from './topbar/TopBar'; +import { SmartDrawHandler } from './smartdraw/SmartDrawHandler'; +import { InkTranscription } from './InkTranscription'; // eslint-disable-next-line @typescript-eslint/no-var-requires, @typescript-eslint/no-require-imports const { LEFT_MENU_WIDTH, TOPBAR_HEIGHT } = require('./global/globalCssVariables.module.scss'); // prettier-ignore @@ -315,6 +317,7 @@ export class MainView extends ObservableReactComponent<object> { fa.faCompass, fa.faSnowflake, fa.faStar, + fa.faSplotch, fa.faMicrophone, fa.faCircleHalfStroke, fa.faKeyboard, @@ -337,6 +340,7 @@ export class MainView extends ObservableReactComponent<object> { fa.faTerminal, fa.faToggleOn, fa.faFile, + fa.faFileExport, fa.faLocationArrow, fa.faSearch, fa.faFileDownload, @@ -376,6 +380,7 @@ export class MainView extends ObservableReactComponent<object> { fa.faXmark, fa.faExclamation, fa.faFileAlt, + fa.faFileArrowDown, fa.faFileAudio, fa.faFileVideo, fa.faFilePdf, @@ -392,6 +397,7 @@ export class MainView extends ObservableReactComponent<object> { fa.faArrowsLeftRight, fa.faPause, fa.faPen, + fa.faUserPen, fa.faPenNib, fa.faPhone, fa.faPlay, @@ -429,6 +435,7 @@ export class MainView extends ObservableReactComponent<object> { fa.faBold, fa.faItalic, fa.faClipboard, + fa.faClipboardCheck, fa.faUnderline, fa.faStrikethrough, fa.faSuperscript, @@ -437,6 +444,7 @@ export class MainView extends ObservableReactComponent<object> { fa.faEyeDropper, fa.faPaintRoller, fa.faBars, + fa.faBarsStaggered, fa.faBrush, fa.faShapes, fa.faEllipsisH, @@ -475,6 +483,8 @@ export class MainView extends ObservableReactComponent<object> { fa.faHashtag, fa.faAlignJustify, fa.faCheckSquare, + fa.faSquarePlus, + fa.faReply, fa.faListUl, fa.faWindowMinimize, fa.faWindowRestore, @@ -569,6 +579,7 @@ export class MainView extends ObservableReactComponent<object> { DocumentManager.removeOverlayViews(); Doc.linkFollowUnhighlight(); const targets = document.elementsFromPoint(e.x, e.y); + const targetClasses = targets.map(target => target.className.toString()); if (targets.length) { let targClass = targets[0].className.toString(); for (let i = 0; i < targets.length - 1; i++) { @@ -576,6 +587,8 @@ export class MainView extends ObservableReactComponent<object> { else break; } !targClass.includes('contextMenu') && ContextMenu.Instance.closeMenu(); + !targetClasses.includes('marqueeView') && !targetClasses.includes('smart-draw-handler') && SmartDrawHandler.Instance.hideSmartDrawHandler(); + !targetClasses.includes('smart-draw-handler') && SmartDrawHandler.Instance.hideRegenerate(); !['timeline-menu-desc', 'timeline-menu-item', 'timeline-menu-input'].includes(targClass) && TimelineMenu.Instance.closeMenu(); } }); @@ -1074,6 +1087,7 @@ export class MainView extends ObservableReactComponent<object> { <TaskCompletionBox /> <ContextMenu /> <ImageLabelHandler /> + <SmartDrawHandler /> <AnchorMenu /> <MapAnchorMenu /> <DirectionsAnchorMenu /> @@ -1081,6 +1095,7 @@ export class MainView extends ObservableReactComponent<object> { <MarqueeOptionsMenu /> <TimelineMenu /> <RichTextMenu /> + <InkTranscription /> {this.snapLines} <LightboxView key="lightbox" PanelWidth={this._windowWidth} addSplit={CollectionDockingView.AddSplit} PanelHeight={this._windowHeight} maxBorder={this.lightboxMaxBorder} /> <GPTPopup key="gptpopup" /> diff --git a/src/client/views/MarqueeAnnotator.tsx b/src/client/views/MarqueeAnnotator.tsx index 8aed34d24..90323086c 100644 --- a/src/client/views/MarqueeAnnotator.tsx +++ b/src/client/views/MarqueeAnnotator.tsx @@ -27,7 +27,7 @@ export interface MarqueeAnnotatorProps { containerOffset?: () => number[]; marqueeContainer: HTMLDivElement; docView: () => DocumentView; - savedAnnotations: () => ObservableMap<number, (HTMLDivElement& { marqueeing?: boolean})[]>; + savedAnnotations: () => ObservableMap<number, (HTMLDivElement & { marqueeing?: boolean })[]>; selectionText: () => string; annotationLayer: HTMLDivElement; addDocument: (doc: Doc) => boolean; @@ -60,7 +60,7 @@ export class MarqueeAnnotator extends ObservableReactComponent<MarqueeAnnotatorP }); @undoBatch - makeAnnotationDocument = (color: string, isLinkButton?: boolean, savedAnnotations?: ObservableMap<number, (HTMLDivElement& { marqueeing?: boolean})[]>): Opt<Doc> => { + makeAnnotationDocument = (color: string, isLinkButton?: boolean, savedAnnotations?: ObservableMap<number, (HTMLDivElement & { marqueeing?: boolean })[]>): Opt<Doc> => { const savedAnnoMap = savedAnnotations?.values() && Array.from(savedAnnotations?.values()).length ? savedAnnotations : this.props.savedAnnotations(); if (savedAnnoMap.size === 0) return undefined; const savedAnnos = Array.from(savedAnnoMap.values())[0]; @@ -137,7 +137,7 @@ export class MarqueeAnnotator extends ObservableReactComponent<MarqueeAnnotatorP return annotationDoc as Doc; }; - public static previewNewAnnotation = action((savedAnnotations: ObservableMap<number, (HTMLDivElement& { marqueeing?: boolean})[]>, annotationLayer: HTMLDivElement & { marqueeing?: boolean}, div: HTMLDivElement, page: number) => { + public static previewNewAnnotation = action((savedAnnotations: ObservableMap<number, (HTMLDivElement & { marqueeing?: boolean })[]>, annotationLayer: HTMLDivElement & { marqueeing?: boolean }, div: HTMLDivElement, page: number) => { div.style.backgroundColor = '#ACCEF7'; div.style.opacity = '0.5'; annotationLayer.append(div); @@ -265,7 +265,7 @@ export class MarqueeAnnotator extends ObservableReactComponent<MarqueeAnnotatorP if (!this.isEmpty && marqueeStyle) { // configure and show the annotation/link menu if a the drag region is big enough // copy the temporary marquee to allow for multiple selections (not currently available though). - const copy: (HTMLDivElement & {marqueeing?: boolean}) = document.createElement('div'); + const copy: HTMLDivElement & { marqueeing?: boolean } = document.createElement('div'); const scale = (this.props.scaling?.() || 1) * NumCast(this.props.Document._freeform_scale, 1); ['border', 'opacity', 'top', 'left', 'width', 'height'].forEach(prop => { copy.style[prop as unknown as number] = marqueeStyle[prop as unknown as number]; // bcz: hack to get around TS type checking for array index with strings diff --git a/src/client/views/PropertiesView.scss b/src/client/views/PropertiesView.scss index 840df41e7..a5e60b831 100644 --- a/src/client/views/PropertiesView.scss +++ b/src/client/views/PropertiesView.scss @@ -638,3 +638,9 @@ padding-left: 8px; background-color: rgb(51, 51, 51); } + +.smooth, +.color, +.smooth-slider { + margin-top: 7px; +} diff --git a/src/client/views/PropertiesView.tsx b/src/client/views/PropertiesView.tsx index daa8e1720..6383d4947 100644 --- a/src/client/views/PropertiesView.tsx +++ b/src/client/views/PropertiesView.tsx @@ -2,7 +2,7 @@ import { IconLookup, IconProp } from '@fortawesome/fontawesome-svg-core'; import { faAnchor, faArrowRight, faWindowMaximize } from '@fortawesome/free-solid-svg-icons'; import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; import { Checkbox, Tooltip } from '@mui/material'; -import { Colors, EditableText, IconButton, NumberInput, Size, Slider, Type } from 'browndash-components'; +import { Colors, EditableText, IconButton, NumberInput, Size, Slider, Toggle, ToggleType, Type } from 'browndash-components'; import { concat } from 'lodash'; import { IReactionDisposer, action, computed, makeObservable, observable, reaction } from 'mobx'; import { observer } from 'mobx-react'; @@ -12,13 +12,13 @@ import * as Icons from 'react-icons/bs'; // {BsCollectionFill, BsFillFileEarmark import ResizeObserver from 'resize-observer-polyfill'; import { ClientUtils, returnEmptyFilter, returnEmptyString, returnFalse, returnTrue, setupMoveUpEvents } from '../../ClientUtils'; import { emptyFunction } from '../../Utils'; -import { Doc, Field, FieldResult, FieldType, HierarchyMapping, NumListCast, Opt, ReverseHierarchyMap, StrListCast, returnEmptyDoclist } from '../../fields/Doc'; +import { Doc, DocListCast, Field, FieldResult, FieldType, HierarchyMapping, NumListCast, Opt, ReverseHierarchyMap, StrListCast, returnEmptyDoclist } from '../../fields/Doc'; import { AclAdmin, DocAcl, DocData } from '../../fields/DocSymbols'; import { Id } from '../../fields/FieldSymbols'; import { InkField } from '../../fields/InkField'; import { List } from '../../fields/List'; import { ComputedField } from '../../fields/ScriptField'; -import { Cast, DocCast, NumCast, StrCast } from '../../fields/Types'; +import { BoolCast, Cast, DocCast, NumCast, StrCast } from '../../fields/Types'; import { GetEffectiveAcl, SharingPermissions, normalizeEmail } from '../../fields/util'; import { CollectionViewType, DocumentType } from '../documents/DocumentTypes'; import { GroupManager } from '../util/GroupManager'; @@ -41,6 +41,10 @@ import { DocumentView } from './nodes/DocumentView'; import { StyleProviderFuncType } from './nodes/FieldView'; import { OpenWhere } from './nodes/OpenWhere'; import { PresBox, PresEffect, PresEffectDirection } from './nodes/trails'; +import { InkingStroke } from './InkingStroke'; +import { SettingsManager } from '../util/SettingsManager'; +import { MarqueeOptionsMenu } from './collections/collectionFreeForm'; +import { SmartDrawHandler } from './smartdraw/SmartDrawHandler'; interface PropertiesViewProps { width: number; @@ -113,6 +117,10 @@ export class PropertiesView extends ObservableReactComponent<PropertiesViewProps @observable openAddSlide: boolean = false; @observable openSlideOptions: boolean = false; + // For ink groups + @observable containsInkDoc: boolean = false; + @observable inkDoc: Doc | undefined = undefined; + @observable _controlButton: boolean = false; private _disposers: { [name: string]: IReactionDisposer } = {}; @@ -805,6 +813,9 @@ export class PropertiesView extends ObservableReactComponent<PropertiesViewProps return Field.toString(this.selectedDoc?.[DocData][key] as FieldType); } + @computed get selectedStrokes() { + return this.containsInkDoc ? DocListCast(this.selectedDoc[DocData].data) : this.selectedDoc ? [this.selectedDoc] : []; + } @computed get shapeXps() { return NumCast(this.selectedDoc?.x); } // prettier-ignore set shapeXps(value) { this.selectedDoc && (this.selectedDoc.x = Math.round(value * 100) / 100); } // prettier-ignore @computed get shapeYps() { return NumCast(this.selectedDoc?.y); } // prettier-ignore @@ -813,8 +824,12 @@ export class PropertiesView extends ObservableReactComponent<PropertiesViewProps set shapeWid(value) { this.selectedDoc && (this.selectedDoc._width = Math.round(value * 100) / 100); } // prettier-ignore @computed get shapeHgt() { return NumCast(this.selectedDoc?._height); } // prettier-ignore set shapeHgt(value) { this.selectedDoc && (this.selectedDoc._height = Math.round(value * 100) / 100); } // prettier-ignore - @computed get strokeThk(){ return NumCast(this.selectedDoc?.[DocData].stroke_width); } // prettier-ignore - set strokeThk(value) { this.selectedDoc && (this.selectedDoc[DocData].stroke_width = Math.round(value * 100) / 100); } // prettier-ignore + @computed get strokeThk(){ return this.containsInkDoc ? NumCast(this.inkDoc?.[DocData].stroke_width) : NumCast(this.selectedDoc?.[DocData].stroke_width); } // prettier-ignore + set strokeThk(value) { + this.selectedStrokes.forEach(doc => { + doc[DocData].stroke_width = Math.round(value * 100) / 100; + }); + } @computed get hgtInput() { return this.inputBoxDuo( @@ -851,10 +866,27 @@ export class PropertiesView extends ObservableReactComponent<PropertiesViewProps private _lastDash: string = '2'; - @computed get colorFil() { return StrCast(this.selectedDoc?.[DocData].fillColor); } // prettier-ignore - set colorFil(value) { this.selectedDoc && (this.selectedDoc[DocData].fillColor = value || undefined); } // prettier-ignore - @computed get colorStk() { return StrCast(this.selectedDoc?.[DocData].color); } // prettier-ignore - set colorStk(value) { this.selectedDoc && (this.selectedDoc[DocData].color = value || undefined); } // prettier-ignore + @computed get colorFil() { return this.containsInkDoc ? StrCast(this.inkDoc?.[DocData].fillColor) : StrCast(this.selectedDoc?.[DocData].fillColor); } // prettier-ignore + set colorFil(value) { + if (this.containsInkDoc) { + const childDocs = DocListCast(this.selectedDoc[DocData].data); + childDocs.forEach(doc => { + const inkStroke = DocumentView.getDocumentView(doc)?.ComponentView as InkingStroke; + const { inkData } = inkStroke.inkScaledData(); + if (InkingStroke.IsClosed(inkData)) { + doc[DocData].fillColor = value || undefined; + } + }); + } else { + this.selectedDoc && (this.selectedDoc[DocData].fillColor = value || undefined); + } + } + @computed get colorStk() { return this.containsInkDoc ? StrCast(this.inkDoc?.[DocData].color) : StrCast(this.selectedDoc?.[DocData].color); } // prettier-ignore + set colorStk(value) { + this.selectedStrokes.forEach(doc => { + doc[DocData].color = value || undefined; + }); + } colorButton(value: string, type: string, setter: () => void) { return ( @@ -925,10 +957,64 @@ export class PropertiesView extends ObservableReactComponent<PropertiesViewProps ); } - @computed get dashdStk() { return StrCast(this.selectedDoc?.stroke_dash); } // prettier-ignore + @computed get smoothAndColor() { + const targetDoc = this.selectedLayoutDoc; + return ( + <div> + {!targetDoc.layout_isSvg && ( + <div className="color"> + <Toggle + text={'Color with GPT'} + color={SettingsManager.userColor} + icon={<FontAwesomeIcon icon="fill-drip" />} + iconPlacement="left" + align="flex-start" + fillWidth + toggleType={ToggleType.BUTTON} + onClick={undoable(() => { + SmartDrawHandler.Instance.colorWithGPT(targetDoc); + }, 'smoothStrokes')} + /> + </div> + )} + <div className="smooth"> + <Toggle + text={'Smooth Ink Strokes'} + color={SettingsManager.userColor} + icon={<FontAwesomeIcon icon="bezier-curve" />} + iconPlacement="left" + align="flex-start" + fillWidth + toggleType={ToggleType.BUTTON} + onClick={undoable(() => { + InkStrokeProperties.Instance.smoothInkStrokes(this.containsInkDoc ? DocListCast(targetDoc.data) : [targetDoc], this.smoothAmt); + }, 'smoothStrokes')} + /> + </div> + <div className="smooth-slider"> + {this.getNumber( + 'Smooth Amount', + '', + 1, + Math.max(20, this.smoothAmt), + this.smoothAmt, + (val: number) => { + !isNaN(val) && (this.smoothAmt = val); + }, + 20, + 1 + )} + </div> + </div> + ); + } + + @computed get dashdStk() { return this.containsInkDoc? this.inkDoc?.stroke_dash || '' : this.selectedDoc?.stroke_dash || ''; } // prettier-ignore set dashdStk(value) { - value && (this._lastDash = value); - this.selectedDoc && (this.selectedDoc[DocData].stroke_dash = value ? this._lastDash : undefined); + value && (this._lastDash = value as string); + this.selectedStrokes.forEach(doc => { + doc[DocData].stroke_dash = value ? this._lastDash : undefined; + }); } @computed get widthStk() { return this.getField('stroke_width') || '1'; } // prettier-ignore set widthStk(value) { @@ -936,15 +1022,25 @@ export class PropertiesView extends ObservableReactComponent<PropertiesViewProps } @computed get markScal() { return Number(this.getField('stroke_markerScale') || '1'); } // prettier-ignore set markScal(value) { - this.selectedDoc && (this.selectedDoc[DocData].stroke_markerScale = Number(value)); + this.selectedStrokes.forEach(doc => { + doc[DocData].stroke_markerScale = Number(value); + }); + } + @computed get smoothAmt() { return Number(this.getField('stroke_smoothAmount') || '10'); } // prettier-ignore + set smoothAmt(value) { + this.selectedDoc && (this.selectedDoc[DocData].stroke_smoothAmount = Number(value)); } @computed get markHead() { return this.getField('stroke_startMarker') || ''; } // prettier-ignore set markHead(value) { - this.selectedDoc && (this.selectedDoc[DocData].stroke_startMarker = value); + this.selectedStrokes.forEach(doc => { + doc[DocData].stroke_startMarker = value; + }); } @computed get markTail() { return this.getField('stroke_endMarker') || ''; } // prettier-ignore set markTail(value) { - this.selectedDoc && (this.selectedDoc[DocData].stroke_endMarker = value); + this.selectedStrokes.forEach(doc => { + doc[DocData].stroke_endMarker = value; + }); } regInput = (key: string, value: string | number | undefined, setter: (val: string) => void) => ( @@ -1056,6 +1152,16 @@ export class PropertiesView extends ObservableReactComponent<PropertiesViewProps <div className="appearance-editor"> {this.widthAndDash} {this.strokeAndFill} + {this.smoothAndColor} + </div> + ); + } + + @computed get inkEditor() { + return ( + <div className="ink-editor"> + {this.widthAndDash} + {this.strokeAndFill} </div> ); } @@ -1185,6 +1291,7 @@ export class PropertiesView extends ObservableReactComponent<PropertiesViewProps } @computed get inkSubMenu() { + this.containsInkDoc = false; return ( // prettier-ignore <> @@ -1198,6 +1305,34 @@ export class PropertiesView extends ObservableReactComponent<PropertiesViewProps ); } + /** + * Determines if a selected collection/group document contains any ink strokes to allow users to edit groups + * of ink strokes in the properties menu. + */ + containsInk = (selectedDoc: Doc) => { + const childDocs: Doc[] = DocListCast(selectedDoc[DocData].data); + for (var i = 0; i < childDocs.length; i++) { + if (DocumentView.getDocumentView(childDocs[i])?.layoutDoc?.layout_isSvg) { + this.inkDoc = childDocs[i]; + this.containsInkDoc = true; + return true; + } + } + this.containsInkDoc = false; + return false; + }; + + @computed get inkCollectionSubMenu() { + return ( + // prettier-ignore + <> + <PropertiesSection title="Ink Appearance" isOpen={this.openAppearance} setIsOpen={bool => { this.openAppearance = bool; }} onDoubleClick={this.CloseAll}> + {this.isGroup && this.containsInk(this.selectedDoc) ? this.appearanceEditor : null} + </PropertiesSection> + </> + ); + } + @computed get fieldsSubMenu() { return ( <PropertiesSection @@ -1749,6 +1884,7 @@ export class PropertiesView extends ObservableReactComponent<PropertiesViewProps {this.linksSubMenu} {!this.selectedLink || !this.openLinks ? null : this.linkProperties} {this.inkSubMenu} + {this.inkCollectionSubMenu} {this.contextsSubMenu} {isNovice ? null : this.sharingSubMenu} {this.filtersSubMenu} diff --git a/src/client/views/ViewBoxInterface.ts b/src/client/views/ViewBoxInterface.ts index dce64ab92..111771ba0 100644 --- a/src/client/views/ViewBoxInterface.ts +++ b/src/client/views/ViewBoxInterface.ts @@ -22,7 +22,7 @@ export abstract class ViewBoxInterface<P> extends ObservableReactComponent<React return ''; // } promoteCollection?: () => void; // moves contents of collection to parent - updateIcon?: () => void; // updates the icon representation of the document + updateIcon?: (usePanelDimensions?: boolean) => void; // updates the icon representation of the document getAnchor?: (addAsAnnotation: boolean, pinData?: PinProps) => Doc; // returns an Anchor Doc that represents the current state of the doc's componentview (e.g., the current playhead location of a an audio/video box) restoreView?: (viewSpec: Doc) => boolean; scrollPreview?: (docView: DocumentView, doc: Doc, focusSpeed: number, options: FocusViewOptions) => Opt<number>; // returns the duration of the focus diff --git a/src/client/views/collections/CollectionCarousel3DView.tsx b/src/client/views/collections/CollectionCarousel3DView.tsx index c799eb3c8..0473bb163 100644 --- a/src/client/views/collections/CollectionCarousel3DView.tsx +++ b/src/client/views/collections/CollectionCarousel3DView.tsx @@ -15,7 +15,6 @@ import { FocusViewOptions } from '../nodes/FocusViewOptions'; import './CollectionCarousel3DView.scss'; import { CollectionSubView, SubCollectionViewProps } from './CollectionSubView'; -// eslint-disable-next-line @typescript-eslint/no-var-requires const { CAROUSEL3D_CENTER_SCALE, CAROUSEL3D_SIDE_SCALE, CAROUSEL3D_TOP } = require('../global/globalCssVariables.module.scss'); @observer @@ -77,7 +76,7 @@ export class CollectionCarousel3DView extends CollectionSubView() { // suppressSetHeight={true} NativeWidth={returnZero} NativeHeight={returnZero} - fitWidth={undefined} + fitWidth={this._props.childLayoutFitWidth} onDoubleClickScript={this.onChildDoubleClick} renderDepth={this._props.renderDepth + 1} LayoutTemplate={this._props.childLayoutTemplate} diff --git a/src/client/views/collections/CollectionCarouselView.tsx b/src/client/views/collections/CollectionCarouselView.tsx index 4bec2d963..7ce4ecc2f 100644 --- a/src/client/views/collections/CollectionCarouselView.tsx +++ b/src/client/views/collections/CollectionCarouselView.tsx @@ -6,7 +6,7 @@ import * as React from 'react'; import { StopEvent, returnFalse, returnOne, returnTrue, returnZero } from '../../../ClientUtils'; import { emptyFunction } from '../../../Utils'; import { Doc, Opt } from '../../../fields/Doc'; -import { DocCast, NumCast, ScriptCast, StrCast } from '../../../fields/Types'; +import { Cast, DocCast, NumCast, ScriptCast, StrCast } from '../../../fields/Types'; import { DocumentType } from '../../documents/DocumentTypes'; import { DragManager } from '../../util/DragManager'; import { ContextMenu } from '../ContextMenu'; @@ -141,6 +141,7 @@ export class CollectionCarouselView extends CollectionSubView() { revealItems.push({description: 'Quiz Cards', event: () => {this.layoutDoc.filterOp = cardMode.QUIZ;}, icon: 'pencil',}); // prettier-ignore !revealOptions && cm.addItem({ description: 'Filter Flashcards', addDivider: false, noexpand: true, subitems: revealItems, icon: 'layer-group' }); }; + childFitWidth = (doc: Doc) => Cast(this.Document.childLayoutFitWidth, 'boolean', this._props.childLayoutFitWidth?.(doc) ?? Cast(doc.layout_fitWidth, 'boolean', null)); @computed get content() { const index = NumCast(this.layoutDoc._carousel_index); const curDoc = this.carouselItems?.[index]; @@ -151,10 +152,11 @@ export class CollectionCarouselView extends CollectionSubView() { <div className="collectionCarouselView-image" key="image"> <DocumentView {...this._props} - NativeWidth={returnZero} - NativeHeight={returnZero} - fitWidth={undefined} + // NativeWidth={returnZero} + // NativeHeight={returnZero} + fitWidth={returnTrue} setContentViewBox={undefined} + containerViewPath={this.DocumentView?.().docViewPath} onDoubleClickScript={this.onContentDoubleClick} onClickScript={this.onContentClick} isDocumentActive={this._props.childDocumentsActive?.() ? this._props.isDocumentActive : this._props.isContentActive} @@ -166,6 +168,7 @@ export class CollectionCarouselView extends CollectionSubView() { Document={curDoc.layout} TemplateDataDocument={DocCast(curDoc.layout.resolvedDataDoc)} PanelHeight={this.panelHeight} + PanelWidth={this._props.PanelWidth} /> </div> {!carouselShowsCaptions ? null : ( diff --git a/src/client/views/collections/CollectionMenu.scss b/src/client/views/collections/CollectionMenu.scss index 3ec875df4..45d9394ed 100644 --- a/src/client/views/collections/CollectionMenu.scss +++ b/src/client/views/collections/CollectionMenu.scss @@ -6,7 +6,7 @@ align-content: center; justify-content: space-between; background-color: $dark-gray; - height: 35px; + height: 40px; border-bottom: $standard-border; padding: 0 10px; align-items: center; diff --git a/src/client/views/collections/TreeView.tsx b/src/client/views/collections/TreeView.tsx index b10a521ca..015f77ffd 100644 --- a/src/client/views/collections/TreeView.tsx +++ b/src/client/views/collections/TreeView.tsx @@ -467,7 +467,7 @@ export class TreeView extends ObservableReactComponent<TreeViewProps> { return false; } - refTransform = (ref: HTMLElement | undefined | null) => { + refTransform = (ref: HTMLDivElement | undefined | null) => { if (!ref) return this.ScreenToLocalTransform(); const { translateX, translateY, scale } = ClientUtils.GetScreenTransform(ref); return new Transform(-translateX, -translateY, 1).scale(1 / scale); diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx index c4cf8dee7..c0f01383d 100644 --- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx +++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx @@ -31,7 +31,7 @@ import { CompileScript } from '../../../util/Scripting'; import { ScriptingGlobals } from '../../../util/ScriptingGlobals'; import { freeformScrollMode, SnappingManager } from '../../../util/SnappingManager'; import { Transform } from '../../../util/Transform'; -import { undoable, UndoManager } from '../../../util/UndoManager'; +import { undoable, undoBatch, UndoManager } from '../../../util/UndoManager'; import { Timeline } from '../../animationtimeline/Timeline'; import { ContextMenu } from '../../ContextMenu'; import { InkingStroke } from '../../InkingStroke'; @@ -43,6 +43,7 @@ import { FocusViewOptions } from '../../nodes/FocusViewOptions'; import { FormattedTextBox } from '../../nodes/formattedText/FormattedTextBox'; import { OpenWhere, OpenWhereMod } from '../../nodes/OpenWhere'; import { PinDocView, PinProps } from '../../PinFuncs'; +import { DrawingOptions, SmartDrawHandler } from '../../smartdraw/SmartDrawHandler'; import { StyleProp } from '../../StyleProp'; import { CollectionSubView, SubCollectionViewProps } from '../CollectionSubView'; import { TreeViewType } from '../CollectionTreeViewType'; @@ -116,6 +117,7 @@ export class CollectionFreeFormView extends CollectionSubView<Partial<collection @observable _panZoomTransition: number = 0; // sets the pan/zoom transform ease time- used by nudge(), focus() etc to smoothly zoom/pan. set to 0 to use document's transition time or default of 0 @observable _firstRender = false; // this turns off rendering of the collection's content so that there's instant feedback when a tab is switched of what content will be shown. could be used for performance improvement @observable _showAnimTimeline = false; + @observable _showDrawingEditor = false; @observable _deleteList: DocumentView[] = []; @observable _timelineRef = React.createRef<Timeline>(); @observable _marqueeViewRef = React.createRef<MarqueeView>(); @@ -493,28 +495,30 @@ export class CollectionFreeFormView extends CollectionSubView<Partial<collection if (!this.Document.isGroup) { // group freeforms don't pan when dragged -- instead let the event go through to allow the group itself to drag // prettier-ignore + const hit = this._clusters.handlePointerDown(this.screenToFreeformContentsXf.transformPoint(e.clientX, e.clientY)); switch (Doc.ActiveTool) { - case InkTool.Highlighter: break; - case InkTool.Write: break; - case InkTool.Pen: break; // the GestureOverlay handles ink stroke input -- either as gestures, or drying as ink strokes that are added to document views + case InkTool.Highlighter: + case InkTool.Write: + case InkTool.Pen: + break; // the GestureOverlay handles ink stroke input -- either as gestures, or drying as ink strokes that are added to document views case InkTool.StrokeEraser: case InkTool.SegmentEraser: - this._batch = UndoManager.StartBatch('collectionErase'); - this._eraserPts.length = 0; - setupMoveUpEvents(this, e, this.onEraserMove, this.onEraserUp, emptyFunction); - break; case InkTool.RadiusEraser: this._batch = UndoManager.StartBatch('collectionErase'); this._eraserPts.length = 0; - setupMoveUpEvents(this, e, this.onRadiusEraserMove, this.onEraserUp, emptyFunction); + setupMoveUpEvents(this, e, this.onEraserMove, this.onEraserUp, this.onEraserClick, hit !== -1); + e.stopPropagation(); break; + case InkTool.SmartDraw: + setupMoveUpEvents(this, e, this.onPointerMove, emptyFunction, this.showSmartDraw, hit !== -1); + e.stopPropagation(); case InkTool.None: if (!(this._props.layoutEngine?.() || StrCast(this.layoutDoc._layoutEngine))) { const hit = this._clusters.handlePointerDown(this.screenToFreeformContentsXf.transformPoint(e.clientX, e.clientY)); setupMoveUpEvents(this, e, this.onPointerMove, emptyFunction, emptyFunction, hit !== -1, false); } break; - default: + default: } } } @@ -537,24 +541,7 @@ export class CollectionFreeFormView extends CollectionSubView<Partial<collection default: { const { points } = ge; const B = this.screenToFreeformContentsXf.transformBounds(ge.bounds.left, ge.bounds.top, ge.bounds.width, ge.bounds.height); - const inkWidth = ActiveInkWidth() * this.ScreenToLocalBoxXf().Scale; - const inkDoc = Docs.Create.InkDocument( - points, - { title: ge.gesture.toString(), - x: B.x - inkWidth / 2, - y: B.y - inkWidth / 2, - _width: B.width + inkWidth, - _height: B.height + inkWidth, - stroke_showLabel: BoolCast(Doc.UserDoc().activeInkHideTextLabels)}, // prettier-ignore - inkWidth, - ActiveInkColor(), - ActiveInkBezierApprox(), - ActiveFillColor(), - ActiveArrowStart(), - ActiveArrowEnd(), - ActiveDash(), - ActiveIsInkMask() - ); + const inkDoc = this.createInkDoc(points, B); if (Doc.ActiveTool === InkTool.Write) { this.unprocessedDocs.push(inkDoc); CollectionFreeFormView.collectionsWithUnprocessedInk.add(this); @@ -606,96 +593,76 @@ export class CollectionFreeFormView extends CollectionSubView<Partial<collection /** * Erases strokes by intersecting them with an invisible "eraser stroke". - * By default this iterates through all intersected ink strokes, determines their segmentation, draws back the non-intersected segments, - * and deletes the original stroke. + * By default this iterates through all intersected ink strokes, determines which parts of a stroke need to be erased based on the type + * of eraser, draws back the ink segments to keep, and deletes the original stroke. + * + * Radius eraser: erases strokes by intersecting them with a circle of variable radius. Essentially creates an InkField for the + * eraser circle, then determines its intersections with other ink strokes. Each stroke's DocumentView and its + * intersection t-values are put into a map, which gets looped through to take out the erased parts. */ - @action - onEraserMove = (e: PointerEvent, down: number[], delta: number[]) => { + erase = (e: PointerEvent, delta: number[]) => { + e.stopImmediatePropagation(); const currPoint = { X: e.clientX, Y: e.clientY }; this._eraserPts.push([currPoint.X, currPoint.Y]); this._eraserPts = this._eraserPts.slice(Math.max(0, this._eraserPts.length - 5)); - // if (this._eraserLock) return false; // leaving this commented out in case the idea is revisited in the future - this.getEraserIntersections({ X: currPoint.X - delta[0], Y: currPoint.Y - delta[1] }, currPoint).forEach(intersect => { - if (!this._deleteList.includes(intersect.inkView)) { - this._deleteList.push(intersect.inkView); - SetActiveInkWidth(StrCast(intersect.inkView.Document.stroke_width?.toString()) || '1'); - SetActiveInkColor(StrCast(intersect.inkView.Document.color?.toString()) || 'black'); - // create a new curve by appending all curves of the current segment together in order to render a single new stroke. - if (Doc.ActiveTool !== InkTool.StrokeEraser) { - // this._eraserLock++; - const segments = this.segmentErase(intersect.inkView, intersect.t); // intersect.t is where the eraser intersected the ink stroke - want to remove the segment that starts at the intersection just before this t value and goes to the one just after it - const newStrokes = segments?.map(segment => { - const points = segment.reduce((data, curve) => [...data, ...curve.points.map(p => intersect.inkView.ComponentView?.ptToScreen?.({ X: p.x, Y: p.y }) ?? { X: 0, Y: 0 })], [] as PointData[]); - const bounds = InkField.getBounds(points); - const B = this.screenToFreeformContentsXf.transformBounds(bounds.left, bounds.top, bounds.width, bounds.height); - const inkWidth = ActiveInkWidth() * this.ScreenToLocalBoxXf().Scale; - return Docs.Create.InkDocument( - points, - { title: 'stroke', - x: B.x - inkWidth / 2, - y: B.y - inkWidth / 2, - _width: B.width + inkWidth, - _height: B.height + inkWidth, - stroke_showLabel: BoolCast(Doc.UserDoc().activeInkHideTextLabels)}, // prettier-ignore - inkWidth, - ActiveInkColor(), - ActiveInkBezierApprox(), - ActiveFillColor(), - ActiveArrowStart(), - ActiveArrowEnd(), - ActiveDash(), - ActiveIsInkMask() - ); - }); - newStrokes && this.addDocument?.(newStrokes); - // setTimeout(() => this._eraserLock--); + if (Doc.ActiveTool === InkTool.RadiusEraser) { + const strokeMap: Map<DocumentView, number[]> = this.getRadiusEraserIntersections({ X: currPoint.X - delta[0], Y: currPoint.Y - delta[1] }, currPoint); + strokeMap.forEach((intersects, stroke) => { + if (!this._deleteList.includes(stroke)) { + this._deleteList.push(stroke); + SetActiveInkWidth(StrCast(stroke.Document.stroke_width?.toString()) || '1'); + SetActiveInkColor(StrCast(stroke.Document.color?.toString()) || 'black'); + const segments = this.radiusErase(stroke, intersects.sort()); + segments?.forEach(segment => + this.forceStrokeGesture( + e, + Gestures.Stroke, + segment.reduce((data, curve) => [...data, ...curve.points.map(p => stroke.ComponentView?.ptToScreen?.({ X: p.x, Y: p.y }) ?? { X: 0, Y: 0 })], [] as PointData[]) + ) + ); } - // Lower ink opacity to give the user a visual indicator of deletion. - intersect.inkView.layoutDoc.opacity = 0; - intersect.inkView.layoutDoc.dontIntersect = true; - } - }); + stroke.layoutDoc.opacity = 0; + stroke.layoutDoc.dontIntersect = true; + }); + } else { + this.getEraserIntersections({ X: currPoint.X - delta[0], Y: currPoint.Y - delta[1] }, currPoint).forEach(intersect => { + if (!this._deleteList.includes(intersect.inkView)) { + this._deleteList.push(intersect.inkView); + SetActiveInkWidth(StrCast(intersect.inkView.Document.stroke_width?.toString()) || '1'); + SetActiveInkColor(StrCast(intersect.inkView.Document.color?.toString()) || 'black'); + // create a new curve by appending all curves of the current segment together in order to render a single new stroke. + if (Doc.ActiveTool !== InkTool.StrokeEraser) { + // this._eraserLock++; + const segments = this.segmentErase(intersect.inkView, intersect.t); // intersect.t is where the eraser intersected the ink stroke - want to remove the segment that starts at the intersection just before this t value and goes to the one just after it + const newStrokes = segments?.map(segment => { + const points = segment.reduce((data, curve) => [...data, ...curve.points.map(p => intersect.inkView.ComponentView?.ptToScreen?.({ X: p.x, Y: p.y }) ?? { X: 0, Y: 0 })], [] as PointData[]); + return this.createInkDoc(points); + }); + newStrokes && this.addDocument?.(newStrokes); + // setTimeout(() => this._eraserLock--); + } + } + }); + } return false; }; - /** - * Erases strokes by intersecting them with a circle of variable radius. Essentially creates an InkField for the - * eraser circle, then determines its intersections with other ink strokes. Each stroke's DocumentView and its - * intersection t-values are put into a map, which gets looped through to take out the erased parts. - * @param e - * @param down - * @param delta - * @returns - */ @action - onRadiusEraserMove = (e: PointerEvent, down: number[], delta: number[]) => { - const currPoint = { X: e.clientX, Y: e.clientY }; - this._eraserPts.push([currPoint.X, currPoint.Y]); - this._eraserPts = this._eraserPts.slice(Math.max(0, this._eraserPts.length - 5)); - const strokeMap: Map<DocumentView, number[]> = this.getRadiusEraserIntersections({ X: currPoint.X - delta[0], Y: currPoint.Y - delta[1] }, currPoint); - - strokeMap.forEach((intersects, stroke) => { - if (!this._deleteList.includes(stroke)) { - this._deleteList.push(stroke); - SetActiveInkWidth(StrCast(stroke.Document.stroke_width?.toString()) || '1'); - SetActiveInkColor(StrCast(stroke.Document.color?.toString()) || 'black'); - const segments = this.radiusErase(stroke, intersects.sort()); - segments?.forEach(segment => - this.forceStrokeGesture( - e, - Gestures.Stroke, - segment.reduce((data, curve) => [...data, ...curve.points.map(p => stroke.ComponentView?.ptToScreen?.({ X: p.x, Y: p.y }) ?? { X: 0, Y: 0 })], [] as PointData[]) - ) - ); - } - stroke.layoutDoc.opacity = 0; - stroke.layoutDoc.dontIntersect = true; - }); + onEraserMove = (e: PointerEvent, down: number[], delta: number[]) => { + this.erase(e, delta); + // if (this._eraserLock) return false; // leaving this commented out in case the idea is revisited in the future return false; }; - forceStrokeGesture = (e: PointerEvent, gesture: Gestures, points: InkData) => { - this.onGesture(e, new GestureUtils.GestureEvent(gesture, points, InkField.getBounds(points))); + @action + onEraserClick = (e: PointerEvent, doubleTap?: boolean) => { + e.preventDefault(); + e.stopImmediatePropagation(); + this.erase(e, [0, 0]); + }; + + forceStrokeGesture = (e: PointerEvent, gesture: Gestures, points: InkData, text?: any) => { + this.onGesture(e, new GestureUtils.GestureEvent(gesture, points, InkField.getBounds(points), text)); }; onPointerMove = (e: PointerEvent) => { @@ -723,7 +690,7 @@ export class CollectionFreeFormView extends CollectionSubView<Partial<collection // increase radius slightly based on the erased stroke's width, added to make eraser look more realistic const radius = ActiveEraserWidth() + 5 + inkStrokeWidth * 0.1; // add 5 to prevent eraser from being too small const c = 0.551915024494; // circle tangent length to side ratio - const movement = { x: endInkCoordsIn.X - startInkCoordsIn.X, y: endInkCoordsIn.Y - startInkCoordsIn.Y }; + const movement = { x: Math.max(endInkCoordsIn.X - startInkCoordsIn.X, 1), y: Math.max(endInkCoordsIn.Y - startInkCoordsIn.Y, 1) }; const moveLen = Math.sqrt(movement.x ** 2 + movement.y ** 2); const direction = { x: (movement.x / moveLen) * radius, y: (movement.y / moveLen) * radius }; const normal = { x: -direction.y, y: direction.x }; // prettier-ignore @@ -1235,6 +1202,108 @@ export class CollectionFreeFormView extends CollectionSubView<Partial<collection return tVals; }; + /** + * Creates an ink document to add to the freeform canvas. + */ + createInkDoc = (points: InkData, transformedBounds?: { x: number; y: number; width: number; height: number }) => { + const bounds = InkField.getBounds(points); + const B = transformedBounds || this.screenToFreeformContentsXf.transformBounds(bounds.left, bounds.top, bounds.width, bounds.height); + const inkWidth = ActiveInkWidth() * this.ScreenToLocalBoxXf().Scale; + return Docs.Create.InkDocument( + points, + { title: 'stroke', + x: B.x - inkWidth / 2, + y: B.y - inkWidth / 2, + _width: B.width + inkWidth, + _height: B.height + inkWidth, + stroke_showLabel: BoolCast(Doc.UserDoc().activeInkHideTextLabels)}, // prettier-ignore + inkWidth, + ActiveInkColor(), + ActiveInkBezierApprox(), + ActiveFillColor(), + ActiveArrowStart(), + ActiveArrowEnd(), + ActiveDash(), + ActiveIsInkMask() + ); + }; + + @action + showSmartDraw = (e: PointerEvent) => { + SmartDrawHandler.Instance.CreateDrawingDoc = this.createDrawingDoc; + SmartDrawHandler.Instance.RemoveDrawing = this.removeDrawing; + SmartDrawHandler.Instance.AddDrawing = this.addDrawing; + SmartDrawHandler.Instance.displaySmartDrawHandler(e.pageX, e.pageY); + }; + + _drawing: Doc[] = []; + _drawingContainer: Doc | undefined = undefined; + /** + * Function that creates a drawing--a group of ink strokes--to go with the smart draw function. + */ + @undoBatch + createDrawingDoc = (strokeData: [InkData, string, string][], opts: DrawingOptions, gptRes: string) => { + this._drawing = []; + const xf = this.screenToFreeformContentsXf; + strokeData.forEach((stroke: [InkData, string, string]) => { + const bounds = InkField.getBounds(stroke[0]); + const B = xf.transformBounds(bounds.left, bounds.top, bounds.width, bounds.height); + const inkWidth = ActiveInkWidth() * this.ScreenToLocalBoxXf().Scale; + const inkDoc = Docs.Create.InkDocument( + stroke[0], + { title: 'stroke', + x: B.x - inkWidth / 2, + y: B.y - inkWidth / 2, + _width: B.width + inkWidth, + _height: B.height + inkWidth, + stroke_showLabel: BoolCast(Doc.UserDoc().activeInkHideTextLabels)}, // prettier-ignore + inkWidth, + opts.autoColor ? stroke[1] : ActiveInkColor(), + ActiveInkBezierApprox(), + stroke[2] === 'none' ? ActiveFillColor() : stroke[2], + ActiveArrowStart(), + ActiveArrowEnd(), + ActiveDash(), + ActiveIsInkMask() + ); + this._drawing.push(inkDoc); + }); + const collection = MarqueeView.getCollection(this._drawing, undefined, true, { left: 1, top: 1, width: 1, height: 1 }); + return collection; + }; + + /** + * Part of regenerating a drawing--deletes the old drawing. + */ + removeDrawing = (doc?: Doc) => { + this._batch = UndoManager.StartBatch('regenerateDrawing'); + if (doc) { + const docData = doc[DocData]; + const children = DocListCast(docData.data); + this._props.removeDocument?.(doc); + this._props.removeDocument?.(children); + } else { + if (this._drawingContainer) this._props.removeDocument?.(this._drawingContainer); + } + this._drawing = []; + }; + + /** + * Adds the created drawing to the freeform canvas and sets the metadata. + */ + addDrawing = (doc: Doc, opts: DrawingOptions, gptRes: string) => { + const docData = doc[DocData]; + docData.title = opts.text.match(/^(.*?)~~~.*$/)?.[1] || opts.text; + docData.drawingInput = opts.text; + docData.drawingComplexity = opts.complexity; + docData.drawingColored = opts.autoColor; + docData.drawingSize = opts.size; + docData.drawingData = gptRes; + this._drawingContainer = doc; + this.addDocument(doc); + this._batch?.end(); + }; + @action zoom = (pointX: number, pointY: number, deltaY: number): void => { if (this.Document.isGroup || this.Document[(this._props.viewField ?? '_') + 'freeform_noZoom']) return; @@ -1807,33 +1876,30 @@ export class CollectionFreeFormView extends CollectionSubView<Partial<collection Object.values(this._disposers).forEach(disposer => disposer?.()); } - updateIcon = () => { - const contentDiv = this.DocumentView?.().ContentDiv; - contentDiv && - UpdateIcon( - this.layoutDoc[Id] + '-icon' + new Date().getTime(), - contentDiv, - NumCast(this.layoutDoc._width), - NumCast(this.layoutDoc._height), - this._props.PanelWidth(), - this._props.PanelHeight(), - 0, - 1, - false, - '', - (iconFile, nativeWidth, nativeHeight) => { - this.dataDoc.icon = new ImageField(iconFile); - this.dataDoc.icon_nativeWidth = nativeWidth; - this.dataDoc.icon_nativeHeight = nativeHeight; - } - ); - }; + updateIcon = (usePanelDimensions?: boolean) => + UpdateIcon( + this.layoutDoc[Id] + '-icon' + new Date().getTime(), + this.DocumentView?.().ContentDiv!, + usePanelDimensions ? this._props.PanelWidth() : NumCast(this.layoutDoc._width), + usePanelDimensions ? this._props.PanelHeight() : NumCast(this.layoutDoc._height), + this._props.PanelWidth(), + this._props.PanelHeight(), + 0, + 1, + false, + '', + (iconFile, nativeWidth, nativeHeight) => { + this.dataDoc.icon = new ImageField(iconFile); + this.dataDoc.icon_nativeWidth = nativeWidth; + this.dataDoc.icon_nativeHeight = nativeHeight; + } + ); @action onCursorMove = (e: React.PointerEvent) => { - this._eraserX = e.clientX; - this._eraserY = e.clientY; - // super.setCursorPosition(this.getTransform().transformPoint(e.clientX, e.clientY)); + const locPt = this.ScreenToLocalBoxXf().transformPoint(e.clientX, e.clientY); + this._eraserX = locPt[0]; + this._eraserY = locPt[1]; }; @action @@ -1914,7 +1980,7 @@ export class CollectionFreeFormView extends CollectionSubView<Partial<collection icon: 'eye', }); appearanceItems.push({ description: `Pin View`, event: () => this._props.pinToPres(this.Document, { pinViewport: MarqueeView.CurViewBounds(this.dataDoc, this._props.PanelWidth(), this._props.PanelHeight()) }), icon: 'map-pin' }); - !Doc.noviceMode && appearanceItems.push({ description: `update icon`, event: this.updateIcon, icon: 'compress-arrows-alt' }); + !Doc.noviceMode && appearanceItems.push({ description: `update icon`, event: () => this.updateIcon(), icon: 'compress-arrows-alt' }); this._props.renderDepth && appearanceItems.push({ description: 'Ungroup collection', event: this.promoteCollection, icon: 'table' }); this.Document.isGroup && this.Document.transcription && appearanceItems.push({ description: 'Ink to text', event: this.transcribeStrokes, icon: 'font' }); @@ -1934,6 +2000,16 @@ export class CollectionFreeFormView extends CollectionSubView<Partial<collection }), icon: 'eye', }); + optionItems.push({ + description: 'Show Drawing Editor', + event: action(() => { + SmartDrawHandler.Instance.CreateDrawingDoc = this.createDrawingDoc; + SmartDrawHandler.Instance.AddDrawing = this.addDrawing; + SmartDrawHandler.Instance.RemoveDrawing = this.removeDrawing; + !SmartDrawHandler.Instance._showRegenerate ? SmartDrawHandler.Instance.displayRegenerate(this._downX, this._downY - 10) : SmartDrawHandler.Instance.hideRegenerate(); + }), + icon: 'pen-to-square', + }); this._props.renderDepth && optionItems.push({ description: 'Use Background Color as Default', @@ -2137,8 +2213,8 @@ export class CollectionFreeFormView extends CollectionSubView<Partial<collection onPointerMove={this.onCursorMove} style={{ position: 'fixed', - left: this._eraserX - 60, - top: this._eraserY - 100, + left: this._eraserX, + top: this._eraserY, width: (ActiveEraserWidth() + 5) * 2, height: (ActiveEraserWidth() + 5) * 2, borderRadius: '50%', diff --git a/src/client/views/collections/collectionFreeForm/ImageLabelHandler.scss b/src/client/views/collections/collectionFreeForm/ImageLabelHandler.scss index e7413bf8e..9b8727e1a 100644 --- a/src/client/views/collections/collectionFreeForm/ImageLabelHandler.scss +++ b/src/client/views/collections/collectionFreeForm/ImageLabelHandler.scss @@ -42,3 +42,17 @@ } } } + +.complexity-slider { + width: 50%; /* Full-width */ + height: 25px; /* Specified height */ + background: #d3d3d3; /* Grey background */ + outline: none; /* Remove outline */ + opacity: 0.7; /* Set transparency (for mouse-over effects on hover) */ + -webkit-transition: 0.2s; /* 0.2 seconds transition on hover */ + transition: opacity 0.2s; + + :hover { + opacity: 1; /* Fully shown on mouse-over */ + } +} diff --git a/src/client/views/collections/collectionFreeForm/MarqueeOptionsMenu.tsx b/src/client/views/collections/collectionFreeForm/MarqueeOptionsMenu.tsx index 44c916ab9..de65b240f 100644 --- a/src/client/views/collections/collectionFreeForm/MarqueeOptionsMenu.tsx +++ b/src/client/views/collections/collectionFreeForm/MarqueeOptionsMenu.tsx @@ -3,6 +3,7 @@ import { IconButton } from 'browndash-components'; import { computed, makeObservable } from 'mobx'; import { observer } from 'mobx-react'; import * as React from 'react'; +import { Doc } from '../../../../fields/Doc'; import { unimplementedFunction } from '../../../../Utils'; import { SettingsManager } from '../../../util/SettingsManager'; import { AntimodeMenu, AntimodeMenuProps } from '../../AntimodeMenu'; @@ -12,7 +13,7 @@ export class MarqueeOptionsMenu extends AntimodeMenu<AntimodeMenuProps> { // eslint-disable-next-line no-use-before-define static Instance: MarqueeOptionsMenu; - public createCollection: (e: KeyboardEvent | React.PointerEvent | undefined, group?: boolean) => void = unimplementedFunction; + public createCollection: (e: KeyboardEvent | React.PointerEvent | undefined, group?: boolean, selection?: Doc[]) => Doc | void = unimplementedFunction; public delete: (e: KeyboardEvent | React.PointerEvent | undefined) => void = unimplementedFunction; public summarize: (e: KeyboardEvent | React.PointerEvent | undefined) => void = unimplementedFunction; public showMarquee: () => void = unimplementedFunction; diff --git a/src/client/views/collections/collectionFreeForm/MarqueeView.tsx b/src/client/views/collections/collectionFreeForm/MarqueeView.tsx index 6cc75aa4b..bfb1d12cb 100644 --- a/src/client/views/collections/collectionFreeForm/MarqueeView.tsx +++ b/src/client/views/collections/collectionFreeForm/MarqueeView.tsx @@ -2,11 +2,11 @@ import { action, computed, makeObservable, observable } from 'mobx'; import { observer } from 'mobx-react'; import * as React from 'react'; import { ClientUtils, lightOrDark, returnFalse } from '../../../../ClientUtils'; -import { intersectRect } from '../../../../Utils'; +import { intersectRect, unimplementedFunction } from '../../../../Utils'; import { Doc, DocListCast, Opt } from '../../../../fields/Doc'; import { AclAdmin, AclAugment, AclEdit, DocData } from '../../../../fields/DocSymbols'; import { Id } from '../../../../fields/FieldSymbols'; -import { InkTool } from '../../../../fields/InkField'; +import { InkData, InkTool } from '../../../../fields/InkField'; import { List } from '../../../../fields/List'; import { Cast, NumCast, StrCast } from '../../../../fields/Types'; import { ImageField } from '../../../../fields/URLField'; @@ -88,6 +88,8 @@ export class MarqueeView extends ObservableReactComponent<SubCollectionViewProps return bounds; } + public AddInkDoc: (points: InkData) => Doc | void = unimplementedFunction; + componentDidMount() { this._props.setPreviewCursor?.(this.setPreviewCursor); } @@ -363,7 +365,7 @@ export class MarqueeView extends ObservableReactComponent<SubCollectionViewProps this.hideMarquee(); }); - getCollection = action((selected: Doc[], creator: Opt<(documents: Array<Doc>, options: DocumentOptions, id?: string) => Doc>, makeGroup: Opt<boolean>) => { + public static getCollection = action((selected: Doc[], creator: Opt<(documents: Array<Doc>, options: DocumentOptions, id?: string) => Doc>, makeGroup: Opt<boolean>, bounds: MarqueeViewBounds) => { const newCollection = creator ? creator(selected, { title: 'nested stack' }) : ((doc: Doc) => { @@ -375,14 +377,13 @@ export class MarqueeView extends ObservableReactComponent<SubCollectionViewProps return doc; })(Doc.MakeCopy(Doc.UserDoc().emptyCollection as Doc, true)); newCollection.isSystem = undefined; - newCollection._width = this.Bounds.width; - newCollection._height = this.Bounds.height; + newCollection._width = bounds.width || 1; // if width/height are unset/0, then groups won't autoexpand to contain their children + newCollection._height = bounds.height || 1; newCollection._dragWhenActive = makeGroup; - newCollection.x = this.Bounds.left; - newCollection.y = this.Bounds.top; + newCollection.x = bounds.left; + newCollection.y = bounds.top; newCollection.layout_fitWidth = true; selected.forEach(d => Doc.SetContainer(d, newCollection)); - this.hideMarquee(); return newCollection; }); @@ -419,7 +420,7 @@ export class MarqueeView extends ObservableReactComponent<SubCollectionViewProps this._props.removeDocument?.(selected); } - const newCollection = this.getCollection(selected, (e as KeyboardEvent)?.key === 't' ? Docs.Create.StackingDocument : undefined, group); + const newCollection = MarqueeView.getCollection(selected, (e as KeyboardEvent)?.key === 't' ? Docs.Create.StackingDocument : undefined, group, this.Bounds); newCollection._freeform_panX = this.Bounds.left + this.Bounds.width / 2; newCollection._freeform_panY = this.Bounds.top + this.Bounds.height / 2; newCollection._currentFrame = activeFrame; @@ -427,6 +428,7 @@ export class MarqueeView extends ObservableReactComponent<SubCollectionViewProps this._props.selectDocuments([newCollection]); MarqueeOptionsMenu.Instance.fadeOut(true); this.hideMarquee(); + return newCollection; }); /** @@ -456,7 +458,7 @@ export class MarqueeView extends ObservableReactComponent<SubCollectionViewProps let y_offset = 0; let row_count = 0; for (const label of labelGroups) { - const newCollection = this.getCollection([], undefined, false); + const newCollection = MarqueeView.getCollection([], undefined, false, { top: 1, left: 1, width: 1, height: 1 }); newCollection._width = 900; newCollection._height = 900; newCollection._x = this.Bounds.left; diff --git a/src/client/views/collections/collectionSchema/CollectionSchemaView.tsx b/src/client/views/collections/collectionSchema/CollectionSchemaView.tsx index 8b0639b3b..325628d53 100644 --- a/src/client/views/collections/collectionSchema/CollectionSchemaView.tsx +++ b/src/client/views/collections/collectionSchema/CollectionSchemaView.tsx @@ -32,7 +32,6 @@ import './CollectionSchemaView.scss'; import { SchemaColumnHeader } from './SchemaColumnHeader'; import { SchemaRowBox } from './SchemaRowBox'; -// eslint-disable-next-line @typescript-eslint/no-var-requires const { SCHEMA_NEW_NODE_HEIGHT } = require('../../global/globalCssVariables.module.scss'); // prettier-ignore export const FInfotoColType: { [key: string]: ColumnType } = { diff --git a/src/client/views/global/globalScripts.ts b/src/client/views/global/globalScripts.ts index 2c7920bdd..68d287ac8 100644 --- a/src/client/views/global/globalScripts.ts +++ b/src/client/views/global/globalScripts.ts @@ -15,7 +15,7 @@ import { ScriptingGlobals } from '../../util/ScriptingGlobals'; import { UndoManager, undoable } from '../../util/UndoManager'; import { GestureOverlay } from '../GestureOverlay'; import { InkingStroke } from '../InkingStroke'; -import { CollectionFreeFormView } from '../collections/collectionFreeForm'; +import { CollectionFreeFormView, MarqueeView } from '../collections/collectionFreeForm'; import { CollectionFreeFormDocumentView } from '../nodes/CollectionFreeFormDocumentView'; import { ActiveEraserWidth, @@ -37,8 +37,6 @@ import { VideoBox } from '../nodes/VideoBox'; import { WebBox } from '../nodes/WebBox'; import { RichTextMenu } from '../nodes/formattedText/RichTextMenu'; -// import { InkTranscription } from '../InkTranscription'; - // eslint-disable-next-line prefer-arrow-callback ScriptingGlobals.add(function IsNoneSelected() { return DocumentView.Selected().length <= 0; @@ -412,7 +410,7 @@ export function createInkGroup(/* inksToGroup?: Doc[], isSubGroup?: boolean */) ); ffView._props.removeDocument?.(selected); // TODO: nda - this is the code to actually get a new grouped collection - const newCollection = marqViewRef?.getCollection(selected, undefined, true); + const newCollection = MarqueeView.getCollection(selected, undefined, true, { top: 1, left: 1, width: 1, height: 1 }); if (newCollection) { newCollection.height = NumCast(newCollection._height); newCollection.width = NumCast(newCollection._width); @@ -423,7 +421,7 @@ export function createInkGroup(/* inksToGroup?: Doc[], isSubGroup?: boolean */) // TODO: nda - will probably need to go through and only remove the unprocessed selected docs ffView.unprocessedDocs = []; - // InkTranscription.Instance.transcribeInk(newCollection, selected, false); + // InkTranscription.Instance.transcribeInk(newCollection, selected, false); }); } CollectionFreeFormView.collectionsWithUnprocessedInk.clear(); @@ -508,7 +506,7 @@ ScriptingGlobals.add(function setInkProperty(option: 'inkMask' | 'labels' | 'fil setMode: () => { SetActiveInkColor(StrCast(value)); selected?.type === DocumentType.INK && setActiveTool(GestureOverlay.Instance.InkShape ?? InkTool.Pen, true, false);}, }], [ 'eraserWidth', { - checkResult: () => ActiveEraserWidth(), + checkResult: () => ActiveEraserWidth() === 0 ? 1 : ActiveEraserWidth(), setInk: (doc: Doc) => { }, setMode: () => { SetEraserWidth(+value);}, }] diff --git a/src/client/views/nodes/DataVizBox/components/TableBox.tsx b/src/client/views/nodes/DataVizBox/components/TableBox.tsx index 7179356b2..e57c9e842 100644 --- a/src/client/views/nodes/DataVizBox/components/TableBox.tsx +++ b/src/client/views/nodes/DataVizBox/components/TableBox.tsx @@ -16,7 +16,6 @@ import { DocumentView } from '../../DocumentView'; import { DataVizView } from '../DataVizBox'; import './Chart.scss'; -// eslint-disable-next-line @typescript-eslint/no-var-requires const { DATA_VIZ_TABLE_ROW_HEIGHT } = require('../../../global/globalCssVariables.module.scss'); // prettier-ignore interface TableBoxProps { diff --git a/src/client/views/nodes/DocumentView.tsx b/src/client/views/nodes/DocumentView.tsx index 4c357cf45..4cc7eb0ff 100644 --- a/src/client/views/nodes/DocumentView.tsx +++ b/src/client/views/nodes/DocumentView.tsx @@ -1090,7 +1090,7 @@ export class DocumentView extends DocComponent<DocumentViewProps>() { /** * The DocumentView below the cursor at the start of a gesture (that receives the pointerDown event). Used by GestureOverlay to determine the doc a gesture should apply to. */ - public static DownDocView: DocumentView | undefined; + public static DownDocView: DocumentView | undefined; // the first DocView that receives a pointerdown event. used by GestureOverlay to determine the doc a gesture should apply to. public get displayName() { return 'DocumentView(' + (this.Document?.title??"") + ')'; } // prettier-ignore private _htmlOverlayEffect: Opt<Doc>; diff --git a/src/client/views/nodes/ScreenshotBox.tsx b/src/client/views/nodes/ScreenshotBox.tsx index 6289470b6..fd1c791d3 100644 --- a/src/client/views/nodes/ScreenshotBox.tsx +++ b/src/client/views/nodes/ScreenshotBox.tsx @@ -149,7 +149,7 @@ export class ScreenshotBox extends ViewBoxAnnotatableComponent<FieldViewProps>() componentDidMount() { this.dataDoc.nativeWidth = this.dataDoc.nativeHeight = 0; - this._props.setContentViewBox?.(this); // this tells the DocumentView that this ScreenshotBox is the "content" of the document. this allows the DocumentView to indirectly call getAnchor() on the AudioBox when making a link. + this._props.setContentViewBox?.(this); // this tells the DocumentView that this Box is the "content" of the document. this allows the DocumentView to indirectly call getAnchor() on the AudioBox when making a link. // this.layoutDoc.videoWall && reaction(() => ({ width: this._props.PanelWidth(), height: this._props.PanelHeight() }), // ({ width, height }) => { // if (this._camera) { diff --git a/src/client/views/nodes/WebBox.tsx b/src/client/views/nodes/WebBox.tsx index 1fd73c226..8088bddd4 100644 --- a/src/client/views/nodes/WebBox.tsx +++ b/src/client/views/nodes/WebBox.tsx @@ -44,7 +44,6 @@ import { LinkInfo } from './LinkDocPreview'; import { OpenWhere } from './OpenWhere'; import './WebBox.scss'; -// eslint-disable-next-line @typescript-eslint/no-var-requires const { CreateImage } = require('./WebBoxRenderer'); @observer diff --git a/src/client/views/nodes/formattedText/FormattedTextBox.tsx b/src/client/views/nodes/formattedText/FormattedTextBox.tsx index a88bd8920..996c6843e 100644 --- a/src/client/views/nodes/formattedText/FormattedTextBox.tsx +++ b/src/client/views/nodes/formattedText/FormattedTextBox.tsx @@ -264,6 +264,14 @@ export class FormattedTextBox extends ViewBoxAnnotatableComponent<FormattedTextB docView && DragManager.StartAnchorAnnoDrag([ele], new DragManager.AnchorAnnoDragData(docView, () => this.getAnchor(true), targetCreator), e.pageX, e.pageY); }); + AnchorMenu.Instance.AddDrawingAnnotation = (drawing: Doc) => { + const container = DocCast(this.Document.embedContainer); + const docView = DocumentView.getDocumentView?.(container); + docView?.ComponentView?._props.addDocument?.(drawing); + drawing.x = NumCast(this.Document.x) + (this.Document.width as number); + drawing.y = NumCast(this.Document.y); + }; + AnchorMenu.Instance.setSelectedText(window.getSelection()?.toString() ?? ''); const coordsB = this._editorView!.coordsAtPos(this._editorView!.state.selection.to); this._props.rootSelected?.() && AnchorMenu.Instance.jumpTo(coordsB.left, coordsB.bottom); diff --git a/src/client/views/pdf/AnchorMenu.tsx b/src/client/views/pdf/AnchorMenu.tsx index 03585a8b7..1a79bbbfe 100644 --- a/src/client/views/pdf/AnchorMenu.tsx +++ b/src/client/views/pdf/AnchorMenu.tsx @@ -1,18 +1,22 @@ import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; import { ColorPicker, Group, IconButton, Popup, Size, Toggle, ToggleType, Type } from 'browndash-components'; -import { IReactionDisposer, ObservableMap, action, computed, makeObservable, observable, reaction } from 'mobx'; +import { IReactionDisposer, ObservableMap, action, computed, makeObservable, observable, reaction, runInAction } from 'mobx'; import { observer } from 'mobx-react'; import * as React from 'react'; import { ColorResult } from 'react-color'; +import ReactLoading from 'react-loading'; import { ClientUtils, returnFalse, setupMoveUpEvents } from '../../../ClientUtils'; import { emptyFunction, unimplementedFunction } from '../../../Utils'; import { Doc, Opt } from '../../../fields/Doc'; +import { DocData } from '../../../fields/DocSymbols'; import { GPTCallType, gptAPICall } from '../../apis/gpt/GPT'; import { Docs } from '../../documents/Documents'; import { SettingsManager } from '../../util/SettingsManager'; +import { undoBatch } from '../../util/UndoManager'; import { AntimodeMenu, AntimodeMenuProps } from '../AntimodeMenu'; import { LinkPopup } from '../linking/LinkPopup'; import { DocumentView } from '../nodes/DocumentView'; +import { DrawingOptions, SmartDrawHandler } from '../smartdraw/SmartDrawHandler'; import './AnchorMenu.scss'; import { GPTPopup, GPTPopupMode } from './GPTPopup/GPTPopup'; @@ -38,6 +42,7 @@ export class AnchorMenu extends AntimodeMenu<AntimodeMenuProps> { // GPT additions @observable private _selectedText: string = ''; + @observable private _isLoading: boolean = false; @action public setSelectedText = (txt: string) => { this._selectedText = txt.trim(); @@ -60,6 +65,7 @@ export class AnchorMenu extends AntimodeMenu<AntimodeMenuProps> { public get Active() { return this._left > 0; } + public AddDrawingAnnotation: (doc: Doc) => void = unimplementedFunction; public addToCollection: ((doc: Doc | Doc[], annotationKey?: string | undefined) => boolean) | undefined; componentWillUnmount() { @@ -136,6 +142,35 @@ export class AnchorMenu extends AntimodeMenu<AntimodeMenuProps> { this.addToCollection?.(newCol); }; + /** + * Creates a GPT drawing based on selected text. + */ + gptDraw = async (e: React.PointerEvent) => { + try { + SmartDrawHandler.Instance.AddDrawing = this.createDrawingAnnotation; + runInAction(() => (this._isLoading = true)); + await SmartDrawHandler.Instance.drawWithGPT({ X: e.clientX, Y: e.clientY }, this._selectedText, 5, 100, true); + runInAction(() => (this._isLoading = false)); + } catch (err) { + console.error(err); + } + }; + + /** + * Defines how a GPT drawing should be added to the current document. + */ + @undoBatch + createDrawingAnnotation = action((drawing: Doc, opts: DrawingOptions, gptRes: string) => { + this.AddDrawingAnnotation(drawing); + const docData = drawing[DocData]; + docData.title = opts.text.match(/^(.*?)~~~.*$/)?.[1] || opts.text; + docData.drawingInput = opts.text; + docData.drawingComplexity = opts.complexity; + docData.drawingColored = opts.autoColor; + docData.drawingSize = opts.size; + docData.drawingData = gptRes; + }); + pointerDown = (e: React.PointerEvent) => { setupMoveUpEvents( this, @@ -225,6 +260,14 @@ export class AnchorMenu extends AntimodeMenu<AntimodeMenuProps> { icon={<FontAwesomeIcon icon="id-card" size="lg" />} color={SettingsManager.userColor} /> + {this._selectedText && ( + <IconButton + tooltip="Create drawing" + onPointerDown={e => this.gptDraw(e)} + icon={this._isLoading ? <ReactLoading type="spin" color={SettingsManager.userVariantColor} width={16} height={20} /> : <FontAwesomeIcon icon="paintbrush" size="lg" />} + color={SettingsManager.userColor} + /> + )} {AnchorMenu.Instance.OnAudio === unimplementedFunction ? null : ( <IconButton tooltip="Click to Record Annotation" // diff --git a/src/client/views/pdf/PDFViewer.tsx b/src/client/views/pdf/PDFViewer.tsx index dee0edfae..20719442a 100644 --- a/src/client/views/pdf/PDFViewer.tsx +++ b/src/client/views/pdf/PDFViewer.tsx @@ -424,6 +424,14 @@ export class PDFViewer extends ObservableReactComponent<IViewerProps> { GPTPopup.Instance.addDoc = this._props.sidebarAddDoc; // allows for creating collection AnchorMenu.Instance.addToCollection = this._props.DocumentView?.()._props.addDocument; + AnchorMenu.Instance.AddDrawingAnnotation = this.addDrawingAnnotation; + }; + + addDrawingAnnotation = (drawing: Doc) => { + // drawing[DocData].x = this._props.pdfBox.ScreenToLocalBoxXf().TranslateX + // const scaleX = this._mainCont.current.offsetWidth / boundingRect.width; + drawing.y = (drawing.y as number) + (this._props.Document.data_sidebar_panY as number); + this._props.addDocument?.(drawing); }; @action @@ -515,7 +523,6 @@ export class PDFViewer extends ObservableReactComponent<IViewerProps> { if (this.inlineTextAnnotations.includes(doc) || this._props.isContentActive() === false) return 'none'; const isInk = doc.layout_isSvg && !props?.LayoutTemplateString; if (isInk) return 'visiblePainted'; - //return isInk ? 'visiblePainted' : 'all'; } return this._props.styleProvider?.(doc, props, property); }; diff --git a/src/client/views/smartdraw/AnnotationPalette.scss b/src/client/views/smartdraw/AnnotationPalette.scss new file mode 100644 index 000000000..9f875f61a --- /dev/null +++ b/src/client/views/smartdraw/AnnotationPalette.scss @@ -0,0 +1,10 @@ +.annotation-palette { + display: flex; + flex-direction: column; + align-items: center; + position: absolute; + right: 14px; + top: 50px; + border-radius: 5px; + margin: auto; +} diff --git a/src/client/views/smartdraw/AnnotationPalette.tsx b/src/client/views/smartdraw/AnnotationPalette.tsx new file mode 100644 index 000000000..22b8b7b67 --- /dev/null +++ b/src/client/views/smartdraw/AnnotationPalette.tsx @@ -0,0 +1,362 @@ +import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; +import { Slider, Switch } from '@mui/material'; +import { Button } from 'browndash-components'; +import { action, makeObservable, observable } from 'mobx'; +import { observer } from 'mobx-react'; +import * as React from 'react'; +import { AiOutlineSend } from 'react-icons/ai'; +import ReactLoading from 'react-loading'; +import { returnEmptyFilter, returnFalse, returnTrue } from '../../../ClientUtils'; +import { emptyFunction } from '../../../Utils'; +import { Doc, DocListCast, returnEmptyDoclist } from '../../../fields/Doc'; +import { DocData } from '../../../fields/DocSymbols'; +import { Copy } from '../../../fields/FieldSymbols'; +import { ImageCast } from '../../../fields/Types'; +import { ImageField } from '../../../fields/URLField'; +import { DocumentType } from '../../documents/DocumentTypes'; +import { Docs } from '../../documents/Documents'; +import { makeUserTemplateImage } from '../../util/DropConverter'; +import { SettingsManager } from '../../util/SettingsManager'; +import { Transform } from '../../util/Transform'; +import { undoBatch } from '../../util/UndoManager'; +import { ObservableReactComponent } from '../ObservableReactComponent'; +import { DefaultStyleProvider, returnEmptyDocViewList } from '../StyleProvider'; +import { DocumentView, DocumentViewInternal } from '../nodes/DocumentView'; +import { FieldView } from '../nodes/FieldView'; +import './AnnotationPalette.scss'; +import { DrawingOptions, SmartDrawHandler } from './SmartDrawHandler'; + +interface AnnotationPaletteProps { + Document: Doc; +} + +@observer +export class AnnotationPalette extends ObservableReactComponent<AnnotationPaletteProps> { + @observable private _paletteMode: 'create' | 'view' = 'view'; + @observable private _userInput: string = ''; + @observable private _isLoading: boolean = false; + @observable private _canInteract: boolean = true; + @observable private _showRegenerate: boolean = false; + @observable private _docView: DocumentView | null = null; + @observable private _docCarouselView: DocumentView | null = null; + @observable private _opts: DrawingOptions = { text: '', complexity: 5, size: 200, autoColor: true, x: 0, y: 0 }; + private _gptRes: string[] = []; + + constructor(props: AnnotationPaletteProps) { + super(props); + makeObservable(this); + } + + public static LayoutString(fieldKey: string) { + return FieldView.LayoutString(AnnotationPalette, fieldKey); + } + + Contains = (view: DocumentView) => { + return (this._docView && (view.containerViewPath?.() ?? []).concat(view).includes(this._docView)) || (this._docCarouselView && (view.containerViewPath?.() ?? []).concat(view).includes(this._docCarouselView)); + }; + + return170 = () => 170; + + @action + handleKeyPress = async (event: React.KeyboardEvent) => { + if (event.key === 'Enter') { + await this.generateDrawings(); + } + }; + + @action + setPaletteMode = (mode: 'create' | 'view') => { + this._paletteMode = mode; + }; + + @action + setUserInput = (input: string) => { + if (!this._isLoading) this._userInput = input; + }; + + @action + setDetail = (detail: number) => { + if (this._canInteract) this._opts.complexity = detail; + }; + + @action + setColor = (autoColor: boolean) => { + if (this._canInteract) this._opts.autoColor = autoColor; + }; + + @action + setSize = (size: number) => { + if (this._canInteract) this._opts.size = size; + }; + + @action + resetPalette = (changePaletteMode: boolean) => { + if (changePaletteMode) this.setPaletteMode('view'); + this.setUserInput(''); + this.setDetail(5); + this.setColor(true); + this.setSize(200); + this._showRegenerate = false; + this._canInteract = true; + this._opts = { text: '', complexity: 5, size: 200, autoColor: true, x: 0, y: 0 }; + this._gptRes = []; + this._props.Document[DocData].data = undefined; + }; + + /** + * Adds a doc to the annotation palette. Gets a snapshot of the document to use as a preview in the palette. When this + * preview is dragged onto a parent document, a copy of that document is added as an annotation. + */ + public static addToPalette = async (doc: Doc) => { + if (!doc.savedAsAnno) { + const clone = await Doc.MakeClone(doc); + clone.clone.title = doc.title; + const image = (await AnnotationPalette.getIcon(doc))?.[Copy](); + if (image) { + const imageTemplate = makeUserTemplateImage(clone.clone, image); + Doc.AddDocToList(Doc.MyAnnos, 'data', imageTemplate); + doc.savedAsAnno = true; + } + } + }; + + public static getIcon(group: Doc) { + const docView = DocumentView.getDocumentView(group); + if (docView) { + docView.ComponentView?.updateIcon?.(true); + return new Promise<ImageField | undefined>(res => setTimeout(() => res(ImageCast(docView.Document.icon)), 1000)); + } + return undefined; + } + + /** + * Calls the draw with GPT functions in SmartDrawHandler to allow users to generate drawings straight from + * the annotation palette. + */ + @undoBatch + generateDrawings = action(async () => { + this._isLoading = true; + this._props.Document[DocData].data = undefined; + for (let i = 0; i < 3; i++) { + try { + SmartDrawHandler.Instance.AddDrawing = this.addDrawing; + this._canInteract = false; + if (this._showRegenerate) { + await SmartDrawHandler.Instance.regenerate(this._opts, this._gptRes[i], this._userInput); + } else { + await SmartDrawHandler.Instance.drawWithGPT({ X: 0, Y: 0 }, this._userInput, this._opts.complexity, this._opts.size, this._opts.autoColor); + } + } catch (e) { + console.log('Error generating drawing', e); + } + } + // eslint-disable-next-line @typescript-eslint/no-unused-expressions + this._opts.text !== '' ? (this._opts.text = `${this._opts.text} ~~~ ${this._userInput}`) : (this._opts.text = this._userInput); + this._userInput = ''; + this._isLoading = false; + this._showRegenerate = true; + }); + + @action + addDrawing = (drawing: Doc, opts: DrawingOptions, gptRes: string) => { + this._gptRes.push(gptRes); + drawing[DocData].freeform_fitContentsToBox = true; + Doc.AddDocToList(this._props.Document, 'data', drawing); + }; + + /** + * Saves the currently showing, newly generated drawing to the annotation palette and sets the metadata. + * AddToPalette() is generically used to add any document to the palette, while this defines the behavior for when a user + * presses the "save drawing" button. + */ + saveDrawing = async () => { + const cIndex: number = this._props.Document.carousel_index as number; + const focusedDrawing = DocListCast(this._props.Document.data)[cIndex]; + const docData = focusedDrawing[DocData]; + docData.title = this._opts.text.match(/^(.*?)~~~.*$/)?.[1] || this._opts.text; + docData.drawingInput = this._opts.text; + docData.drawingComplexity = this._opts.complexity; + docData.drawingColored = this._opts.autoColor; + docData.drawingSize = this._opts.size; + docData.drawingData = this._gptRes[cIndex]; + docData.width = this._opts.size; + await AnnotationPalette.addToPalette(focusedDrawing); + this.resetPalette(true); + }; + + async getIcon(group: Doc) { + const docView = DocumentView.getDocumentView(group); + if (docView) { + docView.ComponentView?.updateIcon?.(true); + return new Promise<ImageField | undefined>(res => setTimeout(() => res(ImageCast(docView.Document.icon)), 1000)); + } + return undefined; + } + + render() { + return ( + <div className="annotation-palette" style={{ zIndex: 1000 }} onClick={e => e.stopPropagation()}> + {this._paletteMode === 'view' && ( + <> + <DocumentView + ref={r => (this._docView = r)} + Document={Doc.MyAnnos} + addDocument={undefined} + addDocTab={DocumentViewInternal.addDocTabFunc} + pinToPres={DocumentView.PinDoc} + containerViewPath={returnEmptyDocViewList} + styleProvider={DefaultStyleProvider} + removeDocument={returnFalse} + ScreenToLocalTransform={Transform.Identity} + PanelWidth={this.return170} + PanelHeight={this.return170} + renderDepth={0} + isContentActive={returnTrue} + focus={emptyFunction} + whenChildContentsActiveChanged={emptyFunction} + childFilters={returnEmptyFilter} + childFiltersByRanges={returnEmptyFilter} + searchFilterDocs={returnEmptyDoclist} + /> + <Button text="Add" icon={<FontAwesomeIcon icon="square-plus" />} color={SettingsManager.userColor} onClick={() => this.setPaletteMode('create')} /> + </> + )} + {this._paletteMode === 'create' && ( + <> + <div style={{ display: 'flex', flexDirection: 'row', width: '170px' }}> + <input + aria-label="label-input" + id="new-label" + type="text" + style={{ color: 'black', width: '170px' }} + value={this._userInput} + onChange={e => { + this.setUserInput(e.target.value); + }} + placeholder={this._showRegenerate ? '(Optional) Enter edits' : 'Enter item to draw'} + onKeyDown={this.handleKeyPress} + /> + <Button + style={{ alignSelf: 'flex-end' }} + tooltip={this._showRegenerate ? 'Regenerate' : 'Send'} + icon={this._isLoading ? <ReactLoading type="spin" color={SettingsManager.userVariantColor} width={16} height={20} /> : this._showRegenerate ? <FontAwesomeIcon icon={'rotate'} /> : <AiOutlineSend />} + iconPlacement="right" + color={SettingsManager.userColor} + onClick={this.generateDrawings} + /> + </div> + <div style={{ display: 'flex', flexDirection: 'row', justifyContent: 'space-between', width: '170px', marginTop: '5px' }}> + <div style={{ display: 'flex', flexDirection: 'column', alignItems: 'center', width: '40px' }}> + Color + <Switch + sx={{ + '& .MuiSwitch-switchBase.Mui-checked': { + color: SettingsManager.userColor, + }, + '& .MuiSwitch-switchBase.Mui-checked + .MuiSwitch-track': { + backgroundColor: SettingsManager.userVariantColor, + }, + }} + defaultChecked={true} + value={this._opts.autoColor} + size="small" + onChange={() => this.setColor(!this._opts.autoColor)} + /> + </div> + <div style={{ display: 'flex', flexDirection: 'column', alignItems: 'center', width: '60px' }}> + Detail + <Slider + sx={{ + '& .MuiSlider-thumb': { + color: SettingsManager.userColor, + '&.Mui-focusVisible, &:hover, &.Mui-active': { + boxShadow: `0px 0px 0px 8px${SettingsManager.userColor.slice(0, 7)}10`, + }, + }, + '& .MuiSlider-track': { + color: SettingsManager.userVariantColor, + }, + '& .MuiSlider-rail': { + color: SettingsManager.userColor, + }, + }} + style={{ width: '80%' }} + min={1} + max={10} + step={1} + size="small" + value={this._opts.complexity} + onChange={(e, val) => { + this.setDetail(val as number); + }} + valueLabelDisplay="auto" + /> + </div> + <div style={{ display: 'flex', flexDirection: 'column', alignItems: 'center', width: '60px' }}> + Size + <Slider + sx={{ + '& .MuiSlider-thumb': { + color: SettingsManager.userColor, + '&.Mui-focusVisible, &:hover, &.Mui-active': { + boxShadow: `0px 0px 0px 8px${SettingsManager.userColor.slice(0, 7)}20`, + }, + }, + '& .MuiSlider-track': { + color: SettingsManager.userVariantColor, + }, + '& .MuiSlider-rail': { + color: SettingsManager.userColor, + }, + }} + style={{ width: '80%' }} + min={50} + max={500} + step={10} + size="small" + value={this._opts.size} + onChange={(e, val) => { + this.setSize(val as number); + }} + valueLabelDisplay="auto" + /> + </div> + </div> + <DocumentView + ref={r => (this._docCarouselView = r)} + Document={this._props.Document} + addDocument={undefined} + addDocTab={DocumentViewInternal.addDocTabFunc} + pinToPres={DocumentView.PinDoc} + containerViewPath={returnEmptyDocViewList} + styleProvider={DefaultStyleProvider} + removeDocument={returnFalse} + ScreenToLocalTransform={Transform.Identity} + PanelWidth={this.return170} + PanelHeight={this.return170} + renderDepth={1} + isContentActive={returnTrue} + focus={emptyFunction} + whenChildContentsActiveChanged={emptyFunction} + childFilters={returnEmptyFilter} + childFiltersByRanges={returnEmptyFilter} + searchFilterDocs={returnEmptyDoclist} + /> + <div style={{ width: '100%', display: 'flex', flexDirection: 'row', justifyContent: 'space-between' }}> + <Button text="Back" tooltip="Back to All Annotations" icon={<FontAwesomeIcon icon="reply" />} color={SettingsManager.userColor} onClick={() => this.resetPalette(true)} /> + <div style={{ display: 'flex', flexDirection: 'row' }}> + <Button tooltip="Save" icon={<FontAwesomeIcon icon="file-arrow-down" />} color={SettingsManager.userColor} onClick={this.saveDrawing} /> + <Button tooltip="Reset" icon={<FontAwesomeIcon icon="rotate-left" />} color={SettingsManager.userColor} onClick={() => this.resetPalette(false)} /> + </div> + </div> + </> + )} + </div> + ); + } +} + +Docs.Prototypes.TemplateMap.set(DocumentType.ANNOPALETTE, { + layout: { view: AnnotationPalette, dataField: 'data' }, + options: { acl: '' }, +}); diff --git a/src/client/views/smartdraw/SmartDrawHandler.scss b/src/client/views/smartdraw/SmartDrawHandler.scss new file mode 100644 index 000000000..6d402a80f --- /dev/null +++ b/src/client/views/smartdraw/SmartDrawHandler.scss @@ -0,0 +1,3 @@ +.smart-draw-handler { + position: absolute; +} diff --git a/src/client/views/smartdraw/SmartDrawHandler.tsx b/src/client/views/smartdraw/SmartDrawHandler.tsx new file mode 100644 index 000000000..aa10dcead --- /dev/null +++ b/src/client/views/smartdraw/SmartDrawHandler.tsx @@ -0,0 +1,553 @@ +import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; +import { action, makeObservable, observable } from 'mobx'; +import { observer } from 'mobx-react'; +import React from 'react'; +import { SettingsManager } from '../../util/SettingsManager'; +import { ObservableReactComponent } from '../ObservableReactComponent'; +import { Button, IconButton } from 'browndash-components'; +import ReactLoading from 'react-loading'; +import { AiOutlineSend } from 'react-icons/ai'; +import { gptAPICall, GPTCallType, gptDrawingColor } from '../../apis/gpt/GPT'; +import { InkData, InkField, InkTool } from '../../../fields/InkField'; +import { SVGToBezier, SVGType } from '../../util/bezierFit'; +import { INode, parse } from 'svgson'; +import { Slider, Switch } from '@mui/material'; +import { Doc, DocListCast } from '../../../fields/Doc'; +import { DocData } from '../../../fields/DocSymbols'; +import { ActiveArrowEnd, ActiveArrowStart, ActiveDash, ActiveFillColor, ActiveInkBezierApprox, ActiveInkColor, ActiveInkWidth, ActiveIsInkMask, DocumentView } from '../nodes/DocumentView'; +import { BoolCast, ImageCast, NumCast, StrCast } from '../../../fields/Types'; +import './SmartDrawHandler.scss'; +import { unimplementedFunction } from '../../../Utils'; +import { Docs } from '../../documents/Documents'; +import { MarqueeView } from '../collections/collectionFreeForm'; +import { ImageField, URLField } from '../../../fields/URLField'; +import { CollectionCardView } from '../collections/CollectionCardDeckView'; +import { InkingStroke } from '../InkingStroke'; +import { undoBatch } from '../../util/UndoManager'; + +export interface DrawingOptions { + text: string; + complexity: number; + size: number; + autoColor: boolean; + x: number; + y: number; +} + +@observer +// eslint-disable-next-line @typescript-eslint/no-empty-object-type +export class SmartDrawHandler extends ObservableReactComponent<{}> { + static Instance: SmartDrawHandler; + + @observable private _display: boolean = false; + @observable private _pageX: number = 0; + @observable private _pageY: number = 0; + @observable private _yRelativeToTop: boolean = true; + @observable private _isLoading: boolean = false; + @observable private _userInput: string = ''; + @observable private _showOptions: boolean = false; + @observable private _showEditBox: boolean = false; + @observable public _showRegenerate: boolean = false; + @observable private _complexity: number = 5; + @observable private _size: number = 200; + @observable private _autoColor: boolean = true; + @observable private _regenInput: string = ''; + @observable private _canInteract: boolean = true; + + private _lastInput: DrawingOptions = { text: '', complexity: 5, size: 350, autoColor: true, x: 0, y: 0 }; + private _lastResponse: string = ''; + private _selectedDoc: Doc | undefined = undefined; + private _errorOccurredOnce = false; + + public RemoveDrawing: (doc?: Doc) => void = unimplementedFunction; + public CreateDrawingDoc: (strokeList: [InkData, string, string][], opts: DrawingOptions, gptRes: string, containerDoc?: Doc) => Doc | undefined = (strokeList: [InkData, string, string][], opts: DrawingOptions) => { + const drawing: Doc[] = []; + strokeList.forEach((stroke: [InkData, string, string]) => { + const bounds = InkField.getBounds(stroke[0]); + const inkWidth = Math.min(5, ActiveInkWidth()); + const inkDoc = Docs.Create.InkDocument( + stroke[0], + { title: 'stroke', + x: bounds.left - inkWidth / 2, + y: bounds.top - inkWidth / 2, + _width: bounds.width + inkWidth, + _height: bounds.height + inkWidth, + stroke_showLabel: BoolCast(Doc.UserDoc().activeInkHideTextLabels)}, // prettier-ignore + inkWidth, + opts.autoColor ? stroke[1] : ActiveInkColor(), + ActiveInkBezierApprox(), + stroke[2] === 'none' ? ActiveFillColor() : stroke[2], + ActiveArrowStart(), + ActiveArrowEnd(), + ActiveDash(), + ActiveIsInkMask() + ); + drawing.push(inkDoc); + }); + + const collection = MarqueeView.getCollection(drawing, undefined, true, { left: 1, top: 1, width: 1, height: 1 }); + return collection; + }; + public AddDrawing: (doc: Doc, opts: DrawingOptions, gptRes: string) => void = unimplementedFunction; + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + constructor(props: any) { + super(props); + makeObservable(this); + SmartDrawHandler.Instance = this; + } + + @action + setUserInput = (input: string) => { + if (this._canInteract) this._userInput = input; + }; + + @action + setRegenInput = (input: string) => { + if (this._canInteract) this._regenInput = input; + }; + + @action + setShowOptions = () => { + this._showOptions = !this._showOptions; + }; + + @action + setComplexity = (val: number) => { + if (this._canInteract) this._complexity = val; + }; + + @action + setSize = (val: number) => { + if (this._canInteract) this._size = val; + }; + + @action + setAutoColor = () => { + if (this._canInteract) this._autoColor = !this._autoColor; + }; + + @action + setEdit = () => { + this._showEditBox = !this._showEditBox; + }; + + @action + displaySmartDrawHandler = (x: number, y: number) => { + this._pageX = x; + this._pageY = y; + this._display = true; + }; + + @action + displayRegenerate = (x: number, y: number) => { + this._selectedDoc = DocumentView.SelectedDocs()?.lastElement(); + this._pageX = x; + this._pageY = y; + this._display = false; + this._showRegenerate = true; + this._showEditBox = false; + const docData = this._selectedDoc[DocData]; + this._lastResponse = StrCast(docData.drawingData); + this._lastInput = { text: StrCast(docData.drawingInput), complexity: NumCast(docData.drawingComplexity), size: NumCast(docData.drawingSize), autoColor: BoolCast(docData.drawingColored), x: this._pageX, y: this._pageY }; + }; + + @action + hideSmartDrawHandler = () => { + this._showRegenerate = false; + this._display = false; + this._isLoading = false; + this._showOptions = false; + this._userInput = ''; + this._complexity = 5; + this._size = 350; + this._autoColor = true; + Doc.ActiveTool = InkTool.None; + }; + + @action + hideRegenerate = () => { + if (!this._isLoading) { + this._showRegenerate = false; + this._isLoading = false; + this._regenInput = ''; + this._lastInput = { text: '', complexity: 5, size: 350, autoColor: true, x: 0, y: 0 }; + } + }; + + @action + handleKeyPress = async (event: React.KeyboardEvent) => { + if (event.key === 'Enter') { + await this.handleSendClick(); + } + }; + + @action + handleSendClick = async () => { + this._isLoading = true; + this._canInteract = false; + if (this._showRegenerate) { + await this.regenerate(); + this._regenInput = ''; + this._showEditBox = false; + } else { + this._showOptions = false; + try { + await this.drawWithGPT({ X: this._pageX, Y: this._pageY }, this._userInput, this._complexity, this._size, this._autoColor); + this.hideSmartDrawHandler(); + this._showRegenerate = true; + } catch (err) { + if (this._errorOccurredOnce) { + console.error('GPT call failed', err); + this._errorOccurredOnce = false; + } else { + this._errorOccurredOnce = true; + await this.drawWithGPT({ X: this._pageX, Y: this._pageY }, this._userInput, this._complexity, this._size, this._autoColor); + } + } + } + this._isLoading = false; + this._canInteract = true; + }; + + /** + * Calls GPT API to create a drawing based on user input + */ + @action + drawWithGPT = async (startPt: { X: number; Y: number }, input: string, complexity: number, size: number, autoColor: boolean) => { + if (input === '') return; + this._lastInput = { text: input, complexity: complexity, size: size, autoColor: autoColor, x: startPt.X, y: startPt.Y }; + const res = await gptAPICall(`"${input}", "${complexity}", "${size}"`, GPTCallType.DRAW, undefined, true); + if (!res) { + console.error('GPT call failed'); + return; + } + console.log(res); + const strokeData = await this.parseSvg(res, startPt, false, autoColor); + const drawingDoc = strokeData && this.CreateDrawingDoc(strokeData?.data, strokeData?.lastInput, strokeData?.lastRes); + // eslint-disable-next-line @typescript-eslint/no-unused-expressions + drawingDoc && this.AddDrawing(drawingDoc, this._lastInput, res); + + this._errorOccurredOnce = false; + return strokeData; + }; + + /** + * Regenerates drawings with the option to add a specific regenerate prompt/request. + */ + @action + regenerate = async (lastInput?: DrawingOptions, lastResponse?: string, regenInput?: string) => { + if (lastInput) this._lastInput = lastInput; + if (lastResponse) this._lastResponse = lastResponse; + if (regenInput) this._regenInput = regenInput; + + try { + let res; + if (this._regenInput !== '') { + const prompt: string = `This is your previously generated svg code: ${this._lastResponse} for the user input "${this._lastInput.text}". Please regenerate it with the provided specifications.`; + res = await gptAPICall(`"${this._regenInput}"`, GPTCallType.DRAW, prompt, true); + this._lastInput.text = `${this._lastInput.text} ~~~ ${this._regenInput}`; + } else { + res = await gptAPICall(`"${this._lastInput.text}", "${this._lastInput.complexity}", "${this._lastInput.size}"`, GPTCallType.DRAW, undefined, true); + } + if (!res) { + console.error('GPT call failed'); + return; + } + console.log(res); + const strokeData = await this.parseSvg(res, { X: this._lastInput.x, Y: this._lastInput.y }, true, lastInput?.autoColor || this._autoColor); + // eslint-disable-next-line @typescript-eslint/no-unused-expressions + this.RemoveDrawing !== unimplementedFunction && this.RemoveDrawing(this._selectedDoc); + const drawingDoc = strokeData && this.CreateDrawingDoc(strokeData?.data, strokeData?.lastInput, strokeData?.lastRes); + // eslint-disable-next-line @typescript-eslint/no-unused-expressions + drawingDoc && this.AddDrawing(drawingDoc, this._lastInput, res); + return strokeData; + } catch (err) { + console.error('Error regenerating drawing', err); + } + }; + + /** + * Parses the svg code that GPT returns into Bezier curves. + */ + @action + parseSvg = async (res: string, startPoint: { X: number; Y: number }, regenerate: boolean, autoColor: boolean) => { + const svg = res.match(/<svg[^>]*>([\s\S]*?)<\/svg>/g); + if (svg) { + this._lastResponse = svg[0]; + const svgObject = await parse(svg[0]); + const svgStrokes: INode[] = svgObject.children; + const strokeData: [InkData, string, string][] = []; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + svgStrokes.forEach(child => { + const convertedBezier: InkData = SVGToBezier(child.name as SVGType, child.attributes); + strokeData.push([ + convertedBezier.map(point => { + return { X: point.X + startPoint.X - this._size / 1.5, Y: point.Y + startPoint.Y - this._size / 2 }; + }), + (regenerate ? this._lastInput.autoColor : autoColor) ? child.attributes.stroke : '', + (regenerate ? this._lastInput.autoColor : autoColor) ? child.attributes.fill : '', + ]); + }); + return { data: strokeData, lastInput: this._lastInput, lastRes: svg[0] }; + } + }; + + /** + * Sends request to GPT API to recolor a selected ink document or group of ink documents. + */ + colorWithGPT = async (drawing: Doc) => { + const img = await this.getIcon(drawing); + const { href } = (img as URLField).url; + const hrefParts = href.split('.'); + const hrefComplete = `${hrefParts[0]}_o.${hrefParts[1]}`; + try { + const hrefBase64 = await CollectionCardView.imageUrlToBase64(hrefComplete); + const strokes = DocListCast(drawing[DocData].data); + const coords: string[] = []; + strokes.forEach((stroke, i) => { + const inkingStroke = DocumentView.getDocumentView(stroke)?.ComponentView as InkingStroke; + const { inkData } = inkingStroke.inkScaledData(); + coords.push( + `${i + 1}. ${inkData + .filter((point, index) => { + return index % 4 === 0 || index == inkData.length - 1; + }) + .map(point => { + return `(${point.X.toString()}, ${point.Y.toString()})`; + })}` + ); + }); + const response = await gptDrawingColor(hrefBase64, coords); + console.log(response); + const colorResponse = await gptAPICall(response, GPTCallType.COLOR, undefined); + console.log(colorResponse); + this.colorStrokes(colorResponse, drawing); + } catch (error) { + console.log('GPT call failed', error); + } + }; + + /** + * Function that parses the GPT color response and sets the selected stroke(s) to the new color. + */ + @undoBatch + colorStrokes = (res: string, drawing: Doc) => { + const colorList = res.match(/\{.*?\}/g); + const strokes = DocListCast(drawing[DocData].data); + colorList?.forEach((colors, index) => { + const strokeAndFill = colors.match(/#[0-9A-Fa-f]{6}/g); + if (strokeAndFill && strokeAndFill.length == 2) { + strokes[index][DocData].color = strokeAndFill[0]; + const inkStroke = DocumentView.getDocumentView(strokes[index])?.ComponentView as InkingStroke; + const { inkData } = inkStroke.inkScaledData(); + // eslint-disable-next-line @typescript-eslint/no-unused-expressions + InkingStroke.IsClosed(inkData) ? (strokes[index][DocData].fillColor = strokeAndFill[1]) : (strokes[index][DocData].fillColor = undefined); + } + }); + }; + + /** + * Gets an image snapshot of a doc. In this class, it's used to snapshot a selected ink stroke/group to use for GPT color. + */ + async getIcon(doc: Doc) { + const docView = DocumentView.getDocumentView(doc); + console.log(doc); + if (docView) { + console.log(docView); + docView.ComponentView?.updateIcon?.(); + return new Promise<ImageField | undefined>(res => setTimeout(() => res(ImageCast(docView.Document.icon)), 1000)); + } + return undefined; + } + + render() { + if (this._display) { + return ( + <div + id="label-handler" + className="smart-draw-handler" + style={{ + display: this._display ? '' : 'none', + left: this._pageX, + ...(this._yRelativeToTop ? { top: Math.max(0, this._pageY) } : { bottom: this._pageY }), + background: SettingsManager.userBackgroundColor, + color: SettingsManager.userColor, + }}> + <div> + <IconButton + tooltip={'Cancel'} + onClick={() => { + this.hideSmartDrawHandler(); + this.hideRegenerate(); + }} + icon={<FontAwesomeIcon icon="xmark" />} + color={SettingsManager.userColor} + style={{ width: '19px' }} + /> + <input + aria-label="Smart Draw Input" + className="smartdraw-input" + id="smartdraw-input" + type="text" + style={{ color: 'black' }} + value={this._userInput} + onChange={e => { + this.setUserInput(e.target.value); + }} + placeholder="Enter item to draw" + onKeyDown={this.handleKeyPress} + /> + <IconButton tooltip="Advanced Options" icon={<FontAwesomeIcon icon={this._showOptions ? 'caret-down' : 'caret-right'} />} color={SettingsManager.userColor} style={{ width: '14px' }} onClick={this.setShowOptions} /> + <Button + style={{ alignSelf: 'flex-end' }} + text="Send" + icon={this._isLoading ? <ReactLoading type="spin" color={SettingsManager.userVariantColor} width={16} height={20} /> : <AiOutlineSend />} + iconPlacement="right" + color={SettingsManager.userColor} + onClick={this.handleSendClick} + /> + </div> + {this._showOptions && ( + <> + <div style={{ display: 'flex', flexDirection: 'row', justifyContent: 'space-around' }}> + <div style={{ display: 'flex', flexDirection: 'column', justifyContent: 'center', width: '30%' }}> + Auto color + <Switch + sx={{ + '& .MuiSwitch-switchBase.Mui-checked': { + color: SettingsManager.userColor, + }, + '& .MuiSwitch-switchBase.Mui-checked + .MuiSwitch-track': { + backgroundColor: SettingsManager.userVariantColor, + }, + }} + defaultChecked={true} + value={this._autoColor} + size="small" + onChange={this.setAutoColor} + /> + </div> + <div style={{ display: 'flex', flexDirection: 'column', justifyContent: 'center', width: '31%' }}> + Complexity + <Slider + sx={{ + '& .MuiSlider-thumb': { + color: SettingsManager.userColor, + '&.Mui-focusVisible, &:hover, &.Mui-active': { + boxShadow: `0px 0px 0px 8px${SettingsManager.userColor.slice(0, 7)}10`, + }, + }, + '& .MuiSlider-track': { + color: SettingsManager.userVariantColor, + }, + '& .MuiSlider-rail': { + color: SettingsManager.userColor, + }, + }} + style={{ width: '80%' }} + min={1} + max={10} + step={1} + size="small" + value={this._complexity} + onChange={(e, val) => { + this.setComplexity(val as number); + }} + valueLabelDisplay="auto" + /> + </div> + <div style={{ display: 'flex', flexDirection: 'column', justifyContent: 'center', width: '39%' }}> + Size (in pixels) + <Slider + sx={{ + '& .MuiSlider-thumb': { + color: SettingsManager.userColor, + '&.Mui-focusVisible, &:hover, &.Mui-active': { + boxShadow: `0px 0px 0px 8px${SettingsManager.userColor.slice(0, 7)}20`, + }, + }, + '& .MuiSlider-track': { + color: SettingsManager.userVariantColor, + }, + '& .MuiSlider-rail': { + color: SettingsManager.userColor, + }, + }} + style={{ width: '80%' }} + min={50} + max={700} + step={10} + size="small" + value={this._size} + onChange={(e, val) => { + this.setSize(val as number); + }} + valueLabelDisplay="auto" + /> + </div> + </div> + </> + )} + </div> + ); + } else if (this._showRegenerate) { + return ( + <div + id="smartdraw-options-menu" + className="smart-draw-handler" + style={{ + left: this._pageX, + ...(this._yRelativeToTop ? { top: Math.max(0, this._pageY) } : { bottom: this._pageY }), + background: SettingsManager.userBackgroundColor, + color: SettingsManager.userColor, + }}> + <div + style={{ + display: 'flex', + flexDirection: 'row', + }}> + <IconButton + tooltip="Regenerate" + icon={this._isLoading && this._regenInput === '' ? <ReactLoading type="spin" color={SettingsManager.userVariantColor} width={16} height={20} /> : <FontAwesomeIcon icon={'rotate'} />} + color={SettingsManager.userColor} + onClick={this.handleSendClick} + /> + <IconButton tooltip="Edit with GPT" icon={<FontAwesomeIcon icon="pen-to-square" />} color={SettingsManager.userColor} onClick={this.setEdit} /> + {this._showEditBox && ( + <div + style={{ + display: 'flex', + flexDirection: 'row', + }}> + <input + aria-label="Edit instructions input" + className="smartdraw-input" + id="regen-input" + type="text" + style={{ color: 'black' }} + value={this._regenInput} + onChange={e => { + this.setRegenInput(e.target.value); + }} + onKeyDown={this.handleKeyPress} + placeholder="Edit instructions" + /> + <Button + style={{ alignSelf: 'flex-end' }} + text="Send" + icon={this._isLoading && this._regenInput !== '' ? <ReactLoading type="spin" color={SettingsManager.userVariantColor} width={16} height={20} /> : <AiOutlineSend />} + iconPlacement="right" + color={SettingsManager.userColor} + onClick={this.handleSendClick} + /> + </div> + )} + </div> + </div> + ); + } else { + return <></>; + } + } +} diff --git a/src/fields/Doc.ts b/src/fields/Doc.ts index 7abba7679..a3e765659 100644 --- a/src/fields/Doc.ts +++ b/src/fields/Doc.ts @@ -181,6 +181,53 @@ export function updateCachedAcls(doc: Doc) { return undefined; } +export function ActiveInkPen(): Doc { return Doc.UserDoc(); } // prettier-ignore +export function ActiveInkColor(): string { return StrCast(ActiveInkPen()?.activeInkColor, 'black'); } // prettier-ignore +export function ActiveFillColor(): string { return StrCast(ActiveInkPen()?.activeFillColor, ''); } // prettier-ignore +export function ActiveIsInkMask(): boolean { return BoolCast(ActiveInkPen()?.activeIsInkMask, false); } // prettier-ignore +export function ActiveInkHideTextLabels(): boolean { return BoolCast(ActiveInkPen().activeInkHideTextLabels, false); } // prettier-ignore +export function ActiveArrowStart(): string { return StrCast(ActiveInkPen()?.activeArrowStart, ''); } // prettier-ignore +export function ActiveArrowEnd(): string { return StrCast(ActiveInkPen()?.activeArrowEnd, ''); } // prettier-ignore +export function ActiveArrowScale(): number { return NumCast(ActiveInkPen()?.activeArrowScale, 1); } // prettier-ignore +export function ActiveDash(): string { return StrCast(ActiveInkPen()?.activeDash, '0'); } // prettier-ignore +export function ActiveInkWidth(): number { return Number(ActiveInkPen()?.activeInkWidth); } // prettier-ignore +export function ActiveInkBezierApprox(): string { return StrCast(ActiveInkPen()?.activeInkBezier); } // prettier-ignore +export function ActiveEraserWidth(): number { return NumCast(ActiveInkPen()?.eraserWidth); } // prettier-ignore + +export function SetActiveInkWidth(width: string): void { + !isNaN(parseInt(width)) && ActiveInkPen() && (ActiveInkPen().activeInkWidth = width); +} +export function SetActiveBezierApprox(bezier: string): void { + ActiveInkPen() && (ActiveInkPen().activeInkBezier = isNaN(parseInt(bezier)) ? '' : bezier); +} +export function SetActiveInkColor(value: string) { + ActiveInkPen() && (ActiveInkPen().activeInkColor = value); +} +export function SetActiveIsInkMask(value: boolean) { + ActiveInkPen() && (ActiveInkPen().activeIsInkMask = value); +} +export function SetActiveInkHideTextLabels(value: boolean) { + ActiveInkPen() && (ActiveInkPen().activeInkHideTextLabels = value); +} +export function SetActiveFillColor(value: string) { + ActiveInkPen() && (ActiveInkPen().activeFillColor = value); +} +export function SetActiveArrowStart(value: string) { + ActiveInkPen() && (ActiveInkPen().activeArrowStart = value); +} +export function SetActiveArrowEnd(value: string) { + ActiveInkPen() && (ActiveInkPen().activeArrowEnd = value); +} +export function SetActiveArrowScale(value: number) { + ActiveInkPen() && (ActiveInkPen().activeArrowScale = value); +} +export function SetActiveDash(dash: string): void { + !isNaN(parseInt(dash)) && ActiveInkPen() && (ActiveInkPen().activeDash = dash); +} +export function SetEraserWidth(width: number): void { + !isNaN(width) && ActiveInkPen() && (ActiveInkPen().eraserWidth = width); +} + @scriptingGlobal @Deserializable('Doc', (obj: unknown) => updateCachedAcls(obj as Doc), ['id']) export class Doc extends RefField { @@ -237,6 +284,8 @@ export class Doc extends RefField { public static get MyPublishedDocs() { return DocListCast(Doc.ActiveDashboard?.myPublishedDocs).concat(DocListCast(DocCast(Doc.UserDoc().myPublishedDocs)?.data)); } // prettier-ignore public static get MyDashboards() { return DocCast(Doc.UserDoc().myDashboards); } // prettier-ignore public static get MyTemplates() { return DocCast(Doc.UserDoc().myTemplates); } // prettier-ignore + public static get MyAnnos() { return DocCast(Doc.UserDoc().myAnnos); } // prettier-ignore + public static get MyLightboxDrawings() { return DocCast(Doc.UserDoc().myLightboxDrawings); } // prettier-ignore public static get MyImports() { return DocCast(Doc.UserDoc().myImports); } // prettier-ignore public static get MyFilesystem() { return DocCast(Doc.UserDoc().myFilesystem); } // prettier-ignore public static get MyTools() { return DocCast(Doc.UserDoc().myTools); } // prettier-ignore diff --git a/src/fields/InkField.ts b/src/fields/InkField.ts index 32abf0076..123d32301 100644 --- a/src/fields/InkField.ts +++ b/src/fields/InkField.ts @@ -17,6 +17,7 @@ export enum InkTool { Stamp = 'stamp', Write = 'write', PresentationPin = 'presentationpin', + SmartDraw = 'smartdraw', } export type Segment = Array<Bezier>; diff --git a/src/pen-gestures/GestureTypes.ts b/src/pen-gestures/GestureTypes.ts index d86562580..2e1c9d16f 100644 --- a/src/pen-gestures/GestureTypes.ts +++ b/src/pen-gestures/GestureTypes.ts @@ -7,6 +7,7 @@ export enum Gestures { Circle = 'circle', Rectangle = 'rectangle', Arrow = 'arrow', + RightAngle = 'rightangle', } // Defines a point in an ink as a pair of x- and y-coordinates. diff --git a/src/pen-gestures/ndollar.ts b/src/pen-gestures/ndollar.ts index ff7f7310b..31a4eb0e8 100644 --- a/src/pen-gestures/ndollar.ts +++ b/src/pen-gestures/ndollar.ts @@ -209,6 +209,7 @@ export class NDollarRecognizer { ]) ) ); + this.Multistrokes.push(new Multistroke(Gestures.Rectangle, useBoundedRotationInvariance, new Array([new Point(30, 143), new Point(106, 146), new Point(106, 225), new Point(30, 222), new Point(30, 146)]))); this.Multistrokes.push(new Multistroke(Gestures.Line, useBoundedRotationInvariance, [[new Point(12, 347), new Point(119, 347)]])); this.Multistrokes.push( new Multistroke( @@ -219,6 +220,13 @@ export class NDollarRecognizer { ); this.Multistrokes.push( new Multistroke( + Gestures.Triangle, // equilateral + useBoundedRotationInvariance, + new Array([new Point(42, 100), new Point(140, 102), new Point(100, 200), new Point(40, 100)]) + ) + ); + this.Multistrokes.push( + new Multistroke( Gestures.Circle, useBoundedRotationInvariance, new Array([ @@ -236,6 +244,94 @@ export class NDollarRecognizer { ]) ) ); + this.Multistrokes.push( + new Multistroke( + Gestures.Circle, + useBoundedRotationInvariance, + new Array([ + new Point(201, 250), + new Point(160, 230), + new Point(151, 210), + new Point(151, 190), + new Point(160, 170), + new Point(200, 150), + new Point(240, 170), + new Point(248, 190), + new Point(248, 210), + new Point(240, 230), + new Point(200, 250), + ]) + ) + ); + this.Multistrokes.push( + new Multistroke( + Gestures.Scribble, + useBoundedRotationInvariance, + new Array([ + new Point(232.43359374999994, 428.3789062499997), + new Point(232.43359374999994, 382.0359155790783), + new Point(265.26320387389393, 340.59025405258205), + new Point(285.92968749999994, 300.6953124999997), + new Point(285.92968749999994, 300.6953124999997), + new Point(290.29904726504463, 292.260623967478), + new Point(294.85514947688137, 276.5586112000384), + new Point(301.80468749999994, 270.1054687499997), + new Point(301.80468749999994, 270.1054687499997), + new Point(303.30831790356257, 268.7092405181201), + new Point(302.3387333271824, 274.174445118092), + new Point(302.58984374999994, 276.2109374999997), + new Point(302.58984374999994, 276.2109374999997), + new Point(303.5898583815426, 284.3210038050238), + new Point(306.34100306922323, 291.42526150092345), + new Point(308.55078124999994, 299.2578124999997), + new Point(308.55078124999994, 299.2578124999997), + new Point(314.02357444430737, 318.65610875195364), + new Point(312.83820955386796, 338.9193216781303), + new Point(317.41015624999994, 358.4374999999997), + new Point(317.41015624999994, 358.4374999999997), + new Point(324.6448511447705, 389.323263646351), + new Point(341.2272901550544, 419.08257022687917), + new Point(351.55468749999994, 449.3085937499997), + new Point(351.55468749999994, 449.3085937499997), + new Point(354.4485190828321, 457.7782031368645), + new Point(359.85272292551673, 488.59621490807643), + new Point(368.55859374999994, 492.7578124999997), + new Point(368.55859374999994, 492.7578124999997), + new Point(368.9336613544369, 492.9371030581646), + new Point(375.30018285197116, 475.54269522741924), + new Point(385.01171874999994, 460.68749999999966), + new Point(385.01171874999994, 460.68749999999966), + new Point(409.01141338031505, 423.9765046563952), + new Point(465.73402430232653, 338.122252279478), + new Point(470.49609374999994, 336.8945312499997), + new Point(470.49609374999994, 336.8945312499997), + new Point(472.34967396580504, 336.41665510061245), + new Point(470.5375229924171, 340.7226912215484), + new Point(470.56249999999994, 342.6367187499997), + new Point(470.56249999999994, 342.6367187499997), + new Point(470.6277554579174, 347.63734752514455), + new Point(471.4666087447205, 352.597933700578), + new Point(472.79687499999994, 357.4218749999997), + new Point(472.79687499999994, 357.4218749999997), + new Point(478.7003095537336, 378.82948542322754), + new Point(492.1754046938961, 397.52358353298115), + new Point(497.80468749999994, 418.6796874999997), + new Point(497.80468749999994, 418.6796874999997), + new Point(499.2975220287686, 424.29009358262533), + new Point(498.6576561843205, 452.5736061983319), + new Point(502.21874999999994, 455.3749999999997), + new Point(502.21874999999994, 455.3749999999997), + new Point(502.3599404176782, 455.4860697952399), + new Point(526.9859878583989, 412.7902963819643), + new Point(528.28515625, 410.5507812499997), + new Point(528.28515625, 410.5507812499997), + new Point(534.3674131478522, 400.0661462732759), + new Point(586.26953125, 311.2520380124085), + new Point(586.26953125, 306.8515624999997), + ]) + ) + ); + this.Multistrokes.push(new Multistroke(Gestures.RightAngle, useBoundedRotationInvariance, new Array([new Point(200, 0), new Point(0, 0), new Point(0, 100)]))); NumMultistrokes = this.Multistrokes.length; // NumMultistrokes flags the end of the non user-defined gstures strokes // // PREDEFINED STROKES |