From ab0e1ac6f5b21f2e10bdce428c882482a0f159b4 Mon Sep 17 00:00:00 2001 From: mehekj Date: Tue, 12 Apr 2022 18:09:03 -0400 Subject: ink to text for groupings --- src/client/documents/Documents.ts | 5 +- src/client/views/GestureOverlay.tsx | 16 +- src/client/views/InkTranscription.scss | 5 + src/client/views/InkTranscription.tsx | 188 +++++++++++++++++++++ src/client/views/MainView.tsx | 4 +- .../collectionFreeForm/CollectionFreeFormView.tsx | 19 ++- .../collectionFreeForm/MarqueeOptionsMenu.tsx | 11 -- .../collections/collectionFreeForm/MarqueeView.tsx | 16 +- src/client/views/nodes/EquationBox.scss | 36 ++++ src/client/views/nodes/EquationBox.tsx | 91 +++++++++- src/fields/InkField.ts | 3 +- src/pen-gestures/GestureUtils.ts | 3 +- src/typings/index.d.ts | 1 + 13 files changed, 366 insertions(+), 32 deletions(-) create mode 100644 src/client/views/InkTranscription.scss create mode 100644 src/client/views/InkTranscription.tsx (limited to 'src') diff --git a/src/client/documents/Documents.ts b/src/client/documents/Documents.ts index 9bcd23aa0..884cc781d 100644 --- a/src/client/documents/Documents.ts +++ b/src/client/documents/Documents.ts @@ -4,7 +4,7 @@ import { DateField } from "../../fields/DateField"; import { Doc, DocListCast, DocListCastAsync, Field, HeightSym, Initializing, Opt, updateCachedAcls, WidthSym } from "../../fields/Doc"; import { Id } from "../../fields/FieldSymbols"; import { HtmlField } from "../../fields/HtmlField"; -import { InkField } from "../../fields/InkField"; +import { InkField, PointData } from "../../fields/InkField"; import { List } from "../../fields/List"; import { ProxyField } from "../../fields/Proxy"; import { RichTextField } from "../../fields/RichTextField"; @@ -738,7 +738,7 @@ export namespace Docs { return linkDoc; } - export function InkDocument(color: string, tool: string, strokeWidth: number, strokeBezier: string, fillColor: string, arrowStart: string, arrowEnd: string, dash: string, points: { X: number, Y: number }[], options: DocumentOptions = {}) { + export function InkDocument(color: string, tool: string, strokeWidth: number, strokeBezier: string, fillColor: string, arrowStart: string, arrowEnd: string, dash: string, points: PointData[], options: DocumentOptions = {}) { const I = new Doc(); I[Initializing] = true; I.type = DocumentType.INK; @@ -1161,6 +1161,7 @@ export namespace DocUtils { created = Docs.Create.AudioDocument((field).url.href, resolved); layout = AudioBox.LayoutString; } else if (field instanceof InkField) { + console.log("Documents " + field.inkData) created = Docs.Create.InkDocument(ActiveInkColor(), CurrentUserUtils.SelectedTool, ActiveInkWidth(), ActiveInkBezierApprox(), ActiveFillColor(), ActiveArrowStart(), ActiveArrowEnd(), ActiveDash(), (field).inkData, resolved); layout = InkingStroke.LayoutString; } else if (field instanceof List && field[0] instanceof Doc) { diff --git a/src/client/views/GestureOverlay.tsx b/src/client/views/GestureOverlay.tsx index 50dca0a99..16a9cfaeb 100644 --- a/src/client/views/GestureOverlay.tsx +++ b/src/client/views/GestureOverlay.tsx @@ -41,6 +41,7 @@ export class GestureOverlay extends Touchable { @observable private _menuY: number = -300; @observable private _pointerY?: number; @observable private _points: { X: number, Y: number }[] = []; + @observable private _timeStamps: number[] = []; @observable private _strokes: InkData[] = []; @observable private _palette?: JSX.Element; @observable private _clipboardDoc?: JSX.Element; @@ -59,6 +60,7 @@ export class GestureOverlay extends Touchable { private pointerIdentifier?: number; private _hands: Map = new Map(); private _holdTimer: NodeJS.Timeout | undefined; + private _strokeStartTime: number = 0; protected _multiTouchDisposer?: InteractionUtils.MultiTouchEventDisposer; @@ -418,6 +420,7 @@ export class GestureOverlay extends Touchable { this._strokes = []; this._points = []; + this._timeStamps = []; this._possibilities = []; document.removeEventListener("touchend", this.handleHandUp); } @@ -493,6 +496,8 @@ export class GestureOverlay extends Touchable { onPointerDown = (e: React.PointerEvent) => { if (InteractionUtils.IsType(e, InteractionUtils.PENTYPE) || [InkTool.Highlighter, InkTool.Pen].includes(CurrentUserUtils.SelectedTool)) { this._points.push({ X: e.clientX, Y: e.clientY }); + this._timeStamps.push(0); + this._strokeStartTime = new Date().getTime(); setupMoveUpEvents(this, e, this.onPointerMove, this.onPointerUp, emptyFunction); // if (CurrentUserUtils.SelectedTool === InkTool.Highlighter) SetActiveInkColor("rgba(245, 230, 95, 0.75)"); } @@ -501,6 +506,7 @@ export class GestureOverlay extends Touchable { @action onPointerMove = (e: PointerEvent) => { this._points.push({ X: e.clientX, Y: e.clientY }); + this._timeStamps.push(new Date().getTime() - this._strokeStartTime); if (this._points.length > 1) { const B = this.svgBounds; @@ -547,6 +553,7 @@ export class GestureOverlay extends Touchable { const points = this._points.map(p => ({ X: p.X - B.left, Y: p.Y - B.top })); //push first points to so interactionUtil knows pointer is up this._points.push({ X: this._points[0].X, Y: this._points[0].Y }); + this._timeStamps.push(this._timeStamps[0]); const initialPoint = this._points[0.]; const xInGlass = initialPoint.X > (this._thumbX ?? Number.MAX_SAFE_INTEGER) && initialPoint.X < (this._thumbX ?? Number.MAX_SAFE_INTEGER) + (this.height); @@ -558,6 +565,7 @@ export class GestureOverlay extends Touchable { case ToolglassTools.InkToText: this._strokes.push(new Array(...this._points)); this._points = []; + this._timeStamps = []; CognitiveServices.Inking.Appliers.InterpretStrokes(this._strokes).then((results) => { const wordResults = results.filter((r: any) => r.category === "line"); const possibilities: string[] = []; @@ -581,6 +589,7 @@ export class GestureOverlay extends Touchable { case ToolglassTools.IgnoreGesture: this.dispatchGesture(GestureUtils.Gestures.Stroke); this._points = []; + this._timeStamps = []; break; } } @@ -589,6 +598,7 @@ export class GestureOverlay extends Touchable { this.makePolygon(this.InkShape, false); this.dispatchGesture(GestureUtils.Gestures.Stroke); this._points = []; + this._timeStamps = []; if (!CollectionFreeFormViewChrome.Instance?._keepPrimitiveMode) { this.InkShape = ""; Doc.UserDoc().activeInkTool = InkTool.None; @@ -635,9 +645,11 @@ export class GestureOverlay extends Touchable { this.dispatchGesture(GestureUtils.Gestures.Stroke); } this._points = []; + this._timeStamps = []; } } else { this._points = []; + this._timeStamps = []; } CollectionFreeFormViewChrome.Instance?.primCreated(); } @@ -686,6 +698,7 @@ export class GestureOverlay extends Touchable { } } this._points = []; + this._timeStamps = []; switch (shape) { //must push an extra point in the end so InteractionUtils knows pointer is up. //must be (points[0].X,points[0]-1) @@ -808,7 +821,8 @@ export class GestureOverlay extends Touchable { points: stroke ?? this._points, gesture: gesture as any, bounds: this.getBounds(stroke ?? this._points), - text: data + text: data, + times: this._timeStamps } } ) diff --git a/src/client/views/InkTranscription.scss b/src/client/views/InkTranscription.scss new file mode 100644 index 000000000..bbb0a1afa --- /dev/null +++ b/src/client/views/InkTranscription.scss @@ -0,0 +1,5 @@ +.ink-transcription { + .error-msg { + display: none !important; + } +} \ No newline at end of file diff --git a/src/client/views/InkTranscription.tsx b/src/client/views/InkTranscription.tsx new file mode 100644 index 000000000..950f622dd --- /dev/null +++ b/src/client/views/InkTranscription.tsx @@ -0,0 +1,188 @@ +import { timeStamp } from 'console'; +import * as iink from 'iink-js'; +import { action, observable } from 'mobx'; +import * as React from 'react'; +import { Doc } from '../../fields/Doc'; +import { InkData, InkField } from "../../fields/InkField"; +import { Cast, NumCast, StrCast } from '../../fields/Types'; +import { Docs } from "../documents/Documents"; +import './InkTranscription.scss'; + +export class InkTranscription extends React.Component { + static Instance: InkTranscription; + + @observable _mathRegister: any; + @observable _mathRef: any; + @observable _textRegister: any; + @observable _textRef: any; + private addDocument?: (doc: Doc | Doc[]) => boolean; + private bounds: { x: number, y: number, width: number, height: number } = { x: 0, y: 0, width: 0, height: 0 }; + + constructor(props: Readonly<{}>) { + super(props); + + InkTranscription.Instance = this; + } + + componentWillUnmount() { + this._mathRef.removeEventListener('exported', (e: any) => this.exportInk(e, this._mathRef)); + this._textRef.removeEventListener('exported', (e: any) => this.exportInk(e, this._textRef)); + + this._mathRef.removeEventListener('changed', (e: any) => this.callExport(this._mathRef)); + this._textRef.removeEventListener('changed', (e: any) => this.callExport(this._textRef)); + } + + @action + setMathRef = (r: any) => { + if (!this._mathRegister) { + this._mathRegister = r ? iink.register(r, { + recognitionParams: { + type: 'MATH', + protocol: 'WEBSOCKET', + server: { + host: 'cloud.myscript.com', + applicationKey: '7277ec34-0c2e-4ee1-9757-ccb657e3f89f', + hmacKey: 'f5cb18f2-1f95-4ddb-96ac-3f7c888dffc1', + websocket: { + pingEnabled: false, + autoReconnect: true + } + }, + iink: { + math: { + mimeTypes: ['application/x-latex', 'application/vnd.myscript.jiix'] + }, + export: { + jiix: { + strokes: true + } + } + } + }, + triggers: { + exportContent: 'DEMAND' + } + }) : null; + } + + r.addEventListener('exported', (e: any) => this.exportInk(e, this._mathRef)); + r.addEventListener('changed', (e: any) => this.callExport(this._mathRef)); + + return this._mathRef = r; + } + + @action + setTextRef = (r: any) => { + if (!this._textRegister) { + this._textRegister = r ? iink.register(r, { + recognitionParams: { + type: 'TEXT', + protocol: 'WEBSOCKET', + server: { + host: 'cloud.myscript.com', + applicationKey: '7277ec34-0c2e-4ee1-9757-ccb657e3f89f', + hmacKey: 'f5cb18f2-1f95-4ddb-96ac-3f7c888dffc1', + websocket: { + pingEnabled: false, + autoReconnect: true + } + }, + iink: { + text: { + mimeTypes: ['text/plain'] + }, + export: { + jiix: { + strokes: true + } + } + } + }, + triggers: { + exportContent: 'DEMAND' + } + }) : null; + } + + r.addEventListener('exported', (e: any) => this.exportInk(e, this._textRef)); + r.addEventListener('changed', (e: any) => this.callExport(this._textRef)); + + return this._textRef = r; + } + + transcribeInk = (inkdocs: Doc[], math: boolean, bounds: { x: number, y: number, width: number, height: number }, addDocument?: (doc: Doc | Doc[]) => boolean) => { + const strokes: InkData[] = []; + inkdocs.filter(i => Cast(i.data, InkField)).forEach(i => { + // TODO: interpolate missing times stamps + const d = Cast(i.data, InkField, null); + const left = Math.min(...d?.inkData.map(pd => pd.X) ?? [0]); + const top = Math.min(...d?.inkData.map(pd => pd.Y) ?? [0]); + strokes.push(d.inkData.map(pd => ({ X: pd.X + NumCast(i.x) - left, Y: pd.Y + NumCast(i.y) - top, time: pd.time }))); + }); + + this.addDocument = addDocument; + this.bounds = bounds; + + const pointerData = { "events": strokes.map(stroke => this.inkJSON(stroke)) }; + // console.log(JSON.stringify(pointerData)); + const processGestures = false; + + if (math) { + this._mathRef.editor.pointerEvents(pointerData, processGestures); + } + else { + this._textRef.editor.pointerEvents(pointerData, processGestures); + } + } + + callExport(ref: any) { + if (ref.editor.canExport) { + ref.editor.export_(); + } + } + + inkJSON = (stroke: InkData) => { + return { + "pointerType": "PEN", + "pointerId": 1, + "x": stroke.map(point => point.X), + "y": stroke.map(point => point.Y), + "t": stroke.map(point => point.time), + "p": new Array(stroke.length).fill(1.0) + }; + } + + exportInk = (e: any, ref: any) => { + const exports = e.detail.exports; + if (exports) { + if (exports['application/x-latex']) { + const latex = exports['application/x-latex']; + console.log(latex); + + this.addDocument?.(Docs.Create.EquationDocument({ title: latex, x: this.bounds.width, y: 0, _width: this.bounds.width, _height: this.bounds.height })); + } + else if (exports['text/plain']) { + const text = exports['text/plain']; + console.log(text); + this.addDocument?.(Docs.Create.TextDocument(text, { title: text, x: this.bounds.width, y: 0, _width: this.bounds.width, _height: this.bounds.height })); + } + + ref.editor.clear(); + } + } + + render() { + return ( +
+
+
+
+
+
+ ) + } +} \ No newline at end of file diff --git a/src/client/views/MainView.tsx b/src/client/views/MainView.tsx index 8c0795881..84c1037dd 100644 --- a/src/client/views/MainView.tsx +++ b/src/client/views/MainView.tsx @@ -44,6 +44,7 @@ import { GestureOverlay } from './GestureOverlay'; import { DASHBOARD_SELECTOR_HEIGHT, LEFT_MENU_WIDTH } from './global/globalCssVariables.scss'; import { Colors } from './global/globalEnums'; import { KeyManager } from './GlobalKeyHandler'; +import { InkTranscription } from './InkTranscription'; import { LightboxView } from './LightboxView'; import { LinkMenu } from './linking/LinkMenu'; import "./MainView.scss"; @@ -184,7 +185,7 @@ export class MainView extends React.Component { fa.faArrowAltCircleDown, fa.faArrowAltCircleUp, fa.faArrowAltCircleLeft, fa.faArrowAltCircleRight, fa.faStopCircle, fa.faCheckCircle, fa.faGripVertical, fa.faSortUp, fa.faSortDown, fa.faTable, fa.faTh, fa.faThList, fa.faProjectDiagram, fa.faSignature, fa.faColumns, fa.faChevronCircleUp, fa.faUpload, fa.faBorderAll, fa.faBraille, fa.faChalkboard, fa.faPencilAlt, fa.faEyeSlash, fa.faSmile, fa.faIndent, fa.faOutdent, fa.faChartBar, fa.faBan, fa.faPhoneSlash, fa.faGripLines, - fa.faSave, fa.faBookmark, fa.faList, fa.faListOl, fa.faFolderPlus, fa.faLightbulb, fa.faBookOpen, fa.faMapMarkerAlt]); + fa.faSave, fa.faBookmark, fa.faList, fa.faListOl, fa.faFolderPlus, fa.faLightbulb, fa.faBookOpen, fa.faMapMarkerAlt, fa.faSquareRootAlt]); this.initAuthenticationRouters(); } @@ -636,6 +637,7 @@ export class MainView extends React.Component { + {this.snapLines}
diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx index e2ea81392..7d60387de 100644 --- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx +++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx @@ -50,6 +50,7 @@ import { CollectionFreeFormRemoteCursors } from "./CollectionFreeFormRemoteCurso import "./CollectionFreeFormView.scss"; import { MarqueeView } from "./MarqueeView"; import React = require("react"); +import { InkTranscription } from "../../InkTranscription"; export const panZoomSchema = createSchema({ _panX: "number", @@ -492,7 +493,9 @@ export class CollectionFreeFormView extends CollectionSubView { return { X: pt.X, Y: pt.Y, time: times[i] } }); + const inkDoc = Docs.Create.InkDocument(ActiveInkColor(), CurrentUserUtils.SelectedTool, ActiveInkWidth(), ActiveInkBezierApprox(), ActiveFillColor(), ActiveArrowStart(), ActiveArrowEnd(), ActiveDash(), strokes, { title: "ink stroke", x: B.x - ActiveInkWidth() / 2, y: B.y - ActiveInkWidth() / 2, _width: B.width + ActiveInkWidth(), _height: B.height + ActiveInkWidth() }); this.addDocument(inkDoc); e.stopPropagation(); @@ -1477,6 +1480,11 @@ export class CollectionFreeFormView extends CollectionSubView this.Document._fitToBox = !this.fitToContent, icon: !this.fitToContent ? "expand-arrows-alt" : "compress-arrows-alt" }); this.props.ContainingCollectionView && appearanceItems.push({ description: "Ungroup collection", event: this.promoteCollection, icon: "table" }); + + this.props.Document._isGroup && this.childDocs.filter(s => s.type === DocumentType.INK).length > 0 && appearanceItems.push({ description: "Ink to text", event: () => this.transcribeStrokes(false), icon: "font" }); + + this.props.Document._isGroup && this.childDocs.filter(s => s.type === DocumentType.INK).length > 0 && appearanceItems.push({ description: "Ink to math", event: () => this.transcribeStrokes(true), icon: "square-root-alt" }); + !Doc.UserDoc().noviceMode ? appearanceItems.push({ description: "Arrange contents in grid", event: this.layoutDocsInGrid, icon: "table" }) : null; !appearance && ContextMenu.Instance.addItem({ description: "Appearance...", subitems: appearanceItems, icon: "eye" }); @@ -1536,6 +1544,15 @@ export class CollectionFreeFormView extends CollectionSubView { + if (this.props.Document._isGroup) { + const inkdocs = this.childDocs.filter(s => s.type === DocumentType.INK); + InkTranscription.Instance.transcribeInk(inkdocs, math, { x: NumCast(this.layoutDoc.x) ?? 0, y: NumCast(this.layoutDoc.y) ?? 0, width: NumCast(this.layoutDoc._width) ?? 0, height: NumCast(this.layoutDoc._height) ?? 0 }, this.addDocument); + } + } + @action setupDragLines = (snapToDraggedDoc: boolean = false) => { const activeDocs = this.getActiveDocuments(); diff --git a/src/client/views/collections/collectionFreeForm/MarqueeOptionsMenu.tsx b/src/client/views/collections/collectionFreeForm/MarqueeOptionsMenu.tsx index 1f59f9732..8a8b528f6 100644 --- a/src/client/views/collections/collectionFreeForm/MarqueeOptionsMenu.tsx +++ b/src/client/views/collections/collectionFreeForm/MarqueeOptionsMenu.tsx @@ -14,7 +14,6 @@ export class MarqueeOptionsMenu extends AntimodeMenu { public createCollection: (e: KeyboardEvent | React.PointerEvent | undefined, group?: boolean) => void = unimplementedFunction; public delete: (e: KeyboardEvent | React.PointerEvent | undefined) => void = unimplementedFunction; public summarize: (e: KeyboardEvent | React.PointerEvent | undefined) => void = unimplementedFunction; - public inkToText: (e: KeyboardEvent | React.PointerEvent | undefined) => void = unimplementedFunction; public showMarquee: () => void = unimplementedFunction; public hideMarquee: () => void = unimplementedFunction; public pinWithView: (e: KeyboardEvent | React.PointerEvent | undefined) => void = unimplementedFunction; @@ -64,16 +63,6 @@ export class MarqueeOptionsMenu extends AntimodeMenu { , ]; - if (false && !SelectionManager.Views().some(v => v.props.Document.type !== DocumentType.INK)) { - buttons.push( - Change to Text
} placement="bottom"> - - ); - } return this.getElement(buttons); } } \ No newline at end of file diff --git a/src/client/views/collections/collectionFreeForm/MarqueeView.tsx b/src/client/views/collections/collectionFreeForm/MarqueeView.tsx index b10b0912f..a828cd981 100644 --- a/src/client/views/collections/collectionFreeForm/MarqueeView.tsx +++ b/src/client/views/collections/collectionFreeForm/MarqueeView.tsx @@ -1,14 +1,15 @@ import { action, computed, observable } from "mobx"; import { observer } from "mobx-react"; -import { AclAugment, AclAdmin, AclEdit, DataSym, Doc, Opt } from "../../../../fields/Doc"; +import { AclAdmin, AclAugment, AclEdit, DataSym, Doc, Opt } from "../../../../fields/Doc"; import { Id } from "../../../../fields/FieldSymbols"; import { InkData, InkField, InkTool } from "../../../../fields/InkField"; import { List } from "../../../../fields/List"; import { RichTextField } from "../../../../fields/RichTextField"; import { SchemaHeaderField } from "../../../../fields/SchemaHeaderField"; import { Cast, FieldValue, NumCast, StrCast } from "../../../../fields/Types"; +import { ImageField } from "../../../../fields/URLField"; import { GetEffectiveAcl } from "../../../../fields/util"; -import { Utils, intersectRect, returnFalse } from "../../../../Utils"; +import { intersectRect, returnFalse, Utils } from "../../../../Utils"; import { CognitiveServices } from "../../../cognitive_services/CognitiveServices"; import { Docs, DocumentOptions, DocUtils } from "../../../documents/Documents"; import { DocumentType } from "../../../documents/DocumentTypes"; @@ -21,18 +22,17 @@ import { ContextMenu } from "../../ContextMenu"; import { FormattedTextBox } from "../../nodes/formattedText/FormattedTextBox"; import { PresBox } from "../../nodes/trails/PresBox"; import { PresMovement } from "../../nodes/trails/PresEnums"; +import { VideoBox } from "../../nodes/VideoBox"; +import { pasteImageBitmap } from "../../nodes/WebBoxRenderer"; import { PreviewCursor } from "../../PreviewCursor"; +import { StyleLayers } from "../../StyleProvider"; import { CollectionDockingView } from "../CollectionDockingView"; import { SubCollectionViewProps } from "../CollectionSubView"; import { CollectionView } from "../CollectionView"; +import { TreeView } from "../TreeView"; import { MarqueeOptionsMenu } from "./MarqueeOptionsMenu"; import "./MarqueeView.scss"; import React = require("react"); -import { StyleLayers } from "../../StyleProvider"; -import { TreeView } from "../TreeView"; -import { VideoBox } from "../../nodes/VideoBox"; -import { ImageField, WebField } from "../../../../fields/URLField"; -import { pasteImageBitmap } from "../../nodes/WebBoxRenderer"; interface MarqueeViewProps { getContainerTransform: () => Transform; @@ -265,7 +265,7 @@ export class MarqueeView extends React.Component() { public static LayoutString(fieldKey: string) { return FieldView.LayoutString(EquationBox, fieldKey); } public static SelectOnLoad: string = ""; + _ref: React.RefObject = React.createRef(); + @observable _inkOpen = false; + @observable _inkEditor: any; + @observable _inkRef: any; + componentDidMount() { if (EquationBox.SelectOnLoad === this.rootDoc[Id] && (!LightboxView.LightboxDoc || LightboxView.IsLightboxDocView(this.props.docViewPath()))) { this.props.select(false); @@ -39,7 +45,11 @@ export class EquationBox extends ViewBoxBaseComponent() { } }, { fireImmediately: true }); } - plot: any; + + componentWillUnmount() { + this._inkRef.removeEventListener('exported', this.exportInk); + } + @action keyPressed = (e: KeyboardEvent) => { const _height = Number(getComputedStyle(this._ref.current!.element.current).height.replace("px", "")); @@ -65,6 +75,7 @@ export class EquationBox extends ViewBoxBaseComponent() { } if (e.key === "Backspace" && !this.dataDoc.text) this.props.removeDocument?.(this.rootDoc); } + onChange = (str: string) => { this.dataDoc.text = str; const style = this._ref.current && getComputedStyle(this._ref.current.element.current); @@ -75,6 +86,58 @@ export class EquationBox extends ViewBoxBaseComponent() { this.layoutDoc._height = Math.max(25, _height); } } + + @action + toggleInk = (e: React.PointerEvent) => { + e.stopPropagation(); + + this._inkOpen = !this._inkOpen; + + if (!this._inkEditor) { + this._inkEditor = this._inkRef ? iink.register(this._inkRef, { + recognitionParams: { + type: 'MATH', + protocol: 'WEBSOCKET', + server: { + host: 'cloud.myscript.com', + applicationKey: '7277ec34-0c2e-4ee1-9757-ccb657e3f89f', + hmacKey: 'f5cb18f2-1f95-4ddb-96ac-3f7c888dffc1', + }, + iink: { + math: { + mimeTypes: ['application/x-latex', 'application/vnd.myscript.jiix'] + }, + export: { + jiix: { + strokes: true + } + } + } + } + }) : null; + + this._inkRef.addEventListener('exported', this.exportInk) + } + } + + convertInk = (e: React.MouseEvent) => { + this._inkRef.editor.convert(); + } + + clearInk = (e: React.MouseEvent) => { + this._inkRef.editor.clear(); + this.onChange(""); + } + + exportInk = (e: any) => { + const exports = e.detail.exports; + if (exports && exports['application/x-latex']) { + this.onChange(exports['application/x-latex']); + console.log(JSON.parse(exports['application/vnd.myscript.jiix']).expressions[0].items[0]); + } + } + + render() { TraceMobx(); const scale = (this.props.scaling?.() || 1) * NumCast(this.layoutDoc._viewScale, 1); @@ -82,18 +145,34 @@ export class EquationBox extends ViewBoxBaseComponent() { onPointerDown={e => !e.ctrlKey && e.stopPropagation()} style={{ transform: `scale(${scale})`, - width: `${100 / scale}%`, + width: `calc(${100 / scale}% + 25px)`, height: `${100 / scale}%`, pointerEvents: !this.props.isSelected() ? "none" : undefined, }} - onKeyDown={e => e.stopPropagation()} - > + onKeyDown={e => e.stopPropagation()}> + + +
+ +
+ +
this._inkRef = r)} + id="editor" onPointerDown={(e) => e.stopPropagation()} + touch-action="none" + style={{ display: this._inkOpen && this.props.isContentActive() ? "block" : "none" }}> + + +
+ ); } } \ No newline at end of file diff --git a/src/fields/InkField.ts b/src/fields/InkField.ts index 31024e805..48271d3d3 100644 --- a/src/fields/InkField.ts +++ b/src/fields/InkField.ts @@ -19,6 +19,7 @@ export enum InkTool { export interface PointData { X: number; Y: number; + time?: number; } export type Segment = Array; @@ -83,7 +84,7 @@ export class InkField extends ObjectField { } [ToScriptString]() { - return "new InkField([" + this.inkData.map(i => `{X: ${i.X}, Y: ${i.Y}} `) + "])"; + return "new InkField([" + this.inkData.map(i => `{X: ${i.X}, Y: ${i.Y}, time: ${i.time || 0}} `) + "])"; } [ToString]() { return "InkField"; diff --git a/src/pen-gestures/GestureUtils.ts b/src/pen-gestures/GestureUtils.ts index 65f2bf80c..cf59cb3c6 100644 --- a/src/pen-gestures/GestureUtils.ts +++ b/src/pen-gestures/GestureUtils.ts @@ -8,7 +8,8 @@ export namespace GestureUtils { readonly gesture: Gestures, readonly points: PointData[], readonly bounds: Rect, - readonly text?: any + readonly text?: any, + readonly times?: number[] ) { } } diff --git a/src/typings/index.d.ts b/src/typings/index.d.ts index 068ac2159..bc7727426 100644 --- a/src/typings/index.d.ts +++ b/src/typings/index.d.ts @@ -8,6 +8,7 @@ declare module 'webrtc-adapter'; declare module 'bezier-curve'; declare module 'fit-curve'; declare module 'react-audio-waveform'; +declare module 'iink-js'; declare module 'reveal'; declare module 'react-reveal'; -- cgit v1.2.3-70-g09d2